From cad83f8d0e9168dd0ae10e614dd35a05788eb5ff Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 31 Oct 2026 15:19:16 +0000 Subject: [PATCH 01/12] chore(bench): measure encode and decode against a hand-written copy The bench builds documents of several sizes and runs encode and decode for crosswalks of identity moves, with and without format, scale and enum rules, beside a hand-written function giving the same output. It prints the ratio of the two medians from one run on one machine and the input reads each side makes per node, counted through a Proxy, so the figures are work done rather than time. --- README.md | 1 + bench/encode.ts | 271 ++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 + tsconfig.json | 2 +- turbo.json | 4 + 5 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 bench/encode.ts diff --git a/README.md b/README.md index 7a33ea8..0e0504d 100644 --- a/README.md +++ b/README.md @@ -276,6 +276,7 @@ Node 22 or later, pnpm, ESM only. See [CONTRIBUTING.md](CONTRIBUTING.md) for the | `pnpm typecheck` | `tsc` over both tsconfig programs | | `pnpm test` | Vitest, tests colocated as `*.unit.test.ts` | | `pnpm build` | tsdown into `dist`, then the JSON Schema generation script | +| `pnpm bench` | Encode and decode against a hand-written copy of the same fields: the ratio of the two on this machine in this run, and the input reads each makes per node | `CLAUDE.md` and `AGENTS.md` are symlinks to this file. diff --git a/bench/encode.ts b/bench/encode.ts new file mode 100644 index 0000000..eedbd8e --- /dev/null +++ b/bench/encode.ts @@ -0,0 +1,271 @@ +/** + * Measures encode and decode against a hand-written copy of the same fields, on the same machine in the same run, and prints the work each does rather than a time: the ratio of crossmap's median run to the baseline's, and the number of input property reads each makes per node (counted through a Proxy over a small document, so the count is exact and independent of load). Run with `pnpm bench`. + */ +import { isDeepStrictEqual } from "node:util"; +import { type Crosswalk, decode, encode, parseCrosswalk } from "../src/index"; + +/** A document small enough that the per-call cost of a run shows beside its per-node cost. */ +const SMALL_SIZE = 1_000; + +/** A document between the two. */ +const MEDIUM_SIZE = 30_000; + +/** A document large enough that the per-node cost dominates: the size the gap to a hand-written copy was first measured at. */ +const LARGE_SIZE = 300_000; + +/** Node counts the ratios are measured at. */ +const SIZES = [SMALL_SIZE, MEDIUM_SIZE, LARGE_SIZE]; + +/** Identity field moves the mixed crosswalk carries besides its non-identity rules, the shape of a catalogue entry that maps one field per rule. */ +const IDENTITY_FIELDS = 6; + +/** Timed rounds per measurement: each round runs the baseline and crossmap once, interleaved so load on a shared host falls on both, and the median of the rounds is kept. */ +const ROUNDS = 7; + +/** Untimed runs of each side before timing, so both are optimised by the JIT before they are compared. */ +const WARMUPS = 2; + +/** Nodes in the document whose reads are counted: the count is per node and exact, so a small document gives the same figure as a large one. */ +const COUNTED_NODES = 100; + +/** The factor the scale rule multiplies by. */ +const SCALE_FACTOR = 1000; + +/** How many distinct kinds the enum rule maps, cycled through the nodes. */ +const KINDS = ["paper", "book", "thesis"] as const; + +/** The enum rule's map from native kind to shared type. */ +const KIND_MAP: Readonly> = { paper: "PAPER", book: "BOOK", thesis: "THESIS" }; + +/** The inverse of KIND_MAP, which the enum rule's mapBack must be. */ +const KIND_MAP_BACK: Readonly> = Object.fromEntries(Object.entries(KIND_MAP).map(([from, to]) => [to, from])); + +const identityFieldNames = Array.from({ length: IDENTITY_FIELDS }, (_unused, position) => `f${String(position)}`); + +/** The three identity moves the problem was first measured with. */ +const identityCrosswalk = parseCrosswalk({ + source: "native", + target: "shared", + rules: [ + { op: "move", from: "meta", to: "meta" }, + { op: "move", from: "nodes[].id", to: "nodes[].id" }, + { op: "move", from: "nodes[].text", to: "nodes[].text" }, + ], +}); + +/** Identity moves for every field, plus one rule each of format, scale and enum. */ +const mixedCrosswalk = parseCrosswalk({ + source: "native", + target: "shared", + rules: [ + { op: "move", from: "meta", to: "meta" }, + ...identityFieldNames.map((name) => ({ op: "move", from: `nodes[].${name}`, to: `nodes[].${name}` })), + { op: "format", from: "nodes[].sizeText", to: "nodes[].size", kind: "numberText" }, + { op: "scale", from: "nodes[].weight", to: "nodes[].weightMilli", factor: SCALE_FACTOR }, + { op: "enum", from: "nodes[].kind", to: "nodes[].type", map: KIND_MAP, mapBack: KIND_MAP_BACK }, + ], +}); + +interface IdentityNode { + readonly id: string; + readonly text: string; +} + +type MixedNode = Readonly>; + +interface Document { + readonly meta: { readonly name: string; readonly version: number }; + readonly nodes: readonly Node[]; +} + +const meta = { name: "bench", version: 1 }; + +function identityDocument(count: number): Document { + return { meta, nodes: Array.from({ length: count }, (_unused, index) => ({ id: `n${String(index)}`, text: `node ${String(index)}` })) }; +} + +function mixedDocument(count: number): Document { + return { + meta, + nodes: Array.from({ length: count }, (_unused, index) => ({ + ...Object.fromEntries(identityFieldNames.map((name) => [name, `${name}-${String(index)}`])), + sizeText: String(index), + weight: index, + kind: KINDS[index % KINDS.length], + })), + }; +} + +/** The hand-written encode of the identity crosswalk: the fields it names, copied. */ +function identityBaseline(doc: Document): unknown { + return { meta: doc.meta, nodes: doc.nodes.map((node) => ({ id: node.id, text: node.text })) }; +} + +/** The hand-written encode of the mixed crosswalk, written as plainly as a caller would write it. */ +function mixedEncodeBaseline(doc: Document): Document { + return { + meta: doc.meta, + nodes: doc.nodes.map((node) => { + const out: Record = {}; + for (const name of identityFieldNames) { + out[name] = node[name]; + } + out["size"] = Number(node["sizeText"]); + out["weightMilli"] = Number(node["weight"]) * SCALE_FACTOR; + out["type"] = KIND_MAP[String(node["kind"])]; + + return out; + }), + }; +} + +/** The hand-written decode of the mixed crosswalk. */ +function mixedDecodeBaseline(doc: Document): unknown { + return { + meta: doc.meta, + nodes: doc.nodes.map((node) => { + const out: Record = {}; + for (const name of identityFieldNames) { + out[name] = node[name]; + } + out["sizeText"] = String(node["size"]); + out["weight"] = Number(node["weightMilli"]) / SCALE_FACTOR; + out["kind"] = KIND_MAP_BACK[String(node["type"])]; + + return out; + }), + }; +} + +function median(values: readonly number[]): number { + const sorted = [...values].sort((left, right) => left - right); + const middle = sorted[Math.floor(sorted.length / 2)]; + if (middle === undefined) { + throw new Error("a median of no values"); + } + + return middle; +} + +/** Runs a function once and returns how long it took, in the high-resolution clock's units; only ever compared with another such reading from the same run. */ +function timed(run: () => unknown): number { + const start = process.hrtime.bigint(); + run(); + + return Number(process.hrtime.bigint() - start); +} + +/** The median ratio of crossmap's run to the baseline's, with the two interleaved round by round. */ +function ratio(baseline: () => unknown, crossmap: () => unknown): number { + for (let warm = 0; warm < WARMUPS; warm += 1) { + baseline(); + crossmap(); + } + const baselineRuns: number[] = []; + const crossmapRuns: number[] = []; + for (let round = 0; round < ROUNDS; round += 1) { + baselineRuns.push(timed(baseline)); + crossmapRuns.push(timed(crossmap)); + } + + return median(crossmapRuns) / median(baselineRuns); +} + +/** A running count of the reads made of a counted document. */ +class ReadCounter { + reads = 0; + + note(): void { + this.reads += 1; + } +} + +/** Wraps a document so every property read, own-key check and key listing on it or anything beneath it is counted. */ +function counted(value: T, counter: ReadCounter): T { + if (typeof value !== "object" || value === null) { + return value; + } + const children = new Map(); + + return new Proxy(value, { + get(target, key, receiver) { + counter.note(); + const found: unknown = Reflect.get(target, key, receiver); + if (typeof found !== "object" || found === null) { + return found; + } + const cached = children.get(key); + if (cached !== undefined) { + return cached; + } + const wrapped = counted(found, counter); + children.set(key, wrapped); + + return wrapped; + }, + getOwnPropertyDescriptor(target, key) { + counter.note(); + + return Reflect.getOwnPropertyDescriptor(target, key); + }, + has(target, key) { + counter.note(); + + return Reflect.has(target, key); + }, + ownKeys(target) { + counter.note(); + + return Reflect.ownKeys(target); + }, + }); +} + +/** Input reads per node one run makes over a document of COUNTED_NODES nodes. */ +function readsPerNode(run: (doc: Doc) => unknown, doc: Doc): number { + const counter = new ReadCounter(); + run(counted(doc, counter)); + + return counter.reads / COUNTED_NODES; +} + +/** One crosswalk in one direction, with the documents it reads and its hand-written equivalent. */ +interface Scenario { + /** The label printed for the scenario. */ + readonly name: string; + /** The crosswalk crossmap runs. */ + readonly crosswalk: Crosswalk; + /** Which of encode and decode crossmap runs. */ + readonly direction: "encode" | "decode"; + /** Builds the input document of a given node count. */ + readonly document: (count: number) => Doc; + /** The hand-written equivalent, which must give the same output. */ + readonly baseline: (doc: Doc) => unknown; +} + +/** + * Measures one scenario against its hand-written baseline and prints the counts and ratios. + * @param scenario - The scenario. + * @throws Error when the baseline and crossmap disagree on a small document, since the ratio would then compare different work. + */ +function measure(scenario: Scenario): void { + const { name, crosswalk, direction, document, baseline } = scenario; + const run = (doc: Doc): unknown => (direction === "encode" ? encode(crosswalk, doc) : decode(crosswalk, doc)); + const small = document(COUNTED_NODES); + if (!isDeepStrictEqual(run(small), baseline(small))) { + throw new Error(`${name}: the baseline and crossmap disagree, so the ratio would compare different work`); + } + const reads = `input reads per node: crossmap ${readsPerNode(run, small).toFixed(1)}, baseline ${readsPerNode(baseline, small).toFixed(1)}`; + const ratios = SIZES.map((size) => { + const doc = document(size); + + return `${String(size)} nodes ${ratio(() => baseline(doc), () => run(doc)).toFixed(1)}x`; + }); + console.log(`${name}\n ${reads}\n ratio to baseline: ${ratios.join(", ")}`); +} + +const mixedName = `${String(IDENTITY_FIELDS + 1)} moves + format, scale, enum`; +measure({ name: "3 identity moves, encode", crosswalk: identityCrosswalk, direction: "encode", document: identityDocument, baseline: identityBaseline }); +measure({ name: "3 identity moves, decode", crosswalk: identityCrosswalk, direction: "decode", document: identityDocument, baseline: identityBaseline }); +measure({ name: `${mixedName}, encode`, crosswalk: mixedCrosswalk, direction: "encode", document: mixedDocument, baseline: mixedEncodeBaseline }); +measure({ name: `${mixedName}, decode`, crosswalk: mixedCrosswalk, direction: "decode", document: (count) => mixedEncodeBaseline(mixedDocument(count)), baseline: mixedDecodeBaseline }); diff --git a/package.json b/package.json index e10c644..85edca1 100644 --- a/package.json +++ b/package.json @@ -70,11 +70,13 @@ "url": "git+https://github.com/ExaDev/crossmap.git" }, "scripts": { + "_bench": "tsx bench/encode.ts", "_build": "tsdown && tsx scripts/generate-json-schema.ts", "_lint": "eslint . --fix --cache --cache-strategy content --max-warnings 0", "_lint:check": "eslint . --cache --cache-strategy content --max-warnings 0", "_test": "vitest run", "_typecheck": "tsc -p tsconfig.json && tsc -p tsconfig.node.json", + "bench": "turbo run _bench", "build": "turbo run _build", "lint": "turbo run _lint", "lint:check": "turbo run _lint:check", diff --git a/tsconfig.json b/tsconfig.json index 4235f3a..688a83c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -27,5 +27,5 @@ "types": ["node"], "verbatimModuleSyntax": true }, - "include": ["src", "cli", "test", "*.config.ts"] + "include": ["src", "cli", "test", "bench", "*.config.ts"] } diff --git a/turbo.json b/turbo.json index c5c619c..ebf2b78 100644 --- a/turbo.json +++ b/turbo.json @@ -7,6 +7,10 @@ // // `_typecheck` and `_test` depend on this package's own `_build`, and that edge is the reason this file exists: scripts/generate-json-schema.ts imports ../dist/index.js by relative path, so the program in tsconfig.node.json cannot typecheck until dist/ exists, and the JSON Schema suite reads schemas/crossmap.schema.json, which the build generates. Encoding the order here means no hook, workflow or person has to remember it. "tasks": { + // A measurement, not a build step: its output is the ratios of this run on this machine, so a cached replay would report another run's figures. + "_bench": { + "cache": false + }, "_build": { // scripts/** covers the schema generation step that runs as part of the build; package.json is an input because the generation reads it. "inputs": ["src/**", "cli/**", "scripts/**", "tsdown.config.ts", "tsconfig.json", "package.json"], From a8d0d3eee7804fe6b0476547162c966bed78e586 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 31 Oct 2026 15:19:42 +0000 Subject: [PATCH 02/12] test(executor): compare the executor with a frozen reference copy of it The reference is the executor as it stands, with its gap search, kept in test support and not shipped. Property tests built on fast-check generate crosswalks of every op, path shape and gate, and documents with absent paths, empty and sparse arrays, fixed indices and values that are not JSON, then require the same output (key order included) or the same error class, message, rule index and path from both, in both directions. Each property also requires that every kind of outcome was met. --- package.json | 1 + pnpm-lock.yaml | 16 ++ src/executor-reference.unit.test.ts | 404 ++++++++++++++++++++++++++++ test/support/reference-executor.ts | 294 ++++++++++++++++++++ 4 files changed, 715 insertions(+) create mode 100644 src/executor-reference.unit.test.ts create mode 100644 test/support/reference-executor.ts diff --git a/package.json b/package.json index 85edca1..420beb0 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "canonicalize": "5.1.0", "conventional-changelog-conventionalcommits": "9.3.1", "eslint": "10.12.0", + "fast-check": "4.10.2", "husky": "9.1.7", "jiti": "2.7.0", "lint-staged": "17.6.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eef8d8c..291921d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -209,6 +209,9 @@ importers: eslint: specifier: 10.12.0 version: 10.12.0(jiti@2.7.0)(supports-color@7.2.0) + fast-check: + specifier: 4.10.2 + version: 4.10.2 husky: specifier: 9.1.7 version: 9.1.7 @@ -1659,6 +1662,10 @@ packages: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} + fast-check@4.10.2: + resolution: {integrity: sha512-iK2f+YrcmoeGqk6fA0ea2bptcu/itMIm4NfEozq6N25+aG6h7s5HZbB/k1aV7b5w5sFLMCbbtRUsTVR+BgC3xw==} + engines: {node: '>=12.17.0'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -2475,6 +2482,9 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + pure-rand@8.4.2: + resolution: {integrity: sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==} + qified@0.10.1: resolution: {integrity: sha512-+Owyggi9IxT1ePKGafcI87ubSmxol6smwJ+RAHDQlx9+9cPwFWDiKFFCPuWhr9ignlGpZ9vDQLw67N4dcTVFEA==} engines: {node: '>=20'} @@ -4537,6 +4547,10 @@ snapshots: expect-type@1.4.0: {} + fast-check@4.10.2: + dependencies: + pure-rand: 8.4.2 + fast-deep-equal@3.1.3: {} fast-json-stable-stringify@2.1.0: {} @@ -5174,6 +5188,8 @@ snapshots: punycode@2.3.1: {} + pure-rand@8.4.2: {} + qified@0.10.1: dependencies: hookified: 2.2.0 diff --git a/src/executor-reference.unit.test.ts b/src/executor-reference.unit.test.ts new file mode 100644 index 0000000..d6b5523 --- /dev/null +++ b/src/executor-reference.unit.test.ts @@ -0,0 +1,404 @@ +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; +import { CORPUS } from "./corpus"; +import { type Crosswalk, parseCrosswalk } from "./crosswalk"; +import { RuleApplicationError } from "./errors"; +import { decode, encode } from "./executor"; +import { type Locus, parsePath } from "./paths"; +import { ruleSides } from "./sides"; +import { decodeReference, encodeReference } from "../test/support/reference-executor"; + +/** What one run of an executor produced: the output in its exact serialisation (so key order counts), or the error's class, message and, for a rule failure, its rule index and path. */ +type Outcome = { readonly output: string; readonly value: unknown } | { readonly error: string; readonly message: string; readonly ruleIndex: number | undefined; readonly path: string | undefined }; + +function outcome(run: () => unknown): Outcome { + try { + const value = run(); + + return { output: JSON.stringify(value), value }; + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + return { + error: error.constructor.name, + message: error.message, + ruleIndex: error instanceof RuleApplicationError ? error.ruleIndex : undefined, + path: error instanceof RuleApplicationError ? error.path : undefined, + }; + } +} + +/** The kinds of outcome a generated case can have, each of which the generators must reach for the comparison to cover it. */ +const OUTCOME_KINDS = ["output", "absent", "conflict", "gap", "operation", "structure", "not JSON"] as const; +type OutcomeKind = (typeof OUTCOME_KINDS)[number]; + +function kindOf(result: Outcome): OutcomeKind | undefined { + if ("output" in result) { + return "output"; + } + if (result.error === "CrosswalkStructureError") { + return "structure"; + } + if (result.error === "ZodError") { + return "not JSON"; + } + if (result.ruleIndex === undefined) { + return undefined; + } + if (result.message.includes("absent from the document")) { + return "absent"; + } + if (result.message.includes("two destinations") || result.message.includes("holds a value where") || result.message.includes("where the document has")) { + return "conflict"; + } + if (result.message.includes("gap")) { + return "gap"; + } + + return "operation"; +} + +/** + * Runs the executor and the reference over one input in one direction and requires the same outcome: the same output, key order included, or the same error class, message, rule index and path. + * @returns The kind of outcome both gave. + */ +function sameOutcome(crosswalk: Crosswalk, doc: unknown, direction: "encode" | "decode"): OutcomeKind | undefined { + const expected = outcome(() => (direction === "encode" ? encodeReference(crosswalk, doc) : decodeReference(crosswalk, doc))); + const actual = outcome(() => (direction === "encode" ? encode(crosswalk, doc) : decode(crosswalk, doc))); + if ("output" in expected && "output" in actual) { + expect(actual.output).toBe(expected.output); + expect(actual.value).toStrictEqual(expected.value); + } else { + expect(actual).toEqual(expected); + } + + return kindOf(expected); +} + +/** Record keys the generated paths and documents share, so a generated path is often present in a generated document. */ +const KEYS = ["a", "b", "c"] as const; + +/** Array keys the generated paths and documents share. */ +const ARRAYS = ["xs", "ys"] as const; + +/** The largest fixed index a generated path uses: past the length of most generated arrays, so an index both hits and misses. */ +const MAX_INDEX = 2; + +/** The longest generated array, long enough that a fixed index can fall inside it or past its end. */ +const MAX_ARRAY_LENGTH = 3; + +/** The deepest a generated document nests records inside records and arrays. */ +const MAX_DEPTH = 3; + +/** The most two-way rules a generated crosswalk carries. */ +const MAX_RULES = 5; + +/** The most one-way rules a generated crosswalk carries. */ +const MAX_ONE_WAY_RULES = 2; + +/** Generated cases per property: enough that every outcome kind is met, which each property checks. */ +const RUNS = 1500; + +/** The time one generated case may take: a handful of encodes and decodes of a small document, with the margin a heavily loaded runner needs, so the budget scales with the cases run rather than being a fixed figure. */ +const MILLISECONDS_PER_CASE = 60; + +/** Each property's own budget, derived from the cases it runs. */ +const PROPERTY_TIMEOUT_MILLISECONDS = RUNS * MILLISECONDS_PER_CASE; + +const key = fc.constantFrom(...KEYS); +const array = fc.constantFrom(...ARRAYS); +const index = fc.integer({ min: 0, max: MAX_INDEX }); + +/** Paths with no wildcard and no index: a key, nested keys, a whole array. */ +const keyPath = fc.oneof( + key, + fc.tuple(key, key).map(([outer, inner]) => `${outer}.${inner}`), + array, +); + +/** Paths with no wildcard, a fixed index among them some of the time. */ +const pathOfArity0 = fc.oneof( + { weight: 4, arbitrary: keyPath }, + { weight: 1, arbitrary: fc.tuple(array, index, key).map(([list, position, field]) => `${list}[${String(position)}].${field}`) }, + { weight: 1, arbitrary: fc.tuple(array, index).map(([list, position]) => `${list}[${String(position)}]`) }, +); + +/** Paths with one wildcard and only keys around it: a field of each element, a nested field, the whole element, an array under a key. */ +const keyedPathOfArity1 = fc.oneof( + fc.tuple(array, key).map(([list, field]) => `${list}[].${field}`), + fc.tuple(array, key, key).map(([list, outer, inner]) => `${list}[].${outer}.${inner}`), + array.map((list) => `${list}[]`), + fc.tuple(key, array, key).map(([outer, list, field]) => `${outer}.${list}[].${field}`), +); + +/** Paths with one wildcard, under a fixed index some of the time. */ +const pathOfArity1 = fc.oneof( + { weight: 4, arbitrary: keyedPathOfArity1 }, + { weight: 1, arbitrary: fc.tuple(array, index, array, key).map(([list, position, inner, field]) => `${list}[${String(position)}].${inner}[].${field}`) }, +); + +/** Paths with two wildcards: a field of each element of each inner array, and each inner element whole. */ +const pathOfArity2 = fc.oneof( + fc.tuple(array, array, key).map(([outer, inner, field]) => `${outer}[].${inner}[].${field}`), + fc.tuple(array, array).map(([outer, inner]) => `${outer}[].${inner}[]`), +); + +/** The array path a rule's read side iterates at its first wildcard, which an element gate's reference must name. */ +function firstArray(path: string): string | undefined { + const at = path.indexOf("[]"); + + return at < 0 ? undefined : path.slice(0, at); +} + +/** A `when` decided once for the whole rule. */ +const wholeRuleWhen = fc.tuple(key, fc.integer({ min: -1, max: 1 })).map(([field, value]) => ({ kind: "compare", op: "gt", left: { kind: "reference", key: field }, right: { kind: "numberLiteral", value } })); + +/** A `when` decided once for the whole rule, or per element over the array the rule's `from` iterates. */ +function whenFor(from: string): fc.Arbitrary { + const list = firstArray(from); + if (list === undefined) { + return wholeRuleWhen; + } + const perElement = fc.tuple(key, fc.constantFrom("open", "closed", "x")).map(([field, value]) => ({ kind: "textCompare", op: "equals", left: { kind: "reference", key: `${list}[].${field}` }, right: { kind: "textLiteral", value } })); + + return fc.oneof(wholeRuleWhen, perElement); +} + +/** A scale factor that is an integer, which multiply-then-divide inverts exactly. */ +const INTEGER_FACTOR = 2; + +/** A scale factor below one. */ +const FRACTION_FACTOR = 0.5; + +/** A scale factor that is a power of ten, the unit change most crosswalks make. */ +const DECIMAL_FACTOR = 1000; + +/** A number with a fractional part, which number text and scaling must carry. */ +const FRACTIONAL = 2.5; + +/** A negative number. */ +const NEGATIVE = -3; + +const ENUM_MAP = { open: "OPEN", closed: "CLOSED" }; +const ENUM_MAP_BACK = { OPEN: "open", CLOSED: "closed" }; + +/** The parameters of each single-path op. */ +const SINGLE_PATH_PARAMETERS = { + move: fc.constant({}), + scale: fc.constantFrom(INTEGER_FACTOR, FRACTION_FACTOR, DECIMAL_FACTOR).map((factor) => ({ factor })), + enum: fc.constant({ map: ENUM_MAP, mapBack: ENUM_MAP_BACK }), + format: fc.constantFrom("isoDateTime", "isoDate", "numberText", "booleanText", "jsonText").map((kind) => ({ kind })), + split: fc.constant({ separator: "," }), +}; + +/** An op with one path on each side, weighted towards `move`, the op most crosswalks are made of. */ +const singlePathOp = fc.constantFrom("move", "move", "move", "scale", "enum", "format", "split"); + +/** + * A single-path rule over paths drawn from the given generator, with a `when` drawn for its `from` some of the time. + * @param path - The paths both sides draw from. + * @param when - The `when` generator for a given `from`. + */ +function singlePathRule(path: fc.Arbitrary, when: (from: string) => fc.Arbitrary): fc.Arbitrary { + return fc.tuple(path, path, singlePathOp).chain(([from, to, op]) => fc.tuple(SINGLE_PATH_PARAMETERS[op], fc.option(when(from), { nil: undefined })).map(([extra, gate]) => ({ op, from, to, ...extra, ...(gate === undefined ? {} : { when: gate }) }))); +} + +/** One two-way rule of every op and every path shape, with a `when` some of the time. */ +const twoWayRule = fc.constantFrom(pathOfArity0, pathOfArity1, pathOfArity1, pathOfArity1, pathOfArity2).chain((path) => { + const template = fc.tuple(path, path, path).map(([given, family, to]) => ({ op: "template", from: { given, family }, to, template: "{family}-{given}" })); + const copy = fc.tuple(path, path, path).map(([from, first, second]) => ({ op: "copy", from, to: [first, second] })); + + return fc.oneof({ weight: 6, arbitrary: singlePathRule(path, whenFor) }, { weight: 1, arbitrary: template }, { weight: 1, arbitrary: copy }); +}); + +/** A single-path rule over keys and one wildcard with only keys around it, gated only on the whole rule: the rule shape most crosswalks are made of, generated on its own so it is met often. */ +const keyedRule = fc.constantFrom(keyPath, keyedPathOfArity1, keyedPathOfArity1).chain((path) => singlePathRule(path, () => wholeRuleWhen)); + +const direction = fc.constantFrom("encode", "decode"); + +/** One one-way rule of every op. */ +const oneWayRule = fc.oneof( + pathOfArity0.map((from) => ({ op: "drop", from, direction: "encode" })), + fc.tuple(pathOfArity0, fc.constantFrom(0, "x", null, { nested: true }), direction).map(([to, value, way]) => ({ op: "default", to, value, direction: way })), + fc.tuple(fc.constantFrom("count", "sum", "min", "max"), pathOfArity1, pathOfArity0, direction).map(([fn, from, to, way]) => ({ op: "aggregate", fn, from, to, direction: way })), +); + +/** Whether two generated rules name a path in common, which validation refuses: the crosswalks are generated without such pairs so that most of them execute. */ +function sharePath(left: object, right: object): boolean { + const paths = (rule: object): readonly string[] => { + try { + const parsed = parseCrosswalk({ source: "n", target: "s", rules: [rule] }).rules[0]; + + return parsed === undefined ? [] : [...ruleSides(parsed).from, ...ruleSides(parsed).to]; + } catch { + // A rule the schema refuses names no path the comparison could keep apart; the crosswalk it is part of is refused in turn. + return []; + } + }; + const theirs = new Set(paths(right)); + + return paths(left).some((path) => theirs.has(path)); +} + +/** + * A crosswalk of generated rules, some of them structurally invalid, which the comparison covers as well. + * @param rule - The two-way rules to draw from. + * @returns A parsed crosswalk, or `undefined` when the schema refuses a generated rule (a template whose fields repeat a path, say), which is no case for the executors. + */ +function crosswalkOf(rule: fc.Arbitrary): fc.Arbitrary { + return fc.tuple(fc.uniqueArray(rule, { minLength: 1, maxLength: MAX_RULES, comparator: sharePath }), fc.array(oneWayRule, { maxLength: MAX_ONE_WAY_RULES })).map(([rules, oneWay]) => { + try { + return parseCrosswalk({ source: "native", target: "shared", rules, oneWay }); + } catch { + // The schema's refusal is this generator's filter, not a result under test. + return undefined; + } + }); +} + +/** JSON scalars, several of which fit an op (enum keys, number text, ISO dates, separated text, JSON text). */ +const jsonLeaf = fc.constantFrom("open", "closed", "x", "1", "2.5", "2026-10-18", "2026-10-18T09:30:00.000Z", "true", '{"k":1}', "p,q", "p-q", 0, 1, FRACTIONAL, NEGATIVE, true, false, null); + +/** Values that are not JSON, or are JSON only in a form the output check would rewrite: each must fail, or survive, exactly as the reference has it. */ +const oddLeaf = fc.constantFrom( + undefined, + Number.NaN, + Number.POSITIVE_INFINITY, + BigInt(1), + Symbol("s"), + () => 0, + new Date(0), + Object.create(null), + JSON.parse('{"__proto__":{"a":1},"a":1}'), + Object.defineProperty([1], "extra", { value: true, enumerable: true }), + { [Symbol("hidden")]: 1, a: 1 }, +); + +const document = fc.letrec<{ leaf: unknown; field: unknown; list: unknown; record: unknown }>((tie) => ({ + leaf: fc.oneof({ weight: 12, arbitrary: jsonLeaf }, { weight: 1, arbitrary: oddLeaf }), + field: fc.oneof({ maxDepth: MAX_DEPTH, depthIdentifier: "document" }, tie("leaf"), tie("leaf"), tie("record")), + list: fc.oneof( + { maxDepth: MAX_DEPTH, depthIdentifier: "document" }, + tie("leaf"), + fc.array(tie("record"), { maxLength: MAX_ARRAY_LENGTH }), + fc.array(tie("record"), { maxLength: MAX_ARRAY_LENGTH }), + fc.array(tie("leaf"), { maxLength: MAX_ARRAY_LENGTH }), + fc.sparseArray(tie("record"), { maxLength: MAX_ARRAY_LENGTH }), + ), + record: fc.record({ a: tie("field"), b: tie("field"), c: tie("field"), xs: tie("list"), ys: tie("list") }, { requiredKeys: [] }), +})).record; + +/** A leaf of a fitted document: mostly JSON, sometimes not. */ +const fittedLeaf = fc.oneof({ weight: 12, arbitrary: jsonLeaf }, { weight: 1, arbitrary: oddLeaf }); + +/** + * Makes every path of a list present in one document, as far as the paths agree on its shape: each key a record, each wildcard or index an array of the given length, each end a drawn leaf. Where two paths disagree (one ends where another goes deeper), the first one placed wins and the other stays absent, so a drawn leaf (which may be a fixture shared between cases) is never written into. + * @param loci - The paths to make present. + * @param length - The length of every array the paths cross. + * @param draw - Draws a leaf. + * @returns The document. + */ +function fitted(loci: readonly Locus[], length: number, draw: () => unknown): Record { + const root: Record = {}; + const created = new WeakSet([root]); + for (const locus of loci) { + place(root, locus, 0, { length, draw, created }); + } + + return root; +} + +/** Places one path into a fitted document from the step at `at` down. */ +function place(node: unknown, locus: Locus, at: number, fit: { readonly length: number; readonly draw: () => unknown; readonly created: WeakSet }): void { + const step = locus[at]; + if (step === undefined || typeof node !== "object" || node === null || !fit.created.has(node)) { + return; + } + const next = locus[at + 1]; + const fresh = (): unknown => { + if (next === undefined) { + return fit.draw(); + } + + const container = next.kind === "key" ? {} : []; + fit.created.add(container); + + return container; + }; + if (step.kind === "key") { + if (Array.isArray(node)) { + return; + } + if (!Object.hasOwn(node, step.key)) { + Reflect.set(node, step.key, fresh()); + } + place(Reflect.get(node, step.key), locus, at + 1, fit); + + return; + } + if (!Array.isArray(node)) { + return; + } + while (node.length < fit.length) { + node.push(fresh()); + } + const positions = step.kind === "wildcard" ? [...node.keys()] : [step.index].filter((position) => position < node.length); + for (const position of positions) { + place(node[position], locus, at + 1, fit); + } +} + +/** + * Compares the two executors over generated crosswalks and documents: a generated native document encoded, a generated shared one decoded, a document fitted to each side's read paths in each direction, and what encode produced decoded (a shared document shaped by the crosswalk itself). + * @returns Every kind of outcome met. + */ +function compareOver(crosswalks: fc.Arbitrary): ReadonlySet { + const seen = new Set(); + const note = (kind: OutcomeKind | undefined): void => { + if (kind !== undefined) { + seen.add(kind); + } + }; + fc.assert( + fc.property(crosswalks, fc.record({ native: document, shared: document, length: fc.integer({ min: 0, max: MAX_ARRAY_LENGTH }), draw: fc.gen() }), (crosswalk, { native, shared, length, draw }) => { + if (crosswalk === undefined) { + return; + } + const sides = crosswalk.rules.map(ruleSides); + const fittedNative = fitted(sides.flatMap((side) => side.from).map(parsePath), length, () => draw(() => fittedLeaf)); + const fittedShared = fitted(sides.flatMap((side) => side.to).map(parsePath), length, () => draw(() => fittedLeaf)); + note(sameOutcome(crosswalk, native, "encode")); + note(sameOutcome(crosswalk, shared, "decode")); + note(sameOutcome(crosswalk, fittedNative, "encode")); + note(sameOutcome(crosswalk, fittedShared, "decode")); + const encoded = outcome(() => encodeReference(crosswalk, fittedNative)); + if ("output" in encoded) { + note(sameOutcome(crosswalk, encoded.value, "decode")); + } + }), + { numRuns: RUNS }, + ); + + return seen; +} + +describe("the executor against the reference executor", () => { + it("gives the same output or the same error as the reference over crosswalks of every op and path shape, both ways", () => { + expect([...compareOver(crosswalkOf(twoWayRule))].sort()).toEqual([...OUTCOME_KINDS].sort()); + }, PROPERTY_TIMEOUT_MILLISECONDS); + + it("gives the same output or the same error as the reference over crosswalks of keyed single-path rules, both ways", () => { + expect([...compareOver(crosswalkOf(keyedRule))].sort()).toEqual([...OUTCOME_KINDS].sort()); + }, PROPERTY_TIMEOUT_MILLISECONDS); + + it("gives the same result as the reference over every corpus sample, both ways", () => { + for (const cell of CORPUS) { + for (const sample of cell.samples) { + sameOutcome(cell.crosswalk, sample, "encode"); + sameOutcome(cell.crosswalk, encodeReference(cell.crosswalk, sample), "decode"); + } + } + }); +}); diff --git a/test/support/reference-executor.ts b/test/support/reference-executor.ts new file mode 100644 index 0000000..07df716 --- /dev/null +++ b/test/support/reference-executor.ts @@ -0,0 +1,294 @@ +/** + * The executor as it was before encode and decode were compiled, kept unchanged as the naive reference the compiled executor is tested against (src/executor-reference.unit.test.ts): one pass per rule, validation and the full output parse on every call. Its gap search is kept here too, since the shipped one in src/paths.ts was rewritten for speed. Not part of the package. + */ +import { type Crosswalk, type TwoWayRule } from "../../src/crosswalk"; +import { CrosswalkStructureError, RuleApplicationError } from "../../src/errors"; +import { enumDecode, enumEncode } from "../../src/ops/enum"; +import { formatDecode, formatEncode } from "../../src/ops/format"; +import { moveValue } from "../../src/ops/move"; +import { applyOneWayRule } from "../../src/ops/one-way"; +import { scaleDecode, scaleEncode } from "../../src/ops/scale"; +import { splitDecode, splitEncode } from "../../src/ops/split"; +import { type JsonValue, JsonValueSchema, type PredicateNode } from "trilean"; +import { bindLocus, type ElementGate, type EmptySite, ensureArray, formatPath, type Locus, parsePathMemo, readTrail, type Trail, walkLocus, wildcardIndexes, writeMatches } from "../../src/paths"; +import { ruleSides } from "../../src/sides"; +import { copyDecode, copyEncode } from "../../src/ops/copy"; +import { parseTemplateMemo, renderTemplate, splitTemplate } from "../../src/ops/template"; +import { validateCrosswalk } from "../../src/validate"; +import { evaluateWhen, whenArity } from "../../src/when"; + +/** + * Encodes a native document into the shared shape. Every rule's `from` is read from `doc` exactly as given and every `to` is written into a fresh output, so the input is never mutated and rule order carries no meaning (validateCrosswalk, run first, enforces the path uniqueness that makes this safe). A rule's `when` is evaluated against `doc`: definite true applies, definite false or indeterminate skips. Two-way rules write first, then the output's absent paths gain any due `default` values. + * @param crosswalk - The parsed crosswalk. + * @param doc - The native (source-side) document. + * @returns The shared (target-side) document. + * @throws CrosswalkStructureError when the crosswalk fails structural validation. + * @throws RuleApplicationError when a rule's `from` path is absent, a value does not fit the operation, two rules claim one output location, or an output array is left with an element no rule wrote (ruleIndex -1, since that gap belongs to no single rule). + */ +export function encodeReference(crosswalk: Crosswalk, doc: unknown): JsonValue { + return run(crosswalk, doc, "encode"); +} + +/** + * Decodes a shared document back into the native shape: the same executor with every two-way rule's paths swapped and each operation's inverse applied, so decode is a true inverse of encode over the paths the crosswalk maps. A rule's `when` is evaluated against `doc` (the target document, the decode input). + * @param crosswalk - The parsed crosswalk. + * @param doc - The shared (target-side) document. + * @returns The native (source-side) document. + * @throws CrosswalkStructureError when the crosswalk fails structural validation. + * @throws RuleApplicationError when a rule's `to` path (the decode read side) is absent, a value does not fit the inverse operation, two rules claim one output location, or an output array is left with a gap (ruleIndex -1). + */ +export function decodeReference(crosswalk: Crosswalk, doc: unknown): JsonValue { + return run(crosswalk, doc, "decode"); +} + +/** The rule index a RuleApplicationError carries when the failure belongs to the rule list as a whole: a gap in an output array is the absence of a write, which no one rule owns. */ +const NO_SINGLE_RULE = -1; + +function run(crosswalk: Crosswalk, doc: unknown, direction: "encode" | "decode"): JsonValue { + const issues = validateCrosswalk(crosswalk); + if (issues.length > 0) { + throw new CrosswalkStructureError(issues); + } + const target: Record = {}; + crosswalk.rules.forEach((rule, index) => { + applyTwoWayRule(rule, index, { direction, doc, target }); + }); + crosswalk.oneWay.forEach((rule, index) => { + if (rule.when !== undefined && !evaluateWhen(rule.when, doc)) { + return; + } + try { + applyOneWayRule(rule, direction, doc, target); + } catch (error) { + throw new RuleApplicationError(crosswalk.rules.length + index, rule.op === "default" ? rule.to : rule.from, errorMessage(error)); + } + }); + const hole = findHole(target); + if (hole !== undefined) { + throw new RuleApplicationError(NO_SINGLE_RULE, formatPath(hole), "no rule wrote this array element, so the array would have a gap"); + } + + return parseJsonValue(target); +} + +/** + * How a rule's `when` applies: a gate with no wildcard references is decided once for the whole rule (`false` skips it, `undefined` lets it run), and one whose references carry wildcards becomes a per-element gate over the read side, evaluated once per element at the depth its references reach. `governing` is the rule's read-side path in this direction, so the gate's references are rebased onto the arrays it iterates. + */ +function elementGate(when: PredicateNode, doc: unknown, governing: Locus): ElementGate | false | undefined { + const arity = whenArity(when); + if (arity === 0) { + return evaluateWhen(when, doc) ? undefined : false; + } + const wildcardPositions = governing.flatMap((step, position) => (step.kind === "wildcard" ? [position] : [])); + const arrayAt = (wildcard: number): Locus => { + const position = wildcardPositions[wildcard]; + if (position === undefined) { + throw new Error(`the rule's path has no wildcard ${String(wildcard)}: validateCrosswalk bounds a reference's wildcards by the rule's`); + } + + return governing.slice(0, position + 1); + }; + + return { arity, admit: (indexes) => evaluateWhen(when, doc, { indexes, arrayAt }) }; +} + +/** + * Applies one two-way rule: walks the governing read path one element at a time, reads the other read paths at the same indexes, transforms each element's values with the operation in this direction, and writes the results to the write-side paths at those indexes as it goes, so nothing per element outlives the element. Empty arrays met on any read side are materialised on every write-side path. + * @throws RuleApplicationError when a read path is absent, two read paths address different elements, the operation rejects a value, or a write conflicts. + */ +function applyTwoWayRule(rule: TwoWayRule, index: number, context: RunContext): void { + const { direction, doc, target } = context; + const sides = ruleSides(rule); + const [governingPath, ...otherReads] = direction === "encode" ? sides.from : sides.to; + const writes = direction === "encode" ? sides.to : sides.from; + if (governingPath === undefined) { + throw new Error("a rule with no read path: ruleSides always returns at least one"); + } + const governing = parsePathMemo(governingPath); + const gate = rule.when === undefined ? undefined : elementGate(rule.when, doc, governing); + if (gate === false) { + return; + } + const writeLoci = writes.map((path) => parsePathMemo(path)); + const readPath = (direction === "encode" ? sides.from : sides.to).join(", "); + const empties: { readonly locus: Locus; readonly empty: EmptySite }[] = []; + let governingCount = 0; + const countWalk = (locus: Locus): { readonly empties: readonly EmptySite[] | undefined; readonly count: number } => { + let count = 0; + const walked = walkLocus(doc, locus, gate, () => { + count += 1; + }); + + return { empties: walked, count }; + }; + const governingWalk = countWalk(governing); + if (governingWalk.empties === undefined) { + throw new RuleApplicationError(index, governingPath, "the path is absent from the document"); + } + governingCount = governingWalk.count; + for (const empty of governingWalk.empties) { + empties.push({ locus: governing, empty }); + } + const otherWalks = otherReads.map((path) => { + const locus = parsePathMemo(path); + const walked = countWalk(locus); + if (walked.empties === undefined) { + throw new RuleApplicationError(index, path, "the path is absent from the document"); + } + if (walked.count !== governingCount) { + throw new RuleApplicationError(index, path, `addresses different elements from ${governingPath}, so the paths cannot be paired element by element`); + } + for (const empty of walked.empties) { + empties.push({ locus, empty }); + } + + return locus; + }); + walkLocus(doc, governing, gate, (trail, value) => { + const indexes = wildcardIndexes(governing, trail); + const values = [value, ...otherWalks.map((locus, position) => { + const found = readTrail(doc, bindLocus(locus, indexes)); + if (!found.found) { + throw new RuleApplicationError(index, otherReads[position] ?? governingPath, `addresses different elements from ${governingPath}, so the paths cannot be paired element by element`); + } + + return found.value; + })]; + const results = transform(rule, direction, values, { index, path: readPath }); + const outputs = writeLoci.map((locus, position) => ({ trail: bindLocus(locus, indexes), value: results[position] })); + try { + writeMatches(target, outputs); + } catch (error) { + throw new RuleApplicationError(index, writes.join(", "), errorMessage(error)); + } + }); + try { + for (const { locus, empty } of empties) { + // An existing empty source array yields no matches, so without this its counterpart would never come into being: decode would find the path absent, or an inner array would leave a gap or a shorter outer array. + const emptyIndexes = wildcardIndexes(locus, empty.trail); + for (const writeLocus of writeLoci) { + ensureArray(target, bindLocus(writeLocus, emptyIndexes, empty.wildcard)); + } + } + } catch (error) { + throw new RuleApplicationError(index, writes.join(", "), errorMessage(error)); + } +} + +/** What every rule in one encode or decode call shares: the direction, the input document and the output under construction. */ +interface RunContext { + readonly direction: "encode" | "decode"; + readonly doc: unknown; + readonly target: Record; +} + +/** Applies one two-way rule's operation to one element's values in the given direction (one value per read path in, one per write path out), turning the operation's own thrown errors into RuleApplicationErrors that name the rule and path. */ +function transform(rule: TwoWayRule, direction: "encode" | "decode", values: readonly unknown[], where: { readonly index: number; readonly path: string }): readonly unknown[] { + try { + switch (rule.op) { + case "template": { + const pieces = parseTemplateMemo(rule.template); + const names = Object.keys(rule.from); + if (direction === "encode") { + return [renderTemplate(pieces, new Map(names.map((name, position) => [name, values[position]])))]; + } + const [text] = values; + if (typeof text !== "string") { + throw new Error(`template decode applies to strings, found: ${JSON.stringify(text)}`); + } + const parts = splitTemplate(pieces, text); + + return names.map((name) => parts.get(name)); + } + case "copy": + return direction === "encode" ? copyEncode(rule, values[0]) : [copyDecode(values)]; + case "move": + case "scale": + case "enum": + case "format": + case "split": + return [applyOp(rule, direction, values[0])]; + default: + return unhandledOp(); + } + } catch (error) { + if (error instanceof RuleApplicationError) { + throw error; + } + throw new RuleApplicationError(where.index, where.path, errorMessage(error)); + } +} + +/** Applies a single-path operation to one value in the given direction. */ +function applyOp(rule: Exclude, direction: "encode" | "decode", value: unknown): unknown { + switch (rule.op) { + case "move": + return moveValue(rule, value); + case "scale": + return direction === "encode" ? scaleEncode(rule, value) : scaleDecode(rule, value); + case "enum": + return direction === "encode" ? enumEncode(rule, value) : enumDecode(rule, value); + case "format": + return direction === "encode" ? formatEncode(rule, value) : formatDecode(rule, value); + case "split": + return direction === "encode" ? splitEncode(rule, value) : splitDecode(rule, value); + default: + return unhandledOp(); + } +} + +/** Unreachable for any rule that passed the schema: the default arm exists so the switch is total, and reaching it means this package's schema and executor disagree. */ +function unhandledOp(): never { + throw new Error("no operation handler for the rule's op: the schema and the executor disagree"); +} + +function errorMessage(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + + return String(error); +} + +/** The output document is only as JSON as the input document was (move passes values through untouched), so the declared JsonValue return type is earned at this boundary: a non-JSON value anywhere in the output fails loudly here rather than reaching a caller as a lie. */ +function parseJsonValue(target: Record): JsonValue { + return JsonValueSchema.parse(target); +} + +/** + * Finds the first array element nothing wrote: a hole left when a rule wrote a later index of an array (a fixed index past the next free one, or a gated rule skipping an element no other rule wrote). Holes are counted before any array is walked, so a large sparse array costs one key listing rather than a pass over its length. + * @param value - The output document under construction. + * @param trail - The location of `value` in the document, prefixed to the returned trail; the root call omits it. + * @returns The trail of the first hole in depth-first order, or `undefined` when there is none. + */ +function findHole(value: unknown, trail: Trail = []): Trail | undefined { + if (Array.isArray(value)) { + if (Object.keys(value).length !== value.length) { + const index = value.findIndex((_element: unknown, position) => !Object.hasOwn(value, position)); + + return [...trail, { kind: "index", index }]; + } + for (const [index, element] of value.entries()) { + const hole = findHole(element, [...trail, { kind: "index", index }]); + if (hole !== undefined) { + return hole; + } + } + + return undefined; + } + if (isRecord(value)) { + for (const [key, child] of Object.entries(value)) { + const hole = findHole(child, [...trail, { kind: "key", key }]); + if (hole !== undefined) { + return hole; + } + } + } + + return undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} From a7537f40a245038ceb3146ac3f3c73a1645fbe27 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 31 Oct 2026 15:22:56 +0000 Subject: [PATCH 03/12] perf(executor): validate and prepare a crosswalk once per crosswalk object compileCrosswalk validates a crosswalk, parses its paths and resolves each rule's sides and operation for both directions, returning a codec whose encode and decode take only the document. encode, decode and assertRoundTrip keep the compiled form in a WeakMap keyed by the crosswalk object, so existing callers validate once however many documents they run. An invalid crosswalk is not kept, so every attempt throws again. --- README.md | 12 +- src/compile.unit.test.ts | 61 +++++++++ src/executor.ts | 288 ++++++++++++++++++++++++++------------- src/index.ts | 3 +- src/roundtrip.ts | 12 +- 5 files changed, 273 insertions(+), 103 deletions(-) create mode 100644 src/compile.unit.test.ts diff --git a/README.md b/README.md index 0e0504d..f1fac2a 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ The exit status is 0 on success, 1 when the library rejects the crosswalk or the ## The format -A crosswalk is `{ source, target, rules, oneWay }`: two labels naming the shapes, a list of two-way rules and a list of one-way rules. `parseCrosswalk` validates the syntax and throws one error carrying every issue. `validateCrosswalk` checks the properties of the rule list as a whole, and `encode`, `decode` and `assertRoundTrip` run it first. +A crosswalk is `{ source, target, rules, oneWay }`: two labels naming the shapes, a list of two-way rules and a list of one-way rules. `parseCrosswalk` validates the syntax and throws one error carrying every issue. `validateCrosswalk` checks the properties of the rule list as a whole, and `compileCrosswalk`, `encode`, `decode` and `assertRoundTrip` run it before anything executes (once per crosswalk object, see [Execution](#execution)). ### Paths @@ -240,6 +240,16 @@ An element that every rule writing into it skips leaves a gap in the output arra The executor is pure and synchronous. Encode reads every `from` from the input as given and writes every `to` into a fresh output, so the input is never mutated. A `from` that is absent from the document is a `RuleApplicationError`, never a silent skip. The output is checked to be valid JSON throughout, so an input value that is not JSON (an `undefined` property, say) fails that check with a Zod error rather than reaching a caller. +A crosswalk is validated and prepared once, the first time it is used: `encode` and `decode` keep the prepared form for each crosswalk object, so running one crosswalk over many documents (every instance of a collection, both halves of a round trip) validates it once. `compileCrosswalk(crosswalk)` does the same step explicitly, throwing `CrosswalkStructureError` there rather than on the first run, and returns `{ encode, decode }` taking only the document. The crosswalk is read when it is prepared, so changing a crosswalk object after it has been used has no effect on it; parse a new one instead. + +```ts +import { compileCrosswalk } from "crossmap"; + +const codec = compileCrosswalk(network); +const shared = codec.encode(exported); +codec.decode(shared); +``` + The errors crossmap defines all extend `CrossmapError`: `CrosswalkSyntaxError` for a document that is not a crosswalk, `CrosswalkStructureError` for a rule list that parses but cannot execute, `RuleApplicationError` naming the rule and path that failed, and `RoundTripMismatchError` naming the sample and path. ## Round trips diff --git a/src/compile.unit.test.ts b/src/compile.unit.test.ts new file mode 100644 index 0000000..a12eea1 --- /dev/null +++ b/src/compile.unit.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it, vi } from "vitest"; +import { parseCrosswalk } from "./crosswalk"; +import { CrosswalkStructureError } from "./errors"; +import { compileCrosswalk, decode, encode } from "./executor"; +import { assertRoundTrip } from "./roundtrip"; +import { validateCrosswalk } from "./validate"; + +vi.mock(import("./validate"), async (original) => { + const actual = await original(); + + return { ...actual, validateCrosswalk: vi.fn(actual.validateCrosswalk) }; +}); + +/** How many encode and decode calls each test makes over one crosswalk object, any number above one showing whether validation repeats. */ +const CALLS = 3; + +const crosswalk = () => parseCrosswalk({ source: "n", target: "s", rules: [{ op: "move", from: "rows[].a", to: "items[].b" }] }); + +describe("compileCrosswalk", () => { + it("encodes and decodes as encode and decode do", () => { + const cw = crosswalk(); + const compiled = compileCrosswalk(cw); + const doc = { rows: [{ a: 1 }, { a: 2 }] }; + expect(compiled.encode(doc)).toEqual(encode(cw, doc)); + expect(compiled.decode({ items: [{ b: 1 }] })).toEqual(decode(cw, { items: [{ b: 1 }] })); + expect(compiled.crosswalk).toBe(cw); + }); + + it("returns the one compiled form for one crosswalk object", () => { + const cw = crosswalk(); + expect(compileCrosswalk(cw)).toBe(compileCrosswalk(cw)); + expect(compileCrosswalk(crosswalk())).not.toBe(compileCrosswalk(cw)); + }); + + it("validates a crosswalk once however many times it is run, through any entry point", () => { + const cw = crosswalk(); + vi.mocked(validateCrosswalk).mockClear(); + for (let call = 0; call < CALLS; call += 1) { + encode(cw, { rows: [{ a: call }] }); + decode(cw, { items: [{ b: call }] }); + compileCrosswalk(cw).encode({ rows: [] }); + } + assertRoundTrip(cw, [{ rows: [{ a: 1 }] }]); + expect(validateCrosswalk).toHaveBeenCalledTimes(1); + }); + + it("throws CrosswalkStructureError for a structurally invalid crosswalk, on every attempt", () => { + const invalid = parseCrosswalk({ + source: "n", + target: "s", + rules: [ + { op: "move", from: "a", to: "x" }, + { op: "move", from: "b", to: "x" }, + ], + }); + for (let call = 0; call < CALLS; call += 1) { + expect(() => compileCrosswalk(invalid)).toThrow(CrosswalkStructureError); + expect(() => encode(invalid, { a: 1, b: 2 })).toThrow(CrosswalkStructureError); + } + }); +}); diff --git a/src/executor.ts b/src/executor.ts index e1552ed..d9816a8 100644 --- a/src/executor.ts +++ b/src/executor.ts @@ -1,4 +1,4 @@ -import { type Crosswalk, type TwoWayRule } from "./crosswalk"; +import { type Crosswalk, type OneWayRule, type TwoWayRule } from "./crosswalk"; import { CrosswalkStructureError, RuleApplicationError } from "./errors"; import { enumDecode, enumEncode } from "./ops/enum"; import { formatDecode, formatEncode } from "./ops/format"; @@ -14,8 +14,59 @@ import { parseTemplateMemo, renderTemplate, splitTemplate } from "./ops/template import { validateCrosswalk } from "./validate"; import { evaluateWhen, whenArity } from "./when"; +/** The two directions a crosswalk runs in. */ +type Direction = "encode" | "decode"; + +/** + * A crosswalk validated and prepared once, for running any number of times: every path parsed, every rule's sides and operation resolved for each direction. Running it has exactly the semantics of {@link encode} and {@link decode} on the crosswalk it was compiled from. + */ +export interface CompiledCrosswalk { + /** The crosswalk this was compiled from. */ + readonly crosswalk: Crosswalk; + /** + * Encodes a native document into the shared shape, as {@link encode} does. + * @throws RuleApplicationError as {@link encode} does. + */ + readonly encode: (doc: unknown) => JsonValue; + /** + * Decodes a shared document back into the native shape, as {@link decode} does. + * @throws RuleApplicationError as {@link decode} does. + */ + readonly decode: (doc: unknown) => JsonValue; +} + +/** Every crosswalk compiled so far, keyed by the crosswalk object, so a caller passing the same crosswalk again pays for validation and preparation once. Weakly held, so a crosswalk no longer in use is not kept alive by its compiled form. */ +const compiledCrosswalks = new WeakMap(); + +/** + * Validates and prepares a crosswalk once, for encoding and decoding any number of documents. The result is kept for the crosswalk object, so compiling the same object again (as {@link encode} and {@link decode} do on every call) returns it at no cost; a crosswalk is read when it is first compiled, so an object changed afterwards keeps the behaviour it was first compiled with. + * @param crosswalk - The parsed crosswalk. + * @returns The compiled crosswalk. + * @throws CrosswalkStructureError when the crosswalk fails structural validation. + */ +export function compileCrosswalk(crosswalk: Crosswalk): CompiledCrosswalk { + const cached = compiledCrosswalks.get(crosswalk); + if (cached !== undefined) { + return cached; + } + const issues = validateCrosswalk(crosswalk); + if (issues.length > 0) { + throw new CrosswalkStructureError(issues); + } + const encodePlan = planFor(crosswalk, "encode"); + const decodePlan = planFor(crosswalk, "decode"); + const compiled: CompiledCrosswalk = { + crosswalk, + encode: (doc) => run(encodePlan, doc), + decode: (doc) => run(decodePlan, doc), + }; + compiledCrosswalks.set(crosswalk, compiled); + + return compiled; +} + /** - * Encodes a native document into the shared shape. Every rule's `from` is read from `doc` exactly as given and every `to` is written into a fresh output, so the input is never mutated and rule order carries no meaning (validateCrosswalk, run first, enforces the path uniqueness that makes this safe). A rule's `when` is evaluated against `doc`: definite true applies, definite false or indeterminate skips. Two-way rules write first, then the output's absent paths gain any due `default` values. + * Encodes a native document into the shared shape. Every rule's `from` is read from `doc` exactly as given and every `to` is written into a fresh output, so the input is never mutated and rule order carries no meaning (validateCrosswalk, run when the crosswalk is first compiled, enforces the path uniqueness that makes this safe). A rule's `when` is evaluated against `doc`: definite true applies, definite false or indeterminate skips. Two-way rules write first, then the output's absent paths gain any due `default` values. The crosswalk is compiled on its first use and the compiled form reused ({@link compileCrosswalk}). * @param crosswalk - The parsed crosswalk. * @param doc - The native (source-side) document. * @returns The shared (target-side) document. @@ -23,11 +74,11 @@ import { evaluateWhen, whenArity } from "./when"; * @throws RuleApplicationError when a rule's `from` path is absent, a value does not fit the operation, two rules claim one output location, or an output array is left with an element no rule wrote (ruleIndex -1, since that gap belongs to no single rule). */ export function encode(crosswalk: Crosswalk, doc: unknown): JsonValue { - return run(crosswalk, doc, "encode"); + return compileCrosswalk(crosswalk).encode(doc); } /** - * Decodes a shared document back into the native shape: the same executor with every two-way rule's paths swapped and each operation's inverse applied, so decode is a true inverse of encode over the paths the crosswalk maps. A rule's `when` is evaluated against `doc` (the target document, the decode input). + * Decodes a shared document back into the native shape: the same executor with every two-way rule's paths swapped and each operation's inverse applied, so decode is a true inverse of encode over the paths the crosswalk maps. A rule's `when` is evaluated against `doc` (the target document, the decode input). The crosswalk is compiled on its first use and the compiled form reused ({@link compileCrosswalk}). * @param crosswalk - The parsed crosswalk. * @param doc - The shared (target-side) document. * @returns The native (source-side) document. @@ -35,31 +86,92 @@ export function encode(crosswalk: Crosswalk, doc: unknown): JsonValue { * @throws RuleApplicationError when a rule's `to` path (the decode read side) is absent, a value does not fit the inverse operation, two rules claim one output location, or an output array is left with a gap (ruleIndex -1). */ export function decode(crosswalk: Crosswalk, doc: unknown): JsonValue { - return run(crosswalk, doc, "decode"); + return compileCrosswalk(crosswalk).decode(doc); } /** The rule index a RuleApplicationError carries when the failure belongs to the rule list as a whole: a gap in an output array is the absence of a write, which no one rule owns. */ const NO_SINGLE_RULE = -1; -function run(crosswalk: Crosswalk, doc: unknown, direction: "encode" | "decode"): JsonValue { - const issues = validateCrosswalk(crosswalk); - if (issues.length > 0) { - throw new CrosswalkStructureError(issues); +/** One read path of a rule in one direction, parsed. */ +interface ReadSide { + readonly path: string; + readonly locus: Locus; +} + +/** One two-way rule prepared for one direction: its read and write paths parsed, the strings its errors name, and its operation resolved. */ +interface RulePlan { + readonly index: number; + readonly when: PredicateNode | undefined; + /** The read path the rule walks, which every other read path must pair with element by element. */ + readonly governing: ReadSide; + readonly otherReads: readonly ReadSide[]; + readonly writeLoci: readonly Locus[]; + /** Every read path, as an operation's failure names them. */ + readonly readPath: string; + /** Every write path, as a write's failure names them. */ + readonly writePath: string; + /** The operation in this direction: one value per read path in, one per write path out. */ + readonly transform: (values: readonly unknown[]) => readonly unknown[]; +} + +/** One one-way rule with the index its failures carry. */ +interface OneWayPlan { + readonly index: number; + readonly rule: OneWayRule; +} + +/** A crosswalk prepared for one direction. */ +interface Plan { + readonly direction: Direction; + readonly rules: readonly RulePlan[]; + readonly oneWay: readonly OneWayPlan[]; +} + +function planFor(crosswalk: Crosswalk, direction: Direction): Plan { + return { + direction, + rules: crosswalk.rules.map((rule, index) => rulePlan(rule, index, direction)), + oneWay: crosswalk.oneWay.map((rule, index) => ({ index: crosswalk.rules.length + index, rule })), + }; +} + +function rulePlan(rule: TwoWayRule, index: number, direction: Direction): RulePlan { + const sides = ruleSides(rule); + const reads = direction === "encode" ? sides.from : sides.to; + const writes = direction === "encode" ? sides.to : sides.from; + const [governingPath, ...otherPaths] = reads; + if (governingPath === undefined) { + throw new Error("a rule with no read path: ruleSides always returns at least one"); } + const readSide = (path: string): ReadSide => ({ path, locus: parsePathMemo(path) }); + + return { + index, + when: rule.when, + governing: readSide(governingPath), + otherReads: otherPaths.map(readSide), + writeLoci: writes.map((path) => parsePathMemo(path)), + readPath: reads.join(", "), + writePath: writes.join(", "), + transform: transformFor(rule, direction), + }; +} + +function run(plan: Plan, doc: unknown): JsonValue { const target: Record = {}; - crosswalk.rules.forEach((rule, index) => { - applyTwoWayRule(rule, index, { direction, doc, target }); - }); - crosswalk.oneWay.forEach((rule, index) => { + for (const rule of plan.rules) { + applyTwoWayRule(rule, doc, target); + } + for (const { index, rule } of plan.oneWay) { if (rule.when !== undefined && !evaluateWhen(rule.when, doc)) { - return; + continue; } try { - applyOneWayRule(rule, direction, doc, target); + applyOneWayRule(rule, plan.direction, doc, target); } catch (error) { - throw new RuleApplicationError(crosswalk.rules.length + index, rule.op === "default" ? rule.to : rule.from, errorMessage(error)); + throw new RuleApplicationError(index, rule.op === "default" ? rule.to : rule.from, errorMessage(error)); } - }); + } const hole = findHole(target); if (hole !== undefined) { throw new RuleApplicationError(NO_SINGLE_RULE, formatPath(hole), "no rule wrote this array element, so the array would have a gap"); @@ -93,23 +205,13 @@ function elementGate(when: PredicateNode, doc: unknown, governing: Locus): Eleme * Applies one two-way rule: walks the governing read path one element at a time, reads the other read paths at the same indexes, transforms each element's values with the operation in this direction, and writes the results to the write-side paths at those indexes as it goes, so nothing per element outlives the element. Empty arrays met on any read side are materialised on every write-side path. * @throws RuleApplicationError when a read path is absent, two read paths address different elements, the operation rejects a value, or a write conflicts. */ -function applyTwoWayRule(rule: TwoWayRule, index: number, context: RunContext): void { - const { direction, doc, target } = context; - const sides = ruleSides(rule); - const [governingPath, ...otherReads] = direction === "encode" ? sides.from : sides.to; - const writes = direction === "encode" ? sides.to : sides.from; - if (governingPath === undefined) { - throw new Error("a rule with no read path: ruleSides always returns at least one"); - } - const governing = parsePathMemo(governingPath); - const gate = rule.when === undefined ? undefined : elementGate(rule.when, doc, governing); +function applyTwoWayRule(rule: RulePlan, doc: unknown, target: Record): void { + const { index, governing, otherReads, writeLoci } = rule; + const gate = rule.when === undefined ? undefined : elementGate(rule.when, doc, governing.locus); if (gate === false) { return; } - const writeLoci = writes.map((path) => parsePathMemo(path)); - const readPath = (direction === "encode" ? sides.from : sides.to).join(", "); const empties: { readonly locus: Locus; readonly empty: EmptySite }[] = []; - let governingCount = 0; const countWalk = (locus: Locus): { readonly empties: readonly EmptySite[] | undefined; readonly count: number } => { let count = 0; const walked = walkLocus(doc, locus, gate, () => { @@ -118,45 +220,45 @@ function applyTwoWayRule(rule: TwoWayRule, index: number, context: RunContext): return { empties: walked, count }; }; - const governingWalk = countWalk(governing); + const governingWalk = countWalk(governing.locus); if (governingWalk.empties === undefined) { - throw new RuleApplicationError(index, governingPath, "the path is absent from the document"); + throw new RuleApplicationError(index, governing.path, "the path is absent from the document"); } - governingCount = governingWalk.count; + const governingCount = governingWalk.count; for (const empty of governingWalk.empties) { - empties.push({ locus: governing, empty }); + empties.push({ locus: governing.locus, empty }); } - const otherWalks = otherReads.map((path) => { - const locus = parsePathMemo(path); - const walked = countWalk(locus); + for (const read of otherReads) { + const walked = countWalk(read.locus); if (walked.empties === undefined) { - throw new RuleApplicationError(index, path, "the path is absent from the document"); + throw new RuleApplicationError(index, read.path, "the path is absent from the document"); } if (walked.count !== governingCount) { - throw new RuleApplicationError(index, path, `addresses different elements from ${governingPath}, so the paths cannot be paired element by element`); + throw new RuleApplicationError(index, read.path, `addresses different elements from ${governing.path}, so the paths cannot be paired element by element`); } for (const empty of walked.empties) { - empties.push({ locus, empty }); + empties.push({ locus: read.locus, empty }); } + } + walkLocus(doc, governing.locus, gate, (trail, value) => { + const indexes = wildcardIndexes(governing.locus, trail); + const values = [ + value, + ...otherReads.map((read) => { + const found = readTrail(doc, bindLocus(read.locus, indexes)); + if (!found.found) { + throw new RuleApplicationError(index, read.path, `addresses different elements from ${governing.path}, so the paths cannot be paired element by element`); + } - return locus; - }); - walkLocus(doc, governing, gate, (trail, value) => { - const indexes = wildcardIndexes(governing, trail); - const values = [value, ...otherWalks.map((locus, position) => { - const found = readTrail(doc, bindLocus(locus, indexes)); - if (!found.found) { - throw new RuleApplicationError(index, otherReads[position] ?? governingPath, `addresses different elements from ${governingPath}, so the paths cannot be paired element by element`); - } - - return found.value; - })]; - const results = transform(rule, direction, values, { index, path: readPath }); + return found.value; + }), + ]; + const results = transformed(rule, values); const outputs = writeLoci.map((locus, position) => ({ trail: bindLocus(locus, indexes), value: results[position] })); try { writeMatches(target, outputs); } catch (error) { - throw new RuleApplicationError(index, writes.join(", "), errorMessage(error)); + throw new RuleApplicationError(index, rule.writePath, errorMessage(error)); } }); try { @@ -168,67 +270,67 @@ function applyTwoWayRule(rule: TwoWayRule, index: number, context: RunContext): } } } catch (error) { - throw new RuleApplicationError(index, writes.join(", "), errorMessage(error)); + throw new RuleApplicationError(index, rule.writePath, errorMessage(error)); } } -/** What every rule in one encode or decode call shares: the direction, the input document and the output under construction. */ -interface RunContext { - readonly direction: "encode" | "decode"; - readonly doc: unknown; - readonly target: Record; +/** Applies a rule's operation to one element's values, turning the operation's own thrown errors into RuleApplicationErrors that name the rule and its read paths. */ +function transformed(rule: RulePlan, values: readonly unknown[]): readonly unknown[] { + try { + return rule.transform(values); + } catch (error) { + throw new RuleApplicationError(rule.index, rule.readPath, errorMessage(error)); + } } -/** Applies one two-way rule's operation to one element's values in the given direction (one value per read path in, one per write path out), turning the operation's own thrown errors into RuleApplicationErrors that name the rule and path. */ -function transform(rule: TwoWayRule, direction: "encode" | "decode", values: readonly unknown[], where: { readonly index: number; readonly path: string }): readonly unknown[] { - try { - switch (rule.op) { - case "template": { - const pieces = parseTemplateMemo(rule.template); - const names = Object.keys(rule.from); - if (direction === "encode") { - return [renderTemplate(pieces, new Map(names.map((name, position) => [name, values[position]])))]; - } - const [text] = values; +/** Resolves a two-way rule's operation in one direction, once, to a function from one value per read path to one value per write path. */ +function transformFor(rule: TwoWayRule, direction: Direction): (values: readonly unknown[]) => readonly unknown[] { + switch (rule.op) { + case "template": { + const pieces = parseTemplateMemo(rule.template); + const names = Object.keys(rule.from); + if (direction === "encode") { + return (values) => [renderTemplate(pieces, new Map(names.map((name, position) => [name, values[position]])))]; + } + + return ([text]) => { if (typeof text !== "string") { throw new Error(`template decode applies to strings, found: ${JSON.stringify(text)}`); } const parts = splitTemplate(pieces, text); return names.map((name) => parts.get(name)); - } - case "copy": - return direction === "encode" ? copyEncode(rule, values[0]) : [copyDecode(values)]; - case "move": - case "scale": - case "enum": - case "format": - case "split": - return [applyOp(rule, direction, values[0])]; - default: - return unhandledOp(); + }; } - } catch (error) { - if (error instanceof RuleApplicationError) { - throw error; + case "copy": + return direction === "encode" ? ([value]) => copyEncode(rule, value) : (values) => [copyDecode(values)]; + case "move": + case "scale": + case "enum": + case "format": + case "split": { + const op = singlePathOp(rule, direction); + + return ([value]) => [op(value)]; } - throw new RuleApplicationError(where.index, where.path, errorMessage(error)); + default: + return unhandledOp(); } } -/** Applies a single-path operation to one value in the given direction. */ -function applyOp(rule: Exclude, direction: "encode" | "decode", value: unknown): unknown { +/** Resolves a single-path operation in one direction to a function of the one value it reads. */ +function singlePathOp(rule: Exclude, direction: Direction): (value: unknown) => unknown { switch (rule.op) { case "move": - return moveValue(rule, value); + return (value) => moveValue(rule, value); case "scale": - return direction === "encode" ? scaleEncode(rule, value) : scaleDecode(rule, value); + return direction === "encode" ? (value) => scaleEncode(rule, value) : (value) => scaleDecode(rule, value); case "enum": - return direction === "encode" ? enumEncode(rule, value) : enumDecode(rule, value); + return direction === "encode" ? (value) => enumEncode(rule, value) : (value) => enumDecode(rule, value); case "format": - return direction === "encode" ? formatEncode(rule, value) : formatDecode(rule, value); + return direction === "encode" ? (value) => formatEncode(rule, value) : (value) => formatDecode(rule, value); case "split": - return direction === "encode" ? splitEncode(rule, value) : splitDecode(rule, value); + return direction === "encode" ? (value) => splitEncode(rule, value) : (value) => splitDecode(rule, value); default: return unhandledOp(); } diff --git a/src/index.ts b/src/index.ts index 9ea2ce1..dc9467e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,7 +19,8 @@ export type { } from "./crosswalk"; export { CrosswalkStructureError, CrosswalkSyntaxError, RuleApplicationError, RoundTripMismatchError, CrossmapError } from "./errors"; export type { CrosswalkIssue, SyntaxIssue } from "./errors"; -export { decode, encode } from "./executor"; +export { compileCrosswalk, decode, encode } from "./executor"; +export type { CompiledCrosswalk } from "./executor"; export { assertRoundTrip } from "./roundtrip"; export { validateCrosswalk } from "./validate"; export type { JsonValue } from "trilean"; diff --git a/src/roundtrip.ts b/src/roundtrip.ts index 4355dc8..8ac445a 100644 --- a/src/roundtrip.ts +++ b/src/roundtrip.ts @@ -1,9 +1,8 @@ import { type Crosswalk } from "./crosswalk"; -import { CrosswalkStructureError, RoundTripMismatchError } from "./errors"; +import { RoundTripMismatchError } from "./errors"; import { formatPath, type Locus, type PathStep, parsePathMemo, type TrailStep } from "./paths"; -import { decode, encode } from "./executor"; +import { compileCrosswalk } from "./executor"; import { setOwn } from "./own"; -import { validateCrosswalk } from "./validate"; /** * Checks that decode inverts encode over every sample, on the paths the crosswalk promises to preserve. The paths one-way rules touch are excluded by construction (a dropped path cannot survive; a decode-direction default or aggregate inserts a path the sample never had), then each sample's round trip is deep-compared with Object.is at the leaves; the first difference throws. @@ -15,10 +14,7 @@ import { validateCrosswalk } from "./validate"; * @throws RoundTripMismatchError naming the sample and path of the first difference. */ export function assertRoundTrip(crosswalk: Crosswalk, samples: readonly unknown[]): void { - const issues = validateCrosswalk(crosswalk); - if (issues.length > 0) { - throw new CrosswalkStructureError(issues); - } + const codec = compileCrosswalk(crosswalk); const excluded = crosswalk.oneWay.flatMap((rule): readonly string[] => { if (rule.op === "drop") { return [rule.from]; @@ -28,7 +24,7 @@ export function assertRoundTrip(crosswalk: Crosswalk, samples: readonly unknown[ }); const excludedLoci = excluded.map(parsePathMemo); samples.forEach((sample, sampleIndex) => { - const roundTripped = decode(crosswalk, encode(crosswalk, sample)); + const roundTripped = codec.decode(codec.encode(sample)); const left = prunePaths(sample, excludedLoci); const right = prunePaths(roundTripped, excludedLoci); compare(left, right, [], sampleIndex); From e490c60e38d3347cf1ac563108a0fc02c7dfe680 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 31 Oct 2026 15:33:27 +0000 Subject: [PATCH 04/12] perf(executor): walk each source array once for all the rules that read it When every two-way rule in a direction is a single-path op over keys and at most one wildcard, gated only on the whole rule, with write paths that never run through one another and each output array fed by one source array, the rules sharing an array are applied element by element in one walk, writing straight into the output element. A move carries its value with no call, a record moved whole is stored in one step, and no trail is built per element. Anything the fused run does not decide for itself (an absent path, a hole, a value an op rejects, a when that throws) makes it give way to the rule-at-a-time run, which then gives the exact error or result. The gap search runs only where a stored value or a one-way write can leave a gap. A property test requires that the fused run never gives way where the reference completes. --- src/executor-reference.unit.test.ts | 29 +++ src/executor.ts | 54 +++-- src/fused.ts | 319 ++++++++++++++++++++++++++++ src/fused.unit.test.ts | 97 +++++++++ src/ops/single-path.ts | 32 +++ src/paths.ts | 8 + 6 files changed, 511 insertions(+), 28 deletions(-) create mode 100644 src/fused.ts create mode 100644 src/fused.unit.test.ts create mode 100644 src/ops/single-path.ts diff --git a/src/executor-reference.unit.test.ts b/src/executor-reference.unit.test.ts index d6b5523..75d1a49 100644 --- a/src/executor-reference.unit.test.ts +++ b/src/executor-reference.unit.test.ts @@ -5,6 +5,7 @@ import { type Crosswalk, parseCrosswalk } from "./crosswalk"; import { RuleApplicationError } from "./errors"; import { decode, encode } from "./executor"; import { type Locus, parsePath } from "./paths"; +import { fusedPlan, runFused } from "./fused"; import { ruleSides } from "./sides"; import { decodeReference, encodeReference } from "../test/support/reference-executor"; @@ -393,6 +394,34 @@ describe("the executor against the reference executor", () => { expect([...compareOver(crosswalkOf(keyedRule))].sort()).toEqual([...OUTCOME_KINDS].sort()); }, PROPERTY_TIMEOUT_MILLISECONDS); + it("never leaves to the general executor a run of keyed single-path rules that the reference completes", () => { + let fusedRuns = 0; + fc.assert( + fc.property(crosswalkOf(keyedRule), fc.integer({ min: 0, max: MAX_ARRAY_LENGTH }), fc.gen(), (crosswalk, length, draw) => { + if (crosswalk === undefined) { + return; + } + for (const way of ["encode", "decode"] as const) { + const plan = fusedPlan(crosswalk.rules, way); + const sides = crosswalk.rules.map(ruleSides); + const doc = fitted(sides.flatMap((side) => (way === "encode" ? side.from : side.to)).map(parsePath), length, () => draw(() => jsonLeaf)); + // The fused run is the two-way rules alone, so the reference runs without the one-way rules that follow it. + const twoWay = { ...crosswalk, oneWay: [] }; + const expected = outcome(() => (way === "encode" ? encodeReference(twoWay, doc) : decodeReference(twoWay, doc))); + if (plan === undefined || !("output" in expected)) { + continue; + } + const fused = runFused(plan, doc); + expect(fused).toBeDefined(); + expect(JSON.stringify(fused?.target)).toBe(expected.output); + fusedRuns += 1; + } + }), + { numRuns: RUNS }, + ); + expect(fusedRuns).toBeGreaterThan(0); + }, PROPERTY_TIMEOUT_MILLISECONDS); + it("gives the same result as the reference over every corpus sample, both ways", () => { for (const cell of CORPUS) { for (const sample of cell.samples) { diff --git a/src/executor.ts b/src/executor.ts index d9816a8..25e027f 100644 --- a/src/executor.ts +++ b/src/executor.ts @@ -1,11 +1,6 @@ import { type Crosswalk, type OneWayRule, type TwoWayRule } from "./crosswalk"; import { CrosswalkStructureError, RuleApplicationError } from "./errors"; -import { enumDecode, enumEncode } from "./ops/enum"; -import { formatDecode, formatEncode } from "./ops/format"; -import { moveValue } from "./ops/move"; import { applyOneWayRule } from "./ops/one-way"; -import { scaleDecode, scaleEncode } from "./ops/scale"; -import { splitDecode, splitEncode } from "./ops/split"; import { type JsonValue, JsonValueSchema, type PredicateNode } from "trilean"; import { bindLocus, type ElementGate, type EmptySite, ensureArray, findHole, formatPath, type Locus, parsePathMemo, readTrail, walkLocus, wildcardIndexes, writeMatches } from "./paths"; import { ruleSides } from "./sides"; @@ -13,6 +8,8 @@ import { copyDecode, copyEncode } from "./ops/copy"; import { parseTemplateMemo, renderTemplate, splitTemplate } from "./ops/template"; import { validateCrosswalk } from "./validate"; import { evaluateWhen, whenArity } from "./when"; +import { type FusedPlan, fusedPlan, runFused } from "./fused"; +import { singlePathOp } from "./ops/single-path"; /** The two directions a crosswalk runs in. */ type Direction = "encode" | "decode"; @@ -125,6 +122,10 @@ interface Plan { readonly direction: Direction; readonly rules: readonly RulePlan[]; readonly oneWay: readonly OneWayPlan[]; + /** The two-way rules arranged to walk each source array once, when every rule has the shape that allows it. */ + readonly fused: FusedPlan | undefined; + /** Whether a one-way rule writes in this direction, which can leave a gap in an output array the fused run built without one. */ + readonly oneWayWrites: boolean; } function planFor(crosswalk: Crosswalk, direction: Direction): Plan { @@ -132,9 +133,12 @@ function planFor(crosswalk: Crosswalk, direction: Direction): Plan { direction, rules: crosswalk.rules.map((rule, index) => rulePlan(rule, index, direction)), oneWay: crosswalk.oneWay.map((rule, index) => ({ index: crosswalk.rules.length + index, rule })), + fused: fusedPlan(crosswalk.rules, direction), + oneWayWrites: crosswalk.oneWay.some((rule) => rule.op !== "drop" && rule.direction === direction), }; } + function rulePlan(rule: TwoWayRule, index: number, direction: Direction): RulePlan { const sides = ruleSides(rule); const reads = direction === "encode" ? sides.from : sides.to; @@ -157,11 +161,12 @@ function rulePlan(rule: TwoWayRule, index: number, direction: Direction): RulePl }; } +/** + * Runs a plan over a document: the fused plan first where there is one, and the rules one at a time when there is none or the fused run gives way (it gives way on anything it does not decide for itself, so every error comes from the rule-at-a-time run, exactly as that run reports it). + */ function run(plan: Plan, doc: unknown): JsonValue { - const target: Record = {}; - for (const rule of plan.rules) { - applyTwoWayRule(rule, doc, target); - } + const fused = plan.fused === undefined ? undefined : runFused(plan.fused, doc); + const target = fused === undefined ? runRules(plan, doc) : fused.target; for (const { index, rule } of plan.oneWay) { if (rule.when !== undefined && !evaluateWhen(rule.when, doc)) { continue; @@ -172,7 +177,8 @@ function run(plan: Plan, doc: unknown): JsonValue { throw new RuleApplicationError(index, rule.op === "default" ? rule.to : rule.from, errorMessage(error)); } } - const hole = findHole(target); + // A fused run fills each output array it builds from the start, so the search can find something only in a value stored whole or after a one-way write. + const hole = fused === undefined || fused.storedGap || plan.oneWayWrites ? findHole(target) : undefined; if (hole !== undefined) { throw new RuleApplicationError(NO_SINGLE_RULE, formatPath(hole), "no rule wrote this array element, so the array would have a gap"); } @@ -180,6 +186,16 @@ function run(plan: Plan, doc: unknown): JsonValue { return parseJsonValue(target); } +/** Applies the two-way rules one at a time, each over the whole document, into a fresh output. */ +function runRules(plan: Plan, doc: unknown): Record { + const target: Record = {}; + for (const rule of plan.rules) { + applyTwoWayRule(rule, doc, target); + } + + return target; +} + /** * How a rule's `when` applies: a gate with no wildcard references is decided once for the whole rule (`false` skips it, `undefined` lets it run), and one whose references carry wildcards becomes a per-element gate over the read side, evaluated once per element at the depth its references reach. `governing` is the rule's read-side path in this direction, so the gate's references are rebased onto the arrays it iterates. */ @@ -318,24 +334,6 @@ function transformFor(rule: TwoWayRule, direction: Direction): (values: readonly } } -/** Resolves a single-path operation in one direction to a function of the one value it reads. */ -function singlePathOp(rule: Exclude, direction: Direction): (value: unknown) => unknown { - switch (rule.op) { - case "move": - return (value) => moveValue(rule, value); - case "scale": - return direction === "encode" ? (value) => scaleEncode(rule, value) : (value) => scaleDecode(rule, value); - case "enum": - return direction === "encode" ? (value) => enumEncode(rule, value) : (value) => enumDecode(rule, value); - case "format": - return direction === "encode" ? (value) => formatEncode(rule, value) : (value) => formatDecode(rule, value); - case "split": - return direction === "encode" ? (value) => splitEncode(rule, value) : (value) => splitDecode(rule, value); - default: - return unhandledOp(); - } -} - /** Unreachable for any rule that passed the schema: the default arm exists so the switch is total, and reaching it means this package's schema and executor disagree. */ function unhandledOp(): never { throw new Error("no operation handler for the rule's op: the schema and the executor disagree"); diff --git a/src/fused.ts b/src/fused.ts new file mode 100644 index 0000000..610b76d --- /dev/null +++ b/src/fused.ts @@ -0,0 +1,319 @@ +import type { PredicateNode } from "trilean"; +import type { TwoWayRule } from "./crosswalk"; +import { singlePathOp } from "./ops/single-path"; +import { findHole, type Locus, markPlaced, parsePathMemo } from "./paths"; +import { evaluateWhen, whenArity } from "./when"; + +/** + * A single-path rule in one direction as the fused executor needs it: its index, its `when`, its read and write paths parsed, and its operation, or `undefined` for a `move`, whose value is carried as it is. + */ +interface FusableRule { + readonly index: number; + readonly when: PredicateNode | undefined; + readonly read: Locus; + readonly write: Locus; + readonly op: ((value: unknown) => unknown) | undefined; +} + +/** One rule of a group: the keys it reads below the element and writes below the output element, and its operation. */ +interface Field { + readonly index: number; + readonly when: PredicateNode | undefined; + readonly read: readonly string[]; + readonly write: readonly string[]; + readonly op: ((value: unknown) => unknown) | undefined; +} + +/** Rules that iterate one source array into one output array, each element of which is walked once for all of them. */ +interface Group { + readonly kind: "group"; + readonly arrayRead: readonly string[]; + readonly arrayWrite: readonly string[]; + readonly fields: readonly Field[]; +} + +/** A rule with no wildcard: one value read and written. */ +interface Single extends Field { + readonly kind: "single"; +} + +/** What a fused run wrote. */ +export class FusedOutput { + /** The output document. */ + readonly target: Record = {}; + + #storedGap = false; + + /** Whether a value stored whole holds an array the gap search stops at: the arrays the run builds itself are filled from the start, so only such a value, or a later one-way write, can make the output's gap search find anything. */ + get storedGap(): boolean { + return this.#storedGap; + } + + /** + * Notes a record or array stored whole as a value: marks it as one, so later writes cannot merge into it, and notes whether it holds an array the gap search stops at. + * @param value - The stored value. + */ + stored(value: object): void { + markPlaced(value); + this.#storedGap ||= findHole(value) !== undefined; + } +} + +/** A crosswalk's two-way rules in one direction, arranged so that each source array is walked once. */ +export interface FusedPlan { + readonly items: readonly (Group | Single)[]; +} + +/** + * Arranges a crosswalk's two-way rules in one direction for fused execution, when every rule has the shape it needs: a single-path operation (not `template` or `copy`), a `when` that is absent or decided once for the whole rule, keys on both sides with at most one wildcard among them, write paths none of which runs through another (no write path is a prefix of another, and no location is both an array one rule iterates and a record another writes into), and each output array fed by one source array. Under these conditions every write lands on a location no other write touches, so the writes need no conflict checks and their order changes nothing but the order keys are created in, which the run keeps. + * @param crosswalkRules - The crosswalk's two-way rules, in rule order. + * @param direction - The direction the plan runs in: decode reads each rule's `to` and writes its `from`. + * @returns The plan, or `undefined` when some rule needs the general executor. + */ +export function fusedPlan(crosswalkRules: readonly TwoWayRule[], direction: "encode" | "decode"): FusedPlan | undefined { + const rules: FusableRule[] = []; + for (const [index, rule] of crosswalkRules.entries()) { + if (rule.op === "template" || rule.op === "copy") { + return undefined; + } + const [read, write] = direction === "encode" ? [rule.from, rule.to] : [rule.to, rule.from]; + rules.push({ index, when: rule.when, read: parsePathMemo(read), write: parsePathMemo(write), op: rule.op === "move" ? undefined : singlePathOp(rule, direction) }); + } + if (!rules.every((rule) => isKeyed(rule.read) && isKeyed(rule.write) && (rule.when === undefined || whenArity(rule.when) === 0)) || !writesAreDisjoint(rules.map((rule) => rule.write))) { + return undefined; + } + const items: (Group | Single)[] = []; + const groups = new Map(); + for (const rule of rules) { + const read = splitAtWildcard(rule.read); + const write = splitAtWildcard(rule.write); + if (read === undefined || write === undefined) { + items.push({ kind: "single", index: rule.index, when: rule.when, read: keysOf(rule.read), write: keysOf(rule.write), op: rule.op }); + continue; + } + const field: Field = { index: rule.index, when: rule.when, read: read.after, write: write.after, op: rule.op }; + // The output array's key list as JSON, in which every key is quoted and escaped, so two arrays share a map key only when they share every step. + const arrayKey = JSON.stringify(write.before); + const existing = groups.get(arrayKey); + if (existing === undefined) { + const fields = [field]; + const group: Group = { kind: "group", arrayRead: read.before, arrayWrite: write.before, fields }; + groups.set(arrayKey, { group, fields }); + items.push(group); + continue; + } + if (!sameKeys(existing.group.arrayRead, read.before)) { + // Two source arrays feeding one output array (which validation allows on the decode side): their elements would be written by separate walks, so the general executor keeps the key order exact. + return undefined; + } + existing.fields.push(field); + } + + return { items }; +} + +/** Whether a locus is only keys and at most one wildcard. */ +function isKeyed(locus: Locus): boolean { + return locus.every((step) => step.kind !== "index") && locus.filter((step) => step.kind === "wildcard").length <= 1; +} + +function keysOf(locus: Locus): readonly string[] { + return locus.flatMap((step) => (step.kind === "key" ? [step.key] : [])); +} + +/** The keys before and after a keyed locus's wildcard, or `undefined` when it has none. */ +function splitAtWildcard(locus: Locus): { readonly before: readonly string[]; readonly after: readonly string[] } | undefined { + const at = locus.findIndex((step) => step.kind === "wildcard"); + if (at < 0) { + return undefined; + } + + return { before: keysOf(locus.slice(0, at)), after: keysOf(locus.slice(at + 1)) }; +} + +function sameKeys(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((key, position) => key === right[position]); +} + +/** A node of the tree the write paths spell, step by step: a wildcard child, key children, and whether a path ends here. */ +interface WriteNode { + ends: boolean; + wildcard: WriteNode | undefined; + readonly keys: Map; +} + +/** Whether no write path runs through another: none ends where another ends or continues, and no location is continued both by a wildcard and by a key. A fixed index is not something the fused plan writes, so a path with one is not disjoint for it. */ +function writesAreDisjoint(writes: readonly Locus[]): boolean { + const root = writeNode(); + for (const write of writes) { + let node = root; + for (const step of write) { + if (node.ends) { + return false; + } + if (step.kind === "wildcard") { + node.wildcard ??= writeNode(); + node = node.wildcard; + continue; + } + if (step.kind === "index") { + return false; + } + const child = node.keys.get(step.key) ?? writeNode(); + node.keys.set(step.key, child); + node = child; + } + if (node.ends || node.wildcard !== undefined || node.keys.size > 0) { + return false; + } + node.ends = true; + } + + return continuedOneWay(root); +} + +function writeNode(): WriteNode { + return { ends: false, wildcard: undefined, keys: new Map() }; +} + +/** Whether no node of the write tree is continued both by a wildcard and by a key. */ +function continuedOneWay(node: WriteNode): boolean { + if (node.wildcard !== undefined && node.keys.size > 0) { + return false; + } + + return [...node.keys.values(), ...(node.wildcard === undefined ? [] : [node.wildcard])].every(continuedOneWay); +} + +/** Thrown to stop a fused run that meets a value or shape it does not decide for itself; nothing about the reason is kept, since the general executor that runs next decides the outcome and reports it exactly. */ +const DECLINED = new Error("the fused executor gives way to the general one"); + +/** + * Runs a fused plan over a document: each source array is walked once, every rule of its group read from each element and written into one output element, and each value a `move` carries is stored as it is (a record or array moved whole is shared with the input, and marked as a stored value so later writes cannot merge into it). + * @param plan - The plan for this direction. + * @param doc - The input document. + * @returns The output, or `undefined` when anything is absent, the wrong shape or rejected by an operation, or a `when` cannot be evaluated: the general executor then runs the crosswalk from the start and gives the exact error or result. + */ +export function runFused(plan: FusedPlan, doc: unknown): FusedOutput | undefined { + const output = new FusedOutput(); + try { + for (const { item, fields } of activeItems(plan, doc)) { + if (item.kind === "single") { + place(output.target, item.write, carried(item, readKeys(doc, item.read), output)); + continue; + } + runGroup(item, fields, doc, output); + } + } catch { + // Whatever stopped the run (a decline above, an operation rejecting a value, a `when` that throws), the general executor runs next and raises it with the rule and path it belongs to, or gives the result; the tests require that a run the general executor completes is never declined. + return undefined; + } + + return output; +} + +/** The items a run applies, in the order the general executor would first write each one, with each group's fields narrowed to those whose `when` admits them. */ +function activeItems(plan: FusedPlan, doc: unknown): readonly { readonly item: Group | Single; readonly fields: readonly Field[] }[] { + const active: { readonly item: Group | Single; readonly fields: readonly Field[]; readonly first: number }[] = []; + for (const item of plan.items) { + const fields = (item.kind === "single" ? [item] : item.fields).filter((field) => field.when === undefined || evaluateWhen(field.when, doc)); + const [first] = fields; + if (first !== undefined) { + active.push({ item, fields, first: first.index }); + } + } + + return active.sort((left, right) => left.first - right.first); +} + +function runGroup(group: Group, fields: readonly Field[], doc: unknown, run: FusedOutput): void { + const source = readKeys(doc, group.arrayRead); + if (!Array.isArray(source)) { + throw DECLINED; + } + const output: unknown[] = []; + place(run.target, group.arrayWrite, output); + const [only] = fields; + if (fields.length === 1 && only?.write.length === 0) { + for (const element of source) { + output.push(carried(only, readElement(element, only.read), run)); + } + + return; + } + for (const element of source) { + const row: Record = {}; + for (const field of fields) { + place(row, field.write, carried(field, readElement(element, field.read), run)); + } + output.push(row); + } +} + +/** Applies a field's operation to the value read, and marks a record or array the field stores as a value, noting whether it holds an array the gap search would stop at. */ +function carried(field: Field, value: unknown, run: FusedOutput): unknown { + const result = field.op === undefined ? value : field.op(value); + if (typeof result === "object" && result !== null) { + run.stored(result); + } + + return result; +} + +/** Reads below one array element; an element that is not there (a hole, which the general executor skips) declines the run. */ +function readElement(element: unknown, keys: readonly string[]): unknown { + if (element === undefined) { + throw DECLINED; + } + + return readKeys(element, keys); +} + +/** Reads a value by own keys, declining the run when any key is absent or meets a value that is not a record. */ +function readKeys(from: unknown, keys: readonly string[]): unknown { + let value = from; + for (const key of keys) { + if (!isRecord(value) || !Object.hasOwn(value, key)) { + throw DECLINED; + } + value = value[key]; + } + + return value; +} + +/** Stores a value at keys below a record of the output, creating the records on the way: the plan's write paths are disjoint, so every record met was created by this run and the final key is free. */ +function place(into: Record, keys: readonly string[], value: unknown): void { + let node = into; + const last = keys.length - 1; + for (let position = 0; position < last; position += 1) { + const key = stepAt(keys, position); + const existing = node[key]; + if (existing === undefined) { + const created: Record = {}; + node[key] = created; + node = created; + continue; + } + // Unreachable while the plan's writes are disjoint, as fusedPlan requires; declining leaves the general executor to report whatever the conflict is. + if (!isRecord(existing)) { + throw DECLINED; + } + node = existing; + } + node[stepAt(keys, last)] = value; +} + +/** The key at a position the caller's loop bounds keep inside the list. */ +function stepAt(keys: readonly string[], position: number): string { + const key = keys[position]; + if (key === undefined) { + throw new Error("a write path with no key at a position inside its length: a fused write path always has at least one key"); + } + + return key; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/src/fused.unit.test.ts b/src/fused.unit.test.ts new file mode 100644 index 0000000..402aa86 --- /dev/null +++ b/src/fused.unit.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from "vitest"; +import { parseCrosswalk } from "./crosswalk"; +import { fusedPlan, runFused } from "./fused"; + +const rules = (list: readonly object[]) => parseCrosswalk({ source: "n", target: "s", rules: list }).rules; + +describe("fusedPlan", () => { + it("fuses keyed single-path rules, grouping the rules of one array into one walk", () => { + const plan = fusedPlan( + rules([ + { op: "move", from: "meta", to: "meta" }, + { op: "move", from: "nodes[].id", to: "nodes[].id" }, + { op: "scale", from: "nodes[].w", to: "nodes[].attributes.weight", factor: 2 }, + { op: "move", from: "edges[].source", to: "links[].from" }, + ]), + "encode", + ); + expect(plan?.items.map((item) => (item.kind === "group" ? { array: item.arrayWrite, fields: item.fields.map((field) => field.write) } : item.write))).toEqual([ + ["meta"], + { array: ["nodes"], fields: [["id"], ["attributes", "weight"]] }, + { array: ["links"], fields: [["from"]] }, + ]); + }); + + it.each([ + ["a rule with several paths on a side", [{ op: "copy", from: "a", to: ["b", "c"] }]], + ["a fixed index", [{ op: "move", from: "rows[0].a", to: "first" }]], + ["two wildcards", [{ op: "move", from: "rows[].cells[]", to: "grid[].cells[]" }]], + ["a gate per element", [{ op: "move", from: "rows[].a", to: "out[].a", when: { kind: "textCompare", op: "equals", left: { kind: "reference", key: "rows[].k" }, right: { kind: "textLiteral", value: "x" } } }]], + [ + "a write path running through another", + [ + { op: "move", from: "a", to: "x" }, + { op: "move", from: "b", to: "x.y" }, + ], + ], + [ + "a location both iterated and written into by key", + [ + { op: "move", from: "rows[].a", to: "x[].a" }, + { op: "move", from: "b", to: "x.b" }, + ], + ], + ])("leaves %s to the general executor", (_name, list) => { + expect(fusedPlan(rules(list), "encode")).toBeUndefined(); + }); + + it("leaves to the general executor two source arrays feeding one output array, which only decode can meet", () => { + const list = rules([ + { op: "move", from: "rows[].a", to: "left[].a" }, + { op: "move", from: "rows[].b", to: "right[].b" }, + ]); + expect(fusedPlan(list, "encode")).toBeDefined(); + expect(fusedPlan(list, "decode")).toBeUndefined(); + }); +}); + +describe("runFused", () => { + const plan = fusedPlan( + rules([ + { op: "move", from: "meta", to: "meta" }, + { op: "move", from: "nodes[].id", to: "items[].id" }, + { op: "move", from: "nodes[].text", to: "items[].label.text" }, + ]), + "encode", + ); + if (plan === undefined) { + throw new Error("the plan under test should fuse"); + } + + it("walks each array once into one output element per element, sharing a record moved whole", () => { + const meta = { name: "g" }; + const output = runFused(plan, { meta, nodes: [{ id: "a", text: "x" }, { id: "b", text: "y" }] }); + expect(output?.target).toEqual({ meta, items: [{ id: "a", label: { text: "x" } }, { id: "b", label: { text: "y" } }] }); + expect(output?.target["meta"]).toBe(meta); + }); + + /** An array whose first element is a hole. */ + const holed = (...after: readonly unknown[]): unknown[] => { + const array: unknown[] = []; + array[1] = undefined; + array.splice(1, 1, ...after); + + return array; + }; + + it("gives way on an absent path, a hole or a value of the wrong shape", () => { + expect(runFused(plan, { meta: {}, nodes: [{ id: "a" }] })).toBeUndefined(); + expect(runFused(plan, { meta: {}, nodes: holed({ id: "a", text: "x" }) })).toBeUndefined(); + expect(runFused(plan, { meta: {}, nodes: { id: "a", text: "x" } })).toBeUndefined(); + }); + + it("notes a value stored whole that holds an array with a hole", () => { + expect(runFused(plan, { meta: { list: holed(2) }, nodes: [] })?.storedGap).toBe(true); + expect(runFused(plan, { meta: { list: [1, 2] }, nodes: [] })?.storedGap).toBe(false); + }); +}); diff --git a/src/ops/single-path.ts b/src/ops/single-path.ts new file mode 100644 index 0000000..a697e18 --- /dev/null +++ b/src/ops/single-path.ts @@ -0,0 +1,32 @@ +import type { TwoWayRule } from "../crosswalk"; +import { enumDecode, enumEncode } from "./enum"; +import { formatDecode, formatEncode } from "./format"; +import { moveValue } from "./move"; +import { scaleDecode, scaleEncode } from "./scale"; +import { splitDecode, splitEncode } from "./split"; + +/** The two-way rules with one path on each side. */ +export type SinglePathRule = Exclude; + +/** Resolves a single-path operation in one direction to a function of the one value it reads. */ +export function singlePathOp(rule: SinglePathRule, direction: "encode" | "decode"): (value: unknown) => unknown { + switch (rule.op) { + case "move": + return (value) => moveValue(rule, value); + case "scale": + return direction === "encode" ? (value) => scaleEncode(rule, value) : (value) => scaleDecode(rule, value); + case "enum": + return direction === "encode" ? (value) => enumEncode(rule, value) : (value) => enumDecode(rule, value); + case "format": + return direction === "encode" ? (value) => formatEncode(rule, value) : (value) => formatDecode(rule, value); + case "split": + return direction === "encode" ? (value) => splitEncode(rule, value) : (value) => splitDecode(rule, value); + default: + return unhandledOp(); + } +} + +/** Unreachable for any rule that passed the schema: the default arm exists so the switch is total, and reaching it means this package's schema and executor disagree. */ +function unhandledOp(): never { + throw new Error("no operation handler for the rule's op: the schema and the executor disagree"); +} diff --git a/src/paths.ts b/src/paths.ts index 372badb..59cae5f 100644 --- a/src/paths.ts +++ b/src/paths.ts @@ -411,6 +411,14 @@ export function readTrail(root: unknown, trail: Trail): { readonly found: true; */ const placedValues = new WeakSet(); +/** + * Records that a container was stored in an output as a value by a writer other than {@link writeMatches}, so that later writes through {@link writeMatches} treat it as the leaf it is rather than as structure to merge into. + * @param value - The object or array stored. + */ +export function markPlaced(value: object): void { + placedValues.add(value); +} + /** * Writes values into a target document, creating every intermediate record and array the trails need (materialising an array to the highest index written into it). An index step creates or enters an array and a key step a record, so the two never stand in for each other. * @param target - The mutable output document; records and arrays along each trail are created in it as needed. From 31408d25ae1c81c96d3db7070f2b4a3c6fb71b68 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 31 Oct 2026 15:37:20 +0000 Subject: [PATCH 05/12] perf(executor): check the output is JSON without copying it The output used to go through the JSON value schema's parse, which traverses and copies the whole document. isExactJson accepts exactly the values that parse would give back unchanged (finite numbers, plain records and ordinary arrays with nothing the schema drops or rejects), so an output it accepts is returned as it is and any other is parsed, which throws or copies just as before. The fused run checks each value as it stores it and gives way on one that is not exact, so a completed fused run needs neither this check nor the gap search. A record or array a move carries whole is now shared with the input rather than copied. --- README.md | 2 +- src/exact-json.ts | 62 ++++++++++++++++++++++++ src/exact-json.unit.test.ts | 45 +++++++++++++++++ src/executor-reference.unit.test.ts | 2 +- src/executor.ts | 22 ++++++--- src/fused.ts | 75 ++++++++++++----------------- src/fused.unit.test.ts | 14 ++++-- src/paths.ts | 2 +- 8 files changed, 165 insertions(+), 59 deletions(-) create mode 100644 src/exact-json.ts create mode 100644 src/exact-json.unit.test.ts diff --git a/README.md b/README.md index f1fac2a..ab8c007 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ An element that every rule writing into it skips leaves a gap in the output arra ### Execution -The executor is pure and synchronous. Encode reads every `from` from the input as given and writes every `to` into a fresh output, so the input is never mutated. A `from` that is absent from the document is a `RuleApplicationError`, never a silent skip. The output is checked to be valid JSON throughout, so an input value that is not JSON (an `undefined` property, say) fails that check with a Zod error rather than reaching a caller. +The executor is pure and synchronous. Encode reads every `from` from the input as given and writes every `to` into a fresh output, so the input is never mutated. A `from` that is absent from the document is a `RuleApplicationError`, never a silent skip. The output is checked to be valid JSON throughout, so an input value that is not JSON (an `undefined` property, say) fails that check with a Zod error rather than reaching a caller. The check does not copy the output: a record or array that a `move` carries whole is the input's own object, shared by the output, so copy it (`structuredClone`) before changing either one in place. A crosswalk is validated and prepared once, the first time it is used: `encode` and `decode` keep the prepared form for each crosswalk object, so running one crosswalk over many documents (every instance of a collection, both halves of a round trip) validates it once. `compileCrosswalk(crosswalk)` does the same step explicitly, throwing `CrosswalkStructureError` there rather than on the first run, and returns `{ encode, decode }` taking only the document. The crosswalk is read when it is prepared, so changing a crosswalk object after it has been used has no effect on it; parse a new one instead. diff --git a/src/exact-json.ts b/src/exact-json.ts new file mode 100644 index 0000000..f95ee86 --- /dev/null +++ b/src/exact-json.ts @@ -0,0 +1,62 @@ +import type { JsonValue } from "trilean"; + +/** + * Whether a value is JSON exactly as trilean's JsonValueSchema would give it back: a string, a finite number, a boolean, null, an array or a plain record of such values, with nothing the schema's parse would drop or reject. When this holds, parsing the value with the schema succeeds and returns a copy deeply and strictly equal to it, so the value can stand for the parsed copy without the copy being made. It is conservative: a value the schema accepts only by rewriting it (a record with a null or other prototype, a non-enumerable or `__proto__` own key, an array holding a hole or a non-index key) fails this check, and the caller then parses it to get the schema's own result. + * @param value - The value to check. + * @returns Whether the value is exact JSON. + */ +export function isExactJson(value: unknown): value is JsonValue { + switch (typeof value) { + case "string": + case "boolean": + return true; + case "number": + // The schema's number refuses NaN and both infinities. + return Number.isFinite(value); + case "object": + if (value === null) { + return true; + } + + return Array.isArray(value) ? isExactArray(value) : isExactRecord(value); + case "bigint": + case "function": + case "symbol": + case "undefined": + return false; + default: + return unknownType(); + } +} + +/** An array the schema copies element for element: an ordinary array whose every element is exact (a hole reads as `undefined`, which is not) and with no key but its indices, since the schema's copy keeps only the indices. */ +function isExactArray(array: readonly unknown[]): boolean { + if (Object.getPrototypeOf(array) !== Array.prototype) { + return false; + } + for (const element of array) { + if (!isExactJson(element)) { + return false; + } + } + + return Object.keys(array).length === array.length; +} + +/** A record the schema copies key for key: an ordinary object whose own keys are all enumerable strings other than `__proto__` (the schema refuses a symbol key and drops the others) and whose every value is exact. */ +function isExactRecord(record: object): boolean { + if (Object.getPrototypeOf(record) !== Object.prototype || Object.getOwnPropertySymbols(record).length > 0) { + return false; + } + const keys = Object.keys(record); + if (keys.length !== Object.getOwnPropertyNames(record).length || keys.includes("__proto__")) { + return false; + } + + return keys.every((key) => isExactJson(Reflect.get(record, key))); +} + +/** Unreachable: `typeof` has exactly the eight results the switch names, and this arm exists so it is total. */ +function unknownType(): never { + throw new Error("a typeof result outside the eight the language defines"); +} diff --git a/src/exact-json.unit.test.ts b/src/exact-json.unit.test.ts new file mode 100644 index 0000000..abfddb2 --- /dev/null +++ b/src/exact-json.unit.test.ts @@ -0,0 +1,45 @@ +import fc from "fast-check"; +import { JsonValueSchema } from "trilean"; +import { describe, expect, it } from "vitest"; +import { isExactJson } from "./exact-json"; + +/** Requires that a value the check accepts is one the schema parses into a strictly equal copy, which is what lets the executor return it without the copy. */ +function acceptedMeansParsedUnchanged(value: unknown): void { + if (isExactJson(value)) { + expect(JsonValueSchema.parse(value)).toStrictEqual(value); + } +} + +/** A number with a fractional part. */ +const FRACTIONAL = 1.5; + +describe("isExactJson", () => { + it("accepts JSON scalars, arrays and plain records", () => { + for (const value of ["", "x", 0, -0, FRACTIONAL, true, false, null, [], [1, "a", null], {}, { a: { b: [1, { c: "d" }] } }]) { + expect(isExactJson(value)).toBe(true); + } + }); + + it("refuses what the schema refuses: non-finite numbers, undefined, bigints, symbols, functions and class instances", () => { + for (const value of [Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY, undefined, BigInt(1), Symbol("s"), () => 0, new Date(0), new Map(), [undefined], { a: undefined }, { [Symbol("k")]: 1 }]) { + expect(isExactJson(value)).toBe(false); + expect(JsonValueSchema.safeParse(value).success).toBe(false); + } + }); + + it("refuses what the schema accepts only by rewriting it, so the caller parses it instead", () => { + const holed: unknown[] = []; + holed[1] = 1; + const bare: unknown = Object.create(null); + const hidden = Object.defineProperty({ a: 1 }, "b", { value: 2, enumerable: false }); + const named = Object.defineProperty([1], "extra", { value: true, enumerable: true }); + const proto: unknown = JSON.parse('{"__proto__":{"a":1},"b":2}'); + for (const value of [holed, bare, hidden, named, proto]) { + expect(isExactJson(value)).toBe(false); + } + }); + + it("accepts a value only when the schema parses it into a strictly equal copy, over generated values of every kind", () => { + fc.assert(fc.property(fc.oneof(fc.jsonValue(), fc.anything({ withBigInt: true, withBoxedValues: true, withDate: true, withMap: true, withSet: true, withNullPrototype: true, withObjectString: true, withSparseArray: true, withTypedArray: true, withUnicodeString: true })), acceptedMeansParsedUnchanged)); + }); +}); diff --git a/src/executor-reference.unit.test.ts b/src/executor-reference.unit.test.ts index 75d1a49..2f669ec 100644 --- a/src/executor-reference.unit.test.ts +++ b/src/executor-reference.unit.test.ts @@ -413,7 +413,7 @@ describe("the executor against the reference executor", () => { } const fused = runFused(plan, doc); expect(fused).toBeDefined(); - expect(JSON.stringify(fused?.target)).toBe(expected.output); + expect(JSON.stringify(fused)).toBe(expected.output); fusedRuns += 1; } }), diff --git a/src/executor.ts b/src/executor.ts index 25e027f..b205598 100644 --- a/src/executor.ts +++ b/src/executor.ts @@ -10,6 +10,7 @@ import { validateCrosswalk } from "./validate"; import { evaluateWhen, whenArity } from "./when"; import { type FusedPlan, fusedPlan, runFused } from "./fused"; import { singlePathOp } from "./ops/single-path"; +import { isExactJson } from "./exact-json"; /** The two directions a crosswalk runs in. */ type Direction = "encode" | "decode"; @@ -124,7 +125,7 @@ interface Plan { readonly oneWay: readonly OneWayPlan[]; /** The two-way rules arranged to walk each source array once, when every rule has the shape that allows it. */ readonly fused: FusedPlan | undefined; - /** Whether a one-way rule writes in this direction, which can leave a gap in an output array the fused run built without one. */ + /** Whether a one-way rule writes in this direction: its write can leave a gap, or a value that is not JSON, in an output the fused run built without either. */ readonly oneWayWrites: boolean; } @@ -166,7 +167,7 @@ function rulePlan(rule: TwoWayRule, index: number, direction: Direction): RulePl */ function run(plan: Plan, doc: unknown): JsonValue { const fused = plan.fused === undefined ? undefined : runFused(plan.fused, doc); - const target = fused === undefined ? runRules(plan, doc) : fused.target; + const target: Record = fused ?? runRules(plan, doc); for (const { index, rule } of plan.oneWay) { if (rule.when !== undefined && !evaluateWhen(rule.when, doc)) { continue; @@ -177,13 +178,16 @@ function run(plan: Plan, doc: unknown): JsonValue { throw new RuleApplicationError(index, rule.op === "default" ? rule.to : rule.from, errorMessage(error)); } } - // A fused run fills each output array it builds from the start, so the search can find something only in a value stored whole or after a one-way write. - const hole = fused === undefined || fused.storedGap || plan.oneWayWrites ? findHole(target) : undefined; + if (fused !== undefined && !plan.oneWayWrites) { + // A completed fused run's output is exact JSON with no gap (runFused), and no one-way rule wrote into it. + return fused; + } + const hole = findHole(target); if (hole !== undefined) { throw new RuleApplicationError(NO_SINGLE_RULE, formatPath(hole), "no rule wrote this array element, so the array would have a gap"); } - return parseJsonValue(target); + return jsonOutput(target); } /** Applies the two-way rules one at a time, each over the whole document, into a fresh output. */ @@ -347,7 +351,9 @@ function errorMessage(error: unknown): string { return String(error); } -/** The output document is only as JSON as the input document was (move passes values through untouched), so the declared JsonValue return type is earned at this boundary: a non-JSON value anywhere in the output fails loudly here rather than reaching a caller as a lie. */ -function parseJsonValue(target: Record): JsonValue { - return JsonValueSchema.parse(target); +/** + * The output document is only as JSON as the input document was (move passes values through untouched), so the declared JsonValue return type is earned at this boundary: a non-JSON value anywhere in the output fails loudly here rather than reaching a caller as a lie. An output that is exact JSON is returned as it is, which is what parsing it would return a copy of; any other goes through the schema, which throws for a value that is not JSON and copies one it accepts by rewriting it. + */ +function jsonOutput(target: Record): JsonValue { + return isExactJson(target) ? target : JsonValueSchema.parse(target); } diff --git a/src/fused.ts b/src/fused.ts index 610b76d..26b8cb7 100644 --- a/src/fused.ts +++ b/src/fused.ts @@ -1,7 +1,8 @@ -import type { PredicateNode } from "trilean"; +import type { JsonValue, PredicateNode } from "trilean"; import type { TwoWayRule } from "./crosswalk"; import { singlePathOp } from "./ops/single-path"; -import { findHole, type Locus, markPlaced, parsePathMemo } from "./paths"; +import { isExactJson } from "./exact-json"; +import { type Locus, markPlaced, parsePathMemo } from "./paths"; import { evaluateWhen, whenArity } from "./when"; /** @@ -37,28 +38,6 @@ interface Single extends Field { readonly kind: "single"; } -/** What a fused run wrote. */ -export class FusedOutput { - /** The output document. */ - readonly target: Record = {}; - - #storedGap = false; - - /** Whether a value stored whole holds an array the gap search stops at: the arrays the run builds itself are filled from the start, so only such a value, or a later one-way write, can make the output's gap search find anything. */ - get storedGap(): boolean { - return this.#storedGap; - } - - /** - * Notes a record or array stored whole as a value: marks it as one, so later writes cannot merge into it, and notes whether it holds an array the gap search stops at. - * @param value - The stored value. - */ - stored(value: object): void { - markPlaced(value); - this.#storedGap ||= findHole(value) !== undefined; - } -} - /** A crosswalk's two-way rules in one direction, arranged so that each source array is walked once. */ export interface FusedPlan { readonly items: readonly (Group | Single)[]; @@ -188,28 +167,31 @@ function continuedOneWay(node: WriteNode): boolean { /** Thrown to stop a fused run that meets a value or shape it does not decide for itself; nothing about the reason is kept, since the general executor that runs next decides the outcome and reports it exactly. */ const DECLINED = new Error("the fused executor gives way to the general one"); +/** A record of the output a fused run builds. */ +type JsonRecord = Record; + /** - * Runs a fused plan over a document: each source array is walked once, every rule of its group read from each element and written into one output element, and each value a `move` carries is stored as it is (a record or array moved whole is shared with the input, and marked as a stored value so later writes cannot merge into it). + * Runs a fused plan over a document: each source array is walked once, every rule of its group read from each element and written into one output element, and each value a `move` carries stored as it is (a record or array moved whole is shared with the input, and marked as a stored value so later writes cannot merge into it). Every value stored is checked to be exact JSON ({@link isExactJson}) as it is stored, and the records and arrays the run builds are exact JSON by construction, so a completed run's output needs no further check, gap search or copy. * @param plan - The plan for this direction. * @param doc - The input document. - * @returns The output, or `undefined` when anything is absent, the wrong shape or rejected by an operation, or a `when` cannot be evaluated: the general executor then runs the crosswalk from the start and gives the exact error or result. + * @returns The output document, or `undefined` when anything is absent, the wrong shape, rejected by an operation or not exact JSON, or a `when` cannot be evaluated: the general executor then runs the crosswalk from the start and gives the exact error or result. */ -export function runFused(plan: FusedPlan, doc: unknown): FusedOutput | undefined { - const output = new FusedOutput(); +export function runFused(plan: FusedPlan, doc: unknown): JsonRecord | undefined { + const target: JsonRecord = {}; try { for (const { item, fields } of activeItems(plan, doc)) { if (item.kind === "single") { - place(output.target, item.write, carried(item, readKeys(doc, item.read), output)); + place(target, item.write, carried(item, readKeys(doc, item.read))); continue; } - runGroup(item, fields, doc, output); + runGroup(item, fields, doc, target); } } catch { // Whatever stopped the run (a decline above, an operation rejecting a value, a `when` that throws), the general executor runs next and raises it with the rule and path it belongs to, or gives the result; the tests require that a run the general executor completes is never declined. return undefined; } - return output; + return target; } /** The items a run applies, in the order the general executor would first write each one, with each group's fields narrowed to those whose `when` admits them. */ @@ -226,35 +208,38 @@ function activeItems(plan: FusedPlan, doc: unknown): readonly { readonly item: G return active.sort((left, right) => left.first - right.first); } -function runGroup(group: Group, fields: readonly Field[], doc: unknown, run: FusedOutput): void { +function runGroup(group: Group, fields: readonly Field[], doc: unknown, target: JsonRecord): void { const source = readKeys(doc, group.arrayRead); if (!Array.isArray(source)) { throw DECLINED; } - const output: unknown[] = []; - place(run.target, group.arrayWrite, output); + const output: JsonValue[] = []; + place(target, group.arrayWrite, output); const [only] = fields; if (fields.length === 1 && only?.write.length === 0) { for (const element of source) { - output.push(carried(only, readElement(element, only.read), run)); + output.push(carried(only, readElement(element, only.read))); } return; } for (const element of source) { - const row: Record = {}; + const row: JsonRecord = {}; for (const field of fields) { - place(row, field.write, carried(field, readElement(element, field.read), run)); + place(row, field.write, carried(field, readElement(element, field.read))); } output.push(row); } } -/** Applies a field's operation to the value read, and marks a record or array the field stores as a value, noting whether it holds an array the gap search would stop at. */ -function carried(field: Field, value: unknown, run: FusedOutput): unknown { +/** Applies a field's operation to the value read, declines the run unless the result is exact JSON, and marks a record or array the field stores as a value. */ +function carried(field: Field, value: unknown): JsonValue { const result = field.op === undefined ? value : field.op(value); + if (!isExactJson(result)) { + throw DECLINED; + } if (typeof result === "object" && result !== null) { - run.stored(result); + markPlaced(result); } return result; @@ -283,20 +268,20 @@ function readKeys(from: unknown, keys: readonly string[]): unknown { } /** Stores a value at keys below a record of the output, creating the records on the way: the plan's write paths are disjoint, so every record met was created by this run and the final key is free. */ -function place(into: Record, keys: readonly string[], value: unknown): void { +function place(into: JsonRecord, keys: readonly string[], value: JsonValue): void { let node = into; const last = keys.length - 1; for (let position = 0; position < last; position += 1) { const key = stepAt(keys, position); const existing = node[key]; if (existing === undefined) { - const created: Record = {}; + const created: JsonRecord = {}; node[key] = created; node = created; continue; } // Unreachable while the plan's writes are disjoint, as fusedPlan requires; declining leaves the general executor to report whatever the conflict is. - if (!isRecord(existing)) { + if (!isJsonRecord(existing)) { throw DECLINED; } node = existing; @@ -317,3 +302,7 @@ function stepAt(keys: readonly string[], position: number): string { function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } + +function isJsonRecord(value: JsonValue): value is JsonRecord { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/src/fused.unit.test.ts b/src/fused.unit.test.ts index 402aa86..b866774 100644 --- a/src/fused.unit.test.ts +++ b/src/fused.unit.test.ts @@ -71,8 +71,8 @@ describe("runFused", () => { it("walks each array once into one output element per element, sharing a record moved whole", () => { const meta = { name: "g" }; const output = runFused(plan, { meta, nodes: [{ id: "a", text: "x" }, { id: "b", text: "y" }] }); - expect(output?.target).toEqual({ meta, items: [{ id: "a", label: { text: "x" } }, { id: "b", label: { text: "y" } }] }); - expect(output?.target["meta"]).toBe(meta); + expect(output).toEqual({ meta, items: [{ id: "a", label: { text: "x" } }, { id: "b", label: { text: "y" } }] }); + expect(output?.["meta"]).toBe(meta); }); /** An array whose first element is a hole. */ @@ -90,8 +90,12 @@ describe("runFused", () => { expect(runFused(plan, { meta: {}, nodes: { id: "a", text: "x" } })).toBeUndefined(); }); - it("notes a value stored whole that holds an array with a hole", () => { - expect(runFused(plan, { meta: { list: holed(2) }, nodes: [] })?.storedGap).toBe(true); - expect(runFused(plan, { meta: { list: [1, 2] }, nodes: [] })?.storedGap).toBe(false); + it("gives way on a value it would store that is not exact JSON, which the general executor reports or rewrites", () => { + expect(runFused(plan, { meta: { list: holed(2) }, nodes: [] })).toBeUndefined(); + expect(runFused(plan, { meta: Number.NaN, nodes: [] })).toBeUndefined(); + expect(runFused(plan, { meta: {}, nodes: [{ id: undefined, text: "x" }] })).toBeUndefined(); + const bare: unknown = Object.create(null); + expect(runFused(plan, { meta: bare, nodes: [] })).toBeUndefined(); + expect(runFused(plan, { meta: { list: [1, 2] }, nodes: [] })).toEqual({ meta: { list: [1, 2] }, items: [] }); }); }); diff --git a/src/paths.ts b/src/paths.ts index 59cae5f..3c2d766 100644 --- a/src/paths.ts +++ b/src/paths.ts @@ -23,7 +23,7 @@ export type Trail = readonly TrailStep[]; const BARE_KEY = "[A-Za-z0-9_-]+"; /** - * A bare segment: `[A-Za-z0-9_-]+` except exactly `__proto__`. Every output passes through trilean's JSON value schema, which drops an own `__proto__` key without a word, so a rule mapping that key would lose its value silently; refusing the segment in the grammar makes it fail loudly where the crosswalk is parsed. + * A bare segment: `[A-Za-z0-9_-]+` except exactly `__proto__`. An output holding an own `__proto__` key is checked by trilean's JSON value schema, which drops the key without a word, so a rule mapping that key would lose its value silently; refusing the segment in the grammar makes it fail loudly where the crosswalk is parsed. */ const SEGMENT = `(?!__proto__(?![A-Za-z0-9_-]))${BARE_KEY}`; From 151b3bf0eb7caae3774e73a4b8f92247429fc482 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 31 Oct 2026 15:43:26 +0000 Subject: [PATCH 06/12] chore(bench): measure a crosswalk the rule-at-a-time executor runs --- bench/encode.ts | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/bench/encode.ts b/bench/encode.ts index eedbd8e..b04487f 100644 --- a/bench/encode.ts +++ b/bench/encode.ts @@ -66,6 +66,17 @@ const mixedCrosswalk = parseCrosswalk({ ], }); +/** Identity moves for every field and one more gated per element, which the rule-at-a-time executor runs (a gate per element is not fused). */ +const gatedCrosswalk = parseCrosswalk({ + source: "native", + target: "shared", + rules: [ + { op: "move", from: "meta", to: "meta" }, + ...identityFieldNames.map((name) => ({ op: "move", from: `nodes[].${name}`, to: `nodes[].${name}` })), + { op: "move", from: "nodes[].weight", to: "nodes[].weight", when: { kind: "compare", op: "gte", left: { kind: "reference", key: "nodes[].weight" }, right: { kind: "numberLiteral", value: 0 } } }, + ], +}); + interface IdentityNode { readonly id: string; readonly text: string; @@ -119,6 +130,24 @@ function mixedEncodeBaseline(doc: Document): Document { }; } +/** The hand-written encode of the gated crosswalk. */ +function gatedBaseline(doc: Document): unknown { + return { + meta: doc.meta, + nodes: doc.nodes.map((node) => { + const out: Record = {}; + for (const name of identityFieldNames) { + out[name] = node[name]; + } + if (Number(node["weight"]) >= 0) { + out["weight"] = node["weight"]; + } + + return out; + }), + }; +} + /** The hand-written decode of the mixed crosswalk. */ function mixedDecodeBaseline(doc: Document): unknown { return { @@ -269,3 +298,4 @@ measure({ name: "3 identity moves, encode", crosswalk: identityCrosswalk, direct measure({ name: "3 identity moves, decode", crosswalk: identityCrosswalk, direction: "decode", document: identityDocument, baseline: identityBaseline }); measure({ name: `${mixedName}, encode`, crosswalk: mixedCrosswalk, direction: "encode", document: mixedDocument, baseline: mixedEncodeBaseline }); measure({ name: `${mixedName}, decode`, crosswalk: mixedCrosswalk, direction: "decode", document: (count) => mixedEncodeBaseline(mixedDocument(count)), baseline: mixedDecodeBaseline }); +measure({ name: `${String(IDENTITY_FIELDS + 1)} moves + 1 gated per element (not fused), encode`, crosswalk: gatedCrosswalk, direction: "encode", document: mixedDocument, baseline: gatedBaseline }); From 0d20232a9b09a6b9a358b6bc2ad2e181f38a1099 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 31 Oct 2026 15:51:01 +0000 Subject: [PATCH 07/12] perf(paths): find gaps with one trail and format a write's path only on failure The gap search copied its trail into a new array at every element and record it entered, and listed every record's entries; it now keeps one trail for the whole walk, enters only records and arrays, and returns the trail it found. A write formatted its full path, and every intermediate container's, for an error that cannot be thrown; the path is now formatted only when that error is raised. Both serve the rule-at-a-time executor and any run with a one-way write. --- src/paths.ts | 70 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/src/paths.ts b/src/paths.ts index 3c2d766..de15a70 100644 --- a/src/paths.ts +++ b/src/paths.ts @@ -450,14 +450,14 @@ function writeOne(target: Record, trail: Trail, value: unknown, } const child: Record | unknown[] = existing ?? (nextStep.kind === "index" ? [] : {}); if (existing === undefined) { - placeInto(node, step, child, formatPath(trail.slice(0, depth + 1))); + placeInto(node, child, trail, depth); } node = child; } if (childOf(node, last, trail, trail.length - 1) !== undefined) { throw new RangeError(`two destinations write the one location ${formatPath(trail)}`); } - placeInto(node, last, value, formatPath(trail)); + placeInto(node, value, trail, trail.length - 1); if (asValue && isContainer(value)) { placedValues.add(value); } @@ -497,8 +497,12 @@ function childOf(node: Record | readonly unknown[], step: Trail return getOwn(node, step.key); } -/** Stores a value one step down from a container. The step's kind has already been checked against the container by {@link childOf}, so the final throw is unreachable for any caller in this module. */ -function placeInto(node: Record | readonly unknown[], step: TrailStep, value: unknown, location: string): void { +/** Stores a value one step down from a container, by the step at `depth` of `trail`. The step's kind has already been checked against the container by {@link childOf}, so the final throw is unreachable for any caller in this module, and the location it names is formatted only there. */ +function placeInto(node: Record | readonly unknown[], value: unknown, trail: Trail, depth: number): void { + const step = trail[depth]; + if (step === undefined) { + throw new RangeError("a trail step is missing inside the trail's own length"); + } if (Array.isArray(node) && step.kind === "index") { Reflect.set(node, step.index, value); @@ -509,41 +513,59 @@ function placeInto(node: Record | readonly unknown[], step: Tra return; } - throw new RangeError(`path step ${location} does not fit the container it meets`); + throw new RangeError(`path step ${formatPath(trail.slice(0, depth + 1))} does not fit the container it meets`); } /** - * Finds the first array element nothing wrote: a hole left when a rule wrote a later index of an array (a fixed index past the next free one, or a gated rule skipping an element no other rule wrote). Holes are counted before any array is walked, so a large sparse array costs one key listing rather than a pass over its length. + * Finds the first array element nothing wrote: a hole left when a rule wrote a later index of an array (a fixed index past the next free one, or a gated rule skipping an element no other rule wrote). Holes are counted before any array is walked, so a large sparse array costs one key listing rather than a pass over its length; an array whose own keys outnumber or fall short of its length is reported at the first index it lacks, or at -1 when it lacks none. The search keeps one trail for the whole walk and copies it only for the hole it returns. * @param value - The output document under construction. - * @param trail - The location of `value` in the document, prefixed to the returned trail; the root call omits it. * @returns The trail of the first hole in depth-first order, or `undefined` when there is none. */ -export function findHole(value: unknown, trail: Trail = []): Trail | undefined { - if (Array.isArray(value)) { - if (Object.keys(value).length !== value.length) { - const index = value.findIndex((_element: unknown, position) => !Object.hasOwn(value, position)); +export function findHole(value: unknown): Trail | undefined { + const trail: TrailStep[] = []; + + /** Searches one value depth first, leaving the steps to a hole pushed onto the trail when it finds one and the trail as it was otherwise. */ + function holeBelow(node: unknown): boolean { + if (Array.isArray(node)) { + if (Object.keys(node).length !== node.length) { + trail.push({ kind: "index", index: node.findIndex((_element: unknown, position) => !Object.hasOwn(node, position)) }); + + return true; + } + for (let index = 0; index < node.length; index += 1) { + if (holeBelowStep(node[index], { kind: "index", index })) { + return true; + } + } - return [...trail, { kind: "index", index }]; + return false; } - for (const [index, element] of value.entries()) { - const hole = findHole(element, [...trail, { kind: "index", index }]); - if (hole !== undefined) { - return hole; + if (isRecord(node)) { + for (const key of Object.keys(node)) { + if (holeBelowStep(node[key], { kind: "key", key })) { + return true; + } } } - return undefined; + return false; } - if (isRecord(value)) { - for (const [key, child] of Object.entries(value)) { - const hole = findHole(child, [...trail, { kind: "key", key }]); - if (hole !== undefined) { - return hole; - } + + /** Searches one child: a scalar holds no array, so only a record or array is entered, with its step pushed for the search and popped unless a hole was found. */ + function holeBelowStep(child: unknown, step: TrailStep): boolean { + if (typeof child !== "object" || child === null) { + return false; } + trail.push(step); + if (holeBelow(child)) { + return true; + } + trail.pop(); + + return false; } - return undefined; + return holeBelow(value) ? trail : undefined; } function isContainer(value: unknown): value is Record | unknown[] { From ffa091d6a81842ebc6cb6e8fead1d659e90f3e68 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 31 Oct 2026 15:54:01 +0000 Subject: [PATCH 08/12] feat(crosswalk): let a single-path rule leave out a to that equals its from A move, scale, enum, format or split rule may omit to, meaning the same path as from, so an identity rule names its path once. Parsing fills the path in, so every parsed crosswalk carries to and nothing past the schema changes. The JSON Schema is now generated from the input side of the Zod schemas, which also stops it requiring the rules and oneWay lists that parseCrosswalk already defaults. --- README.md | 4 +-- scripts/generate-json-schema.ts | 2 ++ scripts/generate-json-schema.unit.test.ts | 16 +++++++++ src/crosswalk.ts | 44 +++++++++++++++++------ src/crosswalk.unit.test.ts | 21 +++++++++++ src/executor.unit.test.ts | 17 +++++++++ 6 files changed, 92 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index ab8c007..f15be1d 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,7 @@ Every `from` path and every `to` path is unique across the crosswalk, compared i ### Two-way rules -Each derives both directions from one declaration. Decode swaps `from` and `to` and applies the inverse. A `template` reads several fields and a `copy` writes several places; all of a rule's paths have the same number of wildcards, a template's fields come from the same element, and each path counts towards the uniqueness of `from` and `to` paths. +Each derives both directions from one declaration. Decode swaps `from` and `to` and applies the inverse. A `template` reads several fields and a `copy` writes several places; all of a rule's paths have the same number of wildcards, a template's fields come from the same element, and each path counts towards the uniqueness of `from` and `to` paths. A rule with one path on each side (`move`, `scale`, `enum`, `format` and `split`) may leave out `to` when it keeps the field at the same path, so `{ op: "move", from: "nodes[].id" }` means `{ op: "move", from: "nodes[].id", to: "nodes[].id" }`; parsing fills it in, so a parsed crosswalk always carries `to`, and the published JSON Schema leaves it optional for those ops. | op | fields | encode | decode | inverse holds when | | --- | --- | --- | --- | --- | @@ -264,7 +264,7 @@ The errors crossmap defines all extend `CrossmapError`: `CrosswalkSyntaxError` f ## JSON Schema -The crosswalk format is published as a JSON Schema 2020-12 document, generated from the Zod definitions as part of the build and canonicalised per RFC 8785. It is exposed through the package exports map as `crossmap/schemas/crossmap.schema.json`. The `when` field is a `$ref` to trilean's published predicate schema, pinned to the trilean version this package depends on, so the predicate grammar has one source of truth. +The crosswalk format is published as a JSON Schema 2020-12 document, generated from the Zod definitions as part of the build and canonicalised per RFC 8785. It is exposed through the package exports map as `crossmap/schemas/crossmap.schema.json`. It describes a crosswalk as a document may be written, so what `parseCrosswalk` fills in (the `rules` and `oneWay` lists, a single-path rule's `to`) is optional in it. The `when` field is a `$ref` to trilean's published predicate schema, pinned to the trilean version this package depends on, so the predicate grammar has one source of truth. ## Not in this version diff --git a/scripts/generate-json-schema.ts b/scripts/generate-json-schema.ts index 625353d..8bb4e94 100644 --- a/scripts/generate-json-schema.ts +++ b/scripts/generate-json-schema.ts @@ -53,7 +53,9 @@ const registry = z.registry(); registry.add(CrosswalkSchema, { id: "Crosswalk" }); registry.add(PredicateNodeSchema, { id: "When" }); +// The input side of each schema: the published schema describes the documents a crosswalk may be written as, so a field the parse fills in (a defaulted rule list, a single-path rule's `to`) is optional in it, as parseCrosswalk accepts it. const { schemas } = z.toJSONSchema(registry, { + io: "input", uri: (id) => (id === "__shared" ? "" : `#/$defs/${id}`), }); diff --git a/scripts/generate-json-schema.unit.test.ts b/scripts/generate-json-schema.unit.test.ts index 797ce91..1b77218 100644 --- a/scripts/generate-json-schema.unit.test.ts +++ b/scripts/generate-json-schema.unit.test.ts @@ -2,6 +2,7 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import canonicalize from "canonicalize"; import { describe, expect, it } from "vitest"; +import { z } from "zod"; const schemaPath = join(import.meta.dirname, "../schemas/crossmap.schema.json"); @@ -37,4 +38,19 @@ describe("the built JSON Schema", () => { } expect(when["$ref"]).toBe(`https://cdn.jsdelivr.net/npm/trilean@${String(trileanVersion)}/schemas/trilean.schema.json#/$defs/PredicateNode`); }); + + it("describes the documents parseCrosswalk accepts: the rule lists, and a single-path rule's to, may be left out", () => { + const shape = z.object({ + $defs: z.object({ + Crosswalk: z.object({ + required: z.array(z.string()), + properties: z.object({ rules: z.object({ items: z.object({ oneOf: z.array(z.object({ properties: z.object({ op: z.object({ const: z.string() }) }), required: z.array(z.string()) })) }) }) }), + }), + }), + }); + const crosswalk = shape.parse(parsed).$defs.Crosswalk; + expect(crosswalk.required.sort()).toEqual(["source", "target"]); + const requiresTo = Object.fromEntries(crosswalk.properties.rules.items.oneOf.map((rule) => [rule.properties.op.const, rule.required.includes("to")])); + expect(requiresTo).toEqual({ move: false, scale: false, enum: false, format: false, split: false, template: true, copy: true }); + }); }); diff --git a/src/crosswalk.ts b/src/crosswalk.ts index ffb8d86..41f7107 100644 --- a/src/crosswalk.ts +++ b/src/crosswalk.ts @@ -13,35 +13,56 @@ const PathField = z.string().regex(PATH_REGEX, { /** A trilean predicate tree, the shape of every rule's optional `when`: evaluated against the governing document (source for encode, target for decode; wildcards forbidden, checked structurally in validateCrosswalk). Exported so consumers building crosswalks in TypeScript have the type without importing trilean separately. */ export type When = z.infer; -export const MoveRuleSchema = z +/** A single-path rule as parsing returns it: its `to` always present. */ +type WithTo = Omit & { readonly to: string }; + +/** + * A single-path rule with its `to` filled in: as written, or the rule's own `from` when `to` is left out, so a rule that keeps a field at the same path names the path once. Every rule the schema returns carries a `to`, so nothing downstream of parsing distinguishes the two spellings. + * @param rule - The rule as the schema read it. + * @returns The rule with `to` present. + */ +function withSameTo(rule: Rule): WithTo { + return { ...rule, to: rule.to ?? rule.from }; +} + +/** The schema of a single-path rule whose `to` may be left out: its object schema, then {@link withSameTo}. Named so that declaration emit refers to it rather than spelling out the whole inferred type, which is longer than the compiler will serialise. */ +type SamePathDefault> = z.ZodPipe>, z.output>>; + +function samePathDefault>(shape: Shape): SamePathDefault { + return shape.transform(withSameTo); +} + +const MoveRuleObject = z .object({ op: z.literal("move"), from: PathField, - to: PathField, + to: PathField.optional(), when: PredicateNodeSchema.optional(), }) .strict(); +export const MoveRuleSchema = samePathDefault(MoveRuleObject); export type MoveRule = z.infer; -export const ScaleRuleSchema = z +const ScaleRuleObject = z .object({ op: z.literal("scale"), from: PathField, - to: PathField, + to: PathField.optional(), factor: z.number().refine((factor) => Number.isFinite(factor) && factor !== 0, { message: "factor must be a finite non-zero number: decode divides by it", }), when: PredicateNodeSchema.optional(), }) .strict(); +export const ScaleRuleSchema = samePathDefault(ScaleRuleObject); export type ScaleRule = z.infer; /** The enum op's pair of mappings is validated as one bijection: mapBack must be map's exact inverse (injective map, mapBack holding exactly map's image, every entry inverted), so decode is determined by encode's vocabulary and cannot drift from it. */ -export const EnumRuleSchema = z +const EnumRuleObject = z .object({ op: z.literal("enum"), from: PathField, - to: PathField, + to: PathField.optional(), map: z.record(z.string(), z.string()), mapBack: z.record(z.string(), z.string()), when: PredicateNodeSchema.optional(), @@ -61,32 +82,35 @@ export const EnumRuleSchema = z } } }); +export const EnumRuleSchema = samePathDefault(EnumRuleObject); export type EnumRule = z.infer; /** The textual forms `format` converts: each is a bijection between its canonical text and a machine value, which is what makes the round trip hold for canonical input. */ export const FormatKindSchema = z.enum(["isoDateTime", "isoDate", "numberText", "booleanText", "jsonText"]); export type FormatKind = z.infer; -export const FormatRuleSchema = z +const FormatRuleObject = z .object({ op: z.literal("format"), from: PathField, - to: PathField, + to: PathField.optional(), kind: FormatKindSchema, when: PredicateNodeSchema.optional(), }) .strict(); +export const FormatRuleSchema = samePathDefault(FormatRuleObject); export type FormatRule = z.infer; -export const SplitRuleSchema = z +const SplitRuleObject = z .object({ op: z.literal("split"), from: PathField, - to: PathField, + to: PathField.optional(), separator: z.string().min(1, "separator must be non-empty: an empty separator makes decode's join ambiguous"), when: PredicateNodeSchema.optional(), }) .strict(); +export const SplitRuleSchema = samePathDefault(SplitRuleObject); export type SplitRule = z.infer; /** Several fields rendered into one text through a template, and split back out of it. `from` maps each placeholder name to the path of its field; the template is validated as having a unique inverse (every field used exactly once, a literal between every two placeholders), and encode refuses a value that contains the literal after its placeholder, so decode always recovers the fields encode rendered. */ diff --git a/src/crosswalk.unit.test.ts b/src/crosswalk.unit.test.ts index 9ff1b8b..4010b5c 100644 --- a/src/crosswalk.unit.test.ts +++ b/src/crosswalk.unit.test.ts @@ -133,4 +133,25 @@ describe("parseCrosswalk", () => { expect(issues.length).toBeGreaterThan(1); } }); + + it.each([ + { op: "move", from: "nodes[].id" }, + { op: "scale", from: "nodes[].weight", factor: 2 }, + { op: "enum", from: "nodes[].kind", map: { a: "A" }, mapBack: { A: "a" } }, + { op: "format", from: "nodes[].at", kind: "isoDateTime" }, + { op: "split", from: "nodes[].tags", separator: ";" }, + ])("reads a $op rule with no to as one whose to is its from", (rule) => { + const [parsed] = parseCrosswalk({ source: "a", target: "b", rules: [rule] }).rules; + expect(parsed).toEqual({ ...rule, to: rule.from }); + }); + + it("keeps a to that is written, and still requires one on template and copy rules", () => { + expect(parseCrosswalk({ source: "a", target: "b", rules: [{ op: "move", from: "x", to: "y" }] }).rules).toEqual([{ op: "move", from: "x", to: "y" }]); + expect(() => parseCrosswalk({ source: "a", target: "b", rules: [{ op: "template", from: { x: "x" }, template: "{x}" }] })).toThrow(/rules\.0\.to/u); + expect(() => parseCrosswalk({ source: "a", target: "b", rules: [{ op: "copy", from: "x" }] })).toThrow(/rules\.0\.to/u); + }); + + it("validates a to left out as the from path it stands for", () => { + expect(() => parseCrosswalk({ source: "a", target: "b", rules: [{ op: "move", from: "a..b" }] })).toThrow(/rules\.0\.from/u); + }); }); diff --git a/src/executor.unit.test.ts b/src/executor.unit.test.ts index 713e258..0f4d815 100644 --- a/src/executor.unit.test.ts +++ b/src/executor.unit.test.ts @@ -435,3 +435,20 @@ describe("encode over a large wildcard array", () => { expect(growthPerEdge).toBeLessThan(BYTES_PER_ELEMENT_BOUND); }, TEST_TIMEOUT_MILLISECONDS); }); + +describe("a rule whose to is left out", () => { + it("writes to the path it reads, both ways", () => { + const cw = crosswalk({ + source: "n", + target: "s", + rules: [ + { op: "move", from: "meta" }, + { op: "move", from: "nodes[].id" }, + { op: "scale", from: "nodes[].weight", factor: 2 }, + ], + }); + const doc = { meta: { name: "g" }, nodes: [{ id: "a", weight: 1 }] }; + expect(encode(cw, doc)).toEqual({ meta: { name: "g" }, nodes: [{ id: "a", weight: 2 }] }); + assertRoundTrip(cw, [doc]); + }); +}); From 58b88bf41b72f6d17fe48a69975b7e67a0c2bf13 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 31 Oct 2026 15:55:38 +0000 Subject: [PATCH 09/12] test(exact-json): refuse an array of a subclass, which the schema copies --- src/exact-json.unit.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/exact-json.unit.test.ts b/src/exact-json.unit.test.ts index abfddb2..12dc16a 100644 --- a/src/exact-json.unit.test.ts +++ b/src/exact-json.unit.test.ts @@ -34,7 +34,10 @@ describe("isExactJson", () => { const hidden = Object.defineProperty({ a: 1 }, "b", { value: 2, enumerable: false }); const named = Object.defineProperty([1], "extra", { value: true, enumerable: true }); const proto: unknown = JSON.parse('{"__proto__":{"a":1},"b":2}'); - for (const value of [holed, bare, hidden, named, proto]) { + /** An array of a subclass, which the schema copies into an ordinary array. */ + class Rows extends Array {} + const rows = Rows.from([1]); + for (const value of [holed, bare, hidden, named, proto, rows]) { expect(isExactJson(value)).toBe(false); } }); From 3a74d69666a9f3901cc2bbea5dfc98608df7f39b Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 31 Oct 2026 15:56:06 +0000 Subject: [PATCH 10/12] docs(contributing): keep the executor and its reference in step --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 44f43ef..639502c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,6 +19,7 @@ - **Task order and caching live in `turbo.json`.** Each task is an underscore-prefixed leaf script (`_build`, `_typecheck`, `_test`, `_lint`) and the public script of the same name is `turbo run _`. Ordering is a `dependsOn` edge, never a sequence of commands in a hook, a workflow or a script, so every entry point (hook, CI, a developer's shell) gets the same order and the same cache. `src/turbo.unit.test.ts` fails when a script, task or hook drifts from this. Incremental state that lives outside turbo's task cache is restored by its own CI cache step: the lint scripts cache by file content (mtimes do not survive a checkout) and `tsc` writes its build info beside each tsconfig, which the CI typecheck job caches. - **Generated files are never hand-edited or committed.** `schemas/crossmap.schema.json` and `dist/` come from `pnpm build` and are gitignored. If the generated schema looks wrong, fix the Zod schema or `scripts/generate-json-schema.ts`. - **No wall-clock figures in tests or docs.** Assert structure and counts of work. +- **The executor answers to a reference.** `test/support/reference-executor.ts` is the plain rule-at-a-time executor, and the property tests in `src/executor-reference.unit.test.ts` require the same output, key order included, or the same error from the shipped one over generated crosswalks and documents. A deliberate change to what encode or decode does is made in both, in the same commit; a change that only makes them faster changes the shipped one and leaves the reference alone. `pnpm bench` measures a change against a hand-written copy, as ratios and input reads per node rather than times. ## Releases From 08cc0c0d6b3febbf38995ce24230aaae9915440d Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 31 Oct 2026 16:05:33 +0000 Subject: [PATCH 11/12] feat(crosswalk): add a keep list of paths carried through unchanged keep is an optional top-level list of paths, each meaning the identity move from that path to itself. parseCrosswalk appends those moves after rules in the order listed and drops the list, so validation, execution, round trips, rule numbering and the path-uniqueness check see only rules. A field that no rule or kept path names still does not survive encode. A property test generates each crosswalk twice, once with keep and once with explicit moves, and requires the same encode and decode outcome from both and from the reference executor. --- README.md | 15 +++++- scripts/generate-json-schema.unit.test.ts | 6 ++- src/crosswalk.ts | 15 +++++- src/crosswalk.unit.test.ts | 35 ++++++++++++++ src/executor-reference.unit.test.ts | 56 +++++++++++++++++++++++ src/executor.unit.test.ts | 14 ++++++ 6 files changed, 136 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f15be1d..33c1704 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ The exit status is 0 on success, 1 when the library rejects the crosswalk or the ## The format -A crosswalk is `{ source, target, rules, oneWay }`: two labels naming the shapes, a list of two-way rules and a list of one-way rules. `parseCrosswalk` validates the syntax and throws one error carrying every issue. `validateCrosswalk` checks the properties of the rule list as a whole, and `compileCrosswalk`, `encode`, `decode` and `assertRoundTrip` run it before anything executes (once per crosswalk object, see [Execution](#execution)). +A crosswalk is `{ source, target, rules, keep, oneWay }`: two labels naming the shapes, a list of two-way rules, a list of paths carried through unchanged and a list of one-way rules. Only `source` and `target` are required; each list defaults to empty. `parseCrosswalk` validates the syntax and throws one error carrying every issue. `validateCrosswalk` checks the properties of the rule list as a whole, and `compileCrosswalk`, `encode`, `decode` and `assertRoundTrip` run it before anything executes (once per crosswalk object, see [Execution](#execution)). ### Paths @@ -209,6 +209,17 @@ For example (a template writes a literal brace as `{{` or `}}`): { op: "copy", from: "items[].id", to: ["nodes[].id", "labels[].text"] } ``` +### Kept paths + +`keep` lists paths whose value goes across unchanged, at the same path, in both directions. Each one means exactly the identity `move` from that path to itself, so these two crosswalks are the same crosswalk: + +```ts +{ source: "export", target: "graph/v1", keep: ["meta", "nodes[].id", "nodes[].label"] } +{ source: "export", target: "graph/v1", rules: [{ op: "move", from: "meta" }, { op: "move", from: "nodes[].id" }, { op: "move", from: "nodes[].label" }] } +``` + +A kept path is any path a `from` may be, wildcards included, and carries no `when`. `parseCrosswalk` turns each one into its `move`, appended after `rules` in the order listed, so a parsed crosswalk has no `keep` and everything else (validation, execution, round trips) sees only rules. Errors number a kept path as that rule: the n-th kept path is rule `rules.length + n`, counting from zero, and one-way rules are numbered after it. A path kept twice, or kept and also named by a rule, is a repeated path like any other. `keep` names each path it carries: a field that neither a rule nor `keep` names still does not survive encode. + ### One-way rules These are the declared lossy remainder, each applying in one named direction. @@ -264,7 +275,7 @@ The errors crossmap defines all extend `CrossmapError`: `CrosswalkSyntaxError` f ## JSON Schema -The crosswalk format is published as a JSON Schema 2020-12 document, generated from the Zod definitions as part of the build and canonicalised per RFC 8785. It is exposed through the package exports map as `crossmap/schemas/crossmap.schema.json`. It describes a crosswalk as a document may be written, so what `parseCrosswalk` fills in (the `rules` and `oneWay` lists, a single-path rule's `to`) is optional in it. The `when` field is a `$ref` to trilean's published predicate schema, pinned to the trilean version this package depends on, so the predicate grammar has one source of truth. +The crosswalk format is published as a JSON Schema 2020-12 document, generated from the Zod definitions as part of the build and canonicalised per RFC 8785. It is exposed through the package exports map as `crossmap/schemas/crossmap.schema.json`. It describes a crosswalk as a document may be written, so what `parseCrosswalk` fills in (the `rules`, `keep` and `oneWay` lists, a single-path rule's `to`) is optional in it. The `when` field is a `$ref` to trilean's published predicate schema, pinned to the trilean version this package depends on, so the predicate grammar has one source of truth. ## Not in this version diff --git a/scripts/generate-json-schema.unit.test.ts b/scripts/generate-json-schema.unit.test.ts index 1b77218..bc430cd 100644 --- a/scripts/generate-json-schema.unit.test.ts +++ b/scripts/generate-json-schema.unit.test.ts @@ -39,12 +39,14 @@ describe("the built JSON Schema", () => { expect(when["$ref"]).toBe(`https://cdn.jsdelivr.net/npm/trilean@${String(trileanVersion)}/schemas/trilean.schema.json#/$defs/PredicateNode`); }); - it("describes the documents parseCrosswalk accepts: the rule lists, and a single-path rule's to, may be left out", () => { + it("describes the documents parseCrosswalk accepts: the rule lists, the keep list, and a single-path rule's to, may be left out", () => { const shape = z.object({ $defs: z.object({ Crosswalk: z.object({ required: z.array(z.string()), - properties: z.object({ rules: z.object({ items: z.object({ oneOf: z.array(z.object({ properties: z.object({ op: z.object({ const: z.string() }) }), required: z.array(z.string()) })) }) }) }), + properties: z.object({ + keep: z.object({ type: z.literal("array"), items: z.object({ type: z.literal("string"), pattern: z.string() }) }), + rules: z.object({ items: z.object({ oneOf: z.array(z.object({ properties: z.object({ op: z.object({ const: z.string() }) }), required: z.array(z.string()) })) }) }) }), }), }), }); diff --git a/src/crosswalk.ts b/src/crosswalk.ts index 41f7107..46bffae 100644 --- a/src/crosswalk.ts +++ b/src/crosswalk.ts @@ -194,6 +194,8 @@ export type OneWayRule = z.infer; /** * The crosswalk document's schema: the rule list that derives both encode and decode. Schema-attached guard per the house pattern (`CrosswalkSchema.is(x)` narrows without a separate hand-written guard). The explicit annotation names the validation surface for declaration emit (the bare schema's inferred type is deeper than the compiler will serialise into a .d.ts); runtime validation is the full strict object below. + * + * `keep` lists paths carried through unchanged: each is read as the two-way rule `{ op: "move", from: path, to: path }`, appended after `rules` in its order, so the parsed crosswalk holds only `rules` and everything past parsing (validation, execution, round trips, error messages, rule numbering) treats a kept path exactly as that rule. A kept path is numbered after every written two-way rule, and a path kept twice, or kept and also named by a rule, is reported by validateCrosswalk's path-uniqueness check like any other repeated path. */ export const CrosswalkSchema: z.ZodType & { is: (value: unknown) => value is Crosswalk } = define( z @@ -201,10 +203,21 @@ export const CrosswalkSchema: z.ZodType & { is: (value: unknown) => v source: z.string(), target: z.string(), rules: z.array(TwoWayRuleSchema).default([]), + keep: z.array(PathField).default([]), oneWay: z.array(OneWayRuleSchema).default([]), }) - .strict(), + .strict() + .transform(({ keep, rules, ...crosswalk }): Crosswalk => ({ ...crosswalk, rules: [...rules, ...keep.map((path) => keptPath(path))] })), ); + +/** + * The two-way rule a `keep` path stands for. + * @param path - A path from the crosswalk's `keep` list. + * @returns The identity move from that path to itself. + */ +function keptPath(path: string): MoveRule { + return { op: "move", from: path, to: path }; +} export interface Crosswalk { readonly source: string; readonly target: string; diff --git a/src/crosswalk.unit.test.ts b/src/crosswalk.unit.test.ts index 4010b5c..9f9c983 100644 --- a/src/crosswalk.unit.test.ts +++ b/src/crosswalk.unit.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { CrosswalkSchema, parseCrosswalk } from "./crosswalk"; import { CrosswalkSyntaxError } from "./errors"; +import { validateCrosswalk } from "./validate"; const validCrosswalk = { source: "native", @@ -155,3 +156,37 @@ describe("parseCrosswalk", () => { expect(() => parseCrosswalk({ source: "a", target: "b", rules: [{ op: "move", from: "a..b" }] })).toThrow(/rules\.0\.from/u); }); }); + +describe("keep", () => { + it("reads each kept path as an identity move appended after the rules, in order", () => { + const parsed = parseCrosswalk({ source: "a", target: "b", rules: [{ op: "scale", from: "w", to: "weight", factor: 2 }], keep: ["meta", "nodes[].id"] }); + expect(parsed.rules).toEqual([ + { op: "scale", from: "w", to: "weight", factor: 2 }, + { op: "move", from: "meta", to: "meta" }, + { op: "move", from: "nodes[].id", to: "nodes[].id" }, + ]); + expect(parsed).not.toHaveProperty("keep"); + }); + + it("is optional, and a crosswalk of kept paths alone needs no rules", () => { + expect(parseCrosswalk({ source: "a", target: "b" }).rules).toEqual([]); + expect(parseCrosswalk({ source: "a", target: "b", keep: ["x"] }).rules).toEqual([{ op: "move", from: "x", to: "x" }]); + }); + + it.each([ + ["a non-string entry", [true]], + ["an empty path", [""]], + ["a malformed path", ["a..b"]], + ["a __proto__ key", ["__proto__"]], + ["a list that is not an array", "meta"], + ])("refuses %s, naming the keep field", (_name, keep) => { + expect(() => parseCrosswalk({ source: "a", target: "b", keep })).toThrow(/keep/u); + }); + + it("leaves a path kept twice, or kept and named by a rule, to the path-uniqueness check", () => { + const twice = validateCrosswalk(parseCrosswalk({ source: "a", target: "b", keep: ["x", '["x"]'] })); + expect(twice.map((issue) => issue.message)).toEqual(["two rules write the same to path: x", "two rules use the same from path: x"]); + const collides = validateCrosswalk(parseCrosswalk({ source: "a", target: "b", rules: [{ op: "move", from: "y", to: "x" }], keep: ["x"] })); + expect(collides.map((issue) => issue.message)).toEqual(["two rules write the same to path: x"]); + }); +}); diff --git a/src/executor-reference.unit.test.ts b/src/executor-reference.unit.test.ts index 2f669ec..40915f0 100644 --- a/src/executor-reference.unit.test.ts +++ b/src/executor-reference.unit.test.ts @@ -259,6 +259,34 @@ function crosswalkOf(rule: fc.Arbitrary): fc.Arbitrary): fc.Arbitrary { + return fc.tuple(fc.array(rule, { maxLength: MAX_RULES }), fc.array(keptPathArbitrary, { minLength: 1, maxLength: MAX_RULES }), fc.array(oneWayRule, { maxLength: MAX_ONE_WAY_RULES })).map(([rules, keep, oneWay]) => { + try { + return { + written: parseCrosswalk({ source: "native", target: "shared", rules, keep, oneWay }), + explicit: parseCrosswalk({ source: "native", target: "shared", rules: [...rules, ...keep.map((path) => ({ op: "move", from: path }))], oneWay }), + }; + } catch { + // The schema's refusal is this generator's filter, not a result under test. + return undefined; + } + }); +} + /** JSON scalars, several of which fit an op (enum keys, number text, ISO dates, separated text, JSON text). */ const jsonLeaf = fc.constantFrom("open", "closed", "x", "1", "2.5", "2026-10-18", "2026-10-18T09:30:00.000Z", "true", '{"k":1}', "p,q", "p-q", 0, 1, FRACTIONAL, NEGATIVE, true, false, null); @@ -422,6 +450,34 @@ describe("the executor against the reference executor", () => { expect(fusedRuns).toBeGreaterThan(0); }, PROPERTY_TIMEOUT_MILLISECONDS); + it("gives a crosswalk written with keep the outcome of the same crosswalk written with explicit moves, and of the reference, both ways", () => { + const seen = new Set(); + const note = (kind: OutcomeKind | undefined): void => { + if (kind !== undefined) { + seen.add(kind); + } + }; + fc.assert( + fc.property(keptPairOf(fc.oneof(keyedRule, twoWayRule)), fc.record({ native: document, length: fc.integer({ min: 0, max: MAX_ARRAY_LENGTH }), draw: fc.gen() }), (pair, { native, length, draw }) => { + if (pair === undefined) { + return; + } + const { written, explicit } = pair; + const sides = explicit.rules.map(ruleSides); + const fittedNative = fitted(sides.flatMap((side) => side.from).map(parsePath), length, () => draw(() => fittedLeaf)); + const fittedShared = fitted(sides.flatMap((side) => side.to).map(parsePath), length, () => draw(() => fittedLeaf)); + for (const [doc, way] of [[native, "encode"], [fittedNative, "encode"], [fittedShared, "decode"]] as const) { + const run = (crosswalk: Crosswalk): Outcome => outcome(() => (way === "encode" ? encode(crosswalk, doc) : decode(crosswalk, doc))); + expect(run(written)).toEqual(run(explicit)); + note(sameOutcome(explicit, doc, way)); + } + }), + { numRuns: RUNS }, + ); + // Both a result and the errors kept paths most often meet were compared. + expect([...seen]).toEqual(expect.arrayContaining(["output", "absent", "structure", "conflict"])); + }, PROPERTY_TIMEOUT_MILLISECONDS); + it("gives the same result as the reference over every corpus sample, both ways", () => { for (const cell of CORPUS) { for (const sample of cell.samples) { diff --git a/src/executor.unit.test.ts b/src/executor.unit.test.ts index 0f4d815..0021f17 100644 --- a/src/executor.unit.test.ts +++ b/src/executor.unit.test.ts @@ -452,3 +452,17 @@ describe("a rule whose to is left out", () => { assertRoundTrip(cw, [doc]); }); }); + +describe("kept paths", () => { + it("carry the paths they name through unchanged both ways, and nothing else", () => { + const cw = crosswalk({ source: "n", target: "s", keep: ["meta", "nodes[].id"] }); + const doc = { meta: { name: "g" }, nodes: [{ id: "a", unnamed: 1 }] }; + expect(encode(cw, doc)).toEqual({ meta: { name: "g" }, nodes: [{ id: "a" }] }); + assertRoundTrip(cw, [{ meta: { name: "g" }, nodes: [{ id: "a" }] }]); + }); + + it("are numbered after the written rules in the errors they raise", () => { + const cw = crosswalk({ source: "n", target: "s", rules: [{ op: "move", from: "a", to: "x" }], keep: ["b", "c"] }); + expect(() => encode(cw, { a: 1, b: 2 })).toThrow(new RuleApplicationError(2, "c", "the path is absent from the document")); + }); +}); From 64ed040d76fddddebdd7a4df55c8ee4d83c751eb Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 31 Oct 2026 16:05:55 +0000 Subject: [PATCH 12/12] chore(bench): measure the identity crosswalk written as a keep list --- bench/encode.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bench/encode.ts b/bench/encode.ts index b04487f..29baeec 100644 --- a/bench/encode.ts +++ b/bench/encode.ts @@ -53,6 +53,9 @@ const identityCrosswalk = parseCrosswalk({ ], }); +/** The identity crosswalk written as a `keep` list, which parses to the same rules and so should cost the same. */ +const keptCrosswalk = parseCrosswalk({ source: "native", target: "shared", keep: ["meta", "nodes[].id", "nodes[].text"] }); + /** Identity moves for every field, plus one rule each of format, scale and enum. */ const mixedCrosswalk = parseCrosswalk({ source: "native", @@ -296,6 +299,8 @@ function measure(scenario: Scenario): void { const mixedName = `${String(IDENTITY_FIELDS + 1)} moves + format, scale, enum`; measure({ name: "3 identity moves, encode", crosswalk: identityCrosswalk, direction: "encode", document: identityDocument, baseline: identityBaseline }); measure({ name: "3 identity moves, decode", crosswalk: identityCrosswalk, direction: "decode", document: identityDocument, baseline: identityBaseline }); +measure({ name: "3 kept paths, encode", crosswalk: keptCrosswalk, direction: "encode", document: identityDocument, baseline: identityBaseline }); +measure({ name: "3 kept paths, decode", crosswalk: keptCrosswalk, direction: "decode", document: identityDocument, baseline: identityBaseline }); measure({ name: `${mixedName}, encode`, crosswalk: mixedCrosswalk, direction: "encode", document: mixedDocument, baseline: mixedEncodeBaseline }); measure({ name: `${mixedName}, decode`, crosswalk: mixedCrosswalk, direction: "decode", document: (count) => mixedEncodeBaseline(mixedDocument(count)), baseline: mixedDecodeBaseline }); measure({ name: `${String(IDENTITY_FIELDS + 1)} moves + 1 gated per element (not fused), encode`, crosswalk: gatedCrosswalk, direction: "encode", document: mixedDocument, baseline: gatedBaseline });