From 9553b3bd38168fad4bba71e8fd1695028af0324c Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Wed, 21 Oct 2026 14:30:14 +0100 Subject: [PATCH] perf(executor): write each wildcard element as it is walked A wildcard rule expanded every read path into one site record per element, held a joined-index map per further read path, and built the whole output list before the first write, so an encode over a large array kept several hundred bytes per element beyond its result and ran out of memory at tens of millions of elements. The governing path is now walked one element at a time with the trail held once, the other read paths are read at the element's indexes, and each element's results are written before the next is visited. Absence and element-pairing failures keep their order: every path is walked for its count first, so an absent path is reported before a misaligned one, exactly as before. --- src/executor.ts | 107 +++++++++++++++++++------------------- src/executor.unit.test.ts | 60 +++++++++++++++++++++ src/paths.ts | 97 ++++++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 53 deletions(-) diff --git a/src/executor.ts b/src/executor.ts index 5fc709d..e1552ed 100644 --- a/src/executor.ts +++ b/src/executor.ts @@ -7,7 +7,7 @@ import { applyOneWayRule } from "./ops/one-way"; import { scaleDecode, scaleEncode } from "./ops/scale"; import { splitDecode, splitEncode } from "./ops/split"; import { type JsonValue, JsonValueSchema, type PredicateNode } from "trilean"; -import { bindLocus, type ElementGate, ensureArray, findHole, formatPath, type Locus, type LocusReading, parsePathMemo, readLocus, wildcardIndexes, writeMatches } from "./paths"; +import { bindLocus, type ElementGate, type EmptySite, ensureArray, findHole, formatPath, type Locus, parsePathMemo, readTrail, walkLocus, wildcardIndexes, writeMatches } from "./paths"; import { ruleSides } from "./sides"; import { copyDecode, copyEncode } from "./ops/copy"; import { parseTemplateMemo, renderTemplate, splitTemplate } from "./ops/template"; @@ -90,7 +90,7 @@ function elementGate(when: PredicateNode, doc: unknown, governing: Locus): Eleme } /** - * Applies one two-way rule: reads every read-side path (index-aligned, one tuple of values per element), transforms each tuple with the operation in this direction, and writes the results to the write-side paths at the same indexes. Empty arrays met on the read side are materialised on every write-side path. + * Applies one two-way rule: walks the governing read path one element at a time, reads the other read paths at the same indexes, transforms each element's values with the operation in this direction, and writes the results to the write-side paths at those indexes as it goes, so nothing per element outlives the element. Empty arrays met on any read side are materialised on every write-side path. * @throws RuleApplicationError when a read path is absent, two read paths address different elements, the operation rejects a value, or a write conflicts. */ function applyTwoWayRule(rule: TwoWayRule, index: number, context: RunContext): void { @@ -106,33 +106,65 @@ function applyTwoWayRule(rule: TwoWayRule, index: number, context: RunContext): if (gate === false) { return; } - const read = (path: string): ReadSide => { + const writeLoci = writes.map((path) => parsePathMemo(path)); + const readPath = (direction === "encode" ? sides.from : sides.to).join(", "); + const empties: { readonly locus: Locus; readonly empty: EmptySite }[] = []; + let governingCount = 0; + const countWalk = (locus: Locus): { readonly empties: readonly EmptySite[] | undefined; readonly count: number } => { + let count = 0; + const walked = walkLocus(doc, locus, gate, () => { + count += 1; + }); + + return { empties: walked, count }; + }; + const governingWalk = countWalk(governing); + if (governingWalk.empties === undefined) { + throw new RuleApplicationError(index, governingPath, "the path is absent from the document"); + } + governingCount = governingWalk.count; + for (const empty of governingWalk.empties) { + empties.push({ locus: governing, empty }); + } + const otherWalks = otherReads.map((path) => { const locus = parsePathMemo(path); - const reading = readLocus(doc, locus, gate); - if (reading === undefined) { + const walked = countWalk(locus); + if (walked.empties === undefined) { throw new RuleApplicationError(index, path, "the path is absent from the document"); } + if (walked.count !== governingCount) { + throw new RuleApplicationError(index, path, `addresses different elements from ${governingPath}, so the paths cannot be paired element by element`); + } + for (const empty of walked.empties) { + empties.push({ locus, empty }); + } - return { path, locus, reading }; - }; - const first = read(governingPath); - const others = otherReads.map(read); - const writeLoci = writes.map((path) => parsePathMemo(path)); - const readPath = [first, ...others].map((side) => side.path).join(", "); - const outputs = alignReadings(first, others, index).flatMap(({ indexes, values }) => { - const results = transform(rule, direction, values, { index, path: readPath }); + return locus; + }); + walkLocus(doc, governing, gate, (trail, value) => { + const indexes = wildcardIndexes(governing, trail); + const values = [value, ...otherWalks.map((locus, position) => { + const found = readTrail(doc, bindLocus(locus, indexes)); + if (!found.found) { + throw new RuleApplicationError(index, otherReads[position] ?? governingPath, `addresses different elements from ${governingPath}, so the paths cannot be paired element by element`); + } - return writeLoci.map((locus, position) => ({ trail: bindLocus(locus, indexes), value: results[position] })); + return found.value; + })]; + const results = transform(rule, direction, values, { index, path: readPath }); + const outputs = writeLoci.map((locus, position) => ({ trail: bindLocus(locus, indexes), value: results[position] })); + try { + writeMatches(target, outputs); + } catch (error) { + throw new RuleApplicationError(index, writes.join(", "), errorMessage(error)); + } }); try { - writeMatches(target, outputs); - for (const side of [first, ...others]) { - for (const empty of side.reading.empties) { - // An existing empty source array yields no matches, so without this its counterpart would never come into being: decode would find the path absent, or an inner array would leave a gap or a shorter outer array. - const emptyIndexes = wildcardIndexes(side.locus, empty.trail); - for (const writeLocus of writeLoci) { - ensureArray(target, bindLocus(writeLocus, emptyIndexes, empty.wildcard)); - } + for (const { locus, empty } of empties) { + // An existing empty source array yields no matches, so without this its counterpart would never come into being: decode would find the path absent, or an inner array would leave a gap or a shorter outer array. + const emptyIndexes = wildcardIndexes(locus, empty.trail); + for (const writeLocus of writeLoci) { + ensureArray(target, bindLocus(writeLocus, emptyIndexes, empty.wildcard)); } } } catch (error) { @@ -147,37 +179,6 @@ interface RunContext { readonly target: Record; } -/** One read-side path of a rule, parsed and read. */ -interface ReadSide { - readonly path: string; - readonly locus: Locus; - readonly reading: LocusReading; -} - -/** - * Pairs the matches of a rule's read paths element by element: the first path's matches fix the order, and every other path must address exactly the same index tuples (a `template` whose fields sit in elements that lack some of them, or copies of an array that no longer agree in length, have no element-wise pairing). - * @returns One entry per element: its index tuple and the value each read path holds there, first path first. - * @throws RuleApplicationError naming the first other path whose elements differ from the first path's. - */ -function alignReadings(first: ReadSide, others: readonly ReadSide[], index: number): readonly { readonly indexes: readonly number[]; readonly values: readonly unknown[] }[] { - const keyed = others.map((side) => { - const byTuple = new Map(); - for (const match of side.reading.matches) { - byTuple.set(wildcardIndexes(side.locus, match.trail).join(","), match.value); - } - - return { side, byTuple }; - }); - const tuples = first.reading.matches.map((match) => ({ match, indexes: wildcardIndexes(first.locus, match.trail) })); - for (const { side, byTuple } of keyed) { - if (byTuple.size !== tuples.length || tuples.some(({ indexes }) => !byTuple.has(indexes.join(",")))) { - throw new RuleApplicationError(index, side.path, `addresses different elements from ${first.path}, so the paths cannot be paired element by element`); - } - } - - return tuples.map(({ match, indexes }) => ({ indexes, values: [match.value, ...keyed.map(({ byTuple }) => byTuple.get(indexes.join(",")))] })); -} - /** Applies one two-way rule's operation to one element's values in the given direction (one value per read path in, one per write path out), turning the operation's own thrown errors into RuleApplicationErrors that name the rule and path. */ function transform(rule: TwoWayRule, direction: "encode" | "decode", values: readonly unknown[], where: { readonly index: number; readonly path: string }): readonly unknown[] { try { diff --git a/src/executor.unit.test.ts b/src/executor.unit.test.ts index 62cb7a2..713e258 100644 --- a/src/executor.unit.test.ts +++ b/src/executor.unit.test.ts @@ -375,3 +375,63 @@ describe("aggregate", () => { expect(() => encode(cw, { xs: ["1"] })).toThrow(RuleApplicationError); }); }); + +describe("encode over a large wildcard array", () => { + /** The elements of the shape under test: enough that a per-element overhead of a few hundred bytes shows as hundreds of megabytes, and small enough that the test fits a loaded runner. */ + const ELEMENTS = 400_000; + /** How much retained growth per element the assertion refuses: the written output and its validated copy need a few hundred, the frontier-and-map design this replaces held about 800 in total. */ + const BYTES_PER_ELEMENT_BOUND = 600; + /** Young-generation sweeps before each measurement, so what the assertion reads is what the run retains rather than what the collector has not swept yet. */ + const SWEEPS = 40; + const SWEEP_SIZE = 250_000; + /** The base of the per-element timestamps, so the input carries a field of each kind the crosswalk reads. */ + const BASE_TIMESTAMP = 1_700_000_000; + /** How many distinct weights the input cycles through. */ + const WEIGHTS = 5; + /** The memory test's own budget: the sweeps and one encode over ELEMENTS, on a runner that may be loaded. */ + const TEST_TIMEOUT_MILLISECONDS = 60_000; + /** The element count of the small correctness check beside the memory one. */ + const SMALL_ELEMENTS = 3; + + const cw = crosswalk({ + source: "n", + target: "s", + rules: [ + { op: "move", from: "meta.id", to: "id" }, + { op: "move", from: "edges[].source", to: "links[].src" }, + { op: "move", from: "edges[].target", to: "links[].dst" }, + { op: "move", from: "edges[].type", to: "links[].rel" }, + { op: "move", from: "edges[].weight", to: "links[].w" }, + { op: "move", from: "edges[].timestamp", to: "links[].at" }, + ], + oneWay: [], + }); + const edges = (count: number) => Array.from({ length: count }, (_, index) => ({ source: `n${String(index)}`, target: `n${String((index + 1) % count)}`, type: "cites", weight: index % WEIGHTS, timestamp: BASE_TIMESTAMP + index })); + const sweep = (): void => { + for (let at = 0; at < SWEEPS; at += 1) { + new Array(SWEEP_SIZE).fill(at); + } + }; + + it("writes the same result over a small array", () => { + expect(encode(cw, { meta: { id: "g" }, edges: edges(SMALL_ELEMENTS) })).toEqual({ + id: "g", + links: [ + { src: "n0", dst: "n1", rel: "cites", w: 0, at: BASE_TIMESTAMP }, + { src: "n1", dst: "n2", rel: "cites", w: 1, at: BASE_TIMESTAMP + 1 }, + { src: "n2", dst: "n0", rel: "cites", w: 2, at: BASE_TIMESTAMP + 2 }, + ], + }); + }); + + it("holds only the output beyond the input, not a record per element per rule", () => { + const input = { meta: { id: "g" }, edges: edges(ELEMENTS) }; + sweep(); + const before = process.memoryUsage().heapUsed; + const shared = encode(cw, input); + expect(shared).toBeTypeOf("object"); + sweep(); + const growthPerEdge = (process.memoryUsage().heapUsed - before) / ELEMENTS; + expect(growthPerEdge).toBeLessThan(BYTES_PER_ELEMENT_BOUND); + }, TEST_TIMEOUT_MILLISECONDS); +}); diff --git a/src/paths.ts b/src/paths.ts index 41372c7..372badb 100644 --- a/src/paths.ts +++ b/src/paths.ts @@ -233,6 +233,103 @@ export function readLocus(root: unknown, locus: Locus, gate?: ElementGate): Locu return { matches: frontier, empties }; } +/** + * Visits every location a locus addresses without holding a record per element: the trail buffer is reused across visits, so a caller that walks a wildcard over millions of elements pays only the output it writes. Semantics match {@link readLocus} exactly: the same absence rules, the same empty-array reporting, the same gate behaviour, and visits in the same index order. + * @param root - The document to read from. + * @param locus - The parsed path to read. + * @param gate - Optional per-element admission, as {@link readLocus} takes it. + * @param visit - Called for each match with the trail (valid only until the next visit; copy it to keep it) and the value found. + * @returns The empty arrays a wildcard step met, or `undefined` under the same conditions {@link readLocus} returns `undefined`. + */ +export function walkLocus(root: unknown, locus: Locus, gate: ElementGate | undefined, visit: (trail: Trail, value: unknown) => void): readonly EmptySite[] | undefined { + const empties: EmptySite[] = []; + const trail: TrailStep[] = []; + + /** + * Walks one step of the locus over one value. + * @param position - The step's position in the locus. + * @param value - The value the steps so far reached. + * @returns False when the locus is absent from the document. + */ + const walk = (position: number, value: unknown): boolean => { + const step = locus[position]; + if (step === undefined) { + visit(trail, value); + + return true; + } + if (step.kind === "key") { + if (!isRecord(value) || !Object.hasOwn(value, step.key)) { + return false; + } + trail.push(step); + const deeper = walk(position + 1, getOwn(value, step.key)); + trail.pop(); + + return deeper; + } + if (!Array.isArray(value)) { + return false; + } + if (step.kind === "index") { + if (step.index >= value.length) { + return false; + } + trail.push(step); + const deeper = walk(position + 1, value[step.index]); + trail.pop(); + + return deeper; + } + if (gate?.arity === countWildcards(locus, position) + 1) { + if (value.length === 0) { + empties.push({ trail: [...trail], wildcard: countWildcards(locus, position) }); + } + for (let index = 0; index < value.length; index += 1) { + trail.push({ kind: "index", index }); + if (Object.hasOwn(value, index) && gate.admit(wildcardIndexes(locus, trail))) { + if (!walk(position + 1, value[index])) { + return false; + } + } + trail.pop(); + } + + return true; + } + if (value.length === 0) { + empties.push({ trail: [...trail], wildcard: countWildcards(locus, position) }); + } + for (let index = 0; index < value.length; index += 1) { + if (!Object.hasOwn(value, index)) { + continue; + } + trail.push({ kind: "index", index }); + const deeper = walk(position + 1, value[index]); + trail.pop(); + if (!deeper) { + return false; + } + } + + return true; + }; + + return walk(0, root) ? empties : undefined; +} + +/** How many wildcard steps a locus holds before a position (the position's own step excluded). */ +function countWildcards(locus: Locus, position: number): number { + let count = 0; + for (let at = 0; at < position; at += 1) { + if (locus[at]?.kind === "wildcard") { + count += 1; + } + } + + return count; +} + /** * The array indexes a trail took at a locus's wildcard steps, in order: the index tuple of one match. The trail was produced by {@link readLocus} over this locus (or a prefix of it), so its step at each wildcard position is an index. * @param locus - The locus the trail was read through.