From 7e1f6d4d9e7c33918adf9ecadfb10aaff4e4e7b8 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 9 Nov 2026 07:44:35 -0700 Subject: [PATCH 1/2] ci(bench): measure the incremental benchmark as a production bundle The benchmark imported db's source through tsx, so it measured development-mode code: unmangled private members, NODE_ENV checks and development-only branches that every production app strips. A new runner, scripts/bench/production.mjs, bundles the benchmark against the built db and db-ivm dist with esbuild, minified, with process.env.NODE_ENV set to production, as the size scenarios do. It refuses a dist older than its package's sources. CI builds both packages on each side and runs the bundle. Co-authored-by: Isaac --- .github/workflows/benchmark.yml | 8 ++--- scripts/bench/README.md | 15 +++++++-- scripts/bench/production.mjs | 55 +++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 scripts/bench/production.mjs diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 48e1a8a6ab..0d94664d4f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -67,8 +67,8 @@ jobs: - name: Run benchmark on PR if: steps.inputs.outputs.changed == 'true' run: | - pnpm --filter @tanstack/db-ivm build - pnpm exec tsx scripts/bench/incremental-update.ts \ + pnpm --filter @tanstack/db-ivm --filter @tanstack/db build + node scripts/bench/production.mjs scripts/bench/incremental-update.ts \ --iterations=100 --warmup=20 --repetitions=3 \ --outFile="$RUNNER_TEMP/candidate.json" - name: Run benchmark on base @@ -83,8 +83,8 @@ jobs: # Run the exact same benchmark code on both sides git checkout "$HEAD_SHA" -- scripts/bench pnpm install --frozen-lockfile - pnpm --filter @tanstack/db-ivm build - pnpm exec tsx scripts/bench/incremental-update.ts \ + pnpm --filter @tanstack/db-ivm --filter @tanstack/db build + node scripts/bench/production.mjs scripts/bench/incremental-update.ts \ --iterations=100 --warmup=20 --repetitions=3 \ --outFile="$RUNNER_TEMP/base.json" - name: Compare results diff --git a/scripts/bench/README.md b/scripts/bench/README.md index d0f1be889e..16dfa7ab9f 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -1,15 +1,22 @@ # Incremental query benchmark -Run `pnpm --filter @tanstack/db-ivm build`, then: +Run `pnpm --filter @tanstack/db-ivm --filter @tanstack/db build`, then: ```sh -node --import tsx scripts/bench/incremental-update.ts \ +node scripts/bench/production.mjs scripts/bench/incremental-update.ts \ --iterations=100 --warmup=20 --repetitions=3 --outFile=candidate.json node --import tsx scripts/bench/compare-incremental-update.ts \ --base=base.json --candidate=candidate.json --failOnRegression=true node --test scripts/bench/compare-incremental-update.test.mjs ``` +The benchmark measures TanStack DB as production apps run it. +`production.mjs` bundles the benchmark against the built `dist` of db and +db-ivm, minified, with `process.env.NODE_ENV` set to `production`, as the +size scenarios do. Development-only code, such as full error messages and +oracle probes, is absent, and db's private members carry their mangled names. +The runner refuses a `dist` that is older than its package's sources. + Each repetition creates fresh fixtures and queries, times their first preload, and measures writes after warmup. Reports retain all cold-load times and per-repetition write medians; displayed statistics are medians across repetitions. @@ -53,6 +60,10 @@ for source controls, validation and limits. ## CI variation measured on 2026-11-05 +These runs measured db's source in development mode, before the benchmark moved +to a production bundle. Their spread is evidence about the runners, not a +calibration of the production-bundle timings. + The [measurement table](ci-variation-2026-11-05.csv) contains query rollups from ten paired runs with no changes to `packages/db/src` or `packages/db-ivm/src`. GitHub's compare API verified that premise (each response had fewer than its diff --git a/scripts/bench/production.mjs b/scripts/bench/production.mjs new file mode 100644 index 0000000000..76bb054f3a --- /dev/null +++ b/scripts/bench/production.mjs @@ -0,0 +1,55 @@ +// Runs a benchmark as an app's production build ships TanStack DB: bundled +// against the built packages/db and packages/db-ivm dist, minified, with +// process.env.NODE_ENV set to `production`. Development-only code, such as +// full error messages and oracle probes, is therefore absent, and db's +// private members carry their mangled names. The arguments after the +// benchmark file pass through to it: +// +// node scripts/bench/production.mjs scripts/bench/incremental-update.ts \ +// --iterations=100 --warmup=20 --repetitions=3 --outFile=candidate.json +import { rmSync } from 'node:fs' +import { mkdtemp } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { build } from 'esbuild' +import { assertPackageDistFresh } from '../../packages/db/scripts/assert-dist-fresh.mjs' + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..') +const [benchmark, ...args] = process.argv.slice(2) +if (!benchmark) + throw new Error('Usage: node scripts/bench/production.mjs [args]') + +const dbEntry = await assertPackageDistFresh(root, 'db') +const bundleDir = await mkdtemp(path.join(tmpdir(), 'tanstack-db-bench-')) +process.on('exit', () => rmSync(bundleDir, { recursive: true, force: true })) +const outfile = path.join(bundleDir, 'benchmark.mjs') +await build({ + entryPoints: [path.resolve(benchmark)], + outfile, + bundle: true, + minify: true, + format: 'esm', + platform: 'node', + // Resolve imports as an app would: the repo's tsconfig paths would + // otherwise inline the workspace packages from their sources. The + // benchmark imports db's source entry for its types; the bundle takes the + // built entry instead. + tsconfigRaw: '{}', + alias: { '@tanstack/db-ivm': await assertPackageDistFresh(root, 'db-ivm') }, + plugins: [ + { + name: 'built-db', + setup(bundle) { + bundle.onResolve({ filter: /\/packages\/db\/src\/index\.js$/ }, () => ({ + path: dbEntry, + })) + }, + }, + ], + define: { 'process.env.NODE_ENV': '"production"' }, + logLevel: 'error', +}) + +process.argv = [process.execPath, outfile, ...args] +await import(pathToFileURL(outfile).href) From df5a78ab581840fe15745ac6a6135e86fc01711b Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 9 Nov 2026 08:32:40 -0700 Subject: [PATCH 2/2] ci(bench): reject source modules in the production bundle Review fixes for the production benchmark runner: - The runner fails when esbuild's metafile lists any packages/*/src input. Before, a second import of db source bundled a development copy of db beside its dist without notice (a planted import of packages/db/src/collection/index.js ran to exit 0; it now fails and names the source files). - The bundle carries an inline source map, and the runner enables source maps before it imports the bundle, so a failed assertion names its line in incremental-update.ts. The bundle directory uses the real temp path, because macOS's /var symlink offset the map's relative paths. - Both freshness checks run before the bundle directory is created. - The base step also takes the freshness helper from the PR head, so both sides run the same runner code. Co-authored-by: Isaac --- .github/workflows/benchmark.yml | 5 +++-- scripts/bench/production.mjs | 25 +++++++++++++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 0d94664d4f..ec61e13616 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -80,8 +80,9 @@ jobs: BASE_SHA=$(git merge-base "origin/$BASE_REF" HEAD) echo "Benchmarking base commit $BASE_SHA" git checkout --force "$BASE_SHA" - # Run the exact same benchmark code on both sides - git checkout "$HEAD_SHA" -- scripts/bench + # Run the exact same benchmark code on both sides, including the + # runner's dist freshness check + git checkout "$HEAD_SHA" -- scripts/bench packages/db/scripts/assert-dist-fresh.mjs pnpm install --frozen-lockfile pnpm --filter @tanstack/db-ivm --filter @tanstack/db build node scripts/bench/production.mjs scripts/bench/incremental-update.ts \ diff --git a/scripts/bench/production.mjs b/scripts/bench/production.mjs index 76bb054f3a..0c2902e02d 100644 --- a/scripts/bench/production.mjs +++ b/scripts/bench/production.mjs @@ -7,7 +7,7 @@ // // node scripts/bench/production.mjs scripts/bench/incremental-update.ts \ // --iterations=100 --warmup=20 --repetitions=3 --outFile=candidate.json -import { rmSync } from 'node:fs' +import { realpathSync, rmSync } from 'node:fs' import { mkdtemp } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' @@ -21,14 +21,19 @@ if (!benchmark) throw new Error('Usage: node scripts/bench/production.mjs [args]') const dbEntry = await assertPackageDistFresh(root, 'db') -const bundleDir = await mkdtemp(path.join(tmpdir(), 'tanstack-db-bench-')) +const ivmEntry = await assertPackageDistFresh(root, 'db-ivm') +const bundleDir = await mkdtemp( + path.join(realpathSync(tmpdir()), 'tanstack-db-bench-'), +) process.on('exit', () => rmSync(bundleDir, { recursive: true, force: true })) const outfile = path.join(bundleDir, 'benchmark.mjs') -await build({ +const { metafile } = await build({ entryPoints: [path.resolve(benchmark)], outfile, bundle: true, minify: true, + sourcemap: 'inline', + metafile: true, format: 'esm', platform: 'node', // Resolve imports as an app would: the repo's tsconfig paths would @@ -36,7 +41,7 @@ await build({ // benchmark imports db's source entry for its types; the bundle takes the // built entry instead. tsconfigRaw: '{}', - alias: { '@tanstack/db-ivm': await assertPackageDistFresh(root, 'db-ivm') }, + alias: { '@tanstack/db-ivm': ivmEntry }, plugins: [ { name: 'built-db', @@ -51,5 +56,17 @@ await build({ logLevel: 'error', }) +// A source module in the bundle would measure development code, or a second +// copy of a package beside its dist. +const sources = Object.keys(metafile.inputs).filter((input) => + /(^|\/)packages\/[^/]+\/src\//.test(input), +) +if (sources.length > 0) + throw new Error( + `The benchmark bundle must use built packages, but it includes sources:\n${sources.join('\n')}`, + ) + +// The stack trace of a failed benchmark assertion names its source line. +process.setSourceMapsEnabled(true) process.argv = [process.execPath, outfile, ...args] await import(pathToFileURL(outfile).href)