diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 48e1a8a6ab..ec61e13616 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -67,8 +67,8 @@ jobs: - name: Run benchmark on PR if: steps.inputs.outputs.changed == 'true' run: | - pnpm --filter @tanstack/db-ivm build - pnpm exec tsx scripts/bench/incremental-update.ts \ + pnpm --filter @tanstack/db-ivm --filter @tanstack/db build + node scripts/bench/production.mjs scripts/bench/incremental-update.ts \ --iterations=100 --warmup=20 --repetitions=3 \ --outFile="$RUNNER_TEMP/candidate.json" - name: Run benchmark on base @@ -80,11 +80,12 @@ jobs: BASE_SHA=$(git merge-base "origin/$BASE_REF" HEAD) echo "Benchmarking base commit $BASE_SHA" git checkout --force "$BASE_SHA" - # Run the exact same benchmark code on both sides - git checkout "$HEAD_SHA" -- scripts/bench + # Run the exact same benchmark code on both sides, including the + # runner's dist freshness check + git checkout "$HEAD_SHA" -- scripts/bench packages/db/scripts/assert-dist-fresh.mjs pnpm install --frozen-lockfile - pnpm --filter @tanstack/db-ivm build - pnpm exec tsx scripts/bench/incremental-update.ts \ + pnpm --filter @tanstack/db-ivm --filter @tanstack/db build + node scripts/bench/production.mjs scripts/bench/incremental-update.ts \ --iterations=100 --warmup=20 --repetitions=3 \ --outFile="$RUNNER_TEMP/base.json" - name: Compare results diff --git a/scripts/bench/README.md b/scripts/bench/README.md index d0f1be889e..16dfa7ab9f 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -1,15 +1,22 @@ # Incremental query benchmark -Run `pnpm --filter @tanstack/db-ivm build`, then: +Run `pnpm --filter @tanstack/db-ivm --filter @tanstack/db build`, then: ```sh -node --import tsx scripts/bench/incremental-update.ts \ +node scripts/bench/production.mjs scripts/bench/incremental-update.ts \ --iterations=100 --warmup=20 --repetitions=3 --outFile=candidate.json node --import tsx scripts/bench/compare-incremental-update.ts \ --base=base.json --candidate=candidate.json --failOnRegression=true node --test scripts/bench/compare-incremental-update.test.mjs ``` +The benchmark measures TanStack DB as production apps run it. +`production.mjs` bundles the benchmark against the built `dist` of db and +db-ivm, minified, with `process.env.NODE_ENV` set to `production`, as the +size scenarios do. Development-only code, such as full error messages and +oracle probes, is absent, and db's private members carry their mangled names. +The runner refuses a `dist` that is older than its package's sources. + Each repetition creates fresh fixtures and queries, times their first preload, and measures writes after warmup. Reports retain all cold-load times and per-repetition write medians; displayed statistics are medians across repetitions. @@ -53,6 +60,10 @@ for source controls, validation and limits. ## CI variation measured on 2026-11-05 +These runs measured db's source in development mode, before the benchmark moved +to a production bundle. Their spread is evidence about the runners, not a +calibration of the production-bundle timings. + The [measurement table](ci-variation-2026-11-05.csv) contains query rollups from ten paired runs with no changes to `packages/db/src` or `packages/db-ivm/src`. GitHub's compare API verified that premise (each response had fewer than its diff --git a/scripts/bench/production.mjs b/scripts/bench/production.mjs new file mode 100644 index 0000000000..0c2902e02d --- /dev/null +++ b/scripts/bench/production.mjs @@ -0,0 +1,72 @@ +// Runs a benchmark as an app's production build ships TanStack DB: bundled +// against the built packages/db and packages/db-ivm dist, minified, with +// process.env.NODE_ENV set to `production`. Development-only code, such as +// full error messages and oracle probes, is therefore absent, and db's +// private members carry their mangled names. The arguments after the +// benchmark file pass through to it: +// +// node scripts/bench/production.mjs scripts/bench/incremental-update.ts \ +// --iterations=100 --warmup=20 --repetitions=3 --outFile=candidate.json +import { realpathSync, rmSync } from 'node:fs' +import { mkdtemp } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { build } from 'esbuild' +import { assertPackageDistFresh } from '../../packages/db/scripts/assert-dist-fresh.mjs' + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..') +const [benchmark, ...args] = process.argv.slice(2) +if (!benchmark) + throw new Error('Usage: node scripts/bench/production.mjs [args]') + +const dbEntry = await assertPackageDistFresh(root, 'db') +const ivmEntry = await assertPackageDistFresh(root, 'db-ivm') +const bundleDir = await mkdtemp( + path.join(realpathSync(tmpdir()), 'tanstack-db-bench-'), +) +process.on('exit', () => rmSync(bundleDir, { recursive: true, force: true })) +const outfile = path.join(bundleDir, 'benchmark.mjs') +const { metafile } = await build({ + entryPoints: [path.resolve(benchmark)], + outfile, + bundle: true, + minify: true, + sourcemap: 'inline', + metafile: true, + format: 'esm', + platform: 'node', + // Resolve imports as an app would: the repo's tsconfig paths would + // otherwise inline the workspace packages from their sources. The + // benchmark imports db's source entry for its types; the bundle takes the + // built entry instead. + tsconfigRaw: '{}', + alias: { '@tanstack/db-ivm': ivmEntry }, + plugins: [ + { + name: 'built-db', + setup(bundle) { + bundle.onResolve({ filter: /\/packages\/db\/src\/index\.js$/ }, () => ({ + path: dbEntry, + })) + }, + }, + ], + define: { 'process.env.NODE_ENV': '"production"' }, + logLevel: 'error', +}) + +// A source module in the bundle would measure development code, or a second +// copy of a package beside its dist. +const sources = Object.keys(metafile.inputs).filter((input) => + /(^|\/)packages\/[^/]+\/src\//.test(input), +) +if (sources.length > 0) + throw new Error( + `The benchmark bundle must use built packages, but it includes sources:\n${sources.join('\n')}`, + ) + +// The stack trace of a failed benchmark assertion names its source line. +process.setSourceMapsEnabled(true) +process.argv = [process.execPath, outfile, ...args] +await import(pathToFileURL(outfile).href)