diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 19d243b..dbdf66b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -33,6 +33,10 @@ jobs: - run: pnpm run format:check - run: pnpm run typecheck - run: pnpm --filter @tanstack/oracles run test + # Checks the partition of the end-to-end gap oracle into CI shards + # (packages/oracles/scripts/e2e-shards.mjs) ahead of the shard jobs, + # which CI does not run yet: every test is in exactly one shard. + - run: pnpm --filter @tanstack/oracles run test:oracle-gaps:shards - run: pnpm run test:publint - run: pnpm run test:sherif test-eslint-plugin: diff --git a/packages/oracles/package.json b/packages/oracles/package.json index b6e094e..bf7b48d 100644 --- a/packages/oracles/package.json +++ b/packages/oracles/package.json @@ -64,6 +64,7 @@ "test:oracle-framework": "vitest --run --config vitest.oracle-framework.config.mjs", "test:oracle-gaps": "vitest --run --config oracle-gaps/vitest.config.mjs", "test:oracle-gaps:fast": "vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static gap-report-classify gap-report-probe-form gap-report-mutant-validity", + "test:oracle-gaps:shards": "node scripts/e2e-shards.mjs check", "typecheck": "tsc --noEmit" }, "dependencies": { diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs new file mode 100644 index 0000000..197a982 --- /dev/null +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -0,0 +1,135 @@ +// The CI shards of the end-to-end gap oracle +// (oracle-gaps/gap-report-oracle.property.test.mjs). +// +// The oracle runs the real CLI on scratch packages. Its work is CPU-bound, so +// shards run as separate CI jobs, never side by side on one runner. Each +// shard names whole tests of the property. A shard's tests are chosen by +// their measured cost, longest first, so that the slowest shard sets the +// job's time. Every test of the file must be in exactly one shard: a test in +// no shard would leave its law out of CI, and a test in two would run twice. +// +// node scripts/e2e-shards.mjs check checks the partition against +// `vitest list` +// node scripts/e2e-shards.mjs pattern prints the shard's `-t` +// pattern +// +// The pattern matches a test's full name, its suite's name and its own, +// joined as Vitest 4 joins them (a space) or as Vitest 5 does (` > `). +import { spawnSync } from 'node:child_process' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const suite = `oracle-gaps.report` + +// The shards balance the costs measured in one full local run on main +// (84cd34a, 2026-11-11, an idle 18-core Mac, one Vitest worker), in local +// seconds: determinism about 345 (all 30 histories, twice each), the random +// seed 160 to 600 by its seed, the fixed seed 157, the refused-run +// calibration 102, the two daemon calibrations 56 and 53, and each other +// calibration 38 to 46. CI took 2.31 times as long. The property registers +// each campaign, each calibration, and the determinism test as one test, so +// a shard cannot be smaller than its largest test, and the determinism test +// alone sets the slowest shard. CI keeps its one fixed-seed job until the +// property registers its parts as units that a shard can split. +export const shards = { + 'e2e-1': [`determinism`], + 'e2e-2': [`random seed`], + 'e2e-3': [ + `fixed seed 20261105`, + `calibration: a refused run is reported as complete`, + ], + 'e2e-4': [ + `calibration: the daemon reports another outcome than the cold run`, + `calibration: a daemon is left after --stop-daemon`, + `calibration: a mutant is dropped from the enumeration`, + `calibration: an owned mutant is reported as unowned`, + ], + 'e2e-5': [ + `calibration: a crash is reported as a kill`, + `calibration: a key drops its statement text`, + `calibration: the summary omits a gap`, + `calibration: an edit moves a key`, + `calibration: production code is reported as probe code`, + ], +} + +const escape = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, `\\$&`) + +/** The `-t` pattern that selects exactly a shard's tests. */ +export function patternOf(shard) { + const tests = shards[shard] + if (tests === undefined) + throw new Error( + `no shard ${JSON.stringify(shard)}; the shards are ${Object.keys(shards).join(`, `)}`, + ) + return `^${escape(suite)}(?: | > )(?:${tests.map(escape).join(`|`)})$` +} + +/** + * The partition's errors against the tests that `vitest list` reports: a + * test in no shard or in several, and a shard entry that names no test. + */ +export function partitionErrors(listed) { + const errors = [] + const patterns = Object.keys(shards).map((shard) => ({ + shard, + pattern: new RegExp(patternOf(shard)), + })) + for (const name of listed) { + // `vitest list` joins the names with ` > `; Vitest 4's `-t` reads them + // joined with a space, so a pattern must select the test both ways. + const joined = [name, name.replace(/ > /, ` `)] + const owners = patterns.filter(({ pattern }) => + joined.every((full) => pattern.test(full)), + ) + if (owners.length !== 1) + errors.push( + `${JSON.stringify(name)} is in ${owners.length === 0 ? `no shard` : `shards ${owners.map(({ shard }) => shard).join(`, `)}`}`, + ) + } + for (const [shard, tests] of Object.entries(shards)) + for (const test of tests) + if (!listed.includes(`${suite} > ${test}`)) + errors.push(`shard ${shard} names ${JSON.stringify(test)}, no test`) + return errors +} + +function listed() { + const run = spawnSync( + `pnpm`, + [ + `exec`, + `vitest`, + `list`, + `--json`, + `--config`, + `oracle-gaps/vitest.config.mjs`, + `gap-report-oracle`, + ], + { + cwd: join(dirname(fileURLToPath(import.meta.url)), `..`), + encoding: `utf8`, + }, + ) + if (run.status !== 0) + throw new Error(`vitest list failed:\n${run.stderr}${run.stdout}`) + return JSON.parse(run.stdout).map((test) => test.name) +} + +const [command, shard] = process.argv.slice(2) +if (command === `pattern`) process.stdout.write(`${patternOf(shard)}\n`) +else if (command === `check`) { + const errors = partitionErrors(listed()) + if (errors.length > 0) { + process.stderr.write( + `The e2e shards do not partition the tests:\n${errors.map((error) => `- ${error}`).join(`\n`)}\nEdit packages/oracles/scripts/e2e-shards.mjs so each test is in exactly one shard.\n`, + ) + process.exit(1) + } + process.stdout.write( + `${Object.keys(shards).length} shards partition the e2e tests\n`, + ) +} else if (command !== undefined) { + process.stderr.write(`usage: e2e-shards.mjs check | pattern \n`) + process.exit(2) +}