From 60fb3e4c0a19d9fb384e69280e6d6ce40515a5e8 Mon Sep 17 00:00:00 2001 From: raiseCatError <315733358+raiseCatError@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:02:03 +0530 Subject: [PATCH 1/2] CI: three test shards per OS from measured weights; Chrome tests share one browser per file Hosted test shards had grown to 8-10 minutes against their 10-minute cap (one shard of the dev push for 47339ff1 was cancelled). The growth predates Mermaid: browser, document and live files added since the 2026-10-04 weights defaulted to weight 1, and several old weights were 3-4x low. Weights now come from the CI logs' per-test durations, the Chrome tests launch one browser per file except the launch, idle and crash tests, and the matrix runs three shards per OS so each lands near five minutes under the unchanged 10-minute cap. --- .github/workflows/ci.yml | 8 +-- docs/development-verification.md | 11 ++-- scripts/test-selection.mjs | 86 +++++++++++++++++++++++-------- tests/browserEngineChrome.test.ts | 31 +++++------ tests/diagramsChrome.test.ts | 19 +++++-- tests/support/browserFixtures.ts | 23 +++++++++ 6 files changed, 132 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6832882e..74826db0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,16 +67,18 @@ jobs: - run: git diff --check tests: - name: Tests (${{ matrix.os }}, ${{ matrix.shard }}/2) + name: Tests (${{ matrix.os }}, ${{ matrix.shard }}/3) needs: changes if: needs.changes.outputs.code == 'true' runs-on: ${{ matrix.os }} + # Three shards per OS keep each job near five minutes (two ran 8-10 minutes by October 2026). The cap stays at twice + # that so a hang still fails the job instead of running on; individual tests keep their own 120 s timeout. timeout-minutes: 10 strategy: fail-fast: false matrix: os: [macos-latest, ubuntu-24.04] - shard: [1, 2] + shard: [1, 2, 3] steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -101,7 +103,7 @@ jobs: echo "NMSH_TEST_MERMAID=$dir/package" >> "$GITHUB_ENV" # Built-launcher tests must execute rather than silently skip on fresh runners. - run: npm run build - - run: npm test -- --shard=${{ matrix.shard }}/2 --test-timeout=120000 + - run: npm test -- --shard=${{ matrix.shard }}/3 --test-timeout=120000 - name: Process leak check if: always() run: bash scripts/check-process-leaks.sh diff --git a/docs/development-verification.md b/docs/development-verification.md index bc31082b..04155ad7 100644 --- a/docs/development-verification.md +++ b/docs/development-verification.md @@ -22,13 +22,16 @@ Local commands reuse node_modules. Clean CI/release environments use ## Canonical suite and sharding -The only shard interface is `npm test -- --shard=1/2` (or `2/2`). No arguments +The only shard interface is `npm test -- --shard=/` (CI uses `1/3`, `2/3`, `3/3`). No arguments still runs every recursively discovered `tests/**/*.test.ts` file. File selection lives in `scripts/test-selection.mjs`; its tests verify deterministic assignment, exact union, no overlap, invalid-input rejection, and curated-list existence. Runtime files are partitioned as whole files by deterministic greedy scheduling. Rounded measured weights for slow files balance PTY work; ordinary/new files -default to one. Reprofile weights only when actual shard imbalance warrants it. `suggestionRanking.test.ts` runs +default to one. Reprofile weights only when actual shard imbalance warrants it: the per-test durations in the +CI logs, attributed to their files, are the source (last done 2026-10-22, when unweighted browser and live files had pushed +shards to 8–10 minutes of a 10-minute cap). Tests that drive a real Chromium share one browser per file where they test +pages rather than the browser's own lifecycle (`sharedLab()` in `tests/support/browserFixtures.ts`). `suggestionRanking.test.ts` runs separately after runtime files, only in shard one, preserving uncontested latency measurement. New files automatically enter the canonical suite and a shard. @@ -40,8 +43,8 @@ session scenarios run fully on Node 26 on both macOS and Ubuntu. ## CI gates -Code PRs run quality on Ubuntu/Node 26, two macOS/Node 26 shards, two Ubuntu/Node -26 shards, and curated Ubuntu/Node 22 compatibility. Each shard builds because +Code PRs run quality on Ubuntu/Node 26, three macOS/Node 26 shards, three Ubuntu/Node +26 shards (10-minute cap each, about twice a shard's normal time), and curated Ubuntu/Node 22 compatibility. Each shard builds because built-launcher tests otherwise skip on a clean checkout. Each integration shard and smoke job checks helper-process leaks, including after a test failure. Existing hard latency assertions remain in the canonical tests. diff --git a/scripts/test-selection.mjs b/scripts/test-selection.mjs index 3077682c..d248e884 100644 --- a/scripts/test-selection.mjs +++ b/scripts/test-selection.mjs @@ -18,36 +18,80 @@ const suites = { fedora: ['linuxPlatform', 'tools', 'update', 'hostProfiles', 'portabilityUninstall'], }; -// Rounded seconds from the macOS Node 26 canonical baseline (2026-10-04). +// Rounded seconds: the slower of the macOS and Ubuntu Node 26 CI shards (run 39756506927, 2026-10-22); files not +// measured there keep their 2026-10-04 macOS baseline. The two Chrome files are estimates after they began sharing one +// browser per file. // Only slow files need weights; new/ordinary files default to one. Greedy // whole-file scheduling avoids clustering the costly PTY files in one shard. const runtimeWeights = { - "bundledCatalog.test.ts": 5, - "compatibilityHarness.test.ts": 28, - "commandQueueLive.test.ts": 93, - "copySelectionLive.test.ts": 13, - "configuredCompletion.test.ts": 6, - "ctrlZJobControl.test.ts": 12, - "detachedOutput.test.ts": 22, + "activityStyleLive.test.ts": 16, + "ansiLiveRegion.test.ts": 15, + "arcadeAll.test.ts": 8, + "arcadeBackgroundLive.test.ts": 52, + "arcadeDashGame.test.ts": 26, + "arcadeDashPhysics.test.ts": 26, + "arcadePinball.test.ts": 20, + "arcadeQueensLogic.test.ts": 15, + "arcadeSudokuLogic.test.ts": 6, + "backgroundSupervisor.test.ts": 11, + "browserEngineChrome.test.ts": 50, + "browserPanel.test.ts": 6, + "bundledCatalog.test.ts": 11, + "chromaV2.test.ts": 6, + "clipsLive.test.ts": 6, + "commandQueueLive.test.ts": 97, + "compareOutputLive.test.ts": 10, + "compatibilityHarness.test.ts": 47, + "configuredCompletion.test.ts": 13, + "contextEngine.test.ts": 13, + "contextHostileWorkspace.test.ts": 9, + "copyExactLive.test.ts": 7, + "copyPayload.test.ts": 5, + "copyReportLive.test.ts": 5, + "copySelectionLive.test.ts": 51, + "ctrlZJobControl.test.ts": 24, + "detachedOutput.test.ts": 38, + "diagramsChrome.test.ts": 25, + "flowComposer.test.ts": 8, "fullChromaQa.test.ts": 5, - "hostProfiles.test.ts": 10, - "inputAwarenessLive.test.ts": 59, - "interactiveCli.test.ts": 14, - "liveHardening.test.ts": 45, - "liveStatus.test.ts": 5, - "muxInterop.test.ts": 31, - "nativeCaptureLifecycle.test.ts": 6, - "sessionLifecycle.test.ts": 28, - "sessionPresets.test.ts": 20, + "fullscreenFailure.test.ts": 8, + "fullscreenResizeSampling.test.ts": 10, + "historyReuseLive.test.ts": 11, + "hostProfiles.test.ts": 18, + "hybrid.test.ts": 6, + "inputAwarenessLive.test.ts": 52, + "inputOwnershipLive.test.ts": 46, + "interactiveCli.test.ts": 37, + "keepAwakeForeground.test.ts": 8, + "liveHardening.test.ts": 88, + "liveStatus.test.ts": 10, + "moduleDiagnosticsLive.test.ts": 7, + "muxInterop.test.ts": 47, + "nativeCaptureLifecycle.test.ts": 8, + "outputReferencesLive.test.ts": 7, "pasteBatchLive.test.ts": 21, + "pinsLive.test.ts": 7, "queueConditionsLive.test.ts": 24, + "queuePreviewLive.test.ts": 33, + "queueSummaryLive.test.ts": 23, + "sessionLifecycle.test.ts": 78, + "sessionPresets.test.ts": 45, "shellAdapters.test.ts": 35, + "shellIntelligence.test.ts": 22, "shellSwitchApp.test.ts": 23, - "startupBlocked.test.ts": 29, - "startupDiscovery.test.ts": 22, - "startupLaunch.test.ts": 8, + "startupBlocked.test.ts": 64, + "startupDiscovery.test.ts": 87, + "startupLaunch.test.ts": 10, + "startupLifecycle.test.ts": 7, + "terminalBaseline.test.ts": 9, + "terminalModeReconciliation.test.ts": 8, + "themeBridgeLive.test.ts": 5, + "tmpdirHygiene.test.ts": 9, + "toolConfig.test.ts": 6, + "trackLive.test.ts": 13, "typingAfterRead.test.ts": 17, - "themeBridgeLive.test.ts": 12 + "viewerLive.test.ts": 13, + "worktreeMutations.test.ts": 11 }; export function discoverTestFiles() { diff --git a/tests/browserEngineChrome.test.ts b/tests/browserEngineChrome.test.ts index 3656290a..baf74149 100644 --- a/tests/browserEngineChrome.test.ts +++ b/tests/browserEngineChrome.test.ts @@ -1,9 +1,9 @@ -import test from 'node:test'; +import test, {after} from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import type {BrowserPage, Frame} from '../src/browser/types.js'; -import {chromeSkipReason, createLab, Events, localSite, pidAlive, scratchDir, waitFor} from './support/browserFixtures.js'; +import {chromeSkipReason, createLab, disposeSharedLab, Events, localSite, pidAlive, scratchDir, sharedLab, waitFor} from './support/browserFixtures.js'; import {createBrowserEngine} from '../src/browser/engine/index.js'; /** @@ -14,6 +14,7 @@ const skip = await chromeSkipReason(); const view = {width: 600, height: 400, scale: 1}; const sleep = (ms: number): Promise => new Promise(resolve => setTimeout(resolve, ms)); const settled = (page: BrowserPage, url?: string): Promise => waitFor(() => (url === undefined || page.url === url) && !page.loading && true, `${url ?? 'the page'} to load`); +after(disposeSharedLab); test('launch is lazy, uses a private profile, and shutdown leaves no process behind', {skip}, async () => { const lab = createLab(); @@ -93,7 +94,7 @@ test('a missing browser is reported in plain words and nothing is started', asyn }); test('web pages load from a local server; navigation, back, forward and reload work', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); const site = await localSite({'/a': 'Page A

a

', '/b': 'Page B

b

'}); try { const page = await lab.engine.open({kind: 'web', viewport: view, url: `${site.origin}/a`}); @@ -122,7 +123,7 @@ test('web pages load from a local server; navigation, back, forward and reload w }); test('addresses that are not http, https, file or about:blank are refused and never navigated', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); try { const page = await lab.engine.open({kind: 'web', viewport: view, url: 'about:blank'}); const events = new Events(page); @@ -138,7 +139,7 @@ test('addresses that are not http, https, file or about:blank are refused and ne }); test('a page that redirects or links to a refused scheme ends on about:blank with a blocked event', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); const site = await localSite({'/': 'xstart'}); try { const page = await lab.engine.open({kind: 'web', viewport: view, url: site.origin + '/'}); @@ -158,7 +159,7 @@ test('a page that redirects or links to a refused scheme ends on about:blank wit const animation = '
'; test('screencast frames arrive at the viewport size, advance with a CSS animation, and a slow listener skips frames', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); try { const handle = await lab.preview(animation); const page = await lab.engine.open({kind: 'preview', viewport: {width: 320, height: 200, scale: 1}, url: handle.url}); @@ -197,7 +198,7 @@ test('screencast frames arrive at the viewport size, advance with a CSS animatio }); test('zoom keeps the frame size and is clamped', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); try { const handle = await lab.preview('

zoom

'); const page = await lab.engine.open({kind: 'preview', viewport: {width: 300, height: 200, scale: 1}, url: handle.url}); @@ -224,7 +225,7 @@ const form = `
'); const page = await lab.engine.open({kind: 'preview', viewport: view, url: handle.url}); @@ -276,7 +277,7 @@ test('wheel events scroll the page', {skip}, async () => { }); test('find works on a preview with scripts off, steps and wraps, and clears', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); try { const handle = await lab.preview('

Needle one

nothing

second needle

needle hidden

NEEDLE last

'); const page = await lab.engine.open({kind: 'preview', viewport: view, url: handle.url}); @@ -299,7 +300,7 @@ test('find works on a preview with scripts off, steps and wraps, and clears', {s }); test('setScripts turns page scripts on and off and reloads', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); try { const handle = await lab.preview('before', {trusted: true}); const page = await lab.engine.open({kind: 'preview', viewport: view, url: handle.url}); @@ -315,7 +316,7 @@ test('setScripts turns page scripts on and off and reloads', {skip}, async () => }); test('popups are closed and reported; dialogs are answered; downloads are refused', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); const site = await localSite({ '/': `pop get @@ -352,7 +353,7 @@ test('popups are closed and reported; dialogs are answered; downloads are refuse }); test('a preview refuses remote resources and navigation, and no request leaves the machine', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); const remote = await localSite(); try { const html = ` @@ -379,7 +380,7 @@ test('a preview refuses remote resources and navigation, and no request leaves t }); test('previews share nothing with browsing: cookies set in a web page are invisible to a preview', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); const site = await localSite({'/': {type: 'text/html', body: 'set', headers: {'Set-Cookie': 'k=v; Path=/'}}}); try { const web = await lab.engine.open({kind: 'web', viewport: view, url: site.origin + '/'}); @@ -395,7 +396,7 @@ test('previews share nothing with browsing: cookies set in a web page are invisi }); test('closing a page closes its target and a preview page disposes its context', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); try { const handle = await lab.preview('

x

'); const page = await lab.engine.open({kind: 'preview', viewport: view, url: handle.url}); diff --git a/tests/diagramsChrome.test.ts b/tests/diagramsChrome.test.ts index b56b19d8..6183af2e 100644 --- a/tests/diagramsChrome.test.ts +++ b/tests/diagramsChrome.test.ts @@ -1,4 +1,4 @@ -import test from 'node:test'; +import test, {after} from 'node:test'; import assert from 'node:assert/strict'; import http from 'node:http'; import type {AddressInfo} from 'node:net'; @@ -31,12 +31,25 @@ async function skip(): Promise { return (await chromeSkipReason()) ?? (await mermaidSkip); } -async function lab(): Promise<{engine: BrowserEngine; dir: string; done(): Promise}> { +interface DiagramLab {engine: BrowserEngine; dir: string; done(): Promise} + +function freshLab(): DiagramLab { const dir = scratchDir('nmsh-diagrams-'); const engine = createBrowserEngine({profileDir: path.join(dir, 'profile'), idleMs: 60_000}); return {engine, dir, done: async () => { await engine.shutdown(); fs.rmSync(dir, {recursive: true, force: true, maxRetries: 10, retryDelay: 200}); }}; } +// Launching Chromium costs seconds on CI runners: tests that only open renderer pages share one browser; each test closes the +// pages it left open, so the next starts with none. The crash test keeps a browser of its own. +let shared: DiagramLab | undefined; +after(async () => { await shared?.done(); shared = undefined; }); + +async function lab(options: {fresh?: boolean} = {}): Promise { + if (options.fresh) return freshLab(); + const base = shared ??= freshLab(); + return {...base, done: async () => { await Promise.all(base.engine.pages().map(page => page.close().catch(() => undefined))); }}; +} + const request = (source: string, extra: Partial = {}): DiagramRenderRequest => ({source, config: mermaidConfig(THEME), scale: 2, timeoutMs: 10_000, ...extra}); const TYPES: Record = { @@ -176,7 +189,7 @@ test('a library that never finishes is stopped at the timeout; the page is throw test('a crashed renderer page is reported, never hangs, and the browser stops cleanly afterwards', async t => { const reason = await skip(); if (reason) { t.skip(reason); return; } - const {engine, done} = await lab(); + const {engine, done} = await lab({fresh: true}); try { const renderer: DiagramRenderer = await engine.openDiagramRenderer!(library!); assert.ok((await renderer.render(request(TYPES.flowchart!))).ok); diff --git a/tests/support/browserFixtures.ts b/tests/support/browserFixtures.ts index dde0fdf8..ada058aa 100644 --- a/tests/support/browserFixtures.ts +++ b/tests/support/browserFixtures.ts @@ -93,6 +93,29 @@ export function createLab(options: {idleMs?: number} = {}): Lab { return lab; } +let shared: Lab | undefined; + +/** + * One browser for every test in a file that only needs pages: launching Chromium costs seconds on CI runners, so tests that do + * not exercise launch, idle shutdown or crashes share it. `dispose()` closes the pages the test left open (each test starts with + * none) and keeps the browser; call `disposeSharedLab()` from the file's `after` hook. Lifecycle tests keep `createLab()`. + */ +export function sharedLab(): Lab { + const lab = shared ??= createLab(); + return { + ...lab, + async dispose() { + await Promise.all(lab.engine.pages().map(page => page.close().catch(() => undefined))); + } + }; +} + +export async function disposeSharedLab(): Promise { + const lab = shared; + shared = undefined; + await lab?.dispose(); +} + /** Collects a page's events and lets a test wait for one. */ export class Events { readonly all: PageEvent[] = []; From a7511cdaa2a5d50e1e68ef98ba7466b421bd5592 Mon Sep 17 00:00:00 2001 From: raiseCatError <315733358+raiseCatError@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:16:25 +0530 Subject: [PATCH 2/2] CI: test shard cap 12 minutes, about 1.5x the slowest measured three-shard job --- .github/workflows/ci.yml | 7 ++++--- docs/development-verification.md | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74826db0..2fee9a73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,9 +71,10 @@ jobs: needs: changes if: needs.changes.outputs.code == 'true' runs-on: ${{ matrix.os }} - # Three shards per OS keep each job near five minutes (two ran 8-10 minutes by October 2026). The cap stays at twice - # that so a hang still fails the job instead of running on; individual tests keep their own 120 s timeout. - timeout-minutes: 10 + # Three shards per OS: test steps measured 4.9-6.6 minutes and whole jobs 6-7.8 minutes on 2026-10-22 (two shards had + # reached 8-10 minutes against a 10-minute cap). The cap is about 1.5x the slowest job, so slow macOS runners pass and a + # hang still fails the job; every test keeps its own 120 s timeout. + timeout-minutes: 12 strategy: fail-fast: false matrix: diff --git a/docs/development-verification.md b/docs/development-verification.md index 04155ad7..ee868ef2 100644 --- a/docs/development-verification.md +++ b/docs/development-verification.md @@ -44,7 +44,7 @@ session scenarios run fully on Node 26 on both macOS and Ubuntu. ## CI gates Code PRs run quality on Ubuntu/Node 26, three macOS/Node 26 shards, three Ubuntu/Node -26 shards (10-minute cap each, about twice a shard's normal time), and curated Ubuntu/Node 22 compatibility. Each shard builds because +26 shards (12-minute cap each, about 1.5 times the slowest measured job, so a hang still fails), and curated Ubuntu/Node 22 compatibility. Each shard builds because built-launcher tests otherwise skip on a clean checkout. Each integration shard and smoke job checks helper-process leaks, including after a test failure. Existing hard latency assertions remain in the canonical tests.