From 7c2095bc7034f2e997bf657b696dd6324d2d108d Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Mon, 2 Nov 2026 17:01:58 +0500 Subject: [PATCH 1/2] feat(protocol,electron): export the pixel video as MP4 (video.mp4), the mobile-canonical name, instead of WebM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default file name of the video FileType is now video.mp4 (video..mp4 per display), and the manifest entry states format mp4, so the worker and the viewer take it as the default H.264 MP4 with no transcode. The host-supplied muxer / MediaRecorder encoder must now produce MP4. Refs #54 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- docs/design/electron.md | 10 +++++----- docs/design/per-window-replay.md | 2 +- packages/core/src/bundle-assembler.test.ts | 12 ++++++------ .../core/src/bundle-stream-assembler.test.ts | 18 +++++++++--------- .../src/display-video.integration.test.ts | 8 ++++---- packages/electron/src/electron-e2e.test.ts | 17 ++++++++++------- .../src/pixel-video-controller.test.ts | 2 +- .../electron/src/pixel-video-controller.ts | 4 ++-- packages/electron/src/video-capture.ts | 2 +- packages/node/src/launch.test.ts | 6 +++--- packages/protocol/src/constants.test.ts | 2 +- packages/protocol/src/constants.ts | 2 +- 12 files changed, 44 insertions(+), 41 deletions(-) diff --git a/docs/design/electron.md b/docs/design/electron.md index 10e538ec..443aee0d 100644 --- a/docs/design/electron.md +++ b/docs/design/electron.md @@ -151,20 +151,20 @@ backend symbolicates. App native-module symbols upload at build time via `bugsee (renderer replay is on by default, E1/E3) and needs no slice; this is the OPT-IN pixel path, built as a report-time snapshot source (the CPU-profile pattern β€” pulled at report assembly, never streamed). Landed across three packages: - **protocol**: the mobile-canonical `'video'` FileType (now also JS-emitted) + `DEFAULT_FILENAMES.video = - 'video.webm'`. + 'video.mp4'`. - **node**: `launchCore` forwards `reportSnapshots` + `fileEncoders` from the launch options into `createClient` (concatenated AFTER the internal profiling snapshot) β€” the generic extension seam for an async-produced report-time artifact + its binary encoder. - **electron**: two pluggable `VideoCaptureSource`s behind injected seams β€” `createCapturePageVideoSource` (MAIN, periodic `webContents.capturePage()` β†’ bounded frame ring β†’ injected muxer; own-window, no TCC) and - `createMediaRecorderVideoSource` (RENDERER, `MediaRecorder` over a MediaStream β†’ webm). **Per the user, the two + `createMediaRecorderVideoSource` (RENDERER, `MediaRecorder` over a MediaStream β†’ MP4, H.264). **Per the user, the two combine**: capturePage supplies frames (no Screen-Recording permission) that a canvasβ†’`MediaRecorder` encodes to - webm β€” composed into ONE source by the adapter, so `createPixelVideoController` stays source-agnostic. The + MP4 β€” composed into ONE source by the adapter, so `createPixelVideoController` stays source-agnostic. The controller is permission-gated (`hasPermission` TCC seam β†’ inert when denied) and exposes the `ReportSnapshotSource`; `encodePixelVideo` is the `video` file encoder. `launchMain` accepts `video: { source, hasPermission? }`, builds the controller, forwards its snapshot + encoder into the node launch, starts it (async permission), and stops it on - `client.stop`. The real Electron/DOM capturers + webm muxer are the app-/adapter-supplied injected seams (no - electron/DOM dependency in the package). The e2e proves an opt-in `video.webm` lands in the SAME merged bundle. + `client.stop`. The real Electron/DOM capturers + MP4 muxer are the app-/adapter-supplied injected seams (no + electron/DOM dependency in the package). The e2e proves an opt-in `video.mp4` lands in the SAME merged bundle. ## 6. Open questions / to confirm at slice time - **Minidump upload path** β€” `crashReporter.submitURL` posting directly to a Bugsee collector minidump endpoint vs diff --git a/docs/design/per-window-replay.md b/docs/design/per-window-replay.md index 53b0f11e..a334e746 100644 --- a/docs/design/per-window-replay.md +++ b/docs/design/per-window-replay.md @@ -25,7 +25,7 @@ base across windows, and a byte-cap reset restarts every recorder. The replay of | `CaptureDataEntry` | `lane?: string` copied from the stored record by the reifier. | | Export | members are grouped by `(type, lane)`. The first lane (by first record) keeps the canonical name (`replay.bin`); later lanes get `replay..bin` with a manifest entry `{type: 'replay', attrs: {window: }}`. Each member has its own stream encoder session (orphan trim and clamp are per window). A viewer that reads only `replay.bin` still gets a coherent single-window replay. | | Electron main | tags each renderer record with its window id as `lane`; `onVisualReset(lane)` restarts ONLY that window's recorder (`checkout` carries the window id; the gate paces per lane). | -| Pixel video | separate slice: main composes the windows of a display with `getBounds()` and z-order into one frame per tick; one `video` member per display (`video..webm`, attrs `{display: id}`). | +| Pixel video | separate slice: main composes the windows of a display with `getBounds()` and z-order into one frame per tick; one `video` member per display (`video..mp4`, attrs `{display: id}`). | ## Slices diff --git a/packages/core/src/bundle-assembler.test.ts b/packages/core/src/bundle-assembler.test.ts index 640380aa..ff663e0b 100644 --- a/packages/core/src/bundle-assembler.test.ts +++ b/packages/core/src/bundle-assembler.test.ts @@ -515,15 +515,15 @@ describe('assembleBundle β€” binary file encoders (replay)', () => { }); describe('assembleBundle β€” the video entry names its container', () => { - // `video.webm` is a WebM file, but a manifest entry with no attrs is read by the worker and the viewer as - // the default (an h264 MP4): the worker skipped the transcode a WebM needs and the viewer labelled it - // `video/mp4`. The format is stated on the entry, taken from the file's own extension. + // The container is stated on the manifest entry, taken from the file's own extension, rather than left to a reader default + // (an entry with no attrs is read by the worker and the viewer as an h264 MP4, and a WebM mislabelled that way skipped the + // transcode it needs). `video.mp4` is the mobile-canonical name, so the default needs no transcode. const manifestOf = (bundle: ReturnType) => JSON.parse(strFromU8(unzipSync(bundle.body)['manifest.json'] as Uint8Array)) as { files: Array<{ filename: string; type: string; attrs?: Record }>; }; - it('puts format "webm" on the video entry', () => { + it('puts format "mp4" on the video entry', () => { const map = new Map([ ['video', [entry('video', 1, new Uint8Array([1, 2, 3]))]], ]); @@ -534,8 +534,8 @@ describe('assembleBundle β€” the video entry names its container', () => { ); const video = manifestOf(bundle).files.find((f) => f.type === 'video'); - expect(video?.filename).toBe('video.webm'); - expect(video?.attrs).toEqual({ format: 'webm' }); + expect(video?.filename).toBe('video.mp4'); + expect(video?.attrs).toEqual({ format: 'mp4' }); }); it('leaves every other entry without attrs', () => { diff --git a/packages/core/src/bundle-stream-assembler.test.ts b/packages/core/src/bundle-stream-assembler.test.ts index 2b1c3f44..fe7a2500 100644 --- a/packages/core/src/bundle-stream-assembler.test.ts +++ b/packages/core/src/bundle-stream-assembler.test.ts @@ -1213,7 +1213,7 @@ describe('assembleBundleStream β€” one stream-encoded member per visual source l files: Array<{ type: string; attrs?: Record }>; }; expect(manifest.files.find((f) => f.type === 'video')?.attrs).toEqual({ - format: 'webm', + format: 'mp4', displayId: 1, }); }); @@ -1244,27 +1244,27 @@ describe('assembleBundleStream β€” one member per display for a buffered-encoder it('writes each display its own video member, encoded on its own, with attrs.displayId', async () => { const { files, manifest } = await run([video('0', [1, 2]), video('1', [3, 4])]); - expect([...(files['video.webm'] as Uint8Array)]).toEqual([1, 2]); - expect([...(files['video.1.webm'] as Uint8Array)]).toEqual([3, 4]); + expect([...(files['video.mp4'] as Uint8Array)]).toEqual([1, 2]); + expect([...(files['video.1.mp4'] as Uint8Array)]).toEqual([3, 4]); expect(manifest.files.filter((f) => f.type === 'video')).toEqual([ - expect.objectContaining({ filename: 'video.webm', attrs: { format: 'webm', displayId: 0 } }), + expect.objectContaining({ filename: 'video.mp4', attrs: { format: 'mp4', displayId: 0 } }), expect.objectContaining({ - filename: 'video.1.webm', - attrs: { format: 'webm', displayId: 1 }, + filename: 'video.1.mp4', + attrs: { format: 'mp4', displayId: 1 }, }), ]); }); it('an unkeyed video stays the one canonical member with no displayId', async () => { const { files, manifest } = await run([video(undefined, [9])]); - expect([...(files['video.webm'] as Uint8Array)]).toEqual([9]); - expect(manifest.files.find((f) => f.type === 'video')?.attrs).toEqual({ format: 'webm' }); + expect([...(files['video.mp4'] as Uint8Array)]).toEqual([9]); + expect(manifest.files.find((f) => f.type === 'video')?.attrs).toEqual({ format: 'mp4' }); }); it('a lane that is not a number is recorded as a window, not a display', async () => { const { manifest } = await run([video('win-a', [1])]); expect(manifest.files.find((f) => f.type === 'video')?.attrs).toEqual({ - format: 'webm', + format: 'mp4', window: 'win-a', }); }); diff --git a/packages/electron/src/display-video.integration.test.ts b/packages/electron/src/display-video.integration.test.ts index 709d1cd1..c99fd7a5 100644 --- a/packages/electron/src/display-video.integration.test.ts +++ b/packages/electron/src/display-video.integration.test.ts @@ -77,16 +77,16 @@ describe('a two-display session, end to end', () => { ); const files = unzipSync(await bundle.body.bytes()); // display 0 = the focused window's display (native 30, window 1); display 1 = native 70 (window 2) - expect([...(files['video.webm'] as Uint8Array)]).toEqual([0, 1]); - expect([...(files['video.1.webm'] as Uint8Array)]).toEqual([1, 2]); + expect([...(files['video.mp4'] as Uint8Array)]).toEqual([0, 1]); + expect([...(files['video.1.mp4'] as Uint8Array)]).toEqual([1, 2]); const manifest = JSON.parse(strFromU8(files['manifest.json'] as Uint8Array)) as { files: Array<{ filename: string; type: string; attrs?: Record }>; }; expect( manifest.files.filter((f) => f.type === 'video').map((f) => [f.filename, f.attrs]), ).toEqual([ - ['video.webm', { format: 'webm', displayId: 0 }], - ['video.1.webm', { format: 'webm', displayId: 1 }], + ['video.mp4', { format: 'mp4', displayId: 0 }], + ['video.1.mp4', { format: 'mp4', displayId: 1 }], ]); }); }); diff --git a/packages/electron/src/electron-e2e.test.ts b/packages/electron/src/electron-e2e.test.ts index 032264c3..d8e384da 100644 --- a/packages/electron/src/electron-e2e.test.ts +++ b/packages/electron/src/electron-e2e.test.ts @@ -199,6 +199,9 @@ async function bootRenderer( }; } +/** The start of an MP4 file: a box of size 24 and type `ftyp` (what an H.264 encoder's output begins with). */ +const MP4_FTYP = [0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70]; + describe('Electron main + 2 renderers converge into one session/bundle', () => { it('handshake, native-crash correlation, ONE merged bundle, and stop/flush propagation', async () => { const bus = fakeElectronBus(); @@ -207,13 +210,13 @@ describe('Electron main + 2 renderers converge into one session/bundle', () => { const crashStarts: CrashReporterStartOptions[] = []; const crashReporter = { start: (o: CrashReporterStartOptions) => crashStarts.push(o) }; - // A pixel-video source (fake capturer + encoder) β€” proves the opt-in path lands a video.webm. + // A pixel-video source (fake capturer + encoder) β€” proves the opt-in path lands a video.mp4. const videoSource: VideoCaptureSource = { start: () => {}, stop: () => {}, clear: () => {}, snapshot: async (now: number): Promise => [ - new CaptureDataEntryBase('video', now, new Uint8Array([0x1a, 0x45, 0xdf, 0xa3])), + new CaptureDataEntryBase('video', now, new Uint8Array(MP4_FTYP)), ], }; @@ -263,8 +266,8 @@ describe('Electron main + 2 renderers converge into one session/bundle', () => { expect.arrayContaining([{ from: 'renderer-1' }, { from: 'renderer-2' }, { from: 'main' }]), ); // The opt-in pixel video rode the report-snapshot path into the SAME bundle (binary, via the encoder). - expect('video.webm' in files).toBe(true); - expect([...(files['video.webm'] as Uint8Array)]).toEqual([0x1a, 0x45, 0xdf, 0xa3]); + expect('video.mp4' in files).toBe(true); + expect([...(files['video.mp4'] as Uint8Array)]).toEqual([...MP4_FTYP]); // ── Control propagation: flush + stop from the main reach both renderers ── await mainClient.flush(); @@ -290,7 +293,7 @@ describe('the main blackout reaches every renderer and the pixel video (privacy) stop: () => void videoCalls.push('stop'), clear: () => void videoCalls.push('clear'), snapshot: async (now: number): Promise => [ - new CaptureDataEntryBase('video', now, new Uint8Array([0x1a, 0x45, 0xdf, 0xa3])), + new CaptureDataEntryBase('video', now, new Uint8Array(MP4_FTYP)), ], }; const mainClient = launchMain('tok', { @@ -325,13 +328,13 @@ describe('the main blackout reaches every renderer and the pixel video (privacy) // A report built DURING the blackout carries no pixel video. await mainClient.logException(new Error('during blackout')); expect(puts).toHaveLength(1); - expect('video.webm' in unzipSync(puts[0] as Uint8Array)).toBe(false); + expect('video.mp4' in unzipSync(puts[0] as Uint8Array)).toBe(false); mainClient.endBlackout(); for (const r of [r1, r2, r3]) expect(r.blackout.isActive()).toBe(false); expect(videoCalls).toEqual(['start', 'stop', 'clear', 'start']); await mainClient.logException(new Error('after blackout')); - expect('video.webm' in unzipSync(puts[1] as Uint8Array)).toBe(true); + expect('video.mp4' in unzipSync(puts[1] as Uint8Array)).toBe(true); await mainClient.stop(); }); diff --git a/packages/electron/src/pixel-video-controller.test.ts b/packages/electron/src/pixel-video-controller.test.ts index e7572134..e0dcadce 100644 --- a/packages/electron/src/pixel-video-controller.test.ts +++ b/packages/electron/src/pixel-video-controller.test.ts @@ -482,7 +482,7 @@ describe('encodePixelVideo (the video fileEncoder)', () => { expect(encodePixelVideo([bytes])).toBe(bytes); }); - it('concatenates multiple payloads (defensive β€” the single video.webm contract)', () => { + it('concatenates multiple payloads (defensive β€” the single video.mp4 contract)', () => { expect(encodePixelVideo([new Uint8Array([1, 2]), new Uint8Array([3])])).toEqual( new Uint8Array([1, 2, 3]), ); diff --git a/packages/electron/src/pixel-video-controller.ts b/packages/electron/src/pixel-video-controller.ts index 3d6d3def..b39d46bc 100644 --- a/packages/electron/src/pixel-video-controller.ts +++ b/packages/electron/src/pixel-video-controller.ts @@ -1,7 +1,7 @@ // The pixel-video controller β€” wraps a VideoCaptureSource behind a permission gate and exposes the // report-time snapshot the bundle assembler pulls. The source is a single VideoCaptureSource; the // "capturePage + MediaRecorder" combination is composed INTO one source by the adapter (capturePage frames -// β†’ canvas β†’ MediaRecorder β†’ webm), so the controller stays source-agnostic. `snapshot` is the +// β†’ canvas β†’ MediaRecorder β†’ mp4), so the controller stays source-agnostic. `snapshot` is the // ReportSnapshotSource contract: given the report's wall-clock time, return the buffered `video` entries. import type { BlackoutGate, CaptureDataEntry } from '@bugsee/core'; import type { VideoCaptureSource } from './video-capture'; @@ -136,7 +136,7 @@ export function createPixelVideoController( /** * The `video` file encoder: the source already produced encoded container bytes, so the common single-entry - * case is verbatim. Multiple `video` entries (belt-and-braces β€” there is only one `video.webm`) are + * case is verbatim. Multiple `video` entries (belt-and-braces β€” there is only one `video.mp4`) are * concatenated. */ export function encodePixelVideo(payloads: readonly unknown[]): Uint8Array { diff --git a/packages/electron/src/video-capture.ts b/packages/electron/src/video-capture.ts index 0dccefd3..e6efef1c 100644 --- a/packages/electron/src/video-capture.ts +++ b/packages/electron/src/video-capture.ts @@ -41,7 +41,7 @@ export interface CapturePageVideoSourceOptions { capturePage: () => Promise; /** Periodic sampler (real: the node global-timer scheduler). */ scheduler: Scheduler; - /** Mux the frame ring into the final container bytes (real: a webm/mjpeg muxer; injected seam). */ + /** Mux the frame ring into the final container bytes (real: an MP4 (H.264) muxer; injected seam). */ encode: (frames: readonly VideoFrame[], fps: number) => Uint8Array; /** Frames per second (default 1). */ fps?: number; diff --git a/packages/node/src/launch.test.ts b/packages/node/src/launch.test.ts index 0efe9076..13503b67 100644 --- a/packages/node/src/launch.test.ts +++ b/packages/node/src/launch.test.ts @@ -2730,9 +2730,9 @@ describe('launch β€” forwarded reportSnapshots + fileEncoders (extension seam)', ); await client.logException(new Error('boom')); const files = bundleFiles(puts); - expect('video.webm' in files).toBe(true); + expect('video.mp4' in files).toBe(true); // The encoder ran over the snapshot's payloads (1 entry) β€” binary bytes, not JSON. - expect([...(files['video.webm'] as Uint8Array)]).toEqual([0xa, 1, 0xb]); + expect([...(files['video.mp4'] as Uint8Array)]).toEqual([0xa, 1, 0xb]); }); it('concatenates the internal profiling snapshot with a forwarded one (neither is dropped)', async () => { @@ -2771,7 +2771,7 @@ describe('launch β€” forwarded reportSnapshots + fileEncoders (extension seam)', await client.logException(new Error('boom')); const files = bundleFiles(puts); expect('profile.json' in files).toBe(true); // internal snapshot kept - expect('video.webm' in files).toBe(true); // forwarded snapshot kept + expect('video.mp4' in files).toBe(true); // forwarded snapshot kept }); }); diff --git a/packages/protocol/src/constants.test.ts b/packages/protocol/src/constants.test.ts index 16208776..4c5280f0 100644 --- a/packages/protocol/src/constants.test.ts +++ b/packages/protocol/src/constants.test.ts @@ -40,7 +40,7 @@ describe('wire constants', () => { }); it('video default filename is the encoded pixel-capture stream (Electron)', () => { - expect(DEFAULT_FILENAMES.video).toBe('video.webm'); + expect(DEFAULT_FILENAMES.video).toBe('video.mp4'); }); // The DEDICATED input stream (mobile-canonical `.input.json`). It exists so SDK-captured diff --git a/packages/protocol/src/constants.ts b/packages/protocol/src/constants.ts index 2e1426ef..2f9e5efb 100644 --- a/packages/protocol/src/constants.ts +++ b/packages/protocol/src/constants.ts @@ -53,7 +53,7 @@ export type FileType = export const DEFAULT_FILENAMES: Readonly, string>> = { replay: 'replay.bin', screenshot: 'screenshot.png', - video: 'video.webm', // encoded pixel-capture video β€” Electron opt-in + video: 'video.mp4', // encoded pixel-capture video (H.264 in MP4, as mobile writes it) β€” Electron opt-in // The geometry of the recording over time: the frame each `input` coordinate was measured in, and // (web) what part of it the person could see. Version 2 of the stream is one shape for every // platform. Named `video.aux` because mobile emits it From 2e2f82fb4166dd6c09575c20eb135aae329ac87d Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Tue, 3 Nov 2026 12:28:41 +0500 Subject: [PATCH 2/2] fix(electron): only MP4 leaves the SDK as video.mp4; document the contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings: the manifest labels the entry format 'mp4' from the file TYPE alone and the worker skips its transcode for an MP4, but nothing checked the bytes, so a host encode/MediaRecorder that still yields WebM (Chromium's MediaRecorder default) would ship WebM labelled MP4 and not play, with no error. - the pixel-video controller checks the ftyp box of every video entry in snapshot(): WebM (named in the message) or anything else is dropped and reported through onError; the bundle still builds, without the video; other entry types are untouched - encode / MediaRecorder seam docs say what they must produce (an H.264 encoder plus an MP4 muxer; an MP4 mimeType) - electron.md states the contract (breaking for a host that supplied WebM); backend-beta-readiness.md's 'Electron video is WebM' item is marked resolved - tests: MP4-shaped fakes; WebM / short / wrong-box / non-bytes dropped with distinct messages; mutants on the check killed πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- docs/design/backend-beta-readiness.md | 5 +- docs/design/electron.md | 5 ++ .../src/display-video.integration.test.ts | 9 ++- packages/electron/src/launch-main.test.ts | 7 +- .../src/pixel-video-controller.test.ts | 79 ++++++++++++++++++- .../electron/src/pixel-video-controller.ts | 34 +++++++- packages/electron/src/video-capture.ts | 12 ++- 7 files changed, 138 insertions(+), 13 deletions(-) diff --git a/docs/design/backend-beta-readiness.md b/docs/design/backend-beta-readiness.md index 41a2ef36..cf967a1b 100644 --- a/docs/design/backend-beta-readiness.md +++ b/docs/design/backend-beta-readiness.md @@ -205,8 +205,9 @@ Found by the same audit; each is a change in this repo: allowed-headers list; Firefox is expected to fail the preflight. - An unrecognised OS is sent as `platform.type: ''`, which the session endpoint rejects as missing (permanent); send `'unknown'`. -- Electron video is WebM, but the manifest carries no codec/format attributes, so the worker and - viewer label it `video/mp4`. +- ~~Electron video is WebM, but the manifest carries no codec/format attributes, so the worker and + viewer label it `video/mp4`.~~ Resolved: the Electron pixel video is `video.mp4` (H.264) with `format: "mp4"`, and the SDK drops + (and reports) any video entry whose bytes are not MP4, so a WebM can no longer be labelled MP4 (`docs/design/electron.md` Β§5). - A cross-origin "Script error." with no filename sends zero frames, so each occurrence is its own issue. - The performance uploader sends no `x-bugsee-internal` header and ignores `{ok:false}`. diff --git a/docs/design/electron.md b/docs/design/electron.md index 443aee0d..b69925e7 100644 --- a/docs/design/electron.md +++ b/docs/design/electron.md @@ -165,6 +165,11 @@ backend symbolicates. App native-module symbols upload at build time via `bugsee controller, forwards its snapshot + encoder into the node launch, starts it (async permission), and stops it on `client.stop`. The real Electron/DOM capturers + MP4 muxer are the app-/adapter-supplied injected seams (no electron/DOM dependency in the package). The e2e proves an opt-in `video.mp4` lands in the SAME merged bundle. + **Contract (breaking for a host that supplied WebM):** the member is `video.mp4` with `format: "mp4"` taken from the file TYPE, + and the worker skips its transcode for an MP4, so the bytes MUST be MP4 (H.264): the `encode` seam and the `MediaRecorder` `mimeType` + must produce it. The pixel-video controller checks the `ftyp` box of every video entry and DROPS one that is not MP4 (a WebM is named + in the message), reporting through `onError`; the bundle still builds, without the video. Chromium's `MediaRecorder` default is WebM, + so a host must ask for an MP4 mime type explicitly. ## 6. Open questions / to confirm at slice time - **Minidump upload path** β€” `crashReporter.submitURL` posting directly to a Bugsee collector minidump endpoint vs diff --git a/packages/electron/src/display-video.integration.test.ts b/packages/electron/src/display-video.integration.test.ts index c99fd7a5..7e30d91f 100644 --- a/packages/electron/src/display-video.integration.test.ts +++ b/packages/electron/src/display-video.integration.test.ts @@ -22,6 +22,9 @@ const win = (id: number, bounds: ReturnType, focused = false): Wind webContents: { capturePage: async () => ({ toJPEG: () => new Uint8Array([id]) }) }, }); +// A box size, `ftyp`, a brand: the smallest bytes the pixel-video controller accepts as MP4 (it drops anything else). +const MP4_HEAD = [0, 0, 0, 12, 0x66, 0x74, 0x79, 0x70, 0x69, 0x73, 0x6f, 0x6d]; + describe('a two-display session, end to end', () => { it('ships one video member per display, each holding only its own display frames, named by attrs.displayId', async () => { const windows = [win(1, rect(10, 10, 100, 100), true), win(2, rect(1100, 10, 100, 100))]; @@ -40,7 +43,7 @@ describe('a two-display session, end to end', () => { // "compose": the first layer's bytes stand for the composed frame of the display compose: async (display, layers) => new Uint8Array([display.id, ...layers.map((l) => l.bytes?.[0] as number)]), - encode: (frames) => Uint8Array.from(frames.flatMap((f) => [...f.bytes])), + encode: (frames) => Uint8Array.from([...MP4_HEAD, ...frames.flatMap((f) => [...f.bytes])]), scheduler: { setInterval: (cb) => { tick = cb; @@ -77,8 +80,8 @@ describe('a two-display session, end to end', () => { ); const files = unzipSync(await bundle.body.bytes()); // display 0 = the focused window's display (native 30, window 1); display 1 = native 70 (window 2) - expect([...(files['video.mp4'] as Uint8Array)]).toEqual([0, 1]); - expect([...(files['video.1.mp4'] as Uint8Array)]).toEqual([1, 2]); + expect([...(files['video.mp4'] as Uint8Array)]).toEqual([...MP4_HEAD, 0, 1]); + expect([...(files['video.1.mp4'] as Uint8Array)]).toEqual([...MP4_HEAD, 1, 2]); const manifest = JSON.parse(strFromU8(files['manifest.json'] as Uint8Array)) as { files: Array<{ filename: string; type: string; attrs?: Record }>; }; diff --git a/packages/electron/src/launch-main.test.ts b/packages/electron/src/launch-main.test.ts index 61f0db15..f619428e 100644 --- a/packages/electron/src/launch-main.test.ts +++ b/packages/electron/src/launch-main.test.ts @@ -524,7 +524,12 @@ describe('launchMain', () => { }, clear() {}, snapshot: vi.fn(async (now: number) => [ - new CaptureDataEntryBase('video', now, new Uint8Array([1])), + // MP4-shaped (a box size, `ftyp`, a brand): the controller drops any other container + new CaptureDataEntryBase( + 'video', + now, + new Uint8Array([0, 0, 0, 12, 0x66, 0x74, 0x79, 0x70, 1, 2, 3, 4]), + ), ]), }; const client = launchMain('tok', { diff --git a/packages/electron/src/pixel-video-controller.test.ts b/packages/electron/src/pixel-video-controller.test.ts index e0dcadce..d0d9264a 100644 --- a/packages/electron/src/pixel-video-controller.test.ts +++ b/packages/electron/src/pixel-video-controller.test.ts @@ -3,7 +3,11 @@ import { describe, expect, it, vi } from 'vitest'; import { createPixelVideoController, encodePixelVideo } from './pixel-video-controller'; import type { VideoCaptureSource } from './video-capture'; -function fakeSource(entriesData: Uint8Array[] = [new Uint8Array([1])]) { +// The smallest thing the controller accepts as MP4: a box size, then `ftyp`, then a brand. +const mp4 = (marker = 1): Uint8Array => + new Uint8Array([0, 0, 0, 12, 0x66, 0x74, 0x79, 0x70, 0x69, 0x73, 0x6f, 0x6d, marker]); + +function fakeSource(entriesData: Uint8Array[] = [mp4()]) { const source: VideoCaptureSource & { started: boolean; stopped: boolean; cleared: number } = { started: false, stopped: false, @@ -51,9 +55,76 @@ describe('createPixelVideoController β€” isActive', () => { }); }); +// Nothing downstream checks the container: the manifest names the entry `video.mp4` / `format: "mp4"` from the file TYPE, and the worker +// skips its transcode for an MP4. A host `encode` / MediaRecorder that still yields WebM must be dropped, never labelled MP4. +describe('createPixelVideoController - only MP4 leaves the SDK', () => { + const webm = new Uint8Array([ + 0x1a, 0x45, 0xdf, 0xa3, 0x9f, 0x42, 0x86, 0x81, 1, 0x42, 0xf7, 0x81, + ]); + + it('passes MP4 bytes through untouched', async () => { + const bytes = mp4(5); + const controller = createPixelVideoController({ source: fakeSource([bytes]) }); + await controller.start(); + const entries = await controller.snapshot(1); + expect(entries).toHaveLength(1); + expect(entries[0]?.data).toBe(bytes); + }); + + it('drops WebM bytes and says so through onError, naming the container', async () => { + const onError = vi.fn(); + const controller = createPixelVideoController({ source: fakeSource([webm]), onError }); + await controller.start(); + expect(await controller.snapshot(1)).toEqual([]); + expect(onError).toHaveBeenCalledTimes(1); + expect(String((onError.mock.calls[0] as [Error])[0].message)).toMatch(/WebM/); + }); + + it('drops anything else that is not MP4 (too short, wrong box, not bytes), and keeps the MP4 beside it', async () => { + const onError = vi.fn(); + const good = mp4(3); + const source = fakeSource([new Uint8Array([1]), good]); + const original = source.snapshot; + source.snapshot = vi.fn(async (now: number) => [ + ...(await original(now)), + new CaptureDataEntryBase('video', now, 'not bytes'), + new CaptureDataEntryBase('video', now, new Uint8Array(16)), + ]); + const controller = createPixelVideoController({ source, onError }); + await controller.start(); + const entries = await controller.snapshot(1); + expect(entries.map((e) => e.data)).toEqual([good]); + expect(onError).toHaveBeenCalledTimes(3); + const reasons = onError.mock.calls.map(([error]) => (error as Error).message); + expect(reasons[0]).toMatch(/not MP4 \(no ftyp box\)/); // one byte + expect(reasons[1]).toMatch(/not bytes/); // a string payload + expect(reasons[2]).toMatch(/not MP4 \(no ftyp box\)/); // 16 zero bytes: not WebM either + }); + + it('leaves entries of other file types alone', async () => { + const source = fakeSource([]); + const other = new CaptureDataEntryBase('console' as never, 1, { not: 'bytes' }); + source.snapshot = vi.fn(async () => [other]); + const controller = createPixelVideoController({ source }); + await controller.start(); + expect(await controller.snapshot(1)).toEqual([other]); + }); + + it('still never throws into the report when onError itself throws', async () => { + const controller = createPixelVideoController({ + source: fakeSource([webm]), + onError: () => { + throw new Error('sink failed'); + }, + }); + await controller.start(); + await expect(controller.snapshot(1)).resolves.toEqual([]); + }); +}); + describe('createPixelVideoController', () => { it('starts the source and pulls its entries on snapshot when permission is granted', async () => { - const source = fakeSource([new Uint8Array([0x9])]); + const source = fakeSource([mp4(9)]); const controller = createPixelVideoController({ source, hasPermission: () => true }); await controller.start(); expect(source.started).toBe(true); @@ -242,7 +313,7 @@ describe('createPixelVideoController - blackout (privacy: fail closed)', () => { source.snapshot = vi.fn( () => new Promise((resolve) => { - release = () => resolve([new CaptureDataEntryBase('video', 1, new Uint8Array([7]))]); + release = () => resolve([new CaptureDataEntryBase('video', 1, mp4(7))]); }), ); const controller = createPixelVideoController({ source, blackout: gate }); @@ -420,7 +491,7 @@ describe('createPixelVideoController - blackout (privacy: fail closed)', () => { source.snapshot = vi.fn( () => new Promise((resolve) => { - release = () => resolve([new CaptureDataEntryBase('video', 1, new Uint8Array([7]))]); + release = () => resolve([new CaptureDataEntryBase('video', 1, mp4(7))]); }), ); const controller = createPixelVideoController({ source, blackout: gate }); diff --git a/packages/electron/src/pixel-video-controller.ts b/packages/electron/src/pixel-video-controller.ts index b39d46bc..96a64a8d 100644 --- a/packages/electron/src/pixel-video-controller.ts +++ b/packages/electron/src/pixel-video-controller.ts @@ -35,6 +35,29 @@ export interface PixelVideoController { snapshot(now: number): Promise; } +// `ftyp` at bytes 4..8 is the ISO base-media (MP4) signature; WebM/Matroska starts with the EBML magic 1A 45 DF A3. +const FTYP = [0x66, 0x74, 0x79, 0x70]; +const EBML = [0x1a, 0x45, 0xdf, 0xa3]; +const startsWith = (bytes: Uint8Array, at: number, magic: readonly number[]): boolean => + bytes.length >= at + magic.length && magic.every((byte, i) => bytes[at + i] === byte); + +/** + * Why `data` cannot go into the bundle as `video.mp4`, or `undefined` when it is MP4. The manifest names the entry `format: "mp4"` + * from the file TYPE alone and the worker skips its transcode for an MP4, so bytes that are really WebM must not get through: they + * would be served as `video/mp4` and not play, with no error anywhere. + */ +function notMp4(data: unknown): string | undefined { + if (!(data instanceof Uint8Array)) { + return 'the payload is not bytes'; + } + if (startsWith(data, 4, FTYP)) { + return undefined; + } + return startsWith(data, 0, EBML) + ? 'the container is WebM' + : 'the container is not MP4 (no ftyp box)'; +} + /** Build the permission-gated pixel-video controller over a single capture source. */ export function createPixelVideoController( options: PixelVideoControllerOptions, @@ -123,7 +146,16 @@ export function createPixelVideoController( } const epoch = blackoutEpoch; try { - const entries = await options.source.snapshot(now); + const produced = await options.source.snapshot(now); + // Only MP4 leaves: a `video` entry that is anything else (a host `encode` or MediaRecorder that still yields WebM) is dropped and + // reported, so the bundle still builds and is never mislabelled. Checked here, not in the encoder, which the assembler does not guard. + const entries = produced.filter((entry) => { + const why = entry.type === 'video' ? notMp4(entry.data) : undefined; + if (why !== undefined) { + report(new Error(`pixel video dropped: ${why}; the SDK ships video.mp4 (H.264) only`)); + } + return why === undefined; + }); // A blackout began while the source was encoding: its result may hold pre-blackout frames. return epoch === blackoutEpoch && options.blackout?.isActive() !== true ? entries : []; } catch (error) { diff --git a/packages/electron/src/video-capture.ts b/packages/electron/src/video-capture.ts index e6efef1c..1101d28d 100644 --- a/packages/electron/src/video-capture.ts +++ b/packages/electron/src/video-capture.ts @@ -41,7 +41,11 @@ export interface CapturePageVideoSourceOptions { capturePage: () => Promise; /** Periodic sampler (real: the node global-timer scheduler). */ scheduler: Scheduler; - /** Mux the frame ring into the final container bytes (real: an MP4 (H.264) muxer; injected seam). */ + /** + * Encode the JPEG frame ring into MP4 (H.264) container bytes (real: an H.264 encoder PLUS an MP4 muxer; a muxer alone does not make + * H.264 out of JPEG frames; injected seam). Anything that is not MP4 (a WebM or Motion-JPEG muxer) is dropped and reported by the + * pixel-video controller, never shipped as `video.mp4`. + */ encode: (frames: readonly VideoFrame[], fps: number) => Uint8Array; /** Frames per second (default 1). */ fps?: number; @@ -119,7 +123,11 @@ export interface MediaRecorderLike { } export interface MediaRecorderVideoSourceOptions { - /** Build a `MediaRecorder` over the capture stream (real: `() => new MediaRecorder(stream, { mimeType })`). */ + /** + * Build a `MediaRecorder` over the capture stream (real: `() => new MediaRecorder(stream, { mimeType })`). The `mimeType` MUST ask for + * MP4 (for example `video/mp4;codecs=avc1`; Chromium's default, with none given, is WebM). Output that is not MP4 is dropped and + * reported by the pixel-video controller, never shipped as `video.mp4`. + */ recorderFactory: () => MediaRecorderLike; /** Concatenate the recorded chunks into the final bytes (real: join Blob `arrayBuffer()`s). */ toBytes: (chunks: unknown[]) => Uint8Array | Promise;