feat(selenium-devtools-py): record a dense filmstrip into the trace

packages/backend/src/baseline/types.ts

@@ -3,6 +3,7 @@ import type {
   ConsoleLog,
   Metadata,
   NetworkRequest,
+  ScreencastFrame,
   TestError,
   TestStatus,
   TraceMutation
@@ -54,4 +55,9 @@ export interface ActiveRun {
   /** Raw `logs` frames, the trace's transcript source. Only the JS adapters
    *  send these, so this is routinely empty. */
   traceLogs: string[]
+  /** Dense screencast frames for the trace filmstrip. The JS adapters hand
+   *  their recorder's buffer straight to the exporter in-process; an adapter
+   *  that exports through here has to send them, so they accumulate like any
+   *  other stream. Empty unless the adapter asked for a filmstrip. */
+  screencastFrames: ScreencastFrame[]
 }

packages/backend/src/baseline/utils.ts

@@ -9,7 +9,8 @@ export function freshRun(): ActiveRun {
     sources: {},
     nodes: new Map(),
     startedAt: Date.now(),
-    traceLogs: []
+    traceLogs: [],
+    screencastFrames: []
   }
 }
 

packages/backend/src/baselineStore.ts

@@ -82,6 +82,11 @@ class BaselineStore {
       case 'logs':
         appendArray(this.#activeRun.traceLogs, data)
         return
+      case 'screencastFrames':
+        // Sent in batches: a run's buffer can reach the recorder's cap, and one
+        // message carrying all of it would sit near the socket's payload limit.
+        appendArray(this.#activeRun.screencastFrames, data)
+        return
       case 'suites':
         this.#ingestSuites(data)
         return

packages/backend/src/trace-export.ts

@@ -82,6 +82,12 @@ export async function exportActiveRunTrace(
     sessionId: request.sessionId,
     ...(request.format ? { format: request.format } : {}),
     ...(request.fileStem ? { fileStem: request.fileStem } : {}),
+    // Omitted when empty rather than passed as []: the exporter treats absence
+    // as "no dense filmstrip" and keeps the sparse per-action one, which is
+    // what an adapter that did not ask for frames should still get.
+    ...(run.screencastFrames.length
+      ? { screencastFrames: run.screencastFrames }
+      : {}),
     testMetadata: testMetadataFromNodes(run.nodes)
   })
 }

packages/backend/tests/trace-export.test.ts

@@ -26,6 +26,13 @@ import {
 import { freshRun } from '../src/baseline/utils.js'
 import type { ActiveRun, TimeWindowNode } from '../src/baseline/types.js'
 
+/** Smallest valid JPEG — the exporter content-addresses frame bytes, so they
+ *  have to be real image data rather than a placeholder string. */
+const JPEG_1PX =
+  '/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0a' +
+  'HBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAABAAEBAREA/8QAFAABAAAAAAAA' +
+  'AAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q=='
+
 const dirs: string[] = []
 
 afterEach(async () => {
@@ -248,6 +255,43 @@ describe('exportActiveRunTrace', () => {
     expect(JSON.stringify(options[0])).toContain('chrome')
   })
 
+  // The JS adapters hand their recorder's buffer straight to the exporter
+  // in-process; an adapter exporting through the backend has to send it, so the
+  // frames arrive as a stream and have to survive the round trip into the zip.
+  it('writes the dense filmstrip when frames were streamed', async () => {
+    const outputDir = await tmpDir()
+    const zipPath = await exportActiveRunTrace(
+      run({
+        screencastFrames: [
+          { data: JPEG_1PX, timestamp: 1100 },
+          { data: JPEG_1PX, timestamp: 1200 }
+        ]
+      }),
+      { outputDir, sessionId: 'sess-film' }
+    )
+
+    const events = await traceEvents(zipPath)
+    const frames = events.filter(
+      (e) => e.type === TRACE_EVENT_TYPES.screencastFrame
+    )
+    expect(frames.length).toBeGreaterThan(0)
+  })
+
+  // Absence means "no dense filmstrip", which keeps the sparse per-action one.
+  // Passing an empty array instead would be a different thing to the exporter.
+  it('writes no screencast-frame events when none were streamed', async () => {
+    const outputDir = await tmpDir()
+    const zipPath = await exportActiveRunTrace(run(), {
+      outputDir,
+      sessionId: 'sess-nofilm'
+    })
+
+    const frames = (await traceEvents(zipPath)).filter(
+      (e) => e.type === TRACE_EVENT_TYPES.screencastFrame
+    )
+    expect(frames).toEqual([])
+  })
+
   it('honours fileStem so a per-test slice can name its own artifact', async () => {
     const outputDir = await tmpDir()
     const zipPath = await exportActiveRunTrace(run(), {

packages/selenium-devtools-py/scripts/gen_contract.py

@@ -42,6 +42,9 @@
     "SCOPE_SCREENCAST": "screencast",
     "SCOPE_SOURCES": "sources",
     "SCOPE_MUTATIONS": "mutations",
+    # The dense filmstrip. The JS adapters hand their recorder's buffer to the
+    # exporter in-process; an adapter exporting through the backend sends it.
+    "SCOPE_SCREENCAST_FRAMES": "screencastFrames",
 }
 
 

packages/selenium-devtools-py/src/selenium_devtools/__init__.py

@@ -30,6 +30,7 @@
     DEFAULT_HOST,
     DEFAULT_PORT,
     ENV_HOST,
+    ENV_FILMSTRIP,
     ENV_PORT,
     ENV_TRACE,
     LOGGER_NAME,
@@ -75,10 +76,22 @@ def _restore_excepthook() -> None:
 _active: dict = {
     "capturer": None, "transport": None, "process": None, "url": None,
     "handle": None, "terminal": None, "logs": None, "excepthook": None,
-    "trace": False, "traced": False,
+    "trace": False, "traced": False, "filmstrip_mark": None,
 }
 
 
+def _filmstrip_enabled(filmstrip: Optional[bool]) -> bool:
+    """Whether trace mode records a dense filmstrip. Default ON, as in the JS
+    adapters (`BaseDevToolsOptions.filmstrip`), opt-out only — the argument
+    wins, then the environment."""
+    if filmstrip is not None:
+        return filmstrip
+    value = os.environ.get(ENV_FILMSTRIP)
+    if value is None:
+        return True
+    return value.lower() not in ("0", "false", "no", "off", "")
+
+
 def _trace_enabled(trace: Optional[bool]) -> bool:
     """Whether this run writes a trace archive. The argument wins over the
     environment so a script can opt out of an exported default."""
@@ -148,6 +161,40 @@ def _export_trace(
         session_id = (
             getattr(capturer, "session_id", None) or resolve_run_id()
         )
+        # Before the request, not with it: the buffer holds up to a couple of
+        # thousand JPEGs, and the backend accumulates them like any other
+        # stream.
+        #
+        # Only what has not gone out already. A failed export is retried at
+        # teardown, and the backend APPENDS these — it has no key to replace or
+        # dedupe on — so resending the buffer would put every frame in the trace
+        # twice.
+        #
+        # Keyed on the newest timestamp sent, NOT on how many were sent. A live
+        # recorder decimates its bounded buffer in place
+        # (`screencast._decimate` halves it, keeping the ends), so between two
+        # attempts the list can SHRINK and every index shift — an offset would
+        # then skip frames it never sent. Decimation drops frames but never
+        # renumbers the survivors, and timestamps are monotonic per recorder, so
+        # the watermark stays meaningful however the buffer is rewritten.
+        # None, not 0: "nothing sent yet" is not a timestamp, and a frame
+        # stamped 0 would be filtered out by one.
+        #
+        # The boundary is INCLUSIVE, so a retry may resend the frame the last
+        # attempt ended on. Timestamps are milliseconds and two frames can share
+        # one, and an exclusive boundary would drop the unsent twin — a gap in
+        # the filmstrip. A resend costs nothing much instead: the exporter
+        # content-addresses frame bytes, so the duplicate shares one resource.
+        # Losing a frame beats duplicating one only if you never look at it.
+        mark = _active["filmstrip_mark"]
+        pending = [
+            f
+            for f in instrumentation.screencast_frames()
+            if mark is None or f.get("timestamp", 0) >= mark
+        ]
+        sent = trace_export.send_frames(_active["transport"], pending)
+        if sent:
+            _active["filmstrip_mark"] = pending[sent - 1].get("timestamp", mark)
         return trace_export.export(
             _active["transport"],
             output_dir=output_dir or resolve_adapter_output_dir(),
@@ -164,6 +211,7 @@ def enable(
     *,
     webdriver_cls: Optional[type] = None,
     trace: Optional[bool] = None,
+    filmstrip: Optional[bool] = None,
 ) -> Optional[SessionCapturer]:
     """Connect to the backend and instrument Selenium. Idempotent.
 
@@ -178,6 +226,7 @@ def enable(
     # Decided before anything reads it: the screencast recorder, the dashboard
     # window and the teardown export all branch on this.
     trace_mode = _trace_enabled(trace)
+    filmstrip_mode = trace_mode and _filmstrip_enabled(filmstrip)
 
     # Before the backend is launched: the directory a rerun spawns in travels
     # through the environment the backend process inherits. A framework plugin
@@ -212,7 +261,9 @@ def enable(
         return None
 
     capturer = SessionCapturer(transport)
-    instrumentation.install(capturer, webdriver_cls, trace=trace_mode)
+    instrumentation.install(
+        capturer, webdriver_cls, trace=trace_mode, filmstrip=filmstrip_mode
+    )
     # Plain scripts only: a framework plugin calls
     # `set_external_suites`, which turns this back off.
     instrumentation.start_assertion_tracing(capturer)
@@ -226,7 +277,7 @@ def enable(
     url = f"http://{host}:{port}"
     _active.update(
         capturer=capturer, transport=transport, process=process, url=url,
-        terminal=term, logs=logs, trace=trace_mode, traced=False,
+        terminal=term, logs=logs, trace=trace_mode, traced=False, filmstrip_mark=None,
     )
 
     # Open the dashboard window and wire exit/signal + control-frame teardown so
@@ -289,7 +340,7 @@ def disable() -> None:
     trace_export.reset()
     _active.update(
         capturer=None, transport=None, process=None, url=None, handle=None,
-        terminal=None, logs=None, excepthook=None, trace=False, traced=False,
+        terminal=None, logs=None, excepthook=None, trace=False, traced=False, filmstrip_mark=None,
     )
 
 

packages/selenium-devtools-py/src/selenium_devtools/_contract.py

@@ -10,6 +10,7 @@
 SCOPE_SCREENCAST = "screencast"
 SCOPE_SOURCES = "sources"
 SCOPE_MUTATIONS = "mutations"
+SCOPE_SCREENCAST_FRAMES = "screencastFrames"
 SCOPE_REPLACE_COMMAND = "replaceCommand"
 
 DATA_SCOPES = frozenset(['actionSnapshots', 'commands', 'config', 'consoleLogs', 'logs', 'metadata', 'mutations', 'networkRequests', 'screencast', 'screencastFrames', 'sources', 'suites'])

packages/selenium-devtools-py/src/selenium_devtools/constants.py

@@ -85,6 +85,16 @@
 #: Opt in to writing a trace archive at the end of the run.
 ENV_TRACE = "DEVTOOLS_TRACE"
 
+#: Opt OUT of the dense filmstrip in trace mode (on by default, as in the JS
+#: adapters). Falsy values disable it.
+ENV_FILMSTRIP = "DEVTOOLS_FILMSTRIP"
+
+#: Filmstrip frames per websocket message. The buffer can hold
+#: SCREENCAST_MAX_BUFFER_FRAMES JPEGs, which in one message would approach the
+#: socket's payload limit; the transport also masks payloads in a per-byte
+#: Python loop (~57 MB/s measured), so smaller messages keep the stall short.
+SCREENCAST_FRAME_BATCH = 50
+
 MIN_NODE_MAJOR = 18
 NODE_VERSION_TIMEOUT_S = 5.0
 

packages/selenium-devtools-py/src/selenium_devtools/instrumentation.py

@@ -246,10 +246,15 @@ def _capture_source(capturer: SessionCapturer, call_src: Optional[str]) -> None:
     # Set by enable()'s excepthook when an exception reaches top level. The
     # synthetic suite's final state reads this rather than assuming success.
     "run_failed": False,
-    # Trace mode. The archive carries per-command screenshots, not the
-    # screencast — `screencastFrames` does not cross the wire yet — so
-    # recording one writes a .webm nothing reads. Set by install().
+    # Trace mode. Set by install().
     "trace": False,
+    # Record a dense filmstrip into the trace. Only meaningful in trace mode;
+    # the recorder runs in live mode regardless, for the dashboard video.
+    "filmstrip": False,
+    # Frames kept from every session's recorder as it finalizes, because that
+    # is the only moment they are reachable. Run-scoped: a run may replace its
+    # driver, and the filmstrip is the whole run.
+    "filmstrip_frames": [],
 }
 
 
@@ -325,6 +330,34 @@ def _attach_performance(
         _log.debug("could not replace the navigation row: %s", exc)
 
 
+def screencast_frames() -> list:
+    """Every frame this run's recorders buffered, in time order.
+
+    Two sources, because a frame is only reachable from one of them at a time:
+
+    * sessions that already quit — kept as they finalized, since
+      `_finalize_screencast` pops the recorder off the entry and `sessions` is
+      keyed weakly by a driver about to be collected;
+    * sessions still live — read straight off their recorder, because an export
+      can run before the last driver quits, and `uninstall` then stops those
+      recorders without keeping anything.
+
+    Sorted rather than concatenated: sessions are appended in quit order, and a
+    replaced session can finish after a later one started.
+    """
+    frames: list = list(_state["filmstrip_frames"])
+    for entry in list(_state.get("sessions", {}).values()):
+        recorder = entry.get("screencast")
+        if recorder is None:
+            continue
+        try:
+            frames.extend(recorder.frames)
+        except Exception as exc:  # noqa: BLE001 — a poorer filmstrip, not a failed run
+            _log.debug("could not read a live recorder's frames: %s", exc)
+    frames.sort(key=lambda f: f.get("timestamp", 0))
+    return frames
+
+
 def resolved_output_dir() -> Optional[str]:
     """The ``test-results`` dir this run resolved from its first test file, or
     None if no command carried a user call source. Screencast videos already
@@ -458,6 +491,21 @@ def _finalize_screencast(
     recorder = entry.pop("screencast", None)
     if recorder is None:
         return
+    # Take the buffer BEFORE anything else. This is the last moment it exists:
+    # the entry drops its recorder on the line above, and `sessions` is keyed
+    # weakly by a driver that quit and is about to be collected — so reading it
+    # at export time, as this first did, always found nothing.
+    if _state["filmstrip"]:
+        try:
+            _state["filmstrip_frames"].extend(recorder.frames)
+        except Exception as exc:  # noqa: BLE001 — a poorer filmstrip, not a failed run
+            _log.debug("could not keep the filmstrip frames: %s", exc)
+    if _state["trace"]:
+        # No video in trace mode: the frames ARE the filmstrip, and a .webm is
+        # a live-dashboard artifact with no dashboard to play it. Stopping
+        # rather than finalizing is what skips the encode.
+        recorder.stop()
+        return
     try:
         info = recorder.finalize(session_id, output_dir=_state.get("output_dir"))
     except Exception as exc:  # noqa: BLE001
@@ -517,9 +565,11 @@ def _ensure_session_setup(driver: Any, capturer: SessionCapturer) -> Optional[di
     except Exception as exc:  # noqa: BLE001 — capture must never break the test
         _log.warning("BiDi attach threw: %s", exc)
     try:
-        if _state["trace"]:
-            # The archive's frames are the per-command screenshots; the video is
-            # a live-dashboard artifact, and trace mode opens no dashboard.
+        if _state["trace"] and not _state["filmstrip"]:
+            # With no filmstrip the archive's frames are the per-command
+            # screenshots alone, and the video is a live-dashboard artifact —
+            # trace mode opens no dashboard, so a recorder here writes a .webm
+            # nothing reads.
             raise _SkipScreencast
         recorder = ScreencastRecorder()
         recorder.start(driver)
@@ -727,6 +777,7 @@ def install(
     webdriver_cls: Optional[type] = None,
     *,
     trace: bool = False,
+    filmstrip: bool = False,
 ) -> None:
     if _state["installed"]:
         return
@@ -807,7 +858,7 @@ def patched_execute(self, driver_command: str, params: Any = None):  # noqa: ANN
     _state.update(
         installed=True, cls=webdriver_cls, orig=orig_execute,
         sessions=weakref.WeakKeyDictionary(), output_dir=None, default_suite=None,
-        trace=trace,
+        trace=trace, filmstrip=filmstrip, filmstrip_frames=[],
     )
 
 
@@ -820,11 +871,20 @@ def uninstall() -> None:
     # per-run state.
     reset_collector_cache()
     # Never leave a recorder running past teardown, for any session still live.
+    # Keep the buffer first: `disable()` uninstalls BEFORE its fallback export,
+    # and `sessions` is replaced below, so a session that never quit would
+    # otherwise have its frames dropped here — the plain-script path exactly.
     for entry in list(_state.get("sessions", {}).values()):
         _stop_push_screencast(entry)
         recorder = entry.get("screencast")
-        if recorder is not None:
-            recorder.stop()
+        if recorder is None:
+            continue
+        if _state["filmstrip"]:
+            try:
+                _state["filmstrip_frames"].extend(recorder.frames)
+            except Exception as exc:  # noqa: BLE001 — a poorer filmstrip, not a failed run
+                _log.debug("could not keep a live recorder's frames: %s", exc)
+        recorder.stop()
     if not _state["installed"]:
         _state.update(
             sessions=weakref.WeakKeyDictionary(),