feat(app-a11y): scan the WDIO config-level before() window, and report it as a hook

.changeset/pr-168.md

@@ -0,0 +1,6 @@
+---
+"@wdio/browserstack-service": minor
+---
+
+- Accessibility scans now run for driver commands issued from your WDIO config's `before()` hook, so screens visited during setup are covered.
+- Hooks in your WDIO config that touch the driver now appear on the build, including when they fail — previously a failing `before()` was reported nowhere.

packages/browserstack-service/src/accessibility-handler.ts

@@ -75,7 +75,9 @@ import accessibilityScripts from './scripts/accessibility-scripts.js'
 import PerformanceTester from './instrumentation/performance/performance-tester.js'
 import * as PERFORMANCE_SDK_EVENTS from './instrumentation/performance/constants.js'
 
+import { PRE_TEST_HOOK_TITLE_FALLBACK, PRE_TEST_HOOK_TITLE_PREFIX, PRE_TEST_SCAN_FRAMEWORKS } from './constants.js'
 import { BStackLogger } from './bstackLogger.js'
+import { getActiveHookName, getPreTestWindowFailure } from './hookInstrumentation.js'
 
 class _AccessibilityHandler {
     /**
@@ -93,6 +95,17 @@ class _AccessibilityHandler {
     private _config: Options.Testrunner
     private _accessibilityOptions?: AccessibilityOptions
     private _autoScanning: boolean = true
+    // Reports the pre-test window as a hook run. Injected by the service, and only for the
+    // frameworks whose Direct-flow hook reporting supports it, so this handler stays unaware of
+    // which one it is serving.
+    private _preTestHookReporter?: {
+        open: (name: string) => Promise<string | undefined>,
+        close: (uuid: string, passed: boolean, message?: string) => Promise<void>
+    }
+    private _preTestHookState: 'idle' | 'attempted' | 'open' | 'closed' = 'idle'
+    // True from before() until the first test/scenario — see preTestWindowActive on the CLI module.
+    private _preTestWindowActive = false
+    private _preTestHookUuid?: string
     private _testIdentifier: string | null = null
     private _testMetadata: TestMetadata = {}
     /* Set while a supported hook is executing; scans fired in this window are stamped with it. */
@@ -188,6 +201,60 @@ class _AccessibilityHandler {
         }
     }
 
+    /** mocha and cucumber only — see PRE_TEST_SCAN_FRAMEWORKS. */
+    private supportsPreTestWindow(): boolean {
+        return PRE_TEST_SCAN_FRAMEWORKS.includes(this._framework as typeof PRE_TEST_SCAN_FRAMEWORKS[number]) &&
+            !this._browser?.isMultiremote
+    }
+
+    setPreTestHookReporter(reporter: {
+        open: (name: string) => Promise<string | undefined>,
+        close: (uuid: string, passed: boolean, message?: string) => Promise<void>
+    }) {
+        this._preTestHookReporter = reporter
+    }
+
+    /** On demand from the scan path, so a suite with no config-level hook reports nothing extra. */
+    private async ensurePreTestHookRun() {
+        if (this._preTestHookState !== 'idle' || !this._preTestHookReporter || !this.supportsPreTestWindow()) {
+            return
+        }
+        this._preTestHookState = 'attempted'
+        try {
+            const hook = getActiveHookName()
+            const name = hook ? `${PRE_TEST_HOOK_TITLE_PREFIX} "${hook}" hook` : PRE_TEST_HOOK_TITLE_FALLBACK
+            this._preTestHookUuid = await this._preTestHookReporter.open(name)
+            if (this._preTestHookUuid) {
+                this._preTestHookState = 'open'
+                // What actually reaches the scan: commandWrapper passes this to performA11yScan as
+                // thHookRunUuid. Without it the hook run exists in TRA and the scans in its window
+                // still arrive with no parent, which is the join this whole thing is for.
+                this._currentHookRunUuid = this._preTestHookUuid
+            }
+        } catch (error) {
+            BStackLogger.debug(`Could not open a hook run for the pre-test window: ${error}`)
+        }
+    }
+
+    private async closePreTestHookRun() {
+        // cleared unconditionally: the window is over whether or not a hook run was ever opened,
+        // and leaving it set would strip the test run uuid from every later test-body scan
+        this._preTestWindowActive = false
+        if (this._preTestHookState !== 'open' || !this._preTestHookUuid || !this._preTestHookReporter) {
+            this._preTestHookState = 'closed'
+            return
+        }
+        this._preTestHookState = 'closed'
+        try {
+            const failure = getPreTestWindowFailure()
+            // cleared before the close so a test-body scan is never stamped as a hook scan
+            this._currentHookRunUuid = null
+            await this._preTestHookReporter.close(this._preTestHookUuid, !failure, failure)
+        } catch (error) {
+            BStackLogger.debug(`Could not close the hook run for the pre-test window: ${error}`)
+        }
+    }
+
     async before(sessionId: string) {
         PerformanceTester.start(PERFORMANCE_SDK_EVENTS.CONFIG_EVENTS.ACCESSIBILITY)
 
@@ -282,6 +349,20 @@ class _AccessibilityHandler {
         if (!this._accessibility) {
             return
         }
+
+        // Gate for the window between driver creation and the first test/scenario. WDIO's
+        // config-level hooks run there and are no test-framework hooks, so nothing else registers
+        // the session and their commands went unscanned. beforeTest/beforeScenario/beforeHook each
+        // recompute it. Same fix as the CLI flow's onBeforeExecute.
+        //
+        // Scoped to mocha and cucumber. Jasmine is in the per-test path too, so opening this for
+        // it would add scans in a window it never scanned — a behaviour change on a framework
+        // where App-A11y is not supported. Multiremote likewise.
+        if (this._autoScanning && this.supportsPreTestWindow()) {
+            AccessibilityHandler._a11yScanSessionMap[sessionId] = true
+            this._preTestWindowActive = true
+        }
+
         if (!('overwriteCommand' in this._browser && Array.isArray(accessibilityScripts.commandsToWrap))) {
             return
         }
@@ -305,6 +386,7 @@ class _AccessibilityHandler {
     }
 
     async beforeTest (suiteTitle: string | undefined, test: Frameworks.Test) {
+        await this.closePreTestHookRun()
         try {
             if (
                 !AccessibilityHandler.TEST_HOOK_FRAMEWORKS.includes(this._framework as string) ||
@@ -385,6 +467,7 @@ class _AccessibilityHandler {
       * Cucumber Only
     */
     async beforeScenario (world: ITestCaseHookParameter) {
+        await this.closePreTestHookRun()
         const pickleData = world.pickle
         const gherkinDocument = world.gherkinDocument
         const featureData = gherkinDocument.feature
@@ -504,6 +587,7 @@ class _AccessibilityHandler {
 
     private async commandWrapper (command: CommandInfo, prevImpl: Function, origFunction: Function, ...args: unknown[]) {
         const skipScanForBidiWindowCommand = AccessibilityHandler.shouldSkipScanForBidiWindowCommand(this._browser, command)
+        await this.ensurePreTestHookRun()
         if (
             this._sessionId && AccessibilityHandler._a11yScanSessionMap[this._sessionId] &&
                 !skipScanForBidiWindowCommand &&
@@ -513,7 +597,7 @@ class _AccessibilityHandler {
                 )
         ) {
             BStackLogger.debug(`Performing scan for ${command.class} ${command.name}`)
-            await performA11yScan(this.isAppAutomate, this._browser, true, true, command.name, undefined, this._currentHookRunUuid)
+            await performA11yScan(this.isAppAutomate, this._browser, true, true, command.name, undefined, this._currentHookRunUuid, this._preTestWindowActive)
         } else if (skipScanForBidiWindowCommand) {
             BStackLogger.debug(`SDK-5047: skipping accessibility scan for BiDi window/context command '${command.name}' to avoid racing the WebdriverIO ContextManager during session-start window churn`)
         }

packages/browserstack-service/src/cli/modules/accessibilityModule.ts

@@ -12,6 +12,8 @@ import type { Command } from '../../scripts/accessibility-scripts.js'
 import accessibilityScripts from '../../scripts/accessibility-scripts.js'
 import { _getParamsForAppAccessibility, formatString, getAppA11yResults, getAppA11yResultsSummary, shouldScanTestForAccessibility, validateCapsWithA11y, validateCapsWithAppA11y, isBrowserstackSession } from '../../util.js'
 import { AutomationFrameworkConstants } from '../frameworks/constants/automationFrameworkConstants.js'
+import { BROWSERSTACK_WDIO_CONFIG_FILE_PATH, PRE_TEST_HOOK_TITLE_FALLBACK, PRE_TEST_HOOK_TITLE_PREFIX } from '../../constants.js'
+import { getActiveHookName, getPreTestWindowFailure } from '../../hookInstrumentation.js'
 import util from 'node:util'
 import type { Accessibility } from '../../grpc/index.js'
 import PerformanceTester from '../../instrumentation/performance/performance-tester.js'
@@ -39,6 +41,16 @@ export default class AccessibilityModule extends BaseModule {
     // Any scan fired while this is set is stamped with it as thHookRunUuid so the backend
     // (SeleniumHub appAllyScan → hook_run_uuid) can reconcile the scan onto the wrapping test.
     currentHookRunUuid: string | null = null
+    // Hook run covering scans fired before any test exists (WDIO's config-level hooks, which get
+    // no hook run of their own). 'attempted' is distinct from 'open' so a failed PRE is neither
+    // retried per command nor closed by a POST that would pair with nothing.
+    private preTestHookState: 'idle' | 'attempted' | 'open' | 'closed' = 'idle'
+    // True from driver creation until the first test. Scans fired in that window come from WDIO's
+    // config-level hooks, which belong to no test, so they must not carry a test run uuid.
+    private preTestWindowActive = false
+    // Captured at PRE: the binary names a hook from the record pushed there, and by POST a
+    // different hook is running.
+    private preTestHookLabel: string | undefined
 
     constructor(accessibilityConfig: Accessibility, isNonBstackA11y: boolean) {
         super()
@@ -102,6 +114,61 @@ export default class AccessibilityModule extends BaseModule {
         this.currentHookRunUuid = null
     }
 
+    /**
+     * Open a BEFORE_ALL hook run for the pre-test window, on demand from the scan path — so a
+     * suite with no config-level hook reports nothing extra. The framework stamps the uuid,
+     * onHookStart picks it up onto the scan, and the binary emits HookRunStarted for it.
+     */
+    private async ensurePreTestHookRun() {
+        if (this.preTestHookState !== 'idle' || this.currentHookRunUuid !== null) {
+            return
+        }
+        // marked before awaiting: concurrent commands must not each open their own hook run
+        this.preTestHookState = 'attempted'
+        try {
+            this.preTestHookLabel = getActiveHookName()
+            const framework = BrowserstackCLI.getInstance()?.getTestFramework()
+            if (!framework) {
+                return
+            }
+            await framework.trackEvent(TestFrameworkState.BEFORE_ALL, HookState.PRE, this.preTestHookArgs())
+            this.preTestHookState = 'open'
+        } catch (error) {
+            this.logger.debug(`Could not open a hook run for the pre-test window: ${error}`)
+        }
+    }
+
+    private async closePreTestHookRun() {
+        if (this.preTestHookState !== 'open') {
+            this.preTestHookState = 'closed'
+            return
+        }
+        this.preTestHookState = 'closed'
+        try {
+            const framework = BrowserstackCLI.getInstance()?.getTestFramework()
+            // WDIO swallows a throwing config hook, so nothing else marks it failed; reporting
+            // `passed` here would assert something false rather than merely omit it.
+            const failure = getPreTestWindowFailure()
+            const result = failure ? { passed: false, error: { message: failure } } : { passed: true }
+            await framework?.trackEvent(TestFrameworkState.BEFORE_ALL, HookState.POST, { ...this.preTestHookArgs(), result })
+        } catch (error) {
+            this.logger.debug(`Could not close the hook run for the pre-test window: ${error}`)
+        }
+    }
+
+    /** The binary runs path.relative() over `file`, so an absent path would drop the event. */
+    private preTestHookArgs() {
+        const hook = this.preTestHookLabel
+        return {
+            test: {
+                file: process.env[BROWSERSTACK_WDIO_CONFIG_FILE_PATH] || process.cwd(),
+                title: hook
+                    ? `${PRE_TEST_HOOK_TITLE_PREFIX} "${hook}" hook`
+                    : PRE_TEST_HOOK_TITLE_FALLBACK
+            }
+        }
+    }
+
     /**
      * browser.reloadSession() hands the worker a NEW session id while the driver object, the
      * wrapped commands and the currently running test all stay exactly the same. The scan gate
@@ -263,6 +330,16 @@ export default class AccessibilityModule extends BaseModule {
                     })
             }
 
+            // Gate for the window between driver creation and the first test: WDIO's config-level
+            // hooks run there, and being no test-framework hooks, nothing else registers the
+            // session — so their commands went unscanned while still counting as expected
+            // coverage. Unconditional like onHookStart; onBeforeTest recomputes it per test.
+            const sessionId = this.currentSessionId()
+            if (this.autoScanning && sessionId !== undefined && sessionId !== null) {
+                this.accessibilityMap.set(sessionId, true)
+                this.preTestWindowActive = true
+            }
+
         } catch (error) {
             this.logger.error(`Error in onBeforeExecute: ${error}`)
         }
@@ -281,8 +358,9 @@ export default class AccessibilityModule extends BaseModule {
                     !command.name.includes('execute') ||
                     !this.shouldPatchExecuteScript(args.length ? args[0] as string : null)
                 ) {
+                    await this.ensurePreTestHookRun()
                     try {
-                        await this.performScanCli(browser, command.name, this.currentHookRunUuid)
+                        await this.performScanCli(browser, command.name, this.currentHookRunUuid, this.preTestWindowActive)
                         this.logger.debug(`Accessibility scan performed after ${command.name} command`)
                     } catch (scanError) {
                         this.logger.debug(`Error performing accessibility scan after ${command.name}: ${scanError}`)
@@ -305,6 +383,11 @@ export default class AccessibilityModule extends BaseModule {
     async onBeforeTest(args: Record<string, unknown>) {
         try {
             this.logger.debug('Accessibility before test hook. Starting accessibility scan for this test case.')
+            // Window over: close its hook run before clearing the uuid, or a spec with no
+            // framework hooks would stamp every test-body scan as a hook scan.
+            await this.closePreTestHookRun()
+            this.preTestWindowActive = false
+            this.currentHookRunUuid = null
             const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || ''
             const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {}
 
@@ -498,7 +581,8 @@ export default class AccessibilityModule extends BaseModule {
     private async performScanCli(
         browser: WebdriverIO.Browser | WebdriverIO.MultiRemoteBrowser,
         commandName?: string,
-        hookRunUuid?: string | null
+        hookRunUuid?: string | null,
+        isGlobalHook?: boolean
     ): Promise<Record<string, unknown> | undefined> {
         return await PerformanceTester.measureWrapper(
             PERFORMANCE_SDK_EVENTS.A11Y_EVENTS.PERFORM_SCAN,
@@ -511,7 +595,7 @@ export default class AccessibilityModule extends BaseModule {
                     if (this.isAppAccessibility) {
                         const testName=this.currentTestName || undefined
                         const results: unknown = await (browser as WebdriverIO.Browser).execute(
-                            formatString(this.scriptInstance.performScan, JSON.stringify(_getParamsForAppAccessibility(commandName, testName, hookRunUuid))) as string,
+                            formatString(this.scriptInstance.performScan, JSON.stringify(_getParamsForAppAccessibility(commandName, testName, hookRunUuid, isGlobalHook))) as string,
                             {}
                         )
                         BStackLogger.debug(util.format(results as string))

packages/browserstack-service/src/constants.ts

@@ -86,6 +86,46 @@ export const CAPTURE_CONFIG_IMPORT_DEPTH = 1
 /* How far to walk up from the config dir looking for the project's package.json */
 export const MAX_PACKAGE_JSON_WALK_UP = 5
 
+/* WDIO config hooks that run in the worker with a live session, so a driver command inside them
+   is meaningful. Excludes launcher hooks (no driver), beforeSession (no session yet),
+   afterSession (session being deleted) and beforeCommand/afterCommand (per-command). */
+export const BROWSER_CONTEXT_HOOKS = [
+    'before',
+    'beforeSuite',
+    'beforeHook',
+    'beforeTest',
+    'afterTest',
+    'afterHook',
+    'afterSuite',
+    'after',
+    'beforeFeature',
+    'beforeScenario',
+    'beforeStep',
+    'afterStep',
+    'afterScenario',
+    'afterFeature'
+] as const
+
+/* Of those, the ones that can run before the first test/scenario: the Mocha adapter registers
+   beforeSuite as a root before-all, and beforeFeature precedes the first scenario. */
+export const PRE_TEST_WINDOW_HOOKS = ['before', 'beforeSuite', 'beforeFeature'] as const
+
+/* Tag on hook-window diagnostics, so they can be grepped out of an uploaded log */
+export const HOOK_WINDOW_LOG_PREFIX = '[hook-window]'
+
+/* Frameworks the pre-test window work applies to. Jasmine is excluded deliberately: it is in
+   the per-test a11y path (TEST_HOOK_FRAMEWORKS) and keeps that behaviour untouched, but App-A11y
+   is not supported there, so nothing new is switched on for it. Multiremote is excluded the same
+   way, and separately never reaches the CLI flow. */
+export const PRE_TEST_SCAN_FRAMEWORKS = ['mocha', 'cucumber'] as const
+
+/* Key the pre-test hook run is stashed under, so its finish can find its start */
+export const PRE_TEST_HOOK_ID = 'bstack:pre-test-hook'
+
+/* Title reported for the hook run covering the pre-test window */
+export const PRE_TEST_HOOK_TITLE_PREFIX = 'wdio'
+export const PRE_TEST_HOOK_TITLE_FALLBACK = `${PRE_TEST_HOOK_TITLE_PREFIX} config-level hook`
+
 /**
  * Keys whose line is scrubbed before a config file enters the archive.
  * `user` / `key` are WDIO's own top-level credential options, hence the bare entries.