From 1edf1b26b9f6c57d51aa15c58e9213e81a069d11 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Sat, 31 Oct 2026 23:48:54 +0530 Subject: [PATCH 01/14] feat(playwright): @keploy/playwright joins a Playwright suite to keploy mock record and replay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A new package in playwright/, @keploy/playwright: a thin client of keploy's scope protocol v1, the same wire go-sdk's keploytest speaks. Specs import test and expect from it instead of @playwright/test. It decides nothing: keploy's agent judges, and the package does what the agent answers. - Inert with KEPLOY_MOCK_AGENT unset or "off"; a value that is neither off nor an http URL fails every test, naming it. A process inside the claimer (KEPLOY_TEST_PID names an ancestor) stays inert. - Claim: a worker-scoped auto fixture claims the run (POST /agent/instance) before the worker's first hook or test, keyed by the Playwright project's name ("." when unnamed), and exports KEPLOY_TEST_PID. A refused claim fails the worker's tests with keploy's message and remedy. - Begin: a test-scoped auto fixture begins the test (POST /agent/scope/begin {name, file, project, canSkip}) before its beforeEach hooks. The file is relative to KEPLOY_MOCK_PROJECT; the name is the describe titles and the test's title joined with " › ". action skip skips the test through Playwright's skip with keploy's reason, and sends no end. - Settle: the context fixture waits, before the context closes, until none of its pages' calls is in flight, counted by request events. A call the browser cancelled (its page loaded another document, its frame went, its page closed) is not waited for; a route change inside the page cancels nothing. A call that never ends fails the test at Playwright's timeout, with a step naming it. - End: after the last teardown, POST /agent/scope/end with the test's own outcome (passed, failed, skipped; test.fail() counts as Playwright does). action fail fails the test with keploy's text. - given(name, value): POST /agent/scope/given while a test is open, with the caller's file:line; outside an open test it fails and sends nothing. An agent without the route is told apart (use matching releases). - Every answer the package cannot trust fails the test: unreachable, non-2xx (keploy's message and remedy), unreadable, over 1 MiB, no "v" or another version, an unknown action. No retry, no timeout. The root tsconfig now excludes playwright/, which builds on its own: without that, the root build compiles playwright/src and fails on @playwright/test. Rows: test/unit.test.mjs (client, join rule, wire names) and test/rows.test.mjs (real Playwright 1.62.1 runs of test/fixtures against a fake agent: inert, claim/begin/given/end order and outcomes with the re-claim after a failed test, skip with reason, judged red, refused begin, refused claim, malformed agent URL, given outside a test, and five settle rows with Chromium). Mutants: 23, all killed. CI: .github/workflows/playwright-sdk.yml, Node 18 and 22. Limits, stated in the README: settle covers the context and page fixtures, not a context the test makes with browser.newContext(); calls beforeAll and afterAll hooks make are answered from the worker's claim, one slice per Playwright project. Signed-off-by: slayerjain --- .github/workflows/playwright-sdk.yml | 31 +++ playwright/.gitignore | 3 + playwright/README.md | 51 ++++ playwright/package-lock.json | 118 ++++++++ playwright/package.json | 32 +++ playwright/src/agent.ts | 197 +++++++++++++ playwright/src/index.ts | 176 ++++++++++++ playwright/src/join.ts | 69 +++++ playwright/src/settle.ts | 90 ++++++ playwright/test/fixtures/agent-says.spec.ts | 10 + playwright/test/fixtures/basic.spec.ts | 24 ++ .../test/fixtures/given-outside.spec.ts | 6 + playwright/test/fixtures/playwright.config.ts | 12 + playwright/test/fixtures/settle.spec.ts | 40 +++ playwright/test/fixtures/steps-reporter.cjs | 11 + playwright/test/rows.test.mjs | 261 ++++++++++++++++++ playwright/test/unit.test.mjs | 175 ++++++++++++ playwright/tsconfig.json | 15 + tsconfig.json | 4 +- 19 files changed, 1324 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/playwright-sdk.yml create mode 100644 playwright/.gitignore create mode 100644 playwright/README.md create mode 100644 playwright/package-lock.json create mode 100644 playwright/package.json create mode 100644 playwright/src/agent.ts create mode 100644 playwright/src/index.ts create mode 100644 playwright/src/join.ts create mode 100644 playwright/src/settle.ts create mode 100644 playwright/test/fixtures/agent-says.spec.ts create mode 100644 playwright/test/fixtures/basic.spec.ts create mode 100644 playwright/test/fixtures/given-outside.spec.ts create mode 100644 playwright/test/fixtures/playwright.config.ts create mode 100644 playwright/test/fixtures/settle.spec.ts create mode 100644 playwright/test/fixtures/steps-reporter.cjs create mode 100644 playwright/test/rows.test.mjs create mode 100644 playwright/test/unit.test.mjs create mode 100644 playwright/tsconfig.json diff --git a/.github/workflows/playwright-sdk.yml b/.github/workflows/playwright-sdk.yml new file mode 100644 index 0000000..dbd5f8e --- /dev/null +++ b/.github/workflows/playwright-sdk.yml @@ -0,0 +1,31 @@ +name: "@keploy/playwright" + +on: + pull_request: + paths: + - "playwright/**" + - ".github/workflows/playwright-sdk.yml" + push: + branches: [main] + paths: + - "playwright/**" + - ".github/workflows/playwright-sdk.yml" + +jobs: + rows: + name: build and rows (Node ${{ matrix.node }}) + runs-on: ubuntu-latest + strategy: + matrix: + node: [18, 22] + defaults: + run: + working-directory: playwright + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - run: npm ci + - run: npx playwright install --with-deps chromium + - run: npm test diff --git a/playwright/.gitignore b/playwright/.gitignore new file mode 100644 index 0000000..0f59a68 --- /dev/null +++ b/playwright/.gitignore @@ -0,0 +1,3 @@ +node_modules +dist +test-results diff --git a/playwright/README.md b/playwright/README.md new file mode 100644 index 0000000..869723b --- /dev/null +++ b/playwright/README.md @@ -0,0 +1,51 @@ +# @keploy/playwright + +Joins a Playwright suite to keploy's Universal Mocker. `keploy mock record` records the calls each test's browser and app server make; `keploy mock replay` answers them from the recording, so the suite runs without its backends. keploy judges every recorded test, and lists each test it cannot replay with a reason. + +This package tells keploy where each test begins and ends. Through it, keploy skips a test it set aside and fails a test whose calls changed. It speaks keploy's scope protocol v1, the same one go-sdk's `keploytest` speaks, and decides nothing itself. + +## Use + +```sh +npm install --save-dev @keploy/playwright +``` + +Import `test` and `expect` from this package instead of `@playwright/test`. A fixture file that extends `test` extends this one: + +```ts +import { test, expect } from '@keploy/playwright'; + +test('lists the apps', async ({ page }) => { + await page.goto('/apps'); + await expect(page.getByRole('row')).toHaveCount(3); +}); +``` + +Run the suite under keploy with one worker: + +```sh +keploy mock record -c "npx playwright test --workers=1" +keploy mock replay -c "npx playwright test --workers=1" +``` + +keploy's first release runs one test process at a time. A second worker that is still alive while another runs is refused by keploy, and its tests fail. + +## Without keploy + +With `KEPLOY_MOCK_AGENT` unset or `off`, nothing changes. Only `keploy mock record` and `keploy mock replay` set that variable. + +## Under keploy + +- **Claim.** Each worker claims keploy's run before its first hook or test (`POST /agent/instance`). The claim uses the worker's Playwright project name as its key, or `.` for an unnamed project. A worker that Playwright starts again after a failed test claims again. +- **Begin.** Each test begins (`POST /agent/scope/begin`) before its `beforeEach` hooks. keploy builds the test's ID as `[project::]file::titles`. The file is relative to the project directory keploy names (`KEPLOY_MOCK_PROJECT`). The titles are the describe groups and the test's own title, joined with ` › `. +- **Skip.** When keploy has set a test aside, the test is skipped through Playwright's own skip, and the report shows keploy's reason (`keploy: ...`). +- **Settle.** Before the test's `context` (and its `page`) closes, the test waits until none of its pages' calls is in flight. Calls are counted by the browser's own request events, and no time window decides. A call that never ends keeps the test waiting until Playwright's test timeout fails it. The test shows a step that names each call still in flight. A call is not waited for once the browser has cancelled it, which happens when its page loads another document, its frame is removed or its page closes. A route change inside the page (the history API) cancels nothing. +- **End.** The test ends (`POST /agent/scope/end`) after its last teardown, with its own outcome. When keploy saw a call with no recorded answer, or a recorded call the test no longer made, the test fails with keploy's text. +- **Given values.** `await given(name, value)` tells keploy a value the test made itself. Examples are a session token it signs and a name it stamps with the time. keploy then follows the value wherever the test's calls carry it, as it follows an incoming request's values in a server test. Give a distinctive value, whole, as the calls carry it. `given` must be called inside a test, or in its `beforeEach` or `afterEach` hooks. It returns the value. + +Any answer from the agent that this package cannot trust fails the test. That covers an agent it cannot reach, an error answer, an unreadable answer, and an answer that does not speak scope protocol v1. Nothing is retried. + +## Limits of this version + +- Settle covers the `context` and `page` fixtures. A context the test makes with `browser.newContext()` is the test's own to close. +- Calls that `beforeAll`/`afterAll` hooks and worker fixtures make outside every test are answered from the worker's claim, one slice per Playwright project. diff --git a/playwright/package-lock.json b/playwright/package-lock.json new file mode 100644 index 0000000..517296d --- /dev/null +++ b/playwright/package-lock.json @@ -0,0 +1,118 @@ +{ + "name": "@keploy/playwright", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@keploy/playwright", + "version": "0.1.0", + "license": "Apache-2.0", + "devDependencies": { + "@playwright/test": "1.62.1", + "@types/node": "22.19.1", + "typescript": "5.9.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@playwright/test": ">=1.40.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz", + "integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@types/node": { + "version": "22.19.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.1.tgz", + "integrity": "sha512-LCCV0HdSZZZb34qifBsyWlUmok6W7ouER+oQIGBScS8EsZsQbrtFTUrDX4hOl+CS6p7cnNC4td+qrSVGSCTUfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/playwright/package.json b/playwright/package.json new file mode 100644 index 0000000..a8b46b7 --- /dev/null +++ b/playwright/package.json @@ -0,0 +1,32 @@ +{ + "name": "@keploy/playwright", + "version": "0.1.0", + "description": "Join a Playwright suite to keploy's Universal Mocker: keploy mock record and keploy mock replay judge each test.", + "license": "Apache-2.0", + "author": "Keploy ", + "repository": { + "type": "git", + "url": "git+https://github.com/keploy/typescript-sdk.git", + "directory": "playwright" + }, + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "dist" + ], + "scripts": { + "build": "tsc -p tsconfig.json", + "test": "npm run build && node --test test/*.test.mjs" + }, + "peerDependencies": { + "@playwright/test": ">=1.40.0" + }, + "devDependencies": { + "@playwright/test": "1.62.1", + "@types/node": "22.19.1", + "typescript": "5.9.3" + }, + "engines": { + "node": ">=18" + } +} diff --git a/playwright/src/agent.ts b/playwright/src/agent.ts new file mode 100644 index 0000000..39b6c62 --- /dev/null +++ b/playwright/src/agent.ts @@ -0,0 +1,197 @@ +// The client of keploy's agent: scope protocol v1, the same wire go-sdk's +// keploytest speaks (POST /agent/instance, /agent/scope/begin, +// /agent/scope/end, /agent/scope/given). +// +// Every answer of the agent carries the protocol version as "v"; an answer +// without it, or with another version, is an error, so this package on an +// old agent (or the reverse) fails loudly instead of running unjudged. Every +// way an answer can fail to be trusted is an error. Nothing is retried, and +// no clock decides anything: the agent answers once the work it owes is done. + +export const PROTOCOL_VERSION = 1; + +export const PATH_INSTANCE = '/agent/instance'; +export const PATH_BEGIN = '/agent/scope/begin'; +export const PATH_END = '/agent/scope/end'; +export const PATH_GIVEN = '/agent/scope/given'; + +export const ACTION_SKIP = 'skip'; +export const ACTION_FAIL = 'fail'; + +// Bounds what is read of one answer, and how much of one an error quotes. +const MAX_ANSWER = 1 << 20; +const MAX_QUOTED = 512; + +/** A 2xx answer of the agent. */ +export interface Answer { + v: number; + status?: string; + action?: string; + reason?: string; + instance?: string; + timeFile?: string; + variable?: string; +} + +/** An answer this package cannot trust, or the agent's own refusal. */ +export class KeployError extends Error { + /** The HTTP status of a non-2xx answer; 0 when there was none. */ + readonly status: number; + /** The scope protocol's error code of a refusal ("not_open", ...). */ + readonly code: string; + constructor(message: string, status = 0, code = '') { + super(message); + this.name = 'KeployError'; + this.status = status; + this.code = code; + } +} + +export function quoted(text: string): string { + const s = text.trim(); + return s.length > MAX_QUOTED ? `${JSON.stringify(s.slice(0, MAX_QUOTED))}...` : JSON.stringify(s); +} + +/** Talks to one keploy agent. */ +export class AgentClient { + constructor(readonly url: string, readonly token: string) {} + + /** Sends one call and reads its answer. */ + async post(path: string, body: unknown): Promise { + const target = this.url.replace(/\/+$/, '') + path; + const headers: Record = { 'content-type': 'application/json' }; + if (this.token) headers.authorization = `Bearer ${this.token}`; + let res: Response; + try { + // A redirect is an answer like any other non-2xx: following it would + // send the call again, or to somewhere else. + res = await fetch(target, { method: 'POST', headers, body: JSON.stringify(body), redirect: 'manual' }); + } catch (err) { + throw new KeployError(`keploy: cannot reach the keploy agent at ${this.url}: ${causeOf(err)}`); + } + let data: string; + try { + data = await readBounded(res); + } catch (err) { + throw new KeployError(`keploy: cannot read the keploy agent's answer to ${path}: ${causeOf(err)}`, res.status); + } + if (res.status < 200 || res.status > 299) { + let e: { error?: unknown; message?: unknown; remedy?: unknown } = {}; + try { e = JSON.parse(data); } catch { /* not the protocol's error body */ } + if (e && typeof e.message === 'string' && e.message !== '') { + const code = typeof e.error === 'string' ? e.error : ''; + const remedy = typeof e.remedy === 'string' && e.remedy !== '' ? ` (${e.remedy})` : ''; + throw new KeployError(`keploy: ${e.message}${remedy}`, res.status, code); + } + throw new KeployError(`keploy: the keploy agent answered ${res.status} ${res.statusText} to ${path}: ${quoted(data)}`, res.status); + } + let a: unknown; + try { + a = JSON.parse(data); + } catch (err) { + throw new KeployError(`keploy: the keploy agent's answer to ${path} cannot be read (${causeOf(err)}): ${quoted(data)}`, res.status); + } + if (a === null || typeof a !== 'object' || Array.isArray(a)) { + throw new KeployError(`keploy: the keploy agent's answer to ${path} cannot be read (not an object): ${quoted(data)}`, res.status); + } + const v = (a as { v?: unknown }).v; + if (v === undefined || v === null) { + throw new KeployError(`keploy: the keploy agent answered ${path} with ${quoted(data)}, which names no scope protocol version; ` + + `this @keploy/playwright needs a keploy agent that speaks scope protocol v${PROTOCOL_VERSION}: upgrade keploy`, res.status); + } + if (v !== PROTOCOL_VERSION) { + throw new KeployError(`keploy: the keploy agent answered ${path} in scope protocol v${String(v)} (${quoted(data)}); ` + + `this @keploy/playwright speaks scope protocol v${PROTOCOL_VERSION}: use a keploy and a @keploy/playwright of the same release`, res.status); + } + return a as Answer; + } + + /** Claims keploy's run for this test process (a Playwright worker). */ + async claim(key: string, dir: string): Promise { + const a = await this.post(PATH_INSTANCE, { key, dir }); + if (!a.instance) { + throw new KeployError('keploy: the keploy agent answered the claim of this test process without an instance'); + } + // a.timeFile names the run's clock when keploy moves it. Node and the + // browser read the clock through libc, which keploy's preload (inherited + // from the test command) moves; unlike a Go test binary, which reads it + // without libc, a Playwright worker needs no launcher to follow it. + return a; + } + + /** Asks the agent to open a test. */ + async begin(test: TestRef): Promise { + const a = await this.post(PATH_BEGIN, { ...test, canSkip: true }); + if (a.status !== 'ok') { + throw new KeployError(`keploy: the keploy agent answered the begin of ${test.name} with status ${JSON.stringify(a.status ?? '')}`); + } + if (a.action && a.action !== ACTION_SKIP) { + throw new KeployError(`keploy: the keploy agent answered the begin of ${test.name} with the unknown action ${JSON.stringify(a.action)}`); + } + return a; + } + + /** Tells the agent the test ended, with its outcome. */ + async end(test: TestRef, outcome: Outcome): Promise { + const a = await this.post(PATH_END, { ...test, outcome }); + if (a.status !== 'ok') { + throw new KeployError(`keploy: the keploy agent answered the end of ${test.name} with status ${JSON.stringify(a.status ?? '')}`); + } + if (a.action && a.action !== ACTION_FAIL) { + throw new KeployError(`keploy: the keploy agent answered the end of ${test.name} with the unknown action ${JSON.stringify(a.action)}`); + } + return a; + } + + /** Gives the open test a value it made itself. Returns the agent's variable name. */ + async given(name: string, value: string, source: string): Promise { + let a: Answer; + try { + a = await this.post(PATH_GIVEN, source ? { name, value, source } : { name, value }); + } catch (err) { + if (err instanceof KeployError && err.status === 404 && err.code === '') { + throw new KeployError(`keploy: this keploy agent does not take a test's given values (${PATH_GIVEN}): use a keploy and a @keploy/playwright of the same release`, 404); + } + throw err; + } + if (a.status !== 'ok' || !a.variable) { + throw new KeployError(`keploy: the keploy agent answered the given value ${name} with ${JSON.stringify({ status: a.status ?? '', variable: a.variable ?? '' })}`); + } + return a.variable; + } +} + +/** A test's name on the wire: keploy builds its ID as [project::]file::name. */ +export interface TestRef { + name: string; + file: string; + project?: string; +} + +export type Outcome = 'passed' | 'failed' | 'skipped'; + +async function readBounded(res: Response): Promise { + if (!res.body) return ''; + const reader = res.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > MAX_ANSWER) { + await reader.cancel(); + throw new Error(`the answer is larger than ${MAX_ANSWER} bytes`); + } + chunks.push(value); + } + return Buffer.concat(chunks).toString('utf8'); +} + +function causeOf(err: unknown): string { + if (err instanceof Error) { + const c = (err as Error & { cause?: unknown }).cause; + return c instanceof Error ? `${err.message}: ${c.message}` : err.message; + } + return String(err); +} diff --git a/playwright/src/index.ts b/playwright/src/index.ts new file mode 100644 index 0000000..53430cd --- /dev/null +++ b/playwright/src/index.ts @@ -0,0 +1,176 @@ +// @keploy/playwright joins a Playwright suite to keploy's Universal Mocker. +// +// keploy records the calls each test's browser and app server make (keploy +// mock record) and replays them (keploy mock replay), so the suite runs with +// every backend answered from the recording. keploy judges every recorded +// test and lists the ones it cannot replay, with a reason. This package is +// how a test tells keploy where it begins and ends, and how keploy skips a +// test it set aside or fails a test whose calls changed. It speaks the same +// scope protocol (v1) as go-sdk's keploytest; it decides nothing itself. +// +// Specs import test (and expect) from this package instead of +// @playwright/test: +// +// import { test, expect } from '@keploy/playwright'; +// +// Without keploy (KEPLOY_MOCK_AGENT unset or "off") nothing changes. Under +// keploy, with one worker (keploy's first release runs one test process at a +// time): +// - each worker claims keploy's run before its first test (POST +// /agent/instance), under its Playwright project's name; +// - each test begins (POST /agent/scope/begin) before its beforeEach hooks; +// a test keploy set aside is skipped with keploy's reason; +// - before the test's browser context closes, the test waits until none of +// its pages' calls is in flight (settle), so its calls end inside it; +// - the test ends (POST /agent/scope/end) after its last teardown, with +// its outcome; when keploy saw a call with no recorded answer, or a +// recorded call the test no longer made, the test fails with keploy's +// text; +// - given(name, value) tells keploy a value the test made itself (a +// session token it signs, a name it stamps with the time), which keploy +// follows where the test's calls carry it. +// Any answer of the agent this package cannot trust fails the test. + +import { test as base, expect } from '@playwright/test'; +import type { TestInfo } from '@playwright/test'; +import fs from 'node:fs'; +import path from 'node:path'; +import { ACTION_FAIL, ACTION_SKIP, AgentClient, KeployError } from './agent'; +import type { Outcome, TestRef } from './agent'; +import { ENV_TEST_PID, decide } from './join'; +import type { Joining } from './join'; +import { Inflight } from './settle'; + +export { expect, KeployError }; +export type { TestRef, Outcome }; + +let joined: Joining | null = null; +function joining(): Joining { + if (joined === null) joined = decide(process.env, process.pid); + return joined; +} + +// This worker's client once it claimed the run; null when inert. +let client: AgentClient | null = null; +// The project directory test files are named relative to. +let projectDir = ''; +// The test this worker has open, if any. +let open: TestRef | null = null; + +type WorkerFixtures = { keployWorker: void }; +type TestFixtures = { keployTest: void }; + +export const test = base.extend({ + keployWorker: [async ({}, use, workerInfo) => { + const j = joining(); + if (j.mode === 'refused') throw new KeployError(j.refusal); + if (j.mode === 'active') { + const c = new AgentClient(j.url, j.token); + projectDir = realpath(j.project || process.cwd()); + // The claim's key names the slice of the recording this worker's + // calls outside tests are answered from: one per Playwright project, + // whose workers run its tests one after another. + await c.claim(workerInfo.project.name || '.', realpath(process.cwd())); + process.env[ENV_TEST_PID] = String(process.pid); + client = c; + } + await use(); + }, { scope: 'worker', auto: true }], + + context: async ({ context }, use) => { + if (client === null) { + await use(context); + return; + } + const inflight = new Inflight(context); + await use(context); + if (inflight.size > 0) { + const calls = inflight.describe(); + const shown = calls.slice(0, 5).join(', ') + (calls.length > 5 ? `, and ${calls.length - 5} more` : ''); + await base.step(`keploy: wait for the test's ${calls.length} call(s) in flight before its browser context closes: ${shown}`, () => inflight.settled()); + } + }, + + keployTest: [async ({ keployWorker }, use, testInfo) => { + void keployWorker; + const c = client; + if (c === null) { + await use(); + return; + } + const ref = testRef(testInfo); + const b = await c.begin(ref); + if (b.action === ACTION_SKIP) { + // The framework's own skip, with keploy's reason; no end follows. + testInfo.skip(true, `keploy: ${b.reason ?? ''}`); + return; + } + open = ref; + await use(); + open = null; + const e = await c.end(ref, outcomeOf(testInfo)); + if (e.action === ACTION_FAIL) throw new KeployError(`keploy: ${e.reason ?? ''}`); + }, { auto: true }], +}); + +/** + * given tells keploy a value the open test made itself, as an incoming + * request's value is for a server test: keploy follows it where the test's + * calls carry it, so a replay that makes a new value is compared by that + * name, never by guessing. Give a distinctive value (a name with a nonce, a + * signed token), whole as the calls carry it. Returns value. + * Without keploy it returns value and does nothing else. + */ +export async function given(name: string, value: string): Promise { + const j = joining(); + if (j.mode === 'inert') return value; + if (j.mode === 'refused') throw new KeployError(j.refusal); + const c = client; + if (c === null || open === null) { + throw new KeployError(`keploy: the value ${name} was given while no test was open: call given inside a test of @keploy/playwright's test, ` + + 'or in its beforeEach or afterEach hooks'); + } + await c.given(name, value, sourceOf(new Error().stack ?? '')); + return value; +} + +/** A test's name on keploy's wire: [project::]file::titles › joined. */ +export function testRef(ti: Pick, dir = projectDir): TestRef { + const file = toSlash(path.relative(dir || process.cwd(), realpath(ti.file))); + // titlePath starts with the file; the rest are the describe groups and + // the test's own title. + const name = ti.titlePath.slice(1).join(' › '); + const project = ti.project.name; + return project ? { name, file, project } : { name, file }; +} + +/** The test's own verdict, as keploy's end takes it. */ +export function outcomeOf(ti: Pick): Outcome { + if (ti.status === 'skipped') return 'skipped'; + return ti.status === ti.expectedStatus ? 'passed' : 'failed'; +} + +// The caller of given, as file:line relative to the project: the first +// frame of the stack outside this package. +const SOURCE_MAX = 1024; +export function sourceOf(stack: string, dir = projectDir): string { + const own = path.dirname(__filename); + for (const line of stack.split('\n').slice(1)) { + const m = /\(?((?:file:\/\/)?[^\s()]+):(\d+):\d+\)?\s*$/.exec(line.trim()); + if (!m) continue; + let file = m[1].startsWith('file://') ? new URL(m[1]).pathname : m[1]; + if (!path.isAbsolute(file) || path.dirname(file) === own || file.startsWith('node:')) continue; + file = toSlash(path.relative(dir || process.cwd(), file)); + const s = `${file}:${m[2]}`; + return s.length > SOURCE_MAX ? `${path.basename(file)}:${m[2]}` : s; + } + return ''; +} + +function realpath(p: string): string { + try { return fs.realpathSync(p); } catch { return path.resolve(p); } +} + +function toSlash(p: string): string { + return p.split(path.sep).join('/'); +} diff --git a/playwright/src/join.ts b/playwright/src/join.ts new file mode 100644 index 0000000..e5c16ed --- /dev/null +++ b/playwright/src/join.ts @@ -0,0 +1,69 @@ +// Whether this process reports to keploy: the same rule go-sdk's keploytest +// follows. keploy mock record and keploy mock replay export +// KEPLOY_MOCK_AGENT (the agent's URL), KEPLOY_MOCK_AGENT_TOKEN and +// KEPLOY_MOCK_PROJECT (the project's directory) into the test command. +// Unset or "off", the package is inert and every test runs as it would +// without keploy. + +import fs from 'node:fs'; + +export const ENV_AGENT = 'KEPLOY_MOCK_AGENT'; +export const ENV_TOKEN = 'KEPLOY_MOCK_AGENT_TOKEN'; +export const ENV_PROJECT = 'KEPLOY_MOCK_PROJECT'; +// The claimer's pid, exported once a test process has claimed the run, so +// the processes its tests start (the browser, a helper) know they belong to +// it and do not claim the run themselves. +export const ENV_TEST_PID = 'KEPLOY_TEST_PID'; + +export type Joining = + | { mode: 'inert' } + | { mode: 'refused'; refusal: string } + | { mode: 'active'; url: string; token: string; project: string }; + +export function decide(env: NodeJS.ProcessEnv, pid: number, runsInside: (pid: number) => boolean = runsInsideProc): Joining { + const v = (env[ENV_AGENT] ?? '').trim(); + if (v === '' || v.toLowerCase() === 'off') return { mode: 'inert' }; + let u: URL | null = null; + try { u = new URL(v); } catch { u = null; } + if (!u || (u.protocol !== 'http:' && u.protocol !== 'https:') || u.host === '') { + return { + mode: 'refused', + refusal: `keploy: ${ENV_AGENT}=${JSON.stringify(v)} is neither off nor the http URL of a keploy agent; set it to off to run without keploy`, + }; + } + const tp = (env[ENV_TEST_PID] ?? '').trim(); + if (tp !== '' && tp !== String(pid)) { + // Another process joined keploy's run. This one is a member only when + // it runs inside that process; a marker naming any other process was + // inherited from elsewhere, and this process claims like any test + // process (the agent refuses a second live one). + const n = Number(tp); + if (Number.isInteger(n) && n > 0 && runsInside(n)) return { mode: 'inert' }; + } + return { mode: 'active', url: v.replace(/\/+$/, ''), token: env[ENV_TOKEN] ?? '', project: (env[ENV_PROJECT] ?? '').trim() }; +} + +// Whether this process runs inside process pid (pid is an ancestor), read +// from /proc. Without /proc (a system other than Linux) it cannot tell, and +// takes the marker at its word. +export function runsInsideProc(pid: number): boolean { + if (!fs.existsSync('/proc/self/stat')) return true; + let p = process.ppid; + for (let i = 0; i < 128 && p > 0; i++) { + if (p === pid) return true; + const next = procParent(p); + if (next === null || next === p) return false; + p = next; + } + return false; +} + +function procParent(pid: number): number | null { + let stat: string; + try { stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf8'); } catch { return null; } + // The command name, in parentheses, may hold spaces and parentheses: the + // fields after it start at the last ')'. + const rest = stat.slice(stat.lastIndexOf(')') + 2).split(' '); + const ppid = Number(rest[1]); + return Number.isInteger(ppid) ? ppid : null; +} diff --git a/playwright/src/settle.ts b/playwright/src/settle.ts new file mode 100644 index 0000000..cdec953 --- /dev/null +++ b/playwright/src/settle.ts @@ -0,0 +1,90 @@ +// Settle: before a test's browser context closes, wait until none of the +// calls its pages made is in flight, so a call the test started is answered +// inside the test instead of being cut by the close (a call keploy recorded +// whole would be cut at a different byte each run). Counted by the +// browser's own request events; no time window decides it. + +import type { BrowserContext, Frame, Page, Request } from '@playwright/test'; + +export class Inflight { + private readonly open = new Map(); + private waiters: (() => void)[] = []; + + constructor(context: BrowserContext) { + context.on('request', (r) => { + let frame: Frame | null = null; + // A service worker's call has no frame. + try { frame = r.frame(); } catch { frame = null; } + this.open.set(r, { frame, nav: r.isNavigationRequest() }); + }); + context.on('requestfinished', (r) => this.done(r)); + context.on('requestfailed', (r) => this.done(r)); + // The frames whose navigation request was answered with a document, which + // the frame commits next. + const loading = new Set(); + context.on('response', (res) => { + const r = res.request(); + if (!r.isNavigationRequest()) return; + let f: Frame | null = null; + try { f = r.frame(); } catch { f = null; } + // A 204 or 205 answer, or a download, leaves the document in place. + const keeps = res.status() === 204 || res.status() === 205 || /attachment/i.test(res.headers()['content-disposition'] ?? ''); + if (f && !keeps) loading.add(f); + }); + const watch = (p: Page): void => { + // The browser reports no end for a call whose document was replaced + // while it ran: a frame that commits a new document has cancelled + // every call the old one still had open. The new document's calls + // start after the commit. A navigation inside the document (the + // history API, a client-side route change) has no navigation request + // and replaces nothing. + p.on('framenavigated', (f) => { + if (!loading.delete(f)) return; + this.drop((o) => o.frame === f && !o.nav); + }); + p.on('framedetached', (f) => { loading.delete(f); this.drop((o) => o.frame === f); }); + p.on('close', () => this.drop((o) => o.frame !== null && pageOf(o.frame) === p)); + }; + context.pages().forEach(watch); + context.on('page', watch); + context.on('close', () => this.drop(() => true)); + } + + /** The calls still in flight, for a step's title. */ + describe(): string[] { + return [...this.open.keys()].map((r) => `${r.method()} ${r.url()}`); + } + + get size(): number { + return this.open.size; + } + + /** Resolves once no call is in flight. */ + settled(): Promise { + if (this.open.size === 0) return Promise.resolve(); + return new Promise((resolve) => this.waiters.push(resolve)); + } + + private done(r: Request): void { + if (this.open.delete(r)) this.wake(); + } + + private drop(gone: (o: { frame: Frame | null; nav: boolean }) => boolean): void { + let n = 0; + for (const [r, o] of this.open) { + if (gone(o)) { this.open.delete(r); n++; } + } + if (n > 0) this.wake(); + } + + private wake(): void { + if (this.open.size !== 0) return; + const w = this.waiters; + this.waiters = []; + w.forEach((f) => f()); + } +} + +function pageOf(f: Frame): Page | null { + try { return f.page(); } catch { return null; } +} diff --git a/playwright/test/fixtures/agent-says.spec.ts b/playwright/test/fixtures/agent-says.spec.ts new file mode 100644 index 0000000..d79f0f7 --- /dev/null +++ b/playwright/test/fixtures/agent-says.spec.ts @@ -0,0 +1,10 @@ +import fs from 'node:fs'; +import { test } from '../../dist'; + +test('set aside', async () => { + fs.writeFileSync(`${process.env.MARKER_DIR}/set-aside-ran`, '1'); +}); +test('judged red', async () => {}); +test('refused', async () => { + fs.writeFileSync(`${process.env.MARKER_DIR}/refused-ran`, '1'); +}); diff --git a/playwright/test/fixtures/basic.spec.ts b/playwright/test/fixtures/basic.spec.ts new file mode 100644 index 0000000..d01013c --- /dev/null +++ b/playwright/test/fixtures/basic.spec.ts @@ -0,0 +1,24 @@ +import fs from 'node:fs'; +import { test, expect, given } from '../../dist'; + +test.describe('g', () => { + // A given needs an open test: the begin comes before beforeEach. + test.beforeEach(async () => { + expect(await given('seed', 'seed-value-1')).toBe('seed-value-1'); + }); + test('a', async () => { + const joined = (process.env.KEPLOY_MOCK_AGENT ?? 'off') !== 'off'; + expect(process.env.KEPLOY_TEST_PID).toBe(joined ? String(process.pid) : undefined); + if (process.env.MARKER_DIR) fs.writeFileSync(`${process.env.MARKER_DIR}/a`, String(process.pid)); + }); + test('b', async () => { + expect(1).toBe(2); + }); + test('c', async () => { + test.skip(true, 'its own skip'); + }); + test('d', async () => { + test.fail(); + expect(1).toBe(2); + }); +}); diff --git a/playwright/test/fixtures/given-outside.spec.ts b/playwright/test/fixtures/given-outside.spec.ts new file mode 100644 index 0000000..5903b54 --- /dev/null +++ b/playwright/test/fixtures/given-outside.spec.ts @@ -0,0 +1,6 @@ +import { test, given } from '../../dist'; + +test.beforeAll(async () => { + await given('early', 'early-value-1'); +}); +test('after a given outside', async () => {}); diff --git a/playwright/test/fixtures/playwright.config.ts b/playwright/test/fixtures/playwright.config.ts new file mode 100644 index 0000000..5aa0baa --- /dev/null +++ b/playwright/test/fixtures/playwright.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from '@playwright/test'; + +// The rows' Playwright project: one worker, no retries, as keploy runs a suite. +export default defineConfig({ + testDir: '.', + workers: 1, + retries: 0, + timeout: Number(process.env.PW_TIMEOUT || 30000), + reporter: [['json'], ['./steps-reporter.cjs']], + outputDir: process.env.PW_OUTPUT_DIR || 'test-results', + projects: [{ name: 'p1', use: { headless: true } }], +}); diff --git a/playwright/test/fixtures/settle.spec.ts b/playwright/test/fixtures/settle.spec.ts new file mode 100644 index 0000000..e46012d --- /dev/null +++ b/playwright/test/fixtures/settle.spec.ts @@ -0,0 +1,40 @@ +import { test } from '../../dist'; + +const base = process.env.SLOW_BASE ?? ''; +test.afterEach(async () => { + if (!process.env.NO_RELEASE) await fetch(`${base}/release`); +}); +test('a call in flight when the body ends', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; +}); +test('a call its page cancelled by loading another document', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + await page.goto(`${base}/page?next`); +}); +test('a call in flight across a route change inside the page', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + const routed = page.waitForEvent('framenavigated'); + await page.evaluate(() => { void fetch('/slow'); history.pushState({}, '', '/page?route'); }); + await sent; + await routed; +}); +test('a call in flight across a navigation answered 204', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + const answered = page.waitForResponse(/\/nocontent$/); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + // A 204 answer to a navigation leaves the document in place: nothing is cancelled. + await page.evaluate(() => { location.href = '/nocontent'; }); + await answered; + const routed = page.waitForEvent('framenavigated'); + await page.evaluate(() => { history.pushState({}, '', '/page?route'); }); + await routed; +}); diff --git a/playwright/test/fixtures/steps-reporter.cjs b/playwright/test/fixtures/steps-reporter.cjs new file mode 100644 index 0000000..0d8555d --- /dev/null +++ b/playwright/test/fixtures/steps-reporter.cjs @@ -0,0 +1,11 @@ +// The rows' reporter of every step a test ran, hooks' and fixtures' included +// (Playwright's JSON report keeps only steps at the test's top level): one +// JSON line {test, step} per step begun, appended to STEPS_FILE. +const fs = require('node:fs'); + +class StepsReporter { + onStepBegin(test, _result, step) { + if (process.env.STEPS_FILE) fs.appendFileSync(process.env.STEPS_FILE, `${JSON.stringify({ test: test.title, step: step.title })}\n`); + } +} +module.exports = StepsReporter; diff --git a/playwright/test/rows.test.mjs b/playwright/test/rows.test.mjs new file mode 100644 index 0000000..bedf46d --- /dev/null +++ b/playwright/test/rows.test.mjs @@ -0,0 +1,261 @@ +// Rows: real Playwright runs of test/fixtures against a fake keploy agent, +// which records every call and answers as each row says. The settle rows +// need Chromium (npx playwright install chromium). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import http from 'node:http'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const PKG = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const CLI = path.join(PKG, 'node_modules', '@playwright', 'test', 'cli.js'); +const CONFIG = path.join(PKG, 'test', 'fixtures', 'playwright.config.ts'); + +// What happened, in order, across the fake agent and the page server. +function journal() { + const events = []; + return { events, add: (e) => events.push(e) }; +} + +// A fake keploy agent. answer(call) returns {status, body} or undefined for +// the protocol's plain "ok". +async function fakeAgent(j, answer = () => undefined) { + const srv = http.createServer((req, res) => { + let body = ''; + req.on('data', (d) => { body += d; }); + req.on('end', () => { + const call = { path: req.url, auth: req.headers.authorization ?? '', body: body ? JSON.parse(body) : null }; + j.add({ agent: call }); + let a = answer(call); + if (!a) { + a = { body: call.path === '/agent/instance' ? { v: 1, instance: 'run', timeFile: '' } + : call.path === '/agent/scope/given' ? { v: 1, status: 'ok', variable: `given:${call.body.name}` } + : { v: 1, status: 'ok' } }; + } + res.writeHead(a.status ?? 200, { 'content-type': 'application/json' }); + res.end(JSON.stringify(a.body)); + }); + }); + await new Promise((r) => srv.listen(0, '127.0.0.1', r)); + return { url: `http://127.0.0.1:${srv.address().port}`, close: () => new Promise((r) => { srv.closeAllConnections(); srv.close(r); }) }; +} + +// The settle rows' page server: /slow is answered only once /release is called. +async function pageServer(j) { + let release = null; + const released = new Promise((r) => { release = r; }); + const srv = http.createServer((req, res) => { + if (req.url === '/page') { res.writeHead(200, { 'content-type': 'text/html' }); res.end('page'); return; } + if (req.url === '/release') { release(); res.end('ok'); return; } + if (req.url === '/nocontent') { res.writeHead(204); res.end(); return; } + if (req.url === '/slow') { + j.add({ page: 'slow sent' }); + res.on('close', () => { if (!res.writableFinished) j.add({ page: 'slow cut' }); }); + res.on('finish', () => j.add({ page: 'slow answered' })); + void released.then(() => res.end('slow')); + return; + } + res.writeHead(404); res.end(); + }); + await new Promise((r) => srv.listen(0, '127.0.0.1', r)); + return { url: `http://127.0.0.1:${srv.address().port}`, close: () => new Promise((r) => { srv.closeAllConnections(); srv.close(r); }) }; +} + +// Runs one Playwright run of the fixture specs; returns each test's result. +async function playwright(specs, env) { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'kpw-')); + const report = path.join(tmp, 'report.json'); + const stepsFile = path.join(tmp, 'steps.jsonl'); + const childEnv = { ...process.env, PLAYWRIGHT_JSON_OUTPUT_NAME: report, PW_OUTPUT_DIR: path.join(tmp, 'out'), MARKER_DIR: tmp, STEPS_FILE: stepsFile, ...env }; + for (const k of Object.keys(childEnv)) if (childEnv[k] === undefined) delete childEnv[k]; + const child = spawn(process.execPath, [CLI, 'test', '-c', CONFIG, ...specs], { cwd: PKG, env: childEnv, stdio: ['ignore', 'pipe', 'pipe'] }); + let out = ''; + child.stdout.on('data', (d) => { out += d; }); + child.stderr.on('data', (d) => { out += d; }); + const code = await new Promise((r) => child.on('close', r)); + if (!fs.existsSync(report)) throw new Error(`playwright wrote no report (exit ${code}):\n${out}`); + const steps = fs.existsSync(stepsFile) ? fs.readFileSync(stepsFile, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)) : []; + const tests = {}; + const walk = (suite, titles) => { + for (const s of suite.suites ?? []) walk(s, s.file && s.title === s.file ? titles : [...titles, s.title]); + for (const spec of suite.specs ?? []) { + const t = spec.tests[0]; + const r = t.results[t.results.length - 1] ?? {}; + tests[[...titles, spec.title].join(' › ')] = { + status: r.status ?? 'none', + error: (r.errors ?? []).map((e) => e.message ?? '').join('\n'), + annotations: r.annotations ?? t.annotations ?? [], + steps: steps.filter((x) => x.test === spec.title).map((x) => x.step), + }; + } + }; + for (const s of JSON.parse(fs.readFileSync(report, 'utf8')).suites) walk(s, []); + return { tests, tmp, out }; +} + +const keployEnv = (url) => ({ KEPLOY_MOCK_AGENT: url, KEPLOY_MOCK_AGENT_TOKEN: 'tok', KEPLOY_MOCK_PROJECT: PKG, KEPLOY_TEST_PID: undefined }); +const calls = (j) => j.events.filter((e) => e.agent).map((e) => e.agent); +const ref = (file, name) => ({ name, file: `test/fixtures/${file}`, project: 'p1' }); + +test('inert without keploy: no call, every test runs as written', async () => { + const j = journal(); + const a = await fakeAgent(j); + try { + for (const v of [undefined, 'off']) { + const { tests, tmp } = await playwright(['basic.spec.ts'], { KEPLOY_MOCK_AGENT: v, KEPLOY_TEST_PID: undefined }); + assert.deepEqual(Object.fromEntries(Object.entries(tests).map(([k, t]) => [k, t.status])), + { 'g › a': 'passed', 'g › b': 'failed', 'g › c': 'skipped', 'g › d': 'failed' }); + assert.ok(fs.existsSync(path.join(tmp, 'a'))); + } + assert.deepEqual(calls(j), []); + } finally { await a.close(); } +}); + +test('a worker claims the run, each test begins before its beforeEach and ends with its outcome', async () => { + const j = journal(); + const a = await fakeAgent(j); + try { + const { tests, tmp } = await playwright(['basic.spec.ts'], keployEnv(a.url)); + assert.deepEqual(Object.fromEntries(Object.entries(tests).map(([k, t]) => [k, t.status])), + { 'g › a': 'passed', 'g › b': 'failed', 'g › c': 'skipped', 'g › d': 'failed' }); + const r = (n) => ref('basic.spec.ts', `g › ${n}`); + const given = { name: 'seed', value: 'seed-value-1', source: 'test/fixtures/basic.spec.ts:7' }; + const claim = { key: 'p1', dir: fs.realpathSync(PKG) }; + // b fails, so Playwright runs c and d in a new worker, which claims again. + assert.deepEqual(calls(j).map((c) => [c.path, c.body]), [ + ['/agent/instance', claim], + ['/agent/scope/begin', { ...r('a'), canSkip: true }], ['/agent/scope/given', given], ['/agent/scope/end', { ...r('a'), outcome: 'passed' }], + ['/agent/scope/begin', { ...r('b'), canSkip: true }], ['/agent/scope/given', given], ['/agent/scope/end', { ...r('b'), outcome: 'failed' }], + ['/agent/instance', claim], + ['/agent/scope/begin', { ...r('c'), canSkip: true }], ['/agent/scope/given', given], ['/agent/scope/end', { ...r('c'), outcome: 'skipped' }], + ['/agent/scope/begin', { ...r('d'), canSkip: true }], ['/agent/scope/given', given], ['/agent/scope/end', { ...r('d'), outcome: 'passed' }], + ]); + assert.ok(calls(j).every((c) => c.auth === 'Bearer tok')); + // KEPLOY_TEST_PID named the claiming worker (test a checks it). + assert.ok(fs.existsSync(path.join(tmp, 'a'))); + } finally { await a.close(); } +}); + +test('keploy skips a set-aside test with its reason, fails a judged-red one, and a refused begin fails the test', async () => { + const j = journal(); + const a = await fakeAgent(j, (c) => { + if (c.path === '/agent/scope/begin' && c.body.name === 'set aside') return { body: { v: 1, status: 'ok', action: 'skip', reason: 'flaky/request_changed: the body changed' } }; + if (c.path === '/agent/scope/end' && c.body.name === 'judged red') return { body: { v: 1, status: 'ok', action: 'fail', reason: 'GET /company: no recorded answer' } }; + if (c.path === '/agent/scope/begin' && c.body.name === 'refused') return { status: 422, body: { error: 'one_test_at_a_time', message: 'refused began while x was open', remedy: 'run one at a time' } }; + return undefined; + }); + try { + const { tests, tmp } = await playwright(['agent-says.spec.ts'], keployEnv(a.url)); + assert.equal(tests['set aside'].status, 'skipped'); + assert.deepEqual(tests['set aside'].annotations.filter((x) => x.type === 'skip').map((x) => x.description), ['keploy: flaky/request_changed: the body changed']); + assert.ok(!fs.existsSync(path.join(tmp, 'set-aside-ran')), 'the set-aside test ran'); + assert.equal(tests['judged red'].status, 'failed'); + assert.match(tests['judged red'].error, /keploy: GET \/company: no recorded answer/); + assert.equal(tests.refused.status, 'failed'); + assert.match(tests.refused.error, /keploy: refused began while x was open \(run one at a time\)/); + assert.ok(!fs.existsSync(path.join(tmp, 'refused-ran')), 'the refused test ran'); + // No end for a skipped test, nor for one whose begin was refused. + assert.deepEqual(calls(j).filter((c) => c.path === '/agent/scope/end').map((c) => c.body.name), ['judged red']); + } finally { await a.close(); } +}); + +test('a refused claim fails every test of the worker, and no test begins', async () => { + const j = journal(); + const a = await fakeAgent(j, (c) => (c.path === '/agent/instance' + ? { status: 422, body: { error: 'one_test_process_at_a_time', message: 'process 2 asked to run tests while process 1 still runs them', remedy: 'run one test process at a time' } } + : undefined)); + try { + const { tests } = await playwright(['agent-says.spec.ts'], keployEnv(a.url)); + for (const t of Object.values(tests)) { + assert.equal(t.status, 'failed'); + assert.match(t.error, /keploy: process 2 asked to run tests while process 1 still runs them \(run one test process at a time\)/); + } + assert.ok(calls(j).every((c) => c.path === '/agent/instance')); + } finally { await a.close(); } +}); + +test('a malformed KEPLOY_MOCK_AGENT fails every test, naming it', async () => { + const { tests } = await playwright(['agent-says.spec.ts'], { ...keployEnv('localhost:16789') }); + for (const t of Object.values(tests)) { + assert.equal(t.status, 'failed'); + assert.match(t.error, /KEPLOY_MOCK_AGENT="localhost:16789" is neither off nor the http URL of a keploy agent/); + } +}); + +test('a value given outside a test fails, and is not sent', async () => { + const j = journal(); + const a = await fakeAgent(j); + try { + const { tests } = await playwright(['given-outside.spec.ts'], keployEnv(a.url)); + assert.equal(tests['after a given outside'].status, 'failed'); + assert.match(tests['after a given outside'].error, /the value early was given while no test was open/); + assert.ok(!calls(j).some((c) => c.path === '/agent/scope/given')); + } finally { await a.close(); } +}); + +test('settle: a call in flight when the body ends is answered before the test ends', async () => { + const j = journal(); + const a = await fakeAgent(j); + const p = await pageServer(j); + try { + const { tests, out } = await playwright(['settle.spec.ts', '-g', 'when the body ends'], { ...keployEnv(a.url), SLOW_BASE: p.url }); + assert.equal(tests['a call in flight when the body ends'].status, 'passed', out); + const order = j.events.map((e) => (e.page ? e.page : e.agent.path)); + assert.deepEqual(order.filter((e) => e.startsWith('slow') || e === '/agent/scope/end'), ['slow sent', 'slow answered', '/agent/scope/end']); + } finally { await p.close(); await a.close(); } +}); + +test('settle: a call that never ends fails the test by name, and is never cut before the end', async () => { + const j = journal(); + const a = await fakeAgent(j); + const p = await pageServer(j); + try { + const { tests, out } = await playwright(['settle.spec.ts', '-g', 'when the body ends'], { ...keployEnv(a.url), SLOW_BASE: p.url, NO_RELEASE: '1', PW_TIMEOUT: '4000' }); + const t = tests['a call in flight when the body ends']; + assert.equal(t.status, 'timedOut', out); + assert.ok(t.steps.some((s) => /^keploy: wait for the test's 1 call\(s\) in flight before its browser context closes: GET http:\/\/127\.0\.0\.1:\d+\/slow$/.test(s)), t.steps.join('\n')); + // The test did not end while its call was open and uncut. + const order = j.events.map((e) => (e.page ? e.page : e.agent.path)); + const end = order.indexOf('/agent/scope/end'); + assert.ok(end === -1 || order.slice(0, end).includes('slow cut'), `ended with the call open: ${order.join(', ')}`); + } finally { await p.close(); await a.close(); } +}); + +test('settle: a call cancelled by its page loading another document is not waited for', async () => { + const j = journal(); + const a = await fakeAgent(j); + const p = await pageServer(j); + try { + const { tests, out } = await playwright(['settle.spec.ts', '-g', 'loading another document'], { ...keployEnv(a.url), SLOW_BASE: p.url, NO_RELEASE: '1', PW_TIMEOUT: '20000' }); + assert.equal(tests['a call its page cancelled by loading another document'].status, 'passed', out); + const order = j.events.map((e) => (e.page ? e.page : e.agent.path)); + assert.deepEqual(order.filter((e) => e.startsWith('slow') || e === '/agent/scope/end'), ['slow sent', 'slow cut', '/agent/scope/end']); + } finally { await p.close(); await a.close(); } +}); + +test('settle: a route change inside the page cancels nothing, so its call in flight is waited for', async () => { + const j = journal(); + const a = await fakeAgent(j); + const p = await pageServer(j); + try { + const { tests, out } = await playwright(['settle.spec.ts', '-g', 'route change inside the page'], { ...keployEnv(a.url), SLOW_BASE: p.url, NO_RELEASE: '1', PW_TIMEOUT: '4000' }); + const t = tests['a call in flight across a route change inside the page']; + assert.equal(t.status, 'timedOut', out); + assert.ok(t.steps.some((s) => /^keploy: wait for the test's 1 call\(s\) in flight .*\/slow$/.test(s)), t.steps.join('\n')); + } finally { await p.close(); await a.close(); } +}); + +test('settle: a navigation answered 204 replaces no document, so a call in flight is still waited for', async () => { + const j = journal(); + const a = await fakeAgent(j); + const p = await pageServer(j); + try { + const { tests, out } = await playwright(['settle.spec.ts', '-g', 'navigation answered 204'], { ...keployEnv(a.url), SLOW_BASE: p.url, NO_RELEASE: '1', PW_TIMEOUT: '4000' }); + const t = tests['a call in flight across a navigation answered 204']; + assert.equal(t.status, 'timedOut', out); + assert.ok(t.steps.some((s) => /^keploy: wait for the test's 1 call\(s\) in flight .*\/slow$/.test(s)), t.steps.join('\n')); + } finally { await p.close(); await a.close(); } +}); diff --git a/playwright/test/unit.test.mjs b/playwright/test/unit.test.mjs new file mode 100644 index 0000000..a5b7065 --- /dev/null +++ b/playwright/test/unit.test.mjs @@ -0,0 +1,175 @@ +// Unit rows: the agent client, the join rule and the wire names, against a +// fake agent. No browser. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; +import path from 'node:path'; +import { AgentClient, KeployError } from '../dist/agent.js'; +import { decide } from '../dist/join.js'; +import { testRef, outcomeOf, sourceOf } from '../dist/index.js'; + +// One fake agent answering every call with answer(path, body). +async function agent(answer) { + const calls = []; + const srv = http.createServer((req, res) => { + let body = ''; + req.on('data', (d) => { body += d; }); + req.on('end', () => { + const call = { path: req.url, auth: req.headers.authorization ?? '', body: body ? JSON.parse(body) : null }; + calls.push(call); + const a = answer(call); + res.writeHead(a.status ?? 200, { 'content-type': 'application/json' }); + res.end(typeof a.body === 'string' ? a.body : JSON.stringify(a.body)); + }); + }); + await new Promise((r) => srv.listen(0, '127.0.0.1', r)); + const url = `http://127.0.0.1:${srv.address().port}`; + return { url, calls, close: () => new Promise((r) => srv.close(r)) }; +} + +async function rejects(p, re) { + await assert.rejects(p, (err) => { + assert.ok(err instanceof KeployError, `not a KeployError: ${err}`); + assert.match(err.message, re); + return true; + }); +} + +test('the client sends the token and the protocol body, and takes v1 answers', async () => { + const a = await agent(({ path: p }) => ({ body: p === '/agent/instance' ? { v: 1, instance: 'run', timeFile: '/tmp/k/t' } : { v: 1, status: 'ok' } })); + try { + const c = new AgentClient(a.url + '/', 'tok'); + await c.claim('chromium', '/proj'); + await c.begin({ name: 'g › a', file: 'tests/a.spec.ts', project: 'chromium' }); + await c.end({ name: 'g › a', file: 'tests/a.spec.ts', project: 'chromium' }, 'passed'); + assert.deepEqual(a.calls.map((x) => [x.path, x.auth, x.body]), [ + ['/agent/instance', 'Bearer tok', { key: 'chromium', dir: '/proj' }], + ['/agent/scope/begin', 'Bearer tok', { name: 'g › a', file: 'tests/a.spec.ts', project: 'chromium', canSkip: true }], + ['/agent/scope/end', 'Bearer tok', { name: 'g › a', file: 'tests/a.spec.ts', project: 'chromium', outcome: 'passed' }], + ]); + } finally { await a.close(); } +}); + +test('no token: no authorization header', async () => { + const a = await agent(() => ({ body: { v: 1, status: 'ok' } })); + try { + await new AgentClient(a.url, '').begin({ name: 'a', file: 'a.spec.ts' }); + assert.equal(a.calls[0].auth, ''); + } finally { await a.close(); } +}); + +test('every answer the client cannot trust is an error', async () => { + const cases = [ + [{ body: { status: 'ok' } }, /names no scope protocol version.*upgrade keploy/], + [{ body: { v: 2, status: 'ok' } }, /scope protocol v2.*same release/], + [{ body: 'not json' }, /cannot be read/], + [{ body: '[1]' }, /cannot be read \(not an object\)/], + [{ body: 'null' }, /cannot be read \(not an object\)/], + [{ status: 422, body: { error: 'one_test_at_a_time', message: 'b began while a was still open', remedy: 'run them one at a time' } }, + /^keploy: b began while a was still open \(run them one at a time\)$/], + [{ status: 400, body: { error: 'bad_request', message: 'invalid' } }, /^keploy: invalid$/], + [{ status: 500, body: 'boom' }, /answered 500 .* to \/agent\/scope\/begin: "boom"/], + [{ status: 302, body: '' }, /answered 302/], + [{ body: { v: 1, status: 'busy' } }, /begin of a with status "busy"/], + [{ body: { v: 1, status: 'ok', action: 'fail' } }, /begin of a with the unknown action "fail"/], + ]; + for (const [answer, re] of cases) { + const a = await agent(() => answer); + try { + await rejects(new AgentClient(a.url, '').begin({ name: 'a', file: 'a.spec.ts' }), re); + } finally { await a.close(); } + } +}); + +test('an end takes only fail as its action', async () => { + const a = await agent(() => ({ body: { v: 1, status: 'ok', action: 'skip' } })); + try { + await rejects(new AgentClient(a.url, '').end({ name: 'a', file: 'a.spec.ts' }, 'passed'), /end of a with the unknown action "skip"/); + } finally { await a.close(); } +}); + +test('a claim without an instance is refused', async () => { + const a = await agent(() => ({ body: { v: 1 } })); + try { + await rejects(new AgentClient(a.url, '').claim('p', '/d'), /without an instance/); + } finally { await a.close(); } +}); + +test('an unreachable agent is an error', async () => { + const a = await agent(() => ({ body: {} })); + await a.close(); + await rejects(new AgentClient(a.url, '').begin({ name: 'a', file: 'a.spec.ts' }), /cannot reach the keploy agent/); +}); + +test('an answer over 1 MiB is an error', async () => { + const a = await agent(() => ({ body: JSON.stringify({ v: 1, status: 'ok', reason: 'x'.repeat(1 << 20) }) })); + try { + await rejects(new AgentClient(a.url, '').begin({ name: 'a', file: 'a.spec.ts' }), /larger than 1048576 bytes/); + } finally { await a.close(); } +}); + +test('given: the variable comes back; an agent without the route says to match releases', async () => { + let a = await agent(() => ({ body: { v: 1, status: 'ok', variable: 'given:session' } })); + try { + assert.equal(await new AgentClient(a.url, '').given('session', 'tok-1', 'tests/a.spec.ts:3'), 'given:session'); + assert.deepEqual(a.calls[0].body, { name: 'session', value: 'tok-1', source: 'tests/a.spec.ts:3' }); + } finally { await a.close(); } + a = await agent(() => ({ status: 404, body: '404 page not found' })); + try { + await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /does not take a test's given values.*same release/); + } finally { await a.close(); } + a = await agent(() => ({ status: 422, body: { error: 'not_open', message: 'a value was given while no test was open' } })); + try { + await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /^keploy: a value was given while no test was open$/); + } finally { await a.close(); } + for (const body of [{ v: 1, status: 'ok' }, { v: 1, status: 'ok', variable: '' }, { v: 1, status: 'busy', variable: 'given:session' }]) { + a = await agent(() => ({ body })); + try { + await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /answered the given value session with/); + } finally { await a.close(); } + } +}); + +test('decide: inert, refused, active', () => { + const never = () => { throw new Error('runsInside called'); }; + assert.deepEqual(decide({}, 10, never), { mode: 'inert' }); + assert.deepEqual(decide({ KEPLOY_MOCK_AGENT: ' OFF ' }, 10, never), { mode: 'inert' }); + assert.equal(decide({ KEPLOY_MOCK_AGENT: 'localhost:16789' }, 10, never).mode, 'refused'); + assert.equal(decide({ KEPLOY_MOCK_AGENT: 'ftp://h' }, 10, never).mode, 'refused'); + assert.deepEqual(decide({ KEPLOY_MOCK_AGENT: 'http://127.0.0.1:16789/', KEPLOY_MOCK_AGENT_TOKEN: 't', KEPLOY_MOCK_PROJECT: '/p' }, 10, never), + { mode: 'active', url: 'http://127.0.0.1:16789', token: 't', project: '/p' }); + // A process inside the claimer (the browser, a helper) stays inert; a + // marker naming no ancestor, or this process, is no claim of another. + const env = { KEPLOY_MOCK_AGENT: 'http://a:1', KEPLOY_TEST_PID: '7' }; + assert.deepEqual(decide(env, 10, (p) => p === 7), { mode: 'inert' }); + assert.equal(decide(env, 10, () => false).mode, 'active'); + assert.equal(decide({ ...env, KEPLOY_TEST_PID: '10' }, 10, never).mode, 'active'); + assert.equal(decide({ ...env, KEPLOY_TEST_PID: 'x' }, 10, never).mode, 'active'); +}); + +test('testRef: the file relative to the project, the titles after it, the project when named', () => { + const ti = { file: path.join(process.cwd(), 'tests', 'a.spec.ts'), titlePath: ['a.spec.ts', 'group', 'inner', 'title'], project: { name: 'chromium' } }; + assert.deepEqual(testRef(ti, process.cwd()), { name: 'group › inner › title', file: 'tests/a.spec.ts', project: 'chromium' }); + assert.deepEqual(testRef({ ...ti, project: { name: '' } }, process.cwd()), { name: 'group › inner › title', file: 'tests/a.spec.ts' }); +}); + +test('outcomeOf: the test\'s own verdict', () => { + assert.equal(outcomeOf({ status: 'passed', expectedStatus: 'passed' }), 'passed'); + assert.equal(outcomeOf({ status: 'failed', expectedStatus: 'passed' }), 'failed'); + assert.equal(outcomeOf({ status: 'timedOut', expectedStatus: 'passed' }), 'failed'); + assert.equal(outcomeOf({ status: 'interrupted', expectedStatus: 'passed' }), 'failed'); + assert.equal(outcomeOf({ status: 'skipped', expectedStatus: 'skipped' }), 'skipped'); + assert.equal(outcomeOf({ status: 'skipped', expectedStatus: 'passed' }), 'skipped'); + // test.fail(): failing is its pass, passing its failure. + assert.equal(outcomeOf({ status: 'failed', expectedStatus: 'failed' }), 'passed'); + assert.equal(outcomeOf({ status: 'passed', expectedStatus: 'failed' }), 'failed'); +}); + +test('sourceOf: the first frame outside the package, relative to the project', () => { + const own = path.resolve('dist', 'index.js'); + const stack = ['Error', ` at given (${own}:10:5)`, ' at /proj/tests/a.spec.ts:12:7', ' at node:internal/x:1:1'].join('\n'); + assert.equal(sourceOf(stack, '/proj'), 'tests/a.spec.ts:12'); + assert.equal(sourceOf(['Error', ' at Object. (/proj/t/b.spec.ts:3:1)'].join('\n'), '/proj'), 't/b.spec.ts:3'); + assert.equal(sourceOf(['Error', ' at x (file:///proj/t/c.spec.mjs:4:2)'].join('\n'), '/proj'), 't/c.spec.mjs:4'); + assert.equal(sourceOf('Error\n at ', '/proj'), ''); +}); diff --git a/playwright/tsconfig.json b/playwright/tsconfig.json new file mode 100644 index 0000000..d9e9d79 --- /dev/null +++ b/playwright/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2022", "DOM"], + "declaration": true, + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src"] +} diff --git a/tsconfig.json b/tsconfig.json index 0801287..961f8da 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -97,5 +97,7 @@ /* Completeness */ // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ "skipLibCheck": true /* Skip type checking all .d.ts files. */ - } + }, + // @keploy/playwright (playwright/) is its own package, with its own build. + "exclude": ["node_modules", "dist", "playwright"] } From 5e66b57bdcf1c5457d1b4b45a38a613382e9c52a Mon Sep 17 00:00:00 2001 From: slayerjain Date: Sun, 1 Nov 2026 00:17:23 +0530 Subject: [PATCH 02/14] fix(playwright): show keploy's reason once, and a fallback when it gives none keploy's judging reason already starts with "keploy: ", and the end of a test prefixed it again: a replay of enterprise-ui's smart-set-pagination failed its two Stripe-beacon tests with "keploy: keploy: GET /v3/.deploy_status_henson.json to js.stripe.com had no recorded answer". A skip reason ("/: ") carries no prefix, so the skip still adds it. keployText now prefixes a reason only when it lacks "keploy: ", and shows the same fallback texts go-sdk's keploytest shows when the agent gives no reason. Rows: the judged-red row uses the agent's real reason shape and refuses a doubled prefix; keployText's unit row covers the prefixed, unprefixed and empty reasons. Mutants prefix-always, prefix-never and no-fallback. Signed-off-by: slayerjain --- playwright/src/index.ts | 15 +++++++++++++-- playwright/test/rows.test.mjs | 6 ++++-- playwright/test/unit.test.mjs | 11 ++++++++++- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/playwright/src/index.ts b/playwright/src/index.ts index 53430cd..76a4849 100644 --- a/playwright/src/index.ts +++ b/playwright/src/index.ts @@ -102,14 +102,14 @@ export const test = base.extend({ const b = await c.begin(ref); if (b.action === ACTION_SKIP) { // The framework's own skip, with keploy's reason; no end follows. - testInfo.skip(true, `keploy: ${b.reason ?? ''}`); + testInfo.skip(true, keployText(b.reason, 'keploy: set aside (the agent gave no reason)')); return; } open = ref; await use(); open = null; const e = await c.end(ref, outcomeOf(testInfo)); - if (e.action === ACTION_FAIL) throw new KeployError(`keploy: ${e.reason ?? ''}`); + if (e.action === ACTION_FAIL) throw new KeployError(keployText(e.reason, 'keploy: the keploy agent failed this test and gave no reason')); }, { auto: true }], }); @@ -144,6 +144,17 @@ export function testRef(ti: Pick, di return project ? { name, file, project } : { name, file }; } +/** + * keploy's reason as the test shows it: once prefixed with "keploy: " (a + * judging reason from the agent already is; a skip reason, "/: + * ", is not), or fallback when the agent gave none, as go-sdk's + * keploytest shows it. + */ +export function keployText(reason: string | undefined, fallback: string): string { + if (!reason) return fallback; + return reason.startsWith('keploy: ') ? reason : `keploy: ${reason}`; +} + /** The test's own verdict, as keploy's end takes it. */ export function outcomeOf(ti: Pick): Outcome { if (ti.status === 'skipped') return 'skipped'; diff --git a/playwright/test/rows.test.mjs b/playwright/test/rows.test.mjs index bedf46d..8dcead3 100644 --- a/playwright/test/rows.test.mjs +++ b/playwright/test/rows.test.mjs @@ -143,7 +143,7 @@ test('keploy skips a set-aside test with its reason, fails a judged-red one, and const j = journal(); const a = await fakeAgent(j, (c) => { if (c.path === '/agent/scope/begin' && c.body.name === 'set aside') return { body: { v: 1, status: 'ok', action: 'skip', reason: 'flaky/request_changed: the body changed' } }; - if (c.path === '/agent/scope/end' && c.body.name === 'judged red') return { body: { v: 1, status: 'ok', action: 'fail', reason: 'GET /company: no recorded answer' } }; + if (c.path === '/agent/scope/end' && c.body.name === 'judged red') return { body: { v: 1, status: 'ok', action: 'fail', reason: 'keploy: GET /company had no recorded answer' } }; if (c.path === '/agent/scope/begin' && c.body.name === 'refused') return { status: 422, body: { error: 'one_test_at_a_time', message: 'refused began while x was open', remedy: 'run one at a time' } }; return undefined; }); @@ -153,7 +153,9 @@ test('keploy skips a set-aside test with its reason, fails a judged-red one, and assert.deepEqual(tests['set aside'].annotations.filter((x) => x.type === 'skip').map((x) => x.description), ['keploy: flaky/request_changed: the body changed']); assert.ok(!fs.existsSync(path.join(tmp, 'set-aside-ran')), 'the set-aside test ran'); assert.equal(tests['judged red'].status, 'failed'); - assert.match(tests['judged red'].error, /keploy: GET \/company: no recorded answer/); + // The agent's judging reason already starts with "keploy: "; the test shows it once (gap13 showed it twice). + assert.match(tests['judged red'].error, /keploy: GET \/company had no recorded answer/); + assert.doesNotMatch(tests['judged red'].error, /keploy: keploy:/); assert.equal(tests.refused.status, 'failed'); assert.match(tests.refused.error, /keploy: refused began while x was open \(run one at a time\)/); assert.ok(!fs.existsSync(path.join(tmp, 'refused-ran')), 'the refused test ran'); diff --git a/playwright/test/unit.test.mjs b/playwright/test/unit.test.mjs index a5b7065..56d5d70 100644 --- a/playwright/test/unit.test.mjs +++ b/playwright/test/unit.test.mjs @@ -6,7 +6,7 @@ import http from 'node:http'; import path from 'node:path'; import { AgentClient, KeployError } from '../dist/agent.js'; import { decide } from '../dist/join.js'; -import { testRef, outcomeOf, sourceOf } from '../dist/index.js'; +import { testRef, outcomeOf, sourceOf, keployText } from '../dist/index.js'; // One fake agent answering every call with answer(path, body). async function agent(answer) { @@ -173,3 +173,12 @@ test('sourceOf: the first frame outside the package, relative to the project', ( assert.equal(sourceOf(['Error', ' at x (file:///proj/t/c.spec.mjs:4:2)'].join('\n'), '/proj'), 't/c.spec.mjs:4'); assert.equal(sourceOf('Error\n at ', '/proj'), ''); }); + +test('keployText: keploy\'s reason once prefixed, or the fallback when it gave none', () => { + assert.equal(keployText('keploy: GET /a had no recorded answer', 'fb'), 'keploy: GET /a had no recorded answer'); + assert.equal(keployText('flaky/request_changed: the body changed', 'fb'), 'keploy: flaky/request_changed: the body changed'); + assert.equal(keployText('', 'keploy: fb'), 'keploy: fb'); + assert.equal(keployText(undefined, 'keploy: fb'), 'keploy: fb'); + // Only the whole word with its separator counts as keploy's prefix. + assert.equal(keployText('keploy:x', 'fb'), 'keploy: keploy:x'); +}); From 32e6550b63f97049b9074d002294349e1e2f722a Mon Sep 17 00:00:00 2001 From: slayerjain Date: Sun, 1 Nov 2026 01:25:05 +0530 Subject: [PATCH 03/14] ci: run the type check with actions/setup-node and the repo's package manager The Check PR workflow failed on every pull request at its first step: borales/actions-yarn@v2.1.0 runs a docker image with no yarn in it ("sh: yarn: not found"). The repo's package manager is npm (package-lock.json, no yarn.lock), so the check now installs with actions/setup-node@v4 (Node.js 22, npm cache) and runs npm ci and npm run build. actions/checkout moves to v4 with it. Signed-off-by: slayerjain --- .github/workflows/ci.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34c1092..7825830 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,24 +12,24 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 + # The repo's package manager is npm (package-lock.json, no yarn.lock). + # borales/actions-yarn@v2.1.0 ran a docker image with no yarn in it + # ("sh: yarn: not found"), so this check failed on every PR. - name: Set up Node - uses: actions/setup-node@v2 + uses: actions/setup-node@v4 with: - node-version: 14 + node-version: 22 + cache: npm - name: Install Dependencies - uses: borales/actions-yarn@v2.1.0 - with: - cmd: install + run: npm ci - name: Build Project - uses: borales/actions-yarn@v2.1.0 - with: - cmd: build + run: npm run build - name: Check commits messages uses: wagoid/commitlint-github-action@v5 From 0ba76693f3526d918623dd68a2927400476cf3f7 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Sun, 1 Nov 2026 01:25:05 +0530 Subject: [PATCH 04/14] ci(playwright): run the rows on Node.js 20, 22 and 24, each leg on its own The Node 18 leg failed at "npx playwright install": Playwright 1.62, the rows' version, needs Node.js 20 or later. Its failure then cancelled the Node 22 leg, whose 24 rows had all passed, through the matrix's default fail-fast. The matrix is now 20, 22 and 24 (the active LTS) with fail-fast off, so one leg's result never hides another's. package.json's engines says node >=20 to match, and the README says it. Signed-off-by: slayerjain --- .github/workflows/playwright-sdk.yml | 7 ++++++- playwright/README.md | 2 ++ playwright/package-lock.json | 2 +- playwright/package.json | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/playwright-sdk.yml b/.github/workflows/playwright-sdk.yml index dbd5f8e..ae3aedf 100644 --- a/.github/workflows/playwright-sdk.yml +++ b/.github/workflows/playwright-sdk.yml @@ -16,8 +16,13 @@ jobs: name: build and rows (Node ${{ matrix.node }}) runs-on: ubuntu-latest strategy: + # One leg failing must not cancel the others: each Node version's + # result is its own. + fail-fast: false matrix: - node: [18, 22] + # Playwright 1.62 (the rows' version) needs Node.js 20 or later, and + # so does this package (package.json engines). + node: [20, 22, 24] defaults: run: working-directory: playwright diff --git a/playwright/README.md b/playwright/README.md index 869723b..39123cd 100644 --- a/playwright/README.md +++ b/playwright/README.md @@ -10,6 +10,8 @@ This package tells keploy where each test begins and ends. Through it, keploy sk npm install --save-dev @keploy/playwright ``` +It needs Node.js 20 or later. Its rows run on Node.js 20, 22 and 24. + Import `test` and `expect` from this package instead of `@playwright/test`. A fixture file that extends `test` extends this one: ```ts diff --git a/playwright/package-lock.json b/playwright/package-lock.json index 517296d..347cd8c 100644 --- a/playwright/package-lock.json +++ b/playwright/package-lock.json @@ -14,7 +14,7 @@ "typescript": "5.9.3" }, "engines": { - "node": ">=18" + "node": ">=20" }, "peerDependencies": { "@playwright/test": ">=1.40.0" diff --git a/playwright/package.json b/playwright/package.json index a8b46b7..b41ecb9 100644 --- a/playwright/package.json +++ b/playwright/package.json @@ -27,6 +27,6 @@ "typescript": "5.9.3" }, "engines": { - "node": ">=18" + "node": ">=20" } } From 81798636730f24bdff3bae8a644aa98f71a78b71 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Sun, 1 Nov 2026 01:26:34 +0530 Subject: [PATCH 05/14] ci(playwright): publish @keploy/playwright to npm from a playwright-v tag A tag playwright-vX.Y.Z runs the rows on the tagged commit (every Node.js leg, through playwright-sdk.yml as a called workflow) and then publishes playwright/ with npm publish --access public --provenance, from GitHub Actions with the repository secret NPM_TOKEN. The publish job refuses, each with a named error, before anything is uploaded: - a tag that does not name package.json's version exactly, or a version that is not a release x.y.z (npm 10 would make a prerelease the package's latest): scripts/release-version.mjs, with rows; - an empty NPM_TOKEN; - an @keploy:registry that npm would use other than registry.npmjs.org (read with every npmrc in it; setup-node pins it); - a token npm whoami cannot authenticate. npm itself refuses a version it already has. package.json gains prepublishOnly: npm run build, so a publish from a fresh checkout (no dist/) carries its build. On pull requests a dry-run job runs the same npm publish with no token and fails when the package would ship without dist/index.js, dist/index.d.ts, package.json or README.md. Signed-off-by: slayerjain --- .github/workflows/playwright-publish.yml | 99 ++++++++++++++++++++++++ .github/workflows/playwright-sdk.yml | 2 + playwright/package.json | 1 + playwright/scripts/release-version.mjs | 30 +++++++ playwright/test/release.test.mjs | 39 ++++++++++ 5 files changed, 171 insertions(+) create mode 100644 .github/workflows/playwright-publish.yml create mode 100644 playwright/scripts/release-version.mjs create mode 100644 playwright/test/release.test.mjs diff --git a/.github/workflows/playwright-publish.yml b/.github/workflows/playwright-publish.yml new file mode 100644 index 0000000..61d5730 --- /dev/null +++ b/.github/workflows/playwright-publish.yml @@ -0,0 +1,99 @@ +name: "@keploy/playwright publish" + +# Publishes @keploy/playwright to npm from a tag. +# +# To release X.Y.Z: set "version": "X.Y.Z" in playwright/package.json on main +# (a pull request, so the dry run below checks it), then push the tag +# playwright-vX.Y.Z on that commit. The rows run on the tagged commit first; +# the publish runs only when every leg passes, and refuses a tag that does not +# name package.json's version (scripts/release-version.mjs). npm itself +# refuses a version it already has. +# +# On a pull request the same package is published as a dry run, with no +# token, so a package that would ship without its build shows before a tag. +on: + push: + tags: ["playwright-v*"] + pull_request: + paths: + - "playwright/**" + - ".github/workflows/playwright-publish.yml" + - ".github/workflows/playwright-sdk.yml" + +permissions: + contents: read + +jobs: + rows: + if: github.event_name == 'push' + uses: ./.github/workflows/playwright-sdk.yml + + publish: + name: publish to npm + if: github.event_name == 'push' + needs: rows + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # npm --provenance signs the package with this run's identity + defaults: + run: + shell: bash # -eo pipefail: a failed command before a pipe fails the step + working-directory: playwright + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + # Writes the token line for registry.npmjs.org and pins the @keploy + # scope to it (@keploy:registry), in the runner's own npmrc. + registry-url: https://registry.npmjs.org + scope: "@keploy" + - name: The tag names package.json's version + run: node scripts/release-version.mjs "$GITHUB_REF_NAME" + - name: The npm token is set + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + run: test -n "$NPM_TOKEN" || { echo "::error::the NPM_TOKEN secret is not set; refusing to publish"; exit 1; } + - name: The @keploy scope publishes to registry.npmjs.org + # npm sends a scoped package to its scope's registry, whatever + # --registry says. Read the setting npm will use, with every npmrc in + # it (a project one included), and refuse any other. + run: | + got=$(npm config get @keploy:registry) + test "$got" = https://registry.npmjs.org/ || { echo "::error::@keploy:registry is '$got', not https://registry.npmjs.org/; refusing to publish"; exit 1; } + - run: npm ci + - name: The token authenticates + # Fails here, named, on a token that is expired, revoked or pasted + # with a stray character, before anything is uploaded. + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm whoami --registry https://registry.npmjs.org/ + - name: Publish + # prepublishOnly builds dist/ first. + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish --access public --provenance + + dry-run: + name: publish dry run + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + defaults: + run: + shell: bash # -eo pipefail: a failed command before a pipe fails the step + working-directory: playwright + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: npm ci + - name: Publish, dry run + run: npm publish --dry-run --access public 2>&1 | tee "$RUNNER_TEMP/publish.log" + - name: The package carries its build + # A fresh checkout has no dist/; only prepublishOnly puts it there. + run: | + for f in dist/index.js dist/index.d.ts package.json README.md; do + grep -qE "npm notice [0-9.]+[kMG]?B +$f\$" "$RUNNER_TEMP/publish.log" || { echo "::error::the package would ship without $f"; exit 1; } + done diff --git a/.github/workflows/playwright-sdk.yml b/.github/workflows/playwright-sdk.yml index ae3aedf..061954d 100644 --- a/.github/workflows/playwright-sdk.yml +++ b/.github/workflows/playwright-sdk.yml @@ -10,6 +10,8 @@ on: paths: - "playwright/**" - ".github/workflows/playwright-sdk.yml" + # playwright-publish.yml runs these rows on a release tag before it publishes. + workflow_call: jobs: rows: diff --git a/playwright/package.json b/playwright/package.json index b41ecb9..7959b42 100644 --- a/playwright/package.json +++ b/playwright/package.json @@ -16,6 +16,7 @@ ], "scripts": { "build": "tsc -p tsconfig.json", + "prepublishOnly": "npm run build", "test": "npm run build && node --test test/*.test.mjs" }, "peerDependencies": { diff --git a/playwright/scripts/release-version.mjs b/playwright/scripts/release-version.mjs new file mode 100644 index 0000000..84c67f0 --- /dev/null +++ b/playwright/scripts/release-version.mjs @@ -0,0 +1,30 @@ +// release-version.mjs TAG: the publish workflow's version check. +// +// TAG is the pushed git tag. It must name package.json's version exactly +// (playwright-v), so the version on npm is always the one a tagged +// commit carries. The version must be a release (x.y.z): npm 10 would make a +// prerelease the package's `latest`, so this workflow refuses one. +// +// Exit 0 with no output when the tag may publish; exit 1 and say why when not. +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +export function refusal(tag, version) { + if (!/^\d+\.\d+\.\d+$/.test(version)) { + return `package.json's version ${version} is not a release (x.y.z); this workflow publishes releases only`; + } + const want = `playwright-v${version}`; + if (tag !== want) { + return `the tag ${tag || '(none)'} does not name package.json's version ${version}: the tag for it is ${want}`; + } + return ''; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); + const why = refusal(process.argv[2] ?? '', pkg.version); + if (why) { + console.error(`refusing to publish: ${why}`); + process.exit(1); + } +} diff --git a/playwright/test/release.test.mjs b/playwright/test/release.test.mjs new file mode 100644 index 0000000..cc2b7f6 --- /dev/null +++ b/playwright/test/release.test.mjs @@ -0,0 +1,39 @@ +// Rows for the publish workflow's version check (scripts/release-version.mjs). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { refusal } from '../scripts/release-version.mjs'; + +const script = fileURLToPath(new URL('../scripts/release-version.mjs', import.meta.url)); +const { version } = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); +const run = (...args) => spawnSync(process.execPath, [script, ...args], { encoding: 'utf8' }); + +test('a tag naming the version publishes', () => { + assert.equal(refusal('playwright-v1.2.3', '1.2.3'), ''); +}); + +test('a tag naming another version is refused, and names the right tag', () => { + assert.match(refusal('playwright-v1.2.4', '1.2.3'), /does not name package\.json's version 1\.2\.3: the tag for it is playwright-v1\.2\.3$/); + assert.match(refusal('playwright-v1.2.3.1', '1.2.3'), /does not name/); + assert.match(refusal('v1.2.3', '1.2.3'), /does not name/); + assert.match(refusal('', '1.2.3'), /the tag \(none\) does not name/); +}); + +test('a prerelease version is refused, even with its own tag', () => { + assert.match(refusal('playwright-v1.2.3-rc.1', '1.2.3-rc.1'), /is not a release \(x\.y\.z\)/); + assert.match(refusal('playwright-v1.2', '1.2'), /is not a release/); +}); + +test('the script reads package.json and exits 1 with the reason on a wrong tag', () => { + const ok = run(`playwright-v${version}`); + assert.equal(ok.status, 0, ok.stderr); + assert.equal(ok.stdout + ok.stderr, ''); + const bad = run('playwright-v0.0.0-not-this'); + assert.equal(bad.status, 1); + assert.match(bad.stderr, new RegExp(`^refusing to publish: the tag playwright-v0\\.0\\.0-not-this does not name package\\.json's version ${version.replace(/\./g, '\\.')}`)); + const none = run(); + assert.equal(none.status, 1); + assert.match(none.stderr, /the tag \(none\)/); +}); From 1c797cebdd3ea71012b7f65290a65cd92cbbb8dc Mon Sep 17 00:00:00 2001 From: slayerjain Date: Sun, 1 Nov 2026 02:34:29 +0530 Subject: [PATCH 06/14] fix(playwright): row every settle rule and agent check; drop a frame rule the browser covers Every behaviour the README states now has a row that fails without it. Settle, against the rows' page server with Chromium: - a navigation answered 205 and a download (a navigation answered with content-disposition attachment) replace no document, so a call in flight across either is still waited for (the test times out naming it); - a call whose page closes is not waited for (the test passes with no wait step); nor is a call whose frame is removed, here a cross-site frame (localhost under a 127.0.0.1 page), as a payment form's is; - a navigation's own request stays in flight after its document commits: a document still loading is waited for, by name. The agent client: - a 307 from the agent is an error, and the call is not sent again to where it points; - an end answered with a status other than ok is an error (begin's was rowed, end's was not). Names: - an unnamed Playwright project claims with the key "." (the agent refuses an empty one) and its tests name no project; - a spec reached through a symbolic link is named by its real path. The page server gains /resetcontent (205), /download (an attachment) and /slowdoc (a document whose last byte waits for /release); the fixture config takes PW_UNNAMED=1 for an unnamed project. The removed-frame rule is gone. Settle dropped a removed frame's calls itself, but Chromium ends them as failed requests: the removed-frame row passes with that rule taken out, for a same-site frame and for a cross-site one alike. A rule no row can fail is surface with no proof, so it goes; the row stays and proves the behaviour. A closed page's calls get no end from the browser, and that rule stays (its row fails without it). Of the review's 17 mutants of the package (16 compile), 9 got past every row. 8 of them are now each killed: 205, attachment, page closed, the navigation request kept on commit, redirect followed, end's status, the "." key and the linked spec. The ninth, frame removed, is the rule removed above. Proof: rows and mutants on Node.js 22, rows on 20 and 24, in node:-bookworm on a test box. Signed-off-by: slayerjain --- playwright/src/settle.ts | 3 +- playwright/test/fixtures/playwright.config.ts | 4 +- playwright/test/fixtures/settle.spec.ts | 63 +++++++++++++++++ playwright/test/rows.test.mjs | 70 ++++++++++++++++++- playwright/test/unit.test.mjs | 31 +++++++- 5 files changed, 167 insertions(+), 4 deletions(-) diff --git a/playwright/src/settle.ts b/playwright/src/settle.ts index cdec953..a5fa911 100644 --- a/playwright/src/settle.ts +++ b/playwright/src/settle.ts @@ -42,7 +42,8 @@ export class Inflight { if (!loading.delete(f)) return; this.drop((o) => o.frame === f && !o.nav); }); - p.on('framedetached', (f) => { loading.delete(f); this.drop((o) => o.frame === f); }); + // Nor for the calls of a page that closes. (A removed frame's calls + // it does end, as failed requests.) p.on('close', () => this.drop((o) => o.frame !== null && pageOf(o.frame) === p)); }; context.pages().forEach(watch); diff --git a/playwright/test/fixtures/playwright.config.ts b/playwright/test/fixtures/playwright.config.ts index 5aa0baa..bde48ea 100644 --- a/playwright/test/fixtures/playwright.config.ts +++ b/playwright/test/fixtures/playwright.config.ts @@ -8,5 +8,7 @@ export default defineConfig({ timeout: Number(process.env.PW_TIMEOUT || 30000), reporter: [['json'], ['./steps-reporter.cjs']], outputDir: process.env.PW_OUTPUT_DIR || 'test-results', - projects: [{ name: 'p1', use: { headless: true } }], + // PW_UNNAMED=1: one unnamed project, Playwright's default when a config + // lists none. + projects: [{ name: process.env.PW_UNNAMED ? '' : 'p1', use: { headless: true } }], }); diff --git a/playwright/test/fixtures/settle.spec.ts b/playwright/test/fixtures/settle.spec.ts index e46012d..6b0022d 100644 --- a/playwright/test/fixtures/settle.spec.ts +++ b/playwright/test/fixtures/settle.spec.ts @@ -38,3 +38,66 @@ test('a call in flight across a navigation answered 204', async ({ page }) => { await page.evaluate(() => { history.pushState({}, '', '/page?route'); }); await routed; }); +test('a call in flight across a navigation answered 205', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + const answered = page.waitForResponse(/\/resetcontent$/); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + // A 205 answer to a navigation leaves the document in place, as a 204 does. + await page.evaluate(() => { location.href = '/resetcontent'; }); + await answered; + const routed = page.waitForEvent('framenavigated'); + await page.evaluate(() => { history.pushState({}, '', '/page?route'); }); + await routed; +}); +test('a call in flight across a download', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + const answered = page.waitForResponse(/\/download$/); + const downloaded = page.waitForEvent('download'); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + // A navigation answered with an attachment is a download: the document stays. + await page.evaluate(() => { location.href = '/download'; }); + await answered; + await downloaded; + const routed = page.waitForEvent('framenavigated'); + await page.evaluate(() => { history.pushState({}, '', '/page?route'); }); + await routed; +}); +test('a call its removed cross-site frame had in flight', async ({ page }) => { + // A frame from another site (localhost under a 127.0.0.1 page), as a + // payment form's is. The browser ends the calls it had in flight when it is + // removed. + await page.goto(`${base}/page`); + const attached = page.waitForEvent('frameattached'); + await page.evaluate((src) => new Promise((resolve) => { + const f = document.createElement('iframe'); + f.id = 'child'; + f.onload = () => resolve(); + f.src = src; + document.body.appendChild(f); + }), `${base.replace('//127.0.0.1:', '//localhost:')}/page?child`); + const child = await attached; + const sent = page.waitForRequest(/\/slow$/); + const detached = page.waitForEvent('framedetached'); + await child.evaluate(() => { void fetch('/slow'); }); + await sent; + await page.evaluate(() => { document.getElementById('child')?.remove(); }); + await detached; +}); +test('a call its closed page had in flight', async ({ page, context }) => { + await page.goto(`${base}/page`); + const other = await context.newPage(); + await other.goto(`${base}/page?other`); + const sent = other.waitForRequest(/\/slow$/); + await other.evaluate(() => { void fetch('/slow'); }); + await sent; + await other.close(); +}); +test('a document still loading after its frame commits', async ({ page }) => { + // The navigation request itself stays in flight until the document's last + // byte: committing the new document cancels only the old document's calls. + await page.goto(`${base}/slowdoc`, { waitUntil: 'commit' }); +}); diff --git a/playwright/test/rows.test.mjs b/playwright/test/rows.test.mjs index 8dcead3..b6eb8b8 100644 --- a/playwright/test/rows.test.mjs +++ b/playwright/test/rows.test.mjs @@ -43,7 +43,8 @@ async function fakeAgent(j, answer = () => undefined) { return { url: `http://127.0.0.1:${srv.address().port}`, close: () => new Promise((r) => { srv.closeAllConnections(); srv.close(r); }) }; } -// The settle rows' page server: /slow is answered only once /release is called. +// The settle rows' page server: /slow and the end of /slowdoc are answered +// only once /release is called. async function pageServer(j) { let release = null; const released = new Promise((r) => { release = r; }); @@ -51,6 +52,16 @@ async function pageServer(j) { if (req.url === '/page') { res.writeHead(200, { 'content-type': 'text/html' }); res.end('page'); return; } if (req.url === '/release') { release(); res.end('ok'); return; } if (req.url === '/nocontent') { res.writeHead(204); res.end(); return; } + if (req.url === '/resetcontent') { res.writeHead(205); res.end(); return; } + if (req.url === '/download') { res.writeHead(200, { 'content-type': 'application/octet-stream', 'content-disposition': 'attachment; filename="d.bin"' }); res.end('bytes'); return; } + if (req.url === '/slowdoc') { + // A document whose last byte comes only once /release is called. + j.add({ page: 'slowdoc sent' }); + res.writeHead(200, { 'content-type': 'text/html' }); + res.write('loading'); + void released.then(() => res.end('')); + return; + } if (req.url === '/slow') { j.add({ page: 'slow sent' }); res.on('close', () => { if (!res.writableFinished) j.add({ page: 'slow cut' }); }); @@ -261,3 +272,60 @@ test('settle: a navigation answered 204 replaces no document, so a call in fligh assert.ok(t.steps.some((s) => /^keploy: wait for the test's 1 call\(s\) in flight .*\/slow$/.test(s)), t.steps.join('\n')); } finally { await p.close(); await a.close(); } }); + +test('an unnamed Playwright project claims with the key ".", and its tests name no project', async () => { + const j = journal(); + const a = await fakeAgent(j); + try { + const { tests, out } = await playwright(['agent-says.spec.ts', '-g', 'judged red'], { ...keployEnv(a.url), PW_UNNAMED: '1' }); + assert.equal(tests['judged red'].status, 'passed', out); + const r = { name: 'judged red', file: 'test/fixtures/agent-says.spec.ts' }; + assert.deepEqual(calls(j).map((c) => [c.path, c.body]), [ + ['/agent/instance', { key: '.', dir: fs.realpathSync(PKG) }], + ['/agent/scope/begin', { ...r, canSkip: true }], + ['/agent/scope/end', { ...r, outcome: 'passed' }], + ]); + } finally { await a.close(); } +}); + +// Settle rows where the browser cancelled a call: the test passes at once, +// with no wait step. The browser ends a removed frame's calls itself; a +// closed page's it does not, and the package drops them. +for (const [title, why] of [ + ['a call its removed cross-site frame had in flight', 'its cross-site frame is removed'], + ['a call its closed page had in flight', 'its page closes'], +]) { + test(`settle: a call is not waited for once ${why}`, async () => { + const j = journal(); + const a = await fakeAgent(j); + const p = await pageServer(j); + try { + const { tests, out } = await playwright(['settle.spec.ts', '-g', title], { ...keployEnv(a.url), SLOW_BASE: p.url, NO_RELEASE: '1', PW_TIMEOUT: '20000' }); + const t = tests[title]; + assert.equal(t.status, 'passed', out); + assert.deepEqual(t.steps.filter((s) => s.startsWith('keploy:')), []); + assert.ok(j.events.some((e) => e.page === 'slow sent')); + } finally { await p.close(); await a.close(); } + }); +} + +// Settle rows where nothing was cancelled: the call is still waited for, and +// the test times out naming it. +for (const [title, why, call] of [ + ['a call in flight across a navigation answered 205', 'a navigation answered 205 replaces no document', '/slow'], + ['a call in flight across a download', 'a navigation answered with an attachment (a download) replaces no document', '/slow'], + ['a document still loading after its frame commits', 'a navigation\'s own request stays in flight after its document commits', '/slowdoc'], +]) { + test(`settle: ${why}, so the call is still waited for`, async () => { + const j = journal(); + const a = await fakeAgent(j); + const p = await pageServer(j); + try { + const { tests, out } = await playwright(['settle.spec.ts', '-g', title], { ...keployEnv(a.url), SLOW_BASE: p.url, NO_RELEASE: '1', PW_TIMEOUT: '4000' }); + const t = tests[title]; + assert.equal(t.status, 'timedOut', out); + const named = new RegExp(`^keploy: wait for the test's 1 call\\(s\\) in flight before its browser context closes: GET http://127\\.0\\.0\\.1:\\d+${call}$`); + assert.ok(t.steps.some((s) => named.test(s)), t.steps.join('\n')); + } finally { await p.close(); await a.close(); } + }); +} diff --git a/playwright/test/unit.test.mjs b/playwright/test/unit.test.mjs index 56d5d70..b645664 100644 --- a/playwright/test/unit.test.mjs +++ b/playwright/test/unit.test.mjs @@ -2,7 +2,9 @@ // fake agent. No browser. import { test } from 'node:test'; import assert from 'node:assert/strict'; +import fs from 'node:fs'; import http from 'node:http'; +import os from 'node:os'; import path from 'node:path'; import { AgentClient, KeployError } from '../dist/agent.js'; import { decide } from '../dist/join.js'; @@ -18,7 +20,7 @@ async function agent(answer) { const call = { path: req.url, auth: req.headers.authorization ?? '', body: body ? JSON.parse(body) : null }; calls.push(call); const a = answer(call); - res.writeHead(a.status ?? 200, { 'content-type': 'application/json' }); + res.writeHead(a.status ?? 200, { 'content-type': 'application/json', ...(a.headers ?? {}) }); res.end(typeof a.body === 'string' ? a.body : JSON.stringify(a.body)); }); }); @@ -81,6 +83,23 @@ test('every answer the client cannot trust is an error', async () => { } }); +test('a redirect from the agent is an error, and is not followed', async () => { + // Following it would send the call again, or to somewhere else. + const a = await agent(({ path: p }) => (p === '/elsewhere' ? { body: { v: 1, status: 'ok' } } + : { status: 307, headers: { location: '/elsewhere' }, body: '' })); + try { + await rejects(new AgentClient(a.url, '').begin({ name: 'a', file: 'a.spec.ts' }), /answered 307 Temporary Redirect to \/agent\/scope\/begin/); + assert.deepEqual(a.calls.map((x) => x.path), ['/agent/scope/begin']); + } finally { await a.close(); } +}); + +test('an end answered with a status other than ok is an error', async () => { + const a = await agent(() => ({ body: { v: 1, status: 'busy' } })); + try { + await rejects(new AgentClient(a.url, '').end({ name: 'a', file: 'a.spec.ts' }, 'passed'), /end of a with status "busy"/); + } finally { await a.close(); } +}); + test('an end takes only fail as its action', async () => { const a = await agent(() => ({ body: { v: 1, status: 'ok', action: 'skip' } })); try { @@ -151,6 +170,16 @@ test('testRef: the file relative to the project, the titles after it, the projec const ti = { file: path.join(process.cwd(), 'tests', 'a.spec.ts'), titlePath: ['a.spec.ts', 'group', 'inner', 'title'], project: { name: 'chromium' } }; assert.deepEqual(testRef(ti, process.cwd()), { name: 'group › inner › title', file: 'tests/a.spec.ts', project: 'chromium' }); assert.deepEqual(testRef({ ...ti, project: { name: '' } }, process.cwd()), { name: 'group › inner › title', file: 'tests/a.spec.ts' }); + // A spec reached through a symbolic link is named by its real path, as the + // project directory is. + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'kpw-ref-'))); + try { + fs.mkdirSync(path.join(root, 'proj', 'tests'), { recursive: true }); + fs.writeFileSync(path.join(root, 'proj', 'tests', 'a.spec.ts'), ''); + fs.symlinkSync(path.join(root, 'proj', 'tests'), path.join(root, 'linked')); + assert.deepEqual(testRef({ ...ti, file: path.join(root, 'linked', 'a.spec.ts') }, path.join(root, 'proj')), + { name: 'group › inner › title', file: 'tests/a.spec.ts', project: 'chromium' }); + } finally { fs.rmSync(root, { recursive: true, force: true }); } }); test('outcomeOf: the test\'s own verdict', () => { From f63d4f43f19d397a808bfd977d774d274170514b Mon Sep 17 00:00:00 2001 From: slayerjain Date: Sun, 1 Nov 2026 02:34:29 +0530 Subject: [PATCH 07/14] fix(playwright): name the route given needs on a 404; document the limits and tag-only releases given() posts to /agent/scope/given, which keploy/keploy#5016 adds and no keploy release serves yet. On a keploy without it, the error said "use a keploy and a @keploy/playwright of the same release", and no such release exists. It now says what happened and what is needed: keploy: this keploy agent answered 404 to POST /agent/scope/given, so it cannot take the given value : given needs a keploy that serves that route, which keploy/keploy#5016 adds Claim, begin, settle and end work with released keploy, so the package is usable without given. README: - Limits: given's route (with the error above), and the third limit the package has: the beforeAll and afterAll hooks of a group whose tests are all set aside still run, because each test is skipped at its own begin. - Releasing: bump the version in a pull request, then push playwright-vX.Y.Z as a plain git tag on main. Never create a GitHub release for it: a published release starts every workflow of the repository that runs on releases, such as the root @keploy/sdk release workflow of #117, which would try to publish under the tag's name. The publish workflow's header says the same. - Run the suite with --retries=0 as well as --workers=1, with the reason. Signed-off-by: slayerjain --- .github/workflows/playwright-publish.yml | 6 ++++++ playwright/README.md | 19 ++++++++++++++----- playwright/src/agent.ts | 5 ++++- playwright/test/unit.test.mjs | 4 ++-- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/.github/workflows/playwright-publish.yml b/.github/workflows/playwright-publish.yml index 61d5730..ee8e073 100644 --- a/.github/workflows/playwright-publish.yml +++ b/.github/workflows/playwright-publish.yml @@ -9,6 +9,12 @@ name: "@keploy/playwright publish" # name package.json's version (scripts/release-version.mjs). npm itself # refuses a version it already has. # +# Push playwright-vX.Y.Z as a plain git tag, never as a GitHub release: a +# published GitHub release also starts any workflow of this repository that +# runs on releases (the root @keploy/sdk release workflow, #117), which would +# try to publish that package under this tag's name. playwright/README.md, +# "Releasing", says the same. +# # On a pull request the same package is published as a dry run, with no # token, so a package that would ship without its build shows before a tag. on: diff --git a/playwright/README.md b/playwright/README.md index 39123cd..b3b95cd 100644 --- a/playwright/README.md +++ b/playwright/README.md @@ -23,14 +23,14 @@ test('lists the apps', async ({ page }) => { }); ``` -Run the suite under keploy with one worker: +Run the suite under keploy with one worker and no retries: ```sh -keploy mock record -c "npx playwright test --workers=1" -keploy mock replay -c "npx playwright test --workers=1" +keploy mock record -c "npx playwright test --workers=1 --retries=0" +keploy mock replay -c "npx playwright test --workers=1 --retries=0" ``` -keploy's first release runs one test process at a time. A second worker that is still alive while another runs is refused by keploy, and its tests fail. +keploy's first release runs one test process at a time. A second worker that is still alive while another runs is refused by keploy, and its tests fail. Turn retries off: keploy sets aside a test it cannot prove, with its reason, and fails a test whose calls changed, and a retry changes neither. ## Without keploy @@ -43,7 +43,7 @@ With `KEPLOY_MOCK_AGENT` unset or `off`, nothing changes. Only `keploy mock reco - **Skip.** When keploy has set a test aside, the test is skipped through Playwright's own skip, and the report shows keploy's reason (`keploy: ...`). - **Settle.** Before the test's `context` (and its `page`) closes, the test waits until none of its pages' calls is in flight. Calls are counted by the browser's own request events, and no time window decides. A call that never ends keeps the test waiting until Playwright's test timeout fails it. The test shows a step that names each call still in flight. A call is not waited for once the browser has cancelled it, which happens when its page loads another document, its frame is removed or its page closes. A route change inside the page (the history API) cancels nothing. - **End.** The test ends (`POST /agent/scope/end`) after its last teardown, with its own outcome. When keploy saw a call with no recorded answer, or a recorded call the test no longer made, the test fails with keploy's text. -- **Given values.** `await given(name, value)` tells keploy a value the test made itself. Examples are a session token it signs and a name it stamps with the time. keploy then follows the value wherever the test's calls carry it, as it follows an incoming request's values in a server test. Give a distinctive value, whole, as the calls carry it. `given` must be called inside a test, or in its `beforeEach` or `afterEach` hooks. It returns the value. +- **Given values.** `await given(name, value)` tells keploy a value the test made itself. Examples are a session token it signs and a name it stamps with the time. keploy then follows the value wherever the test's calls carry it, as it follows an incoming request's values in a server test. Give a distinctive value, whole, as the calls carry it. `given` must be called inside a test, or in its `beforeEach` or `afterEach` hooks. It returns the value. It needs a keploy whose agent serves `POST /agent/scope/given` (see the limits below). Any answer from the agent that this package cannot trust fails the test. That covers an agent it cannot reach, an error answer, an unreadable answer, and an answer that does not speak scope protocol v1. Nothing is retried. @@ -51,3 +51,12 @@ Any answer from the agent that this package cannot trust fails the test. That co - Settle covers the `context` and `page` fixtures. A context the test makes with `browser.newContext()` is the test's own to close. - Calls that `beforeAll`/`afterAll` hooks and worker fixtures make outside every test are answered from the worker's claim, one slice per Playwright project. +- The `beforeAll` and `afterAll` hooks of a describe group whose tests keploy has all set aside still run, and their calls are answered from the claim. Each test is skipped at its begin, after the group's hooks have started; skipping the whole group before its hooks run needs a way to ask keploy for the set-aside tests first, which this version does not have. +- `given` needs the agent route `POST /agent/scope/given`, which keploy/keploy#5016 adds. When this version was published, no keploy release served it yet. On a keploy without it, `given` fails the test with `keploy: this keploy agent answered 404 to POST /agent/scope/given, so it cannot take the given value : given needs a keploy that serves that route, which keploy/keploy#5016 adds`. Claim, begin, settle and end work with released keploy. + +## Releasing + +1. In a pull request, set `"version": "X.Y.Z"` in `playwright/package.json`. The pull request's publish dry run checks the package. +2. Once it is merged, push the tag `playwright-vX.Y.Z` on that commit of `main`: `git tag playwright-vX.Y.Z && git push origin playwright-vX.Y.Z`. The tag runs the rows, then publishes to npm (`.github/workflows/playwright-publish.yml`). + +Push the tag as a plain git tag. Never create a GitHub release for it: a published GitHub release also starts any workflow of this repository that runs on releases, such as the root `@keploy/sdk` release workflow (keploy/typescript-sdk#117), which would try to publish that package under the tag's name and fail. diff --git a/playwright/src/agent.ts b/playwright/src/agent.ts index 39b6c62..4c28f87 100644 --- a/playwright/src/agent.ts +++ b/playwright/src/agent.ts @@ -150,7 +150,10 @@ export class AgentClient { a = await this.post(PATH_GIVEN, source ? { name, value, source } : { name, value }); } catch (err) { if (err instanceof KeployError && err.status === 404 && err.code === '') { - throw new KeployError(`keploy: this keploy agent does not take a test's given values (${PATH_GIVEN}): use a keploy and a @keploy/playwright of the same release`, 404); + // The route came after the rest of the protocol: begin, end and + // settle work on an agent without it. + throw new KeployError(`keploy: this keploy agent answered 404 to POST ${PATH_GIVEN}, so it cannot take the given value ${name}: ` + + 'given needs a keploy that serves that route, which keploy/keploy#5016 adds', 404); } throw err; } diff --git a/playwright/test/unit.test.mjs b/playwright/test/unit.test.mjs index b645664..dc76854 100644 --- a/playwright/test/unit.test.mjs +++ b/playwright/test/unit.test.mjs @@ -127,7 +127,7 @@ test('an answer over 1 MiB is an error', async () => { } finally { await a.close(); } }); -test('given: the variable comes back; an agent without the route says to match releases', async () => { +test('given: the variable comes back; an agent without the route names the route and the keploy change that adds it', async () => { let a = await agent(() => ({ body: { v: 1, status: 'ok', variable: 'given:session' } })); try { assert.equal(await new AgentClient(a.url, '').given('session', 'tok-1', 'tests/a.spec.ts:3'), 'given:session'); @@ -135,7 +135,7 @@ test('given: the variable comes back; an agent without the route says to match r } finally { await a.close(); } a = await agent(() => ({ status: 404, body: '404 page not found' })); try { - await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /does not take a test's given values.*same release/); + await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /^keploy: this keploy agent answered 404 to POST \/agent\/scope\/given, so it cannot take the given value session: given needs a keploy that serves that route, which keploy\/keploy#5016 adds$/); } finally { await a.close(); } a = await agent(() => ({ status: 422, body: { error: 'not_open', message: 'a value was given while no test was open' } })); try { From 81aec5ce34e3a13841eb58860366981b29e71888 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Sun, 1 Nov 2026 06:53:27 +0530 Subject: [PATCH 08/14] fix(playwright): row settle's page-close and navigation guards Two settle guards had no row that fails without them: - A page that closes drops only its own calls. Without that check, a popup that closes (an OAuth or payment window) dropped its opener's calls in flight, and the opener's context then closed on them. Row: "another page closing cancels nothing of this page". - Only a navigation's response marks a frame as loading. Without that check, an answered fetch followed by a route change through the history API dropped the frame's calls in flight. Row: "a route change after an answered call cancels nothing". Row: "the test closes its own context" (a page's call, not waited for). The context's own close rule stays: a later commit rows the call that only it drops. README: a closing page cancels only its own calls, the test closing its own context cancels every call, and the rows that prove which calls the browser ends itself run in Chromium only (a limit for Firefox and WebKit). Signed-off-by: slayerjain --- playwright/README.md | 3 ++- playwright/src/settle.ts | 6 ++++-- playwright/test/fixtures/settle.spec.ts | 26 +++++++++++++++++++++++++ playwright/test/rows.test.mjs | 8 ++++++-- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/playwright/README.md b/playwright/README.md index b3b95cd..0ce6ceb 100644 --- a/playwright/README.md +++ b/playwright/README.md @@ -41,7 +41,7 @@ With `KEPLOY_MOCK_AGENT` unset or `off`, nothing changes. Only `keploy mock reco - **Claim.** Each worker claims keploy's run before its first hook or test (`POST /agent/instance`). The claim uses the worker's Playwright project name as its key, or `.` for an unnamed project. A worker that Playwright starts again after a failed test claims again. - **Begin.** Each test begins (`POST /agent/scope/begin`) before its `beforeEach` hooks. keploy builds the test's ID as `[project::]file::titles`. The file is relative to the project directory keploy names (`KEPLOY_MOCK_PROJECT`). The titles are the describe groups and the test's own title, joined with ` › `. - **Skip.** When keploy has set a test aside, the test is skipped through Playwright's own skip, and the report shows keploy's reason (`keploy: ...`). -- **Settle.** Before the test's `context` (and its `page`) closes, the test waits until none of its pages' calls is in flight. Calls are counted by the browser's own request events, and no time window decides. A call that never ends keeps the test waiting until Playwright's test timeout fails it. The test shows a step that names each call still in flight. A call is not waited for once the browser has cancelled it, which happens when its page loads another document, its frame is removed or its page closes. A route change inside the page (the history API) cancels nothing. +- **Settle.** Before the test's `context` (and its `page`) closes, the test waits until none of its pages' calls is in flight. Calls are counted by the browser's own request events, and no time window decides. A call that never ends keeps the test waiting until Playwright's test timeout fails it. The test shows a step that names each call still in flight. A call is not waited for once the browser has cancelled it, which happens when its page loads another document, its frame is removed, its page closes or the test closes its own context. A closing page cancels only its own calls: a popup that closes cancels nothing of its opener's. A route change inside the page (the history API) cancels nothing, even after an answered call. - **End.** The test ends (`POST /agent/scope/end`) after its last teardown, with its own outcome. When keploy saw a call with no recorded answer, or a recorded call the test no longer made, the test fails with keploy's text. - **Given values.** `await given(name, value)` tells keploy a value the test made itself. Examples are a session token it signs and a name it stamps with the time. keploy then follows the value wherever the test's calls carry it, as it follows an incoming request's values in a server test. Give a distinctive value, whole, as the calls carry it. `given` must be called inside a test, or in its `beforeEach` or `afterEach` hooks. It returns the value. It needs a keploy whose agent serves `POST /agent/scope/given` (see the limits below). @@ -50,6 +50,7 @@ Any answer from the agent that this package cannot trust fails the test. That co ## Limits of this version - Settle covers the `context` and `page` fixtures. A context the test makes with `browser.newContext()` is the test's own to close. +- Settle's rows run in Chromium. Which calls the browser itself ends (a removed frame's) is proven there only; Firefox and WebKit are not rowed. - Calls that `beforeAll`/`afterAll` hooks and worker fixtures make outside every test are answered from the worker's claim, one slice per Playwright project. - The `beforeAll` and `afterAll` hooks of a describe group whose tests keploy has all set aside still run, and their calls are answered from the claim. Each test is skipped at its begin, after the group's hooks have started; skipping the whole group before its hooks run needs a way to ask keploy for the set-aside tests first, which this version does not have. - `given` needs the agent route `POST /agent/scope/given`, which keploy/keploy#5016 adds. When this version was published, no keploy release served it yet. On a keploy without it, `given` fails the test with `keploy: this keploy agent answered 404 to POST /agent/scope/given, so it cannot take the given value : given needs a keploy that serves that route, which keploy/keploy#5016 adds`. Claim, begin, settle and end work with released keploy. diff --git a/playwright/src/settle.ts b/playwright/src/settle.ts index a5fa911..5f84cb3 100644 --- a/playwright/src/settle.ts +++ b/playwright/src/settle.ts @@ -42,12 +42,14 @@ export class Inflight { if (!loading.delete(f)) return; this.drop((o) => o.frame === f && !o.nav); }); - // Nor for the calls of a page that closes. (A removed frame's calls - // it does end, as failed requests.) + // Nor for the calls of a page that closes. Only that page's calls: a + // popup that closes cancels nothing of its opener. (A removed frame's + // calls the browser does end, as failed requests.) p.on('close', () => this.drop((o) => o.frame !== null && pageOf(o.frame) === p)); }; context.pages().forEach(watch); context.on('page', watch); + // Nor for any call when the test closes its own context. context.on('close', () => this.drop(() => true)); } diff --git a/playwright/test/fixtures/settle.spec.ts b/playwright/test/fixtures/settle.spec.ts index 6b0022d..c669ec3 100644 --- a/playwright/test/fixtures/settle.spec.ts +++ b/playwright/test/fixtures/settle.spec.ts @@ -101,3 +101,29 @@ test('a document still loading after its frame commits', async ({ page }) => { // byte: committing the new document cancels only the old document's calls. await page.goto(`${base}/slowdoc`, { waitUntil: 'commit' }); }); +test('a call in flight while another page of the test closes', async ({ page, context }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + const other = await context.newPage(); + await other.goto(`${base}/page?other`); + await other.close(); +}); +test('a call in flight across a route change after an answered call', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + await page.evaluate(() => fetch('/page').then((r) => r.text())); + const routed = page.waitForEvent('framenavigated'); + await page.evaluate(() => { history.pushState({}, '', '/page?route'); }); + await routed; +}); +test('a call in flight when the test closes its own context', async ({ page, context }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + await context.close(); +}); diff --git a/playwright/test/rows.test.mjs b/playwright/test/rows.test.mjs index b6eb8b8..6b218c8 100644 --- a/playwright/test/rows.test.mjs +++ b/playwright/test/rows.test.mjs @@ -289,11 +289,13 @@ test('an unnamed Playwright project claims with the key ".", and its tests name }); // Settle rows where the browser cancelled a call: the test passes at once, -// with no wait step. The browser ends a removed frame's calls itself; a -// closed page's it does not, and the package drops them. +// with no wait step. Chromium ends a removed frame's calls itself; a closed +// page's it does not, and the package drops them, as it drops every call +// when the test closes its own context. for (const [title, why] of [ ['a call its removed cross-site frame had in flight', 'its cross-site frame is removed'], ['a call its closed page had in flight', 'its page closes'], + ['a call in flight when the test closes its own context', 'the test closes its own context'], ]) { test(`settle: a call is not waited for once ${why}`, async () => { const j = journal(); @@ -315,6 +317,8 @@ for (const [title, why, call] of [ ['a call in flight across a navigation answered 205', 'a navigation answered 205 replaces no document', '/slow'], ['a call in flight across a download', 'a navigation answered with an attachment (a download) replaces no document', '/slow'], ['a document still loading after its frame commits', 'a navigation\'s own request stays in flight after its document commits', '/slowdoc'], + ['a call in flight while another page of the test closes', 'another page closing cancels nothing of this page', '/slow'], + ['a call in flight across a route change after an answered call', 'a route change after an answered call cancels nothing', '/slow'], ]) { test(`settle: ${why}, so the call is still waited for`, async () => { const j = journal(); From ebfe3a035a1aabd0f560ea3b04349576a8e00be4 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Sun, 1 Nov 2026 05:16:24 +0530 Subject: [PATCH 09/14] test(playwright): row given's 404 rule, a download's header case and a service worker's call Three guards had no row that fails without them: - given turns a 404 into "needs a keploy that serves that route" only when the answer is not keploy's own refusal (no error code). A 404 that carries keploy's error code, message and remedy now shows them, as any other refusal does. Row: the given row's new 404 case. - A download is a navigation answered with an attachment, in any case (RFC 6266): the download row's server now answers "Attachment", so a case-sensitive check lets the frame's calls be dropped and fails it. - A service worker's call has no frame (Playwright's request.frame() throws for it). Settle counts it like a page's call and no page change drops it. Row: a service worker that calls /slow as it installs keeps the test waiting, with a step that names the call. README: settle counts a service worker's calls. Rows: 40 (15 unit, 4 release, 21 Playwright). Each mutant of the three guards fails its row. Signed-off-by: slayerjain --- playwright/README.md | 2 +- playwright/test/fixtures/settle.spec.ts | 8 ++++++++ playwright/test/rows.test.mjs | 6 +++++- playwright/test/unit.test.mjs | 7 ++++++- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/playwright/README.md b/playwright/README.md index 0ce6ceb..8579cb4 100644 --- a/playwright/README.md +++ b/playwright/README.md @@ -41,7 +41,7 @@ With `KEPLOY_MOCK_AGENT` unset or `off`, nothing changes. Only `keploy mock reco - **Claim.** Each worker claims keploy's run before its first hook or test (`POST /agent/instance`). The claim uses the worker's Playwright project name as its key, or `.` for an unnamed project. A worker that Playwright starts again after a failed test claims again. - **Begin.** Each test begins (`POST /agent/scope/begin`) before its `beforeEach` hooks. keploy builds the test's ID as `[project::]file::titles`. The file is relative to the project directory keploy names (`KEPLOY_MOCK_PROJECT`). The titles are the describe groups and the test's own title, joined with ` › `. - **Skip.** When keploy has set a test aside, the test is skipped through Playwright's own skip, and the report shows keploy's reason (`keploy: ...`). -- **Settle.** Before the test's `context` (and its `page`) closes, the test waits until none of its pages' calls is in flight. Calls are counted by the browser's own request events, and no time window decides. A call that never ends keeps the test waiting until Playwright's test timeout fails it. The test shows a step that names each call still in flight. A call is not waited for once the browser has cancelled it, which happens when its page loads another document, its frame is removed, its page closes or the test closes its own context. A closing page cancels only its own calls: a popup that closes cancels nothing of its opener's. A route change inside the page (the history API) cancels nothing, even after an answered call. +- **Settle.** Before the test's `context` (and its `page`) closes, the test waits until none of its pages' calls is in flight. Calls are counted by the browser's own request events, and no time window decides. A call that never ends keeps the test waiting until Playwright's test timeout fails it. The test shows a step that names each call still in flight. A call is not waited for once the browser has cancelled it, which happens when its page loads another document, its frame is removed, its page closes or the test closes its own context. A closing page cancels only its own calls: a popup that closes cancels nothing of its opener's. A route change inside the page (the history API) cancels nothing, even after an answered call. A service worker's calls are counted too, and no page's change drops them. - **End.** The test ends (`POST /agent/scope/end`) after its last teardown, with its own outcome. When keploy saw a call with no recorded answer, or a recorded call the test no longer made, the test fails with keploy's text. - **Given values.** `await given(name, value)` tells keploy a value the test made itself. Examples are a session token it signs and a name it stamps with the time. keploy then follows the value wherever the test's calls carry it, as it follows an incoming request's values in a server test. Give a distinctive value, whole, as the calls carry it. `given` must be called inside a test, or in its `beforeEach` or `afterEach` hooks. It returns the value. It needs a keploy whose agent serves `POST /agent/scope/given` (see the limits below). diff --git a/playwright/test/fixtures/settle.spec.ts b/playwright/test/fixtures/settle.spec.ts index c669ec3..992f940 100644 --- a/playwright/test/fixtures/settle.spec.ts +++ b/playwright/test/fixtures/settle.spec.ts @@ -127,3 +127,11 @@ test('a call in flight when the test closes its own context', async ({ page, con await sent; await context.close(); }); +test('a call its service worker has in flight', async ({ page, context }) => { + // A service worker's call has no frame (Playwright's request.frame() throws + // for it). It is counted like a page's call, and no page change drops it. + await page.goto(`${base}/page`); + const sent = context.waitForEvent('request', (r) => r.url().endsWith('/slow')); + await page.evaluate(() => { void navigator.serviceWorker.register('/sw.js'); }); + await sent; +}); diff --git a/playwright/test/rows.test.mjs b/playwright/test/rows.test.mjs index 6b218c8..94b64be 100644 --- a/playwright/test/rows.test.mjs +++ b/playwright/test/rows.test.mjs @@ -51,9 +51,12 @@ async function pageServer(j) { const srv = http.createServer((req, res) => { if (req.url === '/page') { res.writeHead(200, { 'content-type': 'text/html' }); res.end('page'); return; } if (req.url === '/release') { release(); res.end('ok'); return; } + // A service worker that calls /slow as it installs. + if (req.url === '/sw.js') { res.writeHead(200, { 'content-type': 'text/javascript' }); res.end("self.addEventListener('install', () => { void fetch('/slow'); });"); return; } if (req.url === '/nocontent') { res.writeHead(204); res.end(); return; } if (req.url === '/resetcontent') { res.writeHead(205); res.end(); return; } - if (req.url === '/download') { res.writeHead(200, { 'content-type': 'application/octet-stream', 'content-disposition': 'attachment; filename="d.bin"' }); res.end('bytes'); return; } + // A disposition's type is case-insensitive (RFC 6266): the browser downloads "Attachment" too. + if (req.url === '/download') { res.writeHead(200, { 'content-type': 'application/octet-stream', 'content-disposition': 'Attachment; filename="d.bin"' }); res.end('bytes'); return; } if (req.url === '/slowdoc') { // A document whose last byte comes only once /release is called. j.add({ page: 'slowdoc sent' }); @@ -319,6 +322,7 @@ for (const [title, why, call] of [ ['a document still loading after its frame commits', 'a navigation\'s own request stays in flight after its document commits', '/slowdoc'], ['a call in flight while another page of the test closes', 'another page closing cancels nothing of this page', '/slow'], ['a call in flight across a route change after an answered call', 'a route change after an answered call cancels nothing', '/slow'], + ['a call its service worker has in flight', 'a service worker\'s call, which has no frame, is counted', '/slow'], ]) { test(`settle: ${why}, so the call is still waited for`, async () => { const j = journal(); diff --git a/playwright/test/unit.test.mjs b/playwright/test/unit.test.mjs index dc76854..4fd6eb6 100644 --- a/playwright/test/unit.test.mjs +++ b/playwright/test/unit.test.mjs @@ -127,7 +127,7 @@ test('an answer over 1 MiB is an error', async () => { } finally { await a.close(); } }); -test('given: the variable comes back; an agent without the route names the route and the keploy change that adds it', async () => { +test('given: the variable comes back; an agent without the route names the route and the keploy change that adds it, and keploy\'s own 404 shows keploy\'s reason', async () => { let a = await agent(() => ({ body: { v: 1, status: 'ok', variable: 'given:session' } })); try { assert.equal(await new AgentClient(a.url, '').given('session', 'tok-1', 'tests/a.spec.ts:3'), 'given:session'); @@ -137,6 +137,11 @@ test('given: the variable comes back; an agent without the route names the route try { await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /^keploy: this keploy agent answered 404 to POST \/agent\/scope\/given, so it cannot take the given value session: given needs a keploy that serves that route, which keploy\/keploy#5016 adds$/); } finally { await a.close(); } + // A 404 that carries keploy's own refusal is keploy's answer, not a missing route. + a = await agent(() => ({ status: 404, body: { error: 'no_test', message: 'no test named a is open', remedy: 'begin it first' } })); + try { + await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /^keploy: no test named a is open \(begin it first\)$/); + } finally { await a.close(); } a = await agent(() => ({ status: 422, body: { error: 'not_open', message: 'a value was given while no test was open' } })); try { await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /^keploy: a value was given while no test was open$/); From d025a195ffb4af91cec5581e897f79117d0f2f26 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Sun, 1 Nov 2026 06:54:21 +0530 Subject: [PATCH 10/14] test(playwright): row the context-close rule, a page close beside a worker, a lowercase download Three settle rules had no row that fails without them: - The test closing its own context drops every call in flight. A page's calls its page close drops too, but a service worker's call has no frame, so no page close drops it, and Chromium ends the worker with its context without reporting an end for the call. Row: a service worker calls /slow as it installs, then the test closes its own context; the test passes at once. Without the rule it waits until the test timeout. - A closing page drops only calls that have a frame in that page. Row: a service worker's call in flight while another page of the test closes is still waited for, and the step names it. - A download is a navigation answered with an attachment in any letter case. The download row answers "Attachment"; a new row answers the common "attachment", so a check for either spelling alone fails one of them. README: settle's service-worker rule names the context close as the one thing that drops a worker's call. There are now 43 rows (15 unit, 4 release, 24 Playwright). Signed-off-by: slayerjain --- playwright/README.md | 2 +- playwright/src/settle.ts | 4 ++- playwright/test/fixtures/settle.spec.ts | 33 +++++++++++++++++++++++++ playwright/test/rows.test.mjs | 10 ++++++-- 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/playwright/README.md b/playwright/README.md index 8579cb4..d74b5dc 100644 --- a/playwright/README.md +++ b/playwright/README.md @@ -41,7 +41,7 @@ With `KEPLOY_MOCK_AGENT` unset or `off`, nothing changes. Only `keploy mock reco - **Claim.** Each worker claims keploy's run before its first hook or test (`POST /agent/instance`). The claim uses the worker's Playwright project name as its key, or `.` for an unnamed project. A worker that Playwright starts again after a failed test claims again. - **Begin.** Each test begins (`POST /agent/scope/begin`) before its `beforeEach` hooks. keploy builds the test's ID as `[project::]file::titles`. The file is relative to the project directory keploy names (`KEPLOY_MOCK_PROJECT`). The titles are the describe groups and the test's own title, joined with ` › `. - **Skip.** When keploy has set a test aside, the test is skipped through Playwright's own skip, and the report shows keploy's reason (`keploy: ...`). -- **Settle.** Before the test's `context` (and its `page`) closes, the test waits until none of its pages' calls is in flight. Calls are counted by the browser's own request events, and no time window decides. A call that never ends keeps the test waiting until Playwright's test timeout fails it. The test shows a step that names each call still in flight. A call is not waited for once the browser has cancelled it, which happens when its page loads another document, its frame is removed, its page closes or the test closes its own context. A closing page cancels only its own calls: a popup that closes cancels nothing of its opener's. A route change inside the page (the history API) cancels nothing, even after an answered call. A service worker's calls are counted too, and no page's change drops them. +- **Settle.** Before the test's `context` (and its `page`) closes, the test waits until none of its pages' calls is in flight. Calls are counted by the browser's own request events, and no time window decides. A call that never ends keeps the test waiting until Playwright's test timeout fails it. The test shows a step that names each call still in flight. A call is not waited for once the browser has cancelled it, which happens when its page loads another document, its frame is removed, its page closes or the test closes its own context. A closing page cancels only its own calls: a popup that closes cancels nothing of its opener's. A route change inside the page (the history API) cancels nothing, even after an answered call. A service worker's calls are counted too. No page's change drops them, but the test closing its own context does, as it ends the worker. - **End.** The test ends (`POST /agent/scope/end`) after its last teardown, with its own outcome. When keploy saw a call with no recorded answer, or a recorded call the test no longer made, the test fails with keploy's text. - **Given values.** `await given(name, value)` tells keploy a value the test made itself. Examples are a session token it signs and a name it stamps with the time. keploy then follows the value wherever the test's calls carry it, as it follows an incoming request's values in a server test. Give a distinctive value, whole, as the calls carry it. `given` must be called inside a test, or in its `beforeEach` or `afterEach` hooks. It returns the value. It needs a keploy whose agent serves `POST /agent/scope/given` (see the limits below). diff --git a/playwright/src/settle.ts b/playwright/src/settle.ts index 5f84cb3..e353d78 100644 --- a/playwright/src/settle.ts +++ b/playwright/src/settle.ts @@ -49,7 +49,9 @@ export class Inflight { }; context.pages().forEach(watch); context.on('page', watch); - // Nor for any call when the test closes its own context. + // Nor for any call when the test closes its own context, which ends its + // service workers too. A service worker's call has no frame, so no + // page's close drops it; this is the only rule that does. context.on('close', () => this.drop(() => true)); } diff --git a/playwright/test/fixtures/settle.spec.ts b/playwright/test/fixtures/settle.spec.ts index 992f940..331e46a 100644 --- a/playwright/test/fixtures/settle.spec.ts +++ b/playwright/test/fixtures/settle.spec.ts @@ -135,3 +135,36 @@ test('a call its service worker has in flight', async ({ page, context }) => { await page.evaluate(() => { void navigator.serviceWorker.register('/sw.js'); }); await sent; }); +test('a service worker call in flight when the test closes its own context', async ({ page, context }) => { + // Closing the context ends its service workers too. Chromium reports no end + // for the worker's call, and no page's close drops it (it has no frame). + await page.goto(`${base}/page`); + const sent = context.waitForEvent('request', (r) => r.url().endsWith('/slow')); + await page.evaluate(() => { void navigator.serviceWorker.register('/sw.js'); }); + await sent; + await context.close(); +}); +test('a service worker call in flight while another page of the test closes', async ({ page, context }) => { + await page.goto(`${base}/page`); + const sent = context.waitForEvent('request', (r) => r.url().endsWith('/slow')); + await page.evaluate(() => { void navigator.serviceWorker.register('/sw.js'); }); + await sent; + const other = await context.newPage(); + await other.goto(`${base}/page?other`); + await other.close(); +}); +test('a call in flight across a lowercase attachment download', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + const answered = page.waitForResponse(/\/download-lower$/); + const downloaded = page.waitForEvent('download'); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + // The common spelling, "attachment", is a download as "Attachment" is. + await page.evaluate(() => { location.href = '/download-lower'; }); + await answered; + await downloaded; + const routed = page.waitForEvent('framenavigated'); + await page.evaluate(() => { history.pushState({}, '', '/page?route'); }); + await routed; +}); diff --git a/playwright/test/rows.test.mjs b/playwright/test/rows.test.mjs index 94b64be..f62dac5 100644 --- a/playwright/test/rows.test.mjs +++ b/playwright/test/rows.test.mjs @@ -57,6 +57,7 @@ async function pageServer(j) { if (req.url === '/resetcontent') { res.writeHead(205); res.end(); return; } // A disposition's type is case-insensitive (RFC 6266): the browser downloads "Attachment" too. if (req.url === '/download') { res.writeHead(200, { 'content-type': 'application/octet-stream', 'content-disposition': 'Attachment; filename="d.bin"' }); res.end('bytes'); return; } + if (req.url === '/download-lower') { res.writeHead(200, { 'content-type': 'application/octet-stream', 'content-disposition': 'attachment; filename="e.bin"' }); res.end('bytes'); return; } if (req.url === '/slowdoc') { // A document whose last byte comes only once /release is called. j.add({ page: 'slowdoc sent' }); @@ -294,11 +295,14 @@ test('an unnamed Playwright project claims with the key ".", and its tests name // Settle rows where the browser cancelled a call: the test passes at once, // with no wait step. Chromium ends a removed frame's calls itself; a closed // page's it does not, and the package drops them, as it drops every call -// when the test closes its own context. +// when the test closes its own context. A service worker's call has no +// frame, so only the context's close drops it: Chromium ends the worker +// with its context and reports no end for the call. for (const [title, why] of [ ['a call its removed cross-site frame had in flight', 'its cross-site frame is removed'], ['a call its closed page had in flight', 'its page closes'], ['a call in flight when the test closes its own context', 'the test closes its own context'], + ['a service worker call in flight when the test closes its own context', 'the test closes its own context, which ends its service worker'], ]) { test(`settle: a call is not waited for once ${why}`, async () => { const j = journal(); @@ -318,11 +322,13 @@ for (const [title, why] of [ // the test times out naming it. for (const [title, why, call] of [ ['a call in flight across a navigation answered 205', 'a navigation answered 205 replaces no document', '/slow'], - ['a call in flight across a download', 'a navigation answered with an attachment (a download) replaces no document', '/slow'], + ['a call in flight across a download', 'a navigation answered with an attachment (a download, "Attachment") replaces no document', '/slow'], + ['a call in flight across a lowercase attachment download', 'a navigation answered with an attachment (a download, "attachment") replaces no document', '/slow'], ['a document still loading after its frame commits', 'a navigation\'s own request stays in flight after its document commits', '/slowdoc'], ['a call in flight while another page of the test closes', 'another page closing cancels nothing of this page', '/slow'], ['a call in flight across a route change after an answered call', 'a route change after an answered call cancels nothing', '/slow'], ['a call its service worker has in flight', 'a service worker\'s call, which has no frame, is counted', '/slow'], + ['a service worker call in flight while another page of the test closes', 'another page closing cancels nothing of a service worker', '/slow'], ]) { test(`settle: ${why}, so the call is still waited for`, async () => { const j = journal(); From 4af8269e9e51ccce289ca0cfefb25216133f9e0e Mon Sep 17 00:00:00 2001 From: slayerjain Date: Sun, 1 Nov 2026 07:07:32 +0530 Subject: [PATCH 11/14] test(playwright): wait for a call to reach the server before a row cancels it Five settle rows cancel a call in flight (a new document, a removed frame, a closed page, the test closing its own context, for a page's call and a service worker's) and then assert that the page server saw the call. Each spec cancelled the call as soon as the browser's request event fired. That event comes before the request reaches the server, so on a loaded box the cancel could win: the test passed for the right reason, and the row failed because the server never logged "slow sent". It happened in this branch's gate run (the context-close row, a box at load 7 to 9 on 16 CPUs). The removed-frame row's one earlier failure, in the second review's run, kept no detail; that row makes the same assertion after the same kind of cancel, so this race can explain it. The page server now answers /seen once /slow has arrived, and each of those specs waits for /seen before it cancels. The order is decided by the server's own event, with no time window. Signed-off-by: slayerjain --- playwright/test/fixtures/settle.spec.ts | 9 +++++++++ playwright/test/rows.test.mjs | 9 ++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/playwright/test/fixtures/settle.spec.ts b/playwright/test/fixtures/settle.spec.ts index 331e46a..f4237ed 100644 --- a/playwright/test/fixtures/settle.spec.ts +++ b/playwright/test/fixtures/settle.spec.ts @@ -1,6 +1,10 @@ import { test } from '../../dist'; const base = process.env.SLOW_BASE ?? ''; +// Resolves once the page server has received /slow. A spec that cancels its +// call waits for this first: the browser's request event comes before the +// call reaches the server, and the row asserts that it did. +const reached = (): Promise => fetch(`${base}/seen`).then((r) => r.text()); test.afterEach(async () => { if (!process.env.NO_RELEASE) await fetch(`${base}/release`); }); @@ -15,6 +19,7 @@ test('a call its page cancelled by loading another document', async ({ page }) = const sent = page.waitForRequest(/\/slow$/); await page.evaluate(() => { void fetch('/slow'); }); await sent; + await reached(); await page.goto(`${base}/page?next`); }); test('a call in flight across a route change inside the page', async ({ page }) => { @@ -84,6 +89,7 @@ test('a call its removed cross-site frame had in flight', async ({ page }) => { const detached = page.waitForEvent('framedetached'); await child.evaluate(() => { void fetch('/slow'); }); await sent; + await reached(); await page.evaluate(() => { document.getElementById('child')?.remove(); }); await detached; }); @@ -94,6 +100,7 @@ test('a call its closed page had in flight', async ({ page, context }) => { const sent = other.waitForRequest(/\/slow$/); await other.evaluate(() => { void fetch('/slow'); }); await sent; + await reached(); await other.close(); }); test('a document still loading after its frame commits', async ({ page }) => { @@ -125,6 +132,7 @@ test('a call in flight when the test closes its own context', async ({ page, con const sent = page.waitForRequest(/\/slow$/); await page.evaluate(() => { void fetch('/slow'); }); await sent; + await reached(); await context.close(); }); test('a call its service worker has in flight', async ({ page, context }) => { @@ -142,6 +150,7 @@ test('a service worker call in flight when the test closes its own context', asy const sent = context.waitForEvent('request', (r) => r.url().endsWith('/slow')); await page.evaluate(() => { void navigator.serviceWorker.register('/sw.js'); }); await sent; + await reached(); await context.close(); }); test('a service worker call in flight while another page of the test closes', async ({ page, context }) => { diff --git a/playwright/test/rows.test.mjs b/playwright/test/rows.test.mjs index f62dac5..63e8373 100644 --- a/playwright/test/rows.test.mjs +++ b/playwright/test/rows.test.mjs @@ -44,13 +44,19 @@ async function fakeAgent(j, answer = () => undefined) { } // The settle rows' page server: /slow and the end of /slowdoc are answered -// only once /release is called. +// only once /release is called. /seen is answered once /slow has arrived, so +// a spec that cancels its call waits for the call to reach the server first: +// the browser's request event comes before that, and a call cancelled before +// it reaches the server proves nothing. async function pageServer(j) { let release = null; const released = new Promise((r) => { release = r; }); + let saw = null; + const seen = new Promise((r) => { saw = r; }); const srv = http.createServer((req, res) => { if (req.url === '/page') { res.writeHead(200, { 'content-type': 'text/html' }); res.end('page'); return; } if (req.url === '/release') { release(); res.end('ok'); return; } + if (req.url === '/seen') { void seen.then(() => res.end('ok')); return; } // A service worker that calls /slow as it installs. if (req.url === '/sw.js') { res.writeHead(200, { 'content-type': 'text/javascript' }); res.end("self.addEventListener('install', () => { void fetch('/slow'); });"); return; } if (req.url === '/nocontent') { res.writeHead(204); res.end(); return; } @@ -68,6 +74,7 @@ async function pageServer(j) { } if (req.url === '/slow') { j.add({ page: 'slow sent' }); + saw(); res.on('close', () => { if (!res.writableFinished) j.add({ page: 'slow cut' }); }); res.on('finish', () => j.add({ page: 'slow answered' })); void released.then(() => res.end('slow')); From 014c369c805deef79bda0ed01344bd6c4d2327a6 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Sun, 1 Nov 2026 08:27:57 +0530 Subject: [PATCH 12/14] fix(playwright): name keploy v3.6.192 and KEPLOY_PROVEN=1 as what given needs keploy/keploy#5016 merged and shipped in keploy v3.6.192, which serves POST /agent/scope/given only under the proven switch: with the switch off, the route answers the plain 404 it gave before it existed. The README said no keploy release served the route, and given's 404 error told a v3.6.192 user whose switch was off that their keploy lacked it. The 404 error and the README's limit now say what given needs: keploy v3.6.192 or later (keploy/keploy#5016), or an enterprise release on keploy v3.6.192 or later, run with KEPLOY_PROVEN=1. An older keploy, or one run without the switch, answers 404 there. Enterprise main still pins keploy v3.6.190, so no enterprise release is named. Row: the given row's 404 case matches the new error exactly. Signed-off-by: slayerjain --- playwright/README.md | 2 +- playwright/src/agent.ts | 6 ++++-- playwright/test/unit.test.mjs | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/playwright/README.md b/playwright/README.md index d74b5dc..88e78e5 100644 --- a/playwright/README.md +++ b/playwright/README.md @@ -53,7 +53,7 @@ Any answer from the agent that this package cannot trust fails the test. That co - Settle's rows run in Chromium. Which calls the browser itself ends (a removed frame's) is proven there only; Firefox and WebKit are not rowed. - Calls that `beforeAll`/`afterAll` hooks and worker fixtures make outside every test are answered from the worker's claim, one slice per Playwright project. - The `beforeAll` and `afterAll` hooks of a describe group whose tests keploy has all set aside still run, and their calls are answered from the claim. Each test is skipped at its begin, after the group's hooks have started; skipping the whole group before its hooks run needs a way to ask keploy for the set-aside tests first, which this version does not have. -- `given` needs the agent route `POST /agent/scope/given`, which keploy/keploy#5016 adds. When this version was published, no keploy release served it yet. On a keploy without it, `given` fails the test with `keploy: this keploy agent answered 404 to POST /agent/scope/given, so it cannot take the given value : given needs a keploy that serves that route, which keploy/keploy#5016 adds`. Claim, begin, settle and end work with released keploy. +- `given` needs the agent route `POST /agent/scope/given` (keploy/keploy#5016). keploy v3.6.192 and later serve it, and only when keploy runs with `KEPLOY_PROVEN=1`; with the enterprise keploy, that is an enterprise release on keploy v3.6.192 or later, run the same way. An older keploy, or one run without `KEPLOY_PROVEN=1`, answers that route with a plain 404, and `given` fails the test with `keploy: this keploy agent answered 404 to POST /agent/scope/given, so it cannot take the given value : given needs keploy v3.6.192 or later (keploy/keploy#5016), or an enterprise release on keploy v3.6.192 or later, run with KEPLOY_PROVEN=1; an older keploy, or one run without KEPLOY_PROVEN=1, answers 404 there`. Claim, begin, settle and end do not need that route. ## Releasing diff --git a/playwright/src/agent.ts b/playwright/src/agent.ts index 4c28f87..c7b3fb0 100644 --- a/playwright/src/agent.ts +++ b/playwright/src/agent.ts @@ -150,10 +150,12 @@ export class AgentClient { a = await this.post(PATH_GIVEN, source ? { name, value, source } : { name, value }); } catch (err) { if (err instanceof KeployError && err.status === 404 && err.code === '') { - // The route came after the rest of the protocol: begin, end and + // The route came after the rest of the protocol (keploy v3.6.192), + // and keploy serves it only under KEPLOY_PROVEN=1: begin, end and // settle work on an agent without it. throw new KeployError(`keploy: this keploy agent answered 404 to POST ${PATH_GIVEN}, so it cannot take the given value ${name}: ` + - 'given needs a keploy that serves that route, which keploy/keploy#5016 adds', 404); + 'given needs keploy v3.6.192 or later (keploy/keploy#5016), or an enterprise release on keploy v3.6.192 or later, ' + + 'run with KEPLOY_PROVEN=1; an older keploy, or one run without KEPLOY_PROVEN=1, answers 404 there', 404); } throw err; } diff --git a/playwright/test/unit.test.mjs b/playwright/test/unit.test.mjs index 4fd6eb6..a8f52ba 100644 --- a/playwright/test/unit.test.mjs +++ b/playwright/test/unit.test.mjs @@ -127,7 +127,7 @@ test('an answer over 1 MiB is an error', async () => { } finally { await a.close(); } }); -test('given: the variable comes back; an agent without the route names the route and the keploy change that adds it, and keploy\'s own 404 shows keploy\'s reason', async () => { +test('given: the variable comes back; an agent without the route names the route, the keploy version that serves it and the switch, and keploy\'s own 404 shows keploy\'s reason', async () => { let a = await agent(() => ({ body: { v: 1, status: 'ok', variable: 'given:session' } })); try { assert.equal(await new AgentClient(a.url, '').given('session', 'tok-1', 'tests/a.spec.ts:3'), 'given:session'); @@ -135,7 +135,7 @@ test('given: the variable comes back; an agent without the route names the route } finally { await a.close(); } a = await agent(() => ({ status: 404, body: '404 page not found' })); try { - await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /^keploy: this keploy agent answered 404 to POST \/agent\/scope\/given, so it cannot take the given value session: given needs a keploy that serves that route, which keploy\/keploy#5016 adds$/); + await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /^keploy: this keploy agent answered 404 to POST \/agent\/scope\/given, so it cannot take the given value session: given needs keploy v3\.6\.192 or later \(keploy\/keploy#5016\), or an enterprise release on keploy v3\.6\.192 or later, run with KEPLOY_PROVEN=1; an older keploy, or one run without KEPLOY_PROVEN=1, answers 404 there$/); } finally { await a.close(); } // A 404 that carries keploy's own refusal is keploy's answer, not a missing route. a = await agent(() => ({ status: 404, body: { error: 'no_test', message: 'no test named a is open', remedy: 'begin it first' } })); From b390fc9a3ad8a0caca5e60809a7b821d76aae82f Mon Sep 17 00:00:00 2001 From: slayerjain Date: Sun, 1 Nov 2026 08:27:57 +0530 Subject: [PATCH 13/14] test(playwright): row settle's wake when a cancel comes while it waits Every cancel in the settle rows came in the test body, before settled() waited, so no row failed with Inflight.drop's wake removed: a test whose page closes, or whose context closes, while settle already waits would then wait until its timeout, against the README's "a call is not waited for once the browser has cancelled it". A unit row drives Inflight with a fake browser context (an EventEmitter), with no browser and no timer. settle() waits for a page's call; another page's close leaves it waiting; the call's own page closing wakes it. With a service worker's call in flight too, that page's close leaves settle waiting for the worker's call, and the test closing its context wakes it. It fails with drop's wake removed, with wake's empty check removed, with no context-close rule or one that drops only calls with a frame, and with a page close that drops every page's calls, none, or a service worker's call too. Rows: 44 (16 unit, 4 release, the rest Playwright). Signed-off-by: slayerjain --- playwright/test/unit.test.mjs | 61 +++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/playwright/test/unit.test.mjs b/playwright/test/unit.test.mjs index a8f52ba..d43d155 100644 --- a/playwright/test/unit.test.mjs +++ b/playwright/test/unit.test.mjs @@ -1,5 +1,5 @@ -// Unit rows: the agent client, the join rule and the wire names, against a -// fake agent. No browser. +// Unit rows: the agent client, the join rule, the wire names and settle's +// wake, against a fake agent and a fake browser context. No browser. import { test } from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; @@ -8,6 +8,8 @@ import os from 'node:os'; import path from 'node:path'; import { AgentClient, KeployError } from '../dist/agent.js'; import { decide } from '../dist/join.js'; +import { Inflight } from '../dist/settle.js'; +import { EventEmitter } from 'node:events'; import { testRef, outcomeOf, sourceOf, keployText } from '../dist/index.js'; // One fake agent answering every call with answer(path, body). @@ -154,6 +156,61 @@ test('given: the variable comes back; an agent without the route names the route } }); +// A fake browser context for Inflight: the events it listens to, and nothing +// else. A page's frame names its page; a service worker's request has none. +function fakeContext() { + const ctx = new EventEmitter(); + ctx.pages = () => []; + return ctx; +} +function fakeRequest(page) { + const frame = page ? { page: () => page } : null; + return { + frame: () => { if (!frame) throw new Error('no frame'); return frame; }, + isNavigationRequest: () => false, + method: () => 'GET', + url: () => 'http://localhost/slow', + }; +} +// Lets every resolved promise's callbacks run; no timer. +async function ticks() { + for (let i = 0; i < 5; i++) await Promise.resolve(); +} + +test('settle: a cancel that comes while settle already waits wakes it (its page closes; then, for a service worker\'s call, the test closes its context)', async () => { + for (const worker of [false, true]) { + const c = worker ? 'a page\'s call and a service worker\'s' : 'a page\'s call'; + const ctx = fakeContext(); + const inflight = new Inflight(ctx); + const page = new EventEmitter(); + const other = new EventEmitter(); + ctx.emit('page', page); + ctx.emit('page', other); + ctx.emit('request', fakeRequest(page)); + if (worker) ctx.emit('request', fakeRequest(null)); + let settled = false; + inflight.settled().then(() => { settled = true; }); + await ticks(); + assert.equal(settled, false, `${c}: settle waits for the call in flight`); + // Another page closing cancels none of these calls. + other.emit('close'); + await ticks(); + assert.equal(settled, false, `${c}: another page's close leaves settle waiting`); + page.emit('close'); + await ticks(); + if (worker) { + // The page's close drops only its own call: the service worker's has + // no page, so settle still waits, until the test closes its context. + assert.equal(inflight.size, 1, `${c}: the page's close leaves the service worker's call`); + assert.equal(settled, false, `${c}: settle still waits for the service worker's call`); + ctx.emit('close'); + await ticks(); + } + assert.equal(inflight.size, 0, `${c}: no call is left in flight`); + assert.equal(settled, true, `${c}: settle is woken`); + } +}); + test('decide: inert, refused, active', () => { const never = () => { throw new Error('runsInside called'); }; assert.deepEqual(decide({}, 10, never), { mode: 'inert' }); From cd02d8319164b40b437180eb9e731547fc1e04dc Mon Sep 17 00:00:00 2001 From: slayerjain Date: Sun, 1 Nov 2026 09:28:37 +0530 Subject: [PATCH 14/14] fix(playwright): name KEPLOY_PROVEN=1 as what the package needs keploy speaks scope protocol v1, this package's wire, only when it runs with KEPLOY_PROVEN=1. With the switch off, keploy's default, the agent has no POST /agent/instance route and answers the worker's claim with a plain 404, so every test of the worker fails. The README's commands had no switch, and named it only for given. - README "Use": the commands are KEPLOY_PROVEN=1 keploy mock record|replay, and the section says once that keploy speaks the package's wire (the claim, begin, end and given) only under the switch. - The claim's plain 404 (no keploy error code) is a named error: the route, the switch and the command, or a keploy from before scope protocol v1. A 404 that carries keploy's own refusal still shows it. - given's 404 names only the version (keploy v3.6.192, or an enterprise release on it): given runs only in a worker whose claim the agent took, so the switch is on there. Row: a claim answered with a plain 404 names the route and the switch; one with keploy's own refusal shows that refusal; a plain 500 keeps the generic text. given's row matches the version-only text. Signed-off-by: slayerjain --- playwright/README.md | 10 ++++++---- playwright/src/agent.ts | 29 +++++++++++++++++++++++------ playwright/test/unit.test.mjs | 21 +++++++++++++++++++-- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/playwright/README.md b/playwright/README.md index 88e78e5..e1f1de8 100644 --- a/playwright/README.md +++ b/playwright/README.md @@ -23,13 +23,15 @@ test('lists the apps', async ({ page }) => { }); ``` -Run the suite under keploy with one worker and no retries: +Run the suite under keploy with `KEPLOY_PROVEN=1`, one worker and no retries: ```sh -keploy mock record -c "npx playwright test --workers=1 --retries=0" -keploy mock replay -c "npx playwright test --workers=1 --retries=0" +KEPLOY_PROVEN=1 keploy mock record -c "npx playwright test --workers=1 --retries=0" +KEPLOY_PROVEN=1 keploy mock replay -c "npx playwright test --workers=1 --retries=0" ``` +`KEPLOY_PROVEN=1` is keploy's switch for its proven model. keploy speaks this package's wire, scope protocol v1 (the claim, begin, end and given), only under it. Without it, keploy answers the worker's claim with a plain 404, and every test of the worker fails with `keploy: this keploy agent answered 404 to POST /agent/instance, so this worker cannot claim keploy's run: ...`, which names the switch. + keploy's first release runs one test process at a time. A second worker that is still alive while another runs is refused by keploy, and its tests fail. Turn retries off: keploy sets aside a test it cannot prove, with its reason, and fails a test whose calls changed, and a retry changes neither. ## Without keploy @@ -53,7 +55,7 @@ Any answer from the agent that this package cannot trust fails the test. That co - Settle's rows run in Chromium. Which calls the browser itself ends (a removed frame's) is proven there only; Firefox and WebKit are not rowed. - Calls that `beforeAll`/`afterAll` hooks and worker fixtures make outside every test are answered from the worker's claim, one slice per Playwright project. - The `beforeAll` and `afterAll` hooks of a describe group whose tests keploy has all set aside still run, and their calls are answered from the claim. Each test is skipped at its begin, after the group's hooks have started; skipping the whole group before its hooks run needs a way to ask keploy for the set-aside tests first, which this version does not have. -- `given` needs the agent route `POST /agent/scope/given` (keploy/keploy#5016). keploy v3.6.192 and later serve it, and only when keploy runs with `KEPLOY_PROVEN=1`; with the enterprise keploy, that is an enterprise release on keploy v3.6.192 or later, run the same way. An older keploy, or one run without `KEPLOY_PROVEN=1`, answers that route with a plain 404, and `given` fails the test with `keploy: this keploy agent answered 404 to POST /agent/scope/given, so it cannot take the given value : given needs keploy v3.6.192 or later (keploy/keploy#5016), or an enterprise release on keploy v3.6.192 or later, run with KEPLOY_PROVEN=1; an older keploy, or one run without KEPLOY_PROVEN=1, answers 404 there`. Claim, begin, settle and end do not need that route. +- `given` needs the agent route `POST /agent/scope/given` (keploy/keploy#5016). keploy v3.6.192 and later serve it; with the enterprise keploy, that is an enterprise release on keploy v3.6.192 or later. An older keploy answers that route with a plain 404, and `given` fails the test with `keploy: this keploy agent answered 404 to POST /agent/scope/given, so it cannot take the given value : given needs keploy v3.6.192 or later (keploy/keploy#5016), or an enterprise release on keploy v3.6.192 or later; an older keploy answers 404 there`. Claim, begin, settle and end do not need that route. ## Releasing diff --git a/playwright/src/agent.ts b/playwright/src/agent.ts index c7b3fb0..fe2a24e 100644 --- a/playwright/src/agent.ts +++ b/playwright/src/agent.ts @@ -108,7 +108,23 @@ export class AgentClient { /** Claims keploy's run for this test process (a Playwright worker). */ async claim(key: string, dir: string): Promise { - const a = await this.post(PATH_INSTANCE, { key, dir }); + let a: Answer; + try { + a = await this.post(PATH_INSTANCE, { key, dir }); + } catch (err) { + if (err instanceof KeployError && err.status === 404 && err.code === '') { + // keploy speaks scope protocol v1 only under its proven switch. With + // it off (keploy's default) the agent has no claim route and answers + // the mux's plain 404, as a keploy from before the protocol does + // (its begin and end answer an older, unversioned wire). The claim is + // this package's first call, so it is the one that names the switch. + throw new KeployError(`keploy: this keploy agent answered 404 to POST ${PATH_INSTANCE}, so this worker cannot claim keploy's run: ` + + 'keploy speaks @keploy/playwright\'s wire, scope protocol v1, only when it runs with KEPLOY_PROVEN=1 ' + + '(KEPLOY_PROVEN=1 keploy mock record|replay -c "npx playwright test --workers=1 --retries=0"); ' + + 'a keploy run without KEPLOY_PROVEN=1, or one from before scope protocol v1, answers 404 there', 404); + } + throw err; + } if (!a.instance) { throw new KeployError('keploy: the keploy agent answered the claim of this test process without an instance'); } @@ -150,12 +166,13 @@ export class AgentClient { a = await this.post(PATH_GIVEN, source ? { name, value, source } : { name, value }); } catch (err) { if (err instanceof KeployError && err.status === 404 && err.code === '') { - // The route came after the rest of the protocol (keploy v3.6.192), - // and keploy serves it only under KEPLOY_PROVEN=1: begin, end and - // settle work on an agent without it. + // The route came after the rest of the protocol (keploy v3.6.192): + // begin, end and settle work on an agent without it. given runs only + // in a worker whose claim the agent took, so the switch is on and + // only the version can be missing. throw new KeployError(`keploy: this keploy agent answered 404 to POST ${PATH_GIVEN}, so it cannot take the given value ${name}: ` + - 'given needs keploy v3.6.192 or later (keploy/keploy#5016), or an enterprise release on keploy v3.6.192 or later, ' + - 'run with KEPLOY_PROVEN=1; an older keploy, or one run without KEPLOY_PROVEN=1, answers 404 there', 404); + 'given needs keploy v3.6.192 or later (keploy/keploy#5016), or an enterprise release on keploy v3.6.192 or later; ' + + 'an older keploy answers 404 there', 404); } throw err; } diff --git a/playwright/test/unit.test.mjs b/playwright/test/unit.test.mjs index d43d155..ca3c6d1 100644 --- a/playwright/test/unit.test.mjs +++ b/playwright/test/unit.test.mjs @@ -116,6 +116,23 @@ test('a claim without an instance is refused', async () => { } finally { await a.close(); } }); +test('a claim answered with a plain 404 names the route and the switch keploy speaks the protocol under; keploy\'s own 404 shows keploy\'s reason', async () => { + let a = await agent(() => ({ status: 404, body: '404 page not found' })); + try { + await rejects(new AgentClient(a.url, '').claim('p', '/d'), /^keploy: this keploy agent answered 404 to POST \/agent\/instance, so this worker cannot claim keploy's run: keploy speaks @keploy\/playwright's wire, scope protocol v1, only when it runs with KEPLOY_PROVEN=1 \(KEPLOY_PROVEN=1 keploy mock record\|replay -c "npx playwright test --workers=1 --retries=0"\); a keploy run without KEPLOY_PROVEN=1, or one from before scope protocol v1, answers 404 there$/); + } finally { await a.close(); } + // A 404 that carries keploy's own refusal is keploy's answer, not a missing route. + a = await agent(() => ({ status: 404, body: { error: 'no_instance', message: 'no run to claim', remedy: 'start keploy mock record' } })); + try { + await rejects(new AgentClient(a.url, '').claim('p', '/d'), /^keploy: no run to claim \(start keploy mock record\)$/); + } finally { await a.close(); } + // Any other plain answer keeps the generic text. + a = await agent(() => ({ status: 500, body: 'boom' })); + try { + await rejects(new AgentClient(a.url, '').claim('p', '/d'), /^keploy: the keploy agent answered 500 Internal Server Error to \/agent\/instance: "boom"$/); + } finally { await a.close(); } +}); + test('an unreachable agent is an error', async () => { const a = await agent(() => ({ body: {} })); await a.close(); @@ -129,7 +146,7 @@ test('an answer over 1 MiB is an error', async () => { } finally { await a.close(); } }); -test('given: the variable comes back; an agent without the route names the route, the keploy version that serves it and the switch, and keploy\'s own 404 shows keploy\'s reason', async () => { +test('given: the variable comes back; an agent without the route names the route and the keploy version that serves it, and keploy\'s own 404 shows keploy\'s reason', async () => { let a = await agent(() => ({ body: { v: 1, status: 'ok', variable: 'given:session' } })); try { assert.equal(await new AgentClient(a.url, '').given('session', 'tok-1', 'tests/a.spec.ts:3'), 'given:session'); @@ -137,7 +154,7 @@ test('given: the variable comes back; an agent without the route names the route } finally { await a.close(); } a = await agent(() => ({ status: 404, body: '404 page not found' })); try { - await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /^keploy: this keploy agent answered 404 to POST \/agent\/scope\/given, so it cannot take the given value session: given needs keploy v3\.6\.192 or later \(keploy\/keploy#5016\), or an enterprise release on keploy v3\.6\.192 or later, run with KEPLOY_PROVEN=1; an older keploy, or one run without KEPLOY_PROVEN=1, answers 404 there$/); + await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /^keploy: this keploy agent answered 404 to POST \/agent\/scope\/given, so it cannot take the given value session: given needs keploy v3\.6\.192 or later \(keploy\/keploy#5016\), or an enterprise release on keploy v3\.6\.192 or later; an older keploy answers 404 there$/); } finally { await a.close(); } // A 404 that carries keploy's own refusal is keploy's answer, not a missing route. a = await agent(() => ({ status: 404, body: { error: 'no_test', message: 'no test named a is open', remedy: 'begin it first' } }));