diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34c1092..7825830 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,24 +12,24 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 + # The repo's package manager is npm (package-lock.json, no yarn.lock). + # borales/actions-yarn@v2.1.0 ran a docker image with no yarn in it + # ("sh: yarn: not found"), so this check failed on every PR. - name: Set up Node - uses: actions/setup-node@v2 + uses: actions/setup-node@v4 with: - node-version: 14 + node-version: 22 + cache: npm - name: Install Dependencies - uses: borales/actions-yarn@v2.1.0 - with: - cmd: install + run: npm ci - name: Build Project - uses: borales/actions-yarn@v2.1.0 - with: - cmd: build + run: npm run build - name: Check commits messages uses: wagoid/commitlint-github-action@v5 diff --git a/.github/workflows/playwright-publish.yml b/.github/workflows/playwright-publish.yml new file mode 100644 index 0000000..ee8e073 --- /dev/null +++ b/.github/workflows/playwright-publish.yml @@ -0,0 +1,105 @@ +name: "@keploy/playwright publish" + +# Publishes @keploy/playwright to npm from a tag. +# +# To release X.Y.Z: set "version": "X.Y.Z" in playwright/package.json on main +# (a pull request, so the dry run below checks it), then push the tag +# playwright-vX.Y.Z on that commit. The rows run on the tagged commit first; +# the publish runs only when every leg passes, and refuses a tag that does not +# name package.json's version (scripts/release-version.mjs). npm itself +# refuses a version it already has. +# +# Push playwright-vX.Y.Z as a plain git tag, never as a GitHub release: a +# published GitHub release also starts any workflow of this repository that +# runs on releases (the root @keploy/sdk release workflow, #117), which would +# try to publish that package under this tag's name. playwright/README.md, +# "Releasing", says the same. +# +# On a pull request the same package is published as a dry run, with no +# token, so a package that would ship without its build shows before a tag. +on: + push: + tags: ["playwright-v*"] + pull_request: + paths: + - "playwright/**" + - ".github/workflows/playwright-publish.yml" + - ".github/workflows/playwright-sdk.yml" + +permissions: + contents: read + +jobs: + rows: + if: github.event_name == 'push' + uses: ./.github/workflows/playwright-sdk.yml + + publish: + name: publish to npm + if: github.event_name == 'push' + needs: rows + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # npm --provenance signs the package with this run's identity + defaults: + run: + shell: bash # -eo pipefail: a failed command before a pipe fails the step + working-directory: playwright + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + # Writes the token line for registry.npmjs.org and pins the @keploy + # scope to it (@keploy:registry), in the runner's own npmrc. + registry-url: https://registry.npmjs.org + scope: "@keploy" + - name: The tag names package.json's version + run: node scripts/release-version.mjs "$GITHUB_REF_NAME" + - name: The npm token is set + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + run: test -n "$NPM_TOKEN" || { echo "::error::the NPM_TOKEN secret is not set; refusing to publish"; exit 1; } + - name: The @keploy scope publishes to registry.npmjs.org + # npm sends a scoped package to its scope's registry, whatever + # --registry says. Read the setting npm will use, with every npmrc in + # it (a project one included), and refuse any other. + run: | + got=$(npm config get @keploy:registry) + test "$got" = https://registry.npmjs.org/ || { echo "::error::@keploy:registry is '$got', not https://registry.npmjs.org/; refusing to publish"; exit 1; } + - run: npm ci + - name: The token authenticates + # Fails here, named, on a token that is expired, revoked or pasted + # with a stray character, before anything is uploaded. + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm whoami --registry https://registry.npmjs.org/ + - name: Publish + # prepublishOnly builds dist/ first. + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish --access public --provenance + + dry-run: + name: publish dry run + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + defaults: + run: + shell: bash # -eo pipefail: a failed command before a pipe fails the step + working-directory: playwright + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: npm ci + - name: Publish, dry run + run: npm publish --dry-run --access public 2>&1 | tee "$RUNNER_TEMP/publish.log" + - name: The package carries its build + # A fresh checkout has no dist/; only prepublishOnly puts it there. + run: | + for f in dist/index.js dist/index.d.ts package.json README.md; do + grep -qE "npm notice [0-9.]+[kMG]?B +$f\$" "$RUNNER_TEMP/publish.log" || { echo "::error::the package would ship without $f"; exit 1; } + done diff --git a/.github/workflows/playwright-sdk.yml b/.github/workflows/playwright-sdk.yml new file mode 100644 index 0000000..061954d --- /dev/null +++ b/.github/workflows/playwright-sdk.yml @@ -0,0 +1,38 @@ +name: "@keploy/playwright" + +on: + pull_request: + paths: + - "playwright/**" + - ".github/workflows/playwright-sdk.yml" + push: + branches: [main] + paths: + - "playwright/**" + - ".github/workflows/playwright-sdk.yml" + # playwright-publish.yml runs these rows on a release tag before it publishes. + workflow_call: + +jobs: + rows: + name: build and rows (Node ${{ matrix.node }}) + runs-on: ubuntu-latest + strategy: + # One leg failing must not cancel the others: each Node version's + # result is its own. + fail-fast: false + matrix: + # Playwright 1.62 (the rows' version) needs Node.js 20 or later, and + # so does this package (package.json engines). + node: [20, 22, 24] + defaults: + run: + working-directory: playwright + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - run: npm ci + - run: npx playwright install --with-deps chromium + - run: npm test diff --git a/playwright/.gitignore b/playwright/.gitignore new file mode 100644 index 0000000..0f59a68 --- /dev/null +++ b/playwright/.gitignore @@ -0,0 +1,3 @@ +node_modules +dist +test-results diff --git a/playwright/README.md b/playwright/README.md new file mode 100644 index 0000000..e1f1de8 --- /dev/null +++ b/playwright/README.md @@ -0,0 +1,65 @@ +# @keploy/playwright + +Joins a Playwright suite to keploy's Universal Mocker. `keploy mock record` records the calls each test's browser and app server make; `keploy mock replay` answers them from the recording, so the suite runs without its backends. keploy judges every recorded test, and lists each test it cannot replay with a reason. + +This package tells keploy where each test begins and ends. Through it, keploy skips a test it set aside and fails a test whose calls changed. It speaks keploy's scope protocol v1, the same one go-sdk's `keploytest` speaks, and decides nothing itself. + +## Use + +```sh +npm install --save-dev @keploy/playwright +``` + +It needs Node.js 20 or later. Its rows run on Node.js 20, 22 and 24. + +Import `test` and `expect` from this package instead of `@playwright/test`. A fixture file that extends `test` extends this one: + +```ts +import { test, expect } from '@keploy/playwright'; + +test('lists the apps', async ({ page }) => { + await page.goto('/apps'); + await expect(page.getByRole('row')).toHaveCount(3); +}); +``` + +Run the suite under keploy with `KEPLOY_PROVEN=1`, one worker and no retries: + +```sh +KEPLOY_PROVEN=1 keploy mock record -c "npx playwright test --workers=1 --retries=0" +KEPLOY_PROVEN=1 keploy mock replay -c "npx playwright test --workers=1 --retries=0" +``` + +`KEPLOY_PROVEN=1` is keploy's switch for its proven model. keploy speaks this package's wire, scope protocol v1 (the claim, begin, end and given), only under it. Without it, keploy answers the worker's claim with a plain 404, and every test of the worker fails with `keploy: this keploy agent answered 404 to POST /agent/instance, so this worker cannot claim keploy's run: ...`, which names the switch. + +keploy's first release runs one test process at a time. A second worker that is still alive while another runs is refused by keploy, and its tests fail. Turn retries off: keploy sets aside a test it cannot prove, with its reason, and fails a test whose calls changed, and a retry changes neither. + +## Without keploy + +With `KEPLOY_MOCK_AGENT` unset or `off`, nothing changes. Only `keploy mock record` and `keploy mock replay` set that variable. + +## Under keploy + +- **Claim.** Each worker claims keploy's run before its first hook or test (`POST /agent/instance`). The claim uses the worker's Playwright project name as its key, or `.` for an unnamed project. A worker that Playwright starts again after a failed test claims again. +- **Begin.** Each test begins (`POST /agent/scope/begin`) before its `beforeEach` hooks. keploy builds the test's ID as `[project::]file::titles`. The file is relative to the project directory keploy names (`KEPLOY_MOCK_PROJECT`). The titles are the describe groups and the test's own title, joined with ` › `. +- **Skip.** When keploy has set a test aside, the test is skipped through Playwright's own skip, and the report shows keploy's reason (`keploy: ...`). +- **Settle.** Before the test's `context` (and its `page`) closes, the test waits until none of its pages' calls is in flight. Calls are counted by the browser's own request events, and no time window decides. A call that never ends keeps the test waiting until Playwright's test timeout fails it. The test shows a step that names each call still in flight. A call is not waited for once the browser has cancelled it, which happens when its page loads another document, its frame is removed, its page closes or the test closes its own context. A closing page cancels only its own calls: a popup that closes cancels nothing of its opener's. A route change inside the page (the history API) cancels nothing, even after an answered call. A service worker's calls are counted too. No page's change drops them, but the test closing its own context does, as it ends the worker. +- **End.** The test ends (`POST /agent/scope/end`) after its last teardown, with its own outcome. When keploy saw a call with no recorded answer, or a recorded call the test no longer made, the test fails with keploy's text. +- **Given values.** `await given(name, value)` tells keploy a value the test made itself. Examples are a session token it signs and a name it stamps with the time. keploy then follows the value wherever the test's calls carry it, as it follows an incoming request's values in a server test. Give a distinctive value, whole, as the calls carry it. `given` must be called inside a test, or in its `beforeEach` or `afterEach` hooks. It returns the value. It needs a keploy whose agent serves `POST /agent/scope/given` (see the limits below). + +Any answer from the agent that this package cannot trust fails the test. That covers an agent it cannot reach, an error answer, an unreadable answer, and an answer that does not speak scope protocol v1. Nothing is retried. + +## Limits of this version + +- Settle covers the `context` and `page` fixtures. A context the test makes with `browser.newContext()` is the test's own to close. +- Settle's rows run in Chromium. Which calls the browser itself ends (a removed frame's) is proven there only; Firefox and WebKit are not rowed. +- Calls that `beforeAll`/`afterAll` hooks and worker fixtures make outside every test are answered from the worker's claim, one slice per Playwright project. +- The `beforeAll` and `afterAll` hooks of a describe group whose tests keploy has all set aside still run, and their calls are answered from the claim. Each test is skipped at its begin, after the group's hooks have started; skipping the whole group before its hooks run needs a way to ask keploy for the set-aside tests first, which this version does not have. +- `given` needs the agent route `POST /agent/scope/given` (keploy/keploy#5016). keploy v3.6.192 and later serve it; with the enterprise keploy, that is an enterprise release on keploy v3.6.192 or later. An older keploy answers that route with a plain 404, and `given` fails the test with `keploy: this keploy agent answered 404 to POST /agent/scope/given, so it cannot take the given value : given needs keploy v3.6.192 or later (keploy/keploy#5016), or an enterprise release on keploy v3.6.192 or later; an older keploy answers 404 there`. Claim, begin, settle and end do not need that route. + +## Releasing + +1. In a pull request, set `"version": "X.Y.Z"` in `playwright/package.json`. The pull request's publish dry run checks the package. +2. Once it is merged, push the tag `playwright-vX.Y.Z` on that commit of `main`: `git tag playwright-vX.Y.Z && git push origin playwright-vX.Y.Z`. The tag runs the rows, then publishes to npm (`.github/workflows/playwright-publish.yml`). + +Push the tag as a plain git tag. Never create a GitHub release for it: a published GitHub release also starts any workflow of this repository that runs on releases, such as the root `@keploy/sdk` release workflow (keploy/typescript-sdk#117), which would try to publish that package under the tag's name and fail. diff --git a/playwright/package-lock.json b/playwright/package-lock.json new file mode 100644 index 0000000..347cd8c --- /dev/null +++ b/playwright/package-lock.json @@ -0,0 +1,118 @@ +{ + "name": "@keploy/playwright", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@keploy/playwright", + "version": "0.1.0", + "license": "Apache-2.0", + "devDependencies": { + "@playwright/test": "1.62.1", + "@types/node": "22.19.1", + "typescript": "5.9.3" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@playwright/test": ">=1.40.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz", + "integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@types/node": { + "version": "22.19.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.1.tgz", + "integrity": "sha512-LCCV0HdSZZZb34qifBsyWlUmok6W7ouER+oQIGBScS8EsZsQbrtFTUrDX4hOl+CS6p7cnNC4td+qrSVGSCTUfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/playwright/package.json b/playwright/package.json new file mode 100644 index 0000000..7959b42 --- /dev/null +++ b/playwright/package.json @@ -0,0 +1,33 @@ +{ + "name": "@keploy/playwright", + "version": "0.1.0", + "description": "Join a Playwright suite to keploy's Universal Mocker: keploy mock record and keploy mock replay judge each test.", + "license": "Apache-2.0", + "author": "Keploy ", + "repository": { + "type": "git", + "url": "git+https://github.com/keploy/typescript-sdk.git", + "directory": "playwright" + }, + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "dist" + ], + "scripts": { + "build": "tsc -p tsconfig.json", + "prepublishOnly": "npm run build", + "test": "npm run build && node --test test/*.test.mjs" + }, + "peerDependencies": { + "@playwright/test": ">=1.40.0" + }, + "devDependencies": { + "@playwright/test": "1.62.1", + "@types/node": "22.19.1", + "typescript": "5.9.3" + }, + "engines": { + "node": ">=20" + } +} diff --git a/playwright/scripts/release-version.mjs b/playwright/scripts/release-version.mjs new file mode 100644 index 0000000..84c67f0 --- /dev/null +++ b/playwright/scripts/release-version.mjs @@ -0,0 +1,30 @@ +// release-version.mjs TAG: the publish workflow's version check. +// +// TAG is the pushed git tag. It must name package.json's version exactly +// (playwright-v), so the version on npm is always the one a tagged +// commit carries. The version must be a release (x.y.z): npm 10 would make a +// prerelease the package's `latest`, so this workflow refuses one. +// +// Exit 0 with no output when the tag may publish; exit 1 and say why when not. +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +export function refusal(tag, version) { + if (!/^\d+\.\d+\.\d+$/.test(version)) { + return `package.json's version ${version} is not a release (x.y.z); this workflow publishes releases only`; + } + const want = `playwright-v${version}`; + if (tag !== want) { + return `the tag ${tag || '(none)'} does not name package.json's version ${version}: the tag for it is ${want}`; + } + return ''; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); + const why = refusal(process.argv[2] ?? '', pkg.version); + if (why) { + console.error(`refusing to publish: ${why}`); + process.exit(1); + } +} diff --git a/playwright/src/agent.ts b/playwright/src/agent.ts new file mode 100644 index 0000000..fe2a24e --- /dev/null +++ b/playwright/src/agent.ts @@ -0,0 +1,219 @@ +// The client of keploy's agent: scope protocol v1, the same wire go-sdk's +// keploytest speaks (POST /agent/instance, /agent/scope/begin, +// /agent/scope/end, /agent/scope/given). +// +// Every answer of the agent carries the protocol version as "v"; an answer +// without it, or with another version, is an error, so this package on an +// old agent (or the reverse) fails loudly instead of running unjudged. Every +// way an answer can fail to be trusted is an error. Nothing is retried, and +// no clock decides anything: the agent answers once the work it owes is done. + +export const PROTOCOL_VERSION = 1; + +export const PATH_INSTANCE = '/agent/instance'; +export const PATH_BEGIN = '/agent/scope/begin'; +export const PATH_END = '/agent/scope/end'; +export const PATH_GIVEN = '/agent/scope/given'; + +export const ACTION_SKIP = 'skip'; +export const ACTION_FAIL = 'fail'; + +// Bounds what is read of one answer, and how much of one an error quotes. +const MAX_ANSWER = 1 << 20; +const MAX_QUOTED = 512; + +/** A 2xx answer of the agent. */ +export interface Answer { + v: number; + status?: string; + action?: string; + reason?: string; + instance?: string; + timeFile?: string; + variable?: string; +} + +/** An answer this package cannot trust, or the agent's own refusal. */ +export class KeployError extends Error { + /** The HTTP status of a non-2xx answer; 0 when there was none. */ + readonly status: number; + /** The scope protocol's error code of a refusal ("not_open", ...). */ + readonly code: string; + constructor(message: string, status = 0, code = '') { + super(message); + this.name = 'KeployError'; + this.status = status; + this.code = code; + } +} + +export function quoted(text: string): string { + const s = text.trim(); + return s.length > MAX_QUOTED ? `${JSON.stringify(s.slice(0, MAX_QUOTED))}...` : JSON.stringify(s); +} + +/** Talks to one keploy agent. */ +export class AgentClient { + constructor(readonly url: string, readonly token: string) {} + + /** Sends one call and reads its answer. */ + async post(path: string, body: unknown): Promise { + const target = this.url.replace(/\/+$/, '') + path; + const headers: Record = { 'content-type': 'application/json' }; + if (this.token) headers.authorization = `Bearer ${this.token}`; + let res: Response; + try { + // A redirect is an answer like any other non-2xx: following it would + // send the call again, or to somewhere else. + res = await fetch(target, { method: 'POST', headers, body: JSON.stringify(body), redirect: 'manual' }); + } catch (err) { + throw new KeployError(`keploy: cannot reach the keploy agent at ${this.url}: ${causeOf(err)}`); + } + let data: string; + try { + data = await readBounded(res); + } catch (err) { + throw new KeployError(`keploy: cannot read the keploy agent's answer to ${path}: ${causeOf(err)}`, res.status); + } + if (res.status < 200 || res.status > 299) { + let e: { error?: unknown; message?: unknown; remedy?: unknown } = {}; + try { e = JSON.parse(data); } catch { /* not the protocol's error body */ } + if (e && typeof e.message === 'string' && e.message !== '') { + const code = typeof e.error === 'string' ? e.error : ''; + const remedy = typeof e.remedy === 'string' && e.remedy !== '' ? ` (${e.remedy})` : ''; + throw new KeployError(`keploy: ${e.message}${remedy}`, res.status, code); + } + throw new KeployError(`keploy: the keploy agent answered ${res.status} ${res.statusText} to ${path}: ${quoted(data)}`, res.status); + } + let a: unknown; + try { + a = JSON.parse(data); + } catch (err) { + throw new KeployError(`keploy: the keploy agent's answer to ${path} cannot be read (${causeOf(err)}): ${quoted(data)}`, res.status); + } + if (a === null || typeof a !== 'object' || Array.isArray(a)) { + throw new KeployError(`keploy: the keploy agent's answer to ${path} cannot be read (not an object): ${quoted(data)}`, res.status); + } + const v = (a as { v?: unknown }).v; + if (v === undefined || v === null) { + throw new KeployError(`keploy: the keploy agent answered ${path} with ${quoted(data)}, which names no scope protocol version; ` + + `this @keploy/playwright needs a keploy agent that speaks scope protocol v${PROTOCOL_VERSION}: upgrade keploy`, res.status); + } + if (v !== PROTOCOL_VERSION) { + throw new KeployError(`keploy: the keploy agent answered ${path} in scope protocol v${String(v)} (${quoted(data)}); ` + + `this @keploy/playwright speaks scope protocol v${PROTOCOL_VERSION}: use a keploy and a @keploy/playwright of the same release`, res.status); + } + return a as Answer; + } + + /** Claims keploy's run for this test process (a Playwright worker). */ + async claim(key: string, dir: string): Promise { + let a: Answer; + try { + a = await this.post(PATH_INSTANCE, { key, dir }); + } catch (err) { + if (err instanceof KeployError && err.status === 404 && err.code === '') { + // keploy speaks scope protocol v1 only under its proven switch. With + // it off (keploy's default) the agent has no claim route and answers + // the mux's plain 404, as a keploy from before the protocol does + // (its begin and end answer an older, unversioned wire). The claim is + // this package's first call, so it is the one that names the switch. + throw new KeployError(`keploy: this keploy agent answered 404 to POST ${PATH_INSTANCE}, so this worker cannot claim keploy's run: ` + + 'keploy speaks @keploy/playwright\'s wire, scope protocol v1, only when it runs with KEPLOY_PROVEN=1 ' + + '(KEPLOY_PROVEN=1 keploy mock record|replay -c "npx playwright test --workers=1 --retries=0"); ' + + 'a keploy run without KEPLOY_PROVEN=1, or one from before scope protocol v1, answers 404 there', 404); + } + throw err; + } + if (!a.instance) { + throw new KeployError('keploy: the keploy agent answered the claim of this test process without an instance'); + } + // a.timeFile names the run's clock when keploy moves it. Node and the + // browser read the clock through libc, which keploy's preload (inherited + // from the test command) moves; unlike a Go test binary, which reads it + // without libc, a Playwright worker needs no launcher to follow it. + return a; + } + + /** Asks the agent to open a test. */ + async begin(test: TestRef): Promise { + const a = await this.post(PATH_BEGIN, { ...test, canSkip: true }); + if (a.status !== 'ok') { + throw new KeployError(`keploy: the keploy agent answered the begin of ${test.name} with status ${JSON.stringify(a.status ?? '')}`); + } + if (a.action && a.action !== ACTION_SKIP) { + throw new KeployError(`keploy: the keploy agent answered the begin of ${test.name} with the unknown action ${JSON.stringify(a.action)}`); + } + return a; + } + + /** Tells the agent the test ended, with its outcome. */ + async end(test: TestRef, outcome: Outcome): Promise { + const a = await this.post(PATH_END, { ...test, outcome }); + if (a.status !== 'ok') { + throw new KeployError(`keploy: the keploy agent answered the end of ${test.name} with status ${JSON.stringify(a.status ?? '')}`); + } + if (a.action && a.action !== ACTION_FAIL) { + throw new KeployError(`keploy: the keploy agent answered the end of ${test.name} with the unknown action ${JSON.stringify(a.action)}`); + } + return a; + } + + /** Gives the open test a value it made itself. Returns the agent's variable name. */ + async given(name: string, value: string, source: string): Promise { + let a: Answer; + try { + a = await this.post(PATH_GIVEN, source ? { name, value, source } : { name, value }); + } catch (err) { + if (err instanceof KeployError && err.status === 404 && err.code === '') { + // The route came after the rest of the protocol (keploy v3.6.192): + // begin, end and settle work on an agent without it. given runs only + // in a worker whose claim the agent took, so the switch is on and + // only the version can be missing. + throw new KeployError(`keploy: this keploy agent answered 404 to POST ${PATH_GIVEN}, so it cannot take the given value ${name}: ` + + 'given needs keploy v3.6.192 or later (keploy/keploy#5016), or an enterprise release on keploy v3.6.192 or later; ' + + 'an older keploy answers 404 there', 404); + } + throw err; + } + if (a.status !== 'ok' || !a.variable) { + throw new KeployError(`keploy: the keploy agent answered the given value ${name} with ${JSON.stringify({ status: a.status ?? '', variable: a.variable ?? '' })}`); + } + return a.variable; + } +} + +/** A test's name on the wire: keploy builds its ID as [project::]file::name. */ +export interface TestRef { + name: string; + file: string; + project?: string; +} + +export type Outcome = 'passed' | 'failed' | 'skipped'; + +async function readBounded(res: Response): Promise { + if (!res.body) return ''; + const reader = res.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > MAX_ANSWER) { + await reader.cancel(); + throw new Error(`the answer is larger than ${MAX_ANSWER} bytes`); + } + chunks.push(value); + } + return Buffer.concat(chunks).toString('utf8'); +} + +function causeOf(err: unknown): string { + if (err instanceof Error) { + const c = (err as Error & { cause?: unknown }).cause; + return c instanceof Error ? `${err.message}: ${c.message}` : err.message; + } + return String(err); +} diff --git a/playwright/src/index.ts b/playwright/src/index.ts new file mode 100644 index 0000000..76a4849 --- /dev/null +++ b/playwright/src/index.ts @@ -0,0 +1,187 @@ +// @keploy/playwright joins a Playwright suite to keploy's Universal Mocker. +// +// keploy records the calls each test's browser and app server make (keploy +// mock record) and replays them (keploy mock replay), so the suite runs with +// every backend answered from the recording. keploy judges every recorded +// test and lists the ones it cannot replay, with a reason. This package is +// how a test tells keploy where it begins and ends, and how keploy skips a +// test it set aside or fails a test whose calls changed. It speaks the same +// scope protocol (v1) as go-sdk's keploytest; it decides nothing itself. +// +// Specs import test (and expect) from this package instead of +// @playwright/test: +// +// import { test, expect } from '@keploy/playwright'; +// +// Without keploy (KEPLOY_MOCK_AGENT unset or "off") nothing changes. Under +// keploy, with one worker (keploy's first release runs one test process at a +// time): +// - each worker claims keploy's run before its first test (POST +// /agent/instance), under its Playwright project's name; +// - each test begins (POST /agent/scope/begin) before its beforeEach hooks; +// a test keploy set aside is skipped with keploy's reason; +// - before the test's browser context closes, the test waits until none of +// its pages' calls is in flight (settle), so its calls end inside it; +// - the test ends (POST /agent/scope/end) after its last teardown, with +// its outcome; when keploy saw a call with no recorded answer, or a +// recorded call the test no longer made, the test fails with keploy's +// text; +// - given(name, value) tells keploy a value the test made itself (a +// session token it signs, a name it stamps with the time), which keploy +// follows where the test's calls carry it. +// Any answer of the agent this package cannot trust fails the test. + +import { test as base, expect } from '@playwright/test'; +import type { TestInfo } from '@playwright/test'; +import fs from 'node:fs'; +import path from 'node:path'; +import { ACTION_FAIL, ACTION_SKIP, AgentClient, KeployError } from './agent'; +import type { Outcome, TestRef } from './agent'; +import { ENV_TEST_PID, decide } from './join'; +import type { Joining } from './join'; +import { Inflight } from './settle'; + +export { expect, KeployError }; +export type { TestRef, Outcome }; + +let joined: Joining | null = null; +function joining(): Joining { + if (joined === null) joined = decide(process.env, process.pid); + return joined; +} + +// This worker's client once it claimed the run; null when inert. +let client: AgentClient | null = null; +// The project directory test files are named relative to. +let projectDir = ''; +// The test this worker has open, if any. +let open: TestRef | null = null; + +type WorkerFixtures = { keployWorker: void }; +type TestFixtures = { keployTest: void }; + +export const test = base.extend({ + keployWorker: [async ({}, use, workerInfo) => { + const j = joining(); + if (j.mode === 'refused') throw new KeployError(j.refusal); + if (j.mode === 'active') { + const c = new AgentClient(j.url, j.token); + projectDir = realpath(j.project || process.cwd()); + // The claim's key names the slice of the recording this worker's + // calls outside tests are answered from: one per Playwright project, + // whose workers run its tests one after another. + await c.claim(workerInfo.project.name || '.', realpath(process.cwd())); + process.env[ENV_TEST_PID] = String(process.pid); + client = c; + } + await use(); + }, { scope: 'worker', auto: true }], + + context: async ({ context }, use) => { + if (client === null) { + await use(context); + return; + } + const inflight = new Inflight(context); + await use(context); + if (inflight.size > 0) { + const calls = inflight.describe(); + const shown = calls.slice(0, 5).join(', ') + (calls.length > 5 ? `, and ${calls.length - 5} more` : ''); + await base.step(`keploy: wait for the test's ${calls.length} call(s) in flight before its browser context closes: ${shown}`, () => inflight.settled()); + } + }, + + keployTest: [async ({ keployWorker }, use, testInfo) => { + void keployWorker; + const c = client; + if (c === null) { + await use(); + return; + } + const ref = testRef(testInfo); + const b = await c.begin(ref); + if (b.action === ACTION_SKIP) { + // The framework's own skip, with keploy's reason; no end follows. + testInfo.skip(true, keployText(b.reason, 'keploy: set aside (the agent gave no reason)')); + return; + } + open = ref; + await use(); + open = null; + const e = await c.end(ref, outcomeOf(testInfo)); + if (e.action === ACTION_FAIL) throw new KeployError(keployText(e.reason, 'keploy: the keploy agent failed this test and gave no reason')); + }, { auto: true }], +}); + +/** + * given tells keploy a value the open test made itself, as an incoming + * request's value is for a server test: keploy follows it where the test's + * calls carry it, so a replay that makes a new value is compared by that + * name, never by guessing. Give a distinctive value (a name with a nonce, a + * signed token), whole as the calls carry it. Returns value. + * Without keploy it returns value and does nothing else. + */ +export async function given(name: string, value: string): Promise { + const j = joining(); + if (j.mode === 'inert') return value; + if (j.mode === 'refused') throw new KeployError(j.refusal); + const c = client; + if (c === null || open === null) { + throw new KeployError(`keploy: the value ${name} was given while no test was open: call given inside a test of @keploy/playwright's test, ` + + 'or in its beforeEach or afterEach hooks'); + } + await c.given(name, value, sourceOf(new Error().stack ?? '')); + return value; +} + +/** A test's name on keploy's wire: [project::]file::titles › joined. */ +export function testRef(ti: Pick, dir = projectDir): TestRef { + const file = toSlash(path.relative(dir || process.cwd(), realpath(ti.file))); + // titlePath starts with the file; the rest are the describe groups and + // the test's own title. + const name = ti.titlePath.slice(1).join(' › '); + const project = ti.project.name; + return project ? { name, file, project } : { name, file }; +} + +/** + * keploy's reason as the test shows it: once prefixed with "keploy: " (a + * judging reason from the agent already is; a skip reason, "/: + * ", is not), or fallback when the agent gave none, as go-sdk's + * keploytest shows it. + */ +export function keployText(reason: string | undefined, fallback: string): string { + if (!reason) return fallback; + return reason.startsWith('keploy: ') ? reason : `keploy: ${reason}`; +} + +/** The test's own verdict, as keploy's end takes it. */ +export function outcomeOf(ti: Pick): Outcome { + if (ti.status === 'skipped') return 'skipped'; + return ti.status === ti.expectedStatus ? 'passed' : 'failed'; +} + +// The caller of given, as file:line relative to the project: the first +// frame of the stack outside this package. +const SOURCE_MAX = 1024; +export function sourceOf(stack: string, dir = projectDir): string { + const own = path.dirname(__filename); + for (const line of stack.split('\n').slice(1)) { + const m = /\(?((?:file:\/\/)?[^\s()]+):(\d+):\d+\)?\s*$/.exec(line.trim()); + if (!m) continue; + let file = m[1].startsWith('file://') ? new URL(m[1]).pathname : m[1]; + if (!path.isAbsolute(file) || path.dirname(file) === own || file.startsWith('node:')) continue; + file = toSlash(path.relative(dir || process.cwd(), file)); + const s = `${file}:${m[2]}`; + return s.length > SOURCE_MAX ? `${path.basename(file)}:${m[2]}` : s; + } + return ''; +} + +function realpath(p: string): string { + try { return fs.realpathSync(p); } catch { return path.resolve(p); } +} + +function toSlash(p: string): string { + return p.split(path.sep).join('/'); +} diff --git a/playwright/src/join.ts b/playwright/src/join.ts new file mode 100644 index 0000000..e5c16ed --- /dev/null +++ b/playwright/src/join.ts @@ -0,0 +1,69 @@ +// Whether this process reports to keploy: the same rule go-sdk's keploytest +// follows. keploy mock record and keploy mock replay export +// KEPLOY_MOCK_AGENT (the agent's URL), KEPLOY_MOCK_AGENT_TOKEN and +// KEPLOY_MOCK_PROJECT (the project's directory) into the test command. +// Unset or "off", the package is inert and every test runs as it would +// without keploy. + +import fs from 'node:fs'; + +export const ENV_AGENT = 'KEPLOY_MOCK_AGENT'; +export const ENV_TOKEN = 'KEPLOY_MOCK_AGENT_TOKEN'; +export const ENV_PROJECT = 'KEPLOY_MOCK_PROJECT'; +// The claimer's pid, exported once a test process has claimed the run, so +// the processes its tests start (the browser, a helper) know they belong to +// it and do not claim the run themselves. +export const ENV_TEST_PID = 'KEPLOY_TEST_PID'; + +export type Joining = + | { mode: 'inert' } + | { mode: 'refused'; refusal: string } + | { mode: 'active'; url: string; token: string; project: string }; + +export function decide(env: NodeJS.ProcessEnv, pid: number, runsInside: (pid: number) => boolean = runsInsideProc): Joining { + const v = (env[ENV_AGENT] ?? '').trim(); + if (v === '' || v.toLowerCase() === 'off') return { mode: 'inert' }; + let u: URL | null = null; + try { u = new URL(v); } catch { u = null; } + if (!u || (u.protocol !== 'http:' && u.protocol !== 'https:') || u.host === '') { + return { + mode: 'refused', + refusal: `keploy: ${ENV_AGENT}=${JSON.stringify(v)} is neither off nor the http URL of a keploy agent; set it to off to run without keploy`, + }; + } + const tp = (env[ENV_TEST_PID] ?? '').trim(); + if (tp !== '' && tp !== String(pid)) { + // Another process joined keploy's run. This one is a member only when + // it runs inside that process; a marker naming any other process was + // inherited from elsewhere, and this process claims like any test + // process (the agent refuses a second live one). + const n = Number(tp); + if (Number.isInteger(n) && n > 0 && runsInside(n)) return { mode: 'inert' }; + } + return { mode: 'active', url: v.replace(/\/+$/, ''), token: env[ENV_TOKEN] ?? '', project: (env[ENV_PROJECT] ?? '').trim() }; +} + +// Whether this process runs inside process pid (pid is an ancestor), read +// from /proc. Without /proc (a system other than Linux) it cannot tell, and +// takes the marker at its word. +export function runsInsideProc(pid: number): boolean { + if (!fs.existsSync('/proc/self/stat')) return true; + let p = process.ppid; + for (let i = 0; i < 128 && p > 0; i++) { + if (p === pid) return true; + const next = procParent(p); + if (next === null || next === p) return false; + p = next; + } + return false; +} + +function procParent(pid: number): number | null { + let stat: string; + try { stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf8'); } catch { return null; } + // The command name, in parentheses, may hold spaces and parentheses: the + // fields after it start at the last ')'. + const rest = stat.slice(stat.lastIndexOf(')') + 2).split(' '); + const ppid = Number(rest[1]); + return Number.isInteger(ppid) ? ppid : null; +} diff --git a/playwright/src/settle.ts b/playwright/src/settle.ts new file mode 100644 index 0000000..e353d78 --- /dev/null +++ b/playwright/src/settle.ts @@ -0,0 +1,95 @@ +// Settle: before a test's browser context closes, wait until none of the +// calls its pages made is in flight, so a call the test started is answered +// inside the test instead of being cut by the close (a call keploy recorded +// whole would be cut at a different byte each run). Counted by the +// browser's own request events; no time window decides it. + +import type { BrowserContext, Frame, Page, Request } from '@playwright/test'; + +export class Inflight { + private readonly open = new Map(); + private waiters: (() => void)[] = []; + + constructor(context: BrowserContext) { + context.on('request', (r) => { + let frame: Frame | null = null; + // A service worker's call has no frame. + try { frame = r.frame(); } catch { frame = null; } + this.open.set(r, { frame, nav: r.isNavigationRequest() }); + }); + context.on('requestfinished', (r) => this.done(r)); + context.on('requestfailed', (r) => this.done(r)); + // The frames whose navigation request was answered with a document, which + // the frame commits next. + const loading = new Set(); + context.on('response', (res) => { + const r = res.request(); + if (!r.isNavigationRequest()) return; + let f: Frame | null = null; + try { f = r.frame(); } catch { f = null; } + // A 204 or 205 answer, or a download, leaves the document in place. + const keeps = res.status() === 204 || res.status() === 205 || /attachment/i.test(res.headers()['content-disposition'] ?? ''); + if (f && !keeps) loading.add(f); + }); + const watch = (p: Page): void => { + // The browser reports no end for a call whose document was replaced + // while it ran: a frame that commits a new document has cancelled + // every call the old one still had open. The new document's calls + // start after the commit. A navigation inside the document (the + // history API, a client-side route change) has no navigation request + // and replaces nothing. + p.on('framenavigated', (f) => { + if (!loading.delete(f)) return; + this.drop((o) => o.frame === f && !o.nav); + }); + // Nor for the calls of a page that closes. Only that page's calls: a + // popup that closes cancels nothing of its opener. (A removed frame's + // calls the browser does end, as failed requests.) + p.on('close', () => this.drop((o) => o.frame !== null && pageOf(o.frame) === p)); + }; + context.pages().forEach(watch); + context.on('page', watch); + // Nor for any call when the test closes its own context, which ends its + // service workers too. A service worker's call has no frame, so no + // page's close drops it; this is the only rule that does. + context.on('close', () => this.drop(() => true)); + } + + /** The calls still in flight, for a step's title. */ + describe(): string[] { + return [...this.open.keys()].map((r) => `${r.method()} ${r.url()}`); + } + + get size(): number { + return this.open.size; + } + + /** Resolves once no call is in flight. */ + settled(): Promise { + if (this.open.size === 0) return Promise.resolve(); + return new Promise((resolve) => this.waiters.push(resolve)); + } + + private done(r: Request): void { + if (this.open.delete(r)) this.wake(); + } + + private drop(gone: (o: { frame: Frame | null; nav: boolean }) => boolean): void { + let n = 0; + for (const [r, o] of this.open) { + if (gone(o)) { this.open.delete(r); n++; } + } + if (n > 0) this.wake(); + } + + private wake(): void { + if (this.open.size !== 0) return; + const w = this.waiters; + this.waiters = []; + w.forEach((f) => f()); + } +} + +function pageOf(f: Frame): Page | null { + try { return f.page(); } catch { return null; } +} diff --git a/playwright/test/fixtures/agent-says.spec.ts b/playwright/test/fixtures/agent-says.spec.ts new file mode 100644 index 0000000..d79f0f7 --- /dev/null +++ b/playwright/test/fixtures/agent-says.spec.ts @@ -0,0 +1,10 @@ +import fs from 'node:fs'; +import { test } from '../../dist'; + +test('set aside', async () => { + fs.writeFileSync(`${process.env.MARKER_DIR}/set-aside-ran`, '1'); +}); +test('judged red', async () => {}); +test('refused', async () => { + fs.writeFileSync(`${process.env.MARKER_DIR}/refused-ran`, '1'); +}); diff --git a/playwright/test/fixtures/basic.spec.ts b/playwright/test/fixtures/basic.spec.ts new file mode 100644 index 0000000..d01013c --- /dev/null +++ b/playwright/test/fixtures/basic.spec.ts @@ -0,0 +1,24 @@ +import fs from 'node:fs'; +import { test, expect, given } from '../../dist'; + +test.describe('g', () => { + // A given needs an open test: the begin comes before beforeEach. + test.beforeEach(async () => { + expect(await given('seed', 'seed-value-1')).toBe('seed-value-1'); + }); + test('a', async () => { + const joined = (process.env.KEPLOY_MOCK_AGENT ?? 'off') !== 'off'; + expect(process.env.KEPLOY_TEST_PID).toBe(joined ? String(process.pid) : undefined); + if (process.env.MARKER_DIR) fs.writeFileSync(`${process.env.MARKER_DIR}/a`, String(process.pid)); + }); + test('b', async () => { + expect(1).toBe(2); + }); + test('c', async () => { + test.skip(true, 'its own skip'); + }); + test('d', async () => { + test.fail(); + expect(1).toBe(2); + }); +}); diff --git a/playwright/test/fixtures/given-outside.spec.ts b/playwright/test/fixtures/given-outside.spec.ts new file mode 100644 index 0000000..5903b54 --- /dev/null +++ b/playwright/test/fixtures/given-outside.spec.ts @@ -0,0 +1,6 @@ +import { test, given } from '../../dist'; + +test.beforeAll(async () => { + await given('early', 'early-value-1'); +}); +test('after a given outside', async () => {}); diff --git a/playwright/test/fixtures/playwright.config.ts b/playwright/test/fixtures/playwright.config.ts new file mode 100644 index 0000000..bde48ea --- /dev/null +++ b/playwright/test/fixtures/playwright.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from '@playwright/test'; + +// The rows' Playwright project: one worker, no retries, as keploy runs a suite. +export default defineConfig({ + testDir: '.', + workers: 1, + retries: 0, + timeout: Number(process.env.PW_TIMEOUT || 30000), + reporter: [['json'], ['./steps-reporter.cjs']], + outputDir: process.env.PW_OUTPUT_DIR || 'test-results', + // PW_UNNAMED=1: one unnamed project, Playwright's default when a config + // lists none. + projects: [{ name: process.env.PW_UNNAMED ? '' : 'p1', use: { headless: true } }], +}); diff --git a/playwright/test/fixtures/settle.spec.ts b/playwright/test/fixtures/settle.spec.ts new file mode 100644 index 0000000..f4237ed --- /dev/null +++ b/playwright/test/fixtures/settle.spec.ts @@ -0,0 +1,179 @@ +import { test } from '../../dist'; + +const base = process.env.SLOW_BASE ?? ''; +// Resolves once the page server has received /slow. A spec that cancels its +// call waits for this first: the browser's request event comes before the +// call reaches the server, and the row asserts that it did. +const reached = (): Promise => fetch(`${base}/seen`).then((r) => r.text()); +test.afterEach(async () => { + if (!process.env.NO_RELEASE) await fetch(`${base}/release`); +}); +test('a call in flight when the body ends', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; +}); +test('a call its page cancelled by loading another document', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + await reached(); + await page.goto(`${base}/page?next`); +}); +test('a call in flight across a route change inside the page', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + const routed = page.waitForEvent('framenavigated'); + await page.evaluate(() => { void fetch('/slow'); history.pushState({}, '', '/page?route'); }); + await sent; + await routed; +}); +test('a call in flight across a navigation answered 204', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + const answered = page.waitForResponse(/\/nocontent$/); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + // A 204 answer to a navigation leaves the document in place: nothing is cancelled. + await page.evaluate(() => { location.href = '/nocontent'; }); + await answered; + const routed = page.waitForEvent('framenavigated'); + await page.evaluate(() => { history.pushState({}, '', '/page?route'); }); + await routed; +}); +test('a call in flight across a navigation answered 205', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + const answered = page.waitForResponse(/\/resetcontent$/); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + // A 205 answer to a navigation leaves the document in place, as a 204 does. + await page.evaluate(() => { location.href = '/resetcontent'; }); + await answered; + const routed = page.waitForEvent('framenavigated'); + await page.evaluate(() => { history.pushState({}, '', '/page?route'); }); + await routed; +}); +test('a call in flight across a download', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + const answered = page.waitForResponse(/\/download$/); + const downloaded = page.waitForEvent('download'); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + // A navigation answered with an attachment is a download: the document stays. + await page.evaluate(() => { location.href = '/download'; }); + await answered; + await downloaded; + const routed = page.waitForEvent('framenavigated'); + await page.evaluate(() => { history.pushState({}, '', '/page?route'); }); + await routed; +}); +test('a call its removed cross-site frame had in flight', async ({ page }) => { + // A frame from another site (localhost under a 127.0.0.1 page), as a + // payment form's is. The browser ends the calls it had in flight when it is + // removed. + await page.goto(`${base}/page`); + const attached = page.waitForEvent('frameattached'); + await page.evaluate((src) => new Promise((resolve) => { + const f = document.createElement('iframe'); + f.id = 'child'; + f.onload = () => resolve(); + f.src = src; + document.body.appendChild(f); + }), `${base.replace('//127.0.0.1:', '//localhost:')}/page?child`); + const child = await attached; + const sent = page.waitForRequest(/\/slow$/); + const detached = page.waitForEvent('framedetached'); + await child.evaluate(() => { void fetch('/slow'); }); + await sent; + await reached(); + await page.evaluate(() => { document.getElementById('child')?.remove(); }); + await detached; +}); +test('a call its closed page had in flight', async ({ page, context }) => { + await page.goto(`${base}/page`); + const other = await context.newPage(); + await other.goto(`${base}/page?other`); + const sent = other.waitForRequest(/\/slow$/); + await other.evaluate(() => { void fetch('/slow'); }); + await sent; + await reached(); + await other.close(); +}); +test('a document still loading after its frame commits', async ({ page }) => { + // The navigation request itself stays in flight until the document's last + // byte: committing the new document cancels only the old document's calls. + await page.goto(`${base}/slowdoc`, { waitUntil: 'commit' }); +}); +test('a call in flight while another page of the test closes', async ({ page, context }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + const other = await context.newPage(); + await other.goto(`${base}/page?other`); + await other.close(); +}); +test('a call in flight across a route change after an answered call', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + await page.evaluate(() => fetch('/page').then((r) => r.text())); + const routed = page.waitForEvent('framenavigated'); + await page.evaluate(() => { history.pushState({}, '', '/page?route'); }); + await routed; +}); +test('a call in flight when the test closes its own context', async ({ page, context }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + await reached(); + await context.close(); +}); +test('a call its service worker has in flight', async ({ page, context }) => { + // A service worker's call has no frame (Playwright's request.frame() throws + // for it). It is counted like a page's call, and no page change drops it. + await page.goto(`${base}/page`); + const sent = context.waitForEvent('request', (r) => r.url().endsWith('/slow')); + await page.evaluate(() => { void navigator.serviceWorker.register('/sw.js'); }); + await sent; +}); +test('a service worker call in flight when the test closes its own context', async ({ page, context }) => { + // Closing the context ends its service workers too. Chromium reports no end + // for the worker's call, and no page's close drops it (it has no frame). + await page.goto(`${base}/page`); + const sent = context.waitForEvent('request', (r) => r.url().endsWith('/slow')); + await page.evaluate(() => { void navigator.serviceWorker.register('/sw.js'); }); + await sent; + await reached(); + await context.close(); +}); +test('a service worker call in flight while another page of the test closes', async ({ page, context }) => { + await page.goto(`${base}/page`); + const sent = context.waitForEvent('request', (r) => r.url().endsWith('/slow')); + await page.evaluate(() => { void navigator.serviceWorker.register('/sw.js'); }); + await sent; + const other = await context.newPage(); + await other.goto(`${base}/page?other`); + await other.close(); +}); +test('a call in flight across a lowercase attachment download', async ({ page }) => { + await page.goto(`${base}/page`); + const sent = page.waitForRequest(/\/slow$/); + const answered = page.waitForResponse(/\/download-lower$/); + const downloaded = page.waitForEvent('download'); + await page.evaluate(() => { void fetch('/slow'); }); + await sent; + // The common spelling, "attachment", is a download as "Attachment" is. + await page.evaluate(() => { location.href = '/download-lower'; }); + await answered; + await downloaded; + const routed = page.waitForEvent('framenavigated'); + await page.evaluate(() => { history.pushState({}, '', '/page?route'); }); + await routed; +}); diff --git a/playwright/test/fixtures/steps-reporter.cjs b/playwright/test/fixtures/steps-reporter.cjs new file mode 100644 index 0000000..0d8555d --- /dev/null +++ b/playwright/test/fixtures/steps-reporter.cjs @@ -0,0 +1,11 @@ +// The rows' reporter of every step a test ran, hooks' and fixtures' included +// (Playwright's JSON report keeps only steps at the test's top level): one +// JSON line {test, step} per step begun, appended to STEPS_FILE. +const fs = require('node:fs'); + +class StepsReporter { + onStepBegin(test, _result, step) { + if (process.env.STEPS_FILE) fs.appendFileSync(process.env.STEPS_FILE, `${JSON.stringify({ test: test.title, step: step.title })}\n`); + } +} +module.exports = StepsReporter; diff --git a/playwright/test/release.test.mjs b/playwright/test/release.test.mjs new file mode 100644 index 0000000..cc2b7f6 --- /dev/null +++ b/playwright/test/release.test.mjs @@ -0,0 +1,39 @@ +// Rows for the publish workflow's version check (scripts/release-version.mjs). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { refusal } from '../scripts/release-version.mjs'; + +const script = fileURLToPath(new URL('../scripts/release-version.mjs', import.meta.url)); +const { version } = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); +const run = (...args) => spawnSync(process.execPath, [script, ...args], { encoding: 'utf8' }); + +test('a tag naming the version publishes', () => { + assert.equal(refusal('playwright-v1.2.3', '1.2.3'), ''); +}); + +test('a tag naming another version is refused, and names the right tag', () => { + assert.match(refusal('playwright-v1.2.4', '1.2.3'), /does not name package\.json's version 1\.2\.3: the tag for it is playwright-v1\.2\.3$/); + assert.match(refusal('playwright-v1.2.3.1', '1.2.3'), /does not name/); + assert.match(refusal('v1.2.3', '1.2.3'), /does not name/); + assert.match(refusal('', '1.2.3'), /the tag \(none\) does not name/); +}); + +test('a prerelease version is refused, even with its own tag', () => { + assert.match(refusal('playwright-v1.2.3-rc.1', '1.2.3-rc.1'), /is not a release \(x\.y\.z\)/); + assert.match(refusal('playwright-v1.2', '1.2'), /is not a release/); +}); + +test('the script reads package.json and exits 1 with the reason on a wrong tag', () => { + const ok = run(`playwright-v${version}`); + assert.equal(ok.status, 0, ok.stderr); + assert.equal(ok.stdout + ok.stderr, ''); + const bad = run('playwright-v0.0.0-not-this'); + assert.equal(bad.status, 1); + assert.match(bad.stderr, new RegExp(`^refusing to publish: the tag playwright-v0\\.0\\.0-not-this does not name package\\.json's version ${version.replace(/\./g, '\\.')}`)); + const none = run(); + assert.equal(none.status, 1); + assert.match(none.stderr, /the tag \(none\)/); +}); diff --git a/playwright/test/rows.test.mjs b/playwright/test/rows.test.mjs new file mode 100644 index 0000000..63e8373 --- /dev/null +++ b/playwright/test/rows.test.mjs @@ -0,0 +1,352 @@ +// Rows: real Playwright runs of test/fixtures against a fake keploy agent, +// which records every call and answers as each row says. The settle rows +// need Chromium (npx playwright install chromium). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import http from 'node:http'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const PKG = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const CLI = path.join(PKG, 'node_modules', '@playwright', 'test', 'cli.js'); +const CONFIG = path.join(PKG, 'test', 'fixtures', 'playwright.config.ts'); + +// What happened, in order, across the fake agent and the page server. +function journal() { + const events = []; + return { events, add: (e) => events.push(e) }; +} + +// A fake keploy agent. answer(call) returns {status, body} or undefined for +// the protocol's plain "ok". +async function fakeAgent(j, answer = () => undefined) { + const srv = http.createServer((req, res) => { + let body = ''; + req.on('data', (d) => { body += d; }); + req.on('end', () => { + const call = { path: req.url, auth: req.headers.authorization ?? '', body: body ? JSON.parse(body) : null }; + j.add({ agent: call }); + let a = answer(call); + if (!a) { + a = { body: call.path === '/agent/instance' ? { v: 1, instance: 'run', timeFile: '' } + : call.path === '/agent/scope/given' ? { v: 1, status: 'ok', variable: `given:${call.body.name}` } + : { v: 1, status: 'ok' } }; + } + res.writeHead(a.status ?? 200, { 'content-type': 'application/json' }); + res.end(JSON.stringify(a.body)); + }); + }); + await new Promise((r) => srv.listen(0, '127.0.0.1', r)); + return { url: `http://127.0.0.1:${srv.address().port}`, close: () => new Promise((r) => { srv.closeAllConnections(); srv.close(r); }) }; +} + +// The settle rows' page server: /slow and the end of /slowdoc are answered +// only once /release is called. /seen is answered once /slow has arrived, so +// a spec that cancels its call waits for the call to reach the server first: +// the browser's request event comes before that, and a call cancelled before +// it reaches the server proves nothing. +async function pageServer(j) { + let release = null; + const released = new Promise((r) => { release = r; }); + let saw = null; + const seen = new Promise((r) => { saw = r; }); + const srv = http.createServer((req, res) => { + if (req.url === '/page') { res.writeHead(200, { 'content-type': 'text/html' }); res.end('page'); return; } + if (req.url === '/release') { release(); res.end('ok'); return; } + if (req.url === '/seen') { void seen.then(() => res.end('ok')); return; } + // A service worker that calls /slow as it installs. + if (req.url === '/sw.js') { res.writeHead(200, { 'content-type': 'text/javascript' }); res.end("self.addEventListener('install', () => { void fetch('/slow'); });"); return; } + if (req.url === '/nocontent') { res.writeHead(204); res.end(); return; } + if (req.url === '/resetcontent') { res.writeHead(205); res.end(); return; } + // A disposition's type is case-insensitive (RFC 6266): the browser downloads "Attachment" too. + if (req.url === '/download') { res.writeHead(200, { 'content-type': 'application/octet-stream', 'content-disposition': 'Attachment; filename="d.bin"' }); res.end('bytes'); return; } + if (req.url === '/download-lower') { res.writeHead(200, { 'content-type': 'application/octet-stream', 'content-disposition': 'attachment; filename="e.bin"' }); res.end('bytes'); return; } + if (req.url === '/slowdoc') { + // A document whose last byte comes only once /release is called. + j.add({ page: 'slowdoc sent' }); + res.writeHead(200, { 'content-type': 'text/html' }); + res.write('loading'); + void released.then(() => res.end('')); + return; + } + if (req.url === '/slow') { + j.add({ page: 'slow sent' }); + saw(); + res.on('close', () => { if (!res.writableFinished) j.add({ page: 'slow cut' }); }); + res.on('finish', () => j.add({ page: 'slow answered' })); + void released.then(() => res.end('slow')); + return; + } + res.writeHead(404); res.end(); + }); + await new Promise((r) => srv.listen(0, '127.0.0.1', r)); + return { url: `http://127.0.0.1:${srv.address().port}`, close: () => new Promise((r) => { srv.closeAllConnections(); srv.close(r); }) }; +} + +// Runs one Playwright run of the fixture specs; returns each test's result. +async function playwright(specs, env) { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'kpw-')); + const report = path.join(tmp, 'report.json'); + const stepsFile = path.join(tmp, 'steps.jsonl'); + const childEnv = { ...process.env, PLAYWRIGHT_JSON_OUTPUT_NAME: report, PW_OUTPUT_DIR: path.join(tmp, 'out'), MARKER_DIR: tmp, STEPS_FILE: stepsFile, ...env }; + for (const k of Object.keys(childEnv)) if (childEnv[k] === undefined) delete childEnv[k]; + const child = spawn(process.execPath, [CLI, 'test', '-c', CONFIG, ...specs], { cwd: PKG, env: childEnv, stdio: ['ignore', 'pipe', 'pipe'] }); + let out = ''; + child.stdout.on('data', (d) => { out += d; }); + child.stderr.on('data', (d) => { out += d; }); + const code = await new Promise((r) => child.on('close', r)); + if (!fs.existsSync(report)) throw new Error(`playwright wrote no report (exit ${code}):\n${out}`); + const steps = fs.existsSync(stepsFile) ? fs.readFileSync(stepsFile, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)) : []; + const tests = {}; + const walk = (suite, titles) => { + for (const s of suite.suites ?? []) walk(s, s.file && s.title === s.file ? titles : [...titles, s.title]); + for (const spec of suite.specs ?? []) { + const t = spec.tests[0]; + const r = t.results[t.results.length - 1] ?? {}; + tests[[...titles, spec.title].join(' › ')] = { + status: r.status ?? 'none', + error: (r.errors ?? []).map((e) => e.message ?? '').join('\n'), + annotations: r.annotations ?? t.annotations ?? [], + steps: steps.filter((x) => x.test === spec.title).map((x) => x.step), + }; + } + }; + for (const s of JSON.parse(fs.readFileSync(report, 'utf8')).suites) walk(s, []); + return { tests, tmp, out }; +} + +const keployEnv = (url) => ({ KEPLOY_MOCK_AGENT: url, KEPLOY_MOCK_AGENT_TOKEN: 'tok', KEPLOY_MOCK_PROJECT: PKG, KEPLOY_TEST_PID: undefined }); +const calls = (j) => j.events.filter((e) => e.agent).map((e) => e.agent); +const ref = (file, name) => ({ name, file: `test/fixtures/${file}`, project: 'p1' }); + +test('inert without keploy: no call, every test runs as written', async () => { + const j = journal(); + const a = await fakeAgent(j); + try { + for (const v of [undefined, 'off']) { + const { tests, tmp } = await playwright(['basic.spec.ts'], { KEPLOY_MOCK_AGENT: v, KEPLOY_TEST_PID: undefined }); + assert.deepEqual(Object.fromEntries(Object.entries(tests).map(([k, t]) => [k, t.status])), + { 'g › a': 'passed', 'g › b': 'failed', 'g › c': 'skipped', 'g › d': 'failed' }); + assert.ok(fs.existsSync(path.join(tmp, 'a'))); + } + assert.deepEqual(calls(j), []); + } finally { await a.close(); } +}); + +test('a worker claims the run, each test begins before its beforeEach and ends with its outcome', async () => { + const j = journal(); + const a = await fakeAgent(j); + try { + const { tests, tmp } = await playwright(['basic.spec.ts'], keployEnv(a.url)); + assert.deepEqual(Object.fromEntries(Object.entries(tests).map(([k, t]) => [k, t.status])), + { 'g › a': 'passed', 'g › b': 'failed', 'g › c': 'skipped', 'g › d': 'failed' }); + const r = (n) => ref('basic.spec.ts', `g › ${n}`); + const given = { name: 'seed', value: 'seed-value-1', source: 'test/fixtures/basic.spec.ts:7' }; + const claim = { key: 'p1', dir: fs.realpathSync(PKG) }; + // b fails, so Playwright runs c and d in a new worker, which claims again. + assert.deepEqual(calls(j).map((c) => [c.path, c.body]), [ + ['/agent/instance', claim], + ['/agent/scope/begin', { ...r('a'), canSkip: true }], ['/agent/scope/given', given], ['/agent/scope/end', { ...r('a'), outcome: 'passed' }], + ['/agent/scope/begin', { ...r('b'), canSkip: true }], ['/agent/scope/given', given], ['/agent/scope/end', { ...r('b'), outcome: 'failed' }], + ['/agent/instance', claim], + ['/agent/scope/begin', { ...r('c'), canSkip: true }], ['/agent/scope/given', given], ['/agent/scope/end', { ...r('c'), outcome: 'skipped' }], + ['/agent/scope/begin', { ...r('d'), canSkip: true }], ['/agent/scope/given', given], ['/agent/scope/end', { ...r('d'), outcome: 'passed' }], + ]); + assert.ok(calls(j).every((c) => c.auth === 'Bearer tok')); + // KEPLOY_TEST_PID named the claiming worker (test a checks it). + assert.ok(fs.existsSync(path.join(tmp, 'a'))); + } finally { await a.close(); } +}); + +test('keploy skips a set-aside test with its reason, fails a judged-red one, and a refused begin fails the test', async () => { + const j = journal(); + const a = await fakeAgent(j, (c) => { + if (c.path === '/agent/scope/begin' && c.body.name === 'set aside') return { body: { v: 1, status: 'ok', action: 'skip', reason: 'flaky/request_changed: the body changed' } }; + if (c.path === '/agent/scope/end' && c.body.name === 'judged red') return { body: { v: 1, status: 'ok', action: 'fail', reason: 'keploy: GET /company had no recorded answer' } }; + if (c.path === '/agent/scope/begin' && c.body.name === 'refused') return { status: 422, body: { error: 'one_test_at_a_time', message: 'refused began while x was open', remedy: 'run one at a time' } }; + return undefined; + }); + try { + const { tests, tmp } = await playwright(['agent-says.spec.ts'], keployEnv(a.url)); + assert.equal(tests['set aside'].status, 'skipped'); + assert.deepEqual(tests['set aside'].annotations.filter((x) => x.type === 'skip').map((x) => x.description), ['keploy: flaky/request_changed: the body changed']); + assert.ok(!fs.existsSync(path.join(tmp, 'set-aside-ran')), 'the set-aside test ran'); + assert.equal(tests['judged red'].status, 'failed'); + // The agent's judging reason already starts with "keploy: "; the test shows it once (gap13 showed it twice). + assert.match(tests['judged red'].error, /keploy: GET \/company had no recorded answer/); + assert.doesNotMatch(tests['judged red'].error, /keploy: keploy:/); + assert.equal(tests.refused.status, 'failed'); + assert.match(tests.refused.error, /keploy: refused began while x was open \(run one at a time\)/); + assert.ok(!fs.existsSync(path.join(tmp, 'refused-ran')), 'the refused test ran'); + // No end for a skipped test, nor for one whose begin was refused. + assert.deepEqual(calls(j).filter((c) => c.path === '/agent/scope/end').map((c) => c.body.name), ['judged red']); + } finally { await a.close(); } +}); + +test('a refused claim fails every test of the worker, and no test begins', async () => { + const j = journal(); + const a = await fakeAgent(j, (c) => (c.path === '/agent/instance' + ? { status: 422, body: { error: 'one_test_process_at_a_time', message: 'process 2 asked to run tests while process 1 still runs them', remedy: 'run one test process at a time' } } + : undefined)); + try { + const { tests } = await playwright(['agent-says.spec.ts'], keployEnv(a.url)); + for (const t of Object.values(tests)) { + assert.equal(t.status, 'failed'); + assert.match(t.error, /keploy: process 2 asked to run tests while process 1 still runs them \(run one test process at a time\)/); + } + assert.ok(calls(j).every((c) => c.path === '/agent/instance')); + } finally { await a.close(); } +}); + +test('a malformed KEPLOY_MOCK_AGENT fails every test, naming it', async () => { + const { tests } = await playwright(['agent-says.spec.ts'], { ...keployEnv('localhost:16789') }); + for (const t of Object.values(tests)) { + assert.equal(t.status, 'failed'); + assert.match(t.error, /KEPLOY_MOCK_AGENT="localhost:16789" is neither off nor the http URL of a keploy agent/); + } +}); + +test('a value given outside a test fails, and is not sent', async () => { + const j = journal(); + const a = await fakeAgent(j); + try { + const { tests } = await playwright(['given-outside.spec.ts'], keployEnv(a.url)); + assert.equal(tests['after a given outside'].status, 'failed'); + assert.match(tests['after a given outside'].error, /the value early was given while no test was open/); + assert.ok(!calls(j).some((c) => c.path === '/agent/scope/given')); + } finally { await a.close(); } +}); + +test('settle: a call in flight when the body ends is answered before the test ends', async () => { + const j = journal(); + const a = await fakeAgent(j); + const p = await pageServer(j); + try { + const { tests, out } = await playwright(['settle.spec.ts', '-g', 'when the body ends'], { ...keployEnv(a.url), SLOW_BASE: p.url }); + assert.equal(tests['a call in flight when the body ends'].status, 'passed', out); + const order = j.events.map((e) => (e.page ? e.page : e.agent.path)); + assert.deepEqual(order.filter((e) => e.startsWith('slow') || e === '/agent/scope/end'), ['slow sent', 'slow answered', '/agent/scope/end']); + } finally { await p.close(); await a.close(); } +}); + +test('settle: a call that never ends fails the test by name, and is never cut before the end', async () => { + const j = journal(); + const a = await fakeAgent(j); + const p = await pageServer(j); + try { + const { tests, out } = await playwright(['settle.spec.ts', '-g', 'when the body ends'], { ...keployEnv(a.url), SLOW_BASE: p.url, NO_RELEASE: '1', PW_TIMEOUT: '4000' }); + const t = tests['a call in flight when the body ends']; + assert.equal(t.status, 'timedOut', out); + assert.ok(t.steps.some((s) => /^keploy: wait for the test's 1 call\(s\) in flight before its browser context closes: GET http:\/\/127\.0\.0\.1:\d+\/slow$/.test(s)), t.steps.join('\n')); + // The test did not end while its call was open and uncut. + const order = j.events.map((e) => (e.page ? e.page : e.agent.path)); + const end = order.indexOf('/agent/scope/end'); + assert.ok(end === -1 || order.slice(0, end).includes('slow cut'), `ended with the call open: ${order.join(', ')}`); + } finally { await p.close(); await a.close(); } +}); + +test('settle: a call cancelled by its page loading another document is not waited for', async () => { + const j = journal(); + const a = await fakeAgent(j); + const p = await pageServer(j); + try { + const { tests, out } = await playwright(['settle.spec.ts', '-g', 'loading another document'], { ...keployEnv(a.url), SLOW_BASE: p.url, NO_RELEASE: '1', PW_TIMEOUT: '20000' }); + assert.equal(tests['a call its page cancelled by loading another document'].status, 'passed', out); + const order = j.events.map((e) => (e.page ? e.page : e.agent.path)); + assert.deepEqual(order.filter((e) => e.startsWith('slow') || e === '/agent/scope/end'), ['slow sent', 'slow cut', '/agent/scope/end']); + } finally { await p.close(); await a.close(); } +}); + +test('settle: a route change inside the page cancels nothing, so its call in flight is waited for', async () => { + const j = journal(); + const a = await fakeAgent(j); + const p = await pageServer(j); + try { + const { tests, out } = await playwright(['settle.spec.ts', '-g', 'route change inside the page'], { ...keployEnv(a.url), SLOW_BASE: p.url, NO_RELEASE: '1', PW_TIMEOUT: '4000' }); + const t = tests['a call in flight across a route change inside the page']; + assert.equal(t.status, 'timedOut', out); + assert.ok(t.steps.some((s) => /^keploy: wait for the test's 1 call\(s\) in flight .*\/slow$/.test(s)), t.steps.join('\n')); + } finally { await p.close(); await a.close(); } +}); + +test('settle: a navigation answered 204 replaces no document, so a call in flight is still waited for', async () => { + const j = journal(); + const a = await fakeAgent(j); + const p = await pageServer(j); + try { + const { tests, out } = await playwright(['settle.spec.ts', '-g', 'navigation answered 204'], { ...keployEnv(a.url), SLOW_BASE: p.url, NO_RELEASE: '1', PW_TIMEOUT: '4000' }); + const t = tests['a call in flight across a navigation answered 204']; + assert.equal(t.status, 'timedOut', out); + assert.ok(t.steps.some((s) => /^keploy: wait for the test's 1 call\(s\) in flight .*\/slow$/.test(s)), t.steps.join('\n')); + } finally { await p.close(); await a.close(); } +}); + +test('an unnamed Playwright project claims with the key ".", and its tests name no project', async () => { + const j = journal(); + const a = await fakeAgent(j); + try { + const { tests, out } = await playwright(['agent-says.spec.ts', '-g', 'judged red'], { ...keployEnv(a.url), PW_UNNAMED: '1' }); + assert.equal(tests['judged red'].status, 'passed', out); + const r = { name: 'judged red', file: 'test/fixtures/agent-says.spec.ts' }; + assert.deepEqual(calls(j).map((c) => [c.path, c.body]), [ + ['/agent/instance', { key: '.', dir: fs.realpathSync(PKG) }], + ['/agent/scope/begin', { ...r, canSkip: true }], + ['/agent/scope/end', { ...r, outcome: 'passed' }], + ]); + } finally { await a.close(); } +}); + +// Settle rows where the browser cancelled a call: the test passes at once, +// with no wait step. Chromium ends a removed frame's calls itself; a closed +// page's it does not, and the package drops them, as it drops every call +// when the test closes its own context. A service worker's call has no +// frame, so only the context's close drops it: Chromium ends the worker +// with its context and reports no end for the call. +for (const [title, why] of [ + ['a call its removed cross-site frame had in flight', 'its cross-site frame is removed'], + ['a call its closed page had in flight', 'its page closes'], + ['a call in flight when the test closes its own context', 'the test closes its own context'], + ['a service worker call in flight when the test closes its own context', 'the test closes its own context, which ends its service worker'], +]) { + test(`settle: a call is not waited for once ${why}`, async () => { + const j = journal(); + const a = await fakeAgent(j); + const p = await pageServer(j); + try { + const { tests, out } = await playwright(['settle.spec.ts', '-g', title], { ...keployEnv(a.url), SLOW_BASE: p.url, NO_RELEASE: '1', PW_TIMEOUT: '20000' }); + const t = tests[title]; + assert.equal(t.status, 'passed', out); + assert.deepEqual(t.steps.filter((s) => s.startsWith('keploy:')), []); + assert.ok(j.events.some((e) => e.page === 'slow sent')); + } finally { await p.close(); await a.close(); } + }); +} + +// Settle rows where nothing was cancelled: the call is still waited for, and +// the test times out naming it. +for (const [title, why, call] of [ + ['a call in flight across a navigation answered 205', 'a navigation answered 205 replaces no document', '/slow'], + ['a call in flight across a download', 'a navigation answered with an attachment (a download, "Attachment") replaces no document', '/slow'], + ['a call in flight across a lowercase attachment download', 'a navigation answered with an attachment (a download, "attachment") replaces no document', '/slow'], + ['a document still loading after its frame commits', 'a navigation\'s own request stays in flight after its document commits', '/slowdoc'], + ['a call in flight while another page of the test closes', 'another page closing cancels nothing of this page', '/slow'], + ['a call in flight across a route change after an answered call', 'a route change after an answered call cancels nothing', '/slow'], + ['a call its service worker has in flight', 'a service worker\'s call, which has no frame, is counted', '/slow'], + ['a service worker call in flight while another page of the test closes', 'another page closing cancels nothing of a service worker', '/slow'], +]) { + test(`settle: ${why}, so the call is still waited for`, async () => { + const j = journal(); + const a = await fakeAgent(j); + const p = await pageServer(j); + try { + const { tests, out } = await playwright(['settle.spec.ts', '-g', title], { ...keployEnv(a.url), SLOW_BASE: p.url, NO_RELEASE: '1', PW_TIMEOUT: '4000' }); + const t = tests[title]; + assert.equal(t.status, 'timedOut', out); + const named = new RegExp(`^keploy: wait for the test's 1 call\\(s\\) in flight before its browser context closes: GET http://127\\.0\\.0\\.1:\\d+${call}$`); + assert.ok(t.steps.some((s) => named.test(s)), t.steps.join('\n')); + } finally { await p.close(); await a.close(); } + }); +} diff --git a/playwright/test/unit.test.mjs b/playwright/test/unit.test.mjs new file mode 100644 index 0000000..ca3c6d1 --- /dev/null +++ b/playwright/test/unit.test.mjs @@ -0,0 +1,292 @@ +// Unit rows: the agent client, the join rule, the wire names and settle's +// wake, against a fake agent and a fake browser context. No browser. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import http from 'node:http'; +import os from 'node:os'; +import path from 'node:path'; +import { AgentClient, KeployError } from '../dist/agent.js'; +import { decide } from '../dist/join.js'; +import { Inflight } from '../dist/settle.js'; +import { EventEmitter } from 'node:events'; +import { testRef, outcomeOf, sourceOf, keployText } from '../dist/index.js'; + +// One fake agent answering every call with answer(path, body). +async function agent(answer) { + const calls = []; + const srv = http.createServer((req, res) => { + let body = ''; + req.on('data', (d) => { body += d; }); + req.on('end', () => { + const call = { path: req.url, auth: req.headers.authorization ?? '', body: body ? JSON.parse(body) : null }; + calls.push(call); + const a = answer(call); + res.writeHead(a.status ?? 200, { 'content-type': 'application/json', ...(a.headers ?? {}) }); + res.end(typeof a.body === 'string' ? a.body : JSON.stringify(a.body)); + }); + }); + await new Promise((r) => srv.listen(0, '127.0.0.1', r)); + const url = `http://127.0.0.1:${srv.address().port}`; + return { url, calls, close: () => new Promise((r) => srv.close(r)) }; +} + +async function rejects(p, re) { + await assert.rejects(p, (err) => { + assert.ok(err instanceof KeployError, `not a KeployError: ${err}`); + assert.match(err.message, re); + return true; + }); +} + +test('the client sends the token and the protocol body, and takes v1 answers', async () => { + const a = await agent(({ path: p }) => ({ body: p === '/agent/instance' ? { v: 1, instance: 'run', timeFile: '/tmp/k/t' } : { v: 1, status: 'ok' } })); + try { + const c = new AgentClient(a.url + '/', 'tok'); + await c.claim('chromium', '/proj'); + await c.begin({ name: 'g › a', file: 'tests/a.spec.ts', project: 'chromium' }); + await c.end({ name: 'g › a', file: 'tests/a.spec.ts', project: 'chromium' }, 'passed'); + assert.deepEqual(a.calls.map((x) => [x.path, x.auth, x.body]), [ + ['/agent/instance', 'Bearer tok', { key: 'chromium', dir: '/proj' }], + ['/agent/scope/begin', 'Bearer tok', { name: 'g › a', file: 'tests/a.spec.ts', project: 'chromium', canSkip: true }], + ['/agent/scope/end', 'Bearer tok', { name: 'g › a', file: 'tests/a.spec.ts', project: 'chromium', outcome: 'passed' }], + ]); + } finally { await a.close(); } +}); + +test('no token: no authorization header', async () => { + const a = await agent(() => ({ body: { v: 1, status: 'ok' } })); + try { + await new AgentClient(a.url, '').begin({ name: 'a', file: 'a.spec.ts' }); + assert.equal(a.calls[0].auth, ''); + } finally { await a.close(); } +}); + +test('every answer the client cannot trust is an error', async () => { + const cases = [ + [{ body: { status: 'ok' } }, /names no scope protocol version.*upgrade keploy/], + [{ body: { v: 2, status: 'ok' } }, /scope protocol v2.*same release/], + [{ body: 'not json' }, /cannot be read/], + [{ body: '[1]' }, /cannot be read \(not an object\)/], + [{ body: 'null' }, /cannot be read \(not an object\)/], + [{ status: 422, body: { error: 'one_test_at_a_time', message: 'b began while a was still open', remedy: 'run them one at a time' } }, + /^keploy: b began while a was still open \(run them one at a time\)$/], + [{ status: 400, body: { error: 'bad_request', message: 'invalid' } }, /^keploy: invalid$/], + [{ status: 500, body: 'boom' }, /answered 500 .* to \/agent\/scope\/begin: "boom"/], + [{ status: 302, body: '' }, /answered 302/], + [{ body: { v: 1, status: 'busy' } }, /begin of a with status "busy"/], + [{ body: { v: 1, status: 'ok', action: 'fail' } }, /begin of a with the unknown action "fail"/], + ]; + for (const [answer, re] of cases) { + const a = await agent(() => answer); + try { + await rejects(new AgentClient(a.url, '').begin({ name: 'a', file: 'a.spec.ts' }), re); + } finally { await a.close(); } + } +}); + +test('a redirect from the agent is an error, and is not followed', async () => { + // Following it would send the call again, or to somewhere else. + const a = await agent(({ path: p }) => (p === '/elsewhere' ? { body: { v: 1, status: 'ok' } } + : { status: 307, headers: { location: '/elsewhere' }, body: '' })); + try { + await rejects(new AgentClient(a.url, '').begin({ name: 'a', file: 'a.spec.ts' }), /answered 307 Temporary Redirect to \/agent\/scope\/begin/); + assert.deepEqual(a.calls.map((x) => x.path), ['/agent/scope/begin']); + } finally { await a.close(); } +}); + +test('an end answered with a status other than ok is an error', async () => { + const a = await agent(() => ({ body: { v: 1, status: 'busy' } })); + try { + await rejects(new AgentClient(a.url, '').end({ name: 'a', file: 'a.spec.ts' }, 'passed'), /end of a with status "busy"/); + } finally { await a.close(); } +}); + +test('an end takes only fail as its action', async () => { + const a = await agent(() => ({ body: { v: 1, status: 'ok', action: 'skip' } })); + try { + await rejects(new AgentClient(a.url, '').end({ name: 'a', file: 'a.spec.ts' }, 'passed'), /end of a with the unknown action "skip"/); + } finally { await a.close(); } +}); + +test('a claim without an instance is refused', async () => { + const a = await agent(() => ({ body: { v: 1 } })); + try { + await rejects(new AgentClient(a.url, '').claim('p', '/d'), /without an instance/); + } finally { await a.close(); } +}); + +test('a claim answered with a plain 404 names the route and the switch keploy speaks the protocol under; keploy\'s own 404 shows keploy\'s reason', async () => { + let a = await agent(() => ({ status: 404, body: '404 page not found' })); + try { + await rejects(new AgentClient(a.url, '').claim('p', '/d'), /^keploy: this keploy agent answered 404 to POST \/agent\/instance, so this worker cannot claim keploy's run: keploy speaks @keploy\/playwright's wire, scope protocol v1, only when it runs with KEPLOY_PROVEN=1 \(KEPLOY_PROVEN=1 keploy mock record\|replay -c "npx playwright test --workers=1 --retries=0"\); a keploy run without KEPLOY_PROVEN=1, or one from before scope protocol v1, answers 404 there$/); + } finally { await a.close(); } + // A 404 that carries keploy's own refusal is keploy's answer, not a missing route. + a = await agent(() => ({ status: 404, body: { error: 'no_instance', message: 'no run to claim', remedy: 'start keploy mock record' } })); + try { + await rejects(new AgentClient(a.url, '').claim('p', '/d'), /^keploy: no run to claim \(start keploy mock record\)$/); + } finally { await a.close(); } + // Any other plain answer keeps the generic text. + a = await agent(() => ({ status: 500, body: 'boom' })); + try { + await rejects(new AgentClient(a.url, '').claim('p', '/d'), /^keploy: the keploy agent answered 500 Internal Server Error to \/agent\/instance: "boom"$/); + } finally { await a.close(); } +}); + +test('an unreachable agent is an error', async () => { + const a = await agent(() => ({ body: {} })); + await a.close(); + await rejects(new AgentClient(a.url, '').begin({ name: 'a', file: 'a.spec.ts' }), /cannot reach the keploy agent/); +}); + +test('an answer over 1 MiB is an error', async () => { + const a = await agent(() => ({ body: JSON.stringify({ v: 1, status: 'ok', reason: 'x'.repeat(1 << 20) }) })); + try { + await rejects(new AgentClient(a.url, '').begin({ name: 'a', file: 'a.spec.ts' }), /larger than 1048576 bytes/); + } finally { await a.close(); } +}); + +test('given: the variable comes back; an agent without the route names the route and the keploy version that serves it, and keploy\'s own 404 shows keploy\'s reason', async () => { + let a = await agent(() => ({ body: { v: 1, status: 'ok', variable: 'given:session' } })); + try { + assert.equal(await new AgentClient(a.url, '').given('session', 'tok-1', 'tests/a.spec.ts:3'), 'given:session'); + assert.deepEqual(a.calls[0].body, { name: 'session', value: 'tok-1', source: 'tests/a.spec.ts:3' }); + } finally { await a.close(); } + a = await agent(() => ({ status: 404, body: '404 page not found' })); + try { + await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /^keploy: this keploy agent answered 404 to POST \/agent\/scope\/given, so it cannot take the given value session: given needs keploy v3\.6\.192 or later \(keploy\/keploy#5016\), or an enterprise release on keploy v3\.6\.192 or later; an older keploy answers 404 there$/); + } finally { await a.close(); } + // A 404 that carries keploy's own refusal is keploy's answer, not a missing route. + a = await agent(() => ({ status: 404, body: { error: 'no_test', message: 'no test named a is open', remedy: 'begin it first' } })); + try { + await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /^keploy: no test named a is open \(begin it first\)$/); + } finally { await a.close(); } + a = await agent(() => ({ status: 422, body: { error: 'not_open', message: 'a value was given while no test was open' } })); + try { + await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /^keploy: a value was given while no test was open$/); + } finally { await a.close(); } + for (const body of [{ v: 1, status: 'ok' }, { v: 1, status: 'ok', variable: '' }, { v: 1, status: 'busy', variable: 'given:session' }]) { + a = await agent(() => ({ body })); + try { + await rejects(new AgentClient(a.url, '').given('session', 'tok-1', ''), /answered the given value session with/); + } finally { await a.close(); } + } +}); + +// A fake browser context for Inflight: the events it listens to, and nothing +// else. A page's frame names its page; a service worker's request has none. +function fakeContext() { + const ctx = new EventEmitter(); + ctx.pages = () => []; + return ctx; +} +function fakeRequest(page) { + const frame = page ? { page: () => page } : null; + return { + frame: () => { if (!frame) throw new Error('no frame'); return frame; }, + isNavigationRequest: () => false, + method: () => 'GET', + url: () => 'http://localhost/slow', + }; +} +// Lets every resolved promise's callbacks run; no timer. +async function ticks() { + for (let i = 0; i < 5; i++) await Promise.resolve(); +} + +test('settle: a cancel that comes while settle already waits wakes it (its page closes; then, for a service worker\'s call, the test closes its context)', async () => { + for (const worker of [false, true]) { + const c = worker ? 'a page\'s call and a service worker\'s' : 'a page\'s call'; + const ctx = fakeContext(); + const inflight = new Inflight(ctx); + const page = new EventEmitter(); + const other = new EventEmitter(); + ctx.emit('page', page); + ctx.emit('page', other); + ctx.emit('request', fakeRequest(page)); + if (worker) ctx.emit('request', fakeRequest(null)); + let settled = false; + inflight.settled().then(() => { settled = true; }); + await ticks(); + assert.equal(settled, false, `${c}: settle waits for the call in flight`); + // Another page closing cancels none of these calls. + other.emit('close'); + await ticks(); + assert.equal(settled, false, `${c}: another page's close leaves settle waiting`); + page.emit('close'); + await ticks(); + if (worker) { + // The page's close drops only its own call: the service worker's has + // no page, so settle still waits, until the test closes its context. + assert.equal(inflight.size, 1, `${c}: the page's close leaves the service worker's call`); + assert.equal(settled, false, `${c}: settle still waits for the service worker's call`); + ctx.emit('close'); + await ticks(); + } + assert.equal(inflight.size, 0, `${c}: no call is left in flight`); + assert.equal(settled, true, `${c}: settle is woken`); + } +}); + +test('decide: inert, refused, active', () => { + const never = () => { throw new Error('runsInside called'); }; + assert.deepEqual(decide({}, 10, never), { mode: 'inert' }); + assert.deepEqual(decide({ KEPLOY_MOCK_AGENT: ' OFF ' }, 10, never), { mode: 'inert' }); + assert.equal(decide({ KEPLOY_MOCK_AGENT: 'localhost:16789' }, 10, never).mode, 'refused'); + assert.equal(decide({ KEPLOY_MOCK_AGENT: 'ftp://h' }, 10, never).mode, 'refused'); + assert.deepEqual(decide({ KEPLOY_MOCK_AGENT: 'http://127.0.0.1:16789/', KEPLOY_MOCK_AGENT_TOKEN: 't', KEPLOY_MOCK_PROJECT: '/p' }, 10, never), + { mode: 'active', url: 'http://127.0.0.1:16789', token: 't', project: '/p' }); + // A process inside the claimer (the browser, a helper) stays inert; a + // marker naming no ancestor, or this process, is no claim of another. + const env = { KEPLOY_MOCK_AGENT: 'http://a:1', KEPLOY_TEST_PID: '7' }; + assert.deepEqual(decide(env, 10, (p) => p === 7), { mode: 'inert' }); + assert.equal(decide(env, 10, () => false).mode, 'active'); + assert.equal(decide({ ...env, KEPLOY_TEST_PID: '10' }, 10, never).mode, 'active'); + assert.equal(decide({ ...env, KEPLOY_TEST_PID: 'x' }, 10, never).mode, 'active'); +}); + +test('testRef: the file relative to the project, the titles after it, the project when named', () => { + const ti = { file: path.join(process.cwd(), 'tests', 'a.spec.ts'), titlePath: ['a.spec.ts', 'group', 'inner', 'title'], project: { name: 'chromium' } }; + assert.deepEqual(testRef(ti, process.cwd()), { name: 'group › inner › title', file: 'tests/a.spec.ts', project: 'chromium' }); + assert.deepEqual(testRef({ ...ti, project: { name: '' } }, process.cwd()), { name: 'group › inner › title', file: 'tests/a.spec.ts' }); + // A spec reached through a symbolic link is named by its real path, as the + // project directory is. + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'kpw-ref-'))); + try { + fs.mkdirSync(path.join(root, 'proj', 'tests'), { recursive: true }); + fs.writeFileSync(path.join(root, 'proj', 'tests', 'a.spec.ts'), ''); + fs.symlinkSync(path.join(root, 'proj', 'tests'), path.join(root, 'linked')); + assert.deepEqual(testRef({ ...ti, file: path.join(root, 'linked', 'a.spec.ts') }, path.join(root, 'proj')), + { name: 'group › inner › title', file: 'tests/a.spec.ts', project: 'chromium' }); + } finally { fs.rmSync(root, { recursive: true, force: true }); } +}); + +test('outcomeOf: the test\'s own verdict', () => { + assert.equal(outcomeOf({ status: 'passed', expectedStatus: 'passed' }), 'passed'); + assert.equal(outcomeOf({ status: 'failed', expectedStatus: 'passed' }), 'failed'); + assert.equal(outcomeOf({ status: 'timedOut', expectedStatus: 'passed' }), 'failed'); + assert.equal(outcomeOf({ status: 'interrupted', expectedStatus: 'passed' }), 'failed'); + assert.equal(outcomeOf({ status: 'skipped', expectedStatus: 'skipped' }), 'skipped'); + assert.equal(outcomeOf({ status: 'skipped', expectedStatus: 'passed' }), 'skipped'); + // test.fail(): failing is its pass, passing its failure. + assert.equal(outcomeOf({ status: 'failed', expectedStatus: 'failed' }), 'passed'); + assert.equal(outcomeOf({ status: 'passed', expectedStatus: 'failed' }), 'failed'); +}); + +test('sourceOf: the first frame outside the package, relative to the project', () => { + const own = path.resolve('dist', 'index.js'); + const stack = ['Error', ` at given (${own}:10:5)`, ' at /proj/tests/a.spec.ts:12:7', ' at node:internal/x:1:1'].join('\n'); + assert.equal(sourceOf(stack, '/proj'), 'tests/a.spec.ts:12'); + assert.equal(sourceOf(['Error', ' at Object. (/proj/t/b.spec.ts:3:1)'].join('\n'), '/proj'), 't/b.spec.ts:3'); + assert.equal(sourceOf(['Error', ' at x (file:///proj/t/c.spec.mjs:4:2)'].join('\n'), '/proj'), 't/c.spec.mjs:4'); + assert.equal(sourceOf('Error\n at ', '/proj'), ''); +}); + +test('keployText: keploy\'s reason once prefixed, or the fallback when it gave none', () => { + assert.equal(keployText('keploy: GET /a had no recorded answer', 'fb'), 'keploy: GET /a had no recorded answer'); + assert.equal(keployText('flaky/request_changed: the body changed', 'fb'), 'keploy: flaky/request_changed: the body changed'); + assert.equal(keployText('', 'keploy: fb'), 'keploy: fb'); + assert.equal(keployText(undefined, 'keploy: fb'), 'keploy: fb'); + // Only the whole word with its separator counts as keploy's prefix. + assert.equal(keployText('keploy:x', 'fb'), 'keploy: keploy:x'); +}); diff --git a/playwright/tsconfig.json b/playwright/tsconfig.json new file mode 100644 index 0000000..d9e9d79 --- /dev/null +++ b/playwright/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2022", "DOM"], + "declaration": true, + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src"] +} diff --git a/tsconfig.json b/tsconfig.json index 0801287..961f8da 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -97,5 +97,7 @@ /* Completeness */ // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ "skipLibCheck": true /* Skip type checking all .d.ts files. */ - } + }, + // @keploy/playwright (playwright/) is its own package, with its own build. + "exclude": ["node_modules", "dist", "playwright"] }