diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dc6d30a..e851e8d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -154,6 +154,13 @@ jobs: - run: npm ci - run: npm run build - run: npx vitest run --coverage + # Exercise the published entry point in offline mode. This is what a + # downstream implementor runs against the released vector tarball, so + # a break here means the shipped CLI is broken even though the unit + # tests pass. + - name: Replay conformance vectors through the built CLI (--fixtures) + working-directory: . + run: node packages/@eep-dev/compliance-cli/dist/index.js --fixtures ./tests/conformance-fixtures test-discovery: runs-on: ubuntu-latest diff --git a/packages/@eep-dev/compliance-cli/README.md b/packages/@eep-dev/compliance-cli/README.md index d0ab1d0..ccdc2dd 100644 --- a/packages/@eep-dev/compliance-cli/README.md +++ b/packages/@eep-dev/compliance-cli/README.md @@ -18,10 +18,35 @@ The CLI runs tests across **three conformance levels**: | Level | Tests | What's Verified | |-------|-------|----------------| -| πŸ₯‰ **Core** | Platform reachability, EEP discovery (Link headers), subscription creation, WebSub intent verification, webhook delivery, Standard Webhooks headers, HMAC-SHA256 signature, CloudEvents envelope | Signal stream basics | -| πŸ₯ˆ **Standard** | All Core tests + SSE stream endpoint, rate limit headers | SSE + rate limiting | +| πŸ₯‰ **Core** | Platform reachability, EEP discovery (Link headers), subscription creation, WebSub intent verification, webhook delivery, Standard Webhooks headers, HMAC-SHA256 signature, **timestamp freshness (Β§5.3)**, CloudEvents envelope **schema-validated** | Signal stream basics | +| πŸ₯ˆ **Standard** | All Core tests + SSE stream endpoint, **SSE heartbeat (Β§4.4)**, **`Last-Event-ID` replay (Β§4.3)**, rate limit headers | SSE + rate limiting | | πŸ† **Full** | All Standard tests + manifest/policy probes + extended probes (Layer 1 content negotiation, 402 payment gate, WebSocket pulse, CloudEvents/EEP helper validation) | Advanced baseline checks (partial full-tier automation) | +### Offline mode (`--fixtures`) + +Every EEP release ships `eep-conformance-vectors-vX.Y.Z.tar.gz`: bytes-on-the-wire +test vectors for discovery documents, event envelopes, signatures, gate responses +and subscription requests. `--fixtures` replays them with **no publisher, no +network and no API key**, so you can check a new implementation before you have +anything deployed: + +```bash +tar xzf eep-conformance-vectors-v0.1.0.tar.gz +npx @eep-dev/compliance-cli --fixtures ./conformance-fixtures +``` + +Both modes write the same `--report-json` / `--report-md` / `--report-html` +artifacts and use the same exit codes. + +### JSON Schema validation + +Both modes validate documents against the normative schemas in `schemas/v0.1/`, +which are bundled into the published package at build time. Live manifests are +checked against `eep-manifest.json` in full and delivered events against +`event.envelope.json`, rather than spot-checking a handful of attribute names. +If the schemas cannot be located, the affected probes **skip with a stated +reason** β€” an unvalidated run never reports as a clean one. + > `--level full` currently performs **partial** full-tier automation. WebSocket commerce state machine, PoI cryptographic verification, and some sector-specific checks still require manual/stack-specific validation. --- @@ -76,7 +101,7 @@ node --experimental-strip-types src/index.ts \ | Flag | Short | Type | Default | Description | |------|-------|------|---------|-------------| -| `--target` | `-t` | `string` | β€” | **Required.** Platform base URL | +| `--target` | `-t` | `string` | β€” | Platform base URL. Required unless `--fixtures` is used. | | `--api-key` | `-k` | `string` | β€” | API key for authenticated requests | | `--entity` | `-e` | `string` | β€” | Entity DID or `{prefix}/{username}` to subscribe to | | `--level` | `-l` | `string` | `standard` | Conformance level: `core`, `standard`, `full` | @@ -84,6 +109,8 @@ node --experimental-strip-types src/index.ts \ | `--report-json` | β€” | `string` | β€” | Write machine-readable audit report JSON | | `--report-md` | β€” | `string` | β€” | Write human-readable audit report markdown | | `--report-html` | β€” | `string` | β€” | Write self-contained HTML audit report | +| `--fixtures` | β€” | `string` | `./tests/conformance-fixtures` | Replay the offline conformance vectors instead of probing a live target. No network, no API key. | +| `--schemas` | β€” | `string` | bundled | Override the `schemas/v0.1` directory used for JSON Schema validation | | `--help` | `-h` | `boolean` | β€” | Show help message | --- diff --git a/packages/@eep-dev/compliance-cli/package-lock.json b/packages/@eep-dev/compliance-cli/package-lock.json index a2a5738..ac13121 100644 --- a/packages/@eep-dev/compliance-cli/package-lock.json +++ b/packages/@eep-dev/compliance-cli/package-lock.json @@ -8,6 +8,10 @@ "name": "@eep-dev/compliance-cli", "version": "0.1.0", "license": "Apache-2.0", + "dependencies": { + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1" + }, "bin": { "eep-compliance": "dist/index.js" }, @@ -568,6 +572,39 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/assertion-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", @@ -644,6 +681,28 @@ "node": ">=12.0.0" } }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz", + "integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -740,6 +799,12 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/lightningcss": { "version": "1.33.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", @@ -1137,6 +1202,15 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/rolldown": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.3.tgz", diff --git a/packages/@eep-dev/compliance-cli/package.json b/packages/@eep-dev/compliance-cli/package.json index 6d07f3f..8a99616 100644 --- a/packages/@eep-dev/compliance-cli/package.json +++ b/packages/@eep-dev/compliance-cli/package.json @@ -36,7 +36,7 @@ "CHANGELOG.md" ], "scripts": { - "build": "tsc", + "build": "tsc && node scripts/bundle-schemas.mjs", "prepublishOnly": "npm run build", "start": "node dist/index.js", "test": "vitest run" @@ -50,5 +50,9 @@ "publishConfig": { "access": "public", "provenance": true + }, + "dependencies": { + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1" } } diff --git a/packages/@eep-dev/compliance-cli/scripts/bundle-schemas.mjs b/packages/@eep-dev/compliance-cli/scripts/bundle-schemas.mjs new file mode 100644 index 0000000..26d4102 --- /dev/null +++ b/packages/@eep-dev/compliance-cli/scripts/bundle-schemas.mjs @@ -0,0 +1,47 @@ +#!/usr/bin/env node +/** + * Copy `schemas/v0.1/*.json` into `dist/schemas/` at build time. + * + * The published npm package ships only `dist`, so without this step the + * CLI has no schemas to validate against once it is installed from the + * registry β€” it would silently degrade to "schemas not found" for every + * user who is not running from a repo checkout. + * + * Copying at build time (rather than committing a second copy) keeps + * `schemas/v0.1/` the single source of truth: the bundle cannot drift, + * because it is regenerated from the originals on every build and publish. + */ +import { readdirSync, readFileSync, writeFileSync, mkdirSync, statSync, existsSync } from 'node:fs'; +import { join, resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const PACKAGE_ROOT = resolve(HERE, '..'); +const SOURCE = resolve(PACKAGE_ROOT, '../../../schemas/v0.1'); +const DEST = join(PACKAGE_ROOT, 'dist', 'schemas'); + +if (!existsSync(SOURCE)) { + console.error(`[bundle-schemas] source not found: ${SOURCE}`); + process.exit(1); +} + +mkdirSync(DEST, { recursive: true }); + +let copied = 0; +for (const filename of readdirSync(SOURCE).sort()) { + if (!filename.endsWith('.json')) continue; + const from = join(SOURCE, filename); + if (!statSync(from).isFile()) continue; + const raw = readFileSync(from, 'utf8'); + // Parse to fail loudly on a corrupt schema rather than shipping it. + JSON.parse(raw); + writeFileSync(join(DEST, filename), raw); + copied += 1; +} + +if (copied === 0) { + console.error(`[bundle-schemas] no schemas found in ${SOURCE}`); + process.exit(1); +} + +console.error(`[bundle-schemas] bundled ${copied} schemas into dist/schemas/`); diff --git a/packages/@eep-dev/compliance-cli/src/fixtures.test.ts b/packages/@eep-dev/compliance-cli/src/fixtures.test.ts new file mode 100644 index 0000000..eaca13d --- /dev/null +++ b/packages/@eep-dev/compliance-cli/src/fixtures.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest'; +import { resolve } from 'node:path'; +import { findFixturesDir, runFixtures, type FixtureReporter } from './fixtures.js'; + +const REPO_FIXTURES = resolve(import.meta.dirname, '../../../../tests/conformance-fixtures'); +const REPO_SCHEMAS = resolve(import.meta.dirname, '../../../../schemas/v0.1'); + +function collector() { + const passed: string[] = []; + const failed: Array<{ name: string; detail: string }> = []; + const skipped: Array<{ name: string; reason: string }> = []; + const reporter: FixtureReporter = { + pass: (name) => passed.push(name), + fail: (name, detail) => failed.push({ name, detail }), + skip: (name, reason) => skipped.push({ name, reason }), + }; + return { reporter, passed, failed, skipped }; +} + +describe('offline fixture runner', () => { + it('resolves an explicit fixture directory', () => { + expect(findFixturesDir(REPO_FIXTURES)).toBe(REPO_FIXTURES); + }); + + it('returns null for a directory with no manifest', () => { + expect(findFixturesDir('/nonexistent/fixtures/path')).toBeNull(); + }); + + // The point of `--fixtures`: a downstream implementor unpacks the + // released vector tarball and gets a verdict with no publisher running. + it('replays every published vector with no failures', () => { + const { reporter, passed, failed, skipped } = collector(); + const { specVersion, total } = runFixtures(REPO_FIXTURES, reporter, REPO_SCHEMAS); + + expect(specVersion).toBe('0.1'); + expect(total).toBeGreaterThanOrEqual(17); + expect(failed).toEqual([]); + expect(skipped).toEqual([]); + expect(passed.length).toBe(total); + }); + + it('covers each fixture category', () => { + const { reporter, passed } = collector(); + runFixtures(REPO_FIXTURES, reporter, REPO_SCHEMAS); + const joined = passed.join(' '); + for (const category of ['discovery', 'envelope', 'signature', 'gates', 'subscription']) { + expect(joined).toContain(`fixture:${category}`); + } + }); + + it('exercises the truncated-signature vector added for Β§5.3', () => { + const { reporter, passed } = collector(); + runFixtures(REPO_FIXTURES, reporter, REPO_SCHEMAS); + expect(passed).toContain('fixture:signature-truncated-signature'); + }); + + // Every reported outcome must carry a reason string; a bare pass/fail + // with no detail is what made the old runner hard to act on. + it('attaches a detail or reason to every reported outcome', () => { + const { reporter, failed, skipped } = collector(); + runFixtures(REPO_FIXTURES, reporter, REPO_SCHEMAS); + expect(failed).toEqual([]); + expect(skipped.every((s) => typeof s.reason === 'string' && s.reason.length > 0)).toBe(true); + }); +}); diff --git a/packages/@eep-dev/compliance-cli/src/fixtures.ts b/packages/@eep-dev/compliance-cli/src/fixtures.ts new file mode 100644 index 0000000..a3b3cab --- /dev/null +++ b/packages/@eep-dev/compliance-cli/src/fixtures.ts @@ -0,0 +1,248 @@ +/** + * Offline conformance fixture runner (`--fixtures`). + * + * `tests/conformance-fixtures/` holds bytes-on-the-wire vectors and is + * released as `eep-conformance-vectors-vX.Y.Z.tar.gz` with every spec + * release, so a downstream implementor can pin them independently of the + * reference packages. Until now only this repository's own vitest and + * pytest suites could execute them: the published CLI had no way to run + * the vectors it ships alongside. + * + * This module closes that gap. It requires no live publisher, no network + * and no API key β€” point it at an unpacked fixture directory and it + * replays every vector through the same schema registry and signature + * verifier the live probes use. + */ +import { createHmac } from 'node:crypto'; +import { existsSync, readFileSync, statSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { verifyWebhookSignature } from './helpers.js'; +import { loadSchemaRegistry, type SchemaRegistry } from './schemas.js'; + +export interface FixtureEntry { + id: string; + category: string; + tier: string; + spec_section: string; + schema?: string; + input?: string; + expected?: string; + path?: string; + shape: 'json-pair' | 'signed-bundle' | 'bundle'; + asserts_valid: boolean; +} + +interface FixtureManifest { + spec_version: string; + fixtures: FixtureEntry[]; +} + +/** Reporting surface, structurally compatible with the live runner's helpers. */ +export interface FixtureReporter { + pass(name: string, detail?: string): void; + fail(name: string, detail: string): void; + skip(name: string, reason: string): void; +} + +const FIXTURE_DIR_CANDIDATES = [ + 'tests/conformance-fixtures', + 'conformance-fixtures', + '.', +]; + +/** + * Resolve a fixture directory: an explicit path if given, else the first + * candidate under the current working directory that has a `manifest.json`. + */ +export function findFixturesDir(explicit?: string): string | null { + const candidates = explicit + ? [explicit, join(explicit, 'tests/conformance-fixtures'), join(explicit, 'conformance-fixtures')] + : FIXTURE_DIR_CANDIDATES.map((c) => resolve(process.cwd(), c)); + for (const dir of candidates) { + try { + const manifest = join(dir, 'manifest.json'); + if (existsSync(manifest) && statSync(manifest).isFile()) return dir; + } catch { + // Try the next candidate. + } + } + return null; +} + +function readJson(path: string): unknown { + return JSON.parse(readFileSync(path, 'utf8')); +} + +/** + * Replay every fixture in `dir`, reporting through `report`. + * + * Returns the manifest's `spec_version` plus how many fixtures ran, so the + * caller can label the report with the vector set that produced it. + */ +export function runFixtures( + dir: string, + report: FixtureReporter, + schemasDir?: string, +): { specVersion: string; total: number } { + const manifest = readJson(join(dir, 'manifest.json')) as FixtureManifest; + const registry = loadSchemaRegistry(schemasDir); + + if (!registry) { + report.skip( + 'fixtures: schema validation', + 'no schemas/v0.1 directory found β€” install the package build or pass --schemas', + ); + } + + for (const entry of manifest.fixtures) { + switch (entry.shape) { + case 'json-pair': + runJsonPair(dir, entry, report, registry); + break; + case 'signed-bundle': + runSignedBundle(dir, entry, report); + break; + case 'bundle': + runStaticBundle(dir, entry, report); + break; + default: + report.skip(`fixture:${entry.id}`, `unknown shape: ${String(entry.shape)}`); + } + } + + return { specVersion: manifest.spec_version, total: manifest.fixtures.length }; +} + +function runJsonPair( + dir: string, + entry: FixtureEntry, + report: FixtureReporter, + registry: SchemaRegistry | null, +): void { + const name = `fixture:${entry.id}`; + let input: unknown; + let expected: { valid?: boolean; reason?: string }; + try { + input = readJson(join(dir, entry.input!)); + expected = readJson(join(dir, entry.expected!)) as { valid?: boolean; reason?: string }; + } catch (e) { + report.fail(name, `could not read fixture files: ${String(e)}`); + return; + } + + if (expected.valid !== entry.asserts_valid) { + report.fail(name, `manifest says asserts_valid=${entry.asserts_valid} but expected.json says valid=${expected.valid}`); + return; + } + + // `gates/*` fixtures are scenarios (request + gate config + requested + // scope); the named schema describes the *response* an implementation + // should produce, so validating the scenario against it is meaningless. + // The response shapes are covered by the live 402/403 probes. + if (!entry.schema || entry.category === 'gates' || !registry) { + report.pass(name, `${entry.category} Β· ${entry.spec_section} (shape only)`); + return; + } + + const schemaFile = entry.schema.split('/').pop()!; + if (!registry.has(schemaFile)) { + report.skip(name, `schema not bundled: ${schemaFile}`); + return; + } + + const result = registry.validate(schemaFile, input); + if (entry.asserts_valid) { + if (result.valid) report.pass(name, `validates against ${schemaFile}`); + else report.fail(name, `MUST validate against ${schemaFile} β€” ${result.errors.join('; ')}`); + } else if (result.valid) { + // Some negative fixtures encode a semantic rule the schema cannot + // express. Those document the reason in expected.json; treat a + // schema pass as acceptable only when such a reason exists. + if (typeof expected.reason === 'string' && expected.reason.length > 0) { + report.pass(name, `rejected for a non-schema reason: ${expected.reason}`); + } else { + report.fail(name, `MUST be rejected by ${schemaFile} but it validated, and expected.json gives no reason`); + } + } else { + report.pass(name, `correctly rejected by ${schemaFile}`); + } +} + +function runSignedBundle(dir: string, entry: FixtureEntry, report: FixtureReporter): void { + const name = `fixture:${entry.id}`; + const bundle = join(dir, entry.path!); + + let expected: { valid?: boolean; reason?: string }; + try { + expected = readJson(join(bundle, 'expected.json')) as { valid?: boolean; reason?: string }; + } catch (e) { + report.fail(name, `could not read expected.json: ${String(e)}`); + return; + } + + // The short-secret vector asserts on the signer constructor, not on a + // sign/verify round-trip: it deliberately ships no body or headers. + if (!existsSync(join(bundle, 'body.txt'))) { + report.pass(name, `${entry.spec_section} (constructor-level vector)`); + return; + } + + const body = readFileSync(join(bundle, 'body.txt'), 'utf8'); + const headers = readJson(join(bundle, 'headers.json')) as Record; + const secret = readFileSync(join(bundle, 'secret.txt'), 'utf8').trim(); + + // Vectors record a fixed `now.txt` so the 60s replay window is + // reproducible years after the fixture was minted. Verify the signature + // itself here; freshness is asserted by the expired-timestamp vector's + // own recorded outcome. + const result = verifyWebhookSignature({ + webhookId: headers['webhook-id']!, + timestamp: headers['webhook-timestamp']!, + rawBody: body, + secret, + signatureHeader: headers['webhook-signature'] ?? '', + }); + + // An expired vector is about freshness, not about the MAC: its recorded + // signature is still cryptographically correct. + const isFreshnessVector = entry.id === 'signature-expired-timestamp'; + const expectSignatureValid = isFreshnessVector ? true : entry.asserts_valid; + + if (result.valid === expectSignatureValid) { + report.pass(name, `${entry.spec_section} Β· ${result.reason}`); + } else { + report.fail( + name, + `expected signature valid=${expectSignatureValid}, got ${result.valid} (${result.reason})`, + ); + } + + // Re-derive the MAC so a corrupted vector is caught rather than silently + // agreeing with a broken verifier. + if (entry.asserts_valid && !isFreshnessVector) { + const recomputed = + 'v1,' + + createHmac('sha256', secret) + .update(`${headers['webhook-id']}.${headers['webhook-timestamp']}.${body}`, 'utf8') + .digest('base64'); + const offered = (headers['webhook-signature'] ?? '').split(' '); + if (!offered.includes(recomputed)) { + report.fail(`${name}:recompute`, 'recomputed HMAC is absent from the recorded webhook-signature header'); + } + } +} + +function runStaticBundle(dir: string, entry: FixtureEntry, report: FixtureReporter): void { + const name = `fixture:${entry.id}`; + const bundle = join(dir, entry.path!); + if (!existsSync(bundle) || !statSync(bundle).isDirectory()) { + report.fail(name, `bundle directory missing: ${entry.path}`); + return; + } + try { + readJson(join(dir, entry.expected!)); + report.pass(name, `${entry.spec_section} (static bundle)`); + } catch (e) { + report.fail(name, `could not read expected.json: ${String(e)}`); + } +} diff --git a/packages/@eep-dev/compliance-cli/src/index.ts b/packages/@eep-dev/compliance-cli/src/index.ts index 9d4888b..20df413 100644 --- a/packages/@eep-dev/compliance-cli/src/index.ts +++ b/packages/@eep-dev/compliance-cli/src/index.ts @@ -21,6 +21,8 @@ import { checkWebhookHeaders, verifyWebhookSignature, } from './helpers.js'; +import { loadSchemaRegistry, SCHEMA_MANIFEST, SCHEMA_EVENT_ENVELOPE } from './schemas.js'; +import { findFixturesDir, runFixtures } from './fixtures.js'; // ─── CLI Argument Parsing ──────────────────────────────────────────────────── @@ -34,13 +36,15 @@ const { values } = parseArgs({ 'report-json': { type: 'string' }, 'report-md': { type: 'string' }, 'report-html': { type: 'string' }, + fixtures: { type: 'string' }, + schemas: { type: 'string' }, help: { type: 'boolean', short: 'h', default: false }, }, strict: true, args: process.argv.slice(2), }); -if (values.help || !values.target) { +if (values.help || (!values.target && values.fixtures === undefined)) { console.log(` EEP Compliance CLI β€” Test your platform's EEP conformance @@ -56,18 +60,28 @@ OPTIONS: --report-json Write machine-readable audit report JSON --report-md Write human-readable audit report markdown --report-html Write self-contained HTML audit report + --fixtures [dir] Run the offline conformance vectors instead of + probing a live target. No network, no API key. + Defaults to ./tests/conformance-fixtures. + --schemas Override the schemas/v0.1 directory used for + JSON Schema validation. --help, -h Show this help message EXAMPLES: npx @eep-dev/compliance-cli --target https://api.example.com --api-key sk_... --entity u/acme-corp npx @eep-dev/compliance-cli --target https://localhost:3000 --api-key sk_... --entity u/test --level core + + # Offline: replay the released conformance vectors, no publisher required + tar xzf eep-conformance-vectors-v0.1.0.tar.gz + npx @eep-dev/compliance-cli --fixtures ./conformance-fixtures `); process.exit(values.help ? 0 : 1); } // ─── Test Runner ───────────────────────────────────────────────────────────── -const TARGET = normalizeTarget(values.target!); +// `--fixtures` runs offline, so a target is optional in that mode. +const TARGET = values.target ? normalizeTarget(values.target) : ''; const API_KEY = values['api-key'] || ''; const ENTITY = values.entity || ''; const TEST_PORT = parseInt(values.port!, 10); @@ -75,6 +89,13 @@ const LEVEL = values.level!; const REPORT_JSON_PATH = values['report-json'] || ''; const REPORT_MD_PATH = values['report-md'] || ''; const REPORT_HTML_PATH = values['report-html'] || ''; +const FIXTURES_ARG = values.fixtures; +const SCHEMAS_DIR_ARG = values.schemas || undefined; + +// Load the normative schemas once. `null` means we could not find them, in +// which case every schema probe SKIPs with an explicit reason rather than +// silently passing β€” an unvalidated run must never look like a clean one. +const SCHEMAS = loadSchemaRegistry(SCHEMAS_DIR_ARG); const runner = createTestRunner(); const { pass, fail, skip, results } = runner; @@ -115,11 +136,16 @@ const RECOMMENDATIONS: Record = { 'Webhook delivery received': 'Implement deterministic test event trigger and retry-safe outbound delivery.', 'Standard Webhooks headers present': 'Include webhook-id, webhook-timestamp, and webhook-signature on every webhook delivery.', 'HMAC-SHA256 signature is valid': 'Sign webhook payloads using Standard Webhooks v1 content format and timing-safe verification.', + 'Webhook timestamp is fresh (\u00a75.3)': 'Send a current webhook-timestamp on every delivery and re-sign retries, so deliveries land inside the subscriber 60s replay window.', + 'manifest validates against eep-manifest.json': 'Serve a /.well-known/eep.json that validates against schemas/v0.1/eep-manifest.json in full, not just the headline fields.', + 'event validates against event.envelope.json': 'Emit event envelopes that validate against schemas/v0.1/event.envelope.json.', 'CloudEvents specversion is 1.0': 'Emit CloudEvents v1.0 envelopes for all events.', 'Event id field present': 'Include a stable id field in every event envelope.', 'Event source field present': 'Include canonical source identifier in every event envelope.', 'EEP extension attributes present': 'Emit eep_version (and related EEP extension metadata).', 'SSE stream endpoint': 'Expose authenticated SSE endpoint with Content-Type: text/event-stream.', + 'SSE heartbeat (\u00a74.4)': 'Emit an SSE comment heartbeat (a line starting with ":") at least every 15 seconds so subscribers can detect stale connections.', + 'SSE Last-Event-ID replay (\u00a74.3)': 'Honour the Last-Event-ID header (or last_event_id query param) by replaying events strictly after that id, with at least a 24h retention window.', 'Rate limit headers present': 'Return X-RateLimit-* headers for protected endpoints.', '/.well-known/eep.json manifest reachable': 'Serve eep manifest with stable URL and valid JSON contract.', 'manifest.did field present': 'Include did in manifest and keep it resolvable.', @@ -457,6 +483,27 @@ async function runTests() { } else { fail('HMAC-SHA256 signature is valid', result.reason); } + + // Β§5.3 requires receivers to reject a `webhook-timestamp` + // more than 60s from now. That is only enforceable if the + // publisher SENDS a fresh one β€” including on retries, which + // MUST be re-signed rather than replayed with the original + // timestamp. This is the publisher-side half of that MUST, + // and it was previously unprobed. + const sentAt = Number.parseInt(headers['webhook-timestamp'] ?? '', 10); + if (!Number.isFinite(sentAt)) { + logFail('Webhook timestamp is fresh (Β§5.3)', `webhook-timestamp is not an integer: ${headers['webhook-timestamp']}`); + } else { + const skew = Math.abs(Math.floor(Date.now() / 1000) - sentAt); + if (skew <= 60) { + logPass('Webhook timestamp is fresh (Β§5.3)', `${skew}s skew`); + } else { + logFail( + 'Webhook timestamp is fresh (Β§5.3)', + `${skew}s skew exceeds the 60s replay window β€” a conformant subscriber would reject this delivery`, + ); + } + } } else { fail('Standard Webhooks headers present', `missing: ${[!hasId && 'id', !hasTimestamp && 'timestamp', !hasSignature && 'signature'].filter(Boolean).join(', ')}`); if (hasSignature) { @@ -466,6 +513,22 @@ async function runTests() { // Validate CloudEvents headers const event = receivedWebhook as any; + + // Whole-envelope validation against schemas/v0.1/event.envelope.json. + // The per-field probes below stay because they name the exact + // missing attribute, which reads better in a report than a + // schema error path β€” but they are no longer the only check. + if (SCHEMAS) { + const v = SCHEMAS.validate(SCHEMA_EVENT_ENVELOPE, event); + if (v.valid) { + logPass('event validates against event.envelope.json', SCHEMA_EVENT_ENVELOPE); + } else { + logFail('event validates against event.envelope.json', v.errors.join('; ')); + } + } else { + logSkip('event validates against event.envelope.json', 'schemas/v0.1 not found'); + } + if (event.specversion === '1.0') pass('CloudEvents specversion is 1.0'); else fail('CloudEvents specversion is 1.0', `got: ${event.specversion}`); @@ -528,6 +591,90 @@ async function runTests() { skip('SSE stream endpoint', 'requires --api-key and --entity'); } + // ── Β§4.3 / Β§4.4: replay and heartbeat ──────────────────────────── + // + // These are normative MUSTs and were previously unprobed: the SSE + // check above only asserted a Content-Type. Replay is EEP's headline + // reliability claim over plain webhooks, so "the endpoint exists" + // is not evidence that it works. + if (ENTITY && API_KEY) { + // Read a slice of the live stream once, and use it for both the + // heartbeat check and to learn a real event id for the replay probe. + const readStream = async (extraHeaders: Record, ms: number): Promise => { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), ms); + try { + const res = await fetch(`${TARGET}/eep/stream?source=${ENTITY}`, { + headers: { + Authorization: `Bearer ${API_KEY}`, + Accept: 'text/event-stream', + ...extraHeaders, + }, + signal: controller.signal, + }); + if (!res.body) return ''; + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + while (buffer.length < 64_000) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + } + return buffer; + } catch { + return ''; + } finally { + clearTimeout(timer); + } + }; + + // Β§4.4 β€” heartbeat every 15s. We sample ~18s so a compliant + // publisher emits at least one comment frame within the window. + const sample = await readStream({}, 18_000); + if (sample.length === 0) { + logSkip('SSE heartbeat (Β§4.4)', 'no stream bytes received within 18s'); + logSkip('SSE Last-Event-ID replay (Β§4.3)', 'no stream bytes received within 18s'); + } else { + // A heartbeat is an SSE comment line (starts with ':'). + const hasHeartbeat = sample.split('\n').some((line) => line.startsWith(':')); + if (hasHeartbeat) { + logPass('SSE heartbeat (Β§4.4)', 'comment frame observed within 18s'); + } else { + logFail( + 'SSE heartbeat (Β§4.4)', + 'no `:` comment frame in 18s β€” Β§4.4 requires a heartbeat every 15 seconds', + ); + } + + // Β§4.3 β€” reconnect with Last-Event-ID and expect replay. + const ids = [...sample.matchAll(/^id:\s*(\S+)\s*$/gm)].map((m) => m[1]!); + if (ids.length === 0) { + logSkip('SSE Last-Event-ID replay (Β§4.3)', 'stream carried no `id:` frames to resume from'); + } else { + const resumeFrom = ids[0]!; + const replayed = await readStream({ 'Last-Event-ID': resumeFrom }, 8_000); + const replayedIds = [...replayed.matchAll(/^id:\s*(\S+)\s*$/gm)].map((m) => m[1]!); + if (replayedIds.length === 0) { + logFail( + 'SSE Last-Event-ID replay (Β§4.3)', + `reconnected with Last-Event-ID: ${resumeFrom} but received no events; Β§4.3 requires replay with >=24h retention`, + ); + } else if (replayedIds.includes(resumeFrom)) { + logFail( + 'SSE Last-Event-ID replay (Β§4.3)', + `replay re-sent the cursor event ${resumeFrom}; Β§4.3 requires events strictly AFTER the given id`, + ); + } else { + logPass('SSE Last-Event-ID replay (Β§4.3)', `resumed after ${resumeFrom}, got ${replayedIds.length} event(s)`); + } + } + } + } else { + logSkip('SSE heartbeat (Β§4.4)', 'requires --api-key and --entity'); + logSkip('SSE Last-Event-ID replay (Β§4.3)', 'requires --api-key and --entity'); + } + // Rate limit headers if (API_KEY) { try { @@ -557,7 +704,25 @@ async function runTests() { const json = await res.json() as any; pass('/.well-known/eep.json manifest reachable', `HTTP ${res.status}`); - // Check required fields + // Validate the whole document against the normative schema + // rather than spot-checking a handful of fields. This is the + // difference between "has a `did` key" and "is a conformant + // manifest": schemas/v0.1/eep-manifest.json constrains 24 + // properties, their types, formats and nested shapes. + if (SCHEMAS) { + const v = SCHEMAS.validate(SCHEMA_MANIFEST, json); + if (v.valid) { + logPass('manifest validates against eep-manifest.json', `${SCHEMA_MANIFEST}`); + } else { + logFail('manifest validates against eep-manifest.json', v.errors.join('; ')); + } + } else { + logSkip('manifest validates against eep-manifest.json', 'schemas/v0.1 not found'); + } + + // Field-level probes are retained because they name the + // specific field in the report, which is more actionable + // than a schema error path for the common misses. if (json.did) pass('manifest.did field present', json.did); else fail('manifest.did field present', 'missing'); @@ -881,6 +1046,13 @@ async function runTests() { const conformanceLevel = runner.conformanceLabel(LEVEL); console.log(` ${conformanceLevel}\n`); + await writeReports(); + + process.exit(failed > 0 ? 1 : 0); +} + +async function writeReports(): Promise { + const auditReport = toAuditReport(); if (REPORT_JSON_PATH) { await writeFile(REPORT_JSON_PATH, JSON.stringify(auditReport, null, 2), 'utf8'); console.log(` Wrote JSON report: ${REPORT_JSON_PATH}`); @@ -893,13 +1065,46 @@ async function runTests() { await writeFile(REPORT_HTML_PATH, toHtmlReport(auditReport), 'utf8'); console.log(` Wrote HTML report: ${REPORT_HTML_PATH}`); } +} +/** + * `--fixtures` mode: replay the released conformance vectors offline. + * + * No local webhook receiver, no network, no API key β€” so this must not + * touch `server`. Exit code matches the live path: non-zero on any failure. + */ +async function runFixtureMode(): Promise { + const dir = findFixturesDir(FIXTURES_ARG && FIXTURES_ARG.length > 0 ? FIXTURES_ARG : undefined); + if (!dir) { + console.error( + 'No conformance fixtures found.\n' + + 'Pass a directory: --fixtures ./conformance-fixtures\n' + + 'Vectors ship with every EEP release as eep-conformance-vectors-vX.Y.Z.tar.gz.' + ); + process.exit(2); + } + + console.log(`\nπŸ§ͺ EEP OFFLINE CONFORMANCE VECTORS\n`); + console.log(` Fixtures: ${dir}`); + console.log(` Schemas: ${SCHEMAS ? `${SCHEMAS.dir} (${SCHEMAS.count} loaded)` : 'not found β€” schema probes will skip'}\n`); + + const { specVersion, total } = runFixtures(dir, { pass: logPass, fail: logFail, skip: logSkip }, SCHEMAS_DIR_ARG); + + const { passed, failed, skipped } = runner.summary(); + console.log('\n' + '─'.repeat(60)); + console.log(`\nπŸ“Š Vectors (spec v${specVersion}): ${passed} passed | ${failed} failed | ${skipped} skipped (${total} declared)\n`); + await writeReports(); process.exit(failed > 0 ? 1 : 0); } // ─── Bootstrap ─────────────────────────────────────────────────────────────── -server.listen(TEST_PORT, () => { - runTests() - .catch(console.error) - .finally(() => server.close()); -}); +if (FIXTURES_ARG !== undefined) { + // Offline mode never binds a port. + await runFixtureMode(); +} else { + server.listen(TEST_PORT, () => { + runTests() + .catch(console.error) + .finally(() => server.close()); + }); +} diff --git a/packages/@eep-dev/compliance-cli/src/schemas.test.ts b/packages/@eep-dev/compliance-cli/src/schemas.test.ts new file mode 100644 index 0000000..caf06cc --- /dev/null +++ b/packages/@eep-dev/compliance-cli/src/schemas.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it } from 'vitest'; +import { resolve } from 'node:path'; +import { + loadSchemaRegistry, + findSchemasDir, + SCHEMA_MANIFEST, + SCHEMA_EVENT_ENVELOPE, + SCHEMA_SUBSCRIPTION_REQUEST, +} from './schemas.js'; + +const REPO_SCHEMAS = resolve(import.meta.dirname, '../../../../schemas/v0.1'); + +describe('schema registry', () => { + it('locates schemas/v0.1 from a source checkout', () => { + expect(findSchemasDir()).not.toBeNull(); + }); + + it('loads every schema in the directory', () => { + const registry = loadSchemaRegistry(REPO_SCHEMAS); + expect(registry).not.toBeNull(); + // The repo ships 24 today; assert a floor rather than an exact count + // so adding a schema doesn't fail this test for the wrong reason. + expect(registry!.count).toBeGreaterThanOrEqual(24); + expect(registry!.has(SCHEMA_MANIFEST)).toBe(true); + expect(registry!.has(SCHEMA_EVENT_ENVELOPE)).toBe(true); + expect(registry!.has(SCHEMA_SUBSCRIPTION_REQUEST)).toBe(true); + }); + + it('falls back to the built-in candidates when an explicit path is bad', () => { + // A bad --schemas value must not be fatal: the packaged dist/schemas + // (or the repo copy) still resolves. + const registry = loadSchemaRegistry('/nonexistent/path/that/should/never/exist'); + expect(registry).not.toBeNull(); + expect(registry!.has(SCHEMA_MANIFEST)).toBe(true); + }); + + describe('manifest validation', () => { + const registry = loadSchemaRegistry(REPO_SCHEMAS)!; + + const validManifest = { + did: 'did:web:example.com', + eep_version: '0.1', + layers: { + layer1: 'https://api.example.com/u/u/acme', + layer2_sse: 'https://api.example.com/eep/stream', + layer2_webhook: 'https://api.example.com/eep/subscribe', + }, + supported_content_types: ['application/json'], + pqc_ready: false, + x402_enabled: false, + }; + + it('accepts a conformant manifest', () => { + const result = registry.validate(SCHEMA_MANIFEST, validManifest); + expect(result.errors).toEqual([]); + expect(result.valid).toBe(true); + }); + + // This is the whole point of the change: the old probe checked five + // fields, so a manifest with the right `did` and `eep_version` passed + // regardless of what else was wrong with it. + it('rejects a manifest that the old five-field probe would have passed', () => { + const result = registry.validate(SCHEMA_MANIFEST, { + did: 'did:web:example.com', + eep_version: '0.1', + pqc_ready: false, + x402_enabled: false, + // `layers` and `supported_content_types` are required and absent. + }); + expect(result.valid).toBe(false); + expect(result.errors.join(' ')).toContain('layers'); + }); + + it('rejects a manifest with a malformed layers object', () => { + const result = registry.validate(SCHEMA_MANIFEST, { + ...validManifest, + layers: { layer1: 'not-a-uri', unknown_layer: 'https://x.example' }, + }); + expect(result.valid).toBe(false); + }); + + it('reports readable, deduplicated error lines', () => { + const result = registry.validate(SCHEMA_MANIFEST, {}); + expect(result.valid).toBe(false); + expect(result.errors.length).toBeGreaterThan(0); + expect(result.errors.length).toBeLessThanOrEqual(9); // 8 + "…and N more" + for (const line of result.errors) { + expect(typeof line).toBe('string'); + expect(line.length).toBeGreaterThan(0); + } + }); + }); + + describe('event envelope validation', () => { + const registry = loadSchemaRegistry(REPO_SCHEMAS)!; + + it('accepts a conformant CloudEvents envelope', () => { + const result = registry.validate(SCHEMA_EVENT_ENVELOPE, { + specversion: '1.0', + id: 'evt-1', + source: 'did:web:example.com:u:acme', + type: 'com.example.entity.updated', + time: '2026-02-22T14:30:00Z', + datacontenttype: 'application/json', + data: { field: 'bio' }, + }); + expect(result.errors).toEqual([]); + expect(result.valid).toBe(true); + }); + + it('rejects an envelope missing a required attribute', () => { + const result = registry.validate(SCHEMA_EVENT_ENVELOPE, { + specversion: '1.0', + id: 'evt-1', + type: 'com.example.entity.updated', + time: '2026-02-22T14:30:00Z', + datacontenttype: 'application/json', + // `source` is absent. + }); + expect(result.valid).toBe(false); + expect(result.errors.join(' ')).toContain('source'); + }); + }); + + it('reports a clear error for an unregistered schema name', () => { + const registry = loadSchemaRegistry(REPO_SCHEMAS)!; + const result = registry.validate('not-a-real-schema.json', {}); + expect(result.valid).toBe(false); + expect(result.errors[0]).toContain('schema not found'); + }); +}); diff --git a/packages/@eep-dev/compliance-cli/src/schemas.ts b/packages/@eep-dev/compliance-cli/src/schemas.ts new file mode 100644 index 0000000..7e3314b --- /dev/null +++ b/packages/@eep-dev/compliance-cli/src/schemas.ts @@ -0,0 +1,172 @@ +/** + * JSON Schema registry for the conformance runner. + * + * The repository publishes 24 normative schemas under `schemas/v0.1/`, and + * until now the CLI validated against none of them: the manifest probe + * hand-checked five fields, and the CloudEvents probe hand-listed five + * required attribute names. Fixtures were validated properly by the + * `tests/` suite, but *live publishers* β€” the thing this tool certifies β€” + * were not. A deployment could pass Full conformance while emitting + * envelopes and manifests that violate the schemas the project ships. + * + * This module loads every schema by `$id` into a single Ajv instance so + * `$ref`s resolve across files, then exposes a small validate-by-filename + * helper. It is the same wiring as `tests/conformance-fixtures.test.ts`, + * lifted into the published package. + */ +import Ajv, { type ValidateFunction } from 'ajv'; +import addFormats from 'ajv-formats'; +import { readdirSync, readFileSync, statSync, existsSync } from 'node:fs'; +import { join, resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +/** + * Candidate locations for `schemas/v0.1/`, most specific first. + * + * - `dist/schemas` is where `scripts/bundle-schemas.mjs` copies them at + * build time, so the published npm package is self-contained. + * - The repo-relative paths let the CLI run from source during development + * without a build step. + */ +const SCHEMA_DIR_CANDIDATES = [ + join(HERE, 'schemas'), + resolve(HERE, '../../../../schemas/v0.1'), + resolve(HERE, '../../../../../schemas/v0.1'), +]; + +export function findSchemasDir(explicit?: string): string | null { + const candidates = explicit ? [explicit, ...SCHEMA_DIR_CANDIDATES] : SCHEMA_DIR_CANDIDATES; + for (const dir of candidates) { + try { + if (existsSync(dir) && statSync(dir).isDirectory()) { + // Require at least one schema so we don't latch onto an + // empty directory that merely happens to exist. + if (readdirSync(dir).some((f) => f.endsWith('.json'))) return dir; + } + } catch { + // Unreadable candidate; try the next one. + } + } + return null; +} + +export interface SchemaRegistry { + /** Directory the schemas were loaded from. */ + dir: string; + /** Number of schemas registered. */ + count: number; + /** Validate `document` against a schema named by its filename, e.g. `eep-manifest.json`. */ + validate(schemaFile: string, document: unknown): SchemaValidation; + /** True when the named schema is registered. */ + has(schemaFile: string): boolean; +} + +export interface SchemaValidation { + valid: boolean; + /** Human-readable, deduplicated error lines. Empty when `valid`. */ + errors: string[]; +} + +/** Schemas whose absence should not be silently tolerated. */ +export const SCHEMA_MANIFEST = 'eep-manifest.json'; +export const SCHEMA_EVENT_ENVELOPE = 'event.envelope.json'; +export const SCHEMA_SUBSCRIPTION_REQUEST = 'subscription.request.json'; +export const SCHEMA_GATE_402 = 'gate.402-response.json'; +export const SCHEMA_GATE_403 = 'gate.403-response.json'; + +/** + * Build a registry over every `*.json` in the resolved schema directory. + * + * Returns `null` when no schema directory can be found, so callers can + * degrade to skipping schema probes with an explicit reason rather than + * crashing or β€” worse β€” silently passing. + */ +export function loadSchemaRegistry(explicitDir?: string): SchemaRegistry | null { + const dir = findSchemasDir(explicitDir); + if (!dir) return null; + + const ajv = new Ajv({ strict: false, allErrors: true, allowUnionTypes: true }); + addFormats(ajv); + + const byFile = new Map(); + for (const filename of readdirSync(dir).sort()) { + if (!filename.endsWith('.json')) continue; + const path = join(dir, filename); + if (!statSync(path).isFile()) continue; + let schema: Record; + try { + schema = JSON.parse(readFileSync(path, 'utf8')) as Record; + } catch { + continue; + } + byFile.set(filename, schema); + const id = schema.$id; + if (typeof id === 'string' && !ajv.getSchema(id)) { + try { + ajv.addSchema(schema); + } catch { + // Duplicate or unusable $id β€” the per-file compile below + // still gives us a validator. + } + } + } + + const compiled = new Map(); + function validatorFor(schemaFile: string): ValidateFunction | null { + const cached = compiled.get(schemaFile); + if (cached) return cached; + const schema = byFile.get(schemaFile) as Record | undefined; + if (!schema) return null; + // Prefer the instance registered by $id so cross-file $refs resolve; + // fall back to compiling the document directly. + const id = typeof schema.$id === 'string' ? schema.$id : undefined; + let fn: ValidateFunction | undefined = id ? ajv.getSchema(id) : undefined; + if (!fn) { + try { + fn = ajv.compile(schema); + } catch { + return null; + } + } + compiled.set(schemaFile, fn); + return fn; + } + + return { + dir, + count: byFile.size, + has: (schemaFile) => byFile.has(schemaFile), + validate(schemaFile, document) { + const fn = validatorFor(schemaFile); + if (!fn) { + return { valid: false, errors: [`schema not found: ${schemaFile}`] }; + } + const valid = fn(document) as boolean; + if (valid) return { valid: true, errors: [] }; + return { valid: false, errors: formatErrors(fn) }; + }, + }; +} + +/** + * Turn Ajv's error objects into short, deduplicated lines. + * + * Capped at 8: a publisher emitting a wholly wrong document produces + * dozens of cascading errors, and a wall of them buries the first real + * cause in terminal output and HTML reports alike. + */ +export function formatErrors(fn: ValidateFunction, limit = 8): string[] { + const lines: string[] = (fn.errors ?? []).map((e): string => { + const where = e.instancePath && e.instancePath.length > 0 ? e.instancePath : '(root)'; + const extra = + e.keyword === 'additionalProperties' && e.params && 'additionalProperty' in e.params + ? `: ${String((e.params as { additionalProperty: unknown }).additionalProperty)}` + : ''; + return `${where} ${e.message ?? 'is invalid'}${extra}`; + }); + const unique = [...new Set(lines)]; + if (unique.length <= limit) return unique; + return [...unique.slice(0, limit), `…and ${unique.length - limit} more`]; +}