From 3b824e3692e851efcf3eba3de00cd490e49af6b0 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Sat, 29 Aug 2026 01:01:12 +0300 Subject: [PATCH 1/7] [TS PBT] Collect per-property TypeScript coverage --- usvm-ts-pbt/DESIGN.md | 61 +- usvm-ts-pbt/README.md | 66 +- .../fast-check-adapter/package-lock.json | 953 ++++++++++++++++++ usvm-ts-pbt/fast-check-adapter/package.json | 6 + .../backend/PropertyBasedTestingBackend.kt | 8 + .../usvm/ts/pbt/backend/PropertyCoverage.kt | 270 +++++ .../org/usvm/ts/pbt/cli/FastCheckCli.kt | 21 + .../org/usvm/ts/pbt/cli/FastCheckOptions.kt | 64 ++ .../ts/pbt/coverage/CoveragePathFilter.kt | 69 ++ .../ts/pbt/coverage/IstanbulCoverageReport.kt | 390 +++++++ .../usvm/ts/pbt/fastcheck/FastCheckBackend.kt | 16 + .../fastcheck/FastCheckExecutionProtocol.kt | 7 + .../pbt/fastcheck/FastCheckProcessClient.kt | 256 ++++- .../ts/pbt/backend/PropertyCoverageTest.kt | 153 +++ .../org/usvm/ts/pbt/cli/FastCheckCliTest.kt | 137 ++- .../InstalledDistributionRegistryProvider.kt | 13 + .../coverage/IstanbulCoverageReportTest.kt | 204 ++++ .../ts/pbt/fastcheck/FastCheckBackendTest.kt | 20 + .../ts/pbt/fastcheck/FastCheckCoverageTest.kt | 131 +++ .../fastcheck/FastCheckProcessClientTest.kt | 197 ++++ .../coverage/istanbul/statement-branch.json | 73 ++ .../properties/coverage/CoverageProperties.ts | 14 + .../properties/coverage/package.json | 3 + .../properties/coverage/source-under-test.ts | 6 + 24 files changed, 3114 insertions(+), 24 deletions(-) create mode 100644 usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt create mode 100644 usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/CoveragePathFilter.kt create mode 100644 usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt create mode 100644 usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/backend/PropertyCoverageTest.kt create mode 100644 usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt create mode 100644 usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt create mode 100644 usvm-ts-pbt/src/test/resources/coverage/istanbul/statement-branch.json create mode 100644 usvm-ts-pbt/src/test/resources/properties/coverage/CoverageProperties.ts create mode 100644 usvm-ts-pbt/src/test/resources/properties/coverage/package.json create mode 100644 usvm-ts-pbt/src/test/resources/properties/coverage/source-under-test.ts diff --git a/usvm-ts-pbt/DESIGN.md b/usvm-ts-pbt/DESIGN.md index ac73bcc1b..9ebf12987 100644 --- a/usvm-ts-pbt/DESIGN.md +++ b/usvm-ts-pbt/DESIGN.md @@ -7,6 +7,7 @@ API and CLI examples, see [README.md](README.md). - Kotlin owns property definitions, validation, registries, orchestration, and public results. - Node is a thin adapter around fast-check and direct TypeScript loading. +- Per-property source coverage is an optional backend capability collected by Kotlin through an isolated c8 run. - The JSON exchange is one request and one response from the same packaged distribution; it has no persistence or compatibility negotiation. - Failures are typed without exposing runtime-dependent Node stack traces. @@ -38,6 +39,7 @@ flowchart LR FastCheck[fast-check] Tsx[tsx] + C8[c8 and Istanbul JSON] UserTS[User TypeScript source] CLI --> Registry @@ -47,6 +49,8 @@ flowchart LR Backend --> Model Backend --> Process Process --> ExecutionCLI + Process --> C8 + C8 --> ExecutionCLI Projection --> ProjectionCLI ExecutionCLI --> Execute Execute --> Domains @@ -67,7 +71,7 @@ flowchart LR | Kotlin model and validation | Define one backend-neutral property and reject invalid structure before execution. | | Registry and CLI | Select Kotlin-defined properties and turn user options into a run configuration. | | `FastCheckBackend` | Validate examples, resolve source roots, and create the adapter request. | -| `FastCheckProcessClient` | Supervise Node with coroutines, bounded I/O, hard deadlines, and response validation. | +| `FastCheckProcessClient` | Supervise Node with coroutines and optionally decode one isolated c8 report. | | `execution-cli.ts` | Read one JSON request, protect protocol stdout from user logging, and write one response. | | `execute-property.ts` | Build the fast-check property, run it, and translate `RunDetails` into the common result. | | `project-domain.ts` | Translate domain descriptors into real `fc.Arbitrary` instances. | @@ -93,8 +97,9 @@ flowchart LR Verify --> Result[PropertyRunResult or PbtBackendException] ``` -The execution request contains `manifest`, `sourceRoots`, optional `seed` and `replayPath`, `numRuns`, -`timeoutMillis`, and tagged `examples`. A response is either: +The private execution request contains `manifest`, `sourceRoots`, optional `seed` and `replayPath`, `numRuns`, +`timeoutMillis`, and tagged `examples`. `coverageRequest` is Kotlin-only transport metadata and is not serialized +to Node or added to `PropertyManifest`. A Node response is either: ```text { status: "ok", result: PropertyRunResult } @@ -105,6 +110,10 @@ There is intentionally no request ID, operation name, schema version, protocol v version. The exchange is private, one-shot, and produced and consumed by the same build. Adding compatibility metadata would create branches that no supported workflow uses. +Coverage does not version this private protocol. When requested, Kotlin starts the same execution CLI under c8, +then reads a separate Istanbul report after a valid response. Backend identity and artifact schema version belong +to `coverageCapability` and `PropertyCoverageArtifact`, not the ordinary property result or private wire response. + Kotlin validates trusted model objects and examples early so callers get local errors. Node validates the decoded JSON again because the process boundary must not trust malformed input. Diagnostic codes have one owner per language: `PbtDiagnosticCode.kt` for Kotlin and `diagnostics.ts` for Node. Node also sends the diagnostic category, @@ -139,6 +148,9 @@ sequenceDiagram FC-->>Node: RunDetails Node-->>Client: one JSON response Client->>Client: validate exit, size, shape, category, and property ID + opt coverage requested + Client->>Client: decode isolated c8 Istanbul report + end Client-->>Backend: PropertyRunResult Backend-->>Caller: PropertyRunResult ``` @@ -167,6 +179,40 @@ Falsification and a timeout cleanly reported by fast-check are completed propert entry-point failures, process failures, malformed responses, and the JVM hard timeout are infrastructure exceptions. +Coverage collection failures use the separate `COVERAGE` infrastructure category. Stable diagnostics distinguish +an unsupported backend or Node runtime, unavailable runtime version, missing collector, missing or malformed +report, and missing or invalid source map. They are never converted into an empty artifact. + +## Coverage collection and filtering + +For each coverage-enabled property the process client creates unique `raw` and `report` directories and runs: + +```text +node /node_modules/c8/bin/c8.js + --config=/c8-config.json + --reporter=json + --reports-dir=/report + --temp-directory=/raw + --exclude-after-remap + --allowExternal + --exclude=__usvm_no_default_excludes__ + node /dist/src/execution-cli.js +``` + +Kotlin first probes the configured Node executable and rejects versions older than 18.18 before creating the c8 +workspace. The verified version is reused as artifact provenance. The c8 process inherits the caller's current +directory so user predicates observe the same environment with and without coverage, while an explicit empty +configuration prevents project-local c8 settings from altering collection. c8 then performs V8-to-Istanbul +conversion and source-map remapping. Kotlin rejects reports larger than 64 MiB before +reading them, validates statement, function, and branch maps and counters, classifies remapped files into +source-under-test, property entry points, generated wrappers, or dependencies, then applies include and exclude +globs. Excludes take precedence. Files and diagnostics are sorted deterministically before constructing artifact +schema version 1. + +A successful or falsified property exits the bridge normally, allowing c8 to flush the report. Process crashes, +invalid protocol responses, and hard kills do not produce a completed property result. The workspace is removed +in all cases, and a new workspace is used for every property. + The execution client starts stdout, stderr, and stdin work concurrently on the coroutine I/O dispatcher. Requests and stdout are limited to 4 MiB; stderr is limited to 64 KiB. These are transport safety bounds, not property-policy limits. The hard deadline is the property timeout plus two seconds for transport, followed by a 250 ms graceful @@ -175,8 +221,8 @@ shutdown before force-kill. The only run-control maximum is `2^31 - 1` milliseco ## Runtime packaging -Gradle installs pinned adapter dependencies, compiles only the private adapter, and packages `dist/src` plus its -runtime dependencies in the application distribution. User TypeScript stays as source. During repository tests, +Gradle installs pinned adapter dependencies, including c8 10.1.3, compiles only the private adapter, and packages +`dist/src` plus its runtime dependencies in the application distribution. User TypeScript stays as source. During repository tests, Gradle passes the adapter directory through a JVM system property; an installed distribution resolves it next to the application libraries. Node.js 18.18 or newer is required, and runtime archives carry an OS/architecture classifier because `tsx` depends on a native esbuild package. @@ -190,6 +236,8 @@ classifier because `tsx` depends on a native esbuild package. fast-check: startup failure, non-zero exit, malformed output, explicit diagnostic categories, and hard timeout. - Backend integration tests execute real uncompiled TypeScript through the packaged adapter, including replay, shrinking, explicit examples, preconditions, async predicates, and timeouts. +- Coverage golden tests assert literal TypeScript statement and branch outcomes for successful and falsified runs, + cross-property isolation, scope and glob filtering, and source-map/report diagnostics. ## Non-goals @@ -197,4 +245,5 @@ classifier because `tsx` depends on a native esbuild package. - Discovering properties by scanning TypeScript source roots. - Compiling user TypeScript as part of the PBT workflow. - Reimplementing generation, replay, skip accounting, or shrinking in Kotlin. -- Recording coverage or other per-run artifacts in this change. +- Mapping Node source locations to EtsIR or constructing symbolic targets from coverage. +- Combining Node source coverage with future EtsIR replay coverage. diff --git a/usvm-ts-pbt/README.md b/usvm-ts-pbt/README.md index ffb531deb..a2c3798d4 100644 --- a/usvm-ts-pbt/README.md +++ b/usvm-ts-pbt/README.md @@ -73,7 +73,7 @@ val result = backend.run( The defaults are 100 successful runs and a 60-second timeout. Configuration also supports replay paths and positional explicit examples. `PropertyRunResult` contains the property ID, status, actual seed, replay path, -counterexample, run/skip/shrink counts, failure details, and elapsed time. +counterexample, run/skip/shrink counts, failure details, elapsed time, and optional per-property coverage. Predicate falsification and a timeout reported by fast-check are normal `FAILURE` results. Invalid input, entry-point, process, and transport failures throw `PbtBackendException`. @@ -82,6 +82,61 @@ Synchronous entry points must return a boolean directly. Asynchronous entry poin resolves to a boolean. A false precondition is passed to fast-check as a skipped input. Generation, replay, explicit examples, checking, and shrinking retain fast-check semantics. +## Per-property TypeScript coverage + +Coverage is a backend capability, not part of `PropertyDefinition` or `PropertyManifest`. +`PropertyBasedTestingBackend.coverageCapability` exposes the backend identity, backend version, collector, and +artifact version before execution. `FastCheckBackend` reports c8 10.1.3 and coverage artifact schema version 1. +An unsupported backend reports `coverage.unsupported` explicitly. + +Kotlin requests coverage for one run through `PropertyRunConfiguration`: + +```kotlin +val result = backend.run( + property = property, + configuration = PropertyRunConfiguration( + seed = 42, + coverageRequest = PropertyCoverageRequest( + scopes = setOf(CoverageScope.SOURCE_UNDER_TEST), + includePatterns = listOf("packages/core/**/*.ts"), + excludePatterns = listOf("**/*.generated.ts"), + ), + ), +) +``` + +The default scope is `SOURCE_UNDER_TEST`. Available scopes are: + +| Scope | Files retained after source-map remapping | +| --- | --- | +| `SOURCE_UNDER_TEST` | Files below a source root except exact predicate and precondition modules | +| `PROPERTY_ENTRY_POINTS` | Exact predicate and optional precondition modules | +| `GENERATED_BACKEND_WRAPPERS` | Files in the private adapter runtime outside `node_modules` | +| `DEPENDENCIES` | Executed files below `node_modules` | + +Include and exclude globs operate on original remapped paths, use `/` separators, and support `*`, `?`, and `**`. +An empty include list retains every file in a selected scope; exclude rules always win. + +For every requested property Kotlin creates a unique c8 workspace, runs the private adapter as +`node c8.js ... node execution-cli.js`, decodes `coverage-final.json`, and removes the workspace. This isolation +prevents coverage from one property contaminating another. The adapter inherits the caller's current directory, +while an explicit empty c8 configuration prevents project-local c8 settings from altering collection. A falsified +property remains a completed Node run, so its artifact preserves coverage collected before falsification. + +Before starting c8, Kotlin probes the configured Node executable once. Versions older than 18.18 are rejected with +`coverage.runtime.unsupported`; an unavailable or unparseable version uses `coverage.runtime.version-unavailable`. +The verified version is recorded in the artifact provenance without a second probe. + +Artifact schema version 1 has kind `NODE_SOURCE` and contains backend/property identity, c8 and Node provenance, +canonical source roots, the original request, and deterministic per-file statement, function, and branch hits. +Lines are one-based and columns are zero-based, following Istanbul. Node source coverage remains separate from +future EtsIR replay coverage. + +Missing or malformed reports use `coverage.report.missing` and `coverage.report.invalid`. Reports larger than +64 MiB are rejected as invalid before they are read. JavaScript below a TypeScript source root that c8 could not +remap produces `coverage.source-map.missing` or `coverage.source-map.invalid`; a missing packaged c8 runtime +produces `coverage.collector.not-found`. + ## Registries and CLI The CLI loads Kotlin property registries through `ServiceLoader`: @@ -107,13 +162,19 @@ java -cp '/opt/usvm-ts-pbt/lib/*:/workspace/example-properties.jar' \ --registry example \ --property array.reverse-twice \ --seed 42 \ - --num-runs 1000 + --num-runs 1000 \ + --coverage \ + --coverage-scope source-under-test \ + --coverage-exclude '**/*.generated.ts' ``` Use `--help` for the complete option list. `--source-root` and `--registry` are repeatable. Without `--registry`, all providers run in registry-ID order; without `--property`, all selected properties run in registry order. Replay paths and explicit examples require exactly one selected property. +`--coverage` enables collection. `--coverage-scope`, `--coverage-include`, and `--coverage-exclude` are repeatable; +scope and path options require `--coverage`. Without an explicit scope, source-under-test coverage is collected. + The CLI writes a JSON array of results to stdout. Exit code `0` means every property succeeded, `1` means at least one property failed, and `2` means a CLI, registry, validation, backend, or transport error. Exit-code-2 diagnostics are written as one JSON object to stderr. @@ -121,6 +182,7 @@ are written as one JSON object to stderr. ## Verification Requires JDK 11, Node.js 18.18 or newer, npm, and the repository Gradle wrapper. +The private distribution pins c8 10.1.3 because it supports the module's Node 18 floor. ```shell npm ci --prefix usvm-ts-pbt/fast-check-adapter --ignore-scripts diff --git a/usvm-ts-pbt/fast-check-adapter/package-lock.json b/usvm-ts-pbt/fast-check-adapter/package-lock.json index 40e9135b5..594dfb598 100644 --- a/usvm-ts-pbt/fast-check-adapter/package-lock.json +++ b/usvm-ts-pbt/fast-check-adapter/package-lock.json @@ -8,6 +8,7 @@ "name": "@usvm/fast-check-adapter", "version": "0.1.0", "dependencies": { + "c8": "10.1.3", "fast-check": "4.9.0", "tsx": "4.23.12" }, @@ -19,6 +20,15 @@ "node": ">=18.18.0" } }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", @@ -435,6 +445,73 @@ "node": ">=18" } }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.6.tgz", + "integrity": "sha512-+Sg6GCR/wy1oSmQDFq4LQDAhm3ETKnorxN+y5nbLULOR3P0c14f2Wurzj3/xqPXtasLFfHd5iRFQ7AJt4KH2cw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "license": "MIT" + }, "node_modules/@types/node": { "version": "18.19.130", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", @@ -445,6 +522,215 @@ "undici-types": "~5.26.4" } }, + "node_modules/ansi-regex": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.3.0.tgz", + "integrity": "sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/c8": { + "version": "10.1.3", + "resolved": "https://registry.npmjs.org/c8/-/c8-10.1.3.tgz", + "integrity": "sha512-LvcyrOAaOnrrlMpW22n690PUvxiq4Uf9WMhQwNJ9vgagkL/ph1+D4uvjvDA5XCbykrc0sx+ay6pVi9YZ1GnhyA==", + "license": "ISC", + "dependencies": { + "@bcoe/v8-coverage": "^1.0.1", + "@istanbuljs/schema": "^0.1.3", + "find-up": "^5.0.0", + "foreground-child": "^3.1.1", + "istanbul-lib-coverage": "^3.2.0", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.1.6", + "test-exclude": "^7.0.1", + "v8-to-istanbul": "^9.0.0", + "yargs": "^17.7.2", + "yargs-parser": "^21.1.1" + }, + "bin": { + "c8": "bin/c8.js" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "monocart-coverage-reports": "^2" + }, + "peerDependenciesMeta": { + "monocart-coverage-reports": { + "optional": true + } + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT" + }, "node_modules/esbuild": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", @@ -486,6 +772,15 @@ "@esbuild/win32-x64": "0.28.2" } }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/fast-check": { "version": "4.9.0", "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.9.0.tgz", @@ -508,6 +803,38 @@ "node": ">=12.17.0" } }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -522,6 +849,247 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "license": "MIT" + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC" + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/pure-rand": { "version": "8.4.2", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.2.tgz", @@ -538,6 +1106,182 @@ ], "license": "MIT" }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/test-exclude": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-7.0.1.tgz", + "integrity": "sha512-pFYqmTw68LXVjeWJMST4+borgQP2AyMNbg1BpZh9LbyhUeNkeaPF9gzfPGUAnSMV3qPYdWUwDIjjCLiSDOl7vg==", + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^10.4.1", + "minimatch": "^9.0.4" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/tsx": { "version": "4.23.12", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.12.tgz", @@ -576,6 +1320,215 @@ "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "dev": true, "license": "MIT" + }, + "node_modules/v8-to-istanbul": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", + "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", + "license": "ISC", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^2.0.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } } } } diff --git a/usvm-ts-pbt/fast-check-adapter/package.json b/usvm-ts-pbt/fast-check-adapter/package.json index c0c905634..de025f782 100644 --- a/usvm-ts-pbt/fast-check-adapter/package.json +++ b/usvm-ts-pbt/fast-check-adapter/package.json @@ -12,6 +12,7 @@ "test:compiled": "node --test dist/test/entry-point.test.js dist/test/execute-property.test.js dist/test/execution-cli.test.js dist/test/js-value.test.js dist/test/project-domain.test.js dist/test/projection-cli.test.js" }, "dependencies": { + "c8": "10.1.3", "fast-check": "4.9.0", "tsx": "4.23.12" }, @@ -19,6 +20,11 @@ "@types/node": "18.19.130", "typescript": "5.9.2" }, + "overrides": { + "c8": { + "test-exclude": "7.0.1" + } + }, "engines": { "node": ">=18.18.0" } diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyBasedTestingBackend.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyBasedTestingBackend.kt index 818e7aef6..c3df828e1 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyBasedTestingBackend.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyBasedTestingBackend.kt @@ -8,6 +8,9 @@ import org.usvm.ts.pbt.model.PropertyId /** Executes validated Kotlin property definitions through one concrete PBT engine. */ interface PropertyBasedTestingBackend { + /** Reports whether this backend version can collect per-property source coverage. */ + val coverageCapability: PropertyCoverageCapability + /** Executes [property] with [configuration] and returns a structured property result. */ fun run( property: PropertyDefinition, @@ -22,6 +25,7 @@ data class PropertyRunConfiguration( val numRuns: Int = DEFAULT_NUM_RUNS, val timeoutMillis: Long = DEFAULT_TIMEOUT_MILLIS, val examples: List> = emptyList(), + val coverageRequest: PropertyCoverageRequest? = null, ) { init { require(numRuns > 0) { "Number of runs must be positive" } @@ -86,12 +90,16 @@ data class PropertyRunResult( val numShrinks: Int, val failure: PropertyFailureDetails?, val executionTimeMillis: Long, + val coverage: PropertyCoverageArtifact? = null, ) { init { require(numRuns >= 0) { "Run count must not be negative" } require(numSkips >= 0) { "Skip count must not be negative" } require(numShrinks >= 0) { "Shrink count must not be negative" } require(executionTimeMillis >= 0) { "Execution time must not be negative" } + require(coverage == null || coverage.propertyId == propertyId) { + "Coverage property ID must match the run result" + } when (status) { PropertyRunStatus.SUCCESS -> { diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt new file mode 100644 index 000000000..bb494841a --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt @@ -0,0 +1,270 @@ +package org.usvm.ts.pbt.backend + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import org.usvm.ts.pbt.model.PropertyId + +const val PROPERTY_COVERAGE_ARTIFACT_VERSION = 1 + +/** Distinguishes Node source coverage from future replay coverage over another representation. */ +@Serializable +enum class CoverageArtifactKind { + @SerialName("node_source") + NODE_SOURCE, +} + +/** Executed-file categories that may be retained in one coverage artifact. */ +@Serializable +enum class CoverageScope { + @SerialName("source_under_test") + SOURCE_UNDER_TEST, + + @SerialName("property_entry_points") + PROPERTY_ENTRY_POINTS, + + @SerialName("generated_backend_wrappers") + GENERATED_BACKEND_WRAPPERS, + + @SerialName("dependencies") + DEPENDENCIES, +} + +/** Whether a concrete backend can produce the common property coverage artifact. */ +@Serializable +enum class CoverageCapabilityLevel { + @SerialName("supported") + SUPPORTED, + + @SerialName("unsupported") + UNSUPPORTED, +} + +/** Stable diagnostic emitted while negotiating or collecting coverage. */ +@Serializable +data class CoverageDiagnostic( + val code: String, + val message: String, + val path: String? = null, +) { + init { + require(code.isNotBlank()) { "Coverage diagnostic code must not be blank" } + require(message.isNotBlank()) { "Coverage diagnostic message must not be blank" } + } +} + +/** Identifies the concrete source coverage collector and its behavior-defining version. */ +@Serializable +data class CoverageCollectorIdentity( + val id: String, + val version: String, +) { + init { + require(id.isNotBlank()) { "Coverage collector ID must not be blank" } + require(version.isNotBlank()) { "Coverage collector version must not be blank" } + } +} + +/** Reports whether a backend version implements property source coverage. */ +@Serializable +data class PropertyCoverageCapability( + val backendId: String, + val backendVersion: String, + val level: CoverageCapabilityLevel, + val collector: CoverageCollectorIdentity? = null, + val artifactVersion: Int? = null, + val diagnostics: List = emptyList(), +) { + init { + require(backendId.isNotBlank()) { "Backend ID must not be blank" } + require(backendVersion.isNotBlank()) { "Backend version must not be blank" } + when (level) { + CoverageCapabilityLevel.SUPPORTED -> { + requireNotNull(collector) { "Supported coverage requires a collector identity" } + require(artifactVersion == PROPERTY_COVERAGE_ARTIFACT_VERSION) { + "Supported coverage requires artifact version $PROPERTY_COVERAGE_ARTIFACT_VERSION" + } + require(diagnostics.isEmpty()) { "Supported coverage must not contain capability diagnostics" } + } + + CoverageCapabilityLevel.UNSUPPORTED -> { + require(collector == null) { "Unsupported coverage must not name a collector" } + require(artifactVersion == null) { "Unsupported coverage must not name an artifact version" } + require(diagnostics.isNotEmpty()) { "Unsupported coverage requires an actionable diagnostic" } + } + } + } + + companion object { + fun supported( + backendId: String, + backendVersion: String, + collector: CoverageCollectorIdentity, + artifactVersion: Int = PROPERTY_COVERAGE_ARTIFACT_VERSION, + ) = PropertyCoverageCapability( + backendId = backendId, + backendVersion = backendVersion, + level = CoverageCapabilityLevel.SUPPORTED, + collector = collector, + artifactVersion = artifactVersion, + ) + + fun unsupported( + backendId: String, + backendVersion: String, + diagnostic: CoverageDiagnostic, + ) = PropertyCoverageCapability( + backendId = backendId, + backendVersion = backendVersion, + level = CoverageCapabilityLevel.UNSUPPORTED, + diagnostics = listOf(diagnostic), + ) + } +} + +/** Backend-neutral controls for source coverage collected during one property run. */ +@Serializable +data class PropertyCoverageRequest( + val scopes: Set = setOf(CoverageScope.SOURCE_UNDER_TEST), + val includePatterns: List = emptyList(), + val excludePatterns: List = emptyList(), +) { + init { + require(scopes.isNotEmpty()) { "At least one coverage scope is required" } + require(includePatterns.all(String::isNotBlank)) { "Coverage include patterns must not be blank" } + require(excludePatterns.all(String::isNotBlank)) { "Coverage exclude patterns must not be blank" } + } +} + +/** One-based line and zero-based column in an original source file. */ +@Serializable +data class SourcePosition( + val line: Int, + val column: Int, +) { + init { + require(line > 0) { "Source line must be positive" } + require(column >= 0) { "Source column must not be negative" } + } +} + +/** Half-open source range as represented by Istanbul. */ +@Serializable +data class SourceRange( + val start: SourcePosition, + val end: SourcePosition, +) { + init { + require(end.line > start.line || end.line == start.line && end.column >= start.column) { + "Source range end must not precede its start" + } + } +} + +/** Hit count for one Istanbul statement. */ +@Serializable +data class StatementCoverage( + val statementId: Int, + val location: SourceRange, + val hits: Long, +) { + init { + require(statementId >= 0) { "Statement ID must not be negative" } + require(hits >= 0) { "Statement hits must not be negative" } + } +} + +/** Hit count for one Istanbul function. */ +@Serializable +data class FunctionCoverage( + val functionId: Int, + val name: String, + val declaration: SourceRange, + val body: SourceRange, + val hits: Long, +) { + init { + require(functionId >= 0) { "Function ID must not be negative" } + require(name.isNotBlank()) { "Function name must not be blank" } + require(hits >= 0) { "Function hits must not be negative" } + } +} + +/** Hit count for one arm of an Istanbul branch. */ +@Serializable +data class BranchArmCoverage( + val location: SourceRange, + val hits: Long, +) { + init { + require(hits >= 0) { "Branch arm hits must not be negative" } + } +} + +/** Hit counts for all arms belonging to one Istanbul branch. */ +@Serializable +data class BranchCoverage( + val branchId: Int, + val type: String, + val location: SourceRange, + val arms: List, +) { + init { + require(branchId >= 0) { "Branch ID must not be negative" } + require(type.isNotBlank()) { "Branch type must not be blank" } + require(arms.isNotEmpty()) { "Branch coverage requires at least one arm" } + } +} + +/** Source-mapped coverage for one original source file. */ +@Serializable +data class SourceFileCoverage( + val path: String, + val statements: List, + val functions: List, + val branches: List, +) { + init { + require(path.isNotBlank()) { "Coverage source path must not be blank" } + } +} + +/** Records the collector inputs needed to interpret or reproduce one artifact. */ +@Serializable +data class CoverageProvenance( + val collector: CoverageCollectorIdentity, + val runtimeId: String, + val runtimeVersion: String, + val sourceRoots: List, + val request: PropertyCoverageRequest, +) { + init { + require(runtimeId.isNotBlank()) { "Coverage runtime ID must not be blank" } + require(runtimeVersion.isNotBlank()) { "Coverage runtime version must not be blank" } + require(sourceRoots.isNotEmpty()) { "Coverage provenance requires source roots" } + require(sourceRoots.all(String::isNotBlank)) { "Coverage source roots must not be blank" } + } +} + +/** Versioned per-property Node source coverage returned by a concrete backend. */ +@Serializable +data class PropertyCoverageArtifact( + val schemaVersion: Int = PROPERTY_COVERAGE_ARTIFACT_VERSION, + val kind: CoverageArtifactKind = CoverageArtifactKind.NODE_SOURCE, + val backendId: String, + val backendVersion: String, + val propertyId: PropertyId, + val provenance: CoverageProvenance, + val files: List, + val diagnostics: List = emptyList(), +) { + init { + require(schemaVersion == PROPERTY_COVERAGE_ARTIFACT_VERSION) { + "Unsupported property coverage artifact version: $schemaVersion" + } + require(backendId.isNotBlank()) { "Coverage backend ID must not be blank" } + require(backendVersion.isNotBlank()) { "Coverage backend version must not be blank" } + require(files.map(SourceFileCoverage::path).distinct().size == files.size) { + "Coverage artifact contains duplicate source paths" + } + } +} diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckCli.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckCli.kt index d08b44a84..762603e6e 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckCli.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckCli.kt @@ -5,6 +5,7 @@ import kotlinx.serialization.SerializationException import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.backend.CoverageCapabilityLevel import org.usvm.ts.pbt.backend.PropertyBasedTestingBackend import org.usvm.ts.pbt.backend.PropertyRunConfiguration import org.usvm.ts.pbt.backend.PropertyRunStatus @@ -119,9 +120,11 @@ class FastCheckCli( numRuns = options.numRuns, timeoutMillis = options.timeoutMillis, examples = examples, + coverageRequest = options.coverageRequest, ) val backend = backendFactory(options.sourceRoots) + requireCoverageSupport(options, backend) val results = properties.map { property -> backend.run(property, configuration) } output.appendLine(PropertyManifestJson.json.encodeToString(results)) @@ -134,6 +137,24 @@ class FastCheckCli( } } + private fun requireCoverageSupport( + options: CliOptions, + backend: PropertyBasedTestingBackend, + ) { + if (options.coverageRequest == null || + backend.coverageCapability.level != CoverageCapabilityLevel.UNSUPPORTED + ) { + return + } + + val diagnostic = backend.coverageCapability.diagnostics.first() + throw CliUsageException( + code = diagnostic.code, + message = diagnostic.message, + path = diagnostic.path, + ) + } + private fun selectProperties( registry: PropertyRegistry, propertyId: PropertyId?, diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt index b6075c901..ca3ef9407 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt @@ -5,12 +5,15 @@ import com.github.ajalt.clikt.core.CliktError import com.github.ajalt.clikt.core.PrintHelpMessage import com.github.ajalt.clikt.core.parse import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.flag import com.github.ajalt.clikt.parameters.options.multiple import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.types.int import com.github.ajalt.clikt.parameters.types.long import com.github.ajalt.clikt.parameters.types.path import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.backend.CoverageScope +import org.usvm.ts.pbt.backend.PropertyCoverageRequest import org.usvm.ts.pbt.backend.PropertyRunConfiguration import org.usvm.ts.pbt.model.PropertyId import java.nio.file.Path @@ -30,6 +33,7 @@ internal data class CliOptions( val numRuns: Int, val timeoutMillis: Long, val examplesFile: Path?, + val coverageRequest: PropertyCoverageRequest?, ) internal class CliUsageException( @@ -98,12 +102,33 @@ private class FastCheckOptionsParser : CliktCommand(name = "usvm-ts-pbt") { help = "JSON file with positional tagged examples", ).path() + private val coverageEnabled by option( + "--coverage", + help = "Collect source-mapped coverage for every property run", + ).flag() + + private val coverageScopes by option( + "--coverage-scope", + help = "Coverage scope; repeat for multiple scopes", + ).multiple() + + private val coverageIncludePatterns by option( + "--coverage-include", + help = "Include glob over remapped source paths; repeat for multiple patterns", + ).multiple() + + private val coverageExcludePatterns by option( + "--coverage-exclude", + help = "Exclude glob over remapped source paths; repeat for multiple patterns", + ).multiple() + lateinit var options: CliOptions private set override fun run() { requireSourceRoots() requirePositiveRunControls() + requireCoverageFlag() options = CliOptions( sourceRoots = sourceRoots, @@ -114,6 +139,7 @@ private class FastCheckOptionsParser : CliktCommand(name = "usvm-ts-pbt") { numRuns = numRuns, timeoutMillis = timeoutMillis, examplesFile = examplesFile, + coverageRequest = buildCoverageRequest(), ) } @@ -144,6 +170,44 @@ private class FastCheckOptionsParser : CliktCommand(name = "usvm-ts-pbt") { ) } } + + private fun requireCoverageFlag() { + val hasCoverageDetails = coverageScopes.isNotEmpty() || + coverageIncludePatterns.isNotEmpty() || + coverageExcludePatterns.isNotEmpty() + if (!coverageEnabled && hasCoverageDetails) { + throw CliUsageException( + code = "cli.coverage.required", + message = "Coverage scope and path rules require --coverage", + path = "coverage", + ) + } + } + + private fun buildCoverageRequest(): PropertyCoverageRequest? { + if (!coverageEnabled) return null + + return PropertyCoverageRequest( + scopes = coverageScopes + .map(::parseCoverageScope) + .toSet() + .ifEmpty { setOf(CoverageScope.SOURCE_UNDER_TEST) }, + includePatterns = coverageIncludePatterns, + excludePatterns = coverageExcludePatterns, + ) + } +} + +private fun parseCoverageScope(value: String): CoverageScope = when (value) { + "source-under-test" -> CoverageScope.SOURCE_UNDER_TEST + "property-entry-points" -> CoverageScope.PROPERTY_ENTRY_POINTS + "generated-backend-wrappers" -> CoverageScope.GENERATED_BACKEND_WRAPPERS + "dependencies" -> CoverageScope.DEPENDENCIES + else -> throw CliUsageException( + code = "cli.coverage.scope.invalid", + message = "Unknown coverage scope $value", + path = "coverageScope", + ) } private fun parsePropertyId(value: String): PropertyId = try { diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/CoveragePathFilter.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/CoveragePathFilter.kt new file mode 100644 index 000000000..88b964003 --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/CoveragePathFilter.kt @@ -0,0 +1,69 @@ +package org.usvm.ts.pbt.coverage + +internal fun matchesCoveragePath( + path: String, + patterns: List, + sourceRoots: List, +): Boolean { + if (patterns.isEmpty()) return true + val candidates = buildList { + add(path) + sourceRoots.forEach { sourceRoot -> + if (isWithin(path, sourceRoot) && path != sourceRoot) { + add(path.removePrefix("$sourceRoot/")) + } + } + } + return patterns.any { pattern -> + val regex = coverageGlobToRegex(pattern) + candidates.any(regex::matches) + } +} + +private fun coverageGlobToRegex(pattern: String): Regex { + val normalized = pattern.replace('\\', '/').removePrefix("./") + val expression = StringBuilder("^") + var index = 0 + while (index < normalized.length) { + val character = normalized[index] + when { + character == '*' && normalized.getOrNull(index + 1) == '*' -> { + if (normalized.getOrNull(index + DOUBLE_WILDCARD_LENGTH) == '/') { + expression.append("(?:.*/)?") + index += DOUBLE_WILDCARD_WITH_SEPARATOR_LENGTH + } else { + expression.append(".*") + index += 2 + } + } + + character == '*' -> { + expression.append("[^/]*") + index++ + } + + character == '?' -> { + expression.append("[^/]") + index++ + } + + character in REGEX_SPECIAL_CHARACTERS -> { + expression.append('\\').append(character) + index++ + } + + else -> { + expression.append(character) + index++ + } + } + } + expression.append('$') + return Regex(expression.toString()) +} + +internal fun isWithin(path: String, root: String): Boolean = path == root || path.startsWith("$root/") + +private const val REGEX_SPECIAL_CHARACTERS = ".+()^$|{}[]" +private const val DOUBLE_WILDCARD_LENGTH = 2 +private const val DOUBLE_WILDCARD_WITH_SEPARATOR_LENGTH = 3 diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt new file mode 100644 index 000000000..a0289dd56 --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt @@ -0,0 +1,390 @@ +package org.usvm.ts.pbt.coverage + +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.intOrNull +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.longOrNull +import org.usvm.ts.pbt.backend.BranchArmCoverage +import org.usvm.ts.pbt.backend.BranchCoverage +import org.usvm.ts.pbt.backend.CoverageArtifactKind +import org.usvm.ts.pbt.backend.CoverageCollectorIdentity +import org.usvm.ts.pbt.backend.CoverageDiagnostic +import org.usvm.ts.pbt.backend.CoverageProvenance +import org.usvm.ts.pbt.backend.CoverageScope +import org.usvm.ts.pbt.backend.FunctionCoverage +import org.usvm.ts.pbt.backend.PROPERTY_COVERAGE_ARTIFACT_VERSION +import org.usvm.ts.pbt.backend.PropertyCoverageArtifact +import org.usvm.ts.pbt.backend.PropertyCoverageRequest +import org.usvm.ts.pbt.backend.SourceFileCoverage +import org.usvm.ts.pbt.backend.SourcePosition +import org.usvm.ts.pbt.backend.SourceRange +import org.usvm.ts.pbt.backend.StatementCoverage +import org.usvm.ts.pbt.manifest.PropertyManifestJson +import org.usvm.ts.pbt.model.PropertyId +import java.io.IOException +import java.nio.file.Files +import java.nio.file.Path + +const val C8_COLLECTOR_VERSION = "10.1.3" + +/** All identities and path boundaries required to convert one isolated c8 report. */ +data class IstanbulCoverageContext( + val backendId: String, + val backendVersion: String, + val propertyId: PropertyId, + val sourceRoots: List, + val propertyEntryPointPaths: Set, + val adapterRoot: String, + val runtimeVersion: String, + val request: PropertyCoverageRequest, +) + +/** Invalid or absent coverage that prevents construction of a trustworthy artifact. */ +class CoverageArtifactException( + val diagnostic: CoverageDiagnostic, + cause: Throwable? = null, +) : IllegalArgumentException(diagnostic.message, cause) + +/** Decodes and filters one source-mapped Istanbul JSON report produced by an isolated c8 run. */ +fun decodeIstanbulCoverageReport( + reportPath: Path, + context: IstanbulCoverageContext, +): PropertyCoverageArtifact = decodeReport(readCoverageReport(reportPath), context) + +private fun readCoverageReport(reportPath: Path): JsonObject { + requireCoverageReport(reportPath) + return parseCoverageReport(readCoverageReportText(reportPath), reportPath) +} + +private fun requireCoverageReport(reportPath: Path) { + if (!Files.isRegularFile(reportPath)) { + throw coverageError( + code = "coverage.report.missing", + message = "c8 did not produce the expected Istanbul report: $reportPath", + path = reportPath.toString(), + ) + } +} + +private fun readCoverageReportText(reportPath: Path): String { + requireCoverageReportSize(reportPath) + + return try { + Files.readString(reportPath) + } catch (error: IOException) { + throw unreadableCoverageReport(reportPath, error) + } +} + +private fun requireCoverageReportSize(reportPath: Path) { + val reportSize = try { + Files.size(reportPath) + } catch (error: IOException) { + throw unreadableCoverageReport(reportPath, error) + } + if (reportSize > MAX_COVERAGE_REPORT_BYTES) { + throw coverageError( + code = "coverage.report.invalid", + message = "Istanbul coverage report exceeds $MAX_COVERAGE_REPORT_BYTES bytes", + path = reportPath.toString(), + ) + } +} + +private fun unreadableCoverageReport( + reportPath: Path, + error: IOException, +): CoverageArtifactException = coverageError( + code = "coverage.report.invalid", + message = "Cannot read Istanbul coverage report: ${error.message}", + path = reportPath.toString(), + cause = error, +) + +private fun parseCoverageReport(text: String, reportPath: Path): JsonObject = try { + PropertyManifestJson.json.parseToJsonElement(text).jsonObject +} catch (error: IllegalArgumentException) { + throw coverageError( + code = "coverage.report.invalid", + message = "Istanbul coverage report is not valid JSON: ${error.message}", + path = reportPath.toString(), + cause = error, + ) +} + +private fun decodeReport( + report: JsonObject, + context: IstanbulCoverageContext, +): PropertyCoverageArtifact { + val normalizedRoots = context.sourceRoots.map(::normalizeCoveragePath) + val normalizedEntryPoints = context.propertyEntryPointPaths.mapTo(mutableSetOf(), ::normalizeCoveragePath) + val normalizedAdapterRoot = normalizeCoveragePath(context.adapterRoot) + val diagnostics = mutableListOf() + val files = report.entries.mapNotNull { (reportKey, element) -> + val fileObject = element.asObject("coverage[$reportKey]") + val path = normalizeCoveragePath( + fileObject.optionalString("path") ?: reportKey, + ) + if (isGeneratedJavaScriptBelowSourceRoot(path, normalizedRoots)) { + val sourceMapExists = Files.exists(Path.of("$path.map")) + diagnostics += CoverageDiagnostic( + code = if (sourceMapExists) "coverage.source-map.invalid" else "coverage.source-map.missing", + message = if (sourceMapExists) { + "Executed JavaScript has a source map that c8 could not remap to its original source" + } else { + "Executed JavaScript below a TypeScript source root has no source map" + }, + path = path, + ) + return@mapNotNull null + } + val scope = classifyCoverageScope( + path = path, + sourceRoots = normalizedRoots, + propertyEntryPoints = normalizedEntryPoints, + adapterRoot = normalizedAdapterRoot, + ) ?: return@mapNotNull null + if (!shouldRetainCoverageFile(path, scope, normalizedRoots, context.request)) { + return@mapNotNull null + } + decodeSourceFileCoverage(fileObject, path, reportKey) + }.sortedBy(SourceFileCoverage::path) + + return PropertyCoverageArtifact( + schemaVersion = PROPERTY_COVERAGE_ARTIFACT_VERSION, + kind = CoverageArtifactKind.NODE_SOURCE, + backendId = context.backendId, + backendVersion = context.backendVersion, + propertyId = context.propertyId, + provenance = CoverageProvenance( + collector = CoverageCollectorIdentity(id = "c8", version = C8_COLLECTOR_VERSION), + runtimeId = "node", + runtimeVersion = context.runtimeVersion, + sourceRoots = normalizedRoots, + request = context.request, + ), + files = files, + diagnostics = diagnostics.sortedWith(compareBy(CoverageDiagnostic::path, CoverageDiagnostic::code)), + ) +} + +private fun shouldRetainCoverageFile( + path: String, + scope: CoverageScope, + sourceRoots: List, + request: PropertyCoverageRequest, +): Boolean { + if (scope !in request.scopes) return false + if (!matchesCoveragePath(path, request.includePatterns, sourceRoots)) return false + return request.excludePatterns.isEmpty() || !matchesCoveragePath(path, request.excludePatterns, sourceRoots) +} + +private fun classifyCoverageScope( + path: String, + sourceRoots: List, + propertyEntryPoints: Set, + adapterRoot: String, +): CoverageScope? = when { + "/node_modules/" in path -> CoverageScope.DEPENDENCIES + isWithin(path, adapterRoot) -> CoverageScope.GENERATED_BACKEND_WRAPPERS + path in propertyEntryPoints -> CoverageScope.PROPERTY_ENTRY_POINTS + sourceRoots.any { root -> isWithin(path, root) } -> CoverageScope.SOURCE_UNDER_TEST + else -> null +} + +private fun isGeneratedJavaScriptBelowSourceRoot(path: String, sourceRoots: List): Boolean { + val extension = path.substringAfterLast('.', missingDelimiterValue = "").lowercase() + return extension in GENERATED_JAVASCRIPT_EXTENSIONS && sourceRoots.any { root -> isWithin(path, root) } +} + +private fun decodeSourceFileCoverage( + file: JsonObject, + path: String, + reportKey: String, +): SourceFileCoverage = try { + SourceFileCoverage( + path = path, + statements = decodeStatements(file, reportKey), + functions = decodeFunctions(file, reportKey), + branches = decodeBranches(file, reportKey), + ) +} catch (error: CoverageArtifactException) { + throw error +} catch (error: IllegalArgumentException) { + throw coverageError( + code = "coverage.report.invalid", + message = "Invalid Istanbul coverage for $path: ${error.message}", + path = "coverage[$reportKey]", + cause = error, + ) +} + +private fun decodeStatements(file: JsonObject, reportKey: String): List { + val locations = file.requiredObject("statementMap", "coverage[$reportKey].statementMap") + val hits = file.requiredObject("s", "coverage[$reportKey].s") + requireMatchingKeys(locations, hits, "coverage[$reportKey].statementMap", "coverage[$reportKey].s") + return locations.keys.map(::parseCoverageId).sorted().map { statementId -> + val id = statementId.toString() + StatementCoverage( + statementId = statementId, + location = locations.getValue(id).decodeRange("coverage[$reportKey].statementMap[$id]"), + hits = hits.getValue(id).decodeHitCount("coverage[$reportKey].s[$id]"), + ) + } +} + +private fun decodeFunctions(file: JsonObject, reportKey: String): List { + val locations = file.requiredObject("fnMap", "coverage[$reportKey].fnMap") + val hits = file.requiredObject("f", "coverage[$reportKey].f") + requireMatchingKeys(locations, hits, "coverage[$reportKey].fnMap", "coverage[$reportKey].f") + return locations.keys.map(::parseCoverageId).sorted().map { functionId -> + val id = functionId.toString() + val function = locations.getValue(id).asObject("coverage[$reportKey].fnMap[$id]") + FunctionCoverage( + functionId = functionId, + name = function.optionalString("name")?.ifBlank { ANONYMOUS_FUNCTION_NAME } ?: ANONYMOUS_FUNCTION_NAME, + declaration = function.requiredElement("decl", "coverage[$reportKey].fnMap[$id].decl") + .decodeRange("coverage[$reportKey].fnMap[$id].decl"), + body = function.requiredElement("loc", "coverage[$reportKey].fnMap[$id].loc") + .decodeRange("coverage[$reportKey].fnMap[$id].loc"), + hits = hits.getValue(id).decodeHitCount("coverage[$reportKey].f[$id]"), + ) + } +} + +private fun decodeBranches(file: JsonObject, reportKey: String): List { + val locations = file.requiredObject("branchMap", "coverage[$reportKey].branchMap") + val hits = file.requiredObject("b", "coverage[$reportKey].b") + requireMatchingKeys(locations, hits, "coverage[$reportKey].branchMap", "coverage[$reportKey].b") + return locations.keys.map(::parseCoverageId).sorted().map { branchId -> + val id = branchId.toString() + val branchPath = "coverage[$reportKey].branchMap[$id]" + val branch = locations.getValue(id).asObject(branchPath) + val (armLocations, armHits) = decodeBranchArrays(branch, hits, reportKey, id, branchPath) + BranchCoverage( + branchId = branchId, + type = branch.requiredString("type", "$branchPath.type"), + location = branch.requiredElement("loc", "$branchPath.loc").decodeRange("$branchPath.loc"), + arms = armLocations.indices.map { armIndex -> + BranchArmCoverage( + location = armLocations[armIndex].decodeRange("$branchPath.locations[$armIndex]"), + hits = armHits[armIndex].decodeHitCount("coverage[$reportKey].b[$id][$armIndex]"), + ) + }, + ) + } +} + +private fun decodeBranchArrays( + branch: JsonObject, + hits: JsonObject, + reportKey: String, + id: String, + branchPath: String, +): Pair { + val armLocations = branch.requiredElement("locations", "$branchPath.locations") as? JsonArray + ?: throw invalidReport("Expected an array", "$branchPath.locations") + val armHitsPath = "coverage[$reportKey].b[$id]" + val armHits = hits.getValue(id) as? JsonArray + ?: throw invalidReport("Expected an array", armHitsPath) + requireMatchingBranchArms(armLocations, armHits, branchPath) + return armLocations to armHits +} + +private fun requireMatchingBranchArms( + armLocations: JsonArray, + armHits: JsonArray, + branchPath: String, +) { + if (armLocations.size != armHits.size) { + throw invalidReport("Branch location and hit counts have different sizes", branchPath) + } +} + +private fun JsonElement.decodeRange(path: String): SourceRange { + val range = asObject(path) + return SourceRange( + start = range.requiredElement("start", "$path.start").decodePosition("$path.start"), + end = range.requiredElement("end", "$path.end").decodePosition("$path.end"), + ) +} + +private fun JsonElement.decodePosition(path: String): SourcePosition { + val position = asObject(path) + return SourcePosition( + line = position.requiredInt("line", "$path.line"), + column = position.requiredInt("column", "$path.column"), + ) +} + +private fun JsonElement.decodeHitCount(path: String): Long { + val primitive = this as? JsonPrimitive ?: throw invalidReport("Expected an integer", path) + return primitive.longOrNull ?: throw invalidReport("Expected an integer", path) +} + +private fun JsonElement.asObject(path: String): JsonObject = this as? JsonObject + ?: throw invalidReport("Expected an object", path) + +private fun JsonObject.requiredObject( + name: String, + path: String, +): JsonObject = requiredElement(name, path).asObject(path) + +private fun JsonObject.requiredElement(name: String, path: String): JsonElement = this[name] + ?: throw invalidReport("Missing required field $name", path) + +private fun JsonObject.requiredString(name: String, path: String): String { + val primitive = this[name] as? JsonPrimitive ?: throw invalidReport("Expected a string", path) + return primitive.contentOrNull ?: throw invalidReport("Expected a string", path) +} + +private fun JsonObject.optionalString(name: String): String? = (this[name] as? JsonPrimitive)?.contentOrNull + +private fun JsonObject.requiredInt(name: String, path: String): Int { + val primitive = this[name] as? JsonPrimitive ?: throw invalidReport("Expected an integer", path) + return primitive.intOrNull ?: throw invalidReport("Expected an integer", path) +} + +private fun requireMatchingKeys( + locations: JsonObject, + hits: JsonObject, + locationsPath: String, + hitsPath: String, +) { + if (locations.keys != hits.keys) { + throw invalidReport("Coverage map keys do not match hit counter keys ($locationsPath)", hitsPath) + } +} + +private fun parseCoverageId(value: String): Int = value.toIntOrNull()?.takeIf { id -> id >= 0 } + ?: throw invalidReport("Coverage ID must be a non-negative integer", value) + +private fun normalizeCoveragePath(path: String): String = Path.of(path) + .toAbsolutePath() + .normalize() + .toString() + .replace('\\', '/') + +private fun invalidReport(message: String, path: String): CoverageArtifactException = coverageError( + code = "coverage.report.invalid", + message = message, + path = path, +) + +private fun coverageError( + code: String, + message: String, + path: String, + cause: Throwable? = null, +) = CoverageArtifactException( + diagnostic = CoverageDiagnostic(code = code, message = message, path = path), + cause = cause, +) + +private const val ANONYMOUS_FUNCTION_NAME = "" +private const val MAX_COVERAGE_REPORT_BYTES = 64L * 1024 * 1024 +private val GENERATED_JAVASCRIPT_EXTENSIONS = setOf("js", "mjs", "cjs") diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt index 66158dbe6..aaac7e9b8 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt @@ -1,9 +1,12 @@ package org.usvm.ts.pbt.fastcheck import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.backend.CoverageCollectorIdentity import org.usvm.ts.pbt.backend.PropertyBasedTestingBackend +import org.usvm.ts.pbt.backend.PropertyCoverageCapability import org.usvm.ts.pbt.backend.PropertyRunConfiguration import org.usvm.ts.pbt.backend.PropertyRunResult +import org.usvm.ts.pbt.coverage.C8_COLLECTOR_VERSION import org.usvm.ts.pbt.manifest.toManifest import org.usvm.ts.pbt.model.JsConcreteValue import org.usvm.ts.pbt.model.JsNumberKind @@ -21,6 +24,15 @@ class FastCheckBackend( nodeExecutable: String = "node", adapterEntryPoint: Path = FastCheckRuntime.executionEntryPoint(), ) : PropertyBasedTestingBackend { + override val coverageCapability: PropertyCoverageCapability = PropertyCoverageCapability.supported( + backendId = FAST_CHECK_BACKEND_ID, + backendVersion = FAST_CHECK_BACKEND_VERSION, + collector = CoverageCollectorIdentity( + id = "c8", + version = C8_COLLECTOR_VERSION, + ), + ) + private val sourceRoots = canonicalizeSourceRoots(sourceRoots) private val client = FastCheckProcessClient( nodeExecutable = nodeExecutable, @@ -43,6 +55,7 @@ class FastCheckBackend( numRuns = configuration.numRuns, timeoutMillis = configuration.timeoutMillis, examples = configuration.examples, + coverageRequest = configuration.coverageRequest, ), ) } @@ -133,6 +146,9 @@ class FastCheckBackend( ) companion object { + const val FAST_CHECK_BACKEND_ID = "fast-check" + const val FAST_CHECK_BACKEND_VERSION = "4.9.0" + private val FINITE_NUMBER_BITS_REGEX = Regex("[0-9a-f]{16}") private fun canonicalizeSourceRoots(sourceRoots: List): List { diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckExecutionProtocol.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckExecutionProtocol.kt index 7b7e507f6..2002aaa04 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckExecutionProtocol.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckExecutionProtocol.kt @@ -2,6 +2,8 @@ package org.usvm.ts.pbt.fastcheck import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable +import kotlinx.serialization.Transient +import org.usvm.ts.pbt.backend.PropertyCoverageRequest import org.usvm.ts.pbt.backend.PropertyRunResult import org.usvm.ts.pbt.manifest.PropertyManifest import org.usvm.ts.pbt.model.JsConcreteValue @@ -15,6 +17,8 @@ internal data class FastCheckExecutionRequest( val numRuns: Int, val timeoutMillis: Long, val examples: List> = emptyList(), + @Transient + val coverageRequest: PropertyCoverageRequest? = null, ) @Serializable @@ -41,6 +45,9 @@ enum class BackendErrorKind { @SerialName("timeout") TIMEOUT, + + @SerialName("coverage") + COVERAGE, } /** Typed failure at the concrete backend boundary. */ diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt index 187417291..79f7a8c37 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt @@ -12,12 +12,18 @@ import kotlinx.coroutines.withTimeoutOrNull import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.backend.CoverageScope import org.usvm.ts.pbt.backend.PropertyRunResult +import org.usvm.ts.pbt.coverage.C8_COLLECTOR_VERSION +import org.usvm.ts.pbt.coverage.CoverageArtifactException +import org.usvm.ts.pbt.coverage.IstanbulCoverageContext +import org.usvm.ts.pbt.coverage.decodeIstanbulCoverageReport import org.usvm.ts.pbt.manifest.PropertyManifestJson import org.usvm.ts.pbt.model.PropertyId import java.io.ByteArrayOutputStream import java.io.IOException import java.io.InputStream +import java.nio.file.Files import java.nio.file.Path import java.util.concurrent.TimeUnit @@ -47,17 +53,22 @@ internal class FastCheckProcessClient( private suspend fun checkSuspending(request: FastCheckExecutionRequest): PropertyRunResult = supervisorScope { val encodedRequest = encodeRequest(request) - val process = startAdapter(request) - val stdout = async(ioDispatcher) { process.inputStream.readBounded(MAX_STDOUT_BYTES) } - val stderr = async(ioDispatcher) { process.errorStream.readBounded(MAX_STDERR_BYTES) } - val writer = async(ioDispatcher) { - process.outputStream.bufferedWriter(Charsets.UTF_8).use { output -> - output.write(encodedRequest) - } - } + val coverageRuntimeVersion = request.coverageRequest?.let { nodeVersion(request) } + val coverageWorkspace = request.coverageRequest?.let { createCoverageWorkspace(request) } + var process: Process? = null try { - awaitProcess(process, request) + val startedProcess = startAdapter(request, coverageWorkspace) + process = startedProcess + val stdout = async(ioDispatcher) { startedProcess.inputStream.readBounded(MAX_STDOUT_BYTES) } + val stderr = async(ioDispatcher) { startedProcess.errorStream.readBounded(MAX_STDERR_BYTES) } + val writer = async(ioDispatcher) { + startedProcess.outputStream.bufferedWriter(Charsets.UTF_8).use { output -> + output.write(encodedRequest) + } + } + + awaitProcess(startedProcess, request) awaitIo( task = writer, @@ -79,14 +90,24 @@ internal class FastCheckProcessClient( request = request, ) - validateProcessExit(process, stderrText, request) + validateProcessExit(startedProcess, stderrText, request) validateStdout(stdoutText, request) val response = decodeResponse(stdoutText.text, request) - decodeSuccessfulResponse(response, request) + val result = decodeSuccessfulResponse(response, request) + + coverageWorkspace?.let { workspace -> + collectCoverage( + result = result, + request = request, + workspace = workspace, + runtimeVersion = requireNotNull(coverageRuntimeVersion), + ) + } ?: result } finally { - if (process.isAlive) terminate(process) + process?.takeIf(Process::isAlive)?.let(::terminate) + coverageWorkspace?.root?.toFile()?.deleteRecursively() } } @@ -160,8 +181,11 @@ internal class FastCheckProcessClient( } } - private fun startAdapter(request: FastCheckExecutionRequest): Process = try { - ProcessBuilder(nodeExecutable, adapterEntryPoint.toString()).start() + private fun startAdapter( + request: FastCheckExecutionRequest, + coverageWorkspace: CoverageWorkspace?, + ): Process = try { + ProcessBuilder(adapterCommand(request, coverageWorkspace)).start() } catch (error: IOException) { throw backendError( kind = BackendErrorKind.PROCESS_FAILURE, @@ -172,6 +196,197 @@ internal class FastCheckProcessClient( ) } + private fun adapterCommand( + request: FastCheckExecutionRequest, + coverageWorkspace: CoverageWorkspace?, + ): List { + if (coverageWorkspace == null) return listOf(nodeExecutable, adapterEntryPoint.toString()) + + val command = mutableListOf( + nodeExecutable, + coverageWorkspace.c8EntryPoint.toString(), + "--config=${coverageWorkspace.configPath}", + "--reporter=json", + "--reports-dir=${coverageWorkspace.reportDirectory}", + "--temp-directory=${coverageWorkspace.rawDirectory}", + "--exclude-after-remap", + "--allowExternal", + "--exclude=__usvm_no_default_excludes__", + ) + if (CoverageScope.DEPENDENCIES in requireNotNull(request.coverageRequest).scopes) { + command += "--exclude-node-modules=false" + } + command += nodeExecutable + command += adapterEntryPoint.toString() + + return command + } + + private fun createCoverageWorkspace(request: FastCheckExecutionRequest): CoverageWorkspace { + val adapterRoot = adapterRoot() + val c8EntryPoint = adapterRoot.resolve("node_modules/c8/bin/c8.js") + if (!Files.isRegularFile(c8EntryPoint)) { + throw backendError( + kind = BackendErrorKind.COVERAGE, + code = "coverage.collector.not-found", + message = "Cannot locate c8 $C8_COLLECTOR_VERSION in the fast-check adapter runtime", + request = request, + path = c8EntryPoint.toString(), + ) + } + + val root = Files.createTempDirectory("usvm-ts-pbt-coverage-") + val configPath = Files.writeString(root.resolve("c8-config.json"), "{}") + return CoverageWorkspace( + root = root, + configPath = configPath, + rawDirectory = root.resolve("raw"), + reportDirectory = root.resolve("report"), + c8EntryPoint = c8EntryPoint, + adapterRoot = adapterRoot, + ) + } + + private fun collectCoverage( + result: PropertyRunResult, + request: FastCheckExecutionRequest, + workspace: CoverageWorkspace, + runtimeVersion: String, + ): PropertyRunResult { + val coverageRequest = requireNotNull(request.coverageRequest) + val entryPointPaths = buildSet { + request.sourceRoots.forEach { sourceRoot -> + val root = Path.of(sourceRoot) + add(root.resolve(request.manifest.predicate.module).normalize().toString()) + request.manifest.precondition?.let { precondition -> + add(root.resolve(precondition.module).normalize().toString()) + } + } + } + val artifact = try { + decodeIstanbulCoverageReport( + reportPath = workspace.reportDirectory.resolve("coverage-final.json"), + context = IstanbulCoverageContext( + backendId = FastCheckBackend.FAST_CHECK_BACKEND_ID, + backendVersion = FastCheckBackend.FAST_CHECK_BACKEND_VERSION, + propertyId = result.propertyId, + sourceRoots = request.sourceRoots, + propertyEntryPointPaths = entryPointPaths, + adapterRoot = workspace.adapterRoot.toString(), + runtimeVersion = runtimeVersion, + request = coverageRequest, + ), + ) + } catch (error: CoverageArtifactException) { + throw backendError( + kind = BackendErrorKind.COVERAGE, + code = error.diagnostic.code, + message = error.diagnostic.message, + request = request, + path = error.diagnostic.path, + cause = error, + ) + } + + return result.copy(coverage = artifact) + } + + private suspend fun nodeVersion(request: FastCheckExecutionRequest): String { + val process = startNodeVersionProcess(request) + + awaitNodeVersionProcess(process, request) + + return readNodeVersion(process, request) + } + + private fun startNodeVersionProcess(request: FastCheckExecutionRequest): Process = try { + ProcessBuilder(nodeExecutable, "--version").start() + } catch (error: IOException) { + throw coverageRuntimeVersionError(request, error) + } + + private suspend fun awaitNodeVersionProcess( + process: Process, + request: FastCheckExecutionRequest, + ) { + val completed = runInterruptible(ioDispatcher) { + process.waitFor(NODE_VERSION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) + } + if (!completed) { + process.destroyForcibly() + + throw backendError( + kind = BackendErrorKind.COVERAGE, + code = "coverage.runtime.version-unavailable", + message = "Timed out while querying the Node.js runtime version", + request = request, + ) + } + } + + private fun readNodeVersion( + process: Process, + request: FastCheckExecutionRequest, + ): String { + val version = process.inputStream.bufferedReader(Charsets.UTF_8).use { input -> + input.readLine().orEmpty().trim() + } + if (process.exitValue() != 0 || version.isBlank()) { + throw backendError( + kind = BackendErrorKind.COVERAGE, + code = "coverage.runtime.version-unavailable", + message = "Cannot query the Node.js runtime version", + request = request, + ) + } + + return requireSupportedNodeVersion(version, request) + } + + private fun requireSupportedNodeVersion( + version: String, + request: FastCheckExecutionRequest, + ): String { + val match = NODE_VERSION_PATTERN.matchEntire(version) + val major = match?.groupValues?.get(1)?.toIntOrNull() + val minor = match?.groupValues?.get(2)?.toIntOrNull() + if (major == null || minor == null) { + throw backendError( + kind = BackendErrorKind.COVERAGE, + code = "coverage.runtime.version-unavailable", + message = "Cannot parse the Node.js runtime version: $version", + request = request, + ) + } + + val isSupported = major > MINIMUM_NODE_MAJOR_VERSION || + major == MINIMUM_NODE_MAJOR_VERSION && minor >= MINIMUM_NODE_MINOR_VERSION + if (!isSupported) { + throw backendError( + kind = BackendErrorKind.COVERAGE, + code = "coverage.runtime.unsupported", + message = "Coverage requires Node.js 18.18 or newer; found $version", + request = request, + ) + } + + return version + } + + private fun coverageRuntimeVersionError( + request: FastCheckExecutionRequest, + error: IOException, + ) = backendError( + kind = BackendErrorKind.COVERAGE, + code = "coverage.runtime.version-unavailable", + message = "Cannot query the Node.js runtime version: ${error.message}", + request = request, + cause = error, + ) + + private fun adapterRoot(): Path = adapterEntryPoint.parent?.parent?.parent + ?: throw IllegalArgumentException("Adapter entry point has no runtime root: $adapterEntryPoint") + private suspend fun awaitIo( task: Deferred, operation: String, @@ -313,9 +528,22 @@ internal class FastCheckProcessClient( const val MAX_STDERR_BYTES = 64 * 1024 const val DEFAULT_TRANSPORT_GRACE_MILLIS = 2_000L const val DEFAULT_SHUTDOWN_GRACE_MILLIS = 250L + const val NODE_VERSION_TIMEOUT_MILLIS = 5_000L + const val MINIMUM_NODE_MAJOR_VERSION = 18 + const val MINIMUM_NODE_MINOR_VERSION = 18 + val NODE_VERSION_PATTERN = Regex("""^v(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$""") } } +private data class CoverageWorkspace( + val root: Path, + val configPath: Path, + val rawDirectory: Path, + val reportDirectory: Path, + val c8EntryPoint: Path, + val adapterRoot: Path, +) + private data class BoundedText(val text: String, val exceeded: Boolean) private fun InputStream.readBounded(limit: Int): BoundedText { diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/backend/PropertyCoverageTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/backend/PropertyCoverageTest.kt new file mode 100644 index 000000000..32b081ec0 --- /dev/null +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/backend/PropertyCoverageTest.kt @@ -0,0 +1,153 @@ +package org.usvm.ts.pbt.backend + +import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.model.PropertyId +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNull + +class PropertyCoverageTest { + @Test + fun `coverage request defaults to source under test only`() { + val request = PropertyCoverageRequest() + + assertEquals(setOf(CoverageScope.SOURCE_UNDER_TEST), request.scopes) + assertEquals(emptyList(), request.includePatterns) + assertEquals(emptyList(), request.excludePatterns) + } + + @Test + fun `supported and unsupported capabilities carry different evidence`() { + val supported = PropertyCoverageCapability.supported( + backendId = "fast-check", + backendVersion = "4.9.0", + collector = CoverageCollectorIdentity( + id = "c8", + version = "10.1.3", + ), + artifactVersion = 1, + ) + + assertEquals(CoverageCapabilityLevel.SUPPORTED, supported.level) + assertEquals("c8", supported.collector?.id) + assertEquals(1, supported.artifactVersion) + assertEquals(emptyList(), supported.diagnostics) + + val unsupported = PropertyCoverageCapability.unsupported( + backendId = "other-backend", + backendVersion = "1.0.0", + diagnostic = CoverageDiagnostic( + code = "coverage.unsupported", + message = "The backend does not collect source coverage", + path = "coverageRequest", + ), + ) + + assertEquals(CoverageCapabilityLevel.UNSUPPORTED, unsupported.level) + assertNull(unsupported.collector) + assertNull(unsupported.artifactVersion) + assertEquals("coverage.unsupported", unsupported.diagnostics.single().code) + } + + @Test + fun `coverage locations and hit counters reject invalid values`() { + assertFailsWith { SourcePosition(line = 0, column = 0) } + assertFailsWith { SourcePosition(line = 1, column = -1) } + assertFailsWith { + SourceRange( + start = SourcePosition(line = 2, column = 0), + end = SourcePosition(line = 1, column = 0), + ) + } + assertFailsWith { + StatementCoverage( + statementId = 0, + location = range, + hits = -1, + ) + } + assertFailsWith { + BranchCoverage( + branchId = 0, + type = "if", + location = range, + arms = emptyList(), + ) + } + } + + @Test + fun `run result requires matching coverage identity`() { + val artifact = artifact() + + successfulResult(coverage = artifact) + + assertFailsWith { + successfulResult( + coverage = artifact.copy(propertyId = PropertyId("different.property")), + ) + } + } + + private fun artifact() = PropertyCoverageArtifact( + schemaVersion = PROPERTY_COVERAGE_ARTIFACT_VERSION, + kind = CoverageArtifactKind.NODE_SOURCE, + backendId = "fast-check", + backendVersion = "4.9.0", + propertyId = PropertyId("example.property"), + provenance = CoverageProvenance( + collector = CoverageCollectorIdentity( + id = "c8", + version = "10.1.3", + ), + runtimeId = "node", + runtimeVersion = "v22.14.0", + sourceRoots = listOf("/workspace/src"), + request = PropertyCoverageRequest(), + ), + files = listOf( + SourceFileCoverage( + path = "/workspace/src/example.ts", + statements = listOf(StatementCoverage(statementId = 0, location = range, hits = 1)), + functions = listOf( + FunctionCoverage( + functionId = 0, + name = "example", + declaration = range, + body = range, + hits = 1, + ), + ), + branches = listOf( + BranchCoverage( + branchId = 0, + type = "if", + location = range, + arms = listOf(BranchArmCoverage(location = range, hits = 1)), + ), + ), + ), + ), + ) + + private fun successfulResult(coverage: PropertyCoverageArtifact?) = PropertyRunResult( + propertyId = PropertyId("example.property"), + status = PropertyRunStatus.SUCCESS, + seed = 42, + replayPath = null, + counterexample = null, + numRuns = 1, + numSkips = 0, + numShrinks = 0, + failure = null, + executionTimeMillis = 1, + coverage = coverage, + ) + + private companion object { + val range = SourceRange( + start = SourcePosition(line = 1, column = 0), + end = SourcePosition(line = 1, column = 10), + ) + } +} diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/FastCheckCliTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/FastCheckCliTest.kt index dd5a057f3..ca5867c70 100644 --- a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/FastCheckCliTest.kt +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/FastCheckCliTest.kt @@ -4,7 +4,14 @@ import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.backend.CoverageArtifactKind +import org.usvm.ts.pbt.backend.CoverageCollectorIdentity +import org.usvm.ts.pbt.backend.CoverageDiagnostic +import org.usvm.ts.pbt.backend.CoverageProvenance +import org.usvm.ts.pbt.backend.PROPERTY_COVERAGE_ARTIFACT_VERSION import org.usvm.ts.pbt.backend.PropertyBasedTestingBackend +import org.usvm.ts.pbt.backend.PropertyCoverageArtifact +import org.usvm.ts.pbt.backend.PropertyCoverageCapability import org.usvm.ts.pbt.backend.PropertyFailureDetails import org.usvm.ts.pbt.backend.PropertyFailureKind import org.usvm.ts.pbt.backend.PropertyRunConfiguration @@ -42,6 +49,7 @@ class FastCheckCliTest { assertEquals("", errors.toString()) assertContains(output, "TypeScript source root") assertContains(output, "--examples") + assertContains(output, "--coverage-scope") } @Test @@ -126,6 +134,84 @@ class FastCheckCliTest { assertEquals(listOf("passing", "failing"), propertyIds) } + @Test + fun `coverage options request and serialize one artifact per property`() { + val output = StringBuilder() + val errors = StringBuilder() + val cli = cli( + providers = listOf(provider(registryId = "examples", propertyIds = arrayOf("covered"))), + output = output, + errors = errors, + ) + + val exitCode = cli.run( + arrayOf( + "--source-root", sourceRoot.toString(), + "--property", "covered", + "--coverage", + "--coverage-scope", "property-entry-points", + "--coverage-include", "**/*.ts", + "--coverage-exclude", "**/ignored.ts", + ), + ) + + val result = PropertyManifestJson.json.parseToJsonElement(output.toString()).jsonArray.single().jsonObject + val coverage = result.getValue("coverage").jsonObject + val request = coverage.getValue("provenance").jsonObject.getValue("request").jsonObject + val includePatterns = request.getValue("includePatterns").jsonArray + val excludePatterns = request.getValue("excludePatterns").jsonArray + + assertEquals(0, exitCode) + assertEquals("", errors.toString()) + assertEquals(1, coverage.getValue("schemaVersion").jsonPrimitive.content.toInt()) + assertEquals( + listOf("property_entry_points"), + request.getValue("scopes").jsonArray.map { scope -> scope.jsonPrimitive.content }, + ) + assertEquals("**/*.ts", includePatterns.single().jsonPrimitive.content) + assertEquals("**/ignored.ts", excludePatterns.single().jsonPrimitive.content) + } + + @Test + fun `coverage details require the coverage flag`() { + val errors = StringBuilder() + + val exitCode = cli( + providers = listOf(provider(registryId = "examples", propertyIds = arrayOf("property"))), + errors = errors, + ).run( + arrayOf( + "--source-root", + sourceRoot.toString(), + "--coverage-scope", + "source-under-test", + ), + ) + + assertEquals(2, exitCode) + assertEquals("cli.coverage.required", diagnosticCode(errors)) + } + + @Test + fun `coverage request reports an unsupported backend before execution`() { + val errors = StringBuilder() + + val exitCode = cli( + providers = listOf(provider(registryId = "examples", propertyIds = arrayOf("property"))), + errors = errors, + supportsCoverage = false, + ).run( + arrayOf( + "--source-root", + sourceRoot.toString(), + "--coverage", + ), + ) + + assertEquals(2, exitCode) + assertEquals("coverage.unsupported", diagnosticCode(errors)) + } + @Test fun `reports unknown registries and duplicate property ids as CLI errors`() { val unknownErrors = StringBuilder() @@ -280,9 +366,15 @@ class FastCheckCliTest { output: Appendable = StringBuilder(), errors: Appendable = StringBuilder(), failures: Set = emptySet(), + supportsCoverage: Boolean = true, ) = FastCheckCli( providers = providers, - backendFactory = { FakeBackend(failures) }, + backendFactory = { + FakeBackend( + failures = failures, + supportsCoverage = supportsCoverage, + ) + }, output = output, errors = errors, ) @@ -309,7 +401,31 @@ class FastCheckCliTest { .jsonPrimitive .content - private class FakeBackend(private val failures: Set) : PropertyBasedTestingBackend { + private class FakeBackend( + private val failures: Set, + supportsCoverage: Boolean, + ) : PropertyBasedTestingBackend { + override val coverageCapability: PropertyCoverageCapability = if (supportsCoverage) { + PropertyCoverageCapability.supported( + backendId = "fake", + backendVersion = "1.0.0", + collector = CoverageCollectorIdentity( + id = "c8", + version = "10.1.3", + ), + ) + } else { + PropertyCoverageCapability.unsupported( + backendId = "fake", + backendVersion = "1.0.0", + diagnostic = CoverageDiagnostic( + code = "coverage.unsupported", + message = "Fake backend does not collect coverage", + path = "coverageRequest", + ), + ) + } + override fun run( property: PropertyDefinition, configuration: PropertyRunConfiguration, @@ -335,6 +451,23 @@ class FastCheckCliTest { null }, executionTimeMillis = 1, + coverage = configuration.coverageRequest?.let { request -> + PropertyCoverageArtifact( + schemaVersion = PROPERTY_COVERAGE_ARTIFACT_VERSION, + kind = CoverageArtifactKind.NODE_SOURCE, + backendId = coverageCapability.backendId, + backendVersion = coverageCapability.backendVersion, + propertyId = property.id, + provenance = CoverageProvenance( + collector = requireNotNull(coverageCapability.collector), + runtimeId = "node", + runtimeVersion = "v22.14.0", + sourceRoots = listOf(sourceRoot.toString()), + request = request, + ), + files = emptyList(), + ) + }, ) } } diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/InstalledDistributionRegistryProvider.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/InstalledDistributionRegistryProvider.kt index 9eaad1836..c7ca9e377 100644 --- a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/InstalledDistributionRegistryProvider.kt +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/InstalledDistributionRegistryProvider.kt @@ -26,6 +26,19 @@ class InstalledDistributionRegistryProvider : PropertyRegistryProvider { exportName = "alwaysTrue", ), ), + PropertyDefinition( + id = PropertyId("distribution.covers-positive"), + inputs = listOf( + PropertyInput( + name = "value", + domain = IntegerDomain(min = 1, max = 10), + ), + ), + predicate = TypeScriptEntryPoint( + module = "properties/coverage/CoverageProperties.ts", + exportName = "coversPositive", + ), + ), ), ) } diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt new file mode 100644 index 000000000..c3e7d6798 --- /dev/null +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt @@ -0,0 +1,204 @@ +package org.usvm.ts.pbt.coverage + +import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.backend.CoverageScope +import org.usvm.ts.pbt.backend.PropertyCoverageRequest +import org.usvm.ts.pbt.model.PropertyId +import java.nio.ByteBuffer +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption +import kotlin.io.path.absolute +import kotlin.io.path.createTempDirectory +import kotlin.io.path.createTempFile +import kotlin.io.path.deleteIfExists +import kotlin.io.path.writeText +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class IstanbulCoverageReportTest { + @Test + fun `decodes literal TypeScript statement function and branch hits`() { + val artifact = decodeIstanbulCoverageReport( + reportPath = goldenReport(), + context = context(), + ) + + val file = artifact.files.single() + val branch = file.branches.single() + val firstArm = branch.arms.first() + + assertEquals("/workspace/src/math.ts", file.path) + assertEquals(listOf(2L, 2L), file.statements.map { statement -> statement.hits }) + assertEquals("classify", file.functions.single().name) + assertEquals(2L, file.functions.single().hits) + assertEquals("if", branch.type) + assertEquals(listOf(1L, 1L), branch.arms.map { arm -> arm.hits }) + assertEquals(2, branch.location.start.line) + assertEquals(2, firstArm.location.start.line) + } + + @Test + fun `classifies entry points wrappers and dependencies independently`() { + val artifact = decodeIstanbulCoverageReport( + reportPath = goldenReport(), + context = context( + request = PropertyCoverageRequest(scopes = CoverageScope.entries.toSet()), + ), + ) + + assertEquals( + listOf( + "/adapter/dist/src/execution-cli.js", + "/adapter/node_modules/example/index.js", + "/workspace/src/math.ts", + "/workspace/src/property.ts", + ), + artifact.files.map { file -> file.path }, + ) + } + + @Test + fun `include and exclude globs filter remapped paths with exclude precedence`() { + val artifact = decodeIstanbulCoverageReport( + reportPath = goldenReport(), + context = context( + request = PropertyCoverageRequest( + scopes = CoverageScope.entries.toSet(), + includePatterns = listOf("**/*.ts"), + excludePatterns = listOf("**/property.?s"), + ), + ), + ) + + assertEquals(listOf("/workspace/src/math.ts"), artifact.files.map { file -> file.path }) + } + + @Test + fun `generated JavaScript below source roots reports a missing source map`() { + val artifact = decodeIstanbulCoverageReport( + reportPath = goldenReport(), + context = context(), + ) + + val diagnostic = artifact.diagnostics.single() + assertEquals("coverage.source-map.missing", diagnostic.code) + assertEquals("/workspace/src/generated-without-map.js", diagnostic.path) + } + + @Test + fun `missing and malformed reports produce stable diagnostics`() { + val missing = assertFailsWith { + decodeIstanbulCoverageReport( + reportPath = Path.of("/definitely/missing/coverage-final.json"), + context = context(), + ) + } + assertEquals("coverage.report.missing", missing.diagnostic.code) + + val malformedReport = createTempFile(suffix = ".json") + try { + malformedReport.writeText("{not-json") + val malformed = assertFailsWith { + decodeIstanbulCoverageReport( + reportPath = malformedReport, + context = context(), + ) + } + assertEquals("coverage.report.invalid", malformed.diagnostic.code) + } finally { + malformedReport.deleteIfExists() + } + } + + @Test + fun `report larger than the decoder limit is rejected before parsing`() { + val oversizedReport = createTempFile(suffix = ".json") + try { + Files.newByteChannel(oversizedReport, StandardOpenOption.WRITE).use { report -> + report.position(TEST_COVERAGE_REPORT_LIMIT_BYTES) + report.write(ByteBuffer.wrap(byteArrayOf(0))) + } + + val oversized = assertFailsWith { + decodeIstanbulCoverageReport( + reportPath = oversizedReport, + context = context(), + ) + } + + assertEquals("coverage.report.invalid", oversized.diagnostic.code) + assertEquals( + "Istanbul coverage report exceeds $TEST_COVERAGE_REPORT_LIMIT_BYTES bytes", + oversized.diagnostic.message, + ) + } finally { + oversizedReport.deleteIfExists() + } + } + + @Test + fun `generated JavaScript with an unusable map reports an invalid source map`() { + val directory = createTempDirectory("invalid-source-map-") + try { + val generated = directory.resolve("generated.js") + generated.writeText("export const value = 1;\n//# sourceMappingURL=generated.js.map") + directory.resolve("generated.js.map").writeText("{not-json") + val report = directory.resolve("coverage-final.json") + report.writeText( + """ + { + "$generated": { + "path": "$generated", + "statementMap": { + "0": { "start": { "line": 1, "column": 0 }, "end": { "line": 1, "column": 23 } } + }, + "fnMap": {}, + "branchMap": {}, + "s": { "0": 1 }, + "f": {}, + "b": {} + } + } + """.trimIndent(), + ) + + val artifact = decodeIstanbulCoverageReport( + reportPath = report, + context = context().copy(sourceRoots = listOf(directory.toString())), + ) + + assertEquals("coverage.source-map.invalid", artifact.diagnostics.single().code) + assertEquals(generated.toString(), artifact.diagnostics.single().path) + } finally { + directory.toFile().deleteRecursively() + } + } + + private fun context( + request: PropertyCoverageRequest = PropertyCoverageRequest(), + ) = IstanbulCoverageContext( + backendId = "fast-check", + backendVersion = "4.9.0", + propertyId = PropertyId("coverage.property"), + sourceRoots = listOf("/workspace/src"), + propertyEntryPointPaths = setOf("/workspace/src/property.ts"), + adapterRoot = "/adapter", + runtimeVersion = "v22.14.0", + request = request, + ) + + private fun goldenReport(): Path = locateFile( + "src/test/resources/coverage/istanbul/statement-branch.json", + "usvm-ts-pbt/src/test/resources/coverage/istanbul/statement-branch.json", + ) + + private fun locateFile(vararg candidates: String): Path = candidates + .map { candidate -> Path.of(candidate).absolute() } + .singleOrNull(Files::isRegularFile) + ?: error("Cannot locate file; checked ${candidates.toList()}") + + private companion object { + const val TEST_COVERAGE_REPORT_LIMIT_BYTES = 64L * 1024 * 1024 + } +} diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackendTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackendTest.kt index c618870e7..764c6fefc 100644 --- a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackendTest.kt +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackendTest.kt @@ -1,6 +1,8 @@ package org.usvm.ts.pbt.fastcheck import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.backend.CoverageScope +import org.usvm.ts.pbt.backend.PropertyCoverageRequest import org.usvm.ts.pbt.backend.PropertyFailureKind import org.usvm.ts.pbt.backend.PropertyRunConfiguration import org.usvm.ts.pbt.backend.PropertyRunStatus @@ -35,6 +37,24 @@ class FastCheckBackendTest { assertEquals(20, result.numRuns) } + @Test + fun `collects source mapped entry point coverage when Kotlin requests it`() { + val result = backend.run( + property = property(predicate = "alwaysTrue"), + configuration = configuration.copy( + coverageRequest = PropertyCoverageRequest( + scopes = setOf(CoverageScope.PROPERTY_ENTRY_POINTS), + ), + ), + ) + + val coverage = assertNotNull(result.coverage) + val file = coverage.files.single { source -> source.path.endsWith(MODULE) } + + assertTrue(file.statements.isNotEmpty()) + assertTrue(file.functions.isNotEmpty()) + } + @Test fun `passes multiple generated inputs to the predicate in declaration order`() { val definition = property( diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt new file mode 100644 index 000000000..1e846000e --- /dev/null +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt @@ -0,0 +1,131 @@ +package org.usvm.ts.pbt.fastcheck + +import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.backend.PropertyCoverageRequest +import org.usvm.ts.pbt.backend.PropertyRunConfiguration +import org.usvm.ts.pbt.backend.PropertyRunStatus +import org.usvm.ts.pbt.backend.SourceFileCoverage +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.PropertyDefinition +import org.usvm.ts.pbt.model.PropertyId +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.model.TypeScriptEntryPoint +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.absolute +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + +class FastCheckCoverageTest { + private val backend = FastCheckBackend( + sourceRoots = listOf(sourceRoot()), + adapterEntryPoint = adapterEntryPoint(), + ) + + @Test + fun `golden TypeScript statements functions and branches are isolated per property`() { + val positive = backend.run( + property = property( + exportName = "coversPositive", + domain = IntegerDomain(min = 1, max = 1), + ), + configuration = configuration, + ) + val nonPositive = backend.run( + property = property( + exportName = "coversNonPositive", + domain = IntegerDomain(min = -1, max = -1), + ), + configuration = configuration, + ) + + val positiveFile = sourceUnderTest(positive) + val nonPositiveFile = sourceUnderTest(nonPositive) + val positiveFunctions = positiveFile.functions.filter { function -> function.name == "classify" } + val nonPositiveFunctions = nonPositiveFile.functions.filter { function -> function.name == "classify" } + assertEquals(1, positiveFunctions.size, positiveFile.toString()) + assertEquals(1, nonPositiveFunctions.size, nonPositiveFile.toString()) + assertEquals(1L, positiveFunctions.single().hits) + assertEquals(1L, nonPositiveFunctions.single().hits) + assertEquals(listOf(5), zeroHitBranchLines(positiveFile)) + assertEquals(listOf(2), zeroHitBranchLines(nonPositiveFile)) + assertEquals(1L, statementHitsAtLine(positiveFile, line = 2)) + assertEquals(1L, statementHitsAtLine(nonPositiveFile, line = 2)) + } + + @Test + fun `falsified property retains coverage collected before failure`() { + val result = backend.run( + property = property( + exportName = "failsAfterClassifying", + domain = IntegerDomain(min = 1, max = 1), + ), + configuration = configuration, + ) + + assertEquals(PropertyRunStatus.FAILURE, result.status) + val file = sourceUnderTest(result) + val functions = file.functions.filter { function -> function.name == "classify" } + assertEquals(1, functions.size, file.toString()) + assertEquals(1L, functions.single().hits) + assertEquals(listOf(5), zeroHitBranchLines(file)) + } + + private fun sourceUnderTest(result: org.usvm.ts.pbt.backend.PropertyRunResult): SourceFileCoverage { + val artifact = assertNotNull(result.coverage) + return artifact.files.single { file -> file.path.endsWith("properties/coverage/source-under-test.ts") } + } + + private fun statementHitsAtLine(file: SourceFileCoverage, line: Int): Long = file.statements + .filter { statement -> statement.location.start.line == line } + .maxOf { statement -> statement.hits } + + private fun zeroHitBranchLines(file: SourceFileCoverage): List = file.branches + .filter { branch -> branch.arms.all { arm -> arm.hits == 0L } } + .map { branch -> branch.location.start.line } + .filter { line -> line > 1 } + .sorted() + + private fun property(exportName: String, domain: IntegerDomain) = PropertyDefinition( + id = PropertyId("coverage.$exportName"), + inputs = listOf( + PropertyInput( + name = "value", + domain = domain, + ), + ), + predicate = TypeScriptEntryPoint( + module = "properties/coverage/CoverageProperties.ts", + exportName = exportName, + ), + ) + + private companion object { + val configuration = PropertyRunConfiguration( + seed = 42, + numRuns = 1, + timeoutMillis = 5_000, + coverageRequest = PropertyCoverageRequest(), + ) + + fun adapterEntryPoint(): Path = locateFile( + "fast-check-adapter/dist/src/execution-cli.js", + "usvm-ts-pbt/fast-check-adapter/dist/src/execution-cli.js", + ) + + fun sourceRoot(): Path = locateDirectory( + "src/test/resources", + "usvm-ts-pbt/src/test/resources", + ) + + fun locateFile(vararg candidates: String): Path = candidates + .map { candidate -> Path.of(candidate).absolute() } + .singleOrNull(Files::isRegularFile) + ?: error("Cannot locate file; checked ${candidates.toList()}") + + fun locateDirectory(vararg candidates: String): Path = candidates + .map { candidate -> Path.of(candidate).absolute() } + .singleOrNull(Files::isDirectory) + ?: error("Cannot locate directory; checked ${candidates.toList()}") + } +} diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClientTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClientTest.kt index 60704169d..c5750410d 100644 --- a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClientTest.kt +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClientTest.kt @@ -1,6 +1,7 @@ package org.usvm.ts.pbt.fastcheck import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.backend.PropertyCoverageRequest import org.usvm.ts.pbt.manifest.toManifest import org.usvm.ts.pbt.model.BooleanDomain import org.usvm.ts.pbt.model.PropertyDefinition @@ -8,9 +9,14 @@ import org.usvm.ts.pbt.model.PropertyId import org.usvm.ts.pbt.model.PropertyInput import org.usvm.ts.pbt.model.TypeScriptEntryPoint import org.usvm.ts.pbt.testResourcesRoot +import java.nio.file.Files import java.nio.file.Path +import kotlin.io.path.createDirectories +import kotlin.io.path.createFile +import kotlin.io.path.createTempDirectory import kotlin.io.path.createTempFile import kotlin.io.path.deleteIfExists +import kotlin.io.path.readText import kotlin.io.path.writeText import kotlin.test.assertEquals import kotlin.test.assertFailsWith @@ -30,6 +36,190 @@ class FastCheckProcessClientTest { assertEquals("backend.process.start.failed", startup.code) } + @Test + fun `coverage workspace is removed when process startup fails`() { + val runtime = createTempDirectory(prefix = "fast-check-runtime-") + val adapter = runtime.resolve("dist/src/execution-cli.js") + val c8 = runtime.resolve("node_modules/c8/bin/c8.js") + val fakeNode = runtime.resolve("fake-node") + adapter.parent.createDirectories() + c8.parent.createDirectories() + adapter.createFile() + c8.createFile() + fakeNode.writeText( + """ + #!/bin/sh + if [ "${'$'}1" = "--version" ]; then + printf 'v22.14.0\n' + rm "${'$'}0" + exit 0 + fi + exit 1 + """.trimIndent(), + ) + check(fakeNode.toFile().setExecutable(true)) + val workspacesBefore = coverageWorkspaces() + + try { + val error = assertFailsWith { + FastCheckProcessClient( + nodeExecutable = fakeNode.toString(), + adapterEntryPoint = adapter, + ).check( + validRequest.copy(coverageRequest = PropertyCoverageRequest()), + ) + } + + assertEquals(BackendErrorKind.PROCESS_FAILURE, error.kind) + assertEquals(workspacesBefore, coverageWorkspaces()) + } finally { + runtime.toFile().deleteRecursively() + coverageWorkspaces() + .minus(workspacesBefore) + .forEach { workspace -> workspace.toFile().deleteRecursively() } + } + } + + @Test + fun `coverage rejects an unsupported Node runtime before starting c8`() { + val runtime = createTempDirectory(prefix = "fast-check-runtime-") + val adapter = runtime.resolve("dist/src/execution-cli.js") + val c8 = runtime.resolve("node_modules/c8/bin/c8.js") + val collectorStarted = runtime.resolve("collector-started") + val fakeNode = runtime.resolve("fake-node") + adapter.parent.createDirectories() + c8.parent.createDirectories() + adapter.createFile() + c8.createFile() + fakeNode.writeText( + """ + #!/bin/sh + if [ "${'$'}1" = "--version" ]; then + printf 'v16.20.2\n' + exit 0 + fi + touch "$collectorStarted" + exit 1 + """.trimIndent(), + ) + check(fakeNode.toFile().setExecutable(true)) + + try { + val error = assertFailsWith { + FastCheckProcessClient( + nodeExecutable = fakeNode.toString(), + adapterEntryPoint = adapter, + ).check( + validRequest.copy(coverageRequest = PropertyCoverageRequest()), + ) + } + + assertEquals(BackendErrorKind.COVERAGE, error.kind) + assertEquals("coverage.runtime.unsupported", error.code) + assertTrue(Files.notExists(collectorStarted)) + } finally { + runtime.toFile().deleteRecursively() + } + } + + @Test + fun `coverage collector preserves the caller working directory`() { + val runtime = createTempDirectory(prefix = "fast-check-runtime-") + val adapter = runtime.resolve("dist/src/execution-cli.js") + val c8 = runtime.resolve("node_modules/c8/bin/c8.js") + val collectorDirectory = runtime.resolve("collector-directory") + val fakeNode = runtime.resolve("fake-node") + adapter.parent.createDirectories() + c8.parent.createDirectories() + adapter.createFile() + c8.createFile() + fakeNode.writeText( + """ + #!/bin/sh + if [ "${'$'}1" = "--version" ]; then + printf 'v22.14.0\n' + exit 0 + fi + pwd > "$collectorDirectory" + exit 1 + """.trimIndent(), + ) + check(fakeNode.toFile().setExecutable(true)) + val callerDirectory = Path.of("").toAbsolutePath().normalize() + + try { + val error = assertFailsWith { + FastCheckProcessClient( + nodeExecutable = fakeNode.toString(), + adapterEntryPoint = adapter, + ).check( + validRequest.copy(coverageRequest = PropertyCoverageRequest()), + ) + } + val startedDirectory = Path.of(collectorDirectory.readText().trim()) + + assertEquals(BackendErrorKind.PROCESS_FAILURE, error.kind) + assertEquals(callerDirectory, startedDirectory) + } finally { + runtime.toFile().deleteRecursively() + } + } + + @Test + fun `coverage collector uses an explicit empty configuration`() { + val runtime = createTempDirectory(prefix = "fast-check-runtime-") + val adapter = runtime.resolve("dist/src/execution-cli.js") + val c8 = runtime.resolve("node_modules/c8/bin/c8.js") + val collectorArguments = runtime.resolve("collector-arguments") + val collectorConfiguration = runtime.resolve("collector-configuration") + val fakeNode = runtime.resolve("fake-node") + adapter.parent.createDirectories() + c8.parent.createDirectories() + adapter.createFile() + c8.createFile() + fakeNode.writeText( + """ + #!/bin/sh + if [ "${'$'}1" = "--version" ]; then + printf 'v22.14.0\n' + exit 0 + fi + printf '%s\n' "${'$'}@" > "$collectorArguments" + for argument in "${'$'}@"; do + case "${'$'}argument" in + --config=*) + config_path="${'$'}{argument#--config=}" + cat "${'$'}config_path" > "$collectorConfiguration" + ;; + esac + done + exit 1 + """.trimIndent(), + ) + check(fakeNode.toFile().setExecutable(true)) + + try { + val error = assertFailsWith { + FastCheckProcessClient( + nodeExecutable = fakeNode.toString(), + adapterEntryPoint = adapter, + ).check( + validRequest.copy(coverageRequest = PropertyCoverageRequest()), + ) + } + val configArguments = collectorArguments.readText() + .lineSequence() + .filter { argument -> argument.startsWith("--config=") } + .toList() + + assertEquals(BackendErrorKind.PROCESS_FAILURE, error.kind) + assertEquals(1, configArguments.size) + assertEquals("{}", collectorConfiguration.readText()) + } finally { + runtime.toFile().deleteRecursively() + } + } + @Test fun `non-zero exit retains stderr`() { withTemporaryAdapter(source = "process.stderr.write('adapter failed'); process.exit(3)") { client -> @@ -130,6 +320,13 @@ class FastCheckProcessClientTest { } } + private fun coverageWorkspaces(): Set { + val temporaryRoot = Path.of(System.getProperty("java.io.tmpdir")) + return Files.newDirectoryStream(temporaryRoot, "usvm-ts-pbt-coverage-*").use { entries -> + entries.toSet() + } + } + private companion object { data class InvalidResponseCase( val script: String, diff --git a/usvm-ts-pbt/src/test/resources/coverage/istanbul/statement-branch.json b/usvm-ts-pbt/src/test/resources/coverage/istanbul/statement-branch.json new file mode 100644 index 000000000..bf272a9f1 --- /dev/null +++ b/usvm-ts-pbt/src/test/resources/coverage/istanbul/statement-branch.json @@ -0,0 +1,73 @@ +{ + "/workspace/src/math.ts": { + "path": "/workspace/src/math.ts", + "statementMap": { + "0": { "start": { "line": 1, "column": 0 }, "end": { "line": 1, "column": 30 } }, + "1": { "start": { "line": 2, "column": 2 }, "end": { "line": 6, "column": 3 } } + }, + "fnMap": { + "0": { + "name": "classify", + "decl": { "start": { "line": 1, "column": 0 }, "end": { "line": 1, "column": 17 } }, + "loc": { "start": { "line": 1, "column": 29 }, "end": { "line": 7, "column": 1 } } + } + }, + "branchMap": { + "0": { + "type": "if", + "loc": { "start": { "line": 2, "column": 2 }, "end": { "line": 6, "column": 3 } }, + "locations": [ + { "start": { "line": 2, "column": 2 }, "end": { "line": 4, "column": 3 } }, + { "start": { "line": 4, "column": 9 }, "end": { "line": 6, "column": 3 } } + ] + } + }, + "s": { "0": 2, "1": 2 }, + "f": { "0": 2 }, + "b": { "0": [1, 1] } + }, + "/workspace/src/property.ts": { + "path": "/workspace/src/property.ts", + "statementMap": { + "0": { "start": { "line": 1, "column": 0 }, "end": { "line": 3, "column": 1 } } + }, + "fnMap": {}, + "branchMap": {}, + "s": { "0": 2 }, + "f": {}, + "b": {} + }, + "/workspace/src/generated-without-map.js": { + "path": "/workspace/src/generated-without-map.js", + "statementMap": { + "0": { "start": { "line": 1, "column": 0 }, "end": { "line": 1, "column": 10 } } + }, + "fnMap": {}, + "branchMap": {}, + "s": { "0": 1 }, + "f": {}, + "b": {} + }, + "/adapter/dist/src/execution-cli.js": { + "path": "/adapter/dist/src/execution-cli.js", + "statementMap": { + "0": { "start": { "line": 1, "column": 0 }, "end": { "line": 1, "column": 10 } } + }, + "fnMap": {}, + "branchMap": {}, + "s": { "0": 1 }, + "f": {}, + "b": {} + }, + "/adapter/node_modules/example/index.js": { + "path": "/adapter/node_modules/example/index.js", + "statementMap": { + "0": { "start": { "line": 1, "column": 0 }, "end": { "line": 1, "column": 10 } } + }, + "fnMap": {}, + "branchMap": {}, + "s": { "0": 1 }, + "f": {}, + "b": {} + } +} diff --git a/usvm-ts-pbt/src/test/resources/properties/coverage/CoverageProperties.ts b/usvm-ts-pbt/src/test/resources/properties/coverage/CoverageProperties.ts new file mode 100644 index 000000000..1b012ceb1 --- /dev/null +++ b/usvm-ts-pbt/src/test/resources/properties/coverage/CoverageProperties.ts @@ -0,0 +1,14 @@ +import { classify } from './source-under-test.ts'; + +export function coversPositive(value: number): boolean { + return classify(value) === 'positive'; +} + +export function coversNonPositive(value: number): boolean { + return classify(value) === 'non-positive'; +} + +export function failsAfterClassifying(value: number): boolean { + classify(value); + return false; +} diff --git a/usvm-ts-pbt/src/test/resources/properties/coverage/package.json b/usvm-ts-pbt/src/test/resources/properties/coverage/package.json new file mode 100644 index 000000000..3dbc1ca59 --- /dev/null +++ b/usvm-ts-pbt/src/test/resources/properties/coverage/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/usvm-ts-pbt/src/test/resources/properties/coverage/source-under-test.ts b/usvm-ts-pbt/src/test/resources/properties/coverage/source-under-test.ts new file mode 100644 index 000000000..eddc082a4 --- /dev/null +++ b/usvm-ts-pbt/src/test/resources/properties/coverage/source-under-test.ts @@ -0,0 +1,6 @@ +export function classify(value: number): string { + if (value > 0) { + return 'positive'; + } + return 'non-positive'; +} From b6baed3f64964061faa9381054d67c1f92671be4 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Sat, 29 Aug 2026 01:22:26 +0300 Subject: [PATCH 2/7] [TS PBT] Drop coverage artifact versioning --- usvm-ts-pbt/DESIGN.md | 16 ++++++++-------- usvm-ts-pbt/README.md | 14 +++++++------- .../org/usvm/ts/pbt/backend/PropertyCoverage.kt | 15 +-------------- .../ts/pbt/coverage/IstanbulCoverageReport.kt | 2 -- .../usvm/ts/pbt/backend/PropertyCoverageTest.kt | 16 ++++++++++++---- .../org/usvm/ts/pbt/cli/FastCheckCliTest.kt | 5 ++--- 6 files changed, 30 insertions(+), 38 deletions(-) diff --git a/usvm-ts-pbt/DESIGN.md b/usvm-ts-pbt/DESIGN.md index 9ebf12987..82dfe6214 100644 --- a/usvm-ts-pbt/DESIGN.md +++ b/usvm-ts-pbt/DESIGN.md @@ -106,13 +106,13 @@ to Node or added to `PropertyManifest`. A Node response is either: { status: "error", diagnostics: [{ kind, code, message, path }] } ``` -There is intentionally no request ID, operation name, schema version, protocol version, backend ID, or backend -version. The exchange is private, one-shot, and produced and consumed by the same build. Adding compatibility -metadata would create branches that no supported workflow uses. +There is intentionally no request ID, operation name, backend ID, or backend version. The exchange is private, +one-shot, and produced and consumed by the same build. Adding compatibility metadata would create branches that +no supported workflow uses. -Coverage does not version this private protocol. When requested, Kotlin starts the same execution CLI under c8, -then reads a separate Istanbul report after a valid response. Backend identity and artifact schema version belong -to `coverageCapability` and `PropertyCoverageArtifact`, not the ordinary property result or private wire response. +Coverage uses the same private protocol. When requested, Kotlin starts the execution CLI under c8, then reads a +separate Istanbul report after a valid response. Backend identity belongs to `coverageCapability` and +`PropertyCoverageArtifact`, not the ordinary property result or private wire response. Kotlin validates trusted model objects and examples early so callers get local errors. Node validates the decoded JSON again because the process boundary must not trust malformed input. Diagnostic codes have one owner per @@ -206,8 +206,8 @@ configuration prevents project-local c8 settings from altering collection. c8 th conversion and source-map remapping. Kotlin rejects reports larger than 64 MiB before reading them, validates statement, function, and branch maps and counters, classifies remapped files into source-under-test, property entry points, generated wrappers, or dependencies, then applies include and exclude -globs. Excludes take precedence. Files and diagnostics are sorted deterministically before constructing artifact -schema version 1. +globs. Excludes take precedence. Files and diagnostics are sorted deterministically before constructing the +artifact. A successful or falsified property exits the bridge normally, allowing c8 to flush the report. Process crashes, invalid protocol responses, and hard kills do not produce a completed property result. The workspace is removed diff --git a/usvm-ts-pbt/README.md b/usvm-ts-pbt/README.md index a2c3798d4..14a942a9c 100644 --- a/usvm-ts-pbt/README.md +++ b/usvm-ts-pbt/README.md @@ -85,9 +85,9 @@ examples, checking, and shrinking retain fast-check semantics. ## Per-property TypeScript coverage Coverage is a backend capability, not part of `PropertyDefinition` or `PropertyManifest`. -`PropertyBasedTestingBackend.coverageCapability` exposes the backend identity, backend version, collector, and -artifact version before execution. `FastCheckBackend` reports c8 10.1.3 and coverage artifact schema version 1. -An unsupported backend reports `coverage.unsupported` explicitly. +`PropertyBasedTestingBackend.coverageCapability` exposes the backend identity, backend version, and collector +before execution. `FastCheckBackend` reports c8 10.1.3. An unsupported backend reports +`coverage.unsupported` explicitly. Kotlin requests coverage for one run through `PropertyRunConfiguration`: @@ -127,10 +127,10 @@ Before starting c8, Kotlin probes the configured Node executable once. Versions `coverage.runtime.unsupported`; an unavailable or unparseable version uses `coverage.runtime.version-unavailable`. The verified version is recorded in the artifact provenance without a second probe. -Artifact schema version 1 has kind `NODE_SOURCE` and contains backend/property identity, c8 and Node provenance, -canonical source roots, the original request, and deterministic per-file statement, function, and branch hits. -Lines are one-based and columns are zero-based, following Istanbul. Node source coverage remains separate from -future EtsIR replay coverage. +The artifact has kind `NODE_SOURCE` and contains backend/property identity, c8 and Node provenance, canonical +source roots, the original request, and deterministic per-file statement, function, and branch hits. Lines are +one-based and columns are zero-based, following Istanbul. Node source coverage remains separate from future EtsIR +replay coverage. Missing or malformed reports use `coverage.report.missing` and `coverage.report.invalid`. Reports larger than 64 MiB are rejected as invalid before they are read. JavaScript below a TypeScript source root that c8 could not diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt index bb494841a..bbc0ff5db 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt @@ -4,8 +4,6 @@ import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import org.usvm.ts.pbt.model.PropertyId -const val PROPERTY_COVERAGE_ARTIFACT_VERSION = 1 - /** Distinguishes Node source coverage from future replay coverage over another representation. */ @Serializable enum class CoverageArtifactKind { @@ -71,7 +69,6 @@ data class PropertyCoverageCapability( val backendVersion: String, val level: CoverageCapabilityLevel, val collector: CoverageCollectorIdentity? = null, - val artifactVersion: Int? = null, val diagnostics: List = emptyList(), ) { init { @@ -80,15 +77,11 @@ data class PropertyCoverageCapability( when (level) { CoverageCapabilityLevel.SUPPORTED -> { requireNotNull(collector) { "Supported coverage requires a collector identity" } - require(artifactVersion == PROPERTY_COVERAGE_ARTIFACT_VERSION) { - "Supported coverage requires artifact version $PROPERTY_COVERAGE_ARTIFACT_VERSION" - } require(diagnostics.isEmpty()) { "Supported coverage must not contain capability diagnostics" } } CoverageCapabilityLevel.UNSUPPORTED -> { require(collector == null) { "Unsupported coverage must not name a collector" } - require(artifactVersion == null) { "Unsupported coverage must not name an artifact version" } require(diagnostics.isNotEmpty()) { "Unsupported coverage requires an actionable diagnostic" } } } @@ -99,13 +92,11 @@ data class PropertyCoverageCapability( backendId: String, backendVersion: String, collector: CoverageCollectorIdentity, - artifactVersion: Int = PROPERTY_COVERAGE_ARTIFACT_VERSION, ) = PropertyCoverageCapability( backendId = backendId, backendVersion = backendVersion, level = CoverageCapabilityLevel.SUPPORTED, collector = collector, - artifactVersion = artifactVersion, ) fun unsupported( @@ -245,10 +236,9 @@ data class CoverageProvenance( } } -/** Versioned per-property Node source coverage returned by a concrete backend. */ +/** Per-property Node source coverage returned by a concrete backend. */ @Serializable data class PropertyCoverageArtifact( - val schemaVersion: Int = PROPERTY_COVERAGE_ARTIFACT_VERSION, val kind: CoverageArtifactKind = CoverageArtifactKind.NODE_SOURCE, val backendId: String, val backendVersion: String, @@ -258,9 +248,6 @@ data class PropertyCoverageArtifact( val diagnostics: List = emptyList(), ) { init { - require(schemaVersion == PROPERTY_COVERAGE_ARTIFACT_VERSION) { - "Unsupported property coverage artifact version: $schemaVersion" - } require(backendId.isNotBlank()) { "Coverage backend ID must not be blank" } require(backendVersion.isNotBlank()) { "Coverage backend version must not be blank" } require(files.map(SourceFileCoverage::path).distinct().size == files.size) { diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt index a0289dd56..918b6ac03 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt @@ -16,7 +16,6 @@ import org.usvm.ts.pbt.backend.CoverageDiagnostic import org.usvm.ts.pbt.backend.CoverageProvenance import org.usvm.ts.pbt.backend.CoverageScope import org.usvm.ts.pbt.backend.FunctionCoverage -import org.usvm.ts.pbt.backend.PROPERTY_COVERAGE_ARTIFACT_VERSION import org.usvm.ts.pbt.backend.PropertyCoverageArtifact import org.usvm.ts.pbt.backend.PropertyCoverageRequest import org.usvm.ts.pbt.backend.SourceFileCoverage @@ -155,7 +154,6 @@ private fun decodeReport( }.sortedBy(SourceFileCoverage::path) return PropertyCoverageArtifact( - schemaVersion = PROPERTY_COVERAGE_ARTIFACT_VERSION, kind = CoverageArtifactKind.NODE_SOURCE, backendId = context.backendId, backendVersion = context.backendVersion, diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/backend/PropertyCoverageTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/backend/PropertyCoverageTest.kt index 32b081ec0..42d8d4f82 100644 --- a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/backend/PropertyCoverageTest.kt +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/backend/PropertyCoverageTest.kt @@ -1,9 +1,13 @@ package org.usvm.ts.pbt.backend +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.jsonObject import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.manifest.PropertyManifestJson import org.usvm.ts.pbt.model.PropertyId import kotlin.test.assertEquals import kotlin.test.assertFailsWith +import kotlin.test.assertFalse import kotlin.test.assertNull class PropertyCoverageTest { @@ -25,12 +29,14 @@ class PropertyCoverageTest { id = "c8", version = "10.1.3", ), - artifactVersion = 1, ) + val supportedJson = PropertyManifestJson.json + .parseToJsonElement(PropertyManifestJson.json.encodeToString(supported)) + .jsonObject assertEquals(CoverageCapabilityLevel.SUPPORTED, supported.level) assertEquals("c8", supported.collector?.id) - assertEquals(1, supported.artifactVersion) + assertFalse("artifactVersion" in supportedJson) assertEquals(emptyList(), supported.diagnostics) val unsupported = PropertyCoverageCapability.unsupported( @@ -42,10 +48,13 @@ class PropertyCoverageTest { path = "coverageRequest", ), ) + val unsupportedJson = PropertyManifestJson.json + .parseToJsonElement(PropertyManifestJson.json.encodeToString(unsupported)) + .jsonObject assertEquals(CoverageCapabilityLevel.UNSUPPORTED, unsupported.level) assertNull(unsupported.collector) - assertNull(unsupported.artifactVersion) + assertFalse("artifactVersion" in unsupportedJson) assertEquals("coverage.unsupported", unsupported.diagnostics.single().code) } @@ -90,7 +99,6 @@ class PropertyCoverageTest { } private fun artifact() = PropertyCoverageArtifact( - schemaVersion = PROPERTY_COVERAGE_ARTIFACT_VERSION, kind = CoverageArtifactKind.NODE_SOURCE, backendId = "fast-check", backendVersion = "4.9.0", diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/FastCheckCliTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/FastCheckCliTest.kt index ca5867c70..d00526bec 100644 --- a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/FastCheckCliTest.kt +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/FastCheckCliTest.kt @@ -8,7 +8,6 @@ import org.usvm.ts.pbt.backend.CoverageArtifactKind import org.usvm.ts.pbt.backend.CoverageCollectorIdentity import org.usvm.ts.pbt.backend.CoverageDiagnostic import org.usvm.ts.pbt.backend.CoverageProvenance -import org.usvm.ts.pbt.backend.PROPERTY_COVERAGE_ARTIFACT_VERSION import org.usvm.ts.pbt.backend.PropertyBasedTestingBackend import org.usvm.ts.pbt.backend.PropertyCoverageArtifact import org.usvm.ts.pbt.backend.PropertyCoverageCapability @@ -32,6 +31,7 @@ import java.nio.file.Files import java.nio.file.Path import kotlin.test.assertContains import kotlin.test.assertEquals +import kotlin.test.assertFalse class FastCheckCliTest { @Test @@ -163,7 +163,7 @@ class FastCheckCliTest { assertEquals(0, exitCode) assertEquals("", errors.toString()) - assertEquals(1, coverage.getValue("schemaVersion").jsonPrimitive.content.toInt()) + assertFalse("schemaVersion" in coverage) assertEquals( listOf("property_entry_points"), request.getValue("scopes").jsonArray.map { scope -> scope.jsonPrimitive.content }, @@ -453,7 +453,6 @@ class FastCheckCliTest { executionTimeMillis = 1, coverage = configuration.coverageRequest?.let { request -> PropertyCoverageArtifact( - schemaVersion = PROPERTY_COVERAGE_ARTIFACT_VERSION, kind = CoverageArtifactKind.NODE_SOURCE, backendId = coverageCapability.backendId, backendVersion = coverageCapability.backendVersion, From f0fcc070a61dd882abd802f5114d35aa78f1df03 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Sat, 29 Aug 2026 18:35:23 +0300 Subject: [PATCH 3/7] [TS PBT] Centralize coverage diagnostic codes --- .../org/usvm/ts/pbt/PbtDiagnosticCode.kt | 10 ++++++++++ .../org/usvm/ts/pbt/cli/FastCheckOptions.kt | 4 ++-- .../ts/pbt/coverage/IstanbulCoverageReport.kt | 19 ++++++++++++------- .../pbt/fastcheck/FastCheckProcessClient.kt | 12 ++++++------ 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/PbtDiagnosticCode.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/PbtDiagnosticCode.kt index fac25f84f..7ac238b1e 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/PbtDiagnosticCode.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/PbtDiagnosticCode.kt @@ -3,6 +3,8 @@ package org.usvm.ts.pbt /** Stable identifiers for diagnostics created on the Kotlin side of the PBT boundary. */ internal object PbtDiagnosticCode { const val CLI_ARGUMENT_INVALID = "cli.argument.invalid" + const val CLI_COVERAGE_REQUIRED = "cli.coverage.required" + const val CLI_COVERAGE_SCOPE_INVALID = "cli.coverage.scope.invalid" const val CLI_EXAMPLES_INVALID = "cli.examples.invalid" const val CLI_NUM_RUNS_INVALID = "cli.num-runs.invalid" const val CLI_PROPERTY_EMPTY = "cli.property.empty" @@ -35,6 +37,14 @@ internal object PbtDiagnosticCode { const val BACKEND_RESPONSE_TOO_LARGE = "backend.response.too-large" const val BACKEND_RUNTIME_NOT_FOUND = "backend.runtime.not-found" + const val COVERAGE_COLLECTOR_NOT_FOUND = "coverage.collector.not-found" + const val COVERAGE_REPORT_INVALID = "coverage.report.invalid" + const val COVERAGE_REPORT_MISSING = "coverage.report.missing" + const val COVERAGE_RUNTIME_UNSUPPORTED = "coverage.runtime.unsupported" + const val COVERAGE_RUNTIME_VERSION_UNAVAILABLE = "coverage.runtime.version-unavailable" + const val COVERAGE_SOURCE_MAP_INVALID = "coverage.source-map.invalid" + const val COVERAGE_SOURCE_MAP_MISSING = "coverage.source-map.missing" + const val PROTOCOL_REQUEST_INVALID = "protocol.request.invalid" const val SOURCE_ROOT_INVALID = "source-root.invalid" diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt index ca3ef9407..aca380a56 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt @@ -177,7 +177,7 @@ private class FastCheckOptionsParser : CliktCommand(name = "usvm-ts-pbt") { coverageExcludePatterns.isNotEmpty() if (!coverageEnabled && hasCoverageDetails) { throw CliUsageException( - code = "cli.coverage.required", + code = PbtDiagnosticCode.CLI_COVERAGE_REQUIRED, message = "Coverage scope and path rules require --coverage", path = "coverage", ) @@ -204,7 +204,7 @@ private fun parseCoverageScope(value: String): CoverageScope = when (value) { "generated-backend-wrappers" -> CoverageScope.GENERATED_BACKEND_WRAPPERS "dependencies" -> CoverageScope.DEPENDENCIES else -> throw CliUsageException( - code = "cli.coverage.scope.invalid", + code = PbtDiagnosticCode.CLI_COVERAGE_SCOPE_INVALID, message = "Unknown coverage scope $value", path = "coverageScope", ) diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt index 918b6ac03..1f0fb8312 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt @@ -8,6 +8,7 @@ import kotlinx.serialization.json.contentOrNull import kotlinx.serialization.json.intOrNull import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.longOrNull +import org.usvm.ts.pbt.PbtDiagnosticCode import org.usvm.ts.pbt.backend.BranchArmCoverage import org.usvm.ts.pbt.backend.BranchCoverage import org.usvm.ts.pbt.backend.CoverageArtifactKind @@ -62,7 +63,7 @@ private fun readCoverageReport(reportPath: Path): JsonObject { private fun requireCoverageReport(reportPath: Path) { if (!Files.isRegularFile(reportPath)) { throw coverageError( - code = "coverage.report.missing", + code = PbtDiagnosticCode.COVERAGE_REPORT_MISSING, message = "c8 did not produce the expected Istanbul report: $reportPath", path = reportPath.toString(), ) @@ -87,7 +88,7 @@ private fun requireCoverageReportSize(reportPath: Path) { } if (reportSize > MAX_COVERAGE_REPORT_BYTES) { throw coverageError( - code = "coverage.report.invalid", + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, message = "Istanbul coverage report exceeds $MAX_COVERAGE_REPORT_BYTES bytes", path = reportPath.toString(), ) @@ -98,7 +99,7 @@ private fun unreadableCoverageReport( reportPath: Path, error: IOException, ): CoverageArtifactException = coverageError( - code = "coverage.report.invalid", + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, message = "Cannot read Istanbul coverage report: ${error.message}", path = reportPath.toString(), cause = error, @@ -108,7 +109,7 @@ private fun parseCoverageReport(text: String, reportPath: Path): JsonObject = tr PropertyManifestJson.json.parseToJsonElement(text).jsonObject } catch (error: IllegalArgumentException) { throw coverageError( - code = "coverage.report.invalid", + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, message = "Istanbul coverage report is not valid JSON: ${error.message}", path = reportPath.toString(), cause = error, @@ -131,7 +132,11 @@ private fun decodeReport( if (isGeneratedJavaScriptBelowSourceRoot(path, normalizedRoots)) { val sourceMapExists = Files.exists(Path.of("$path.map")) diagnostics += CoverageDiagnostic( - code = if (sourceMapExists) "coverage.source-map.invalid" else "coverage.source-map.missing", + code = if (sourceMapExists) { + PbtDiagnosticCode.COVERAGE_SOURCE_MAP_INVALID + } else { + PbtDiagnosticCode.COVERAGE_SOURCE_MAP_MISSING + }, message = if (sourceMapExists) { "Executed JavaScript has a source map that c8 could not remap to its original source" } else { @@ -214,7 +219,7 @@ private fun decodeSourceFileCoverage( throw error } catch (error: IllegalArgumentException) { throw coverageError( - code = "coverage.report.invalid", + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, message = "Invalid Istanbul coverage for $path: ${error.message}", path = "coverage[$reportKey]", cause = error, @@ -368,7 +373,7 @@ private fun normalizeCoveragePath(path: String): String = Path.of(path) .replace('\\', '/') private fun invalidReport(message: String, path: String): CoverageArtifactException = coverageError( - code = "coverage.report.invalid", + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, message = message, path = path, ) diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt index 79f7a8c37..f17a8417b 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt @@ -228,7 +228,7 @@ internal class FastCheckProcessClient( if (!Files.isRegularFile(c8EntryPoint)) { throw backendError( kind = BackendErrorKind.COVERAGE, - code = "coverage.collector.not-found", + code = PbtDiagnosticCode.COVERAGE_COLLECTOR_NOT_FOUND, message = "Cannot locate c8 $C8_COLLECTOR_VERSION in the fast-check adapter runtime", request = request, path = c8EntryPoint.toString(), @@ -317,7 +317,7 @@ internal class FastCheckProcessClient( throw backendError( kind = BackendErrorKind.COVERAGE, - code = "coverage.runtime.version-unavailable", + code = PbtDiagnosticCode.COVERAGE_RUNTIME_VERSION_UNAVAILABLE, message = "Timed out while querying the Node.js runtime version", request = request, ) @@ -334,7 +334,7 @@ internal class FastCheckProcessClient( if (process.exitValue() != 0 || version.isBlank()) { throw backendError( kind = BackendErrorKind.COVERAGE, - code = "coverage.runtime.version-unavailable", + code = PbtDiagnosticCode.COVERAGE_RUNTIME_VERSION_UNAVAILABLE, message = "Cannot query the Node.js runtime version", request = request, ) @@ -353,7 +353,7 @@ internal class FastCheckProcessClient( if (major == null || minor == null) { throw backendError( kind = BackendErrorKind.COVERAGE, - code = "coverage.runtime.version-unavailable", + code = PbtDiagnosticCode.COVERAGE_RUNTIME_VERSION_UNAVAILABLE, message = "Cannot parse the Node.js runtime version: $version", request = request, ) @@ -364,7 +364,7 @@ internal class FastCheckProcessClient( if (!isSupported) { throw backendError( kind = BackendErrorKind.COVERAGE, - code = "coverage.runtime.unsupported", + code = PbtDiagnosticCode.COVERAGE_RUNTIME_UNSUPPORTED, message = "Coverage requires Node.js 18.18 or newer; found $version", request = request, ) @@ -378,7 +378,7 @@ internal class FastCheckProcessClient( error: IOException, ) = backendError( kind = BackendErrorKind.COVERAGE, - code = "coverage.runtime.version-unavailable", + code = PbtDiagnosticCode.COVERAGE_RUNTIME_VERSION_UNAVAILABLE, message = "Cannot query the Node.js runtime version: ${error.message}", request = request, cause = error, From 39a2f4d2de13f9b22c2763a6ca44a1caa51c4f99 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Sat, 29 Aug 2026 18:42:05 +0300 Subject: [PATCH 4/7] [TS PBT] Avoid ordered sets where order is irrelevant --- .../main/kotlin/org/usvm/ts/pbt/cli/FastCheckCli.kt | 4 ++-- .../kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt | 4 +--- .../usvm/ts/pbt/coverage/IstanbulCoverageReport.kt | 2 +- .../usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt | 13 ++++++------- .../ts/pbt/coverage/IstanbulCoverageReportTest.kt | 4 ++-- .../ts/pbt/fastcheck/FastCheckProcessClientTest.kt | 2 +- 6 files changed, 13 insertions(+), 16 deletions(-) diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckCli.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckCli.kt index 762603e6e..198ab3d74 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckCli.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckCli.kt @@ -204,8 +204,8 @@ class FastCheckCli( if (registryIds.isEmpty()) return orderedProviders - val requestedIds = registryIds.toSet() - val availableIds = orderedProviders.map(ProviderRegistration::id).toSet() + val requestedIds = registryIds.toHashSet() + val availableIds = orderedProviders.mapTo(hashSetOf(), ProviderRegistration::id) val unknown = requestedIds.minus(availableIds).minOrNull() if (unknown != null) { diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt index aca380a56..17b2e1e32 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt @@ -188,9 +188,7 @@ private class FastCheckOptionsParser : CliktCommand(name = "usvm-ts-pbt") { if (!coverageEnabled) return null return PropertyCoverageRequest( - scopes = coverageScopes - .map(::parseCoverageScope) - .toSet() + scopes = coverageScopes.mapTo(hashSetOf(), ::parseCoverageScope) .ifEmpty { setOf(CoverageScope.SOURCE_UNDER_TEST) }, includePatterns = coverageIncludePatterns, excludePatterns = coverageExcludePatterns, diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt index 1f0fb8312..26157ee69 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt @@ -121,7 +121,7 @@ private fun decodeReport( context: IstanbulCoverageContext, ): PropertyCoverageArtifact { val normalizedRoots = context.sourceRoots.map(::normalizeCoveragePath) - val normalizedEntryPoints = context.propertyEntryPointPaths.mapTo(mutableSetOf(), ::normalizeCoveragePath) + val normalizedEntryPoints = context.propertyEntryPointPaths.mapTo(hashSetOf(), ::normalizeCoveragePath) val normalizedAdapterRoot = normalizeCoveragePath(context.adapterRoot) val diagnostics = mutableListOf() val files = report.entries.mapNotNull { (reportKey, element) -> diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt index f17a8417b..13318ccd4 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt @@ -254,13 +254,12 @@ internal class FastCheckProcessClient( runtimeVersion: String, ): PropertyRunResult { val coverageRequest = requireNotNull(request.coverageRequest) - val entryPointPaths = buildSet { - request.sourceRoots.forEach { sourceRoot -> - val root = Path.of(sourceRoot) - add(root.resolve(request.manifest.predicate.module).normalize().toString()) - request.manifest.precondition?.let { precondition -> - add(root.resolve(precondition.module).normalize().toString()) - } + val entryPointPaths = hashSetOf() + request.sourceRoots.forEach { sourceRoot -> + val root = Path.of(sourceRoot) + entryPointPaths += root.resolve(request.manifest.predicate.module).normalize().toString() + request.manifest.precondition?.let { precondition -> + entryPointPaths += root.resolve(precondition.module).normalize().toString() } } val artifact = try { diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt index c3e7d6798..f34336791 100644 --- a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt @@ -43,7 +43,7 @@ class IstanbulCoverageReportTest { val artifact = decodeIstanbulCoverageReport( reportPath = goldenReport(), context = context( - request = PropertyCoverageRequest(scopes = CoverageScope.entries.toSet()), + request = PropertyCoverageRequest(scopes = CoverageScope.entries.toHashSet()), ), ) @@ -64,7 +64,7 @@ class IstanbulCoverageReportTest { reportPath = goldenReport(), context = context( request = PropertyCoverageRequest( - scopes = CoverageScope.entries.toSet(), + scopes = CoverageScope.entries.toHashSet(), includePatterns = listOf("**/*.ts"), excludePatterns = listOf("**/property.?s"), ), diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClientTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClientTest.kt index c5750410d..008bd2d5f 100644 --- a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClientTest.kt +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClientTest.kt @@ -323,7 +323,7 @@ class FastCheckProcessClientTest { private fun coverageWorkspaces(): Set { val temporaryRoot = Path.of(System.getProperty("java.io.tmpdir")) return Files.newDirectoryStream(temporaryRoot, "usvm-ts-pbt-coverage-*").use { entries -> - entries.toSet() + entries.toHashSet() } } From 5a3a1ade32898f0088c3c3aee79b0e905d0ed20f Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Sat, 29 Aug 2026 18:44:26 +0300 Subject: [PATCH 5/7] [TS PBT] Explain coverage glob translation --- .../org/usvm/ts/pbt/coverage/CoveragePathFilter.kt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/CoveragePathFilter.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/CoveragePathFilter.kt index 88b964003..7643b439f 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/CoveragePathFilter.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/CoveragePathFilter.kt @@ -20,7 +20,14 @@ internal fun matchesCoveragePath( } } +/** + * Converts a coverage glob into a regex over normalized forward-slash paths. + * + * A single `*` or `?` stays within one path segment, while `**` may cross directory separators. + * The resulting regex matches the complete candidate path rather than an arbitrary substring. + */ private fun coverageGlobToRegex(pattern: String): Regex { + // Treat Windows and Unix patterns uniformly and accept the common relative-path prefix. val normalized = pattern.replace('\\', '/').removePrefix("./") val expression = StringBuilder("^") var index = 0 @@ -29,25 +36,30 @@ private fun coverageGlobToRegex(pattern: String): Regex { when { character == '*' && normalized.getOrNull(index + 1) == '*' -> { if (normalized.getOrNull(index + DOUBLE_WILDCARD_LENGTH) == '/') { + // `**/` covers zero or more whole segments: `**/*.ts` also matches `file.ts`. expression.append("(?:.*/)?") index += DOUBLE_WILDCARD_WITH_SEPARATOR_LENGTH } else { + // A bare `**` consumes any characters, including directory separators. expression.append(".*") index += 2 } } character == '*' -> { + // A single wildcard consumes characters only inside the current path segment. expression.append("[^/]*") index++ } character == '?' -> { + // `?` consumes exactly one non-separator character in the current segment. expression.append("[^/]") index++ } character in REGEX_SPECIAL_CHARACTERS -> { + // Non-glob regex metacharacters are literals in coverage patterns. expression.append('\\').append(character) index++ } From 12419a510694b42ee5c3d6810cb7c8776f919e4f Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Sat, 29 Aug 2026 18:56:59 +0300 Subject: [PATCH 6/7] [TS PBT] Source runtime versions from the lockfile --- usvm-ts-pbt/build.gradle.kts | 51 ++++++++++++++++++- .../ts/pbt/coverage/IstanbulCoverageReport.kt | 5 +- .../usvm/ts/pbt/fastcheck/FastCheckBackend.kt | 10 +--- .../pbt/fastcheck/FastCheckProcessClient.kt | 7 +-- .../pbt/fastcheck/FastCheckRuntimeMetadata.kt | 29 +++++++++++ .../coverage/IstanbulCoverageReportTest.kt | 2 + .../fastcheck/FastCheckRuntimeMetadataTest.kt | 49 ++++++++++++++++++ 7 files changed, 137 insertions(+), 16 deletions(-) create mode 100644 usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntimeMetadata.kt create mode 100644 usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntimeMetadataTest.kt diff --git a/usvm-ts-pbt/build.gradle.kts b/usvm-ts-pbt/build.gradle.kts index aa7bcd608..afc82d9f4 100644 --- a/usvm-ts-pbt/build.gradle.kts +++ b/usvm-ts-pbt/build.gradle.kts @@ -1,3 +1,5 @@ +import groovy.json.JsonSlurper + plugins { id("usvm.kotlin-conventions") kotlin("plugin.serialization") version Versions.kotlin @@ -14,7 +16,12 @@ dependencies { } val fastCheckAdapterDir = layout.projectDirectory.dir("fast-check-adapter") +val fastCheckAdapterPackageJson = fastCheckAdapterDir.file("package.json") +val fastCheckAdapterPackageLock = fastCheckAdapterDir.file("package-lock.json") val fastCheckRuntimeProperty = "org.usvm.ts.pbt.fastcheck.runtime" +val generatedFastCheckRuntimeMetadataDirectory = layout.buildDirectory.dir( + "generated/resources/fastCheckRuntimeMetadata", +) val hostOperatingSystem = System.getProperty("os.name").lowercase() val hostPlatform = when { hostOperatingSystem.contains("mac") -> "darwin" @@ -31,12 +38,52 @@ val hostArchitecture = when (val architecture = System.getProperty("os.arch").lo val fastCheckRuntimeClassifier = "$hostPlatform-$hostArchitecture" val npmExecutable = if (hostPlatform == "win32") "npm.cmd" else "npm" +val generateFastCheckRuntimeMetadata = tasks.register("generateFastCheckRuntimeMetadata") { + inputs.file(fastCheckAdapterPackageLock) + outputs.dir(generatedFastCheckRuntimeMetadataDirectory) + + doLast { + val packageLock = JsonSlurper().parse(fastCheckAdapterPackageLock.asFile) as? Map<*, *> + ?: error("Invalid fast-check adapter package lock") + val packages = packageLock["packages"] as? Map<*, *> + ?: error("Missing packages in fast-check adapter package lock") + fun dependencyVersion(dependency: String): String { + val metadata = packages["node_modules/$dependency"] as? Map<*, *> + ?: error("Missing locked fast-check adapter dependency: $dependency") + + return (metadata["version"] as? String) + ?.takeIf(String::isNotBlank) + ?: error("Missing locked fast-check adapter dependency version: $dependency") + } + + val metadataFile = generatedFastCheckRuntimeMetadataDirectory.get() + .file("org/usvm/ts/pbt/fastcheck/runtime-dependencies.properties") + .asFile + metadataFile.parentFile.mkdirs() + metadataFile.writeText( + """ + fast-check.version=${dependencyVersion("fast-check")} + c8.version=${dependencyVersion("c8")} + """.trimIndent() + "\n", + Charsets.UTF_8, + ) + } +} + +sourceSets.main { + resources.srcDir(generatedFastCheckRuntimeMetadataDirectory) +} + +tasks.processResources { + dependsOn(generateFastCheckRuntimeMetadata) +} + val installFastCheckAdapter = tasks.register("installFastCheckAdapter") { workingDir(fastCheckAdapterDir) commandLine(npmExecutable, "ci", "--ignore-scripts") inputs.files( - fastCheckAdapterDir.file("package.json"), - fastCheckAdapterDir.file("package-lock.json"), + fastCheckAdapterPackageJson, + fastCheckAdapterPackageLock, ) inputs.property("runtimeClassifier", fastCheckRuntimeClassifier) outputs.dir(fastCheckAdapterDir.dir("node_modules")) diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt index 26157ee69..419e5e99c 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt @@ -29,8 +29,6 @@ import java.io.IOException import java.nio.file.Files import java.nio.file.Path -const val C8_COLLECTOR_VERSION = "10.1.3" - /** All identities and path boundaries required to convert one isolated c8 report. */ data class IstanbulCoverageContext( val backendId: String, @@ -40,6 +38,7 @@ data class IstanbulCoverageContext( val propertyEntryPointPaths: Set, val adapterRoot: String, val runtimeVersion: String, + val collector: CoverageCollectorIdentity, val request: PropertyCoverageRequest, ) @@ -164,7 +163,7 @@ private fun decodeReport( backendVersion = context.backendVersion, propertyId = context.propertyId, provenance = CoverageProvenance( - collector = CoverageCollectorIdentity(id = "c8", version = C8_COLLECTOR_VERSION), + collector = context.collector, runtimeId = "node", runtimeVersion = context.runtimeVersion, sourceRoots = normalizedRoots, diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt index aaac7e9b8..e17ddf93c 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt @@ -1,12 +1,10 @@ package org.usvm.ts.pbt.fastcheck import org.usvm.ts.pbt.PbtDiagnosticCode -import org.usvm.ts.pbt.backend.CoverageCollectorIdentity import org.usvm.ts.pbt.backend.PropertyBasedTestingBackend import org.usvm.ts.pbt.backend.PropertyCoverageCapability import org.usvm.ts.pbt.backend.PropertyRunConfiguration import org.usvm.ts.pbt.backend.PropertyRunResult -import org.usvm.ts.pbt.coverage.C8_COLLECTOR_VERSION import org.usvm.ts.pbt.manifest.toManifest import org.usvm.ts.pbt.model.JsConcreteValue import org.usvm.ts.pbt.model.JsNumberKind @@ -26,11 +24,8 @@ class FastCheckBackend( ) : PropertyBasedTestingBackend { override val coverageCapability: PropertyCoverageCapability = PropertyCoverageCapability.supported( backendId = FAST_CHECK_BACKEND_ID, - backendVersion = FAST_CHECK_BACKEND_VERSION, - collector = CoverageCollectorIdentity( - id = "c8", - version = C8_COLLECTOR_VERSION, - ), + backendVersion = FastCheckRuntimeMetadata.fastCheckVersion, + collector = FastCheckRuntimeMetadata.coverageCollector, ) private val sourceRoots = canonicalizeSourceRoots(sourceRoots) @@ -147,7 +142,6 @@ class FastCheckBackend( companion object { const val FAST_CHECK_BACKEND_ID = "fast-check" - const val FAST_CHECK_BACKEND_VERSION = "4.9.0" private val FINITE_NUMBER_BITS_REGEX = Regex("[0-9a-f]{16}") diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt index 13318ccd4..85085343b 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt @@ -14,7 +14,6 @@ import kotlinx.serialization.encodeToString import org.usvm.ts.pbt.PbtDiagnosticCode import org.usvm.ts.pbt.backend.CoverageScope import org.usvm.ts.pbt.backend.PropertyRunResult -import org.usvm.ts.pbt.coverage.C8_COLLECTOR_VERSION import org.usvm.ts.pbt.coverage.CoverageArtifactException import org.usvm.ts.pbt.coverage.IstanbulCoverageContext import org.usvm.ts.pbt.coverage.decodeIstanbulCoverageReport @@ -229,7 +228,8 @@ internal class FastCheckProcessClient( throw backendError( kind = BackendErrorKind.COVERAGE, code = PbtDiagnosticCode.COVERAGE_COLLECTOR_NOT_FOUND, - message = "Cannot locate c8 $C8_COLLECTOR_VERSION in the fast-check adapter runtime", + message = "Cannot locate c8 ${FastCheckRuntimeMetadata.coverageCollector.version} " + + "in the fast-check adapter runtime", request = request, path = c8EntryPoint.toString(), ) @@ -267,12 +267,13 @@ internal class FastCheckProcessClient( reportPath = workspace.reportDirectory.resolve("coverage-final.json"), context = IstanbulCoverageContext( backendId = FastCheckBackend.FAST_CHECK_BACKEND_ID, - backendVersion = FastCheckBackend.FAST_CHECK_BACKEND_VERSION, + backendVersion = FastCheckRuntimeMetadata.fastCheckVersion, propertyId = result.propertyId, sourceRoots = request.sourceRoots, propertyEntryPointPaths = entryPointPaths, adapterRoot = workspace.adapterRoot.toString(), runtimeVersion = runtimeVersion, + collector = FastCheckRuntimeMetadata.coverageCollector, request = coverageRequest, ), ) diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntimeMetadata.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntimeMetadata.kt new file mode 100644 index 000000000..e68827f61 --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntimeMetadata.kt @@ -0,0 +1,29 @@ +package org.usvm.ts.pbt.fastcheck + +import org.usvm.ts.pbt.backend.CoverageCollectorIdentity +import java.util.Properties + +/** Dependency versions generated from the packaged fast-check adapter lockfile. */ +internal object FastCheckRuntimeMetadata { + private val properties = Properties().apply { + val resource = checkNotNull( + FastCheckRuntimeMetadata::class.java.getResourceAsStream(RUNTIME_METADATA_RESOURCE), + ) { + "Missing generated fast-check runtime metadata: $RUNTIME_METADATA_RESOURCE" + } + resource.use(::load) + } + + val fastCheckVersion: String = requireVersion("fast-check.version") + + val coverageCollector = CoverageCollectorIdentity( + id = "c8", + version = requireVersion("c8.version"), + ) + + private fun requireVersion(name: String): String = properties.getProperty(name) + ?.takeIf(String::isNotBlank) + ?: error("Missing fast-check runtime dependency version: $name") + + private const val RUNTIME_METADATA_RESOURCE = "/org/usvm/ts/pbt/fastcheck/runtime-dependencies.properties" +} diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt index f34336791..81e2ade2b 100644 --- a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt @@ -1,6 +1,7 @@ package org.usvm.ts.pbt.coverage import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.backend.CoverageCollectorIdentity import org.usvm.ts.pbt.backend.CoverageScope import org.usvm.ts.pbt.backend.PropertyCoverageRequest import org.usvm.ts.pbt.model.PropertyId @@ -185,6 +186,7 @@ class IstanbulCoverageReportTest { propertyEntryPointPaths = setOf("/workspace/src/property.ts"), adapterRoot = "/adapter", runtimeVersion = "v22.14.0", + collector = CoverageCollectorIdentity(id = "c8", version = "10.1.3"), request = request, ) diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntimeMetadataTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntimeMetadataTest.kt new file mode 100644 index 000000000..f4c7396ec --- /dev/null +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntimeMetadataTest.kt @@ -0,0 +1,49 @@ +package org.usvm.ts.pbt.fastcheck + +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.manifest.PropertyManifestJson +import org.usvm.ts.pbt.testResourcesRoot +import java.nio.file.Files +import java.util.Properties +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + +class FastCheckRuntimeMetadataTest { + @Test + fun `runtime dependency versions are packaged from the adapter lockfile`() { + val adapterRoot = FastCheckRuntime.executionEntryPoint().parent.parent.parent + val packageLock = PropertyManifestJson.json + .parseToJsonElement(Files.readString(adapterRoot.resolve("package-lock.json"))) + .jsonObject + val packages = packageLock.getValue("packages").jsonObject + val expectedFastCheckVersion = packages.getValue("node_modules/fast-check") + .jsonObject + .getValue("version") + .jsonPrimitive + .content + val expectedC8Version = packages.getValue("node_modules/c8") + .jsonObject + .getValue("version") + .jsonPrimitive + .content + + val metadata = Properties().apply { + val resource = assertNotNull( + FastCheckRuntimeMetadataTest::class.java.getResourceAsStream(RUNTIME_METADATA_RESOURCE), + ) + resource.use(::load) + } + val backend = FastCheckBackend(sourceRoots = listOf(testResourcesRoot())) + + assertEquals(expectedFastCheckVersion, metadata.getProperty("fast-check.version")) + assertEquals(expectedC8Version, metadata.getProperty("c8.version")) + assertEquals(expectedFastCheckVersion, backend.coverageCapability.backendVersion) + assertEquals(expectedC8Version, backend.coverageCapability.collector?.version) + } + + private companion object { + const val RUNTIME_METADATA_RESOURCE = "/org/usvm/ts/pbt/fastcheck/runtime-dependencies.properties" + } +} From bf5e8c10ee908b1548a0471d70c6b6e71edfa828 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Sat, 29 Aug 2026 19:10:24 +0300 Subject: [PATCH 7/7] Refactor Istanbul coverage decoding --- .../pbt/coverage/IstanbulCoverageDecoder.kt | 165 ++++++++ .../ts/pbt/coverage/IstanbulCoverageReport.kt | 376 +----------------- .../coverage/IstanbulCoverageReportReader.kt | 76 ++++ .../usvm/ts/pbt/coverage/IstanbulJsonValue.kt | 120 ++++++ .../pbt/coverage/IstanbulSourceFileDecoder.kt | 241 +++++++++++ 5 files changed, 621 insertions(+), 357 deletions(-) create mode 100644 usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageDecoder.kt create mode 100644 usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportReader.kt create mode 100644 usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulJsonValue.kt create mode 100644 usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulSourceFileDecoder.kt diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageDecoder.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageDecoder.kt new file mode 100644 index 000000000..c2a64f707 --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageDecoder.kt @@ -0,0 +1,165 @@ +package org.usvm.ts.pbt.coverage + +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.backend.CoverageArtifactKind +import org.usvm.ts.pbt.backend.CoverageDiagnostic +import org.usvm.ts.pbt.backend.CoverageProvenance +import org.usvm.ts.pbt.backend.CoverageScope +import org.usvm.ts.pbt.backend.PropertyCoverageArtifact +import org.usvm.ts.pbt.backend.SourceFileCoverage +import java.nio.file.Files +import java.nio.file.Path + +/** Selects relevant report entries and assembles the coverage artifact for one property run. */ +internal class IstanbulCoverageDecoder( + private val context: IstanbulCoverageContext, +) { + private val sourceRoots = context.sourceRoots.map(::normalizeCoveragePath) + private val propertyEntryPoints = context.propertyEntryPointPaths + .mapTo(hashSetOf(), ::normalizeCoveragePath) + private val adapterRoot = normalizeCoveragePath(context.adapterRoot) + + fun decode(report: JsonObject): PropertyCoverageArtifact { + val diagnostics = mutableListOf() + val files = decodeFiles(report, diagnostics) + val sortedDiagnostics = diagnostics.sortedWith( + compareBy(CoverageDiagnostic::path, CoverageDiagnostic::code), + ) + + return buildArtifact(files, sortedDiagnostics) + } + + private fun decodeFiles( + report: JsonObject, + diagnostics: MutableList, + ): List { + val decodedFiles = buildList { + for ((reportKey, reportEntry) in report) { + val decodedFile = decodeFile(reportKey, reportEntry, diagnostics) + if (decodedFile != null) { + add(decodedFile) + } + } + } + + return decodedFiles.sortedBy(SourceFileCoverage::path) + } + + private fun decodeFile( + reportKey: String, + reportEntry: JsonElement, + diagnostics: MutableList, + ): SourceFileCoverage? { + val coveragePath = "coverage[$reportKey]" + val fileJson = IstanbulJsonValue(reportEntry, coveragePath) + val fileObject = fileJson.asObject() + val reportedPath = fileJson.optionalString(name = "path") + val path = normalizeCoveragePath(reportedPath ?: reportKey) + + if (isGeneratedJavaScriptBelowSourceRoot(path)) { + diagnostics += sourceMapDiagnostic(path) + return null + } + + val scope = classifyScope(path) ?: return null + if (!shouldRetain(path, scope)) { + return null + } + + return IstanbulSourceFileDecoder( + file = fileObject, + path = path, + reportKey = reportKey, + ).decode() + } + + private fun sourceMapDiagnostic(path: String): CoverageDiagnostic { + val sourceMapPath = Path.of("$path.map") + val sourceMapExists = Files.exists(sourceMapPath) + + val diagnosticCode = if (sourceMapExists) { + PbtDiagnosticCode.COVERAGE_SOURCE_MAP_INVALID + } else { + PbtDiagnosticCode.COVERAGE_SOURCE_MAP_MISSING + } + val diagnosticMessage = if (sourceMapExists) { + "Executed JavaScript has a source map that c8 could not remap to its original source" + } else { + "Executed JavaScript below a TypeScript source root has no source map" + } + + return CoverageDiagnostic( + code = diagnosticCode, + message = diagnosticMessage, + path = path, + ) + } + + private fun classifyScope(path: String): CoverageScope? = when { + "/node_modules/" in path -> CoverageScope.DEPENDENCIES + isWithin(path, adapterRoot) -> CoverageScope.GENERATED_BACKEND_WRAPPERS + path in propertyEntryPoints -> CoverageScope.PROPERTY_ENTRY_POINTS + sourceRoots.any { sourceRoot -> isWithin(path, sourceRoot) } -> CoverageScope.SOURCE_UNDER_TEST + else -> null + } + + private fun shouldRetain(path: String, scope: CoverageScope): Boolean { + val scopeWasRequested = scope in context.request.scopes + val matchesIncludePatterns = matchesCoveragePath( + path = path, + patterns = context.request.includePatterns, + sourceRoots = sourceRoots, + ) + val matchesExcludePatterns = context.request.excludePatterns.isNotEmpty() && + matchesCoveragePath( + path = path, + patterns = context.request.excludePatterns, + sourceRoots = sourceRoots, + ) + + return scopeWasRequested && matchesIncludePatterns && !matchesExcludePatterns + } + + private fun isGeneratedJavaScriptBelowSourceRoot(path: String): Boolean { + val extension = path.substringAfterLast('.', missingDelimiterValue = "").lowercase() + val isGeneratedJavaScript = extension in GENERATED_JAVASCRIPT_EXTENSIONS + val isBelowSourceRoot = sourceRoots.any { sourceRoot -> isWithin(path, sourceRoot) } + + return isGeneratedJavaScript && isBelowSourceRoot + } + + private fun buildArtifact( + files: List, + diagnostics: List, + ): PropertyCoverageArtifact { + val provenance = CoverageProvenance( + collector = context.collector, + runtimeId = "node", + runtimeVersion = context.runtimeVersion, + sourceRoots = sourceRoots, + request = context.request, + ) + + return PropertyCoverageArtifact( + kind = CoverageArtifactKind.NODE_SOURCE, + backendId = context.backendId, + backendVersion = context.backendVersion, + propertyId = context.propertyId, + provenance = provenance, + files = files, + diagnostics = diagnostics, + ) + } + + private companion object { + val GENERATED_JAVASCRIPT_EXTENSIONS = hashSetOf("js", "mjs", "cjs") + + fun normalizeCoveragePath(path: String): String = Path.of(path) + .toAbsolutePath() + .normalize() + .toString() + .replace('\\', '/') + } +} diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt index 419e5e99c..700602872 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt @@ -1,32 +1,10 @@ package org.usvm.ts.pbt.coverage -import kotlinx.serialization.json.JsonArray -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive -import kotlinx.serialization.json.contentOrNull -import kotlinx.serialization.json.intOrNull -import kotlinx.serialization.json.jsonObject -import kotlinx.serialization.json.longOrNull -import org.usvm.ts.pbt.PbtDiagnosticCode -import org.usvm.ts.pbt.backend.BranchArmCoverage -import org.usvm.ts.pbt.backend.BranchCoverage -import org.usvm.ts.pbt.backend.CoverageArtifactKind import org.usvm.ts.pbt.backend.CoverageCollectorIdentity import org.usvm.ts.pbt.backend.CoverageDiagnostic -import org.usvm.ts.pbt.backend.CoverageProvenance -import org.usvm.ts.pbt.backend.CoverageScope -import org.usvm.ts.pbt.backend.FunctionCoverage import org.usvm.ts.pbt.backend.PropertyCoverageArtifact import org.usvm.ts.pbt.backend.PropertyCoverageRequest -import org.usvm.ts.pbt.backend.SourceFileCoverage -import org.usvm.ts.pbt.backend.SourcePosition -import org.usvm.ts.pbt.backend.SourceRange -import org.usvm.ts.pbt.backend.StatementCoverage -import org.usvm.ts.pbt.manifest.PropertyManifestJson import org.usvm.ts.pbt.model.PropertyId -import java.io.IOException -import java.nio.file.Files import java.nio.file.Path /** All identities and path boundaries required to convert one isolated c8 report. */ @@ -46,347 +24,31 @@ data class IstanbulCoverageContext( class CoverageArtifactException( val diagnostic: CoverageDiagnostic, cause: Throwable? = null, -) : IllegalArgumentException(diagnostic.message, cause) - -/** Decodes and filters one source-mapped Istanbul JSON report produced by an isolated c8 run. */ -fun decodeIstanbulCoverageReport( - reportPath: Path, - context: IstanbulCoverageContext, -): PropertyCoverageArtifact = decodeReport(readCoverageReport(reportPath), context) - -private fun readCoverageReport(reportPath: Path): JsonObject { - requireCoverageReport(reportPath) - return parseCoverageReport(readCoverageReportText(reportPath), reportPath) -} - -private fun requireCoverageReport(reportPath: Path) { - if (!Files.isRegularFile(reportPath)) { - throw coverageError( - code = PbtDiagnosticCode.COVERAGE_REPORT_MISSING, - message = "c8 did not produce the expected Istanbul report: $reportPath", - path = reportPath.toString(), - ) - } -} - -private fun readCoverageReportText(reportPath: Path): String { - requireCoverageReportSize(reportPath) - - return try { - Files.readString(reportPath) - } catch (error: IOException) { - throw unreadableCoverageReport(reportPath, error) - } -} - -private fun requireCoverageReportSize(reportPath: Path) { - val reportSize = try { - Files.size(reportPath) - } catch (error: IOException) { - throw unreadableCoverageReport(reportPath, error) - } - if (reportSize > MAX_COVERAGE_REPORT_BYTES) { - throw coverageError( - code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, - message = "Istanbul coverage report exceeds $MAX_COVERAGE_REPORT_BYTES bytes", - path = reportPath.toString(), +) : IllegalArgumentException(diagnostic.message, cause) { + internal companion object { + fun create( + code: String, + message: String, + path: String, + cause: Throwable? = null, + ) = CoverageArtifactException( + diagnostic = CoverageDiagnostic( + code = code, + message = message, + path = path, + ), + cause = cause, ) } } -private fun unreadableCoverageReport( +/** Decodes and filters one source-mapped Istanbul JSON report produced by an isolated c8 run. */ +fun decodeIstanbulCoverageReport( reportPath: Path, - error: IOException, -): CoverageArtifactException = coverageError( - code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, - message = "Cannot read Istanbul coverage report: ${error.message}", - path = reportPath.toString(), - cause = error, -) - -private fun parseCoverageReport(text: String, reportPath: Path): JsonObject = try { - PropertyManifestJson.json.parseToJsonElement(text).jsonObject -} catch (error: IllegalArgumentException) { - throw coverageError( - code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, - message = "Istanbul coverage report is not valid JSON: ${error.message}", - path = reportPath.toString(), - cause = error, - ) -} - -private fun decodeReport( - report: JsonObject, context: IstanbulCoverageContext, ): PropertyCoverageArtifact { - val normalizedRoots = context.sourceRoots.map(::normalizeCoveragePath) - val normalizedEntryPoints = context.propertyEntryPointPaths.mapTo(hashSetOf(), ::normalizeCoveragePath) - val normalizedAdapterRoot = normalizeCoveragePath(context.adapterRoot) - val diagnostics = mutableListOf() - val files = report.entries.mapNotNull { (reportKey, element) -> - val fileObject = element.asObject("coverage[$reportKey]") - val path = normalizeCoveragePath( - fileObject.optionalString("path") ?: reportKey, - ) - if (isGeneratedJavaScriptBelowSourceRoot(path, normalizedRoots)) { - val sourceMapExists = Files.exists(Path.of("$path.map")) - diagnostics += CoverageDiagnostic( - code = if (sourceMapExists) { - PbtDiagnosticCode.COVERAGE_SOURCE_MAP_INVALID - } else { - PbtDiagnosticCode.COVERAGE_SOURCE_MAP_MISSING - }, - message = if (sourceMapExists) { - "Executed JavaScript has a source map that c8 could not remap to its original source" - } else { - "Executed JavaScript below a TypeScript source root has no source map" - }, - path = path, - ) - return@mapNotNull null - } - val scope = classifyCoverageScope( - path = path, - sourceRoots = normalizedRoots, - propertyEntryPoints = normalizedEntryPoints, - adapterRoot = normalizedAdapterRoot, - ) ?: return@mapNotNull null - if (!shouldRetainCoverageFile(path, scope, normalizedRoots, context.request)) { - return@mapNotNull null - } - decodeSourceFileCoverage(fileObject, path, reportKey) - }.sortedBy(SourceFileCoverage::path) - - return PropertyCoverageArtifact( - kind = CoverageArtifactKind.NODE_SOURCE, - backendId = context.backendId, - backendVersion = context.backendVersion, - propertyId = context.propertyId, - provenance = CoverageProvenance( - collector = context.collector, - runtimeId = "node", - runtimeVersion = context.runtimeVersion, - sourceRoots = normalizedRoots, - request = context.request, - ), - files = files, - diagnostics = diagnostics.sortedWith(compareBy(CoverageDiagnostic::path, CoverageDiagnostic::code)), - ) -} - -private fun shouldRetainCoverageFile( - path: String, - scope: CoverageScope, - sourceRoots: List, - request: PropertyCoverageRequest, -): Boolean { - if (scope !in request.scopes) return false - if (!matchesCoveragePath(path, request.includePatterns, sourceRoots)) return false - return request.excludePatterns.isEmpty() || !matchesCoveragePath(path, request.excludePatterns, sourceRoots) -} - -private fun classifyCoverageScope( - path: String, - sourceRoots: List, - propertyEntryPoints: Set, - adapterRoot: String, -): CoverageScope? = when { - "/node_modules/" in path -> CoverageScope.DEPENDENCIES - isWithin(path, adapterRoot) -> CoverageScope.GENERATED_BACKEND_WRAPPERS - path in propertyEntryPoints -> CoverageScope.PROPERTY_ENTRY_POINTS - sourceRoots.any { root -> isWithin(path, root) } -> CoverageScope.SOURCE_UNDER_TEST - else -> null -} - -private fun isGeneratedJavaScriptBelowSourceRoot(path: String, sourceRoots: List): Boolean { - val extension = path.substringAfterLast('.', missingDelimiterValue = "").lowercase() - return extension in GENERATED_JAVASCRIPT_EXTENSIONS && sourceRoots.any { root -> isWithin(path, root) } -} - -private fun decodeSourceFileCoverage( - file: JsonObject, - path: String, - reportKey: String, -): SourceFileCoverage = try { - SourceFileCoverage( - path = path, - statements = decodeStatements(file, reportKey), - functions = decodeFunctions(file, reportKey), - branches = decodeBranches(file, reportKey), - ) -} catch (error: CoverageArtifactException) { - throw error -} catch (error: IllegalArgumentException) { - throw coverageError( - code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, - message = "Invalid Istanbul coverage for $path: ${error.message}", - path = "coverage[$reportKey]", - cause = error, - ) -} - -private fun decodeStatements(file: JsonObject, reportKey: String): List { - val locations = file.requiredObject("statementMap", "coverage[$reportKey].statementMap") - val hits = file.requiredObject("s", "coverage[$reportKey].s") - requireMatchingKeys(locations, hits, "coverage[$reportKey].statementMap", "coverage[$reportKey].s") - return locations.keys.map(::parseCoverageId).sorted().map { statementId -> - val id = statementId.toString() - StatementCoverage( - statementId = statementId, - location = locations.getValue(id).decodeRange("coverage[$reportKey].statementMap[$id]"), - hits = hits.getValue(id).decodeHitCount("coverage[$reportKey].s[$id]"), - ) - } -} - -private fun decodeFunctions(file: JsonObject, reportKey: String): List { - val locations = file.requiredObject("fnMap", "coverage[$reportKey].fnMap") - val hits = file.requiredObject("f", "coverage[$reportKey].f") - requireMatchingKeys(locations, hits, "coverage[$reportKey].fnMap", "coverage[$reportKey].f") - return locations.keys.map(::parseCoverageId).sorted().map { functionId -> - val id = functionId.toString() - val function = locations.getValue(id).asObject("coverage[$reportKey].fnMap[$id]") - FunctionCoverage( - functionId = functionId, - name = function.optionalString("name")?.ifBlank { ANONYMOUS_FUNCTION_NAME } ?: ANONYMOUS_FUNCTION_NAME, - declaration = function.requiredElement("decl", "coverage[$reportKey].fnMap[$id].decl") - .decodeRange("coverage[$reportKey].fnMap[$id].decl"), - body = function.requiredElement("loc", "coverage[$reportKey].fnMap[$id].loc") - .decodeRange("coverage[$reportKey].fnMap[$id].loc"), - hits = hits.getValue(id).decodeHitCount("coverage[$reportKey].f[$id]"), - ) - } -} - -private fun decodeBranches(file: JsonObject, reportKey: String): List { - val locations = file.requiredObject("branchMap", "coverage[$reportKey].branchMap") - val hits = file.requiredObject("b", "coverage[$reportKey].b") - requireMatchingKeys(locations, hits, "coverage[$reportKey].branchMap", "coverage[$reportKey].b") - return locations.keys.map(::parseCoverageId).sorted().map { branchId -> - val id = branchId.toString() - val branchPath = "coverage[$reportKey].branchMap[$id]" - val branch = locations.getValue(id).asObject(branchPath) - val (armLocations, armHits) = decodeBranchArrays(branch, hits, reportKey, id, branchPath) - BranchCoverage( - branchId = branchId, - type = branch.requiredString("type", "$branchPath.type"), - location = branch.requiredElement("loc", "$branchPath.loc").decodeRange("$branchPath.loc"), - arms = armLocations.indices.map { armIndex -> - BranchArmCoverage( - location = armLocations[armIndex].decodeRange("$branchPath.locations[$armIndex]"), - hits = armHits[armIndex].decodeHitCount("coverage[$reportKey].b[$id][$armIndex]"), - ) - }, - ) - } -} - -private fun decodeBranchArrays( - branch: JsonObject, - hits: JsonObject, - reportKey: String, - id: String, - branchPath: String, -): Pair { - val armLocations = branch.requiredElement("locations", "$branchPath.locations") as? JsonArray - ?: throw invalidReport("Expected an array", "$branchPath.locations") - val armHitsPath = "coverage[$reportKey].b[$id]" - val armHits = hits.getValue(id) as? JsonArray - ?: throw invalidReport("Expected an array", armHitsPath) - requireMatchingBranchArms(armLocations, armHits, branchPath) - return armLocations to armHits -} - -private fun requireMatchingBranchArms( - armLocations: JsonArray, - armHits: JsonArray, - branchPath: String, -) { - if (armLocations.size != armHits.size) { - throw invalidReport("Branch location and hit counts have different sizes", branchPath) - } -} - -private fun JsonElement.decodeRange(path: String): SourceRange { - val range = asObject(path) - return SourceRange( - start = range.requiredElement("start", "$path.start").decodePosition("$path.start"), - end = range.requiredElement("end", "$path.end").decodePosition("$path.end"), - ) -} - -private fun JsonElement.decodePosition(path: String): SourcePosition { - val position = asObject(path) - return SourcePosition( - line = position.requiredInt("line", "$path.line"), - column = position.requiredInt("column", "$path.column"), - ) -} + val report = IstanbulCoverageReportReader.read(reportPath) + val decoder = IstanbulCoverageDecoder(context) -private fun JsonElement.decodeHitCount(path: String): Long { - val primitive = this as? JsonPrimitive ?: throw invalidReport("Expected an integer", path) - return primitive.longOrNull ?: throw invalidReport("Expected an integer", path) + return decoder.decode(report) } - -private fun JsonElement.asObject(path: String): JsonObject = this as? JsonObject - ?: throw invalidReport("Expected an object", path) - -private fun JsonObject.requiredObject( - name: String, - path: String, -): JsonObject = requiredElement(name, path).asObject(path) - -private fun JsonObject.requiredElement(name: String, path: String): JsonElement = this[name] - ?: throw invalidReport("Missing required field $name", path) - -private fun JsonObject.requiredString(name: String, path: String): String { - val primitive = this[name] as? JsonPrimitive ?: throw invalidReport("Expected a string", path) - return primitive.contentOrNull ?: throw invalidReport("Expected a string", path) -} - -private fun JsonObject.optionalString(name: String): String? = (this[name] as? JsonPrimitive)?.contentOrNull - -private fun JsonObject.requiredInt(name: String, path: String): Int { - val primitive = this[name] as? JsonPrimitive ?: throw invalidReport("Expected an integer", path) - return primitive.intOrNull ?: throw invalidReport("Expected an integer", path) -} - -private fun requireMatchingKeys( - locations: JsonObject, - hits: JsonObject, - locationsPath: String, - hitsPath: String, -) { - if (locations.keys != hits.keys) { - throw invalidReport("Coverage map keys do not match hit counter keys ($locationsPath)", hitsPath) - } -} - -private fun parseCoverageId(value: String): Int = value.toIntOrNull()?.takeIf { id -> id >= 0 } - ?: throw invalidReport("Coverage ID must be a non-negative integer", value) - -private fun normalizeCoveragePath(path: String): String = Path.of(path) - .toAbsolutePath() - .normalize() - .toString() - .replace('\\', '/') - -private fun invalidReport(message: String, path: String): CoverageArtifactException = coverageError( - code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, - message = message, - path = path, -) - -private fun coverageError( - code: String, - message: String, - path: String, - cause: Throwable? = null, -) = CoverageArtifactException( - diagnostic = CoverageDiagnostic(code = code, message = message, path = path), - cause = cause, -) - -private const val ANONYMOUS_FUNCTION_NAME = "" -private const val MAX_COVERAGE_REPORT_BYTES = 64L * 1024 * 1024 -private val GENERATED_JAVASCRIPT_EXTENSIONS = setOf("js", "mjs", "cjs") diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportReader.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportReader.kt new file mode 100644 index 000000000..e401833fa --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportReader.kt @@ -0,0 +1,76 @@ +package org.usvm.ts.pbt.coverage + +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject +import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.manifest.PropertyManifestJson +import java.io.IOException +import java.nio.file.Files +import java.nio.file.Path + +/** Reads one bounded Istanbul report and converts I/O and JSON failures into coverage diagnostics. */ +internal object IstanbulCoverageReportReader { + fun read(reportPath: Path): JsonObject { + requireRegularFile(reportPath) + requireAllowedSize(reportPath) + + val reportText = readText(reportPath) + + return parse(reportText, reportPath) + } + + private fun requireRegularFile(reportPath: Path) { + if (!Files.isRegularFile(reportPath)) { + throw CoverageArtifactException.create( + code = PbtDiagnosticCode.COVERAGE_REPORT_MISSING, + message = "c8 did not produce the expected Istanbul report: $reportPath", + path = reportPath.toString(), + ) + } + } + + private fun requireAllowedSize(reportPath: Path) { + val reportSize = try { + Files.size(reportPath) + } catch (error: IOException) { + throw unreadableReport(reportPath, error) + } + + if (reportSize > MAX_COVERAGE_REPORT_BYTES) { + throw CoverageArtifactException.create( + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, + message = "Istanbul coverage report exceeds $MAX_COVERAGE_REPORT_BYTES bytes", + path = reportPath.toString(), + ) + } + } + + private fun readText(reportPath: Path): String = try { + Files.readString(reportPath) + } catch (error: IOException) { + throw unreadableReport(reportPath, error) + } + + private fun parse(reportText: String, reportPath: Path): JsonObject = try { + PropertyManifestJson.json.parseToJsonElement(reportText).jsonObject + } catch (error: IllegalArgumentException) { + throw CoverageArtifactException.create( + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, + message = "Istanbul coverage report is not valid JSON: ${error.message}", + path = reportPath.toString(), + cause = error, + ) + } + + private fun unreadableReport( + reportPath: Path, + error: IOException, + ): CoverageArtifactException = CoverageArtifactException.create( + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, + message = "Cannot read Istanbul coverage report: ${error.message}", + path = reportPath.toString(), + cause = error, + ) + + private const val MAX_COVERAGE_REPORT_BYTES = 64L * 1024 * 1024 +} diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulJsonValue.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulJsonValue.kt new file mode 100644 index 000000000..6f84a5dac --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulJsonValue.kt @@ -0,0 +1,120 @@ +package org.usvm.ts.pbt.coverage + +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.intOrNull +import kotlinx.serialization.json.longOrNull +import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.backend.SourcePosition +import org.usvm.ts.pbt.backend.SourceRange + +/** A JSON value paired with its Istanbul path for precise validation diagnostics. */ +internal class IstanbulJsonValue( + private val value: JsonElement, + val path: String, +) { + fun asObject(): JsonObject = value as? JsonObject + ?: throw invalidReport( + message = "Expected an object", + path = path, + ) + + fun asArray(): JsonArray = value as? JsonArray + ?: throw invalidReport( + message = "Expected an array", + path = path, + ) + + fun required(name: String): IstanbulJsonValue { + val childPath = "$path.$name" + val child = asObject()[name] + ?: throw invalidReport( + message = "Missing required field $name", + path = childPath, + ) + + return IstanbulJsonValue(child, childPath) + } + + fun requiredObject(name: String): JsonObject = required(name).asObject() + + fun requiredString(name: String): String { + val childPath = "$path.$name" + val primitive = asObject()[name] as? JsonPrimitive + ?: throw invalidReport( + message = "Expected a string", + path = childPath, + ) + + return primitive.contentOrNull + ?: throw invalidReport( + message = "Expected a string", + path = childPath, + ) + } + + fun optionalString(name: String): String? { + val primitive = asObject()[name] as? JsonPrimitive + + return primitive?.contentOrNull + } + + fun asRange(): SourceRange { + val range = asObject() + val rangeJson = IstanbulJsonValue(range, path) + val start = rangeJson.required(name = "start").asPosition() + val end = rangeJson.required(name = "end").asPosition() + + return SourceRange(start = start, end = end) + } + + fun asPosition(): SourcePosition { + val position = asObject() + val positionJson = IstanbulJsonValue(position, path) + val line = positionJson.requiredInt(name = "line") + val column = positionJson.requiredInt(name = "column") + + return SourcePosition(line = line, column = column) + } + + fun asHitCount(): Long { + val primitive = value as? JsonPrimitive + ?: throw invalidReport( + message = "Expected an integer", + path = path, + ) + + return primitive.longOrNull + ?: throw invalidReport( + message = "Expected an integer", + path = path, + ) + } + + private fun requiredInt(name: String): Int { + val childPath = "$path.$name" + val primitive = asObject()[name] as? JsonPrimitive + ?: throw invalidReport( + message = "Expected an integer", + path = childPath, + ) + + return primitive.intOrNull + ?: throw invalidReport( + message = "Expected an integer", + path = childPath, + ) + } + + internal companion object { + fun invalidReport(message: String, path: String): CoverageArtifactException = + CoverageArtifactException.create( + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, + message = message, + path = path, + ) + } +} diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulSourceFileDecoder.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulSourceFileDecoder.kt new file mode 100644 index 000000000..ec03fc317 --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulSourceFileDecoder.kt @@ -0,0 +1,241 @@ +package org.usvm.ts.pbt.coverage + +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.backend.BranchArmCoverage +import org.usvm.ts.pbt.backend.BranchCoverage +import org.usvm.ts.pbt.backend.FunctionCoverage +import org.usvm.ts.pbt.backend.SourceFileCoverage +import org.usvm.ts.pbt.backend.StatementCoverage + +/** Decodes statement, function, and branch maps for one retained Istanbul file entry. */ +internal class IstanbulSourceFileDecoder( + file: JsonObject, + private val path: String, + reportKey: String, +) { + private val coveragePath = "coverage[$reportKey]" + private val fileJson = IstanbulJsonValue(file, coveragePath) + + fun decode(): SourceFileCoverage = try { + val statements = decodeStatements() + val functions = decodeFunctions() + val branches = decodeBranches() + + SourceFileCoverage( + path = path, + statements = statements, + functions = functions, + branches = branches, + ) + } catch (error: CoverageArtifactException) { + throw error + } catch (error: IllegalArgumentException) { + throw CoverageArtifactException.create( + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, + message = "Invalid Istanbul coverage for $path: ${error.message}", + path = coveragePath, + cause = error, + ) + } + + private fun decodeStatements(): List { + val coverageMap = readCoverageMap( + locationsName = "statementMap", + hitsName = "s", + ) + + return coverageMap.ids.map { statementId -> + val location = coverageMap.location(statementId).asRange() + val hitCount = coverageMap.hitCount(statementId).asHitCount() + + StatementCoverage( + statementId = statementId, + location = location, + hits = hitCount, + ) + } + } + + private fun decodeFunctions(): List { + val coverageMap = readCoverageMap( + locationsName = "fnMap", + hitsName = "f", + ) + + return coverageMap.ids.map { functionId -> + decodeFunction( + functionId = functionId, + coverageMap = coverageMap, + ) + } + } + + private fun decodeFunction( + functionId: Int, + coverageMap: CoverageMap, + ): FunctionCoverage { + val functionJson = coverageMap.location(functionId) + val functionName = functionJson.optionalString(name = "name") + ?.ifBlank { ANONYMOUS_FUNCTION_NAME } + ?: ANONYMOUS_FUNCTION_NAME + val declaration = functionJson.required(name = "decl").asRange() + val body = functionJson.required(name = "loc").asRange() + val hitCount = coverageMap.hitCount(functionId).asHitCount() + + return FunctionCoverage( + functionId = functionId, + name = functionName, + declaration = declaration, + body = body, + hits = hitCount, + ) + } + + private fun decodeBranches(): List { + val coverageMap = readCoverageMap( + locationsName = "branchMap", + hitsName = "b", + ) + + return coverageMap.ids.map { branchId -> + decodeBranch( + branchId = branchId, + coverageMap = coverageMap, + ) + } + } + + private fun decodeBranch( + branchId: Int, + coverageMap: CoverageMap, + ): BranchCoverage { + val branchJson = coverageMap.location(branchId) + val branchHitsJson = coverageMap.hitCount(branchId) + val branchArms = decodeBranchArms(branchJson, branchHitsJson) + + val type = branchJson.requiredString(name = "type") + val location = branchJson.required(name = "loc").asRange() + val arms = branchArms.locations.indices.map { armIndex -> + BranchArmCoverage( + location = branchArms.location(armIndex).asRange(), + hits = branchArms.hitCount(armIndex).asHitCount(), + ) + } + + return BranchCoverage( + branchId = branchId, + type = type, + location = location, + arms = arms, + ) + } + + private fun decodeBranchArms( + branchJson: IstanbulJsonValue, + branchHitsJson: IstanbulJsonValue, + ): BranchArms { + val locationsJson = branchJson.required(name = "locations") + val locations = locationsJson.asArray() + val hits = branchHitsJson.asArray() + + if (locations.size != hits.size) { + throw IstanbulJsonValue.invalidReport( + message = "Branch location and hit counts have different sizes", + path = branchJson.path, + ) + } + + return BranchArms( + locations = locations, + hits = hits, + locationsPath = locationsJson.path, + hitsPath = branchHitsJson.path, + ) + } + + private fun readCoverageMap( + locationsName: String, + hitsName: String, + ): CoverageMap { + val locationsJson = fileJson.required(name = locationsName) + val hitsJson = fileJson.required(name = hitsName) + val locations = locationsJson.asObject() + val hits = hitsJson.asObject() + + if (locations.keys != hits.keys) { + throw IstanbulJsonValue.invalidReport( + message = "Coverage map keys do not match hit counter keys (${locationsJson.path})", + path = hitsJson.path, + ) + } + + val ids = locations.keys.map(::parseCoverageId).sorted() + + return CoverageMap( + ids = ids, + locations = locations, + hits = hits, + locationsPath = locationsJson.path, + hitsPath = hitsJson.path, + ) + } + + private fun parseCoverageId(value: String): Int { + val coverageId = value.toIntOrNull() + + return coverageId?.takeIf { id -> id >= 0 } + ?: throw IstanbulJsonValue.invalidReport( + message = "Coverage ID must be a non-negative integer", + path = value, + ) + } + + private data class CoverageMap( + val ids: List, + val locations: JsonObject, + val hits: JsonObject, + val locationsPath: String, + val hitsPath: String, + ) { + fun location(id: Int): IstanbulJsonValue { + val key = id.toString() + + return IstanbulJsonValue( + value = locations.getValue(key), + path = "$locationsPath[$key]", + ) + } + + fun hitCount(id: Int): IstanbulJsonValue { + val key = id.toString() + + return IstanbulJsonValue( + value = hits.getValue(key), + path = "$hitsPath[$key]", + ) + } + } + + private data class BranchArms( + val locations: JsonArray, + val hits: JsonArray, + val locationsPath: String, + val hitsPath: String, + ) { + fun location(index: Int) = IstanbulJsonValue( + value = locations[index], + path = "$locationsPath[$index]", + ) + + fun hitCount(index: Int) = IstanbulJsonValue( + value = hits[index], + path = "$hitsPath[$index]", + ) + } + + private companion object { + const val ANONYMOUS_FUNCTION_NAME = "" + } +}