diff --git a/usvm-ts-pbt/DESIGN.md b/usvm-ts-pbt/DESIGN.md index ac73bcc1b..82dfe6214 100644 --- a/usvm-ts-pbt/DESIGN.md +++ b/usvm-ts-pbt/DESIGN.md @@ -7,6 +7,7 @@ API and CLI examples, see [README.md](README.md). - Kotlin owns property definitions, validation, registries, orchestration, and public results. - Node is a thin adapter around fast-check and direct TypeScript loading. +- Per-property source coverage is an optional backend capability collected by Kotlin through an isolated c8 run. - The JSON exchange is one request and one response from the same packaged distribution; it has no persistence or compatibility negotiation. - Failures are typed without exposing runtime-dependent Node stack traces. @@ -38,6 +39,7 @@ flowchart LR FastCheck[fast-check] Tsx[tsx] + C8[c8 and Istanbul JSON] UserTS[User TypeScript source] CLI --> Registry @@ -47,6 +49,8 @@ flowchart LR Backend --> Model Backend --> Process Process --> ExecutionCLI + Process --> C8 + C8 --> ExecutionCLI Projection --> ProjectionCLI ExecutionCLI --> Execute Execute --> Domains @@ -67,7 +71,7 @@ flowchart LR | Kotlin model and validation | Define one backend-neutral property and reject invalid structure before execution. | | Registry and CLI | Select Kotlin-defined properties and turn user options into a run configuration. | | `FastCheckBackend` | Validate examples, resolve source roots, and create the adapter request. | -| `FastCheckProcessClient` | Supervise Node with coroutines, bounded I/O, hard deadlines, and response validation. | +| `FastCheckProcessClient` | Supervise Node with coroutines and optionally decode one isolated c8 report. | | `execution-cli.ts` | Read one JSON request, protect protocol stdout from user logging, and write one response. | | `execute-property.ts` | Build the fast-check property, run it, and translate `RunDetails` into the common result. | | `project-domain.ts` | Translate domain descriptors into real `fc.Arbitrary` instances. | @@ -93,17 +97,22 @@ flowchart LR Verify --> Result[PropertyRunResult or PbtBackendException] ``` -The execution request contains `manifest`, `sourceRoots`, optional `seed` and `replayPath`, `numRuns`, -`timeoutMillis`, and tagged `examples`. A response is either: +The private execution request contains `manifest`, `sourceRoots`, optional `seed` and `replayPath`, `numRuns`, +`timeoutMillis`, and tagged `examples`. `coverageRequest` is Kotlin-only transport metadata and is not serialized +to Node or added to `PropertyManifest`. A Node response is either: ```text { status: "ok", result: PropertyRunResult } { status: "error", diagnostics: [{ kind, code, message, path }] } ``` -There is intentionally no request ID, operation name, schema version, protocol version, backend ID, or backend -version. The exchange is private, one-shot, and produced and consumed by the same build. Adding compatibility -metadata would create branches that no supported workflow uses. +There is intentionally no request ID, operation name, backend ID, or backend version. The exchange is private, +one-shot, and produced and consumed by the same build. Adding compatibility metadata would create branches that +no supported workflow uses. + +Coverage uses the same private protocol. When requested, Kotlin starts the execution CLI under c8, then reads a +separate Istanbul report after a valid response. Backend identity belongs to `coverageCapability` and +`PropertyCoverageArtifact`, not the ordinary property result or private wire response. Kotlin validates trusted model objects and examples early so callers get local errors. Node validates the decoded JSON again because the process boundary must not trust malformed input. Diagnostic codes have one owner per @@ -139,6 +148,9 @@ sequenceDiagram FC-->>Node: RunDetails Node-->>Client: one JSON response Client->>Client: validate exit, size, shape, category, and property ID + opt coverage requested + Client->>Client: decode isolated c8 Istanbul report + end Client-->>Backend: PropertyRunResult Backend-->>Caller: PropertyRunResult ``` @@ -167,6 +179,40 @@ Falsification and a timeout cleanly reported by fast-check are completed propert entry-point failures, process failures, malformed responses, and the JVM hard timeout are infrastructure exceptions. +Coverage collection failures use the separate `COVERAGE` infrastructure category. Stable diagnostics distinguish +an unsupported backend or Node runtime, unavailable runtime version, missing collector, missing or malformed +report, and missing or invalid source map. They are never converted into an empty artifact. + +## Coverage collection and filtering + +For each coverage-enabled property the process client creates unique `raw` and `report` directories and runs: + +```text +node /node_modules/c8/bin/c8.js + --config=/c8-config.json + --reporter=json + --reports-dir=/report + --temp-directory=/raw + --exclude-after-remap + --allowExternal + --exclude=__usvm_no_default_excludes__ + node /dist/src/execution-cli.js +``` + +Kotlin first probes the configured Node executable and rejects versions older than 18.18 before creating the c8 +workspace. The verified version is reused as artifact provenance. The c8 process inherits the caller's current +directory so user predicates observe the same environment with and without coverage, while an explicit empty +configuration prevents project-local c8 settings from altering collection. c8 then performs V8-to-Istanbul +conversion and source-map remapping. Kotlin rejects reports larger than 64 MiB before +reading them, validates statement, function, and branch maps and counters, classifies remapped files into +source-under-test, property entry points, generated wrappers, or dependencies, then applies include and exclude +globs. Excludes take precedence. Files and diagnostics are sorted deterministically before constructing the +artifact. + +A successful or falsified property exits the bridge normally, allowing c8 to flush the report. Process crashes, +invalid protocol responses, and hard kills do not produce a completed property result. The workspace is removed +in all cases, and a new workspace is used for every property. + The execution client starts stdout, stderr, and stdin work concurrently on the coroutine I/O dispatcher. Requests and stdout are limited to 4 MiB; stderr is limited to 64 KiB. These are transport safety bounds, not property-policy limits. The hard deadline is the property timeout plus two seconds for transport, followed by a 250 ms graceful @@ -175,8 +221,8 @@ shutdown before force-kill. The only run-control maximum is `2^31 - 1` milliseco ## Runtime packaging -Gradle installs pinned adapter dependencies, compiles only the private adapter, and packages `dist/src` plus its -runtime dependencies in the application distribution. User TypeScript stays as source. During repository tests, +Gradle installs pinned adapter dependencies, including c8 10.1.3, compiles only the private adapter, and packages +`dist/src` plus its runtime dependencies in the application distribution. User TypeScript stays as source. During repository tests, Gradle passes the adapter directory through a JVM system property; an installed distribution resolves it next to the application libraries. Node.js 18.18 or newer is required, and runtime archives carry an OS/architecture classifier because `tsx` depends on a native esbuild package. @@ -190,6 +236,8 @@ classifier because `tsx` depends on a native esbuild package. fast-check: startup failure, non-zero exit, malformed output, explicit diagnostic categories, and hard timeout. - Backend integration tests execute real uncompiled TypeScript through the packaged adapter, including replay, shrinking, explicit examples, preconditions, async predicates, and timeouts. +- Coverage golden tests assert literal TypeScript statement and branch outcomes for successful and falsified runs, + cross-property isolation, scope and glob filtering, and source-map/report diagnostics. ## Non-goals @@ -197,4 +245,5 @@ classifier because `tsx` depends on a native esbuild package. - Discovering properties by scanning TypeScript source roots. - Compiling user TypeScript as part of the PBT workflow. - Reimplementing generation, replay, skip accounting, or shrinking in Kotlin. -- Recording coverage or other per-run artifacts in this change. +- Mapping Node source locations to EtsIR or constructing symbolic targets from coverage. +- Combining Node source coverage with future EtsIR replay coverage. diff --git a/usvm-ts-pbt/README.md b/usvm-ts-pbt/README.md index ffb531deb..14a942a9c 100644 --- a/usvm-ts-pbt/README.md +++ b/usvm-ts-pbt/README.md @@ -73,7 +73,7 @@ val result = backend.run( The defaults are 100 successful runs and a 60-second timeout. Configuration also supports replay paths and positional explicit examples. `PropertyRunResult` contains the property ID, status, actual seed, replay path, -counterexample, run/skip/shrink counts, failure details, and elapsed time. +counterexample, run/skip/shrink counts, failure details, elapsed time, and optional per-property coverage. Predicate falsification and a timeout reported by fast-check are normal `FAILURE` results. Invalid input, entry-point, process, and transport failures throw `PbtBackendException`. @@ -82,6 +82,61 @@ Synchronous entry points must return a boolean directly. Asynchronous entry poin resolves to a boolean. A false precondition is passed to fast-check as a skipped input. Generation, replay, explicit examples, checking, and shrinking retain fast-check semantics. +## Per-property TypeScript coverage + +Coverage is a backend capability, not part of `PropertyDefinition` or `PropertyManifest`. +`PropertyBasedTestingBackend.coverageCapability` exposes the backend identity, backend version, and collector +before execution. `FastCheckBackend` reports c8 10.1.3. An unsupported backend reports +`coverage.unsupported` explicitly. + +Kotlin requests coverage for one run through `PropertyRunConfiguration`: + +```kotlin +val result = backend.run( + property = property, + configuration = PropertyRunConfiguration( + seed = 42, + coverageRequest = PropertyCoverageRequest( + scopes = setOf(CoverageScope.SOURCE_UNDER_TEST), + includePatterns = listOf("packages/core/**/*.ts"), + excludePatterns = listOf("**/*.generated.ts"), + ), + ), +) +``` + +The default scope is `SOURCE_UNDER_TEST`. Available scopes are: + +| Scope | Files retained after source-map remapping | +| --- | --- | +| `SOURCE_UNDER_TEST` | Files below a source root except exact predicate and precondition modules | +| `PROPERTY_ENTRY_POINTS` | Exact predicate and optional precondition modules | +| `GENERATED_BACKEND_WRAPPERS` | Files in the private adapter runtime outside `node_modules` | +| `DEPENDENCIES` | Executed files below `node_modules` | + +Include and exclude globs operate on original remapped paths, use `/` separators, and support `*`, `?`, and `**`. +An empty include list retains every file in a selected scope; exclude rules always win. + +For every requested property Kotlin creates a unique c8 workspace, runs the private adapter as +`node c8.js ... node execution-cli.js`, decodes `coverage-final.json`, and removes the workspace. This isolation +prevents coverage from one property contaminating another. The adapter inherits the caller's current directory, +while an explicit empty c8 configuration prevents project-local c8 settings from altering collection. A falsified +property remains a completed Node run, so its artifact preserves coverage collected before falsification. + +Before starting c8, Kotlin probes the configured Node executable once. Versions older than 18.18 are rejected with +`coverage.runtime.unsupported`; an unavailable or unparseable version uses `coverage.runtime.version-unavailable`. +The verified version is recorded in the artifact provenance without a second probe. + +The artifact has kind `NODE_SOURCE` and contains backend/property identity, c8 and Node provenance, canonical +source roots, the original request, and deterministic per-file statement, function, and branch hits. Lines are +one-based and columns are zero-based, following Istanbul. Node source coverage remains separate from future EtsIR +replay coverage. + +Missing or malformed reports use `coverage.report.missing` and `coverage.report.invalid`. Reports larger than +64 MiB are rejected as invalid before they are read. JavaScript below a TypeScript source root that c8 could not +remap produces `coverage.source-map.missing` or `coverage.source-map.invalid`; a missing packaged c8 runtime +produces `coverage.collector.not-found`. + ## Registries and CLI The CLI loads Kotlin property registries through `ServiceLoader`: @@ -107,13 +162,19 @@ java -cp '/opt/usvm-ts-pbt/lib/*:/workspace/example-properties.jar' \ --registry example \ --property array.reverse-twice \ --seed 42 \ - --num-runs 1000 + --num-runs 1000 \ + --coverage \ + --coverage-scope source-under-test \ + --coverage-exclude '**/*.generated.ts' ``` Use `--help` for the complete option list. `--source-root` and `--registry` are repeatable. Without `--registry`, all providers run in registry-ID order; without `--property`, all selected properties run in registry order. Replay paths and explicit examples require exactly one selected property. +`--coverage` enables collection. `--coverage-scope`, `--coverage-include`, and `--coverage-exclude` are repeatable; +scope and path options require `--coverage`. Without an explicit scope, source-under-test coverage is collected. + The CLI writes a JSON array of results to stdout. Exit code `0` means every property succeeded, `1` means at least one property failed, and `2` means a CLI, registry, validation, backend, or transport error. Exit-code-2 diagnostics are written as one JSON object to stderr. @@ -121,6 +182,7 @@ are written as one JSON object to stderr. ## Verification Requires JDK 11, Node.js 18.18 or newer, npm, and the repository Gradle wrapper. +The private distribution pins c8 10.1.3 because it supports the module's Node 18 floor. ```shell npm ci --prefix usvm-ts-pbt/fast-check-adapter --ignore-scripts diff --git a/usvm-ts-pbt/build.gradle.kts b/usvm-ts-pbt/build.gradle.kts index aa7bcd608..afc82d9f4 100644 --- a/usvm-ts-pbt/build.gradle.kts +++ b/usvm-ts-pbt/build.gradle.kts @@ -1,3 +1,5 @@ +import groovy.json.JsonSlurper + plugins { id("usvm.kotlin-conventions") kotlin("plugin.serialization") version Versions.kotlin @@ -14,7 +16,12 @@ dependencies { } val fastCheckAdapterDir = layout.projectDirectory.dir("fast-check-adapter") +val fastCheckAdapterPackageJson = fastCheckAdapterDir.file("package.json") +val fastCheckAdapterPackageLock = fastCheckAdapterDir.file("package-lock.json") val fastCheckRuntimeProperty = "org.usvm.ts.pbt.fastcheck.runtime" +val generatedFastCheckRuntimeMetadataDirectory = layout.buildDirectory.dir( + "generated/resources/fastCheckRuntimeMetadata", +) val hostOperatingSystem = System.getProperty("os.name").lowercase() val hostPlatform = when { hostOperatingSystem.contains("mac") -> "darwin" @@ -31,12 +38,52 @@ val hostArchitecture = when (val architecture = System.getProperty("os.arch").lo val fastCheckRuntimeClassifier = "$hostPlatform-$hostArchitecture" val npmExecutable = if (hostPlatform == "win32") "npm.cmd" else "npm" +val generateFastCheckRuntimeMetadata = tasks.register("generateFastCheckRuntimeMetadata") { + inputs.file(fastCheckAdapterPackageLock) + outputs.dir(generatedFastCheckRuntimeMetadataDirectory) + + doLast { + val packageLock = JsonSlurper().parse(fastCheckAdapterPackageLock.asFile) as? Map<*, *> + ?: error("Invalid fast-check adapter package lock") + val packages = packageLock["packages"] as? Map<*, *> + ?: error("Missing packages in fast-check adapter package lock") + fun dependencyVersion(dependency: String): String { + val metadata = packages["node_modules/$dependency"] as? Map<*, *> + ?: error("Missing locked fast-check adapter dependency: $dependency") + + return (metadata["version"] as? String) + ?.takeIf(String::isNotBlank) + ?: error("Missing locked fast-check adapter dependency version: $dependency") + } + + val metadataFile = generatedFastCheckRuntimeMetadataDirectory.get() + .file("org/usvm/ts/pbt/fastcheck/runtime-dependencies.properties") + .asFile + metadataFile.parentFile.mkdirs() + metadataFile.writeText( + """ + fast-check.version=${dependencyVersion("fast-check")} + c8.version=${dependencyVersion("c8")} + """.trimIndent() + "\n", + Charsets.UTF_8, + ) + } +} + +sourceSets.main { + resources.srcDir(generatedFastCheckRuntimeMetadataDirectory) +} + +tasks.processResources { + dependsOn(generateFastCheckRuntimeMetadata) +} + val installFastCheckAdapter = tasks.register("installFastCheckAdapter") { workingDir(fastCheckAdapterDir) commandLine(npmExecutable, "ci", "--ignore-scripts") inputs.files( - fastCheckAdapterDir.file("package.json"), - fastCheckAdapterDir.file("package-lock.json"), + fastCheckAdapterPackageJson, + fastCheckAdapterPackageLock, ) inputs.property("runtimeClassifier", fastCheckRuntimeClassifier) outputs.dir(fastCheckAdapterDir.dir("node_modules")) diff --git a/usvm-ts-pbt/fast-check-adapter/package-lock.json b/usvm-ts-pbt/fast-check-adapter/package-lock.json index 40e9135b5..594dfb598 100644 --- a/usvm-ts-pbt/fast-check-adapter/package-lock.json +++ b/usvm-ts-pbt/fast-check-adapter/package-lock.json @@ -8,6 +8,7 @@ "name": "@usvm/fast-check-adapter", "version": "0.1.0", "dependencies": { + "c8": "10.1.3", "fast-check": "4.9.0", "tsx": "4.23.12" }, @@ -19,6 +20,15 @@ "node": ">=18.18.0" } }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", @@ -435,6 +445,73 @@ "node": ">=18" } }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.6.tgz", + "integrity": "sha512-+Sg6GCR/wy1oSmQDFq4LQDAhm3ETKnorxN+y5nbLULOR3P0c14f2Wurzj3/xqPXtasLFfHd5iRFQ7AJt4KH2cw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "license": "MIT" + }, "node_modules/@types/node": { "version": "18.19.130", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", @@ -445,6 +522,215 @@ "undici-types": "~5.26.4" } }, + "node_modules/ansi-regex": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.3.0.tgz", + "integrity": "sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/c8": { + "version": "10.1.3", + "resolved": "https://registry.npmjs.org/c8/-/c8-10.1.3.tgz", + "integrity": "sha512-LvcyrOAaOnrrlMpW22n690PUvxiq4Uf9WMhQwNJ9vgagkL/ph1+D4uvjvDA5XCbykrc0sx+ay6pVi9YZ1GnhyA==", + "license": "ISC", + "dependencies": { + "@bcoe/v8-coverage": "^1.0.1", + "@istanbuljs/schema": "^0.1.3", + "find-up": "^5.0.0", + "foreground-child": "^3.1.1", + "istanbul-lib-coverage": "^3.2.0", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.1.6", + "test-exclude": "^7.0.1", + "v8-to-istanbul": "^9.0.0", + "yargs": "^17.7.2", + "yargs-parser": "^21.1.1" + }, + "bin": { + "c8": "bin/c8.js" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "monocart-coverage-reports": "^2" + }, + "peerDependenciesMeta": { + "monocart-coverage-reports": { + "optional": true + } + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT" + }, "node_modules/esbuild": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", @@ -486,6 +772,15 @@ "@esbuild/win32-x64": "0.28.2" } }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/fast-check": { "version": "4.9.0", "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.9.0.tgz", @@ -508,6 +803,38 @@ "node": ">=12.17.0" } }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -522,6 +849,247 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "license": "MIT" + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC" + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/pure-rand": { "version": "8.4.2", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.2.tgz", @@ -538,6 +1106,182 @@ ], "license": "MIT" }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/test-exclude": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-7.0.1.tgz", + "integrity": "sha512-pFYqmTw68LXVjeWJMST4+borgQP2AyMNbg1BpZh9LbyhUeNkeaPF9gzfPGUAnSMV3qPYdWUwDIjjCLiSDOl7vg==", + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^10.4.1", + "minimatch": "^9.0.4" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/tsx": { "version": "4.23.12", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.12.tgz", @@ -576,6 +1320,215 @@ "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "dev": true, "license": "MIT" + }, + "node_modules/v8-to-istanbul": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", + "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", + "license": "ISC", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^2.0.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } } } } diff --git a/usvm-ts-pbt/fast-check-adapter/package.json b/usvm-ts-pbt/fast-check-adapter/package.json index c0c905634..de025f782 100644 --- a/usvm-ts-pbt/fast-check-adapter/package.json +++ b/usvm-ts-pbt/fast-check-adapter/package.json @@ -12,6 +12,7 @@ "test:compiled": "node --test dist/test/entry-point.test.js dist/test/execute-property.test.js dist/test/execution-cli.test.js dist/test/js-value.test.js dist/test/project-domain.test.js dist/test/projection-cli.test.js" }, "dependencies": { + "c8": "10.1.3", "fast-check": "4.9.0", "tsx": "4.23.12" }, @@ -19,6 +20,11 @@ "@types/node": "18.19.130", "typescript": "5.9.2" }, + "overrides": { + "c8": { + "test-exclude": "7.0.1" + } + }, "engines": { "node": ">=18.18.0" } diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/PbtDiagnosticCode.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/PbtDiagnosticCode.kt index fac25f84f..7ac238b1e 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/PbtDiagnosticCode.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/PbtDiagnosticCode.kt @@ -3,6 +3,8 @@ package org.usvm.ts.pbt /** Stable identifiers for diagnostics created on the Kotlin side of the PBT boundary. */ internal object PbtDiagnosticCode { const val CLI_ARGUMENT_INVALID = "cli.argument.invalid" + const val CLI_COVERAGE_REQUIRED = "cli.coverage.required" + const val CLI_COVERAGE_SCOPE_INVALID = "cli.coverage.scope.invalid" const val CLI_EXAMPLES_INVALID = "cli.examples.invalid" const val CLI_NUM_RUNS_INVALID = "cli.num-runs.invalid" const val CLI_PROPERTY_EMPTY = "cli.property.empty" @@ -35,6 +37,14 @@ internal object PbtDiagnosticCode { const val BACKEND_RESPONSE_TOO_LARGE = "backend.response.too-large" const val BACKEND_RUNTIME_NOT_FOUND = "backend.runtime.not-found" + const val COVERAGE_COLLECTOR_NOT_FOUND = "coverage.collector.not-found" + const val COVERAGE_REPORT_INVALID = "coverage.report.invalid" + const val COVERAGE_REPORT_MISSING = "coverage.report.missing" + const val COVERAGE_RUNTIME_UNSUPPORTED = "coverage.runtime.unsupported" + const val COVERAGE_RUNTIME_VERSION_UNAVAILABLE = "coverage.runtime.version-unavailable" + const val COVERAGE_SOURCE_MAP_INVALID = "coverage.source-map.invalid" + const val COVERAGE_SOURCE_MAP_MISSING = "coverage.source-map.missing" + const val PROTOCOL_REQUEST_INVALID = "protocol.request.invalid" const val SOURCE_ROOT_INVALID = "source-root.invalid" diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyBasedTestingBackend.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyBasedTestingBackend.kt index 818e7aef6..c3df828e1 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyBasedTestingBackend.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyBasedTestingBackend.kt @@ -8,6 +8,9 @@ import org.usvm.ts.pbt.model.PropertyId /** Executes validated Kotlin property definitions through one concrete PBT engine. */ interface PropertyBasedTestingBackend { + /** Reports whether this backend version can collect per-property source coverage. */ + val coverageCapability: PropertyCoverageCapability + /** Executes [property] with [configuration] and returns a structured property result. */ fun run( property: PropertyDefinition, @@ -22,6 +25,7 @@ data class PropertyRunConfiguration( val numRuns: Int = DEFAULT_NUM_RUNS, val timeoutMillis: Long = DEFAULT_TIMEOUT_MILLIS, val examples: List> = emptyList(), + val coverageRequest: PropertyCoverageRequest? = null, ) { init { require(numRuns > 0) { "Number of runs must be positive" } @@ -86,12 +90,16 @@ data class PropertyRunResult( val numShrinks: Int, val failure: PropertyFailureDetails?, val executionTimeMillis: Long, + val coverage: PropertyCoverageArtifact? = null, ) { init { require(numRuns >= 0) { "Run count must not be negative" } require(numSkips >= 0) { "Skip count must not be negative" } require(numShrinks >= 0) { "Shrink count must not be negative" } require(executionTimeMillis >= 0) { "Execution time must not be negative" } + require(coverage == null || coverage.propertyId == propertyId) { + "Coverage property ID must match the run result" + } when (status) { PropertyRunStatus.SUCCESS -> { diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt new file mode 100644 index 000000000..bbc0ff5db --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt @@ -0,0 +1,257 @@ +package org.usvm.ts.pbt.backend + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import org.usvm.ts.pbt.model.PropertyId + +/** Distinguishes Node source coverage from future replay coverage over another representation. */ +@Serializable +enum class CoverageArtifactKind { + @SerialName("node_source") + NODE_SOURCE, +} + +/** Executed-file categories that may be retained in one coverage artifact. */ +@Serializable +enum class CoverageScope { + @SerialName("source_under_test") + SOURCE_UNDER_TEST, + + @SerialName("property_entry_points") + PROPERTY_ENTRY_POINTS, + + @SerialName("generated_backend_wrappers") + GENERATED_BACKEND_WRAPPERS, + + @SerialName("dependencies") + DEPENDENCIES, +} + +/** Whether a concrete backend can produce the common property coverage artifact. */ +@Serializable +enum class CoverageCapabilityLevel { + @SerialName("supported") + SUPPORTED, + + @SerialName("unsupported") + UNSUPPORTED, +} + +/** Stable diagnostic emitted while negotiating or collecting coverage. */ +@Serializable +data class CoverageDiagnostic( + val code: String, + val message: String, + val path: String? = null, +) { + init { + require(code.isNotBlank()) { "Coverage diagnostic code must not be blank" } + require(message.isNotBlank()) { "Coverage diagnostic message must not be blank" } + } +} + +/** Identifies the concrete source coverage collector and its behavior-defining version. */ +@Serializable +data class CoverageCollectorIdentity( + val id: String, + val version: String, +) { + init { + require(id.isNotBlank()) { "Coverage collector ID must not be blank" } + require(version.isNotBlank()) { "Coverage collector version must not be blank" } + } +} + +/** Reports whether a backend version implements property source coverage. */ +@Serializable +data class PropertyCoverageCapability( + val backendId: String, + val backendVersion: String, + val level: CoverageCapabilityLevel, + val collector: CoverageCollectorIdentity? = null, + val diagnostics: List = emptyList(), +) { + init { + require(backendId.isNotBlank()) { "Backend ID must not be blank" } + require(backendVersion.isNotBlank()) { "Backend version must not be blank" } + when (level) { + CoverageCapabilityLevel.SUPPORTED -> { + requireNotNull(collector) { "Supported coverage requires a collector identity" } + require(diagnostics.isEmpty()) { "Supported coverage must not contain capability diagnostics" } + } + + CoverageCapabilityLevel.UNSUPPORTED -> { + require(collector == null) { "Unsupported coverage must not name a collector" } + require(diagnostics.isNotEmpty()) { "Unsupported coverage requires an actionable diagnostic" } + } + } + } + + companion object { + fun supported( + backendId: String, + backendVersion: String, + collector: CoverageCollectorIdentity, + ) = PropertyCoverageCapability( + backendId = backendId, + backendVersion = backendVersion, + level = CoverageCapabilityLevel.SUPPORTED, + collector = collector, + ) + + fun unsupported( + backendId: String, + backendVersion: String, + diagnostic: CoverageDiagnostic, + ) = PropertyCoverageCapability( + backendId = backendId, + backendVersion = backendVersion, + level = CoverageCapabilityLevel.UNSUPPORTED, + diagnostics = listOf(diagnostic), + ) + } +} + +/** Backend-neutral controls for source coverage collected during one property run. */ +@Serializable +data class PropertyCoverageRequest( + val scopes: Set = setOf(CoverageScope.SOURCE_UNDER_TEST), + val includePatterns: List = emptyList(), + val excludePatterns: List = emptyList(), +) { + init { + require(scopes.isNotEmpty()) { "At least one coverage scope is required" } + require(includePatterns.all(String::isNotBlank)) { "Coverage include patterns must not be blank" } + require(excludePatterns.all(String::isNotBlank)) { "Coverage exclude patterns must not be blank" } + } +} + +/** One-based line and zero-based column in an original source file. */ +@Serializable +data class SourcePosition( + val line: Int, + val column: Int, +) { + init { + require(line > 0) { "Source line must be positive" } + require(column >= 0) { "Source column must not be negative" } + } +} + +/** Half-open source range as represented by Istanbul. */ +@Serializable +data class SourceRange( + val start: SourcePosition, + val end: SourcePosition, +) { + init { + require(end.line > start.line || end.line == start.line && end.column >= start.column) { + "Source range end must not precede its start" + } + } +} + +/** Hit count for one Istanbul statement. */ +@Serializable +data class StatementCoverage( + val statementId: Int, + val location: SourceRange, + val hits: Long, +) { + init { + require(statementId >= 0) { "Statement ID must not be negative" } + require(hits >= 0) { "Statement hits must not be negative" } + } +} + +/** Hit count for one Istanbul function. */ +@Serializable +data class FunctionCoverage( + val functionId: Int, + val name: String, + val declaration: SourceRange, + val body: SourceRange, + val hits: Long, +) { + init { + require(functionId >= 0) { "Function ID must not be negative" } + require(name.isNotBlank()) { "Function name must not be blank" } + require(hits >= 0) { "Function hits must not be negative" } + } +} + +/** Hit count for one arm of an Istanbul branch. */ +@Serializable +data class BranchArmCoverage( + val location: SourceRange, + val hits: Long, +) { + init { + require(hits >= 0) { "Branch arm hits must not be negative" } + } +} + +/** Hit counts for all arms belonging to one Istanbul branch. */ +@Serializable +data class BranchCoverage( + val branchId: Int, + val type: String, + val location: SourceRange, + val arms: List, +) { + init { + require(branchId >= 0) { "Branch ID must not be negative" } + require(type.isNotBlank()) { "Branch type must not be blank" } + require(arms.isNotEmpty()) { "Branch coverage requires at least one arm" } + } +} + +/** Source-mapped coverage for one original source file. */ +@Serializable +data class SourceFileCoverage( + val path: String, + val statements: List, + val functions: List, + val branches: List, +) { + init { + require(path.isNotBlank()) { "Coverage source path must not be blank" } + } +} + +/** Records the collector inputs needed to interpret or reproduce one artifact. */ +@Serializable +data class CoverageProvenance( + val collector: CoverageCollectorIdentity, + val runtimeId: String, + val runtimeVersion: String, + val sourceRoots: List, + val request: PropertyCoverageRequest, +) { + init { + require(runtimeId.isNotBlank()) { "Coverage runtime ID must not be blank" } + require(runtimeVersion.isNotBlank()) { "Coverage runtime version must not be blank" } + require(sourceRoots.isNotEmpty()) { "Coverage provenance requires source roots" } + require(sourceRoots.all(String::isNotBlank)) { "Coverage source roots must not be blank" } + } +} + +/** Per-property Node source coverage returned by a concrete backend. */ +@Serializable +data class PropertyCoverageArtifact( + val kind: CoverageArtifactKind = CoverageArtifactKind.NODE_SOURCE, + val backendId: String, + val backendVersion: String, + val propertyId: PropertyId, + val provenance: CoverageProvenance, + val files: List, + val diagnostics: List = emptyList(), +) { + init { + require(backendId.isNotBlank()) { "Coverage backend ID must not be blank" } + require(backendVersion.isNotBlank()) { "Coverage backend version must not be blank" } + require(files.map(SourceFileCoverage::path).distinct().size == files.size) { + "Coverage artifact contains duplicate source paths" + } + } +} diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckCli.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckCli.kt index d08b44a84..198ab3d74 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckCli.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckCli.kt @@ -5,6 +5,7 @@ import kotlinx.serialization.SerializationException import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.backend.CoverageCapabilityLevel import org.usvm.ts.pbt.backend.PropertyBasedTestingBackend import org.usvm.ts.pbt.backend.PropertyRunConfiguration import org.usvm.ts.pbt.backend.PropertyRunStatus @@ -119,9 +120,11 @@ class FastCheckCli( numRuns = options.numRuns, timeoutMillis = options.timeoutMillis, examples = examples, + coverageRequest = options.coverageRequest, ) val backend = backendFactory(options.sourceRoots) + requireCoverageSupport(options, backend) val results = properties.map { property -> backend.run(property, configuration) } output.appendLine(PropertyManifestJson.json.encodeToString(results)) @@ -134,6 +137,24 @@ class FastCheckCli( } } + private fun requireCoverageSupport( + options: CliOptions, + backend: PropertyBasedTestingBackend, + ) { + if (options.coverageRequest == null || + backend.coverageCapability.level != CoverageCapabilityLevel.UNSUPPORTED + ) { + return + } + + val diagnostic = backend.coverageCapability.diagnostics.first() + throw CliUsageException( + code = diagnostic.code, + message = diagnostic.message, + path = diagnostic.path, + ) + } + private fun selectProperties( registry: PropertyRegistry, propertyId: PropertyId?, @@ -183,8 +204,8 @@ class FastCheckCli( if (registryIds.isEmpty()) return orderedProviders - val requestedIds = registryIds.toSet() - val availableIds = orderedProviders.map(ProviderRegistration::id).toSet() + val requestedIds = registryIds.toHashSet() + val availableIds = orderedProviders.mapTo(hashSetOf(), ProviderRegistration::id) val unknown = requestedIds.minus(availableIds).minOrNull() if (unknown != null) { diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt index b6075c901..17b2e1e32 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/cli/FastCheckOptions.kt @@ -5,12 +5,15 @@ import com.github.ajalt.clikt.core.CliktError import com.github.ajalt.clikt.core.PrintHelpMessage import com.github.ajalt.clikt.core.parse import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.flag import com.github.ajalt.clikt.parameters.options.multiple import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.types.int import com.github.ajalt.clikt.parameters.types.long import com.github.ajalt.clikt.parameters.types.path import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.backend.CoverageScope +import org.usvm.ts.pbt.backend.PropertyCoverageRequest import org.usvm.ts.pbt.backend.PropertyRunConfiguration import org.usvm.ts.pbt.model.PropertyId import java.nio.file.Path @@ -30,6 +33,7 @@ internal data class CliOptions( val numRuns: Int, val timeoutMillis: Long, val examplesFile: Path?, + val coverageRequest: PropertyCoverageRequest?, ) internal class CliUsageException( @@ -98,12 +102,33 @@ private class FastCheckOptionsParser : CliktCommand(name = "usvm-ts-pbt") { help = "JSON file with positional tagged examples", ).path() + private val coverageEnabled by option( + "--coverage", + help = "Collect source-mapped coverage for every property run", + ).flag() + + private val coverageScopes by option( + "--coverage-scope", + help = "Coverage scope; repeat for multiple scopes", + ).multiple() + + private val coverageIncludePatterns by option( + "--coverage-include", + help = "Include glob over remapped source paths; repeat for multiple patterns", + ).multiple() + + private val coverageExcludePatterns by option( + "--coverage-exclude", + help = "Exclude glob over remapped source paths; repeat for multiple patterns", + ).multiple() + lateinit var options: CliOptions private set override fun run() { requireSourceRoots() requirePositiveRunControls() + requireCoverageFlag() options = CliOptions( sourceRoots = sourceRoots, @@ -114,6 +139,7 @@ private class FastCheckOptionsParser : CliktCommand(name = "usvm-ts-pbt") { numRuns = numRuns, timeoutMillis = timeoutMillis, examplesFile = examplesFile, + coverageRequest = buildCoverageRequest(), ) } @@ -144,6 +170,42 @@ private class FastCheckOptionsParser : CliktCommand(name = "usvm-ts-pbt") { ) } } + + private fun requireCoverageFlag() { + val hasCoverageDetails = coverageScopes.isNotEmpty() || + coverageIncludePatterns.isNotEmpty() || + coverageExcludePatterns.isNotEmpty() + if (!coverageEnabled && hasCoverageDetails) { + throw CliUsageException( + code = PbtDiagnosticCode.CLI_COVERAGE_REQUIRED, + message = "Coverage scope and path rules require --coverage", + path = "coverage", + ) + } + } + + private fun buildCoverageRequest(): PropertyCoverageRequest? { + if (!coverageEnabled) return null + + return PropertyCoverageRequest( + scopes = coverageScopes.mapTo(hashSetOf(), ::parseCoverageScope) + .ifEmpty { setOf(CoverageScope.SOURCE_UNDER_TEST) }, + includePatterns = coverageIncludePatterns, + excludePatterns = coverageExcludePatterns, + ) + } +} + +private fun parseCoverageScope(value: String): CoverageScope = when (value) { + "source-under-test" -> CoverageScope.SOURCE_UNDER_TEST + "property-entry-points" -> CoverageScope.PROPERTY_ENTRY_POINTS + "generated-backend-wrappers" -> CoverageScope.GENERATED_BACKEND_WRAPPERS + "dependencies" -> CoverageScope.DEPENDENCIES + else -> throw CliUsageException( + code = PbtDiagnosticCode.CLI_COVERAGE_SCOPE_INVALID, + message = "Unknown coverage scope $value", + path = "coverageScope", + ) } private fun parsePropertyId(value: String): PropertyId = try { diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/CoveragePathFilter.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/CoveragePathFilter.kt new file mode 100644 index 000000000..7643b439f --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/CoveragePathFilter.kt @@ -0,0 +1,81 @@ +package org.usvm.ts.pbt.coverage + +internal fun matchesCoveragePath( + path: String, + patterns: List, + sourceRoots: List, +): Boolean { + if (patterns.isEmpty()) return true + val candidates = buildList { + add(path) + sourceRoots.forEach { sourceRoot -> + if (isWithin(path, sourceRoot) && path != sourceRoot) { + add(path.removePrefix("$sourceRoot/")) + } + } + } + return patterns.any { pattern -> + val regex = coverageGlobToRegex(pattern) + candidates.any(regex::matches) + } +} + +/** + * Converts a coverage glob into a regex over normalized forward-slash paths. + * + * A single `*` or `?` stays within one path segment, while `**` may cross directory separators. + * The resulting regex matches the complete candidate path rather than an arbitrary substring. + */ +private fun coverageGlobToRegex(pattern: String): Regex { + // Treat Windows and Unix patterns uniformly and accept the common relative-path prefix. + val normalized = pattern.replace('\\', '/').removePrefix("./") + val expression = StringBuilder("^") + var index = 0 + while (index < normalized.length) { + val character = normalized[index] + when { + character == '*' && normalized.getOrNull(index + 1) == '*' -> { + if (normalized.getOrNull(index + DOUBLE_WILDCARD_LENGTH) == '/') { + // `**/` covers zero or more whole segments: `**/*.ts` also matches `file.ts`. + expression.append("(?:.*/)?") + index += DOUBLE_WILDCARD_WITH_SEPARATOR_LENGTH + } else { + // A bare `**` consumes any characters, including directory separators. + expression.append(".*") + index += 2 + } + } + + character == '*' -> { + // A single wildcard consumes characters only inside the current path segment. + expression.append("[^/]*") + index++ + } + + character == '?' -> { + // `?` consumes exactly one non-separator character in the current segment. + expression.append("[^/]") + index++ + } + + character in REGEX_SPECIAL_CHARACTERS -> { + // Non-glob regex metacharacters are literals in coverage patterns. + expression.append('\\').append(character) + index++ + } + + else -> { + expression.append(character) + index++ + } + } + } + expression.append('$') + return Regex(expression.toString()) +} + +internal fun isWithin(path: String, root: String): Boolean = path == root || path.startsWith("$root/") + +private const val REGEX_SPECIAL_CHARACTERS = ".+()^$|{}[]" +private const val DOUBLE_WILDCARD_LENGTH = 2 +private const val DOUBLE_WILDCARD_WITH_SEPARATOR_LENGTH = 3 diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageDecoder.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageDecoder.kt new file mode 100644 index 000000000..c2a64f707 --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageDecoder.kt @@ -0,0 +1,165 @@ +package org.usvm.ts.pbt.coverage + +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.backend.CoverageArtifactKind +import org.usvm.ts.pbt.backend.CoverageDiagnostic +import org.usvm.ts.pbt.backend.CoverageProvenance +import org.usvm.ts.pbt.backend.CoverageScope +import org.usvm.ts.pbt.backend.PropertyCoverageArtifact +import org.usvm.ts.pbt.backend.SourceFileCoverage +import java.nio.file.Files +import java.nio.file.Path + +/** Selects relevant report entries and assembles the coverage artifact for one property run. */ +internal class IstanbulCoverageDecoder( + private val context: IstanbulCoverageContext, +) { + private val sourceRoots = context.sourceRoots.map(::normalizeCoveragePath) + private val propertyEntryPoints = context.propertyEntryPointPaths + .mapTo(hashSetOf(), ::normalizeCoveragePath) + private val adapterRoot = normalizeCoveragePath(context.adapterRoot) + + fun decode(report: JsonObject): PropertyCoverageArtifact { + val diagnostics = mutableListOf() + val files = decodeFiles(report, diagnostics) + val sortedDiagnostics = diagnostics.sortedWith( + compareBy(CoverageDiagnostic::path, CoverageDiagnostic::code), + ) + + return buildArtifact(files, sortedDiagnostics) + } + + private fun decodeFiles( + report: JsonObject, + diagnostics: MutableList, + ): List { + val decodedFiles = buildList { + for ((reportKey, reportEntry) in report) { + val decodedFile = decodeFile(reportKey, reportEntry, diagnostics) + if (decodedFile != null) { + add(decodedFile) + } + } + } + + return decodedFiles.sortedBy(SourceFileCoverage::path) + } + + private fun decodeFile( + reportKey: String, + reportEntry: JsonElement, + diagnostics: MutableList, + ): SourceFileCoverage? { + val coveragePath = "coverage[$reportKey]" + val fileJson = IstanbulJsonValue(reportEntry, coveragePath) + val fileObject = fileJson.asObject() + val reportedPath = fileJson.optionalString(name = "path") + val path = normalizeCoveragePath(reportedPath ?: reportKey) + + if (isGeneratedJavaScriptBelowSourceRoot(path)) { + diagnostics += sourceMapDiagnostic(path) + return null + } + + val scope = classifyScope(path) ?: return null + if (!shouldRetain(path, scope)) { + return null + } + + return IstanbulSourceFileDecoder( + file = fileObject, + path = path, + reportKey = reportKey, + ).decode() + } + + private fun sourceMapDiagnostic(path: String): CoverageDiagnostic { + val sourceMapPath = Path.of("$path.map") + val sourceMapExists = Files.exists(sourceMapPath) + + val diagnosticCode = if (sourceMapExists) { + PbtDiagnosticCode.COVERAGE_SOURCE_MAP_INVALID + } else { + PbtDiagnosticCode.COVERAGE_SOURCE_MAP_MISSING + } + val diagnosticMessage = if (sourceMapExists) { + "Executed JavaScript has a source map that c8 could not remap to its original source" + } else { + "Executed JavaScript below a TypeScript source root has no source map" + } + + return CoverageDiagnostic( + code = diagnosticCode, + message = diagnosticMessage, + path = path, + ) + } + + private fun classifyScope(path: String): CoverageScope? = when { + "/node_modules/" in path -> CoverageScope.DEPENDENCIES + isWithin(path, adapterRoot) -> CoverageScope.GENERATED_BACKEND_WRAPPERS + path in propertyEntryPoints -> CoverageScope.PROPERTY_ENTRY_POINTS + sourceRoots.any { sourceRoot -> isWithin(path, sourceRoot) } -> CoverageScope.SOURCE_UNDER_TEST + else -> null + } + + private fun shouldRetain(path: String, scope: CoverageScope): Boolean { + val scopeWasRequested = scope in context.request.scopes + val matchesIncludePatterns = matchesCoveragePath( + path = path, + patterns = context.request.includePatterns, + sourceRoots = sourceRoots, + ) + val matchesExcludePatterns = context.request.excludePatterns.isNotEmpty() && + matchesCoveragePath( + path = path, + patterns = context.request.excludePatterns, + sourceRoots = sourceRoots, + ) + + return scopeWasRequested && matchesIncludePatterns && !matchesExcludePatterns + } + + private fun isGeneratedJavaScriptBelowSourceRoot(path: String): Boolean { + val extension = path.substringAfterLast('.', missingDelimiterValue = "").lowercase() + val isGeneratedJavaScript = extension in GENERATED_JAVASCRIPT_EXTENSIONS + val isBelowSourceRoot = sourceRoots.any { sourceRoot -> isWithin(path, sourceRoot) } + + return isGeneratedJavaScript && isBelowSourceRoot + } + + private fun buildArtifact( + files: List, + diagnostics: List, + ): PropertyCoverageArtifact { + val provenance = CoverageProvenance( + collector = context.collector, + runtimeId = "node", + runtimeVersion = context.runtimeVersion, + sourceRoots = sourceRoots, + request = context.request, + ) + + return PropertyCoverageArtifact( + kind = CoverageArtifactKind.NODE_SOURCE, + backendId = context.backendId, + backendVersion = context.backendVersion, + propertyId = context.propertyId, + provenance = provenance, + files = files, + diagnostics = diagnostics, + ) + } + + private companion object { + val GENERATED_JAVASCRIPT_EXTENSIONS = hashSetOf("js", "mjs", "cjs") + + fun normalizeCoveragePath(path: String): String = Path.of(path) + .toAbsolutePath() + .normalize() + .toString() + .replace('\\', '/') + } +} diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt new file mode 100644 index 000000000..700602872 --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReport.kt @@ -0,0 +1,54 @@ +package org.usvm.ts.pbt.coverage + +import org.usvm.ts.pbt.backend.CoverageCollectorIdentity +import org.usvm.ts.pbt.backend.CoverageDiagnostic +import org.usvm.ts.pbt.backend.PropertyCoverageArtifact +import org.usvm.ts.pbt.backend.PropertyCoverageRequest +import org.usvm.ts.pbt.model.PropertyId +import java.nio.file.Path + +/** All identities and path boundaries required to convert one isolated c8 report. */ +data class IstanbulCoverageContext( + val backendId: String, + val backendVersion: String, + val propertyId: PropertyId, + val sourceRoots: List, + val propertyEntryPointPaths: Set, + val adapterRoot: String, + val runtimeVersion: String, + val collector: CoverageCollectorIdentity, + val request: PropertyCoverageRequest, +) + +/** Invalid or absent coverage that prevents construction of a trustworthy artifact. */ +class CoverageArtifactException( + val diagnostic: CoverageDiagnostic, + cause: Throwable? = null, +) : IllegalArgumentException(diagnostic.message, cause) { + internal companion object { + fun create( + code: String, + message: String, + path: String, + cause: Throwable? = null, + ) = CoverageArtifactException( + diagnostic = CoverageDiagnostic( + code = code, + message = message, + path = path, + ), + cause = cause, + ) + } +} + +/** Decodes and filters one source-mapped Istanbul JSON report produced by an isolated c8 run. */ +fun decodeIstanbulCoverageReport( + reportPath: Path, + context: IstanbulCoverageContext, +): PropertyCoverageArtifact { + val report = IstanbulCoverageReportReader.read(reportPath) + val decoder = IstanbulCoverageDecoder(context) + + return decoder.decode(report) +} diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportReader.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportReader.kt new file mode 100644 index 000000000..e401833fa --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportReader.kt @@ -0,0 +1,76 @@ +package org.usvm.ts.pbt.coverage + +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject +import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.manifest.PropertyManifestJson +import java.io.IOException +import java.nio.file.Files +import java.nio.file.Path + +/** Reads one bounded Istanbul report and converts I/O and JSON failures into coverage diagnostics. */ +internal object IstanbulCoverageReportReader { + fun read(reportPath: Path): JsonObject { + requireRegularFile(reportPath) + requireAllowedSize(reportPath) + + val reportText = readText(reportPath) + + return parse(reportText, reportPath) + } + + private fun requireRegularFile(reportPath: Path) { + if (!Files.isRegularFile(reportPath)) { + throw CoverageArtifactException.create( + code = PbtDiagnosticCode.COVERAGE_REPORT_MISSING, + message = "c8 did not produce the expected Istanbul report: $reportPath", + path = reportPath.toString(), + ) + } + } + + private fun requireAllowedSize(reportPath: Path) { + val reportSize = try { + Files.size(reportPath) + } catch (error: IOException) { + throw unreadableReport(reportPath, error) + } + + if (reportSize > MAX_COVERAGE_REPORT_BYTES) { + throw CoverageArtifactException.create( + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, + message = "Istanbul coverage report exceeds $MAX_COVERAGE_REPORT_BYTES bytes", + path = reportPath.toString(), + ) + } + } + + private fun readText(reportPath: Path): String = try { + Files.readString(reportPath) + } catch (error: IOException) { + throw unreadableReport(reportPath, error) + } + + private fun parse(reportText: String, reportPath: Path): JsonObject = try { + PropertyManifestJson.json.parseToJsonElement(reportText).jsonObject + } catch (error: IllegalArgumentException) { + throw CoverageArtifactException.create( + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, + message = "Istanbul coverage report is not valid JSON: ${error.message}", + path = reportPath.toString(), + cause = error, + ) + } + + private fun unreadableReport( + reportPath: Path, + error: IOException, + ): CoverageArtifactException = CoverageArtifactException.create( + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, + message = "Cannot read Istanbul coverage report: ${error.message}", + path = reportPath.toString(), + cause = error, + ) + + private const val MAX_COVERAGE_REPORT_BYTES = 64L * 1024 * 1024 +} diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulJsonValue.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulJsonValue.kt new file mode 100644 index 000000000..6f84a5dac --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulJsonValue.kt @@ -0,0 +1,120 @@ +package org.usvm.ts.pbt.coverage + +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.intOrNull +import kotlinx.serialization.json.longOrNull +import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.backend.SourcePosition +import org.usvm.ts.pbt.backend.SourceRange + +/** A JSON value paired with its Istanbul path for precise validation diagnostics. */ +internal class IstanbulJsonValue( + private val value: JsonElement, + val path: String, +) { + fun asObject(): JsonObject = value as? JsonObject + ?: throw invalidReport( + message = "Expected an object", + path = path, + ) + + fun asArray(): JsonArray = value as? JsonArray + ?: throw invalidReport( + message = "Expected an array", + path = path, + ) + + fun required(name: String): IstanbulJsonValue { + val childPath = "$path.$name" + val child = asObject()[name] + ?: throw invalidReport( + message = "Missing required field $name", + path = childPath, + ) + + return IstanbulJsonValue(child, childPath) + } + + fun requiredObject(name: String): JsonObject = required(name).asObject() + + fun requiredString(name: String): String { + val childPath = "$path.$name" + val primitive = asObject()[name] as? JsonPrimitive + ?: throw invalidReport( + message = "Expected a string", + path = childPath, + ) + + return primitive.contentOrNull + ?: throw invalidReport( + message = "Expected a string", + path = childPath, + ) + } + + fun optionalString(name: String): String? { + val primitive = asObject()[name] as? JsonPrimitive + + return primitive?.contentOrNull + } + + fun asRange(): SourceRange { + val range = asObject() + val rangeJson = IstanbulJsonValue(range, path) + val start = rangeJson.required(name = "start").asPosition() + val end = rangeJson.required(name = "end").asPosition() + + return SourceRange(start = start, end = end) + } + + fun asPosition(): SourcePosition { + val position = asObject() + val positionJson = IstanbulJsonValue(position, path) + val line = positionJson.requiredInt(name = "line") + val column = positionJson.requiredInt(name = "column") + + return SourcePosition(line = line, column = column) + } + + fun asHitCount(): Long { + val primitive = value as? JsonPrimitive + ?: throw invalidReport( + message = "Expected an integer", + path = path, + ) + + return primitive.longOrNull + ?: throw invalidReport( + message = "Expected an integer", + path = path, + ) + } + + private fun requiredInt(name: String): Int { + val childPath = "$path.$name" + val primitive = asObject()[name] as? JsonPrimitive + ?: throw invalidReport( + message = "Expected an integer", + path = childPath, + ) + + return primitive.intOrNull + ?: throw invalidReport( + message = "Expected an integer", + path = childPath, + ) + } + + internal companion object { + fun invalidReport(message: String, path: String): CoverageArtifactException = + CoverageArtifactException.create( + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, + message = message, + path = path, + ) + } +} diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulSourceFileDecoder.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulSourceFileDecoder.kt new file mode 100644 index 000000000..ec03fc317 --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/IstanbulSourceFileDecoder.kt @@ -0,0 +1,241 @@ +package org.usvm.ts.pbt.coverage + +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.backend.BranchArmCoverage +import org.usvm.ts.pbt.backend.BranchCoverage +import org.usvm.ts.pbt.backend.FunctionCoverage +import org.usvm.ts.pbt.backend.SourceFileCoverage +import org.usvm.ts.pbt.backend.StatementCoverage + +/** Decodes statement, function, and branch maps for one retained Istanbul file entry. */ +internal class IstanbulSourceFileDecoder( + file: JsonObject, + private val path: String, + reportKey: String, +) { + private val coveragePath = "coverage[$reportKey]" + private val fileJson = IstanbulJsonValue(file, coveragePath) + + fun decode(): SourceFileCoverage = try { + val statements = decodeStatements() + val functions = decodeFunctions() + val branches = decodeBranches() + + SourceFileCoverage( + path = path, + statements = statements, + functions = functions, + branches = branches, + ) + } catch (error: CoverageArtifactException) { + throw error + } catch (error: IllegalArgumentException) { + throw CoverageArtifactException.create( + code = PbtDiagnosticCode.COVERAGE_REPORT_INVALID, + message = "Invalid Istanbul coverage for $path: ${error.message}", + path = coveragePath, + cause = error, + ) + } + + private fun decodeStatements(): List { + val coverageMap = readCoverageMap( + locationsName = "statementMap", + hitsName = "s", + ) + + return coverageMap.ids.map { statementId -> + val location = coverageMap.location(statementId).asRange() + val hitCount = coverageMap.hitCount(statementId).asHitCount() + + StatementCoverage( + statementId = statementId, + location = location, + hits = hitCount, + ) + } + } + + private fun decodeFunctions(): List { + val coverageMap = readCoverageMap( + locationsName = "fnMap", + hitsName = "f", + ) + + return coverageMap.ids.map { functionId -> + decodeFunction( + functionId = functionId, + coverageMap = coverageMap, + ) + } + } + + private fun decodeFunction( + functionId: Int, + coverageMap: CoverageMap, + ): FunctionCoverage { + val functionJson = coverageMap.location(functionId) + val functionName = functionJson.optionalString(name = "name") + ?.ifBlank { ANONYMOUS_FUNCTION_NAME } + ?: ANONYMOUS_FUNCTION_NAME + val declaration = functionJson.required(name = "decl").asRange() + val body = functionJson.required(name = "loc").asRange() + val hitCount = coverageMap.hitCount(functionId).asHitCount() + + return FunctionCoverage( + functionId = functionId, + name = functionName, + declaration = declaration, + body = body, + hits = hitCount, + ) + } + + private fun decodeBranches(): List { + val coverageMap = readCoverageMap( + locationsName = "branchMap", + hitsName = "b", + ) + + return coverageMap.ids.map { branchId -> + decodeBranch( + branchId = branchId, + coverageMap = coverageMap, + ) + } + } + + private fun decodeBranch( + branchId: Int, + coverageMap: CoverageMap, + ): BranchCoverage { + val branchJson = coverageMap.location(branchId) + val branchHitsJson = coverageMap.hitCount(branchId) + val branchArms = decodeBranchArms(branchJson, branchHitsJson) + + val type = branchJson.requiredString(name = "type") + val location = branchJson.required(name = "loc").asRange() + val arms = branchArms.locations.indices.map { armIndex -> + BranchArmCoverage( + location = branchArms.location(armIndex).asRange(), + hits = branchArms.hitCount(armIndex).asHitCount(), + ) + } + + return BranchCoverage( + branchId = branchId, + type = type, + location = location, + arms = arms, + ) + } + + private fun decodeBranchArms( + branchJson: IstanbulJsonValue, + branchHitsJson: IstanbulJsonValue, + ): BranchArms { + val locationsJson = branchJson.required(name = "locations") + val locations = locationsJson.asArray() + val hits = branchHitsJson.asArray() + + if (locations.size != hits.size) { + throw IstanbulJsonValue.invalidReport( + message = "Branch location and hit counts have different sizes", + path = branchJson.path, + ) + } + + return BranchArms( + locations = locations, + hits = hits, + locationsPath = locationsJson.path, + hitsPath = branchHitsJson.path, + ) + } + + private fun readCoverageMap( + locationsName: String, + hitsName: String, + ): CoverageMap { + val locationsJson = fileJson.required(name = locationsName) + val hitsJson = fileJson.required(name = hitsName) + val locations = locationsJson.asObject() + val hits = hitsJson.asObject() + + if (locations.keys != hits.keys) { + throw IstanbulJsonValue.invalidReport( + message = "Coverage map keys do not match hit counter keys (${locationsJson.path})", + path = hitsJson.path, + ) + } + + val ids = locations.keys.map(::parseCoverageId).sorted() + + return CoverageMap( + ids = ids, + locations = locations, + hits = hits, + locationsPath = locationsJson.path, + hitsPath = hitsJson.path, + ) + } + + private fun parseCoverageId(value: String): Int { + val coverageId = value.toIntOrNull() + + return coverageId?.takeIf { id -> id >= 0 } + ?: throw IstanbulJsonValue.invalidReport( + message = "Coverage ID must be a non-negative integer", + path = value, + ) + } + + private data class CoverageMap( + val ids: List, + val locations: JsonObject, + val hits: JsonObject, + val locationsPath: String, + val hitsPath: String, + ) { + fun location(id: Int): IstanbulJsonValue { + val key = id.toString() + + return IstanbulJsonValue( + value = locations.getValue(key), + path = "$locationsPath[$key]", + ) + } + + fun hitCount(id: Int): IstanbulJsonValue { + val key = id.toString() + + return IstanbulJsonValue( + value = hits.getValue(key), + path = "$hitsPath[$key]", + ) + } + } + + private data class BranchArms( + val locations: JsonArray, + val hits: JsonArray, + val locationsPath: String, + val hitsPath: String, + ) { + fun location(index: Int) = IstanbulJsonValue( + value = locations[index], + path = "$locationsPath[$index]", + ) + + fun hitCount(index: Int) = IstanbulJsonValue( + value = hits[index], + path = "$hitsPath[$index]", + ) + } + + private companion object { + const val ANONYMOUS_FUNCTION_NAME = "" + } +} diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt index 66158dbe6..e17ddf93c 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt @@ -2,6 +2,7 @@ package org.usvm.ts.pbt.fastcheck import org.usvm.ts.pbt.PbtDiagnosticCode import org.usvm.ts.pbt.backend.PropertyBasedTestingBackend +import org.usvm.ts.pbt.backend.PropertyCoverageCapability import org.usvm.ts.pbt.backend.PropertyRunConfiguration import org.usvm.ts.pbt.backend.PropertyRunResult import org.usvm.ts.pbt.manifest.toManifest @@ -21,6 +22,12 @@ class FastCheckBackend( nodeExecutable: String = "node", adapterEntryPoint: Path = FastCheckRuntime.executionEntryPoint(), ) : PropertyBasedTestingBackend { + override val coverageCapability: PropertyCoverageCapability = PropertyCoverageCapability.supported( + backendId = FAST_CHECK_BACKEND_ID, + backendVersion = FastCheckRuntimeMetadata.fastCheckVersion, + collector = FastCheckRuntimeMetadata.coverageCollector, + ) + private val sourceRoots = canonicalizeSourceRoots(sourceRoots) private val client = FastCheckProcessClient( nodeExecutable = nodeExecutable, @@ -43,6 +50,7 @@ class FastCheckBackend( numRuns = configuration.numRuns, timeoutMillis = configuration.timeoutMillis, examples = configuration.examples, + coverageRequest = configuration.coverageRequest, ), ) } @@ -133,6 +141,8 @@ class FastCheckBackend( ) companion object { + const val FAST_CHECK_BACKEND_ID = "fast-check" + private val FINITE_NUMBER_BITS_REGEX = Regex("[0-9a-f]{16}") private fun canonicalizeSourceRoots(sourceRoots: List): List { diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckExecutionProtocol.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckExecutionProtocol.kt index 7b7e507f6..2002aaa04 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckExecutionProtocol.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckExecutionProtocol.kt @@ -2,6 +2,8 @@ package org.usvm.ts.pbt.fastcheck import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable +import kotlinx.serialization.Transient +import org.usvm.ts.pbt.backend.PropertyCoverageRequest import org.usvm.ts.pbt.backend.PropertyRunResult import org.usvm.ts.pbt.manifest.PropertyManifest import org.usvm.ts.pbt.model.JsConcreteValue @@ -15,6 +17,8 @@ internal data class FastCheckExecutionRequest( val numRuns: Int, val timeoutMillis: Long, val examples: List> = emptyList(), + @Transient + val coverageRequest: PropertyCoverageRequest? = null, ) @Serializable @@ -41,6 +45,9 @@ enum class BackendErrorKind { @SerialName("timeout") TIMEOUT, + + @SerialName("coverage") + COVERAGE, } /** Typed failure at the concrete backend boundary. */ diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt index 187417291..85085343b 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClient.kt @@ -12,12 +12,17 @@ import kotlinx.coroutines.withTimeoutOrNull import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString import org.usvm.ts.pbt.PbtDiagnosticCode +import org.usvm.ts.pbt.backend.CoverageScope import org.usvm.ts.pbt.backend.PropertyRunResult +import org.usvm.ts.pbt.coverage.CoverageArtifactException +import org.usvm.ts.pbt.coverage.IstanbulCoverageContext +import org.usvm.ts.pbt.coverage.decodeIstanbulCoverageReport import org.usvm.ts.pbt.manifest.PropertyManifestJson import org.usvm.ts.pbt.model.PropertyId import java.io.ByteArrayOutputStream import java.io.IOException import java.io.InputStream +import java.nio.file.Files import java.nio.file.Path import java.util.concurrent.TimeUnit @@ -47,17 +52,22 @@ internal class FastCheckProcessClient( private suspend fun checkSuspending(request: FastCheckExecutionRequest): PropertyRunResult = supervisorScope { val encodedRequest = encodeRequest(request) - val process = startAdapter(request) - val stdout = async(ioDispatcher) { process.inputStream.readBounded(MAX_STDOUT_BYTES) } - val stderr = async(ioDispatcher) { process.errorStream.readBounded(MAX_STDERR_BYTES) } - val writer = async(ioDispatcher) { - process.outputStream.bufferedWriter(Charsets.UTF_8).use { output -> - output.write(encodedRequest) - } - } + val coverageRuntimeVersion = request.coverageRequest?.let { nodeVersion(request) } + val coverageWorkspace = request.coverageRequest?.let { createCoverageWorkspace(request) } + var process: Process? = null try { - awaitProcess(process, request) + val startedProcess = startAdapter(request, coverageWorkspace) + process = startedProcess + val stdout = async(ioDispatcher) { startedProcess.inputStream.readBounded(MAX_STDOUT_BYTES) } + val stderr = async(ioDispatcher) { startedProcess.errorStream.readBounded(MAX_STDERR_BYTES) } + val writer = async(ioDispatcher) { + startedProcess.outputStream.bufferedWriter(Charsets.UTF_8).use { output -> + output.write(encodedRequest) + } + } + + awaitProcess(startedProcess, request) awaitIo( task = writer, @@ -79,14 +89,24 @@ internal class FastCheckProcessClient( request = request, ) - validateProcessExit(process, stderrText, request) + validateProcessExit(startedProcess, stderrText, request) validateStdout(stdoutText, request) val response = decodeResponse(stdoutText.text, request) - decodeSuccessfulResponse(response, request) + val result = decodeSuccessfulResponse(response, request) + + coverageWorkspace?.let { workspace -> + collectCoverage( + result = result, + request = request, + workspace = workspace, + runtimeVersion = requireNotNull(coverageRuntimeVersion), + ) + } ?: result } finally { - if (process.isAlive) terminate(process) + process?.takeIf(Process::isAlive)?.let(::terminate) + coverageWorkspace?.root?.toFile()?.deleteRecursively() } } @@ -160,8 +180,11 @@ internal class FastCheckProcessClient( } } - private fun startAdapter(request: FastCheckExecutionRequest): Process = try { - ProcessBuilder(nodeExecutable, adapterEntryPoint.toString()).start() + private fun startAdapter( + request: FastCheckExecutionRequest, + coverageWorkspace: CoverageWorkspace?, + ): Process = try { + ProcessBuilder(adapterCommand(request, coverageWorkspace)).start() } catch (error: IOException) { throw backendError( kind = BackendErrorKind.PROCESS_FAILURE, @@ -172,6 +195,198 @@ internal class FastCheckProcessClient( ) } + private fun adapterCommand( + request: FastCheckExecutionRequest, + coverageWorkspace: CoverageWorkspace?, + ): List { + if (coverageWorkspace == null) return listOf(nodeExecutable, adapterEntryPoint.toString()) + + val command = mutableListOf( + nodeExecutable, + coverageWorkspace.c8EntryPoint.toString(), + "--config=${coverageWorkspace.configPath}", + "--reporter=json", + "--reports-dir=${coverageWorkspace.reportDirectory}", + "--temp-directory=${coverageWorkspace.rawDirectory}", + "--exclude-after-remap", + "--allowExternal", + "--exclude=__usvm_no_default_excludes__", + ) + if (CoverageScope.DEPENDENCIES in requireNotNull(request.coverageRequest).scopes) { + command += "--exclude-node-modules=false" + } + command += nodeExecutable + command += adapterEntryPoint.toString() + + return command + } + + private fun createCoverageWorkspace(request: FastCheckExecutionRequest): CoverageWorkspace { + val adapterRoot = adapterRoot() + val c8EntryPoint = adapterRoot.resolve("node_modules/c8/bin/c8.js") + if (!Files.isRegularFile(c8EntryPoint)) { + throw backendError( + kind = BackendErrorKind.COVERAGE, + code = PbtDiagnosticCode.COVERAGE_COLLECTOR_NOT_FOUND, + message = "Cannot locate c8 ${FastCheckRuntimeMetadata.coverageCollector.version} " + + "in the fast-check adapter runtime", + request = request, + path = c8EntryPoint.toString(), + ) + } + + val root = Files.createTempDirectory("usvm-ts-pbt-coverage-") + val configPath = Files.writeString(root.resolve("c8-config.json"), "{}") + return CoverageWorkspace( + root = root, + configPath = configPath, + rawDirectory = root.resolve("raw"), + reportDirectory = root.resolve("report"), + c8EntryPoint = c8EntryPoint, + adapterRoot = adapterRoot, + ) + } + + private fun collectCoverage( + result: PropertyRunResult, + request: FastCheckExecutionRequest, + workspace: CoverageWorkspace, + runtimeVersion: String, + ): PropertyRunResult { + val coverageRequest = requireNotNull(request.coverageRequest) + val entryPointPaths = hashSetOf() + request.sourceRoots.forEach { sourceRoot -> + val root = Path.of(sourceRoot) + entryPointPaths += root.resolve(request.manifest.predicate.module).normalize().toString() + request.manifest.precondition?.let { precondition -> + entryPointPaths += root.resolve(precondition.module).normalize().toString() + } + } + val artifact = try { + decodeIstanbulCoverageReport( + reportPath = workspace.reportDirectory.resolve("coverage-final.json"), + context = IstanbulCoverageContext( + backendId = FastCheckBackend.FAST_CHECK_BACKEND_ID, + backendVersion = FastCheckRuntimeMetadata.fastCheckVersion, + propertyId = result.propertyId, + sourceRoots = request.sourceRoots, + propertyEntryPointPaths = entryPointPaths, + adapterRoot = workspace.adapterRoot.toString(), + runtimeVersion = runtimeVersion, + collector = FastCheckRuntimeMetadata.coverageCollector, + request = coverageRequest, + ), + ) + } catch (error: CoverageArtifactException) { + throw backendError( + kind = BackendErrorKind.COVERAGE, + code = error.diagnostic.code, + message = error.diagnostic.message, + request = request, + path = error.diagnostic.path, + cause = error, + ) + } + + return result.copy(coverage = artifact) + } + + private suspend fun nodeVersion(request: FastCheckExecutionRequest): String { + val process = startNodeVersionProcess(request) + + awaitNodeVersionProcess(process, request) + + return readNodeVersion(process, request) + } + + private fun startNodeVersionProcess(request: FastCheckExecutionRequest): Process = try { + ProcessBuilder(nodeExecutable, "--version").start() + } catch (error: IOException) { + throw coverageRuntimeVersionError(request, error) + } + + private suspend fun awaitNodeVersionProcess( + process: Process, + request: FastCheckExecutionRequest, + ) { + val completed = runInterruptible(ioDispatcher) { + process.waitFor(NODE_VERSION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) + } + if (!completed) { + process.destroyForcibly() + + throw backendError( + kind = BackendErrorKind.COVERAGE, + code = PbtDiagnosticCode.COVERAGE_RUNTIME_VERSION_UNAVAILABLE, + message = "Timed out while querying the Node.js runtime version", + request = request, + ) + } + } + + private fun readNodeVersion( + process: Process, + request: FastCheckExecutionRequest, + ): String { + val version = process.inputStream.bufferedReader(Charsets.UTF_8).use { input -> + input.readLine().orEmpty().trim() + } + if (process.exitValue() != 0 || version.isBlank()) { + throw backendError( + kind = BackendErrorKind.COVERAGE, + code = PbtDiagnosticCode.COVERAGE_RUNTIME_VERSION_UNAVAILABLE, + message = "Cannot query the Node.js runtime version", + request = request, + ) + } + + return requireSupportedNodeVersion(version, request) + } + + private fun requireSupportedNodeVersion( + version: String, + request: FastCheckExecutionRequest, + ): String { + val match = NODE_VERSION_PATTERN.matchEntire(version) + val major = match?.groupValues?.get(1)?.toIntOrNull() + val minor = match?.groupValues?.get(2)?.toIntOrNull() + if (major == null || minor == null) { + throw backendError( + kind = BackendErrorKind.COVERAGE, + code = PbtDiagnosticCode.COVERAGE_RUNTIME_VERSION_UNAVAILABLE, + message = "Cannot parse the Node.js runtime version: $version", + request = request, + ) + } + + val isSupported = major > MINIMUM_NODE_MAJOR_VERSION || + major == MINIMUM_NODE_MAJOR_VERSION && minor >= MINIMUM_NODE_MINOR_VERSION + if (!isSupported) { + throw backendError( + kind = BackendErrorKind.COVERAGE, + code = PbtDiagnosticCode.COVERAGE_RUNTIME_UNSUPPORTED, + message = "Coverage requires Node.js 18.18 or newer; found $version", + request = request, + ) + } + + return version + } + + private fun coverageRuntimeVersionError( + request: FastCheckExecutionRequest, + error: IOException, + ) = backendError( + kind = BackendErrorKind.COVERAGE, + code = PbtDiagnosticCode.COVERAGE_RUNTIME_VERSION_UNAVAILABLE, + message = "Cannot query the Node.js runtime version: ${error.message}", + request = request, + cause = error, + ) + + private fun adapterRoot(): Path = adapterEntryPoint.parent?.parent?.parent + ?: throw IllegalArgumentException("Adapter entry point has no runtime root: $adapterEntryPoint") + private suspend fun awaitIo( task: Deferred, operation: String, @@ -313,9 +528,22 @@ internal class FastCheckProcessClient( const val MAX_STDERR_BYTES = 64 * 1024 const val DEFAULT_TRANSPORT_GRACE_MILLIS = 2_000L const val DEFAULT_SHUTDOWN_GRACE_MILLIS = 250L + const val NODE_VERSION_TIMEOUT_MILLIS = 5_000L + const val MINIMUM_NODE_MAJOR_VERSION = 18 + const val MINIMUM_NODE_MINOR_VERSION = 18 + val NODE_VERSION_PATTERN = Regex("""^v(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$""") } } +private data class CoverageWorkspace( + val root: Path, + val configPath: Path, + val rawDirectory: Path, + val reportDirectory: Path, + val c8EntryPoint: Path, + val adapterRoot: Path, +) + private data class BoundedText(val text: String, val exceeded: Boolean) private fun InputStream.readBounded(limit: Int): BoundedText { diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntimeMetadata.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntimeMetadata.kt new file mode 100644 index 000000000..e68827f61 --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntimeMetadata.kt @@ -0,0 +1,29 @@ +package org.usvm.ts.pbt.fastcheck + +import org.usvm.ts.pbt.backend.CoverageCollectorIdentity +import java.util.Properties + +/** Dependency versions generated from the packaged fast-check adapter lockfile. */ +internal object FastCheckRuntimeMetadata { + private val properties = Properties().apply { + val resource = checkNotNull( + FastCheckRuntimeMetadata::class.java.getResourceAsStream(RUNTIME_METADATA_RESOURCE), + ) { + "Missing generated fast-check runtime metadata: $RUNTIME_METADATA_RESOURCE" + } + resource.use(::load) + } + + val fastCheckVersion: String = requireVersion("fast-check.version") + + val coverageCollector = CoverageCollectorIdentity( + id = "c8", + version = requireVersion("c8.version"), + ) + + private fun requireVersion(name: String): String = properties.getProperty(name) + ?.takeIf(String::isNotBlank) + ?: error("Missing fast-check runtime dependency version: $name") + + private const val RUNTIME_METADATA_RESOURCE = "/org/usvm/ts/pbt/fastcheck/runtime-dependencies.properties" +} diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/backend/PropertyCoverageTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/backend/PropertyCoverageTest.kt new file mode 100644 index 000000000..42d8d4f82 --- /dev/null +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/backend/PropertyCoverageTest.kt @@ -0,0 +1,161 @@ +package org.usvm.ts.pbt.backend + +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.jsonObject +import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.manifest.PropertyManifestJson +import org.usvm.ts.pbt.model.PropertyId +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNull + +class PropertyCoverageTest { + @Test + fun `coverage request defaults to source under test only`() { + val request = PropertyCoverageRequest() + + assertEquals(setOf(CoverageScope.SOURCE_UNDER_TEST), request.scopes) + assertEquals(emptyList(), request.includePatterns) + assertEquals(emptyList(), request.excludePatterns) + } + + @Test + fun `supported and unsupported capabilities carry different evidence`() { + val supported = PropertyCoverageCapability.supported( + backendId = "fast-check", + backendVersion = "4.9.0", + collector = CoverageCollectorIdentity( + id = "c8", + version = "10.1.3", + ), + ) + val supportedJson = PropertyManifestJson.json + .parseToJsonElement(PropertyManifestJson.json.encodeToString(supported)) + .jsonObject + + assertEquals(CoverageCapabilityLevel.SUPPORTED, supported.level) + assertEquals("c8", supported.collector?.id) + assertFalse("artifactVersion" in supportedJson) + assertEquals(emptyList(), supported.diagnostics) + + val unsupported = PropertyCoverageCapability.unsupported( + backendId = "other-backend", + backendVersion = "1.0.0", + diagnostic = CoverageDiagnostic( + code = "coverage.unsupported", + message = "The backend does not collect source coverage", + path = "coverageRequest", + ), + ) + val unsupportedJson = PropertyManifestJson.json + .parseToJsonElement(PropertyManifestJson.json.encodeToString(unsupported)) + .jsonObject + + assertEquals(CoverageCapabilityLevel.UNSUPPORTED, unsupported.level) + assertNull(unsupported.collector) + assertFalse("artifactVersion" in unsupportedJson) + assertEquals("coverage.unsupported", unsupported.diagnostics.single().code) + } + + @Test + fun `coverage locations and hit counters reject invalid values`() { + assertFailsWith { SourcePosition(line = 0, column = 0) } + assertFailsWith { SourcePosition(line = 1, column = -1) } + assertFailsWith { + SourceRange( + start = SourcePosition(line = 2, column = 0), + end = SourcePosition(line = 1, column = 0), + ) + } + assertFailsWith { + StatementCoverage( + statementId = 0, + location = range, + hits = -1, + ) + } + assertFailsWith { + BranchCoverage( + branchId = 0, + type = "if", + location = range, + arms = emptyList(), + ) + } + } + + @Test + fun `run result requires matching coverage identity`() { + val artifact = artifact() + + successfulResult(coverage = artifact) + + assertFailsWith { + successfulResult( + coverage = artifact.copy(propertyId = PropertyId("different.property")), + ) + } + } + + private fun artifact() = PropertyCoverageArtifact( + kind = CoverageArtifactKind.NODE_SOURCE, + backendId = "fast-check", + backendVersion = "4.9.0", + propertyId = PropertyId("example.property"), + provenance = CoverageProvenance( + collector = CoverageCollectorIdentity( + id = "c8", + version = "10.1.3", + ), + runtimeId = "node", + runtimeVersion = "v22.14.0", + sourceRoots = listOf("/workspace/src"), + request = PropertyCoverageRequest(), + ), + files = listOf( + SourceFileCoverage( + path = "/workspace/src/example.ts", + statements = listOf(StatementCoverage(statementId = 0, location = range, hits = 1)), + functions = listOf( + FunctionCoverage( + functionId = 0, + name = "example", + declaration = range, + body = range, + hits = 1, + ), + ), + branches = listOf( + BranchCoverage( + branchId = 0, + type = "if", + location = range, + arms = listOf(BranchArmCoverage(location = range, hits = 1)), + ), + ), + ), + ), + ) + + private fun successfulResult(coverage: PropertyCoverageArtifact?) = PropertyRunResult( + propertyId = PropertyId("example.property"), + status = PropertyRunStatus.SUCCESS, + seed = 42, + replayPath = null, + counterexample = null, + numRuns = 1, + numSkips = 0, + numShrinks = 0, + failure = null, + executionTimeMillis = 1, + coverage = coverage, + ) + + private companion object { + val range = SourceRange( + start = SourcePosition(line = 1, column = 0), + end = SourcePosition(line = 1, column = 10), + ) + } +} diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/FastCheckCliTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/FastCheckCliTest.kt index dd5a057f3..d00526bec 100644 --- a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/FastCheckCliTest.kt +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/FastCheckCliTest.kt @@ -4,7 +4,13 @@ import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.backend.CoverageArtifactKind +import org.usvm.ts.pbt.backend.CoverageCollectorIdentity +import org.usvm.ts.pbt.backend.CoverageDiagnostic +import org.usvm.ts.pbt.backend.CoverageProvenance import org.usvm.ts.pbt.backend.PropertyBasedTestingBackend +import org.usvm.ts.pbt.backend.PropertyCoverageArtifact +import org.usvm.ts.pbt.backend.PropertyCoverageCapability import org.usvm.ts.pbt.backend.PropertyFailureDetails import org.usvm.ts.pbt.backend.PropertyFailureKind import org.usvm.ts.pbt.backend.PropertyRunConfiguration @@ -25,6 +31,7 @@ import java.nio.file.Files import java.nio.file.Path import kotlin.test.assertContains import kotlin.test.assertEquals +import kotlin.test.assertFalse class FastCheckCliTest { @Test @@ -42,6 +49,7 @@ class FastCheckCliTest { assertEquals("", errors.toString()) assertContains(output, "TypeScript source root") assertContains(output, "--examples") + assertContains(output, "--coverage-scope") } @Test @@ -126,6 +134,84 @@ class FastCheckCliTest { assertEquals(listOf("passing", "failing"), propertyIds) } + @Test + fun `coverage options request and serialize one artifact per property`() { + val output = StringBuilder() + val errors = StringBuilder() + val cli = cli( + providers = listOf(provider(registryId = "examples", propertyIds = arrayOf("covered"))), + output = output, + errors = errors, + ) + + val exitCode = cli.run( + arrayOf( + "--source-root", sourceRoot.toString(), + "--property", "covered", + "--coverage", + "--coverage-scope", "property-entry-points", + "--coverage-include", "**/*.ts", + "--coverage-exclude", "**/ignored.ts", + ), + ) + + val result = PropertyManifestJson.json.parseToJsonElement(output.toString()).jsonArray.single().jsonObject + val coverage = result.getValue("coverage").jsonObject + val request = coverage.getValue("provenance").jsonObject.getValue("request").jsonObject + val includePatterns = request.getValue("includePatterns").jsonArray + val excludePatterns = request.getValue("excludePatterns").jsonArray + + assertEquals(0, exitCode) + assertEquals("", errors.toString()) + assertFalse("schemaVersion" in coverage) + assertEquals( + listOf("property_entry_points"), + request.getValue("scopes").jsonArray.map { scope -> scope.jsonPrimitive.content }, + ) + assertEquals("**/*.ts", includePatterns.single().jsonPrimitive.content) + assertEquals("**/ignored.ts", excludePatterns.single().jsonPrimitive.content) + } + + @Test + fun `coverage details require the coverage flag`() { + val errors = StringBuilder() + + val exitCode = cli( + providers = listOf(provider(registryId = "examples", propertyIds = arrayOf("property"))), + errors = errors, + ).run( + arrayOf( + "--source-root", + sourceRoot.toString(), + "--coverage-scope", + "source-under-test", + ), + ) + + assertEquals(2, exitCode) + assertEquals("cli.coverage.required", diagnosticCode(errors)) + } + + @Test + fun `coverage request reports an unsupported backend before execution`() { + val errors = StringBuilder() + + val exitCode = cli( + providers = listOf(provider(registryId = "examples", propertyIds = arrayOf("property"))), + errors = errors, + supportsCoverage = false, + ).run( + arrayOf( + "--source-root", + sourceRoot.toString(), + "--coverage", + ), + ) + + assertEquals(2, exitCode) + assertEquals("coverage.unsupported", diagnosticCode(errors)) + } + @Test fun `reports unknown registries and duplicate property ids as CLI errors`() { val unknownErrors = StringBuilder() @@ -280,9 +366,15 @@ class FastCheckCliTest { output: Appendable = StringBuilder(), errors: Appendable = StringBuilder(), failures: Set = emptySet(), + supportsCoverage: Boolean = true, ) = FastCheckCli( providers = providers, - backendFactory = { FakeBackend(failures) }, + backendFactory = { + FakeBackend( + failures = failures, + supportsCoverage = supportsCoverage, + ) + }, output = output, errors = errors, ) @@ -309,7 +401,31 @@ class FastCheckCliTest { .jsonPrimitive .content - private class FakeBackend(private val failures: Set) : PropertyBasedTestingBackend { + private class FakeBackend( + private val failures: Set, + supportsCoverage: Boolean, + ) : PropertyBasedTestingBackend { + override val coverageCapability: PropertyCoverageCapability = if (supportsCoverage) { + PropertyCoverageCapability.supported( + backendId = "fake", + backendVersion = "1.0.0", + collector = CoverageCollectorIdentity( + id = "c8", + version = "10.1.3", + ), + ) + } else { + PropertyCoverageCapability.unsupported( + backendId = "fake", + backendVersion = "1.0.0", + diagnostic = CoverageDiagnostic( + code = "coverage.unsupported", + message = "Fake backend does not collect coverage", + path = "coverageRequest", + ), + ) + } + override fun run( property: PropertyDefinition, configuration: PropertyRunConfiguration, @@ -335,6 +451,22 @@ class FastCheckCliTest { null }, executionTimeMillis = 1, + coverage = configuration.coverageRequest?.let { request -> + PropertyCoverageArtifact( + kind = CoverageArtifactKind.NODE_SOURCE, + backendId = coverageCapability.backendId, + backendVersion = coverageCapability.backendVersion, + propertyId = property.id, + provenance = CoverageProvenance( + collector = requireNotNull(coverageCapability.collector), + runtimeId = "node", + runtimeVersion = "v22.14.0", + sourceRoots = listOf(sourceRoot.toString()), + request = request, + ), + files = emptyList(), + ) + }, ) } } diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/InstalledDistributionRegistryProvider.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/InstalledDistributionRegistryProvider.kt index 9eaad1836..c7ca9e377 100644 --- a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/InstalledDistributionRegistryProvider.kt +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/cli/InstalledDistributionRegistryProvider.kt @@ -26,6 +26,19 @@ class InstalledDistributionRegistryProvider : PropertyRegistryProvider { exportName = "alwaysTrue", ), ), + PropertyDefinition( + id = PropertyId("distribution.covers-positive"), + inputs = listOf( + PropertyInput( + name = "value", + domain = IntegerDomain(min = 1, max = 10), + ), + ), + predicate = TypeScriptEntryPoint( + module = "properties/coverage/CoverageProperties.ts", + exportName = "coversPositive", + ), + ), ), ) } diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt new file mode 100644 index 000000000..81e2ade2b --- /dev/null +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/coverage/IstanbulCoverageReportTest.kt @@ -0,0 +1,206 @@ +package org.usvm.ts.pbt.coverage + +import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.backend.CoverageCollectorIdentity +import org.usvm.ts.pbt.backend.CoverageScope +import org.usvm.ts.pbt.backend.PropertyCoverageRequest +import org.usvm.ts.pbt.model.PropertyId +import java.nio.ByteBuffer +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption +import kotlin.io.path.absolute +import kotlin.io.path.createTempDirectory +import kotlin.io.path.createTempFile +import kotlin.io.path.deleteIfExists +import kotlin.io.path.writeText +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class IstanbulCoverageReportTest { + @Test + fun `decodes literal TypeScript statement function and branch hits`() { + val artifact = decodeIstanbulCoverageReport( + reportPath = goldenReport(), + context = context(), + ) + + val file = artifact.files.single() + val branch = file.branches.single() + val firstArm = branch.arms.first() + + assertEquals("/workspace/src/math.ts", file.path) + assertEquals(listOf(2L, 2L), file.statements.map { statement -> statement.hits }) + assertEquals("classify", file.functions.single().name) + assertEquals(2L, file.functions.single().hits) + assertEquals("if", branch.type) + assertEquals(listOf(1L, 1L), branch.arms.map { arm -> arm.hits }) + assertEquals(2, branch.location.start.line) + assertEquals(2, firstArm.location.start.line) + } + + @Test + fun `classifies entry points wrappers and dependencies independently`() { + val artifact = decodeIstanbulCoverageReport( + reportPath = goldenReport(), + context = context( + request = PropertyCoverageRequest(scopes = CoverageScope.entries.toHashSet()), + ), + ) + + assertEquals( + listOf( + "/adapter/dist/src/execution-cli.js", + "/adapter/node_modules/example/index.js", + "/workspace/src/math.ts", + "/workspace/src/property.ts", + ), + artifact.files.map { file -> file.path }, + ) + } + + @Test + fun `include and exclude globs filter remapped paths with exclude precedence`() { + val artifact = decodeIstanbulCoverageReport( + reportPath = goldenReport(), + context = context( + request = PropertyCoverageRequest( + scopes = CoverageScope.entries.toHashSet(), + includePatterns = listOf("**/*.ts"), + excludePatterns = listOf("**/property.?s"), + ), + ), + ) + + assertEquals(listOf("/workspace/src/math.ts"), artifact.files.map { file -> file.path }) + } + + @Test + fun `generated JavaScript below source roots reports a missing source map`() { + val artifact = decodeIstanbulCoverageReport( + reportPath = goldenReport(), + context = context(), + ) + + val diagnostic = artifact.diagnostics.single() + assertEquals("coverage.source-map.missing", diagnostic.code) + assertEquals("/workspace/src/generated-without-map.js", diagnostic.path) + } + + @Test + fun `missing and malformed reports produce stable diagnostics`() { + val missing = assertFailsWith { + decodeIstanbulCoverageReport( + reportPath = Path.of("/definitely/missing/coverage-final.json"), + context = context(), + ) + } + assertEquals("coverage.report.missing", missing.diagnostic.code) + + val malformedReport = createTempFile(suffix = ".json") + try { + malformedReport.writeText("{not-json") + val malformed = assertFailsWith { + decodeIstanbulCoverageReport( + reportPath = malformedReport, + context = context(), + ) + } + assertEquals("coverage.report.invalid", malformed.diagnostic.code) + } finally { + malformedReport.deleteIfExists() + } + } + + @Test + fun `report larger than the decoder limit is rejected before parsing`() { + val oversizedReport = createTempFile(suffix = ".json") + try { + Files.newByteChannel(oversizedReport, StandardOpenOption.WRITE).use { report -> + report.position(TEST_COVERAGE_REPORT_LIMIT_BYTES) + report.write(ByteBuffer.wrap(byteArrayOf(0))) + } + + val oversized = assertFailsWith { + decodeIstanbulCoverageReport( + reportPath = oversizedReport, + context = context(), + ) + } + + assertEquals("coverage.report.invalid", oversized.diagnostic.code) + assertEquals( + "Istanbul coverage report exceeds $TEST_COVERAGE_REPORT_LIMIT_BYTES bytes", + oversized.diagnostic.message, + ) + } finally { + oversizedReport.deleteIfExists() + } + } + + @Test + fun `generated JavaScript with an unusable map reports an invalid source map`() { + val directory = createTempDirectory("invalid-source-map-") + try { + val generated = directory.resolve("generated.js") + generated.writeText("export const value = 1;\n//# sourceMappingURL=generated.js.map") + directory.resolve("generated.js.map").writeText("{not-json") + val report = directory.resolve("coverage-final.json") + report.writeText( + """ + { + "$generated": { + "path": "$generated", + "statementMap": { + "0": { "start": { "line": 1, "column": 0 }, "end": { "line": 1, "column": 23 } } + }, + "fnMap": {}, + "branchMap": {}, + "s": { "0": 1 }, + "f": {}, + "b": {} + } + } + """.trimIndent(), + ) + + val artifact = decodeIstanbulCoverageReport( + reportPath = report, + context = context().copy(sourceRoots = listOf(directory.toString())), + ) + + assertEquals("coverage.source-map.invalid", artifact.diagnostics.single().code) + assertEquals(generated.toString(), artifact.diagnostics.single().path) + } finally { + directory.toFile().deleteRecursively() + } + } + + private fun context( + request: PropertyCoverageRequest = PropertyCoverageRequest(), + ) = IstanbulCoverageContext( + backendId = "fast-check", + backendVersion = "4.9.0", + propertyId = PropertyId("coverage.property"), + sourceRoots = listOf("/workspace/src"), + propertyEntryPointPaths = setOf("/workspace/src/property.ts"), + adapterRoot = "/adapter", + runtimeVersion = "v22.14.0", + collector = CoverageCollectorIdentity(id = "c8", version = "10.1.3"), + request = request, + ) + + private fun goldenReport(): Path = locateFile( + "src/test/resources/coverage/istanbul/statement-branch.json", + "usvm-ts-pbt/src/test/resources/coverage/istanbul/statement-branch.json", + ) + + private fun locateFile(vararg candidates: String): Path = candidates + .map { candidate -> Path.of(candidate).absolute() } + .singleOrNull(Files::isRegularFile) + ?: error("Cannot locate file; checked ${candidates.toList()}") + + private companion object { + const val TEST_COVERAGE_REPORT_LIMIT_BYTES = 64L * 1024 * 1024 + } +} diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackendTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackendTest.kt index c618870e7..764c6fefc 100644 --- a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackendTest.kt +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackendTest.kt @@ -1,6 +1,8 @@ package org.usvm.ts.pbt.fastcheck import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.backend.CoverageScope +import org.usvm.ts.pbt.backend.PropertyCoverageRequest import org.usvm.ts.pbt.backend.PropertyFailureKind import org.usvm.ts.pbt.backend.PropertyRunConfiguration import org.usvm.ts.pbt.backend.PropertyRunStatus @@ -35,6 +37,24 @@ class FastCheckBackendTest { assertEquals(20, result.numRuns) } + @Test + fun `collects source mapped entry point coverage when Kotlin requests it`() { + val result = backend.run( + property = property(predicate = "alwaysTrue"), + configuration = configuration.copy( + coverageRequest = PropertyCoverageRequest( + scopes = setOf(CoverageScope.PROPERTY_ENTRY_POINTS), + ), + ), + ) + + val coverage = assertNotNull(result.coverage) + val file = coverage.files.single { source -> source.path.endsWith(MODULE) } + + assertTrue(file.statements.isNotEmpty()) + assertTrue(file.functions.isNotEmpty()) + } + @Test fun `passes multiple generated inputs to the predicate in declaration order`() { val definition = property( diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt new file mode 100644 index 000000000..1e846000e --- /dev/null +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt @@ -0,0 +1,131 @@ +package org.usvm.ts.pbt.fastcheck + +import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.backend.PropertyCoverageRequest +import org.usvm.ts.pbt.backend.PropertyRunConfiguration +import org.usvm.ts.pbt.backend.PropertyRunStatus +import org.usvm.ts.pbt.backend.SourceFileCoverage +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.PropertyDefinition +import org.usvm.ts.pbt.model.PropertyId +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.model.TypeScriptEntryPoint +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.absolute +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + +class FastCheckCoverageTest { + private val backend = FastCheckBackend( + sourceRoots = listOf(sourceRoot()), + adapterEntryPoint = adapterEntryPoint(), + ) + + @Test + fun `golden TypeScript statements functions and branches are isolated per property`() { + val positive = backend.run( + property = property( + exportName = "coversPositive", + domain = IntegerDomain(min = 1, max = 1), + ), + configuration = configuration, + ) + val nonPositive = backend.run( + property = property( + exportName = "coversNonPositive", + domain = IntegerDomain(min = -1, max = -1), + ), + configuration = configuration, + ) + + val positiveFile = sourceUnderTest(positive) + val nonPositiveFile = sourceUnderTest(nonPositive) + val positiveFunctions = positiveFile.functions.filter { function -> function.name == "classify" } + val nonPositiveFunctions = nonPositiveFile.functions.filter { function -> function.name == "classify" } + assertEquals(1, positiveFunctions.size, positiveFile.toString()) + assertEquals(1, nonPositiveFunctions.size, nonPositiveFile.toString()) + assertEquals(1L, positiveFunctions.single().hits) + assertEquals(1L, nonPositiveFunctions.single().hits) + assertEquals(listOf(5), zeroHitBranchLines(positiveFile)) + assertEquals(listOf(2), zeroHitBranchLines(nonPositiveFile)) + assertEquals(1L, statementHitsAtLine(positiveFile, line = 2)) + assertEquals(1L, statementHitsAtLine(nonPositiveFile, line = 2)) + } + + @Test + fun `falsified property retains coverage collected before failure`() { + val result = backend.run( + property = property( + exportName = "failsAfterClassifying", + domain = IntegerDomain(min = 1, max = 1), + ), + configuration = configuration, + ) + + assertEquals(PropertyRunStatus.FAILURE, result.status) + val file = sourceUnderTest(result) + val functions = file.functions.filter { function -> function.name == "classify" } + assertEquals(1, functions.size, file.toString()) + assertEquals(1L, functions.single().hits) + assertEquals(listOf(5), zeroHitBranchLines(file)) + } + + private fun sourceUnderTest(result: org.usvm.ts.pbt.backend.PropertyRunResult): SourceFileCoverage { + val artifact = assertNotNull(result.coverage) + return artifact.files.single { file -> file.path.endsWith("properties/coverage/source-under-test.ts") } + } + + private fun statementHitsAtLine(file: SourceFileCoverage, line: Int): Long = file.statements + .filter { statement -> statement.location.start.line == line } + .maxOf { statement -> statement.hits } + + private fun zeroHitBranchLines(file: SourceFileCoverage): List = file.branches + .filter { branch -> branch.arms.all { arm -> arm.hits == 0L } } + .map { branch -> branch.location.start.line } + .filter { line -> line > 1 } + .sorted() + + private fun property(exportName: String, domain: IntegerDomain) = PropertyDefinition( + id = PropertyId("coverage.$exportName"), + inputs = listOf( + PropertyInput( + name = "value", + domain = domain, + ), + ), + predicate = TypeScriptEntryPoint( + module = "properties/coverage/CoverageProperties.ts", + exportName = exportName, + ), + ) + + private companion object { + val configuration = PropertyRunConfiguration( + seed = 42, + numRuns = 1, + timeoutMillis = 5_000, + coverageRequest = PropertyCoverageRequest(), + ) + + fun adapterEntryPoint(): Path = locateFile( + "fast-check-adapter/dist/src/execution-cli.js", + "usvm-ts-pbt/fast-check-adapter/dist/src/execution-cli.js", + ) + + fun sourceRoot(): Path = locateDirectory( + "src/test/resources", + "usvm-ts-pbt/src/test/resources", + ) + + fun locateFile(vararg candidates: String): Path = candidates + .map { candidate -> Path.of(candidate).absolute() } + .singleOrNull(Files::isRegularFile) + ?: error("Cannot locate file; checked ${candidates.toList()}") + + fun locateDirectory(vararg candidates: String): Path = candidates + .map { candidate -> Path.of(candidate).absolute() } + .singleOrNull(Files::isDirectory) + ?: error("Cannot locate directory; checked ${candidates.toList()}") + } +} diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClientTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClientTest.kt index 60704169d..008bd2d5f 100644 --- a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClientTest.kt +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckProcessClientTest.kt @@ -1,6 +1,7 @@ package org.usvm.ts.pbt.fastcheck import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.backend.PropertyCoverageRequest import org.usvm.ts.pbt.manifest.toManifest import org.usvm.ts.pbt.model.BooleanDomain import org.usvm.ts.pbt.model.PropertyDefinition @@ -8,9 +9,14 @@ import org.usvm.ts.pbt.model.PropertyId import org.usvm.ts.pbt.model.PropertyInput import org.usvm.ts.pbt.model.TypeScriptEntryPoint import org.usvm.ts.pbt.testResourcesRoot +import java.nio.file.Files import java.nio.file.Path +import kotlin.io.path.createDirectories +import kotlin.io.path.createFile +import kotlin.io.path.createTempDirectory import kotlin.io.path.createTempFile import kotlin.io.path.deleteIfExists +import kotlin.io.path.readText import kotlin.io.path.writeText import kotlin.test.assertEquals import kotlin.test.assertFailsWith @@ -30,6 +36,190 @@ class FastCheckProcessClientTest { assertEquals("backend.process.start.failed", startup.code) } + @Test + fun `coverage workspace is removed when process startup fails`() { + val runtime = createTempDirectory(prefix = "fast-check-runtime-") + val adapter = runtime.resolve("dist/src/execution-cli.js") + val c8 = runtime.resolve("node_modules/c8/bin/c8.js") + val fakeNode = runtime.resolve("fake-node") + adapter.parent.createDirectories() + c8.parent.createDirectories() + adapter.createFile() + c8.createFile() + fakeNode.writeText( + """ + #!/bin/sh + if [ "${'$'}1" = "--version" ]; then + printf 'v22.14.0\n' + rm "${'$'}0" + exit 0 + fi + exit 1 + """.trimIndent(), + ) + check(fakeNode.toFile().setExecutable(true)) + val workspacesBefore = coverageWorkspaces() + + try { + val error = assertFailsWith { + FastCheckProcessClient( + nodeExecutable = fakeNode.toString(), + adapterEntryPoint = adapter, + ).check( + validRequest.copy(coverageRequest = PropertyCoverageRequest()), + ) + } + + assertEquals(BackendErrorKind.PROCESS_FAILURE, error.kind) + assertEquals(workspacesBefore, coverageWorkspaces()) + } finally { + runtime.toFile().deleteRecursively() + coverageWorkspaces() + .minus(workspacesBefore) + .forEach { workspace -> workspace.toFile().deleteRecursively() } + } + } + + @Test + fun `coverage rejects an unsupported Node runtime before starting c8`() { + val runtime = createTempDirectory(prefix = "fast-check-runtime-") + val adapter = runtime.resolve("dist/src/execution-cli.js") + val c8 = runtime.resolve("node_modules/c8/bin/c8.js") + val collectorStarted = runtime.resolve("collector-started") + val fakeNode = runtime.resolve("fake-node") + adapter.parent.createDirectories() + c8.parent.createDirectories() + adapter.createFile() + c8.createFile() + fakeNode.writeText( + """ + #!/bin/sh + if [ "${'$'}1" = "--version" ]; then + printf 'v16.20.2\n' + exit 0 + fi + touch "$collectorStarted" + exit 1 + """.trimIndent(), + ) + check(fakeNode.toFile().setExecutable(true)) + + try { + val error = assertFailsWith { + FastCheckProcessClient( + nodeExecutable = fakeNode.toString(), + adapterEntryPoint = adapter, + ).check( + validRequest.copy(coverageRequest = PropertyCoverageRequest()), + ) + } + + assertEquals(BackendErrorKind.COVERAGE, error.kind) + assertEquals("coverage.runtime.unsupported", error.code) + assertTrue(Files.notExists(collectorStarted)) + } finally { + runtime.toFile().deleteRecursively() + } + } + + @Test + fun `coverage collector preserves the caller working directory`() { + val runtime = createTempDirectory(prefix = "fast-check-runtime-") + val adapter = runtime.resolve("dist/src/execution-cli.js") + val c8 = runtime.resolve("node_modules/c8/bin/c8.js") + val collectorDirectory = runtime.resolve("collector-directory") + val fakeNode = runtime.resolve("fake-node") + adapter.parent.createDirectories() + c8.parent.createDirectories() + adapter.createFile() + c8.createFile() + fakeNode.writeText( + """ + #!/bin/sh + if [ "${'$'}1" = "--version" ]; then + printf 'v22.14.0\n' + exit 0 + fi + pwd > "$collectorDirectory" + exit 1 + """.trimIndent(), + ) + check(fakeNode.toFile().setExecutable(true)) + val callerDirectory = Path.of("").toAbsolutePath().normalize() + + try { + val error = assertFailsWith { + FastCheckProcessClient( + nodeExecutable = fakeNode.toString(), + adapterEntryPoint = adapter, + ).check( + validRequest.copy(coverageRequest = PropertyCoverageRequest()), + ) + } + val startedDirectory = Path.of(collectorDirectory.readText().trim()) + + assertEquals(BackendErrorKind.PROCESS_FAILURE, error.kind) + assertEquals(callerDirectory, startedDirectory) + } finally { + runtime.toFile().deleteRecursively() + } + } + + @Test + fun `coverage collector uses an explicit empty configuration`() { + val runtime = createTempDirectory(prefix = "fast-check-runtime-") + val adapter = runtime.resolve("dist/src/execution-cli.js") + val c8 = runtime.resolve("node_modules/c8/bin/c8.js") + val collectorArguments = runtime.resolve("collector-arguments") + val collectorConfiguration = runtime.resolve("collector-configuration") + val fakeNode = runtime.resolve("fake-node") + adapter.parent.createDirectories() + c8.parent.createDirectories() + adapter.createFile() + c8.createFile() + fakeNode.writeText( + """ + #!/bin/sh + if [ "${'$'}1" = "--version" ]; then + printf 'v22.14.0\n' + exit 0 + fi + printf '%s\n' "${'$'}@" > "$collectorArguments" + for argument in "${'$'}@"; do + case "${'$'}argument" in + --config=*) + config_path="${'$'}{argument#--config=}" + cat "${'$'}config_path" > "$collectorConfiguration" + ;; + esac + done + exit 1 + """.trimIndent(), + ) + check(fakeNode.toFile().setExecutable(true)) + + try { + val error = assertFailsWith { + FastCheckProcessClient( + nodeExecutable = fakeNode.toString(), + adapterEntryPoint = adapter, + ).check( + validRequest.copy(coverageRequest = PropertyCoverageRequest()), + ) + } + val configArguments = collectorArguments.readText() + .lineSequence() + .filter { argument -> argument.startsWith("--config=") } + .toList() + + assertEquals(BackendErrorKind.PROCESS_FAILURE, error.kind) + assertEquals(1, configArguments.size) + assertEquals("{}", collectorConfiguration.readText()) + } finally { + runtime.toFile().deleteRecursively() + } + } + @Test fun `non-zero exit retains stderr`() { withTemporaryAdapter(source = "process.stderr.write('adapter failed'); process.exit(3)") { client -> @@ -130,6 +320,13 @@ class FastCheckProcessClientTest { } } + private fun coverageWorkspaces(): Set { + val temporaryRoot = Path.of(System.getProperty("java.io.tmpdir")) + return Files.newDirectoryStream(temporaryRoot, "usvm-ts-pbt-coverage-*").use { entries -> + entries.toHashSet() + } + } + private companion object { data class InvalidResponseCase( val script: String, diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntimeMetadataTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntimeMetadataTest.kt new file mode 100644 index 000000000..f4c7396ec --- /dev/null +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntimeMetadataTest.kt @@ -0,0 +1,49 @@ +package org.usvm.ts.pbt.fastcheck + +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.manifest.PropertyManifestJson +import org.usvm.ts.pbt.testResourcesRoot +import java.nio.file.Files +import java.util.Properties +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + +class FastCheckRuntimeMetadataTest { + @Test + fun `runtime dependency versions are packaged from the adapter lockfile`() { + val adapterRoot = FastCheckRuntime.executionEntryPoint().parent.parent.parent + val packageLock = PropertyManifestJson.json + .parseToJsonElement(Files.readString(adapterRoot.resolve("package-lock.json"))) + .jsonObject + val packages = packageLock.getValue("packages").jsonObject + val expectedFastCheckVersion = packages.getValue("node_modules/fast-check") + .jsonObject + .getValue("version") + .jsonPrimitive + .content + val expectedC8Version = packages.getValue("node_modules/c8") + .jsonObject + .getValue("version") + .jsonPrimitive + .content + + val metadata = Properties().apply { + val resource = assertNotNull( + FastCheckRuntimeMetadataTest::class.java.getResourceAsStream(RUNTIME_METADATA_RESOURCE), + ) + resource.use(::load) + } + val backend = FastCheckBackend(sourceRoots = listOf(testResourcesRoot())) + + assertEquals(expectedFastCheckVersion, metadata.getProperty("fast-check.version")) + assertEquals(expectedC8Version, metadata.getProperty("c8.version")) + assertEquals(expectedFastCheckVersion, backend.coverageCapability.backendVersion) + assertEquals(expectedC8Version, backend.coverageCapability.collector?.version) + } + + private companion object { + const val RUNTIME_METADATA_RESOURCE = "/org/usvm/ts/pbt/fastcheck/runtime-dependencies.properties" + } +} diff --git a/usvm-ts-pbt/src/test/resources/coverage/istanbul/statement-branch.json b/usvm-ts-pbt/src/test/resources/coverage/istanbul/statement-branch.json new file mode 100644 index 000000000..bf272a9f1 --- /dev/null +++ b/usvm-ts-pbt/src/test/resources/coverage/istanbul/statement-branch.json @@ -0,0 +1,73 @@ +{ + "/workspace/src/math.ts": { + "path": "/workspace/src/math.ts", + "statementMap": { + "0": { "start": { "line": 1, "column": 0 }, "end": { "line": 1, "column": 30 } }, + "1": { "start": { "line": 2, "column": 2 }, "end": { "line": 6, "column": 3 } } + }, + "fnMap": { + "0": { + "name": "classify", + "decl": { "start": { "line": 1, "column": 0 }, "end": { "line": 1, "column": 17 } }, + "loc": { "start": { "line": 1, "column": 29 }, "end": { "line": 7, "column": 1 } } + } + }, + "branchMap": { + "0": { + "type": "if", + "loc": { "start": { "line": 2, "column": 2 }, "end": { "line": 6, "column": 3 } }, + "locations": [ + { "start": { "line": 2, "column": 2 }, "end": { "line": 4, "column": 3 } }, + { "start": { "line": 4, "column": 9 }, "end": { "line": 6, "column": 3 } } + ] + } + }, + "s": { "0": 2, "1": 2 }, + "f": { "0": 2 }, + "b": { "0": [1, 1] } + }, + "/workspace/src/property.ts": { + "path": "/workspace/src/property.ts", + "statementMap": { + "0": { "start": { "line": 1, "column": 0 }, "end": { "line": 3, "column": 1 } } + }, + "fnMap": {}, + "branchMap": {}, + "s": { "0": 2 }, + "f": {}, + "b": {} + }, + "/workspace/src/generated-without-map.js": { + "path": "/workspace/src/generated-without-map.js", + "statementMap": { + "0": { "start": { "line": 1, "column": 0 }, "end": { "line": 1, "column": 10 } } + }, + "fnMap": {}, + "branchMap": {}, + "s": { "0": 1 }, + "f": {}, + "b": {} + }, + "/adapter/dist/src/execution-cli.js": { + "path": "/adapter/dist/src/execution-cli.js", + "statementMap": { + "0": { "start": { "line": 1, "column": 0 }, "end": { "line": 1, "column": 10 } } + }, + "fnMap": {}, + "branchMap": {}, + "s": { "0": 1 }, + "f": {}, + "b": {} + }, + "/adapter/node_modules/example/index.js": { + "path": "/adapter/node_modules/example/index.js", + "statementMap": { + "0": { "start": { "line": 1, "column": 0 }, "end": { "line": 1, "column": 10 } } + }, + "fnMap": {}, + "branchMap": {}, + "s": { "0": 1 }, + "f": {}, + "b": {} + } +} diff --git a/usvm-ts-pbt/src/test/resources/properties/coverage/CoverageProperties.ts b/usvm-ts-pbt/src/test/resources/properties/coverage/CoverageProperties.ts new file mode 100644 index 000000000..1b012ceb1 --- /dev/null +++ b/usvm-ts-pbt/src/test/resources/properties/coverage/CoverageProperties.ts @@ -0,0 +1,14 @@ +import { classify } from './source-under-test.ts'; + +export function coversPositive(value: number): boolean { + return classify(value) === 'positive'; +} + +export function coversNonPositive(value: number): boolean { + return classify(value) === 'non-positive'; +} + +export function failsAfterClassifying(value: number): boolean { + classify(value); + return false; +} diff --git a/usvm-ts-pbt/src/test/resources/properties/coverage/package.json b/usvm-ts-pbt/src/test/resources/properties/coverage/package.json new file mode 100644 index 000000000..3dbc1ca59 --- /dev/null +++ b/usvm-ts-pbt/src/test/resources/properties/coverage/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/usvm-ts-pbt/src/test/resources/properties/coverage/source-under-test.ts b/usvm-ts-pbt/src/test/resources/properties/coverage/source-under-test.ts new file mode 100644 index 000000000..eddc082a4 --- /dev/null +++ b/usvm-ts-pbt/src/test/resources/properties/coverage/source-under-test.ts @@ -0,0 +1,6 @@ +export function classify(value: number): string { + if (value > 0) { + return 'positive'; + } + return 'non-positive'; +}