From 5f3792b92dda7589544692cd2aabf8bfc9d8b4a9 Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Mon, 19 Oct 2026 14:24:56 +0200 Subject: [PATCH 1/3] Mark missing ops in source and surface them in the coverage matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Known gaps (TransposedConv2d, GroupNormalization, SDPA backward, missing StableHLO converters) only showed up when code threw at runtime; the ops status matrix had "โ€”" in almost every backend cell and no layer, lowering or autograd view. - @NotImplemented gains reason/issue; the docs KSP processor gives it precedence over @InProgress and over a detected backend override, and reads method-level markers on @Backend classes. - The processor now also runs in skainet-backend-cpu; generateDocs merges its statusByBackend, so the cpu column reflects real overrides. - Matrix gains StableHLO and Backward columns (from jvmTest dumps of the converter registry and the backward dispatch table) and an NN-layers table; โŒ cells link the tracking issue. - TensorOps.convTranspose2d added as a marked placeholder; TransposedConv2d and GroupNormalization marked @NotImplemented. - Removed stale @InProgress markers on implemented DefaultCpuOps methods and the fake Metal markers in VoidTensorOps. - checkUnmarkedStubs fails on NotImplementedError / TODO( in production code that is neither marked nor in config/stub-allowlist.txt (new CI leg stub-markers); docs.yml fails on a stale committed matrix. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 4 + .github/workflows/docs.yml | 29 ++- .github/workflows/documentation.yml | 3 +- .../src/main/kotlin/CheckUnmarkedStubsTask.kt | 151 ++++++++++++++ .../src/main/kotlin/DocumentationExtension.kt | 18 ++ .../src/main/kotlin/DocumentationPlugin.kt | 21 ++ .../main/kotlin/GenerateDocumentationTask.kt | 162 ++++++++++++++- .../main/kotlin/models/DocumentationModels.kt | 19 +- .../schemas/operator-doc-schema-v1.json | 37 +++- build.gradle.kts | 11 +- config/stub-allowlist.txt | 33 +++ .../pages/contributing/darc-workflow.adoc | 4 +- .../pages/contributing/issue-taxonomy.adoc | 23 +++ .../pages/explanation/operator-design.adoc | 6 +- .../reference/operators/generated/index.adoc | 2 +- .../operators/generated/similarity.adoc | 2 +- .../operators/generated/tensorops.adoc | 30 +++ .../pages/reference/ops-status-matrix.adoc | 193 +++++++++++------- .../api/jvm/skainet-backend-cpu.api | 1 + .../skainet-backend-cpu/build.gradle.kts | 14 ++ .../sk/ainet/exec/tensor/ops/DefaultCpuOps.kt | 36 ---- .../ainet/lang/graph/DefaultExecutionTape.kt | 9 + .../exec/autograd/BackwardCoverageDumpTest.kt | 34 +++ .../ainet/compile/hlo/StableHloConverter.kt | 3 + .../ainet/compile/hlo/HloCoverageDumpTest.kt | 21 ++ .../api/jvm/skainet-lang-core.api | 7 + .../sk/ainet/lang/nn/TransposedConv2d.kt | 6 +- .../nn/normalization/GroupNormalization.kt | 2 + .../sk/ainet/lang/tensor/ops/TensorOps.kt | 22 ++ .../sk/ainet/lang/tensor/ops/VoidTensorOps.kt | 3 - .../api/jvm/skainet-lang-dag.api | 1 + .../kotlin/sk/ainet/lang/ops/TensorOp.kt | 21 +- .../lang/ops/ksp/OperatorDocProcessor.kt | 173 +++++++++++++++- .../lang/ops/ksp/OperatorDocProcessorTest.kt | 83 +++++++- 34 files changed, 1037 insertions(+), 147 deletions(-) create mode 100644 build-logic/convention/src/main/kotlin/CheckUnmarkedStubsTask.kt create mode 100644 config/stub-allowlist.txt create mode 100644 skainet-compile/skainet-compile-dag/src/jvmTest/kotlin/sk/ainet/exec/autograd/BackwardCoverageDumpTest.kt create mode 100644 skainet-compile/skainet-compile-hlo/src/jvmTest/kotlin/sk/ainet/compile/hlo/HloCoverageDumpTest.kt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b8f340c9c..0db0e143b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,6 +76,10 @@ jobs: # associates with dumps (#1171 merged a stale dump exactly that way; #1174 repaired it). - name: api-compatibility tasks: apiCheck + # Every NotImplementedError / TODO( in production code is either @NotImplemented + # (so it shows in the ops coverage matrix) or listed in config/stub-allowlist.txt. + - name: stub-markers + tasks: checkUnmarkedStubs permissions: contents: read diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index fa1ccc0e2..13eeebadb 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -25,14 +25,20 @@ on: - '.github/workflows/docs.yml' - 'build.gradle.kts' - 'build-logic/**' - - 'skainet-lang/skainet-lang-core/**' + - 'skainet-lang/**' + - 'skainet-backends/skainet-backend-cpu/**' + - 'skainet-compile/skainet-compile-hlo/**' + - 'skainet-compile/skainet-compile-dag/**' pull_request: paths: - 'docs/**' - '.github/workflows/docs.yml' - 'build.gradle.kts' - 'build-logic/**' - - 'skainet-lang/skainet-lang-core/**' + - 'skainet-lang/**' + - 'skainet-backends/skainet-backend-cpu/**' + - 'skainet-compile/skainet-compile-hlo/**' + - 'skainet-compile/skainet-compile-dag/**' workflow_dispatch: concurrency: @@ -81,8 +87,25 @@ jobs: # bundle it; running both here means commit 6 is a pure # workflow-step + Gradle-task-registration change with no # Gradle re-run cost. + # The StableHLO / Backward matrix columns come from two jvmTest dumps + # (HloCoverageDumpTest, BackwardCoverageDumpTest); run just those first. - name: Generate operator docs and Dokka - run: ./gradlew --no-daemon generateDocs dokkaGenerate + run: >- + ./gradlew --no-daemon + :skainet-compile:skainet-compile-hlo:jvmTest --tests '*HloCoverageDumpTest' + :skainet-compile:skainet-compile-dag:jvmTest --tests '*BackwardCoverageDumpTest' + generateDocs dokkaGenerate + + # The committed coverage matrix and operator pages must match the code: + # a new @NotImplemented, a new backend override or a new StableHLO + # converter has to land together with the regenerated page. The + # "Generated from โ€ฆ on " stamp is ignored. + - name: Check generated docs are up to date + run: | + if ! git diff --exit-code -I 'Generated from' -- docs/modules/ROOT/pages/reference/; then + echo "::error::Generated operator docs are stale. Run the generateDocs command from build.gradle.kts (documentation block) and commit docs/modules/ROOT/pages/reference/." + exit 1 + fi - name: Set up Docker Buildx uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4.4.0 diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 3f4882f1f..296b97361 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -62,7 +62,8 @@ jobs: with: name: operator-documentation path: | - docs/modules/operators/_generated_/** + docs/modules/ROOT/pages/reference/operators/generated/** + docs/modules/ROOT/pages/reference/ops-status-matrix.adoc skainet-lang/skainet-lang-core/build/generated/ksp/metadata/commonMain/resources/operators.json retention-days: 30 diff --git a/build-logic/convention/src/main/kotlin/CheckUnmarkedStubsTask.kt b/build-logic/convention/src/main/kotlin/CheckUnmarkedStubsTask.kt new file mode 100644 index 000000000..aca9f81d5 --- /dev/null +++ b/build-logic/convention/src/main/kotlin/CheckUnmarkedStubsTask.kt @@ -0,0 +1,151 @@ +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.tasks.* +import java.io.File + +/** + * Fails the build when production Kotlin code throws `NotImplementedError` / calls `TODO(` + * without the gap being declared, so every missing op or layer shows up in the operator + * coverage matrix and has a tracking issue. + * + * A stub is declared when either + * - it sits inside a declaration annotated `@NotImplemented` / `@InProgress` (the KSP docs + * processor renders these as โŒ / ๐Ÿšง), or + * - its file is listed in [allowlistFile] as `path|count|issue or reason`. Non-op gaps + * (loaders, platform actuals, DSL) use this; `count` is the exact number of unmarked stubs + * expected in the file, so adding one more fails until the entry is updated. + * + * Comment lines are ignored, so `// TODO:` notes are not stubs. + */ +@CacheableTask +abstract class CheckUnmarkedStubsTask : DefaultTask() { + + @get:InputFiles + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val sources: ConfigurableFileCollection + + /** Repository root that report and allowlist paths are relative to. */ + @get:Internal + abstract val rootDirectory: DirectoryProperty + + @get:InputFile + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val allowlistFile: RegularFileProperty + + @get:OutputFile + abstract val reportFile: RegularFileProperty + + private val stubCall = Regex("""\bNotImplementedError\s*\(|\bTODO\s*\(""") + private val mainSourceDir = Regex("""/src/[A-Za-z0-9]*[Mm]ain/""") + private val marker = Regex("""^\s*@(NotImplemented|InProgress)\b""") + + @TaskAction + fun check() { + val root = rootDirectory.get().asFile + val allowlist = parseAllowlist(allowlistFile.get().asFile) + val unmarked = mutableMapOf>() + + sources.files + .filter { it.extension == "kt" } + .forEach { file -> + val rel = file.relativeTo(root).invariantSeparatorsPath + if (!mainSourceDir.containsMatchIn("/$rel")) return@forEach + val lines = file.readLines() + val covered = markedRanges(lines) + lines.forEachIndexed { index, line -> + val trimmed = line.trimStart() + if (trimmed.startsWith("//") || trimmed.startsWith("*") || trimmed.startsWith("/*")) return@forEachIndexed + // Mentions inside string literals (messages, regexes) are not calls. + if (!stubCall.containsMatchIn(stripStrings(line))) return@forEachIndexed + if (covered.any { index in it }) return@forEachIndexed + unmarked.getOrPut(rel) { mutableListOf() } += index + 1 + } + } + + val problems = mutableListOf() + unmarked.toSortedMap().forEach { (path, hits) -> + val expected = allowlist[path] + when { + expected == null -> hits.forEach { problems += "$path:$it โ€” unmarked stub" } + expected != hits.size -> problems += "$path โ€” ${hits.size} unmarked stubs, allowlist expects $expected (lines ${hits.joinToString()})" + } + } + allowlist.keys.filter { it !in unmarked }.sorted().forEach { path -> + problems += "$path โ€” allowlisted but has no unmarked stubs any more; remove the entry" + } + + reportFile.get().asFile.apply { parentFile.mkdirs() }.writeText( + unmarked.toSortedMap().entries.joinToString("\n", postfix = "\n") { (path, hits) -> "$path ${hits.joinToString(",")}" }, + ) + + if (problems.isNotEmpty()) { + throw GradleException( + buildString { + appendLine("Unmarked NotImplementedError / TODO( stubs in production code:") + problems.forEach { appendLine(" $it") } + appendLine() + appendLine("Mark a missing op or layer with @NotImplemented(reason = โ€ฆ, issue = โ€ฆ) so it shows in") + appendLine("docs/modules/ROOT/pages/reference/ops-status-matrix.adoc, or add/update the file's entry in") + appendLine("${allowlistFile.get().asFile.relativeTo(root).invariantSeparatorsPath} (path|count|issue).") + }, + ) + } + logger.lifecycle("โœ… No unmarked stubs (${unmarked.values.sumOf { it.size }} allowlisted in ${unmarked.size} files)") + } + + /** Blank out the contents of `"โ€ฆ"` literals on one line (escape-aware). */ + private fun stripStrings(line: String): String = buildString { + var inString = false + var i = 0 + while (i < line.length) { + val ch = line[i] + when { + inString && ch == '\\' -> i++ + ch == '"' -> { inString = !inString; append(ch) } + !inString -> append(ch) + } + i++ + } + } + + private fun parseAllowlist(file: File): Map = + file.readLines() + .map { it.trim() } + .filter { it.isNotEmpty() && !it.startsWith("#") } + .associate { line -> + val parts = line.split("|").map { it.trim() } + if (parts.size < 3 || parts[2].isEmpty()) { + throw GradleException("Malformed allowlist line (want path|count|issue or reason): $line") + } + parts[0] to (parts[1].toIntOrNull() ?: throw GradleException("Bad count in allowlist line: $line")) + } + + /** + * Line ranges (0-based, inclusive) of declarations carrying a status marker: from the + * annotation to the matching closing brace, or to the `= throw` / `= TODO(` line of an + * expression-bodied function. + */ + private fun markedRanges(lines: List): List { + val ranges = mutableListOf() + lines.forEachIndexed { start, line -> + if (!marker.containsMatchIn(line)) return@forEachIndexed + var depth = 0 + var opened = false + var end = lines.lastIndex + loop@ for (i in start until lines.size) { + val text = lines[i] + if (!opened && (text.contains("= throw") || text.contains("= TODO("))) { end = i; break@loop } + for (ch in text) { + if (ch == '{') { depth++; opened = true } + if (ch == '}') depth-- + if (opened && depth == 0) { end = i; break@loop } + } + } + ranges += start..end + } + return ranges + } +} diff --git a/build-logic/convention/src/main/kotlin/DocumentationExtension.kt b/build-logic/convention/src/main/kotlin/DocumentationExtension.kt index d58fe7fcd..ee07208f0 100644 --- a/build-logic/convention/src/main/kotlin/DocumentationExtension.kt +++ b/build-logic/convention/src/main/kotlin/DocumentationExtension.kt @@ -1,5 +1,6 @@ import models.DocumentationFormat import org.gradle.api.Project +import org.gradle.api.file.ConfigurableFileCollection import org.gradle.api.file.DirectoryProperty import org.gradle.api.file.RegularFileProperty import org.gradle.api.provider.Property @@ -15,6 +16,23 @@ open class DocumentationExtension(project: Project) { val generateIndex: Property = project.objects.property(Boolean::class.java) .convention(true) + /** + * operators.json files emitted by running the docs KSP processor in backend modules + * (e.g. skainet-backend-cpu). Only their `statusByBackend` maps and notes are merged into + * [inputFile]; missing files are skipped. + */ + val backendStatusFiles: ConfigurableFileCollection = project.objects.fileCollection() + + /** hlo-coverage.json (op names the StableHLO converter lowers), from HloCoverageDumpTest. */ + val hloCoverageFile: RegularFileProperty = project.objects.fileProperty() + + /** backward-coverage.json (op name โ†’ implemented / stubbed / missing), from BackwardCoverageDumpTest. */ + val backwardCoverageFile: RegularFileProperty = project.objects.fileProperty() + + /** Base URL issue references like `#123` resolve against in rendered matrix cells. */ + val issueBaseUrl: Property = project.objects.property(String::class.java) + .convention("https://github.com/SKaiNET-developers/SKaiNET/issues/") + /** kernel-support.json emitted by KernelSupportMatrixTest (registry introspection). */ val kernelInputFile: RegularFileProperty = project.objects.fileProperty() diff --git a/build-logic/convention/src/main/kotlin/DocumentationPlugin.kt b/build-logic/convention/src/main/kotlin/DocumentationPlugin.kt index 265768a74..df18da929 100644 --- a/build-logic/convention/src/main/kotlin/DocumentationPlugin.kt +++ b/build-logic/convention/src/main/kotlin/DocumentationPlugin.kt @@ -17,6 +17,10 @@ class DocumentationPlugin : Plugin { task.format.set(extension.format) task.includeBackendStatus.set(extension.includeBackendStatus) task.generateIndex.set(extension.generateIndex) + task.backendStatusFiles.from(extension.backendStatusFiles) + task.hloCoverageFile.set(extension.hloCoverageFile) + task.backwardCoverageFile.set(extension.backwardCoverageFile) + task.issueBaseUrl.set(extension.issueBaseUrl) } }) @@ -31,6 +35,23 @@ class DocumentationPlugin : Plugin { } }) + // Guard: every NotImplementedError / TODO( in production code is either inside a + // @NotImplemented / @InProgress declaration (and so in the coverage matrix) or allowlisted. + project.tasks.register("checkUnmarkedStubs", CheckUnmarkedStubsTask::class.java, object : Action { + override fun execute(task: CheckUnmarkedStubsTask) { + task.group = "verification" + task.description = "Fail on NotImplementedError / TODO( stubs that are neither @NotImplemented nor allowlisted" + val root = project.rootProject.layout.projectDirectory + task.sources.from(root.asFileTree.matching { + include("**/src/**/*.kt") + exclude("**/build/**", ".gradle/**", "**/node_modules/**") + }) + task.rootDirectory.set(root) + task.allowlistFile.set(root.file("config/stub-allowlist.txt")) + task.reportFile.set(project.layout.buildDirectory.file("reports/stubs/unmarked-stubs.txt")) + } + }) + // Register schema validation task in plugin (migrated from skainet-lang-export-ops) val validateTaskProvider = project.tasks.register("validateOperatorSchema", SchemaValidationTask::class.java, object : Action { override fun execute(task: SchemaValidationTask) { diff --git a/build-logic/convention/src/main/kotlin/GenerateDocumentationTask.kt b/build-logic/convention/src/main/kotlin/GenerateDocumentationTask.kt index e6f540819..081369cbb 100644 --- a/build-logic/convention/src/main/kotlin/GenerateDocumentationTask.kt +++ b/build-logic/convention/src/main/kotlin/GenerateDocumentationTask.kt @@ -1,6 +1,7 @@ import models.* import kotlinx.serialization.json.Json import org.gradle.api.DefaultTask +import org.gradle.api.file.ConfigurableFileCollection import org.gradle.api.file.DirectoryProperty import org.gradle.api.file.RegularFileProperty import org.gradle.api.provider.Property @@ -34,6 +35,28 @@ abstract class GenerateDocumentationTask : DefaultTask() { @get:Input @get:Optional abstract val generateIndex: Property + + /** Backend-module operators.json files whose `statusByBackend` maps are merged in. */ + @get:InputFiles + @get:Optional + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val backendStatusFiles: ConfigurableFileCollection + + /** Op names the StableHLO converter lowers; drives the StableHLO column when present. */ + @get:InputFiles + @get:Optional + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val hloCoverageFile: RegularFileProperty + + /** Op name โ†’ implemented / stubbed / missing; drives the Backward column when present. */ + @get:InputFiles + @get:Optional + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val backwardCoverageFile: RegularFileProperty + + @get:Input + @get:Optional + abstract val issueBaseUrl: Property @TaskAction fun generateDocumentation() { @@ -45,7 +68,7 @@ abstract class GenerateDocumentationTask : DefaultTask() { val jsonContent = input.readText() val json = Json { ignoreUnknownKeys = true } - val module = json.decodeFromString(jsonContent) + val module = expandAllBackends(mergeBackendStatus(json.decodeFromString(jsonContent), json)) when (format.get()) { DocumentationFormat.ASCIIDOC -> generateAsciidoc(module, output) @@ -63,6 +86,75 @@ abstract class GenerateDocumentationTask : DefaultTask() { } } + /** + * Merge `statusByBackend` and notes from backend-module operators.json files. Backend + * modules see `TensorOps` only as a library class, so their SOURCE-retained annotations + * (`@Diff`, `@DarcValidated`, `@Lowering`) are lost there โ€” only status is taken from them. + */ + private fun mergeBackendStatus(module: OperatorDocModule, json: Json): OperatorDocModule { + val fragments = backendStatusFiles.files.filter { it.isFile } + if (fragments.isEmpty()) return module + val byFunction = mutableMapOf, FunctionDoc>() + fragments.forEach { file -> + logger.lifecycle("๐Ÿ”— Merging backend status from: ${file.absolutePath}") + json.decodeFromString(file.readText()).operators.forEach { op -> + op.functions.forEach { fn -> byFunction[op.name to fn.name] = fn } + } + } + return module.copy(operators = module.operators.map { op -> + op.copy(functions = op.functions.map { fn -> + val extra = byFunction[op.name to fn.name] ?: return@map fn + val status = fn.statusByBackend.toMutableMap() + extra.statusByBackend.forEach { (backend, value) -> status[backend] = mergeStatus(status[backend], value) } + fn.copy(statusByBackend = status, notes = fn.notes + extra.notes) + }) + }) + } + + /** Expand a `"*"` status (marker without a backend list) onto every backend column. */ + private fun expandAllBackends(module: OperatorDocModule): OperatorDocModule { + val backends = module.operators.flatMap { op -> op.functions.flatMap { it.statusByBackend.keys } }.toSet() - "*" + return module.copy(operators = module.operators.map { op -> + op.copy(functions = op.functions.map { fn -> + val all = fn.statusByBackend["*"] ?: return@map fn + val status = fn.statusByBackend.toMutableMap() + status.remove("*") + backends.forEach { backend -> status[backend] = mergeStatus(status[backend], all) } + fn.copy(statusByBackend = status) + }) + }) + } + + /** Status precedence: not_implemented > in_progress > implemented > inherited. */ + private fun mergeStatus(current: String?, incoming: String): String { + val rank = listOf("not_implemented", "in_progress", "implemented", "inherited") + if (current == null) return incoming + val a = rank.indexOf(current).let { if (it < 0) rank.size else it } + val b = rank.indexOf(incoming).let { if (it < 0) rank.size else it } + return if (b < a) incoming else current + } + + private fun readHloCoverage(): Set? { + val file = hloCoverageFile.orNull?.asFile?.takeIf { it.isFile } ?: return null + return Json.decodeFromString>(file.readText()).map { it.lowercase() }.toSet() + } + + private fun readBackwardCoverage(): Map? { + val file = backwardCoverageFile.orNull?.asFile?.takeIf { it.isFile } ?: return null + return Json.decodeFromString>(file.readText()) + } + + /** Render an issue reference (`#123`, a URL, or free text) as an AsciiDoc link. */ + private fun issueLink(issue: String): String { + val trimmed = issue.trim() + return when { + trimmed.startsWith("http") -> "$trimmed[${trimmed.substringAfterLast('/').let { if (it.all(Char::isDigit)) "#$it" else "issue" }}]" + trimmed.startsWith("#") && trimmed.drop(1).all(Char::isDigit) -> + "${issueBaseUrl.getOrElse("https://github.com/SKaiNET-developers/SKaiNET/issues/")}${trimmed.drop(1)}[$trimmed]" + else -> trimmed + } + } + private fun generateAsciidoc(module: OperatorDocModule, outputDir: File) { outputDir.mkdirs() @@ -105,8 +197,11 @@ abstract class GenerateDocumentationTask : DefaultTask() { // column order is stable across runs. val allBackends: List = module.operators .flatMap { op -> op.functions.flatMap { it.statusByBackend.keys } } + .filter { it != "*" } .toSortedSet() .toList() + val hloOps = readHloCoverage() + val backward = readBackwardCoverage() // Row view carries the per-function status plus DARC validation // signal so the matrix can render both in one pass. @@ -130,7 +225,13 @@ abstract class GenerateDocumentationTask : DefaultTask() { appendLine("") appendLine("Generated from `operators.json` version `${module.version}` on ${formatTimestamp(module.timestamp)}.") appendLine("") - appendLine("Rows are `Operator.function` pairs. The `Validated` column shows whether the function's documentation has been DARC-validated by a reviewer (see xref:contributing/darc-workflow.adoc[DARC workflow]). The `Lowering` column lists the compile-path forms the function declares with `@Lowering` (`composite` = a `stablehlo.composite` when a target keeps it fused, `decompose` = StableHLO/CHLO primitives, `native` = a 1:1 target op); `โ€”` means undeclared. Remaining columns are backends that appear in any function's `statusByBackend` map โ€” a missing entry means the backend makes no claim about the function (treat it as \"unknown\", not \"not supported\").") + appendLine("Rows are `Operator.function` pairs. A gap is marked in source with `@NotImplemented(reason = โ€ฆ, issue = โ€ฆ)` (or `@InProgress`) and shows up here as โŒ (or ๐Ÿšง) linked to its tracking issue; the PR that closes the gap removes the annotation. See xref:contributing/issue-taxonomy.adoc[Issue taxonomy] to pick one up.") + appendLine("") + appendLine("* *Validated* โ€” whether the function's documentation has been DARC-validated by a reviewer (see xref:contributing/darc-workflow.adoc[DARC workflow]).") + appendLine("* *Lowering* โ€” the compile-path forms the function declares with `@Lowering` (`composite` = a `stablehlo.composite` when a target keeps it fused, `decompose` = StableHLO/CHLO primitives, `native` = a 1:1 target op); `โ€”` means undeclared.") + if (hloOps != null) appendLine("* *StableHLO* โ€” โœ… the StableHLO converter can lower the op; โŒ export fails with \"No converter found\".") + if (backward != null) appendLine("* *Backward* โ€” for `@Diff` ops: โœ… a backward rule is wired; โŒ the rule is missing or a stub returning null gradients; `โ€”` the op is not differentiable.") + appendLine("* *Backend columns* โ€” โœ… the backend overrides the function; โ†ช the backend relies on the `TensorOps` default; ๐Ÿšง in progress; โŒ not implemented; `โ€”` the backend makes no claim (unknown, not \"unsupported\").") appendLine("") if (rows.isEmpty() || allBackends.isEmpty()) { appendLine("NOTE: No backend status information found in the source data.") @@ -139,10 +240,12 @@ abstract class GenerateDocumentationTask : DefaultTask() { } // Header: row label, validation column, then one column per backend. - val colSpec = (listOf("2", "1", "1") + List(allBackends.size) { "1" }).joinToString(",") + val extraCols = listOfNotNull(hloOps?.let { "StableHLO" }, backward?.let { "Backward" }) + val colSpec = (listOf("2", "1", "1") + List(extraCols.size + allBackends.size) { "1" }).joinToString(",") appendLine("[cols=\"$colSpec\", options=\"header\"]") appendLine("|===") append("| Operator.function | Validated | Lowering ") + extraCols.forEach { append("| $it ") } allBackends.forEach { append("| $it ") } appendLine("") appendLine("") @@ -151,10 +254,14 @@ abstract class GenerateDocumentationTask : DefaultTask() { append("| `${row.operator}.${row.function.name}` ") append("| ${darcCell(row.function, row.hasPartial)} ") append("| ${row.function.lowering.ifEmpty { "โ€”" }} ") + if (hloOps != null) append("| ${if (row.function.name.lowercase() in hloOps) "โœ…" else "โŒ"} ") + if (backward != null) append("| ${backwardCell(row.function, backward)} ") allBackends.forEach { backend -> val raw = row.function.statusByBackend[backend] val cell = if (raw == null) "โ€”" else shortStatus(raw) - append("| $cell ") + val issue = row.function.notes.firstOrNull { it.type == "issue" && (it.backend == backend || it.backend == "*") } + val link = if (issue != null && raw != null && !isDone(raw)) " ${issueLink(issue.message)}" else "" + append("| $cell$link ") } appendLine("") } @@ -164,6 +271,11 @@ abstract class GenerateDocumentationTask : DefaultTask() { val validatedCount = rows.count { it.function.validated } val loweringCount = rows.count { it.function.lowering.isNotEmpty() } append("| *Done* | *$validatedCount / ${rows.size}* | *$loweringCount / ${rows.size}* ") + if (hloOps != null) append("| *${rows.count { it.function.name.lowercase() in hloOps }} / ${rows.size}* ") + if (backward != null) { + val diffRows = rows.filter { it.function.isDifferentiable } + append("| *${diffRows.count { backward[it.function.name] == "implemented" }} / ${diffRows.size}* ") + } allBackends.forEach { backend -> val n = rows.count { isDone(it.function.statusByBackend[backend]) } append("| *$n / ${rows.size}* ") @@ -172,9 +284,44 @@ abstract class GenerateDocumentationTask : DefaultTask() { appendLine("|===") appendLine("") appendLine("Per-function detail including notes lives in xref:reference/operators/generated/index.adoc[Operator reference].") + appendLayersTable(module.layers) }) } + private fun backwardCell(function: FunctionDoc, backward: Map): String { + if (!function.isDifferentiable) return "โ€”" + return when (backward[function.name]) { + "implemented" -> "โœ…" + "stubbed" -> "โŒ stub" + else -> "โŒ" + } + } + + /** `nn` layer table: one row per concrete `Module` subclass, โŒ when `@NotImplemented`. */ + private fun StringBuilder.appendLayersTable(layers: List) { + if (layers.isEmpty()) return + appendLine("") + appendLine("== NN layers") + appendLine("") + appendLine("Every concrete `sk.ainet.lang.nn` `Module` subclass. โŒ means the layer's `forward` throws today; the class carries `@NotImplemented`.") + appendLine("") + appendLine("[cols=\"2,3,1,3\", options=\"header\"]") + appendLine("|===") + appendLine("| Layer | Package | Status | Missing") + appendLine("") + layers.forEach { layer -> + val missing = listOf(layer.reason, layer.issue.takeIf { it.isNotBlank() }?.let { issueLink(it) }.orEmpty()) + .filter { it.isNotBlank() } + .joinToString(" โ€” ") + .ifEmpty { "โ€”" } + appendLine("| `${layer.name}` | `${layer.packageName}` | ${shortStatus(layer.status)} | $missing") + } + appendLine("") + val done = layers.count { isDone(it.status) } + appendLine("| *Done* | | *$done / ${layers.size}* |") + appendLine("|===") + } + /** * One-line DARC validation badge rendered above each function's * signature on the generated page. @@ -225,7 +372,8 @@ abstract class GenerateDocumentationTask : DefaultTask() { private fun shortStatus(status: String): String = when (status.lowercase()) { "supported", "implemented", "done" -> "โœ…" "partial" -> "โš ๏ธ" - "not_supported", "missing", "unsupported" -> "โŒ" + "not_supported", "not_implemented", "missing", "unsupported" -> "โŒ" + "inherited" -> "โ†ช" "planned" -> "โณ" "in_progress", "wip" -> "๐Ÿšง" else -> status @@ -415,7 +563,9 @@ abstract class GenerateDocumentationTask : DefaultTask() { appendLine("=== Notes") appendLine("") function.notes.forEach { note -> - appendLine("TIP: *${note.backend}*: ${note.message}") + val who = if (note.backend == "*") "all backends" else note.backend + val text = if (note.type == "issue") issueLink(note.message) else note.message + appendLine("TIP: *$who*: $text") appendLine("") } } diff --git a/build-logic/convention/src/main/kotlin/models/DocumentationModels.kt b/build-logic/convention/src/main/kotlin/models/DocumentationModels.kt index 758995073..207dac36e 100644 --- a/build-logic/convention/src/main/kotlin/models/DocumentationModels.kt +++ b/build-logic/convention/src/main/kotlin/models/DocumentationModels.kt @@ -9,7 +9,18 @@ data class OperatorDocModule( val commit: String, val timestamp: String, val module: String, - val operators: List + val operators: List, + /** nn `Module` subclasses with their implementation status. */ + val layers: List = emptyList(), +) + +@Serializable +data class LayerDoc( + val name: String, + val packageName: String, + val status: String, + val reason: String = "", + val issue: String = "", ) @Serializable @@ -30,6 +41,8 @@ data class FunctionDoc( val returnType: String, val statusByBackend: Map = emptyMap(), val notes: List = emptyList(), + /** Whether the op carries `@Diff`, i.e. must have a wired backward rule. */ + val isDifferentiable: Boolean = false, val validated: Boolean = false, val validatedBy: String = "", val validatedOn: String = "", @@ -47,9 +60,11 @@ data class ParameterDoc( ) @Serializable - data class Note( +data class Note( val type: String = "", val backend: String = "", + // The KSP processor emits the note text as `content`. + @kotlinx.serialization.json.JsonNames("content") val message: String = "" ) diff --git a/build-logic/convention/src/main/resources/schemas/operator-doc-schema-v1.json b/build-logic/convention/src/main/resources/schemas/operator-doc-schema-v1.json index ca7f81fe4..923e683bb 100644 --- a/build-logic/convention/src/main/resources/schemas/operator-doc-schema-v1.json +++ b/build-logic/convention/src/main/resources/schemas/operator-doc-schema-v1.json @@ -37,6 +37,13 @@ "$ref": "#/$defs/OperatorDoc" }, "description": "Array of operator documentation objects" + }, + "layers": { + "type": "array", + "items": { + "$ref": "#/$defs/LayerDoc" + }, + "description": "nn Module subclasses with their implementation status (sourced from @NotImplemented on the class)" } }, "required": ["schema", "version", "commit", "timestamp", "module", "operators"], @@ -99,14 +106,14 @@ "statusByBackend": { "type": "object", "patternProperties": { - "^[a-zA-Z][a-zA-Z0-9_]*$": { + "^([a-zA-Z][a-zA-Z0-9_]*|\\*)$": { "type": "string", "enum": ["implemented", "inherited", "in_progress", "not_implemented"], "description": "Implementation status for this backend" } }, "additionalProperties": false, - "description": "Map of backend names to implementation status" + "description": "Map of backend names to implementation status. The key '*' means every backend (a @NotImplemented on the interface function with no backend list)." }, "notes": { "type": "array", @@ -173,13 +180,13 @@ "properties": { "type": { "type": "string", - "enum": ["owner", "issue", "description"], + "enum": ["owner", "issue", "description", "reason"], "description": "Type of the note" }, "backend": { "type": "string", - "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", - "description": "Backend this note applies to" + "pattern": "^([a-zA-Z][a-zA-Z0-9_]*|\\*)$", + "description": "Backend this note applies to ('*' = every backend, 'all' = backend-agnostic)" }, "content": { "type": "string", @@ -189,6 +196,26 @@ }, "required": ["type", "backend", "content"], "additionalProperties": false + }, + "LayerDoc": { + "type": "object", + "properties": { + "name": { "type": "string", "minLength": 1, "description": "Simple class name of the layer" }, + "packageName": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]*(\\.[a-z][a-z0-9_]*)*$", + "description": "Fully qualified package name" + }, + "status": { + "type": "string", + "enum": ["implemented", "in_progress", "not_implemented"], + "description": "Whether the layer's forward pass works" + }, + "reason": { "type": "string", "description": "What is missing, from @NotImplemented(reason=...)" }, + "issue": { "type": "string", "description": "Tracking issue, from @NotImplemented(issue=...)" } + }, + "required": ["name", "packageName", "status"], + "additionalProperties": false } } } diff --git a/build.gradle.kts b/build.gradle.kts index b393c2edf..4a0a8853d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -133,7 +133,7 @@ tasks.register("generateOperatorDocs") { // Configure inputs for incremental builds inputs.files("skainet-lang/skainet-lang-core/build/generated/ksp/metadata/commonMain/resources/operators.json") // Configure outputs for incremental builds - outputs.dir("docs/modules/operators/_generated_") + outputs.dir("docs/modules/ROOT/pages/reference/operators/generated") outputs.cacheIf { true } // Depend on KSP processing @@ -155,6 +155,14 @@ documentation { outputDirectory.set(file("docs/modules/ROOT/pages/reference/operators/generated")) includeBackendStatus.set(true) generateIndex.set(true) + // Backend `statusByBackend` maps + StableHLO / backward coverage dumps for the matrix. + // The two coverage JSONs come from jvmTests; refresh them before generateDocs with: + // ./gradlew :skainet-compile:skainet-compile-hlo:jvmTest --tests '*HloCoverageDumpTest' \ + // :skainet-compile:skainet-compile-dag:jvmTest --tests '*BackwardCoverageDumpTest' generateDocs + // (without them the StableHLO / Backward columns are omitted). + backendStatusFiles.from(file("skainet-backends/skainet-backend-cpu/build/generated/ksp/metadata/commonMain/resources/operators.json")) + hloCoverageFile.set(file("skainet-compile/skainet-compile-hlo/build/generated/coverage/hlo-coverage.json")) + backwardCoverageFile.set(file("skainet-compile/skainet-compile-dag/build/generated/coverage/backward-coverage.json")) // Kernel ร— platform matrix โ€” registry-introspected JSON (emitted by KernelSupportMatrixTest) // rendered to an Antora page, mirroring the operators.json โ†’ ops-status-matrix.adoc pipeline. kernelInputFile.set(file("skainet-backends/skainet-backend-native-cpu/build/generated/kernel-support/kernel-support.json")) @@ -163,6 +171,7 @@ documentation { tasks.named("generateDocs") { dependsOn(":skainet-lang:skainet-lang-core:kspCommonMainKotlinMetadata") + dependsOn(":skainet-backends:skainet-backend-cpu:kspCommonMainKotlinMetadata") } tasks.named("generateKernelMatrix") { diff --git a/config/stub-allowlist.txt b/config/stub-allowlist.txt new file mode 100644 index 000000000..a2b215472 --- /dev/null +++ b/config/stub-allowlist.txt @@ -0,0 +1,33 @@ +# Production-code stubs (NotImplementedError / TODO() that are NOT ops or layers, so they +# cannot carry @NotImplemented into the operator coverage matrix. Checked by +# `./gradlew checkUnmarkedStubs`. +# +# Format: path|count|issue or reason +# count = exact number of unmarked stubs expected in the file. Closing a gap means lowering +# the count (or deleting the line); adding a stub means raising it โ€” and filing an issue. +# Ops and nn layers do not belong here: annotate them with @NotImplemented(reason, issue). + +# --- Docs generator ---------------------------------------------------------------------- +build-logic/convention/src/main/kotlin/GenerateDocumentationTask.kt|2|Markdown / HTML output formats (AsciiDoc only) + +# --- Compile / export: writeExportToFile has only a JVM actual --------------------------- +skainet-compile/skainet-compile-json/src/androidNativeMain/kotlin/sk/ainet/compile/json/Serialization.androidNative.kt|1|writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/iosArm64Main/kotlin/sk/ainet/compile/json/Serialization.ios.kt|1|writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/iosSimulatorArm64Main/kotlin/sk/ainet/compile/json/Serialization.iossim.kt|1|writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/jsMain/kotlin/sk/ainet/compile/json/Serialization.js.kt|1|writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/linuxArm64Main/kotlin/sk/ainet/compile/json/Serialization.linuxarm64.kt|1|writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/linuxX64Main/kotlin/sk/ainet/compile/json/Serialization.linuxx64.kt|1|writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/macosArm64Main/kotlin/sk/ainet/compile/json/Serialization.macos.kt|1|writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/wasmJsMain/kotlin/sk/ainet/compile/json/Serialization.wasm.kt|1|writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/wasmWasiMain/kotlin/sk/ainet/compile/json/Serialization.wasmWasi.kt|1|writeExportToFile on non-JVM targets + +# --- IO ------------------------------------------------------------------------------------ +skainet-io/skainet-io-iree-params/src/nativeMain/kotlin/sk/ainet/io/irpa/FileBackedTransfer.native.kt|1|IRPA FileBacked transfer on Kotlin/Native +skainet-io/skainet-io-iree-params/src/wasmJsMain/kotlin/sk/ainet/io/irpa/FileBackedTransfer.wasmjs.kt|1|platform limitation: no file mapping in the browser +skainet-io/skainet-io-iree-params/src/wasmWasiMain/kotlin/sk/ainet/io/irpa/FileBackedTransfer.wasmwasi.kt|1|IRPA FileBacked transfer on wasmWasi +skainet-io/skainet-io-onnx/src/commonMain/kotlin/sk/ainet/io/onnx/OnnxModelReader.kt|1|streaming tensor loading for ONNX + +# --- Lang core: DSL and tensor views --------------------------------------------------------- +skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/dsl/NetworkBuilder.kt|6|mixed-precision dense() / stage() (needs a SKEEP) +skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/TensorSlicingExtensions.kt|1|sliceCopy {} materialization +skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/dsl/TensorDSL.kt|1|TensorDefineDslImpl.create() (unused; implement or delete) diff --git a/docs/modules/ROOT/pages/contributing/darc-workflow.adoc b/docs/modules/ROOT/pages/contributing/darc-workflow.adoc index a31b51a62..fd4545bf6 100644 --- a/docs/modules/ROOT/pages/contributing/darc-workflow.adoc +++ b/docs/modules/ROOT/pages/contributing/darc-workflow.adoc @@ -237,7 +237,9 @@ All of the following must be true before the annotation goes in: been spot-checked. . *R โ€” Research.* Every citation resolves and supports the specific claim it backs. Broken or drifted links are a hard fail. -. *C โ€” Code.* `statusByBackend` reflects reality. +. *C โ€” Code.* `statusByBackend` reflects reality. A function or + backend override that still throws carries `@NotImplemented` (see + xref:contributing/issue-taxonomy.adoc#marked-gaps[marked gaps]). Set `referencesChecked = false` only as a deliberate signal โ€” e.g. a citation behind a paywall the reviewer could not open. The badge diff --git a/docs/modules/ROOT/pages/contributing/issue-taxonomy.adoc b/docs/modules/ROOT/pages/contributing/issue-taxonomy.adoc index 15d560834..77300aa34 100644 --- a/docs/modules/ROOT/pages/contributing/issue-taxonomy.adoc +++ b/docs/modules/ROOT/pages/contributing/issue-taxonomy.adoc @@ -202,6 +202,29 @@ DRY_RUN=1 .github/scripts/sync-labels.sh The script only creates and updates; it never deletes a label that is not in `labels.txt`. To add a label, add a line to the file and re-run. +[[marked-gaps]] +== From a marked gap to an issue + +Known gaps live in source, not only in the tracker, so the +xref:reference/ops-status-matrix.adoc[operator coverage matrix] can +show them: + +* A missing `TensorOps` function, a backend override that still + throws, or an `nn` layer whose `forward` throws carries + `@NotImplemented(reason = "โ€ฆ", issue = "#NNN")`. The matrix renders + it as โŒ linked to the issue; the NN-layers table does the same for + layers. +* Other stubs (format loaders, platform `actual`s, DSL corners) are + listed in `config/stub-allowlist.txt` with their issue. +* `./gradlew checkUnmarkedStubs` (CI leg `stub-markers`) fails on any + `NotImplementedError` / `TODO(` in production code that is neither + marked nor allowlisted. The docs workflow fails when the committed + matrix is stale. + +The lane PR that closes a gap removes the annotation (or lowers the +allowlist count) and commits the regenerated matrix, so the โŒ turns โœ… +in the same change. + == Lane task anatomy A good sub-issue answers, in this order, without the reader opening diff --git a/docs/modules/ROOT/pages/explanation/operator-design.adoc b/docs/modules/ROOT/pages/explanation/operator-design.adoc index ff8a56514..e37737b91 100644 --- a/docs/modules/ROOT/pages/explanation/operator-design.adoc +++ b/docs/modules/ROOT/pages/explanation/operator-design.adoc @@ -42,7 +42,11 @@ Your article must be written in AsciiDoc and include the following sections (use [source,kotlin] ---- @Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) - annotation class NotImplemented(vararg val backends: String) + annotation class NotImplemented( + vararg val backends: String, // empty = every backend + val reason: String = "", + val issue: String = "" // e.g. "#1234" + ) @Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) annotation class InProgress( diff --git a/docs/modules/ROOT/pages/reference/operators/generated/index.adoc b/docs/modules/ROOT/pages/reference/operators/generated/index.adoc index 8aad78d17..b3f9e096d 100644 --- a/docs/modules/ROOT/pages/reference/operators/generated/index.adoc +++ b/docs/modules/ROOT/pages/reference/operators/generated/index.adoc @@ -1,6 +1,6 @@ = AI-NET Operators Reference -Generated from version `0.61.0` on 2026-10-16 +Generated from version `0.61.0` on 2026-10-19 == Operators by Modality diff --git a/docs/modules/ROOT/pages/reference/operators/generated/similarity.adoc b/docs/modules/ROOT/pages/reference/operators/generated/similarity.adoc index f3c0cde52..c36e22234 100644 --- a/docs/modules/ROOT/pages/reference/operators/generated/similarity.adoc +++ b/docs/modules/ROOT/pages/reference/operators/generated/similarity.adoc @@ -30,5 +30,5 @@ fun cosineDistance(other:Tensor, dim:Int, eps:Double): Tensor === Notes -TIP: *all*: +TIP: *all*: Calculates the cosine distance between two tensors along a given dimension (1 - cosine similarity). diff --git a/docs/modules/ROOT/pages/reference/operators/generated/tensorops.adoc b/docs/modules/ROOT/pages/reference/operators/generated/tensorops.adoc index 38c9e227c..02c6b546c 100644 --- a/docs/modules/ROOT/pages/reference/operators/generated/tensorops.adoc +++ b/docs/modules/ROOT/pages/reference/operators/generated/tensorops.adoc @@ -425,6 +425,36 @@ fun convTranspose1d(input:Tensor, weight:Tensor, bias:Tensor, stride:Int, paddin `Tensor` +== convTranspose2d + +[.darc-none]#โœ– Generated facts only (no human prose)# + +=== Signature + +[source,kotlin] +---- +fun convTranspose2d(input:Tensor, weight:Tensor, bias:Tensor, stride:Pair, padding:Pair, outputPadding:Pair, dilation:Pair, groups:Int): Tensor +---- + +=== Parameters + +* `input: Tensor` +* `weight: Tensor` +* `bias: Tensor` +* `stride: Pair` +* `padding: Pair` +* `outputPadding: Pair` +* `dilation: Pair` +* `groups: Int` + +=== Return Type + +`Tensor` + +=== Notes + +TIP: *all backends*: no backend kernel yet; TransposedConv2d.forward depends on it + == maxPool2d [.darc-none]#โœ– Generated facts only (no human prose)# diff --git a/docs/modules/ROOT/pages/reference/ops-status-matrix.adoc b/docs/modules/ROOT/pages/reference/ops-status-matrix.adoc index 5c918791a..ab13249f8 100644 --- a/docs/modules/ROOT/pages/reference/ops-status-matrix.adoc +++ b/docs/modules/ROOT/pages/reference/ops-status-matrix.adoc @@ -1,82 +1,133 @@ = Operator Coverage Matrix :description: Cross-backend status for every operator function in SKaiNET. -Generated from `operators.json` version `0.61.0` on 2026-10-16. +Generated from `operators.json` version `0.61.0` on 2026-10-19. -Rows are `Operator.function` pairs. The `Validated` column shows whether the function's documentation has been DARC-validated by a reviewer (see xref:contributing/darc-workflow.adoc[DARC workflow]). The `Lowering` column lists the compile-path forms the function declares with `@Lowering` (`composite` = a `stablehlo.composite` when a target keeps it fused, `decompose` = StableHLO/CHLO primitives, `native` = a 1:1 target op); `โ€”` means undeclared. Remaining columns are backends that appear in any function's `statusByBackend` map โ€” a missing entry means the backend makes no claim about the function (treat it as "unknown", not "not supported"). +Rows are `Operator.function` pairs. A gap is marked in source with `@NotImplemented(reason = โ€ฆ, issue = โ€ฆ)` (or `@InProgress`) and shows up here as โŒ (or ๐Ÿšง) linked to its tracking issue; the PR that closes the gap removes the annotation. See xref:contributing/issue-taxonomy.adoc[Issue taxonomy] to pick one up. -[cols="2,1,1,1,1,1", options="header"] +* *Validated* โ€” whether the function's documentation has been DARC-validated by a reviewer (see xref:contributing/darc-workflow.adoc[DARC workflow]). +* *Lowering* โ€” the compile-path forms the function declares with `@Lowering` (`composite` = a `stablehlo.composite` when a target keeps it fused, `decompose` = StableHLO/CHLO primitives, `native` = a 1:1 target op); `โ€”` means undeclared. +* *StableHLO* โ€” โœ… the StableHLO converter can lower the op; โŒ export fails with "No converter found". +* *Backward* โ€” for `@Diff` ops: โœ… a backward rule is wired; โŒ the rule is missing or a stub returning null gradients; `โ€”` the op is not differentiable. +* *Backend columns* โ€” โœ… the backend overrides the function; โ†ช the backend relies on the `TensorOps` default; ๐Ÿšง in progress; โŒ not implemented; `โ€”` the backend makes no claim (unknown, not "unsupported"). + +[cols="2,1,1,1,1,1,1,1", options="header"] |=== -| Operator.function | Validated | Lowering | apple | cpu | wasm +| Operator.function | Validated | Lowering | StableHLO | Backward | apple | cpu | wasm -| `TensorOps.add` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.subtract` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.multiply` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.divide` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.addScalar` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.subScalar` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.mulScalar` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.divScalar` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.rsubScalar` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.rdivScalar` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.matmul` | โœ… | โ€” | โ€” | โ€” | โ€” -| `TensorOps.matmulWeightTransposed` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.relayoutPackedWeightForKernels` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.transpose` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.permute` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.conv1d` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.conv2d` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.conv3d` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.convTranspose1d` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.maxPool2d` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.avgPool2d` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.upsample2d` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.reshape` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.flatten` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.concat` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.split` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.squeeze` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.unsqueeze` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.relu` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.leakyRelu` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.elu` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.softmax` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.logSoftmax` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.sigmoid` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.tanh` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.silu` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.gelu` | โœ– | composite ยท decompose | โ€” | โ€” | โ€” -| `TensorOps.sum` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.mean` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.variance` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.argMax` | โš  | โ€” | โ€” | โ€” | โ€” -| `TensorOps.sqrt` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.pow` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.powScalar` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.log` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.log2` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.log10` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.abs` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.sign` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.clamp` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.narrow` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.pad2d` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.unfold` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.lt` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.ge` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.tril` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.convert` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.gather` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.indexSelect` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.exp` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.expm1` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.erf` | โœ– | composite ยท decompose | โ€” | โ€” | โ€” -| `TensorOps.sin` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.cos` | โœ– | โ€” | โ€” | โ€” | โ€” -| `TensorOps.scaledDotProductAttention` | โœ– | composite ยท decompose | โ€” | โ€” | โ€” -| `Similarity.cosineDistance` | โœ– | โ€” | โœ… | โœ… | โœ… +| `TensorOps.add` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.subtract` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.multiply` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.divide` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.addScalar` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.subScalar` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.mulScalar` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.divScalar` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.rsubScalar` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.rdivScalar` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.matmul` | โœ… | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.matmulWeightTransposed` | โœ– | โ€” | โŒ | โ€” | โ€” | โœ… | โ€” +| `TensorOps.relayoutPackedWeightForKernels` | โœ– | โ€” | โŒ | โ€” | โ€” | โœ… | โ€” +| `TensorOps.transpose` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.permute` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.conv1d` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.conv2d` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.conv3d` | โœ– | โ€” | โŒ | โœ… | โ€” | โœ… | โ€” +| `TensorOps.convTranspose1d` | โœ– | โ€” | โŒ | โœ… | โ€” | โœ… | โ€” +| `TensorOps.convTranspose2d` | โœ– | โ€” | โŒ | โ€” | โŒ | โŒ | โŒ +| `TensorOps.maxPool2d` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.avgPool2d` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.upsample2d` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.reshape` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.flatten` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.concat` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.split` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.squeeze` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.unsqueeze` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.relu` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.leakyRelu` | โœ– | โ€” | โŒ | โœ… | โ€” | โœ… | โ€” +| `TensorOps.elu` | โœ– | โ€” | โŒ | โœ… | โ€” | โœ… | โ€” +| `TensorOps.softmax` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.logSoftmax` | โœ– | โ€” | โŒ | โœ… | โ€” | โœ… | โ€” +| `TensorOps.sigmoid` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.tanh` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.silu` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.gelu` | โœ– | composite ยท decompose | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.sum` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.mean` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.variance` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.argMax` | โš  | โ€” | โœ… | โ€” | โ€” | โœ… | โ€” +| `TensorOps.sqrt` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.pow` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.powScalar` | โœ– | โ€” | โŒ | โœ… | โ€” | โœ… | โ€” +| `TensorOps.log` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.log2` | โœ– | โ€” | โŒ | โœ… | โ€” | โœ… | โ€” +| `TensorOps.log10` | โœ– | โ€” | โŒ | โœ… | โ€” | โœ… | โ€” +| `TensorOps.abs` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.sign` | โœ– | โ€” | โœ… | โ€” | โ€” | โœ… | โ€” +| `TensorOps.clamp` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.narrow` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.pad2d` | โœ– | โ€” | โŒ | โœ… | โ€” | โœ… | โ€” +| `TensorOps.unfold` | โœ– | โ€” | โŒ | โœ… | โ€” | โœ… | โ€” +| `TensorOps.lt` | โœ– | โ€” | โŒ | โ€” | โ€” | โœ… | โ€” +| `TensorOps.ge` | โœ– | โ€” | โŒ | โ€” | โ€” | โœ… | โ€” +| `TensorOps.tril` | โœ– | โ€” | โŒ | โœ… | โ€” | โœ… | โ€” +| `TensorOps.convert` | โœ– | โ€” | โœ… | โ€” | โ€” | โœ… | โ€” +| `TensorOps.gather` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.indexSelect` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.exp` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.expm1` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.erf` | โœ– | composite ยท decompose | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.sin` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.cos` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” +| `TensorOps.scaledDotProductAttention` | โœ– | composite ยท decompose | โœ… | โŒ stub | โ€” | โœ… | โ€” +| `Similarity.cosineDistance` | โœ– | โ€” | โŒ | โ€” | โœ… | โœ… | โœ… -| *Done* | *1 / 66* | *3 / 66* | *1 / 66* | *1 / 66* | *1 / 66* +| *Done* | *1 / 67* | *3 / 67* | *50 / 67* | *57 / 58* | *1 / 67* | *66 / 67* | *1 / 67* |=== Per-function detail including notes lives in xref:reference/operators/generated/index.adoc[Operator reference]. + +== NN layers + +Every concrete `sk.ainet.lang.nn` `Module` subclass. โŒ means the layer's `forward` throws today; the class carries `@NotImplemented`. + +[cols="2,3,1,3", options="header"] +|=== +| Layer | Package | Status | Missing + +| `AvgPool2d` | `sk.ainet.lang.nn` | โœ… | โ€” +| `Conv1d` | `sk.ainet.lang.nn` | โœ… | โ€” +| `Conv2d` | `sk.ainet.lang.nn` | โœ… | โ€” +| `Conv3d` | `sk.ainet.lang.nn` | โœ… | โ€” +| `DepthwiseSeparableConv2d` | `sk.ainet.lang.nn` | โœ… | โ€” +| `DilatedConv2d` | `sk.ainet.lang.nn` | โœ… | โ€” +| `Flatten` | `sk.ainet.lang.nn` | โœ… | โ€” +| `GroupedConv2d` | `sk.ainet.lang.nn` | โœ… | โ€” +| `Gru` | `sk.ainet.lang.nn` | โœ… | โ€” +| `Input` | `sk.ainet.lang.nn` | โœ… | โ€” +| `LayerScale` | `sk.ainet.lang.nn` | โœ… | โ€” +| `Linear` | `sk.ainet.lang.nn` | โœ… | โ€” +| `Lstm` | `sk.ainet.lang.nn` | โœ… | โ€” +| `MaxPool2d` | `sk.ainet.lang.nn` | โœ… | โ€” +| `TransposedConv1d` | `sk.ainet.lang.nn` | โœ… | โ€” +| `TransposedConv2d` | `sk.ainet.lang.nn` | โŒ | forward needs TensorOps.convTranspose2d, which no backend implements yet +| `Upsample2d` | `sk.ainet.lang.nn` | โœ… | โ€” +| `ActivationsWrapperModule` | `sk.ainet.lang.nn.activations` | โœ… | โ€” +| `ELU` | `sk.ainet.lang.nn.activations` | โœ… | โ€” +| `GELU` | `sk.ainet.lang.nn.activations` | โœ… | โ€” +| `LeakyReLU` | `sk.ainet.lang.nn.activations` | โœ… | โ€” +| `ReLU` | `sk.ainet.lang.nn.activations` | โœ… | โ€” +| `SiLU` | `sk.ainet.lang.nn.activations` | โœ… | โ€” +| `Sigmoid` | `sk.ainet.lang.nn.activations` | โœ… | โ€” +| `Snake` | `sk.ainet.lang.nn.activations` | โœ… | โ€” +| `Softmax` | `sk.ainet.lang.nn.activations` | โœ… | โ€” +| `Dropout` | `sk.ainet.lang.nn.layers` | โœ… | โ€” +| `LossModule` | `sk.ainet.lang.nn.loss` | โœ… | โ€” +| `BatchNormalization` | `sk.ainet.lang.nn.normalization` | โœ… | โ€” +| `GroupNormalization` | `sk.ainet.lang.nn.normalization` | โŒ | forward helpers (group reshape, mean, variance) are TODO(); eps and gamma/beta are not applied +| `LayerNormalization` | `sk.ainet.lang.nn.normalization` | โœ… | โ€” +| `MLP` | `sk.ainet.lang.nn.topology` | โœ… | โ€” + +| *Done* | | *30 / 32* | +|=== diff --git a/skainet-backends/skainet-backend-cpu/api/jvm/skainet-backend-cpu.api b/skainet-backends/skainet-backend-cpu/api/jvm/skainet-backend-cpu.api index e0b809808..54182f2c5 100644 --- a/skainet-backends/skainet-backend-cpu/api/jvm/skainet-backend-cpu.api +++ b/skainet-backends/skainet-backend-cpu/api/jvm/skainet-backend-cpu.api @@ -292,6 +292,7 @@ public class sk/ainet/exec/tensor/ops/DefaultCpuOpsBase : sk/ainet/context/sched public fun conv2d (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;I)Lsk/ainet/lang/tensor/Tensor; public fun conv3d (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Triple;Lkotlin/Triple;Lkotlin/Triple;I)Lsk/ainet/lang/tensor/Tensor; public fun convTranspose1d (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;IIIII)Lsk/ainet/lang/tensor/Tensor; + public fun convTranspose2d (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;I)Lsk/ainet/lang/tensor/Tensor; public fun convert (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/types/DType;)Lsk/ainet/lang/tensor/Tensor; public fun cos (Lsk/ainet/lang/tensor/Tensor;)Lsk/ainet/lang/tensor/Tensor; protected final fun dispatchScope ()Lsk/ainet/lang/memory/Scope; diff --git a/skainet-backends/skainet-backend-cpu/build.gradle.kts b/skainet-backends/skainet-backend-cpu/build.gradle.kts index 7b56fbdec..469f1ff12 100644 --- a/skainet-backends/skainet-backend-cpu/build.gradle.kts +++ b/skainet-backends/skainet-backend-cpu/build.gradle.kts @@ -4,6 +4,7 @@ plugins { alias(libs.plugins.vanniktech.mavenPublish) alias(libs.plugins.binary.compatibility.validator) id("sk.ainet.dokka") + alias(libs.plugins.ksp) } skainet { @@ -62,6 +63,19 @@ kotlin { } } +// Run the operator-doc KSP processor over commonMain so the coverage matrix gets a real +// `cpu` column: it sees `TensorOps` (from skainet-lang-core) plus the `@Backend("cpu")` +// `DefaultCpuOps` overrides and their @NotImplemented / @InProgress markers. Root +// `generateDocs` merges the emitted operators.json via `documentation.backendStatusFiles`. +dependencies { + add("kspCommonMainMetadata", project(":skainet-lang:skainet-lang-ksp-processor")) +} + +ksp { + arg("skainet.version", providers.gradleProperty("VERSION_NAME").getOrElse("unknown")) + arg("skainet.module", "skainet-backend-cpu") +} + tasks.withType().configureEach { jvmArgs("--enable-preview", "--add-modules", "jdk.incubator.vector") } diff --git a/skainet-backends/skainet-backend-cpu/src/commonMain/kotlin/sk/ainet/exec/tensor/ops/DefaultCpuOps.kt b/skainet-backends/skainet-backend-cpu/src/commonMain/kotlin/sk/ainet/exec/tensor/ops/DefaultCpuOps.kt index db6da6530..9c99162a9 100644 --- a/skainet-backends/skainet-backend-cpu/src/commonMain/kotlin/sk/ainet/exec/tensor/ops/DefaultCpuOps.kt +++ b/skainet-backends/skainet-backend-cpu/src/commonMain/kotlin/sk/ainet/exec/tensor/ops/DefaultCpuOps.kt @@ -7,7 +7,6 @@ import sk.ainet.lang.tensor.ops.TensorOps import sk.ainet.lang.types.DType import sk.ainet.lang.ops.Backend import sk.ainet.lang.ops.TensorOp -import sk.ainet.lang.ops.InProgress import sk.ainet.backend.api.kernel.KernelProvider import sk.ainet.backend.api.kernel.KernelRegistry import sk.ainet.lang.tensor.data.RowDequantSource @@ -55,7 +54,6 @@ import kotlin.reflect.KClass internal const val SDPA_PARALLEL_MIN_WORK: Long = 1L shl 20 @Backend(id = "cpu", displayName = "CPU") -@InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#defaultcpuops") public open class DefaultCpuOpsBase( protected val dataFactory: TensorDataFactory, /** How this ops instance maps independent work onto cores (SKEEP-005); [Schedule.Sequential] by default. */ @@ -401,7 +399,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-add") override fun add( a: Tensor, b: Tensor @@ -425,7 +422,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-subtract") override fun subtract( a: Tensor, b: Tensor @@ -449,7 +445,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-multiply") override fun multiply( a: Tensor, b: Tensor @@ -473,7 +468,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-divide") override fun divide( a: Tensor, b: Tensor @@ -1461,7 +1455,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-conv2d") override fun conv2d( input: Tensor, weight: Tensor, @@ -1989,7 +1982,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-upsample2d") override fun upsample2d( input: Tensor, scale: Pair, @@ -2064,7 +2056,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-maxpool2d") override fun maxPool2d( input: Tensor, kernelSize: Pair, @@ -2230,7 +2221,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-reshape") override fun reshape( tensor: Tensor, newShape: Shape @@ -2293,7 +2283,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-flatten") override fun flatten( tensor: Tensor, startDim: Int, @@ -2322,7 +2311,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-concat") override fun concat( tensors: List>, dim: Int @@ -2403,7 +2391,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-split") override fun split( tensor: Tensor, splitSize: Int, @@ -2436,7 +2423,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-squeeze") override fun squeeze( tensor: Tensor, dim: Int? @@ -2460,7 +2446,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-unsqueeze") override fun unsqueeze( tensor: Tensor, dim: Int @@ -2476,7 +2461,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-relu") override fun relu(tensor: Tensor): Tensor { floatUnaryFast(tensor) { x -> if (x < 0f) 0f else x }?.let { return it } val outData = dataFactory.init(tensor.shape, tensor.dtype) { idx -> @@ -2535,7 +2519,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-softmax") override fun softmax( tensor: Tensor, dim: Int @@ -2599,7 +2582,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-logsoftmax") override fun logSoftmax( tensor: Tensor, dim: Int @@ -2656,7 +2638,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-sigmoid") override fun sigmoid(tensor: Tensor): Tensor { floatUnaryFast(tensor) { x -> 1.0f / (1.0f + kotlin.math.exp(-x)) }?.let { return it } val outData = dataFactory.init(tensor.shape, tensor.dtype) { idx -> @@ -2673,7 +2654,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-silu") override fun silu(tensor: Tensor): Tensor { floatUnaryFast(tensor) { x -> x * (1.0f / (1.0f + kotlin.math.exp(-x))) }?.let { return it } val outData = dataFactory.init(tensor.shape, tensor.dtype) { idx -> @@ -2749,7 +2729,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-tril") override fun tril(tensor: Tensor, k: Int): Tensor { val rank = tensor.rank // Apply over last two dims; for rank < 2, just copy @@ -2784,7 +2763,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-sum") override fun sum( tensor: Tensor, dim: Int? @@ -2930,7 +2908,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-argmax") override fun argMax(tensor: Tensor, dim: Int): Tensor { val rank = tensor.rank require(rank >= 1) { "argMax: input must have rank >= 1" } @@ -2970,7 +2947,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-mean") override fun mean( tensor: Tensor, dim: Int? @@ -3126,7 +3102,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-variance") override fun variance( tensor: Tensor, dim: Int? @@ -3273,7 +3248,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-sqrt") override fun sqrt(tensor: Tensor): Tensor { require( tensor.dtype == sk.ainet.lang.types.FP32::class || @@ -3403,7 +3377,6 @@ public open class DefaultCpuOpsBase( // ---- TinyFoA ops: abs, sign, clamp, lt, ge ---- @TensorOp() - @InProgress("cpu", owner = "team:tinyfoa", issue = "PRD-tinyFoA#op-abs") override fun abs(tensor: Tensor): Tensor { floatUnaryFast(tensor) { x -> kotlin.math.abs(x) }?.let { return it } val outData = dataFactory.init(tensor.shape, tensor.dtype) { idx -> @@ -3425,7 +3398,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:tinyfoa", issue = "PRD-tinyFoA#op-sign") override fun sign(tensor: Tensor): Tensor { floatUnaryFast(tensor) { x -> if (x > 0f) 1f else if (x < 0f) -1f else 0f }?.let { return it } val outData = dataFactory.init(tensor.shape, tensor.dtype) { idx -> @@ -3447,7 +3419,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:tinyfoa", issue = "PRD-tinyFoA#op-clamp") override fun clamp(tensor: Tensor, minVal: Float, maxVal: Float): Tensor { require(minVal <= maxVal) { "clamp: minVal ($minVal) must be <= maxVal ($maxVal)" } floatUnaryFast(tensor) { x -> x.coerceIn(minVal, maxVal) }?.let { return it } @@ -3470,7 +3441,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:tinyfoa", issue = "PRD-tinyFoA#op-lt") override fun lt(tensor: Tensor, value: Float): Tensor { floatUnaryFast(tensor) { x -> if (x < value) 1f else 0f }?.let { return it } val outData = dataFactory.init(tensor.shape, tensor.dtype) { idx -> @@ -3492,7 +3462,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:tinyfoa", issue = "PRD-tinyFoA#op-ge") override fun ge(tensor: Tensor, value: Float): Tensor { floatUnaryFast(tensor) { x -> if (x >= value) 1f else 0f }?.let { return it } val outData = dataFactory.init(tensor.shape, tensor.dtype) { idx -> @@ -3516,7 +3485,6 @@ public open class DefaultCpuOpsBase( // ---- narrow, pad2d, unfold ---- @TensorOp() - @InProgress("cpu", owner = "team:tinyfoa", issue = "PRD-tinyFoA#op-narrow") override fun narrow(tensor: Tensor, dim: Int, start: Int, length: Int): Tensor { val actualDim = if (dim < 0) tensor.shape.rank + dim else dim require(actualDim in 0 until tensor.shape.rank) { "narrow dim $dim out of bounds for rank ${tensor.shape.rank}" } @@ -3535,7 +3503,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:tinyfoa", issue = "PRD-tinyFoA#op-pad2d") override fun pad2d(tensor: Tensor, padLeft: Int, padRight: Int, padTop: Int, padBottom: Int): Tensor { require(tensor.shape.rank == 4) { "pad2d requires 4D tensor [N,C,H,W], got rank ${tensor.shape.rank}" } val (n, c, h, w) = tensor.shape.dimensions.toList() @@ -3564,7 +3531,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:tinyfoa", issue = "PRD-tinyFoA#op-unfold") override fun unfold(tensor: Tensor, dim: Int, size: Int, step: Int): Tensor { val actualDim = if (dim < 0) tensor.shape.rank + dim else dim require(actualDim in 0 until tensor.shape.rank) { "unfold dim $dim out of bounds for rank ${tensor.shape.rank}" } @@ -3594,7 +3560,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-convert") override fun convert( tensor: Tensor, targetType: TTo @@ -3790,7 +3755,6 @@ public open class DefaultCpuOpsBase( } @TensorOp() - @InProgress("cpu", owner = "team:cpu", issue = "task-ops.md#op-tanh") override fun tanh(tensor: Tensor): Tensor { floatUnaryFast(tensor) { x -> kotlin.math.tanh(x) }?.let { return it } val outData = dataFactory.init(tensor.shape, tensor.dtype) { idx -> diff --git a/skainet-compile/skainet-compile-dag/src/commonMain/kotlin/sk/ainet/lang/graph/DefaultExecutionTape.kt b/skainet-compile/skainet-compile-dag/src/commonMain/kotlin/sk/ainet/lang/graph/DefaultExecutionTape.kt index b7acd1356..5c16e6ee5 100644 --- a/skainet-compile/skainet-compile-dag/src/commonMain/kotlin/sk/ainet/lang/graph/DefaultExecutionTape.kt +++ b/skainet-compile/skainet-compile-dag/src/commonMain/kotlin/sk/ainet/lang/graph/DefaultExecutionTape.kt @@ -1055,6 +1055,7 @@ public class DefaultGradientTape( override fun scaledDotProductAttentionBackward(upstream: Tensor, output: Tensor, inputs: List>, attributes: Map): List?> { // SDPA backward is complex; stub returns null gradients (not differentiable through this path). // Full backward would require recomputing attention weights from Q,K,V. + // Listed in stubbedBackwardOpNames so the coverage matrix shows the gap. return inputs.map { null } } @@ -1321,6 +1322,14 @@ public class DefaultGradientTape( /** Trace op names with a wired backward rule. Consumed by the autodiff-coverage guard test. */ internal val dispatchedOpNames: Set get() = backwardDispatch.keys + /** + * Dispatched ops whose backward rule is still a stub that returns `null` gradients, so + * gradients silently stop there. Rendered as โŒ in the Backward column of the operator + * coverage matrix (BackwardCoverageDumpTest). Remove an entry in the change that + * implements its rule. + */ + internal val stubbedBackwardOpNames: Set = setOf("scaledDotProductAttention") + private fun buildBackwardFromTrace( trace: OpTrace, inputs: List>, diff --git a/skainet-compile/skainet-compile-dag/src/jvmTest/kotlin/sk/ainet/exec/autograd/BackwardCoverageDumpTest.kt b/skainet-compile/skainet-compile-dag/src/jvmTest/kotlin/sk/ainet/exec/autograd/BackwardCoverageDumpTest.kt new file mode 100644 index 000000000..cfea47e86 --- /dev/null +++ b/skainet-compile/skainet-compile-dag/src/jvmTest/kotlin/sk/ainet/exec/autograd/BackwardCoverageDumpTest.kt @@ -0,0 +1,34 @@ +package sk.ainet.exec.autograd + +import sk.ainet.lang.graph.DefaultGradientTape +import sk.ainet.lang.tensor.ops.DifferentiableTensorOpsRules +import java.io.File +import kotlin.test.Test + +/** + * Dumps the autodiff backward coverage to `build/generated/coverage/backward-coverage.json` + * for the operator coverage matrix's Backward column (`generateDocs`). + * + * `implemented` = dispatched backward rule; `stubbed` = dispatched but listed in + * `stubbedBackwardOpNames` (returns null gradients); `missing` = `@Diff` op with no dispatch arm. + */ +class BackwardCoverageDumpTest { + + @Test + fun dump_backward_coverage() { + val tape = DefaultGradientTape() + val dispatched = tape.dispatchedOpNames + val stubbed = tape.stubbedBackwardOpNames + val ops = (DifferentiableTensorOpsRules.ruleNames + dispatched).toSortedSet() + val json = ops.joinToString(",\n", prefix = "{\n", postfix = "\n}\n") { op -> + val status = when { + op in stubbed -> "stubbed" + op in dispatched -> "implemented" + else -> "missing" + } + " \"$op\": \"$status\"" + } + val outDir = File("build/generated/coverage").apply { mkdirs() } + File(outDir, "backward-coverage.json").writeText(json) + } +} diff --git a/skainet-compile/skainet-compile-hlo/src/commonMain/kotlin/sk/ainet/compile/hlo/StableHloConverter.kt b/skainet-compile/skainet-compile-hlo/src/commonMain/kotlin/sk/ainet/compile/hlo/StableHloConverter.kt index 80e527a67..51b65a83e 100644 --- a/skainet-compile/skainet-compile-hlo/src/commonMain/kotlin/sk/ainet/compile/hlo/StableHloConverter.kt +++ b/skainet-compile/skainet-compile-hlo/src/commonMain/kotlin/sk/ainet/compile/hlo/StableHloConverter.kt @@ -37,6 +37,9 @@ public class StableHloConverter @kotlin.jvm.JvmOverloads constructor( private val errorPolicy: ConversionErrorPolicy = ConversionErrorPolicy.STRICT ) { + /** Op names this converter can lower. Consumed by the docs coverage dump (HloCoverageDumpTest). */ + internal val supportedOperationNames: Set get() = registry.getSupportedOperations() + /** * Convert a ComputeGraph to StableHLO MLIR format */ diff --git a/skainet-compile/skainet-compile-hlo/src/jvmTest/kotlin/sk/ainet/compile/hlo/HloCoverageDumpTest.kt b/skainet-compile/skainet-compile-hlo/src/jvmTest/kotlin/sk/ainet/compile/hlo/HloCoverageDumpTest.kt new file mode 100644 index 000000000..08ee19ad2 --- /dev/null +++ b/skainet-compile/skainet-compile-hlo/src/jvmTest/kotlin/sk/ainet/compile/hlo/HloCoverageDumpTest.kt @@ -0,0 +1,21 @@ +package sk.ainet.compile.hlo + +import java.io.File +import kotlin.test.Test + +/** + * Dumps the op names the extended StableHLO converter can lower to + * `build/generated/coverage/hlo-coverage.json`, for the operator coverage matrix's + * StableHLO column (`generateDocs`). An op missing from this list throws + * "No converter found" at export time. + */ +class HloCoverageDumpTest { + + @Test + fun dump_stablehlo_coverage() { + val ops = StableHloConverterFactory.createExtended().supportedOperationNames.toSortedSet() + val json = ops.joinToString(",\n", prefix = "[\n", postfix = "\n]\n") { " \"$it\"" } + val outDir = File("build/generated/coverage").apply { mkdirs() } + File(outDir, "hlo-coverage.json").writeText(json) + } +} diff --git a/skainet-lang/skainet-lang-core/api/jvm/skainet-lang-core.api b/skainet-lang/skainet-lang-core/api/jvm/skainet-lang-core.api index c2564589a..08deb2d08 100644 --- a/skainet-lang/skainet-lang-core/api/jvm/skainet-lang-core.api +++ b/skainet-lang/skainet-lang-core/api/jvm/skainet-lang-core.api @@ -6482,6 +6482,7 @@ public final class sk/ainet/lang/tensor/ops/KspTensorOps : sk/ainet/lang/tensor/ public fun conv2d (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;I)Lsk/ainet/lang/tensor/Tensor; public fun conv3d (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Triple;Lkotlin/Triple;Lkotlin/Triple;I)Lsk/ainet/lang/tensor/Tensor; public fun convTranspose1d (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;IIIII)Lsk/ainet/lang/tensor/Tensor; + public fun convTranspose2d (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;I)Lsk/ainet/lang/tensor/Tensor; public fun convert (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/types/DType;)Lsk/ainet/lang/tensor/Tensor; public fun cos (Lsk/ainet/lang/tensor/Tensor;)Lsk/ainet/lang/tensor/Tensor; public fun divScalar (Lsk/ainet/lang/tensor/Tensor;Ljava/lang/Number;)Lsk/ainet/lang/tensor/Tensor; @@ -6594,6 +6595,7 @@ public abstract interface class sk/ainet/lang/tensor/ops/MixedPrecisionTensorOps } public final class sk/ainet/lang/tensor/ops/MixedPrecisionTensorOps$DefaultImpls { + public static fun convTranspose2d (Lsk/ainet/lang/tensor/ops/MixedPrecisionTensorOps;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;I)Lsk/ainet/lang/tensor/Tensor; public static fun matmulWeightTransposed (Lsk/ainet/lang/tensor/ops/MixedPrecisionTensorOps;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;)Lsk/ainet/lang/tensor/Tensor; public static fun relayoutPackedWeightForKernels (Lsk/ainet/lang/tensor/ops/MixedPrecisionTensorOps;Lsk/ainet/lang/tensor/Tensor;)Lsk/ainet/lang/tensor/Tensor; } @@ -6767,6 +6769,8 @@ public abstract interface class sk/ainet/lang/tensor/ops/TensorOps { public static synthetic fun conv3d$default (Lsk/ainet/lang/tensor/ops/TensorOps;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Triple;Lkotlin/Triple;Lkotlin/Triple;IILjava/lang/Object;)Lsk/ainet/lang/tensor/Tensor; public abstract fun convTranspose1d (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;IIIII)Lsk/ainet/lang/tensor/Tensor; public static synthetic fun convTranspose1d$default (Lsk/ainet/lang/tensor/ops/TensorOps;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;IIIIIILjava/lang/Object;)Lsk/ainet/lang/tensor/Tensor; + public fun convTranspose2d (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;I)Lsk/ainet/lang/tensor/Tensor; + public static synthetic fun convTranspose2d$default (Lsk/ainet/lang/tensor/ops/TensorOps;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;IILjava/lang/Object;)Lsk/ainet/lang/tensor/Tensor; public abstract fun convert (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/types/DType;)Lsk/ainet/lang/tensor/Tensor; public abstract fun cos (Lsk/ainet/lang/tensor/Tensor;)Lsk/ainet/lang/tensor/Tensor; public abstract fun divScalar (Lsk/ainet/lang/tensor/Tensor;Ljava/lang/Number;)Lsk/ainet/lang/tensor/Tensor; @@ -6845,6 +6849,8 @@ public final class sk/ainet/lang/tensor/ops/TensorOps$DefaultImpls { public static synthetic fun conv2d$default (Lsk/ainet/lang/tensor/ops/TensorOps;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;IILjava/lang/Object;)Lsk/ainet/lang/tensor/Tensor; public static synthetic fun conv3d$default (Lsk/ainet/lang/tensor/ops/TensorOps;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Triple;Lkotlin/Triple;Lkotlin/Triple;IILjava/lang/Object;)Lsk/ainet/lang/tensor/Tensor; public static synthetic fun convTranspose1d$default (Lsk/ainet/lang/tensor/ops/TensorOps;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;IIIIIILjava/lang/Object;)Lsk/ainet/lang/tensor/Tensor; + public static fun convTranspose2d (Lsk/ainet/lang/tensor/ops/TensorOps;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;I)Lsk/ainet/lang/tensor/Tensor; + public static synthetic fun convTranspose2d$default (Lsk/ainet/lang/tensor/ops/TensorOps;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;IILjava/lang/Object;)Lsk/ainet/lang/tensor/Tensor; public static synthetic fun elu$default (Lsk/ainet/lang/tensor/ops/TensorOps;Lsk/ainet/lang/tensor/Tensor;FILjava/lang/Object;)Lsk/ainet/lang/tensor/Tensor; public static synthetic fun flatten$default (Lsk/ainet/lang/tensor/ops/TensorOps;Lsk/ainet/lang/tensor/Tensor;IIILjava/lang/Object;)Lsk/ainet/lang/tensor/Tensor; public static synthetic fun gather$default (Lsk/ainet/lang/tensor/ops/TensorOps;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;IILjava/lang/Object;)Lsk/ainet/lang/tensor/Tensor; @@ -6981,6 +6987,7 @@ public final class sk/ainet/lang/tensor/ops/VoidTensorOps : sk/ainet/lang/tensor public fun conv2d (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;I)Lsk/ainet/lang/tensor/Tensor; public fun conv3d (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Triple;Lkotlin/Triple;Lkotlin/Triple;I)Lsk/ainet/lang/tensor/Tensor; public fun convTranspose1d (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;IIIII)Lsk/ainet/lang/tensor/Tensor; + public fun convTranspose2d (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;I)Lsk/ainet/lang/tensor/Tensor; public fun convert (Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/types/DType;)Lsk/ainet/lang/tensor/Tensor; public fun cos (Lsk/ainet/lang/tensor/Tensor;)Lsk/ainet/lang/tensor/Tensor; public fun divScalar (Lsk/ainet/lang/tensor/Tensor;Ljava/lang/Number;)Lsk/ainet/lang/tensor/Tensor; diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/TransposedConv2d.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/TransposedConv2d.kt index 40b46f018..0d018a2c1 100644 --- a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/TransposedConv2d.kt +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/TransposedConv2d.kt @@ -5,6 +5,7 @@ import sk.ainet.lang.tensor.Tensor import sk.ainet.lang.types.DType import sk.ainet.lang.nn.topology.ModuleParameter import sk.ainet.lang.nn.topology.ModuleParameters +import sk.ainet.lang.ops.NotImplemented /** * Transposed Convolution (Deconvolution) layer. @@ -36,6 +37,7 @@ import sk.ainet.lang.nn.topology.ModuleParameters * @param initWeights Initial weights tensor * @param initBias Initial bias tensor (if bias is true) */ +@NotImplemented(reason = "forward needs TensorOps.convTranspose2d, which no backend implements yet") public class TransposedConv2d( public val inChannels: Int, public val outChannels: Int, @@ -80,8 +82,8 @@ public class TransposedConv2d( sk.ainet.lang.nn.hooks.withForwardHooks(ctx, this, input) { val weight = params.first { it.name.endsWith(".weight") }.value val biasValue = if (bias) params.firstOrNull { it.name.endsWith(".bias") }?.value else null - // TODO: Implement actual transposed convolution operation - throw NotImplementedError("TransposedConv2d forward pass requires tensor transposed convolution operations to be implemented") + // Throws until a backend implements convTranspose2d (see @NotImplemented above). + input.ops.convTranspose2d(input, weight, biasValue, stride, padding, outputPadding, dilation, groups) } /** diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/normalization/GroupNormalization.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/normalization/GroupNormalization.kt index 4140e5156..64d605a2e 100644 --- a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/normalization/GroupNormalization.kt +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/normalization/GroupNormalization.kt @@ -4,6 +4,7 @@ import sk.ainet.context.ExecutionContext import sk.ainet.lang.nn.Module import sk.ainet.lang.nn.topology.ModuleParameter import sk.ainet.lang.nn.topology.ModuleParameters +import sk.ainet.lang.ops.NotImplemented import sk.ainet.lang.tensor.Tensor import sk.ainet.lang.tensor.Shape import sk.ainet.lang.tensor.* @@ -22,6 +23,7 @@ import kotlin.reflect.KClass * @param initGamma Initial gamma (scale) parameter * @param initBeta Initial beta (shift) parameter */ +@NotImplemented(reason = "forward helpers (group reshape, mean, variance) are TODO(); eps and gamma/beta are not applied") public class GroupNormalization( private val numGroups: Int, private val numChannels: Int, diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/TensorOps.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/TensorOps.kt index fffb31dca..de37d6617 100644 --- a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/TensorOps.kt +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/TensorOps.kt @@ -9,6 +9,7 @@ import sk.ainet.lang.nn.dsl.GenerateNetworkDsl import sk.ainet.lang.nn.dsl.ActivationDsl import sk.ainet.lang.ops.DarcValidated import sk.ainet.lang.ops.Lowering +import sk.ainet.lang.ops.NotImplemented @GenerateTracingWrapper @GenerateNetworkDsl @@ -159,6 +160,27 @@ public interface TensorOps { groups: Int = 1 ): Tensor + /** + * 2D transposed convolution (a.k.a. deconvolution) over NCHW [input]. + * + * [weight] is laid out `[C_in, C_out / groups, kH, kW]` (PyTorch `ConvTranspose2d`). Output + * spatial size per axis is `(in - 1) * stride - 2 * padding + dilation * (k - 1) + outputPadding + 1`. + * + * Not implemented on any backend yet: the default throws. Once a backend kernel lands this + * becomes an abstract `@Diff` op like [convTranspose1d]. + */ + @NotImplemented(reason = "no backend kernel yet; TransposedConv2d.forward depends on it") + public fun convTranspose2d( + input: Tensor, + weight: Tensor, + bias: Tensor? = null, + stride: Pair = 1 to 1, + padding: Pair = 0 to 0, + outputPadding: Pair = 0 to 0, + dilation: Pair = 1 to 1, + groups: Int = 1 + ): Tensor = throw NotImplementedError("convTranspose2d is not implemented on this backend yet") + // Pooling operations @Diff public fun maxPool2d( diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/VoidTensorOps.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/VoidTensorOps.kt index 8d3a764ba..8ca0194a6 100644 --- a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/VoidTensorOps.kt +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/VoidTensorOps.kt @@ -1,7 +1,6 @@ package sk.ainet.lang.tensor.ops import sk.ainet.lang.ops.Backend -import sk.ainet.lang.ops.InProgress import sk.ainet.lang.tensor.Dim import sk.ainet.lang.tensor.Shape import sk.ainet.lang.tensor.Tensor @@ -150,7 +149,6 @@ public class VoidTensorOps : TensorOps { return VoidOpsTensor(resultData, b.dtype) } - @InProgress("Metal", owner="ops-team", issue="GH-1234") override fun matmul(a: Tensor, b: Tensor): Tensor { validateMatmulShapes(a.shape, b.shape) val resultShape = calculateMatmulShape(a.shape, b.shape) @@ -178,7 +176,6 @@ public class VoidTensorOps : TensorOps { return VoidOpsTensor(resultData, x.dtype) } - @InProgress("Metal", owner="ops-team", issue="GH-1234") override fun transpose(tensor: Tensor): Tensor { val resultShape = calculateTransposeShape(tensor.shape) val resultData = dataFactory.zeros(resultShape, tensor.dtype) diff --git a/skainet-lang/skainet-lang-dag/api/jvm/skainet-lang-dag.api b/skainet-lang/skainet-lang-dag/api/jvm/skainet-lang-dag.api index 5f37b32cc..4969d8fec 100644 --- a/skainet-lang/skainet-lang-dag/api/jvm/skainet-lang-dag.api +++ b/skainet-lang/skainet-lang-dag/api/jvm/skainet-lang-dag.api @@ -36,6 +36,7 @@ public abstract interface class sk/ainet/lang/dag/GraphDslOps : sk/ainet/lang/te } public final class sk/ainet/lang/dag/GraphDslOps$DefaultImpls { + public static fun convTranspose2d (Lsk/ainet/lang/dag/GraphDslOps;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;Lkotlin/Pair;I)Lsk/ainet/lang/tensor/Tensor; public static fun matmulWeightTransposed (Lsk/ainet/lang/dag/GraphDslOps;Lsk/ainet/lang/tensor/Tensor;Lsk/ainet/lang/tensor/Tensor;)Lsk/ainet/lang/tensor/Tensor; public static fun relayoutPackedWeightForKernels (Lsk/ainet/lang/dag/GraphDslOps;Lsk/ainet/lang/tensor/Tensor;)Lsk/ainet/lang/tensor/Tensor; } diff --git a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/sk/ainet/lang/ops/TensorOp.kt b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/sk/ainet/lang/ops/TensorOp.kt index 95ddb74ff..4e66e6a74 100644 --- a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/sk/ainet/lang/ops/TensorOp.kt +++ b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/sk/ainet/lang/ops/TensorOp.kt @@ -30,12 +30,27 @@ public annotation class TensorOp(val mode: ComputationMode = ComputationMode.INF /** * Annotation to mark classes or functions as not implemented for specific backends. - * - * @param backends List of backend names where this feature is not implemented + * + * This is the single source-level marker for a known gap: a `TensorOps` + * function no backend implements yet, a backend override that still throws, + * or an `nn` layer whose `forward` throws. The docs KSP processor renders it + * as โŒ in the operator coverage matrix and links [issue] from the cell. + * It wins over [InProgress] and over a detected backend override. + * + * Remove the annotation in the same change that closes the gap. + * + * @param backends List of backend names where this feature is not implemented. + * Empty means "every backend". + * @param reason One-line explanation of what is missing, shown in the docs + * @param issue URL or identifier (e.g. `"#1234"`) of the tracking issue */ @Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) @Retention(AnnotationRetention.SOURCE) -public annotation class NotImplemented(vararg val backends: String) +public annotation class NotImplemented( + vararg val backends: String, + val reason: String = "", + val issue: String = "" +) /** * Annotation to mark classes or functions as in progress for specific backends. diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt index 930c27a2b..82e4f9ea2 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt @@ -1,6 +1,7 @@ package sk.ainet.lang.ops.ksp import com.google.devtools.ksp.getClassDeclarationByName +import com.google.devtools.ksp.getAllSuperTypes import com.google.devtools.ksp.getDeclaredFunctions import com.google.devtools.ksp.processing.* import com.google.devtools.ksp.symbol.* @@ -14,7 +15,18 @@ data class OperatorDocModule( val commit: String, val timestamp: String, val module: String, - val operators: List + val operators: List, + // nn `Module` subclasses visible in this compilation unit, with their + // implementation status (`implemented` / `not_implemented`). + val layers: List = emptyList(), +) + +data class LayerDoc( + val name: String, + val packageName: String, + val status: String, + val reason: String = "", + val issue: String = "", ) data class OperatorDoc( @@ -112,7 +124,10 @@ class OperatorDocProcessor( logger.info("Found ${allSymbols.size} annotated symbols (dropped ${rawSymbols.size - allSymbols.size} on @Backend classes)") // Group annotation-discovered symbols by their containing class/package to create operators - val annotationOps = if (allSymbols.isNotEmpty()) groupSymbolsByOperator(allSymbols) else emptyList() + // Class-level markers on nn layers land in `layers`, not as empty operators. + val annotationOps = if (allSymbols.isNotEmpty()) { + groupSymbolsByOperator(allSymbols).filter { it.functions.isNotEmpty() } + } else emptyList() // Additionally discover the full TensorOps surface by walking the interface // and any `@Backend`-tagged implementors visible in this compilation unit. @@ -135,8 +150,9 @@ class OperatorDocProcessor( version = extractVersion(), commit = extractCommitSha(), timestamp = Instant.now().toString(), - module = "skainet-lang-core", // TODO: Extract from module info - operators = operatorDocs + module = options["skainet.module"]?.takeIf { it.isNotBlank() } ?: "skainet-lang-core", + operators = operatorDocs, + layers = discoverLayers(resolver), ) // Generate JSON output @@ -187,12 +203,32 @@ class OperatorDocProcessor( val functionDocs = interfaceFunctions.map { fn -> val statusByBackend = mutableMapOf() + val notes = mutableListOf() for ((backendId, backendClass) in backendClasses) { - val overrides = backendClass.getAllFunctions().any { candidate -> + val override = backendClass.getAllFunctions().firstOrNull { candidate -> candidate.simpleName.asString() == fn.simpleName.asString() && candidate.findOverridee()?.simpleName?.asString() == fn.simpleName.asString() } - statusByBackend[backendId] = if (overrides) "implemented" else "inherited" + // A status marker on the override itself (method level only โ€” + // a class-level marker would blanket the whole backend) wins + // over the plain "overrides it" signal. + val marker = override?.let { statusMarker(it) }?.takeIf { it.appliesTo(backendId) } + statusByBackend[backendId] = when { + marker != null -> marker.status + override != null -> "implemented" + else -> "inherited" + } + marker?.let { notes += it.notes(backendId) } + } + // A marker on the interface function applies across backends. An + // empty backend list is recorded under "*" and expanded to every + // backend column when the matrix is rendered. + statusMarker(fn)?.let { marker -> + val targets = marker.backends.ifEmpty { backendClasses.map { it.first } + "*" } + targets.forEach { backend -> + statusByBackend[backend] = mergeStatus(statusByBackend[backend], marker.status) + notes += marker.notes(backend) + } } val validation = extractDarcValidation(fn) FunctionDoc( @@ -201,7 +237,7 @@ class OperatorDocProcessor( parameters = extractParameters(fn), returnType = extractReturnType(fn), statusByBackend = statusByBackend, - notes = emptyList(), + notes = notes, isDifferentiable = extractDiffRule(fn) != null, diffRuleName = extractDiffRule(fn).orEmpty(), validated = validation.validated, @@ -467,9 +503,109 @@ class OperatorDocProcessor( statusMap[backend] = "in_progress" } } + + // @NotImplemented wins over @DslOp and @InProgress. + declaration.annotations.find { + it.shortName.asString() == "NotImplemented" + }?.let { + val marker = statusMarker(declaration) ?: return@let + marker.backends.ifEmpty { listOf("*") }.forEach { backend -> + statusMap[backend] = "not_implemented" + } + } return statusMap } + /** + * A `@NotImplemented` or `@InProgress` marker read off a declaration. + * When both are present `@NotImplemented` wins. + */ + private data class StatusMarker( + val status: String, + val backends: List, + val reason: String, + val owner: String, + val issue: String, + ) { + fun appliesTo(backend: String): Boolean = backends.isEmpty() || backends.any { it.equals(backend, ignoreCase = true) } + + fun notes(backend: String): List = buildList { + if (reason.isNotEmpty()) add(Note("reason", backend, reason)) + if (owner.isNotEmpty()) add(Note("owner", backend, owner)) + if (issue.isNotEmpty()) add(Note("issue", backend, issue)) + } + } + + private fun statusMarker(declaration: KSDeclaration): StatusMarker? { + val annotation = declaration.annotations.find { it.shortName.asString() == "NotImplemented" } + ?: declaration.annotations.find { it.shortName.asString() == "InProgress" } + ?: return null + fun arg(name: String): String = + annotation.arguments.find { it.name?.asString() == name }?.value?.toString().orEmpty() + val backendsArg = annotation.arguments.find { it.name?.asString() == "backends" } + ?: annotation.arguments.firstOrNull() + // KSP can hand back the first named argument in the vararg slot when + // the vararg is left empty (`@NotImplemented(reason = "...")`), so a + // bare String that equals another argument is not a backend id. + val otherValues = setOf(arg("reason"), arg("owner"), arg("issue")) + val backends = when (val value = backendsArg?.value) { + is List<*> -> value.map { it.toString() } + is String -> if (value in otherValues) emptyList() else listOf(value) + else -> emptyList() + }.filter { it.isNotBlank() } + return StatusMarker( + status = if (annotation.shortName.asString() == "NotImplemented") "not_implemented" else "in_progress", + backends = backends, + reason = arg("reason"), + owner = arg("owner"), + issue = arg("issue"), + ) + } + + /** Status precedence: not_implemented > in_progress > implemented > inherited. */ + private fun mergeStatus(current: String?, incoming: String): String { + val rank = listOf("not_implemented", "in_progress", "implemented", "inherited") + if (current == null) return incoming + val a = rank.indexOf(current).let { if (it < 0) rank.size else it } + val b = rank.indexOf(incoming).let { if (it < 0) rank.size else it } + return if (b < a) incoming else current + } + + /** + * Every concrete `sk.ainet.lang.nn.Module` subclass declared in this + * compilation unit, with `not_implemented` when the class carries + * `@NotImplemented` (its `forward` throws) and `implemented` otherwise. + */ + private fun discoverLayers(resolver: Resolver): List { + val moduleName = "sk.ainet.lang.nn.Module" + fun KSClassDeclaration.allClasses(): Sequence = + sequenceOf(this) + declarations.filterIsInstance().flatMap { it.allClasses() } + return resolver.getAllFiles() + .flatMap { it.declarations.filterIsInstance() } + .flatMap { it.allClasses() } + .filter { cls -> + cls.classKind == ClassKind.CLASS && + Modifier.ABSTRACT !in cls.modifiers && + Modifier.PRIVATE !in cls.modifiers && + Modifier.INTERNAL !in cls.modifiers && + cls.packageName.asString().startsWith("sk.ainet.lang.nn") && + cls.getAllSuperTypes().any { it.declaration.qualifiedName?.asString() == moduleName } + } + .map { cls -> + val marker = statusMarker(cls) + LayerDoc( + name = cls.simpleName.asString(), + packageName = cls.packageName.asString(), + status = marker?.status ?: "implemented", + reason = marker?.reason.orEmpty(), + issue = marker?.issue.orEmpty(), + ) + } + .distinctBy { it.packageName + "." + it.name } + .sortedWith(compareBy({ it.packageName }, { it.name })) + .toList() + } + private fun deriveNotes(declaration: KSDeclaration): List { val notes = mutableListOf() @@ -510,6 +646,11 @@ class OperatorDocProcessor( } } + // Extract reason/issue from @NotImplemented + statusMarker(declaration)?.takeIf { it.status == "not_implemented" }?.let { marker -> + marker.backends.ifEmpty { listOf("*") }.forEach { backend -> notes += marker.notes(backend) } + } + return notes } @@ -643,8 +784,22 @@ class OperatorDocProcessor( append("\n") } - append(" ]\n") - append("}") + append(" ]") + if (module.layers.isNotEmpty()) { + append(",\n \"layers\": [\n") + module.layers.forEachIndexed { index, layer -> + append(" {\"name\": \"${escapeJson(layer.name)}\", ") + append("\"packageName\": \"${escapeJson(layer.packageName)}\", ") + append("\"status\": \"${escapeJson(layer.status)}\"") + if (layer.reason.isNotEmpty()) append(", \"reason\": \"${escapeJson(layer.reason)}\"") + if (layer.issue.isNotEmpty()) append(", \"issue\": \"${escapeJson(layer.issue)}\"") + append("}") + if (index < module.layers.size - 1) append(",") + append("\n") + } + append(" ]") + } + append("\n}") } val file = codeGenerator.createNewFile( diff --git a/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessorTest.kt b/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessorTest.kt index 81623efa1..73441ad6c 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessorTest.kt +++ b/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessorTest.kt @@ -241,4 +241,85 @@ class OperatorDocProcessorTest { assertTrue(output.contains("Generated operators.json"), "Processor should generate operators.json file") } -} \ No newline at end of file + + @Test + fun testNotImplementedMarkersFlowIntoStatusAndLayers() { + val annotations = SourceFile.kotlin("sk/ainet/lang/ops/Annotations.kt", """ + package sk.ainet.lang.ops + + @Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) + @Retention(AnnotationRetention.SOURCE) + annotation class NotImplemented(vararg val backends: String, val reason: String = "", val issue: String = "") + + @Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) + @Retention(AnnotationRetention.SOURCE) + annotation class InProgress(vararg val backends: String, val owner: String = "", val issue: String = "") + + @Target(AnnotationTarget.CLASS) + @Retention(AnnotationRetention.SOURCE) + annotation class Backend(val id: String, val displayName: String = "", val internal: Boolean = false) + """.trimIndent()) + val ops = SourceFile.kotlin("sk/ainet/lang/tensor/ops/TensorOps.kt", """ + package sk.ainet.lang.tensor.ops + + import sk.ainet.lang.ops.Backend + import sk.ainet.lang.ops.InProgress + import sk.ainet.lang.ops.NotImplemented + + interface TensorOps { + fun add(): Int + fun partial(): Int + @NotImplemented(reason = "no kernel yet", issue = "#42") + fun missing(): Int + } + + @Backend(id = "cpu") + class CpuOps : TensorOps { + override fun add(): Int = 1 + @NotImplemented("cpu", reason = "FP32 only", issue = "#7") + @InProgress("cpu") + override fun partial(): Int = 2 + override fun missing(): Int = throw NotImplementedError() + } + """.trimIndent()) + val nn = SourceFile.kotlin("sk/ainet/lang/nn/Layers.kt", """ + package sk.ainet.lang.nn + + import sk.ainet.lang.ops.NotImplemented + + abstract class Module + class Dense : Module() + @NotImplemented(reason = "forward throws", issue = "#99") + class Deconv : Module() + """.trimIndent()) + + val compilation = KotlinCompilation().apply { + sources = listOf(annotations, ops, nn) + configureKsp {} + symbolProcessorProviders = mutableListOf(OperatorDocProcessorProvider()) + inheritClassPath = true + messageOutputStream = System.out + } + compilation.compile() + val text = compilation.workingDir.walkTopDown() + .first { it.isFile && it.name == "operators.json" }.readText() + + fun statusOf(fn: String): String = + Regex("\"name\": \"$fn\"[\\s\\S]*?\"statusByBackend\": \\{([^}]*)}").find(text)!!.groupValues[1] + + assertTrue(statusOf("add").contains("\"cpu\": \"implemented\""), text) + // @NotImplemented on the override wins over @InProgress and over "overrides it" + assertTrue(statusOf("partial").contains("\"cpu\": \"not_implemented\""), text) + // @NotImplemented on the interface function, no backend list -> "*" + assertTrue(statusOf("missing").contains("\"*\": \"not_implemented\""), text) + // ...and it overrides a backend that overrides the function but still throws + assertTrue(statusOf("missing").contains("\"cpu\": \"not_implemented\""), text) + assertTrue(!text.contains("\"name\": \"Deconv\",\n"), "class-level layer marker must not become an operator: $text") + assertTrue(text.contains("\"content\": \"#42\""), text) + assertTrue(text.contains("\"content\": \"FP32 only\""), text) + // Layers table + assertTrue(text.contains("{\"name\": \"Dense\", \"packageName\": \"sk.ainet.lang.nn\", \"status\": \"implemented\"}"), text) + assertTrue(text.contains("\"name\": \"Deconv\", \"packageName\": \"sk.ainet.lang.nn\", \"status\": \"not_implemented\", \"reason\": \"forward throws\", \"issue\": \"#99\""), text) + assertTrue(!text.contains("\"name\": \"Module\""), "abstract Module must not be listed: $text") + } +} From d42af95f90f925e36db8d15b739ef5cd96775eeb Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Mon, 19 Oct 2026 14:32:15 +0200 Subject: [PATCH 2/3] Link coverage gaps to their DARC tracking issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fill issue = "#โ€ฆ" on the @NotImplemented markers (convTranspose2d and TransposedConv2d #1371, GroupNormalization #1379), note the SDPA stub backward issue (#1384) and the stub-allowlist entries, and regenerate the coverage matrix so every โŒ links its issue. Scope checkUnmarkedStubs to per-project src/ trees so it no longer overlaps the generated docs output. Co-Authored-By: Claude Opus 5.5 --- .../src/main/kotlin/DocumentationPlugin.kt | 9 ++++-- config/stub-allowlist.txt | 32 +++++++++---------- .../operators/generated/tensorops.adoc | 2 ++ .../pages/reference/ops-status-matrix.adoc | 6 ++-- .../ainet/lang/graph/DefaultExecutionTape.kt | 2 +- .../sk/ainet/lang/nn/TransposedConv2d.kt | 2 +- .../nn/normalization/GroupNormalization.kt | 2 +- .../sk/ainet/lang/tensor/ops/TensorOps.kt | 4 +-- 8 files changed, 33 insertions(+), 26 deletions(-) diff --git a/build-logic/convention/src/main/kotlin/DocumentationPlugin.kt b/build-logic/convention/src/main/kotlin/DocumentationPlugin.kt index df18da929..b2fbd42f4 100644 --- a/build-logic/convention/src/main/kotlin/DocumentationPlugin.kt +++ b/build-logic/convention/src/main/kotlin/DocumentationPlugin.kt @@ -42,9 +42,14 @@ class DocumentationPlugin : Plugin { task.group = "verification" task.description = "Fail on NotImplementedError / TODO( stubs that are neither @NotImplemented nor allowlisted" val root = project.rootProject.layout.projectDirectory - task.sources.from(root.asFileTree.matching { + // Per-project src/ trees (not the repo root) so the scan never overlaps other + // tasks' outputs such as the generated docs pages. + project.rootProject.allprojects.forEach { p -> + task.sources.from(p.fileTree("src") { include("**/*.kt") }) + } + task.sources.from(root.dir("build-logic").asFileTree.matching { include("**/src/**/*.kt") - exclude("**/build/**", ".gradle/**", "**/node_modules/**") + exclude("**/build/**", "**/.gradle/**") }) task.rootDirectory.set(root) task.allowlistFile.set(root.file("config/stub-allowlist.txt")) diff --git a/config/stub-allowlist.txt b/config/stub-allowlist.txt index a2b215472..1b40ad3bb 100644 --- a/config/stub-allowlist.txt +++ b/config/stub-allowlist.txt @@ -8,26 +8,26 @@ # Ops and nn layers do not belong here: annotate them with @NotImplemented(reason, issue). # --- Docs generator ---------------------------------------------------------------------- -build-logic/convention/src/main/kotlin/GenerateDocumentationTask.kt|2|Markdown / HTML output formats (AsciiDoc only) +build-logic/convention/src/main/kotlin/GenerateDocumentationTask.kt|2|#1418 Markdown / HTML output formats (AsciiDoc only) # --- Compile / export: writeExportToFile has only a JVM actual --------------------------- -skainet-compile/skainet-compile-json/src/androidNativeMain/kotlin/sk/ainet/compile/json/Serialization.androidNative.kt|1|writeExportToFile on non-JVM targets -skainet-compile/skainet-compile-json/src/iosArm64Main/kotlin/sk/ainet/compile/json/Serialization.ios.kt|1|writeExportToFile on non-JVM targets -skainet-compile/skainet-compile-json/src/iosSimulatorArm64Main/kotlin/sk/ainet/compile/json/Serialization.iossim.kt|1|writeExportToFile on non-JVM targets -skainet-compile/skainet-compile-json/src/jsMain/kotlin/sk/ainet/compile/json/Serialization.js.kt|1|writeExportToFile on non-JVM targets -skainet-compile/skainet-compile-json/src/linuxArm64Main/kotlin/sk/ainet/compile/json/Serialization.linuxarm64.kt|1|writeExportToFile on non-JVM targets -skainet-compile/skainet-compile-json/src/linuxX64Main/kotlin/sk/ainet/compile/json/Serialization.linuxx64.kt|1|writeExportToFile on non-JVM targets -skainet-compile/skainet-compile-json/src/macosArm64Main/kotlin/sk/ainet/compile/json/Serialization.macos.kt|1|writeExportToFile on non-JVM targets -skainet-compile/skainet-compile-json/src/wasmJsMain/kotlin/sk/ainet/compile/json/Serialization.wasm.kt|1|writeExportToFile on non-JVM targets -skainet-compile/skainet-compile-json/src/wasmWasiMain/kotlin/sk/ainet/compile/json/Serialization.wasmWasi.kt|1|writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/androidNativeMain/kotlin/sk/ainet/compile/json/Serialization.androidNative.kt|1|#1420 writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/iosArm64Main/kotlin/sk/ainet/compile/json/Serialization.ios.kt|1|#1420 writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/iosSimulatorArm64Main/kotlin/sk/ainet/compile/json/Serialization.iossim.kt|1|#1420 writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/jsMain/kotlin/sk/ainet/compile/json/Serialization.js.kt|1|#1420 writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/linuxArm64Main/kotlin/sk/ainet/compile/json/Serialization.linuxarm64.kt|1|#1420 writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/linuxX64Main/kotlin/sk/ainet/compile/json/Serialization.linuxx64.kt|1|#1420 writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/macosArm64Main/kotlin/sk/ainet/compile/json/Serialization.macos.kt|1|#1420 writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/wasmJsMain/kotlin/sk/ainet/compile/json/Serialization.wasm.kt|1|#1420 writeExportToFile on non-JVM targets +skainet-compile/skainet-compile-json/src/wasmWasiMain/kotlin/sk/ainet/compile/json/Serialization.wasmWasi.kt|1|#1420 writeExportToFile on non-JVM targets # --- IO ------------------------------------------------------------------------------------ -skainet-io/skainet-io-iree-params/src/nativeMain/kotlin/sk/ainet/io/irpa/FileBackedTransfer.native.kt|1|IRPA FileBacked transfer on Kotlin/Native +skainet-io/skainet-io-iree-params/src/nativeMain/kotlin/sk/ainet/io/irpa/FileBackedTransfer.native.kt|1|#1423 IRPA FileBacked transfer on Kotlin/Native skainet-io/skainet-io-iree-params/src/wasmJsMain/kotlin/sk/ainet/io/irpa/FileBackedTransfer.wasmjs.kt|1|platform limitation: no file mapping in the browser -skainet-io/skainet-io-iree-params/src/wasmWasiMain/kotlin/sk/ainet/io/irpa/FileBackedTransfer.wasmwasi.kt|1|IRPA FileBacked transfer on wasmWasi -skainet-io/skainet-io-onnx/src/commonMain/kotlin/sk/ainet/io/onnx/OnnxModelReader.kt|1|streaming tensor loading for ONNX +skainet-io/skainet-io-iree-params/src/wasmWasiMain/kotlin/sk/ainet/io/irpa/FileBackedTransfer.wasmwasi.kt|1|IRPA FileBacked transfer on wasmWasi (follow-up of #1423) +skainet-io/skainet-io-onnx/src/commonMain/kotlin/sk/ainet/io/onnx/OnnxModelReader.kt|1|#1407 streaming tensor loading for ONNX # --- Lang core: DSL and tensor views --------------------------------------------------------- -skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/dsl/NetworkBuilder.kt|6|mixed-precision dense() / stage() (needs a SKEEP) -skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/TensorSlicingExtensions.kt|1|sliceCopy {} materialization -skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/dsl/TensorDSL.kt|1|TensorDefineDslImpl.create() (unused; implement or delete) +skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/dsl/NetworkBuilder.kt|6|mixed-precision dense() / stage() โ€” needs a SKEEP before an issue +skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/TensorSlicingExtensions.kt|1|#1415 sliceCopy {} materialization +skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/dsl/TensorDSL.kt|1|#1417 TensorDefineDslImpl.create() (unused; implement or delete) diff --git a/docs/modules/ROOT/pages/reference/operators/generated/tensorops.adoc b/docs/modules/ROOT/pages/reference/operators/generated/tensorops.adoc index 02c6b546c..4a9eefd0b 100644 --- a/docs/modules/ROOT/pages/reference/operators/generated/tensorops.adoc +++ b/docs/modules/ROOT/pages/reference/operators/generated/tensorops.adoc @@ -455,6 +455,8 @@ fun convTranspose2d(input:Tensor, weight:Tensor, bias:Tensor, stride:Pair, paddi TIP: *all backends*: no backend kernel yet; TransposedConv2d.forward depends on it +TIP: *all backends*: https://github.com/SKaiNET-developers/SKaiNET/issues/1371[#1371] + == maxPool2d [.darc-none]#โœ– Generated facts only (no human prose)# diff --git a/docs/modules/ROOT/pages/reference/ops-status-matrix.adoc b/docs/modules/ROOT/pages/reference/ops-status-matrix.adoc index ab13249f8..7619bd0c2 100644 --- a/docs/modules/ROOT/pages/reference/ops-status-matrix.adoc +++ b/docs/modules/ROOT/pages/reference/ops-status-matrix.adoc @@ -34,7 +34,7 @@ Rows are `Operator.function` pairs. A gap is marked in source with `@NotImplemen | `TensorOps.conv2d` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” | `TensorOps.conv3d` | โœ– | โ€” | โŒ | โœ… | โ€” | โœ… | โ€” | `TensorOps.convTranspose1d` | โœ– | โ€” | โŒ | โœ… | โ€” | โœ… | โ€” -| `TensorOps.convTranspose2d` | โœ– | โ€” | โŒ | โ€” | โŒ | โŒ | โŒ +| `TensorOps.convTranspose2d` | โœ– | โ€” | โŒ | โ€” | โŒ https://github.com/SKaiNET-developers/SKaiNET/issues/1371[#1371] | โŒ https://github.com/SKaiNET-developers/SKaiNET/issues/1371[#1371] | โŒ https://github.com/SKaiNET-developers/SKaiNET/issues/1371[#1371] | `TensorOps.maxPool2d` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” | `TensorOps.avgPool2d` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” | `TensorOps.upsample2d` | โœ– | โ€” | โœ… | โœ… | โ€” | โœ… | โ€” @@ -111,7 +111,7 @@ Every concrete `sk.ainet.lang.nn` `Module` subclass. โŒ means the layer's `forw | `Lstm` | `sk.ainet.lang.nn` | โœ… | โ€” | `MaxPool2d` | `sk.ainet.lang.nn` | โœ… | โ€” | `TransposedConv1d` | `sk.ainet.lang.nn` | โœ… | โ€” -| `TransposedConv2d` | `sk.ainet.lang.nn` | โŒ | forward needs TensorOps.convTranspose2d, which no backend implements yet +| `TransposedConv2d` | `sk.ainet.lang.nn` | โŒ | forward needs TensorOps.convTranspose2d, which no backend implements yet โ€” https://github.com/SKaiNET-developers/SKaiNET/issues/1371[#1371] | `Upsample2d` | `sk.ainet.lang.nn` | โœ… | โ€” | `ActivationsWrapperModule` | `sk.ainet.lang.nn.activations` | โœ… | โ€” | `ELU` | `sk.ainet.lang.nn.activations` | โœ… | โ€” @@ -125,7 +125,7 @@ Every concrete `sk.ainet.lang.nn` `Module` subclass. โŒ means the layer's `forw | `Dropout` | `sk.ainet.lang.nn.layers` | โœ… | โ€” | `LossModule` | `sk.ainet.lang.nn.loss` | โœ… | โ€” | `BatchNormalization` | `sk.ainet.lang.nn.normalization` | โœ… | โ€” -| `GroupNormalization` | `sk.ainet.lang.nn.normalization` | โŒ | forward helpers (group reshape, mean, variance) are TODO(); eps and gamma/beta are not applied +| `GroupNormalization` | `sk.ainet.lang.nn.normalization` | โŒ | forward helpers (group reshape, mean, variance) are TODO(); eps and gamma/beta are not applied โ€” https://github.com/SKaiNET-developers/SKaiNET/issues/1379[#1379] | `LayerNormalization` | `sk.ainet.lang.nn.normalization` | โœ… | โ€” | `MLP` | `sk.ainet.lang.nn.topology` | โœ… | โ€” diff --git a/skainet-compile/skainet-compile-dag/src/commonMain/kotlin/sk/ainet/lang/graph/DefaultExecutionTape.kt b/skainet-compile/skainet-compile-dag/src/commonMain/kotlin/sk/ainet/lang/graph/DefaultExecutionTape.kt index 5c16e6ee5..358536066 100644 --- a/skainet-compile/skainet-compile-dag/src/commonMain/kotlin/sk/ainet/lang/graph/DefaultExecutionTape.kt +++ b/skainet-compile/skainet-compile-dag/src/commonMain/kotlin/sk/ainet/lang/graph/DefaultExecutionTape.kt @@ -1326,7 +1326,7 @@ public class DefaultGradientTape( * Dispatched ops whose backward rule is still a stub that returns `null` gradients, so * gradients silently stop there. Rendered as โŒ in the Backward column of the operator * coverage matrix (BackwardCoverageDumpTest). Remove an entry in the change that - * implements its rule. + * implements its rule. scaledDotProductAttention: #1384. */ internal val stubbedBackwardOpNames: Set = setOf("scaledDotProductAttention") diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/TransposedConv2d.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/TransposedConv2d.kt index 0d018a2c1..ddd8fbea7 100644 --- a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/TransposedConv2d.kt +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/TransposedConv2d.kt @@ -37,7 +37,7 @@ import sk.ainet.lang.ops.NotImplemented * @param initWeights Initial weights tensor * @param initBias Initial bias tensor (if bias is true) */ -@NotImplemented(reason = "forward needs TensorOps.convTranspose2d, which no backend implements yet") +@NotImplemented(reason = "forward needs TensorOps.convTranspose2d, which no backend implements yet", issue = "#1371") public class TransposedConv2d( public val inChannels: Int, public val outChannels: Int, diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/normalization/GroupNormalization.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/normalization/GroupNormalization.kt index 64d605a2e..231d395af 100644 --- a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/normalization/GroupNormalization.kt +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/nn/normalization/GroupNormalization.kt @@ -23,7 +23,7 @@ import kotlin.reflect.KClass * @param initGamma Initial gamma (scale) parameter * @param initBeta Initial beta (shift) parameter */ -@NotImplemented(reason = "forward helpers (group reshape, mean, variance) are TODO(); eps and gamma/beta are not applied") +@NotImplemented(reason = "forward helpers (group reshape, mean, variance) are TODO(); eps and gamma/beta are not applied", issue = "#1379") public class GroupNormalization( private val numGroups: Int, private val numChannels: Int, diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/TensorOps.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/TensorOps.kt index de37d6617..2ade46ad8 100644 --- a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/TensorOps.kt +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/TensorOps.kt @@ -169,7 +169,7 @@ public interface TensorOps { * Not implemented on any backend yet: the default throws. Once a backend kernel lands this * becomes an abstract `@Diff` op like [convTranspose1d]. */ - @NotImplemented(reason = "no backend kernel yet; TransposedConv2d.forward depends on it") + @NotImplemented(reason = "no backend kernel yet; TransposedConv2d.forward depends on it", issue = "#1371") public fun convTranspose2d( input: Tensor, weight: Tensor, @@ -179,7 +179,7 @@ public interface TensorOps { outputPadding: Pair = 0 to 0, dilation: Pair = 1 to 1, groups: Int = 1 - ): Tensor = throw NotImplementedError("convTranspose2d is not implemented on this backend yet") + ): Tensor = throw NotImplementedError("convTranspose2d is not implemented on this backend yet (see #1371)") // Pooling operations @Diff From bd841e623f3036ab67647641f604282fe19deb27 Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Mon, 19 Oct 2026 17:09:15 +0200 Subject: [PATCH 3/3] Run coverage dumps before generateDocs in the docs workflow generateDocs had no ordering against the HloCoverageDumpTest / BackwardCoverageDumpTest runs, so on a clean CI checkout it rendered the matrix before the JSON existed and dropped the StableHLO and Backward columns, failing the drift check. Order it after both jvmTest tasks and declare the dump directory as a test output so up-to-date and cached test runs still provide the files. Co-Authored-By: Claude Opus 5.5 --- build.gradle.kts | 6 ++++++ skainet-compile/skainet-compile-dag/build.gradle.kts | 6 ++++++ skainet-compile/skainet-compile-hlo/build.gradle.kts | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index 4a0a8853d..3f9db1536 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -172,6 +172,12 @@ documentation { tasks.named("generateDocs") { dependsOn(":skainet-lang:skainet-lang-core:kspCommonMainKotlinMetadata") dependsOn(":skainet-backends:skainet-backend-cpu:kspCommonMainKotlinMetadata") + // The coverage dumps are opt-in (see the regenerate command above), but when they are + // requested in the same invocation they must finish before the matrix is rendered. + mustRunAfter( + ":skainet-compile:skainet-compile-hlo:jvmTest", + ":skainet-compile:skainet-compile-dag:jvmTest", + ) } tasks.named("generateKernelMatrix") { diff --git a/skainet-compile/skainet-compile-dag/build.gradle.kts b/skainet-compile/skainet-compile-dag/build.gradle.kts index 0562de4c5..34cf6970d 100644 --- a/skainet-compile/skainet-compile-dag/build.gradle.kts +++ b/skainet-compile/skainet-compile-dag/build.gradle.kts @@ -66,3 +66,9 @@ kotlin { } } } + +// BackwardCoverageDumpTest writes the Backward column data for the ops coverage matrix here; +// declaring it as an output keeps up-to-date checks and the build cache from skipping the dump. +tasks.withType().matching { it.name == "jvmTest" }.configureEach { + outputs.dir(layout.buildDirectory.dir("generated/coverage")) +} diff --git a/skainet-compile/skainet-compile-hlo/build.gradle.kts b/skainet-compile/skainet-compile-hlo/build.gradle.kts index d2e08e072..d6ce1633b 100644 --- a/skainet-compile/skainet-compile-hlo/build.gradle.kts +++ b/skainet-compile/skainet-compile-hlo/build.gradle.kts @@ -96,3 +96,9 @@ tasks.register("generateHlo") { providers.gradleProperty("batch").orNull?.let { argsList += "--batch=$it" } args = argsList } + +// HloCoverageDumpTest writes the StableHLO column data for the ops coverage matrix here; +// declaring it as an output keeps up-to-date checks and the build cache from skipping the dump. +tasks.withType().matching { it.name == "jvmTest" }.configureEach { + outputs.dir(layout.buildDirectory.dir("generated/coverage")) +}