diff --git a/.github/ISSUE_TEMPLATE/darc_feature_request.md b/.github/ISSUE_TEMPLATE/darc_feature_request.md index e1475b3bb..59862c892 100644 --- a/.github/ISSUE_TEMPLATE/darc_feature_request.md +++ b/.github/ISSUE_TEMPLATE/darc_feature_request.md @@ -6,7 +6,7 @@ labels: enhancement assignees: "" --- -# ๐Ÿง  D: DEFINE โ€” Problem & Opportunity +# ๐Ÿง  D: DOCUMENT โ€” Problem & Opportunity **What is the problem, limitation, or opportunity? Why does this matter for SKaiNET?** @@ -60,7 +60,7 @@ Document research tasks or open questions that must be answered before implement --- -# ๐Ÿ› ๏ธ C: CONTRIBUTE โ€” Implementation Plan +# ๐Ÿ› ๏ธ C: CODE โ€” Implementation Plan Break down actionable steps required to deliver this feature: diff --git a/build-logic/convention/src/main/kotlin/GenerateDocumentationTask.kt b/build-logic/convention/src/main/kotlin/GenerateDocumentationTask.kt index 459d3fa68..dabbae093 100644 --- a/build-logic/convention/src/main/kotlin/GenerateDocumentationTask.kt +++ b/build-logic/convention/src/main/kotlin/GenerateDocumentationTask.kt @@ -108,10 +108,20 @@ abstract class GenerateDocumentationTask : DefaultTask() { .toSortedSet() .toList() - // Row view: (operator, function) pair -> per-backend status. - data class Row(val operator: String, val function: String, val status: Map) + // Row view carries the per-function status plus DARC validation + // signal so the matrix can render both in one pass. + val partialsRoot = derivePartialsRoot(outputDir) + data class Row( + val operator: String, + val function: FunctionDoc, + val hasPartial: Boolean, + ) val rows: List = module.operators.flatMap { op -> - op.functions.map { fn -> Row(op.name, fn.name, fn.statusByBackend) } + op.functions.map { fn -> + val relative = "ops/${op.name.lowercase()}/${fn.name.lowercase()}.adoc" + val present = partialsRoot?.let { File(it, relative).isFile } == true + Row(op.name, fn, present) + } } matrixFile.writeText(buildString { @@ -120,7 +130,7 @@ abstract class GenerateDocumentationTask : DefaultTask() { appendLine("") appendLine("Generated from `operators.json` version `${module.version}` on ${formatTimestamp(module.timestamp)}.") appendLine("") - appendLine("Rows are `Operator.function` pairs; columns are backends that appear in any function's `statusByBackend` map. A missing entry means the backend makes no claim about the function โ€” treat it as \"unknown\", not \"not supported\".") + appendLine("Rows are `Operator.function` pairs. The `Validated` column shows whether the function's documentation has been DARC-validated by a reviewer (see xref:contributing/darc-workflow.adoc[DARC workflow]). Remaining columns are backends that appear in any function's `statusByBackend` map โ€” a missing entry means the backend makes no claim about the function (treat it as \"unknown\", not \"not supported\").") appendLine("") if (rows.isEmpty() || allBackends.isEmpty()) { appendLine("NOTE: No backend status information found in the source data.") @@ -128,32 +138,32 @@ abstract class GenerateDocumentationTask : DefaultTask() { return@buildString } - // Table header: 1 col for the row label + 1 col per backend. - val colSpec = (listOf("2") + List(allBackends.size) { "1" }).joinToString(",") + // Header: row label, validation column, then one column per backend. + val colSpec = (listOf("2", "1") + List(allBackends.size) { "1" }).joinToString(",") appendLine("[cols=\"$colSpec\", options=\"header\"]") appendLine("|===") - append("| Operator.function ") + append("| Operator.function | Validated ") allBackends.forEach { append("| $it ") } appendLine("") appendLine("") rows.forEach { row -> - append("| `${row.operator}.${row.function}` ") + append("| `${row.operator}.${row.function.name}` ") + append("| ${darcCell(row.function, row.hasPartial)} ") allBackends.forEach { backend -> - val raw = row.status[backend] + val raw = row.function.statusByBackend[backend] val cell = if (raw == null) "โ€”" else shortStatus(raw) append("| $cell ") } appendLine("") } - // Totals footer: number of "done" rows per backend out - // of total row count. A status counts as done when it - // maps to the green check in shortStatus. + // Totals footer: validated count followed by per-backend done count. appendLine("") - append("| *Done* ") + val validatedCount = rows.count { it.function.validated } + append("| *Done* | *$validatedCount / ${rows.size}* ") allBackends.forEach { backend -> - val n = rows.count { isDone(it.status[backend]) } + val n = rows.count { isDone(it.function.statusByBackend[backend]) } append("| *$n / ${rows.size}* ") } appendLine("") @@ -163,6 +173,42 @@ abstract class GenerateDocumentationTask : DefaultTask() { }) } + /** + * One-line DARC validation badge rendered above each function's + * signature on the generated page. + * + * Three states, ordered from strongest to weakest signal: + * - validated: a reviewer (not the original author) has signed off + * on the partial prose. Carries the validator and date. + * - prose without validation: a partial exists but no + * `@DarcValidated` annotation backs it. Treated as a stub. + * - no prose: only auto-generated facts (signature, parameters, + * return type). The reader is told explicitly so they don't + * mistake terseness for completeness. + * + * The bracketed CSS class is a hook for the Antora UI bundle to + * style the badge; the visible text is the contract. + */ + private fun darcBadge(function: FunctionDoc, hasPartial: Boolean): String = when { + function.validated -> { + val on = function.validatedOn.ifBlank { "an unspecified date" } + val by = function.validatedBy.ifBlank { "an unspecified reviewer" } + "[.darc-validated]#โœ… DARC-validated by $by on $on#" + } + hasPartial -> "[.darc-stub]#โš  Prose present but not DARC-validated#" + else -> "[.darc-none]#โœ– Generated facts only (no human prose)#" + } + + /** + * Emoji-only DARC status for the coverage matrix column. + * Matches the three branches of [darcBadge]. + */ + private fun darcCell(function: FunctionDoc, hasPartial: Boolean): String = when { + function.validated -> "โœ…" + hasPartial -> "โš " + else -> "โœ–" + } + /** * Short emoji-only rendering of a backend status, for use in the * compact matrix cells. @@ -316,6 +362,8 @@ abstract class GenerateDocumentationTask : DefaultTask() { builder.apply { appendLine("== ${function.name}") appendLine("") + appendLine(darcBadge(function, hasPartial)) + appendLine("") appendLine("=== Signature") appendLine("") appendLine("[source,kotlin]") diff --git a/build-logic/convention/src/main/kotlin/models/DocumentationModels.kt b/build-logic/convention/src/main/kotlin/models/DocumentationModels.kt index fd176a336..7b8717d9f 100644 --- a/build-logic/convention/src/main/kotlin/models/DocumentationModels.kt +++ b/build-logic/convention/src/main/kotlin/models/DocumentationModels.kt @@ -29,7 +29,12 @@ data class FunctionDoc( val parameters: List = emptyList(), val returnType: String, val statusByBackend: Map = emptyMap(), - val notes: List = emptyList() + val notes: List = emptyList(), + val validated: Boolean = false, + val validatedBy: String = "", + val validatedOn: String = "", + val validatedCommit: String = "", + val referencesChecked: Boolean = true, ) @Serializable diff --git a/build-logic/convention/src/main/resources/schemas/operator-doc-schema-v1.json b/build-logic/convention/src/main/resources/schemas/operator-doc-schema-v1.json index 05e361e07..53b50512e 100644 --- a/build-logic/convention/src/main/resources/schemas/operator-doc-schema-v1.json +++ b/build-logic/convention/src/main/resources/schemas/operator-doc-schema-v1.json @@ -13,8 +13,8 @@ }, "version": { "type": "string", - "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9.-]+)?$", - "description": "Semantic version of the framework" + "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9.-]+)?$|^unknown$", + "description": "Semantic version of the framework, or 'unknown' when the KSP processor was invoked without the skainet.version option (e.g. from a unit-test fixture)." }, "commit": { "type": "string", @@ -57,7 +57,7 @@ }, "modality": { "type": "string", - "enum": ["core", "vision", "nlp"], + "enum": ["core", "composite", "vision", "nlp"], "description": "Modality category of the operator" }, "functions": { @@ -101,7 +101,7 @@ "patternProperties": { "^[a-zA-Z][a-zA-Z0-9_]*$": { "type": "string", - "enum": ["implemented", "not_implemented", "in_progress"], + "enum": ["implemented", "inherited", "in_progress", "not_implemented"], "description": "Implementation status for this backend" } }, @@ -114,6 +114,26 @@ "$ref": "#/$defs/Note" }, "description": "Array of notes associated with the function" + }, + "validated": { + "type": "boolean", + "description": "Whether the function's documentation has been DARC-validated by a reviewer. Sourced from the @DarcValidated annotation on the function." + }, + "validatedBy": { + "type": "string", + "description": "Identity of the reviewer who DARC-validated this function's documentation." + }, + "validatedOn": { + "type": "string", + "description": "ISO-8601 date the documentation was DARC-validated." + }, + "validatedCommit": { + "type": "string", + "description": "Optional short git SHA pinning the validated prose to a specific revision." + }, + "referencesChecked": { + "type": "boolean", + "description": "Whether the reviewer verified that every citation in the partial still resolves and supports its claim." } }, "required": ["name", "signature", "parameters", "returnType", "statusByBackend", "notes"], @@ -145,7 +165,7 @@ "properties": { "type": { "type": "string", - "enum": ["owner", "issue"], + "enum": ["owner", "issue", "description"], "description": "Type of the note" }, "backend": { diff --git a/docs/modules/ROOT/images/darc-logo.png b/docs/modules/ROOT/images/darc-logo.png new file mode 100644 index 000000000..393abf9da Binary files /dev/null and b/docs/modules/ROOT/images/darc-logo.png differ diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 136a3b98e..7899d82ce 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -33,9 +33,9 @@ .Contributing * xref:contributing/index.adoc[Audience and scope] +* xref:contributing/darc-workflow.adoc[DARC: advanced contribution workflow] * xref:contributing/build-from-source.adoc[Build from source] * xref:contributing/dtype-model.adoc[The SKaiNET dtype model] * xref:contributing/benchmarks.adoc[Engine benchmark program] * xref:contributing/matmul-kernels.adoc[Reading the matmul benchmark] * xref:contributing/register-bench-runner.adoc[Register a self-hosted bench runner] -* xref:contributing/native-ffm-plan.adoc[Plan: native FFM kernel provider] diff --git a/docs/modules/ROOT/pages/contributing/darc-workflow.adoc b/docs/modules/ROOT/pages/contributing/darc-workflow.adoc new file mode 100644 index 000000000..d552fb2e3 --- /dev/null +++ b/docs/modules/ROOT/pages/contributing/darc-workflow.adoc @@ -0,0 +1,229 @@ += DARC: Advanced Contribution Workflow +:description: SKaiNET's general Document / Assess / Research / Code workflow for advanced contributions, plus its specialisation for operator documentation. + +image::darc-logo.png[DARC,240] + +[NOTE] +==== +**Audience: SKaiNET maintainers and contributors taking on non-trivial +work.** Library *consumers* do not need this page. Routine fixes โ€” a +typo, an obvious bug, a CI tweak โ€” also do not need DARC; the workflow +exists for contributions where the open question "is this even the +right thing to build?" is real. +==== + +== What DARC is + +DARC is SKaiNET's cyclical, document-driven workflow for advanced +contributions: new operators, new modules, algorithmic changes, kernel +strategies, format readers, anything where the design decisions matter +as much as the code. This page is the authoritative definition โ€” the +four phases, when they apply, and how their outcomes get encoded in +source. + +DARC is *cyclical* and *document-driven*: the prose is the deliverable +that survives the iteration, the code follows. Teams can enter at any +phase and revisit earlier ones โ€” a code review that surfaces a stale +assumption sends the work back to Research, not forward to release. + +== The four phases + +[cols="1,4",options="header"] +|=== +| Phase | Purpose + +| *D โ€” Document* +| Capture *what* and *why* in source-controlled prose before the code +exists. The artefact lives in `docs/modules/ROOT/...` as AsciiDoc, or +on the relevant feature proposal issue (see the +https://github.com/SKaiNET-developers/SKaiNET/blob/develop/.github/ISSUE_TEMPLATE/darc_feature_request.md[DARC +Feature Proposal] template). Documentation is a first-class part of +the design, not a write-up of an already-shipped decision. + +| *A โ€” Assess* +| Validate the proposal against ground truth: reference implementations +(PyTorch, JAX, NumPy), benchmarks, numerical-stability edge cases, +cross-platform behaviour (JVM, Android, native). The output is a +decision: approve, send back to Research, or back to Document if the +problem statement turned out to be wrong. + +| *R โ€” Research* +| Survey existing solutions, dependencies, prior art, papers. Resolve +the open questions raised in Document and Assess. Every claim that +ends up in the final prose has a citation a reader can follow. + +| *C โ€” Code* +| Implement against the documented design with Gitflow, KtLint / +Detekt, unit and integration tests, CI green. The Code phase produces +running software, but the *design* it implements was settled in the +prior phases โ€” Code is execution, not invention. +|=== + +Repeat until the artefact converges. The minimum is one full cycle; +in practice most non-trivial contributions go through the loop more +than once. + +== When DARC applies + +[cols="1,1",options="header"] +|=== +| Use DARC when | Skip DARC when + +| Adding a new operator, layer, module, format reader, or backend. +| Fixing a typo or an obvious one-line bug. + +| Changing public API surface or numerical behaviour of existing code. +| Reformatting, renaming a variable, removing dead code. + +| Introducing a new dependency, build tool, or CI workflow. +| Bumping a dependency version with no behavioural change. + +| Algorithmic work where the design decisions are not self-evident from +the diff (kernel strategies, quantisation schemes, attention variants). +| Adding a missing test for already-shipped behaviour. +|=== + +The signal for "DARC applies" is whether a reasonable maintainer six +months from now would want to know *why* the change is shaped the way +it is. If yes, the work belongs in DARC; if the diff speaks for +itself, it doesn't. + +== Specialisation: operator documentation + +Operator documentation is the largest single application of DARC in +the project today โ€” a four-figure number of method-level pages, each +fusing auto-derived API facts with human-written math, intuition, +examples, and references. Because the surface is huge and uneven, +operator docs are the only DARC artefact that carries a +machine-checkable validation flag. + +=== Mapping DARC to operator-doc work + +[cols="1,4",options="header"] +|=== +| Phase | What it means for an operator + +| *D โ€” Document* +| Write or refresh the partial at +`docs/modules/ROOT/partials/ops/\{operator}/\{function}.adoc`. The +partial is sliced into the generated reference page by the `math`, +`intuition`, `examples`, `references` AsciiDoc tags. + +| *A โ€” Assess* +| Validate the prose against ground truth: a reference implementation +(PyTorch / JAX / NumPy), known numerical-stability edge cases, and +SKaiNET's runtime. Run every documented example end-to-end; shapes, +dtypes, and numerical answers must match. + +| *R โ€” Research* +| Every non-trivial claim has a citation. Reviewer clicks every link +and confirms it still resolves and supports the claim. Dead or +drifted citations are a hard fail. + +| *C โ€” Code* +| Confirm the implementation matches the documented contract. +`statusByBackend` reflects reality โ€” no `inherited` rows that should +be `implemented`, no `implemented` rows whose backend currently +throws. +|=== + +=== The `@DarcValidated` annotation + +When an operator function has passed DARC end-to-end, the *reviewer* +(not the original author) annotates the function in source: + +[source,kotlin] +---- +import sk.ainet.lang.ops.DarcValidated + +public interface TensorOps { + @DarcValidated(by = "First Last ", on = "2026-05-24") + public fun matmul(a: Tensor, b: Tensor): Tensor +} +---- + +The KSP processor (`OperatorDocProcessor`) picks up the annotation +and threads it through `operators.json` into the generated reference +page and the coverage matrix: + +[cols="1,4",options="header"] +|=== +| Signal | Meaning + +| `โœ… DARC-validated by โ€ฆ on โ€ฆ` +| `@DarcValidated` is present. A reviewer has gone through all four +phases. + +| `โš  Prose present but not DARC-validated` +| A partial exists at `partials/ops/\{op}/\{fn}.adoc` but no +annotation backs it. Treat the prose as a stub. + +| `โœ– Generated facts only (no human prose)` +| No partial on disk and no annotation. The reader sees only the +auto-derived signature, parameters, return type, and backend status. +|=== + +The same three states appear as `โœ… / โš  / โœ–` in the *Validated* column +of xref:reference/ops-status-matrix.adoc[the operator coverage matrix]. + +=== Criteria for setting `@DarcValidated` + +All of the following must be true before the annotation goes in: + +. The reviewer is not the original author. Two pairs of eyes on the + prose is the entire point. +. *D โ€” Document.* The partial has non-empty `math`, `intuition`, + `examples`, and `references` tags. None are placeholders. +. *A โ€” Assess.* Every example in the partial has been executed + against the SKaiNET implementation and produces the documented + result. Boundary cases for any numerical-stability claims have + been spot-checked. +. *R โ€” Research.* Every citation resolves and supports the specific + claim it backs. Broken or drifted links are a hard fail. +. *C โ€” Code.* `statusByBackend` reflects reality. + +Set `referencesChecked = false` only as a deliberate signal โ€” e.g. a +citation behind a paywall the reviewer could not open. The badge +still renders as validated; the flag is there so a future reviewer +knows what was skipped. + +=== Updating an already-validated operator + +Editing the prose of a `@DarcValidated` function invalidates the +validation. The contributor making the change must: + +. Remove the `@DarcValidated` annotation in the same change that + edits the partial. The page falls back to the โš  badge until a + fresh review. +. Open a new review request once the change is in. + +This is the price of keeping the validation signal in code: any +prose change has to flow through a fresh review before the badge +comes back. + +== Where DARC lives in the repo + +[cols="2,3",options="header"] +|=== +| Location | Role + +| This page (`docs/modules/ROOT/pages/contributing/darc-workflow.adoc`) +| Authoritative definition of the workflow, when it applies, and how +its outcomes are encoded. + +| `.github/ISSUE_TEMPLATE/darc_feature_request.md` +| Issue template for new DARC proposals. Section headers match the +four phases: Document / Assess / Research / Code. + +| `skainet-lang-ksp-annotations/.../DarcValidated.kt` +| The annotation that records a passed DARC review on an operator +function. `SOURCE` retention, `FUNCTION` target. + +| `skainet-lang-ksp-processor/.../OperatorDocProcessor.kt` +| Reads `@DarcValidated` per function and writes the validation +fields into `operators.json`. + +| `build-logic/convention/.../GenerateDocumentationTask.kt` +| Renders the badge above each function's signature and the +*Validated* column in the coverage matrix. +|=== diff --git a/docs/modules/ROOT/pages/contributing/index.adoc b/docs/modules/ROOT/pages/contributing/index.adoc index 4e6320f27..5d1d3fbb8 100644 --- a/docs/modules/ROOT/pages/contributing/index.adoc +++ b/docs/modules/ROOT/pages/contributing/index.adoc @@ -32,7 +32,6 @@ Concretely: | xref:contributing/benchmarks.adoc[Engine benchmark program] | The Phoronix Test Suite / OpenBenchmarking publication path: methodology, manifest, lanes, CI workflow, replay. | xref:contributing/matmul-kernels.adoc[Reading the matmul benchmark] | What the published numbers mean โ€” scalar vs. Panama vs. quantized regimes, roofline reasoning, and a checklist for interpreting any new measurement. | xref:contributing/register-bench-runner.adoc[Register a self-hosted bench runner] | The one-time operator setup that lights up the full-publish CI lane on a Linux x86 box. -| xref:contributing/native-ffm-plan.adoc[Plan: native FFM kernel provider] | Forward-looking design doc for the priority-100 kernel provider that closes the gap to native BLAS. |=== == What this section deliberately does *not* cover diff --git a/docs/modules/ROOT/pages/contributing/matmul-kernels.adoc b/docs/modules/ROOT/pages/contributing/matmul-kernels.adoc index 4f546793e..66d6c2d41 100644 --- a/docs/modules/ROOT/pages/contributing/matmul-kernels.adoc +++ b/docs/modules/ROOT/pages/contributing/matmul-kernels.adoc @@ -314,8 +314,7 @@ This kernel is now **compute-bound on a single core, near the single-core ceiling**. The remaining headroom requires architectural moves: AVX-512 (16 lanes โ€” 2ร— headroom; needs a wider microkernel set), multi-threading (6 cores โ€” 6ร— headroom; needs `ith`/`nth` -SPI), or a hand-tuned native kernel via FFM -(xref:contributing/native-ffm-plan.adoc[]). +SPI), or a hand-tuned native kernel via FFM. If you take only one comparison away from this page: **the scalar โ†’ Panama jump (1ร— โ†’ 13ร—) is the biggest single performance @@ -540,4 +539,3 @@ It is **not** the right answer when: * xref:explanation/perf/simd-kernels.adoc[How SIMD kernels are built] โ€” the FP32 Panama kernel walk-through and the full kernel-provider SPI rationale. * xref:explanation/perf/quantized-simd-kernels.adoc[How quantized SIMD kernels are built] โ€” inner loops for Q4_0 / Q4_K / Q6_K / Q8_0. * xref:explanation/perf/jvm-cpu.adoc[JVM CPU performance] โ€” JVM flags, vector module enablement, JIT considerations. -* xref:contributing/native-ffm-plan.adoc[Plan: native FFM kernel provider] โ€” the path to the priority-100 native kernels. diff --git a/docs/modules/ROOT/pages/contributing/native-ffm-plan.adoc b/docs/modules/ROOT/pages/contributing/native-ffm-plan.adoc deleted file mode 100644 index 436a72c14..000000000 --- a/docs/modules/ROOT/pages/contributing/native-ffm-plan.adoc +++ /dev/null @@ -1,285 +0,0 @@ -= Plan: Native (FFM) Kernel Provider -:description: Where the JVM Vector kernels stop, what a native priority-100 provider would look like, and when to build it. - -[NOTE] -==== -**Audience: SKaiNET contributors.** Design doc for a kernel backend -that is not yet shipped. Library users do not need to read this; the -in-process Panama Vector kernels are the production code path today. -==== - -This page is a *plan*, not shipped code. The intent is to capture -enough detail that the design doesn't drift between the time someone -decides to start the work and the moment a PR is opened. The earlier -content of this page lived briefly in `NATIVE_FFM_KERNEL_PROVIDER.md` -at the repo root and was removed on advice of "ship the release first, -keep the plan in docs"; this is its permanent home. - -== Where the JVM Vector kernels run out - -After the M5 milestone work landed (PRs #554โ€“#565 across the 0.21.0 -release), every CPU matmul path goes through the kernel SPI โ€” see -xref:explanation/perf/simd-kernels.adoc[] and -xref:explanation/perf/quantized-simd-kernels.adoc[]. The Panama Vector -provider runs at: - -* ~73 GFLOPS on FP32 4096ยฒ matmul (Apple Silicon NEON) -* ~73 GFLOPS on Q4_K 4096ยฒ matmul-vector (same regime; fused dequant -adds essentially zero cost on top of the FMA) - -That's already in the ggml NEON ballpark in absolute terms. But -ggml's hand-tuned NEON / AVX2 still outruns the JVM Vector API on: - -* dense FLOPs/cycle on shapes the Vector API can't tile-block -optimally (the 8ร—8ร—128 default is heuristic) -* AVX-512 VNNI fused INT8 dot products -* NEON `bf16` / `fp16` SDOT instructions -* future SVE / SME โ€” none of which the Vector API exposes portably -today - -A native provider closes that gap and unlocks two follow-ons that -*can't* be built on the Vector API alone: - -. *M4 โ†” M5 zero-copy.* Mmap'd Q4_K weights stay as `MemorySegment` -views; a native kernel reads the same pages with no heap copy and -no staging buffer. -. *Hardware-specific lanes* unreachable from portable Vector code. - -== Provider shape - -[cols="1,1,1",options="header"] -|=== -| Priority | Provider | Status -| 0 | `ScalarKernelProvider` | shipped (PR #554) -| 50 | `PanamaVectorKernelProvider` | shipped (PRs #557, #560 + ServiceLoader #559) -| *100* | *`NativeKernelProvider` (FFM)* | *this plan* -|=== - -The `KernelRegistry.bestAvailable()` cascade means: when the native -lib loads, native wins; when it doesn't (sandbox, missing arch, JDK -without FFM, kill-switch flipped), Panama wins; on Native targets and -JS / Wasm where neither is available, scalar wins. No code change -above the registry layer. - -== Goals - -. *A `NativeKernelProvider` registered at priority 100* that on JDK -21+ wins `KernelRegistry.bestAvailable()` over Panama whenever the -native lib loads successfully. -. *A first concrete kernel: native Q4_K matmul.* It must: -.. take a `MemorySegment` for both input (FP32) and packed Q4_K -weights (canonical ggml layout โ€” same as `Q4_KBlockTensorData` -and `matmulF32Q4_KMemSeg`); -.. produce numerically equivalent output to -`PanamaVectorQ4KMatmulKernel` within `1e-4` relative tolerance -(same parity bar `PanamaVectorQ4KMatmulKernelTest` uses); -.. clear *โ‰ฅ2.5ร—* over the prior Q4_K scalar dequant baseline โ€” the -M5 success metric โ€” on the bench shapes from -`QuantizedMatmulBench` (1024ยฒ, 4096ร—1024, 4096ยฒ). -. *Optional follow-on kernels* โ€” Q6_K, Q8_0, FP32 โ€” share the build -system but each ship as a separate small PR. -. *One supported architecture for the first PR* (likely Apple -Silicon NEON since that's the development hardware in use), with a -clear extension path for `linuxX64` AVX2 / `linuxArm64` NEON. - -== Non-goals - -* *JNI.* The roadmap explicitly says "FFM not JNI". JNI's per-call -overhead and the global JNI lock are wrong for hot per-token -kernels; FFM (Java 22 stable, Java 21 preview) gives near-zero -overhead native calls and direct `MemorySegment` ABI. -* *Cross-compilation matrix on day one.* The first PR can ship just -one (host-arch) variant; CI cross-arch builds come later. -* *Replacing Panama.* Panama remains the priority-50 fallback for -environments that can't load native libs (sandboxes, Wasm, Native -targets, JDK without `jdk.incubator.vector`). -* *Distribution via pre-built native artifacts on Maven Central.* -Out of scope for the first PR โ€” local build only. Publishing -classifier JARs comes in a separate plan. - -== Architecture - -=== Module layout - -[source] ----- -skainet-backends/ - skainet-backend-native-cpu/ # NEW - src/ - jvmMain/kotlin/sk/ainet/exec/kernel/ # Kotlin side - NativeKernelProvider.kt # priority=100, isAvailable()=libLoaded - NativeQ4KMatmulKernel.kt # implements Q4KMatmulKernel via FFM - NativeLibraryLoader.kt # System.loadLibrary, locate, version - jvmMain/resources/META-INF/services/ - sk.ainet.backend.api.kernel.KernelProvider # appends NativeKernelProviderFactory - jvmTest/kotlin/sk/ainet/exec/kernel/ - NativeQ4KMatmulKernelTest.kt # parity vs PanamaVectorQ4KMatmulKernel - native/ # native source tree - c/ - q4k_matmul.c # ggml-style hand-tuned kernel - q4k_matmul.h - CMakeLists.txt # or Bazel BUILD - build.gradle.kts # Gradle wrapper that invokes CMake ----- - -The native library compiles to a shared object (`libskainet_kernels.dylib` -on macOS, `.so` on Linux, `.dll` on Windows) and is packaged into the -module's resources for `System.loadLibrary` discovery. - -=== FFM binding pattern - -Single C entry point per kernel: - -[source,c] ----- -// q4k_matmul.h -void skainet_q4k_matmul( - const float* input, // FP32 input vector, length input_dim - const uint8_t* weight, // packed Q4_K bytes (canonical ggml layout) - int32_t weight_byte_offset, - int32_t input_dim, - int32_t output_dim, - float* output, // FP32 output, length output_dim - int32_t output_offset -); ----- - -Kotlin side: - -[source,kotlin] ----- -internal object NativeQ4KMatmulKernel : Q4KMatmulKernel { - private val handle: MethodHandle = run { - val arena = Arena.ofAuto() - val symbol = NativeLibraryLoader.lib.find("skainet_q4k_matmul").orElseThrow() - Linker.nativeLinker().downcallHandle( - symbol, - FunctionDescriptor.ofVoid( - ValueLayout.ADDRESS, ValueLayout.ADDRESS, ValueLayout.JAVA_INT, - ValueLayout.JAVA_INT, ValueLayout.JAVA_INT, - ValueLayout.ADDRESS, ValueLayout.JAVA_INT, - ), - ) - } - - override fun matmul( - input: FloatArray, inputOffset: Int, - weight: ByteArray, weightByteOffset: Int, - inputDim: Int, outputDim: Int, - output: FloatArray, outputOffset: Int, - ) { - // Heap arrays: pass via temporary off-heap MemorySegment + bulk copy, - // OR (preferred) overload with a MemorySegment-input variant for - // mmap'd weights to avoid the copy. - } -} ----- - -The cleaner path is to introduce a sibling `Q4KMemSegMatmulKernel` -SPI (mentioned as out-of-scope in PR #563) that takes `MemorySegment` -directly, and have the native provider implement *that* โ€” no heap -copy. The `Q4KMatmulKernel` (`ByteArray`) variant can wrap the -MemSeg one with a temporary `Arena.ofConfined()` copy if needed for -legacy callers. - -=== Build system - -*Gradle + CMake* is the path of least resistance: - -* A new Gradle module (or hand-rolled `Exec` tasks) invokes CMake -for the native module's `build` task. -* Native artifacts land in `build/native//` and are copied -into `src/jvmMain/resources/native/-/` so -`System.loadLibrary` finds them. -* Kotlin compile depends on the native artifact being built first. - -The xnnpack backend already in the repo -(`skainet-backends/skainet-backend-xnnpack/`) demonstrates a similar -pattern โ€” Gradle invokes CMake to build a native lib via cinterop. -*Reuse that template* rather than reinventing. - -== Staged delivery - -PRs in order, each independently mergeable: - -. *`skainet-backend-native-cpu` module scaffolding.* Gradle module, -`build.gradle.kts` wired to invoke CMake, a *trivial* C kernel -(e.g. just multiplies its first input by 2.0) to prove the FFM -pipeline end-to-end. `NativeKernelProvider` that's `isAvailable() -= false` until the real kernel lands. Sets up CI artifact path on -host arch. -. *First real native kernel: Q4_K matmul (Apple Silicon NEON).* -Hand-tuned kernel, parity tests vs `PanamaVectorQ4KMatmulKernel`, -JMH bench variant added to `QuantizedMatmulBench`. -. *`Q4KMemSegMatmulKernel` SPI sibling + native variant.* Closes -the M4โ†”M5 zero-copy story for mmap'd weights. -. *`linuxX64` AVX2 variant + cross-arch CI build.* The -cross-compilation matrix story. -. *Optional: native FP32 matmul, native Q6_K, native Q8_0.* Same -shape as PRs 2โ€“3, one per format. - -The first PR is the largest in scaffolding terms (~500โ€“800 LoC of -build glue + 1 trivial kernel), but every subsequent PR is small and -template-able. - -== Success metrics - -* *PR 2 sign-off*: native Q4_K matmul on Apple Silicon clears *โ‰ฅ2.5ร—* -over the scalar Q4_K dequant-then-matmul baseline at 4096ยฒ (the M5 -milestone target). For reference: Panama Q4_K SIMD already exceeds -this metric (~73 GFLOPS, see -xref:explanation/perf/quantized-simd-kernels.adoc[]), so the bar is -"beats Panama by a meaningful margin", probably โ‰ฅ1.5ร— over Panama. -* *PR 3 sign-off*: Q4_K MemSeg native path is faster than the Panama -Q4_K MemSeg path from PR #563, with no heap copy in the timed -region. -* *No regression on JVM-only environments* โ€” when the native lib -fails to load (sandbox, missing arch, kill-switch), `bestAvailable()` -cleanly falls through to Panama, and existing tests / benches show -the same numbers as today. - -== Risks & open questions - -. *JDK 21 preview FFM vs JDK 22 stable.* FFM left preview in Java 22. -The repo currently builds on JDK 21 with `--enable-preview ---add-modules jdk.incubator.vector`. Recommendation: stay on 21 -preview; flip to 22 in a separate toolchain-bump PR. -. *`MethodHandle` invocation overhead.* Even with FFM, each native -call has a small fixed cost (~ยตs). For the smallest matmul shapes -(e.g. 256ยฒ FP32) this could swamp the FLOPs win. Mitigation: route -small inputs to Panama and large inputs to native at the -registry/provider level, OR accept that the win is sized for -production-relevant shapes (4096ยฒ+). -. *Native code quality and maintenance.* Hand-tuned NEON / AVX2 in C -is harder to audit than Kotlin Vector API code. Mitigation: keep -kernels small (<300 LoC each), parity-test exhaustively, prefer -porting from ggml's reference (BSD-licensed, well-vetted) over -writing from scratch. -. *Distribution.* Native artifacts complicate Maven Central -publication (need `` per OS/arch). Not a blocker for -the first internal-use PR; a separate "publish native classifier -JARs" plan will be needed before community use. -. *Cross-arch CI cost.* Building NEON natively on Apple Silicon CI -plus AVX2 on linuxX64 plus Android NDK doubles or triples build -time. The xnnpack backend's existing CI matrix is a precedent โ€” -reuse the same approach. -. *Native `MemorySegment` lifetime.* The Kotlin caller owns the -`Arena` for arrays it copies in. The native kernel must NOT retain -pointers past the FFM call return. Document this contract in -`NativeQ4KMatmulKernel.matmul` kdoc. - -== When to start - -Trigger conditions (any one): - -* Real workload demands the native โ‰ฅ2.5ร— target (Panama Q4_K stops -being fast enough on a customer machine). -* A community contributor offers a hand-tuned NEON / AVX2 Q4_K -kernel that's measurably faster than Panama. -* A second M5 metric (e.g. SDPA throughput, training-loop -throughput) needs hand-tuned native code. - -Until then: *pause.* The Panama provider is doing the -milestone-equivalent work in absolute terms, and adding a native -build system is a meaningful complexity tax to take on -speculatively. diff --git a/docs/modules/ROOT/pages/explanation/perf/quantized-simd-kernels.adoc b/docs/modules/ROOT/pages/explanation/perf/quantized-simd-kernels.adoc index 23ab9e9d0..f3cf8a0c9 100644 --- a/docs/modules/ROOT/pages/explanation/perf/quantized-simd-kernels.adoc +++ b/docs/modules/ROOT/pages/explanation/perf/quantized-simd-kernels.adoc @@ -230,6 +230,4 @@ shape as the Q4_K rewrite, with the lane-interleave done via |=== For the kernel SPI itself, see -xref:explanation/perf/simd-kernels.adoc[]. For the planned native FFM -provider that would replace the Vector path on supported hosts, see -xref:contributing/native-ffm-plan.adoc[]. +xref:explanation/perf/simd-kernels.adoc[]. diff --git a/docs/modules/ROOT/pages/explanation/perf/simd-kernels.adoc b/docs/modules/ROOT/pages/explanation/perf/simd-kernels.adoc index 810b32381..ea2c3081f 100644 --- a/docs/modules/ROOT/pages/explanation/perf/simd-kernels.adoc +++ b/docs/modules/ROOT/pages/explanation/perf/simd-kernels.adoc @@ -31,7 +31,7 @@ don't carry that"). Three providers ship with the CPU backend today: | Provider | Priority | When available | Notes | `ScalarKernelProvider` | 0 | always | Three-loop reference; the parity baseline. | `PanamaVectorKernelProvider` | 50 | JDK 21+ with `--add-modules jdk.incubator.vector` and `skainet.cpu.vector.enabled != false` | Tile-blocked FMA; the production winner on every supported JVM. -| (future) `NativeKernelProvider` | 100 | JDK 22+ with the native lib loaded | Captured as a plan in xref:contributing/native-ffm-plan.adoc[]; not yet shipped. +| (future) `NativeKernelProvider` | 100 | JDK 22+ with the native lib loaded | Designed but not yet shipped. |=== == Why the SPI exists @@ -239,6 +239,3 @@ no-regression change. For quantized matmul (Q4_K, Q6_K, Q8_0, Q4_0) โ€” same story, different inner loop โ€” see xref:explanation/perf/quantized-simd-kernels.adoc[]. - -For the still-unbuilt native FFM provider, see -xref:contributing/native-ffm-plan.adoc[]. diff --git a/docs/modules/ROOT/pages/reference/architecture.adoc b/docs/modules/ROOT/pages/reference/architecture.adoc index 0430a8992..9fb4759d9 100644 --- a/docs/modules/ROOT/pages/reference/architecture.adoc +++ b/docs/modules/ROOT/pages/reference/architecture.adoc @@ -131,8 +131,7 @@ Introduced in 0.21.0 (PRs #554, #559, #562). The static structure: ---- Three live providers ship; a fourth (priority 100, native FFM) is -captured as a plan in xref:contributing/native-ffm-plan.adoc[] -and not yet built. For *how* the kernels are implemented, see +designed but not yet built. For *how* the kernels are implemented, see xref:explanation/perf/simd-kernels.adoc[] (FP32) and xref:explanation/perf/quantized-simd-kernels.adoc[] (quantized). @@ -268,8 +267,7 @@ canary. * *No native FFM provider yet.* The literal M5 milestone metric (`โ‰ฅ2.5ร—` for Q4_K) is met by Panama in absolute terms but not in the "native vs JVM" framing the metric originally specified. -Mitigation: `xref:contributing/native-ffm-plan.adoc[]` -documents what shipping it would look like. +The priority-100 native provider is designed but not yet shipped. * *Two reverted optimizations on develop history.* MemSeg pool (commit 8642b322) and intra-op matmul parallelism (commit 9ed633b6) were both tried and reverted. Re-attempts need a diff --git a/docs/modules/ROOT/pages/reference/operators/generated/index.adoc b/docs/modules/ROOT/pages/reference/operators/generated/index.adoc index 9eab83e0b..3c2fd15dc 100644 --- a/docs/modules/ROOT/pages/reference/operators/generated/index.adoc +++ b/docs/modules/ROOT/pages/reference/operators/generated/index.adoc @@ -1,6 +1,6 @@ = AI-NET Operators Reference -Generated from version `0.19.0` on 2026-04-15 +Generated from version `0.23.0` on 2026-05-24 == Operators by Modality diff --git a/docs/modules/ROOT/pages/reference/operators/generated/similarity.adoc b/docs/modules/ROOT/pages/reference/operators/generated/similarity.adoc index 0901bb891..f3c0cde52 100644 --- a/docs/modules/ROOT/pages/reference/operators/generated/similarity.adoc +++ b/docs/modules/ROOT/pages/reference/operators/generated/similarity.adoc @@ -6,6 +6,8 @@ Modality: Composite == cosineDistance +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] diff --git a/docs/modules/ROOT/pages/reference/operators/generated/tensorops.adoc b/docs/modules/ROOT/pages/reference/operators/generated/tensorops.adoc index cd3c671b7..fd54fe42c 100644 --- a/docs/modules/ROOT/pages/reference/operators/generated/tensorops.adoc +++ b/docs/modules/ROOT/pages/reference/operators/generated/tensorops.adoc @@ -6,6 +6,8 @@ Modality: Core == add +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -24,6 +26,8 @@ fun add(a:Tensor, b:Tensor): Tensor == subtract +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -42,6 +46,8 @@ fun subtract(a:Tensor, b:Tensor): Tensor == multiply +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -60,6 +66,8 @@ fun multiply(a:Tensor, b:Tensor): Tensor == divide +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -78,6 +86,8 @@ fun divide(a:Tensor, b:Tensor): Tensor == addScalar +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -96,6 +106,8 @@ fun addScalar(a:Tensor, b:Number): Tensor == subScalar +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -114,6 +126,8 @@ fun subScalar(a:Tensor, b:Number): Tensor == mulScalar +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -132,6 +146,8 @@ fun mulScalar(a:Tensor, b:Number): Tensor == divScalar +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -150,6 +166,8 @@ fun divScalar(a:Tensor, b:Number): Tensor == rsubScalar +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -168,6 +186,8 @@ fun rsubScalar(a:Number, b:Tensor): Tensor == rdivScalar +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -186,6 +206,8 @@ fun rdivScalar(a:Number, b:Tensor): Tensor == matmul +[.darc-validated]#โœ… DARC-validated by SKaiNET docs maintainers on 2026-05-24# + === Signature [source,kotlin] @@ -222,6 +244,8 @@ include::partial$ops/tensorops/matmul.adoc[tag=references,optional] == transpose +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -237,8 +261,32 @@ fun transpose(tensor:Tensor): Tensor `Tensor` +== permute + +[.darc-none]#โœ– Generated facts only (no human prose)# + +=== Signature + +[source,kotlin] +---- +fun permute(tensor:Tensor, axes:IntArray): Tensor +---- + +=== Parameters + +* `tensor: Tensor` + input tensor, any rank โ‰ฅ 1 +* `axes: IntArray` + a permutation of `0..tensor.rank-1` (length must equal `tensor.rank`, every value in `[0, rank)` exactly once) + +=== Return Type + +`Tensor` + == conv1d +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -262,6 +310,8 @@ fun conv1d(input:Tensor, weight:Tensor, bias:Tensor, stride:Int, padding:Int, di == conv2d +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -285,6 +335,8 @@ fun conv2d(input:Tensor, weight:Tensor, bias:Tensor, stride:Pair, padding:Pair, == conv3d +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -308,6 +360,8 @@ fun conv3d(input:Tensor, weight:Tensor, bias:Tensor, stride:Triple, padding:Trip == convTranspose1d +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -332,6 +386,8 @@ fun convTranspose1d(input:Tensor, weight:Tensor, bias:Tensor, stride:Int, paddin == maxPool2d +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -352,6 +408,8 @@ fun maxPool2d(input:Tensor, kernelSize:Pair, stride:Pair, padding:Pair): Tensor == avgPool2d +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -373,6 +431,8 @@ fun avgPool2d(input:Tensor, kernelSize:Pair, stride:Pair, padding:Pair, countInc == upsample2d +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -393,6 +453,8 @@ fun upsample2d(input:Tensor, scale:Pair, mode:UpsampleMode, alignCorners:Boolean == reshape +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -411,6 +473,8 @@ fun reshape(tensor:Tensor, newShape:Shape): Tensor == flatten +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -430,6 +494,8 @@ fun flatten(tensor:Tensor, startDim:Int, endDim:Int): Tensor == concat +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -448,6 +514,8 @@ fun concat(tensors:List, dim:Int): Tensor == split +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -467,6 +535,8 @@ fun split(tensor:Tensor, splitSize:Int, dim:Int): List == squeeze +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -485,6 +555,8 @@ fun squeeze(tensor:Tensor, dim:Int): Tensor == unsqueeze +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -503,6 +575,8 @@ fun unsqueeze(tensor:Tensor, dim:Int): Tensor == relu +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -520,6 +594,8 @@ fun relu(tensor:Tensor): Tensor == leakyRelu +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -538,6 +614,8 @@ fun leakyRelu(tensor:Tensor, negativeSlope:Float): Tensor == elu +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -556,6 +634,8 @@ fun elu(tensor:Tensor, alpha:Float): Tensor == softmax +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -574,6 +654,8 @@ fun softmax(tensor:Tensor, dim:Int): Tensor == logSoftmax +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -592,6 +674,8 @@ fun logSoftmax(tensor:Tensor, dim:Int): Tensor == sigmoid +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -609,6 +693,8 @@ fun sigmoid(tensor:Tensor): Tensor == silu +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -626,6 +712,8 @@ fun silu(tensor:Tensor): Tensor == gelu +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -643,6 +731,8 @@ fun gelu(tensor:Tensor): Tensor == sum +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -661,6 +751,8 @@ fun sum(tensor:Tensor, dim:Int): Tensor == mean +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -679,6 +771,8 @@ fun mean(tensor:Tensor, dim:Int): Tensor == variance +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -697,6 +791,8 @@ fun variance(tensor:Tensor, dim:Int): Tensor == sqrt +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -712,8 +808,107 @@ fun sqrt(tensor:Tensor): Tensor `Tensor` +== pow + +[.darc-none]#โœ– Generated facts only (no human prose)# + +=== Signature + +[source,kotlin] +---- +fun pow(a:Tensor, b:Tensor): Tensor +---- + +=== Parameters + +* `a: Tensor` +* `b: Tensor` + +=== Return Type + +`Tensor` + +== powScalar + +[.darc-none]#โœ– Generated facts only (no human prose)# + +=== Signature + +[source,kotlin] +---- +fun powScalar(a:Tensor, n:Number): Tensor +---- + +=== Parameters + +* `a: Tensor` +* `n: Number` + +=== Return Type + +`Tensor` + +== log + +[.darc-none]#โœ– Generated facts only (no human prose)# + +=== Signature + +[source,kotlin] +---- +fun log(tensor:Tensor): Tensor +---- + +=== Parameters + +* `tensor: Tensor` + +=== Return Type + +`Tensor` + +== log2 + +[.darc-none]#โœ– Generated facts only (no human prose)# + +=== Signature + +[source,kotlin] +---- +fun log2(tensor:Tensor): Tensor +---- + +=== Parameters + +* `tensor: Tensor` + +=== Return Type + +`Tensor` + +== log10 + +[.darc-none]#โœ– Generated facts only (no human prose)# + +=== Signature + +[source,kotlin] +---- +fun log10(tensor:Tensor): Tensor +---- + +=== Parameters + +* `tensor: Tensor` + +=== Return Type + +`Tensor` + == abs +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -731,6 +926,8 @@ fun abs(tensor:Tensor): Tensor == sign +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -748,6 +945,8 @@ fun sign(tensor:Tensor): Tensor == clamp +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -767,6 +966,8 @@ fun clamp(tensor:Tensor, minVal:Float, maxVal:Float): Tensor == narrow +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -787,6 +988,8 @@ fun narrow(tensor:Tensor, dim:Int, start:Int, length:Int): Tensor == pad2d +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -808,6 +1011,8 @@ fun pad2d(tensor:Tensor, padLeft:Int, padRight:Int, padTop:Int, padBottom:Int): == unfold +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -828,6 +1033,8 @@ fun unfold(tensor:Tensor, dim:Int, size:Int, step:Int): Tensor == lt +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -846,6 +1053,8 @@ fun lt(tensor:Tensor, value:Float): Tensor == ge +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -864,6 +1073,8 @@ fun ge(tensor:Tensor, value:Float): Tensor == tril +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -882,6 +1093,8 @@ fun tril(tensor:Tensor, k:Int): Tensor == convert +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -900,6 +1113,8 @@ fun convert(tensor:Tensor, targetType:TTo): Tensor == gather +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -919,6 +1134,8 @@ fun gather(input:Tensor, indices:Tensor, dim:Int): Tensor == indexSelect +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -938,6 +1155,8 @@ fun indexSelect(input:Tensor, indices:Tensor, dim:Int): Tensor == exp +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -955,6 +1174,8 @@ fun exp(tensor:Tensor): Tensor == expm1 +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -972,6 +1193,8 @@ fun expm1(tensor:Tensor): Tensor == sin +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -989,6 +1212,8 @@ fun sin(tensor:Tensor): Tensor == cos +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -1006,6 +1231,8 @@ fun cos(tensor:Tensor): Tensor == tanh +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] @@ -1023,6 +1250,8 @@ fun tanh(tensor:Tensor): Tensor == scaledDotProductAttention +[.darc-none]#โœ– Generated facts only (no human prose)# + === Signature [source,kotlin] diff --git a/docs/modules/ROOT/pages/reference/ops-status-matrix.adoc b/docs/modules/ROOT/pages/reference/ops-status-matrix.adoc index 46ace7bef..2398a9c93 100644 --- a/docs/modules/ROOT/pages/reference/ops-status-matrix.adoc +++ b/docs/modules/ROOT/pages/reference/ops-status-matrix.adoc @@ -1,72 +1,78 @@ = Operator Coverage Matrix :description: Cross-backend status for every operator function in SKaiNET. -Generated from `operators.json` version `0.19.0` on 2026-04-15. +Generated from `operators.json` version `0.23.0` on 2026-05-24. -Rows are `Operator.function` pairs; columns are backends that appear in any function's `statusByBackend` map. A missing entry means the backend makes no claim about the function โ€” treat it as "unknown", not "not supported". +Rows are `Operator.function` pairs. The `Validated` column shows whether the function's documentation has been DARC-validated by a reviewer (see xref:contributing/darc-workflow.adoc[DARC workflow]). Remaining columns are backends that appear in any function's `statusByBackend` map โ€” a missing entry means the backend makes no claim about the function (treat it as "unknown", not "not supported"). -[cols="2,1,1,1", options="header"] +[cols="2,1,1,1,1", options="header"] |=== -| Operator.function | apple | cpu | wasm +| Operator.function | Validated | apple | cpu | wasm -| `TensorOps.add` | โ€” | โ€” | โ€” -| `TensorOps.subtract` | โ€” | โ€” | โ€” -| `TensorOps.multiply` | โ€” | โ€” | โ€” -| `TensorOps.divide` | โ€” | โ€” | โ€” -| `TensorOps.addScalar` | โ€” | โ€” | โ€” -| `TensorOps.subScalar` | โ€” | โ€” | โ€” -| `TensorOps.mulScalar` | โ€” | โ€” | โ€” -| `TensorOps.divScalar` | โ€” | โ€” | โ€” -| `TensorOps.rsubScalar` | โ€” | โ€” | โ€” -| `TensorOps.rdivScalar` | โ€” | โ€” | โ€” -| `TensorOps.matmul` | โ€” | โ€” | โ€” -| `TensorOps.transpose` | โ€” | โ€” | โ€” -| `TensorOps.conv1d` | โ€” | โ€” | โ€” -| `TensorOps.conv2d` | โ€” | โ€” | โ€” -| `TensorOps.conv3d` | โ€” | โ€” | โ€” -| `TensorOps.convTranspose1d` | โ€” | โ€” | โ€” -| `TensorOps.maxPool2d` | โ€” | โ€” | โ€” -| `TensorOps.avgPool2d` | โ€” | โ€” | โ€” -| `TensorOps.upsample2d` | โ€” | โ€” | โ€” -| `TensorOps.reshape` | โ€” | โ€” | โ€” -| `TensorOps.flatten` | โ€” | โ€” | โ€” -| `TensorOps.concat` | โ€” | โ€” | โ€” -| `TensorOps.split` | โ€” | โ€” | โ€” -| `TensorOps.squeeze` | โ€” | โ€” | โ€” -| `TensorOps.unsqueeze` | โ€” | โ€” | โ€” -| `TensorOps.relu` | โ€” | โ€” | โ€” -| `TensorOps.leakyRelu` | โ€” | โ€” | โ€” -| `TensorOps.elu` | โ€” | โ€” | โ€” -| `TensorOps.softmax` | โ€” | โ€” | โ€” -| `TensorOps.logSoftmax` | โ€” | โ€” | โ€” -| `TensorOps.sigmoid` | โ€” | โ€” | โ€” -| `TensorOps.silu` | โ€” | โ€” | โ€” -| `TensorOps.gelu` | โ€” | โ€” | โ€” -| `TensorOps.sum` | โ€” | โ€” | โ€” -| `TensorOps.mean` | โ€” | โ€” | โ€” -| `TensorOps.variance` | โ€” | โ€” | โ€” -| `TensorOps.sqrt` | โ€” | โ€” | โ€” -| `TensorOps.abs` | โ€” | โ€” | โ€” -| `TensorOps.sign` | โ€” | โ€” | โ€” -| `TensorOps.clamp` | โ€” | โ€” | โ€” -| `TensorOps.narrow` | โ€” | โ€” | โ€” -| `TensorOps.pad2d` | โ€” | โ€” | โ€” -| `TensorOps.unfold` | โ€” | โ€” | โ€” -| `TensorOps.lt` | โ€” | โ€” | โ€” -| `TensorOps.ge` | โ€” | โ€” | โ€” -| `TensorOps.tril` | โ€” | โ€” | โ€” -| `TensorOps.convert` | โ€” | โ€” | โ€” -| `TensorOps.gather` | โ€” | โ€” | โ€” -| `TensorOps.indexSelect` | โ€” | โ€” | โ€” -| `TensorOps.exp` | โ€” | โ€” | โ€” -| `TensorOps.expm1` | โ€” | โ€” | โ€” -| `TensorOps.sin` | โ€” | โ€” | โ€” -| `TensorOps.cos` | โ€” | โ€” | โ€” -| `TensorOps.tanh` | โ€” | โ€” | โ€” -| `TensorOps.scaledDotProductAttention` | โ€” | โ€” | โ€” -| `Similarity.cosineDistance` | โœ… | โœ… | โœ… +| `TensorOps.add` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.subtract` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.multiply` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.divide` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.addScalar` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.subScalar` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.mulScalar` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.divScalar` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.rsubScalar` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.rdivScalar` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.matmul` | โœ… | โ€” | โ€” | โ€” +| `TensorOps.transpose` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.permute` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.conv1d` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.conv2d` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.conv3d` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.convTranspose1d` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.maxPool2d` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.avgPool2d` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.upsample2d` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.reshape` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.flatten` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.concat` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.split` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.squeeze` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.unsqueeze` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.relu` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.leakyRelu` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.elu` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.softmax` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.logSoftmax` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.sigmoid` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.silu` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.gelu` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.sum` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.mean` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.variance` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.sqrt` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.pow` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.powScalar` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.log` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.log2` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.log10` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.abs` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.sign` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.clamp` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.narrow` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.pad2d` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.unfold` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.lt` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.ge` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.tril` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.convert` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.gather` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.indexSelect` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.exp` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.expm1` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.sin` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.cos` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.tanh` | โœ– | โ€” | โ€” | โ€” +| `TensorOps.scaledDotProductAttention` | โœ– | โ€” | โ€” | โ€” +| `Similarity.cosineDistance` | โœ– | โœ… | โœ… | โœ… -| *Done* | *1 / 56* | *1 / 56* | *1 / 56* +| *Done* | *1 / 62* | *1 / 62* | *1 / 62* | *1 / 62* |=== Per-function detail including notes lives in xref:reference/operators/generated/index.adoc[Operator reference]. diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/TensorOps.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/TensorOps.kt index 6d2962e13..7a0f5082f 100644 --- a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/TensorOps.kt +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/TensorOps.kt @@ -7,6 +7,7 @@ import sk.ainet.lang.trace.GenerateTracingWrapper import sk.ainet.lang.trace.Diff import sk.ainet.lang.nn.dsl.GenerateNetworkDsl import sk.ainet.lang.nn.dsl.ActivationDsl +import sk.ainet.lang.ops.DarcValidated @GenerateTracingWrapper @GenerateNetworkDsl @@ -49,6 +50,7 @@ public interface TensorOps { * broadcast against [a] using the usual broadcasting rules. */ @Diff + @DarcValidated(by = "SKaiNET docs maintainers", on = "2026-05-24") public fun matmul(a: Tensor, b: Tensor): Tensor @Diff public fun transpose(tensor: Tensor): Tensor diff --git a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/sk/ainet/lang/ops/DarcValidated.kt b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/sk/ainet/lang/ops/DarcValidated.kt new file mode 100644 index 000000000..a6a086b6a --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/sk/ainet/lang/ops/DarcValidated.kt @@ -0,0 +1,33 @@ +package sk.ainet.lang.ops + +/** + * Marks a [TensorOp] function as DARC-validated. + * + * A reviewer (not the original author) has gone through Document, Assess, + * Research, and Code for this function โ€” read the partial prose, checked the + * math against a reference implementation, verified the citations, and + * confirmed the runtime behaviour matches the documented contract. + * + * Picked up by `OperatorDocProcessor` and rendered as a badge on the + * generated operator page plus a column in the ops coverage matrix. + * + * See `contributing/darc-workflow.adoc` for the exact criteria. + * + * @param by Validator identity. Free-form, but `"First Last "` + * is the convention so the same string can be linked back to git history. + * @param on ISO-8601 date the validation completed, e.g. `"2026-05-24"`. + * @param commit Optional short SHA pinning the validated prose. Empty when + * the validation is not pinned to a specific revision. + * @param referencesChecked Whether the reviewer verified that every link and + * citation in the partial still resolves and supports the claim it backs. + * Defaults to `true` because if it weren't, the validation would not have + * passed; set to `false` only as a deliberate documentation signal. + */ +@Target(AnnotationTarget.FUNCTION) +@Retention(AnnotationRetention.SOURCE) +public annotation class DarcValidated( + val by: String, + val on: String, + val commit: String = "", + val referencesChecked: Boolean = true, +) diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt index e50b6455d..6472c9562 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt @@ -30,7 +30,14 @@ data class FunctionDoc( val parameters: List, val returnType: String, val statusByBackend: Map, - val notes: List + val notes: List, + // DARC validation metadata. `validated = false` means @DarcValidated is + // absent โ€” the generator will render a "not validated" badge. + val validated: Boolean = false, + val validatedBy: String = "", + val validatedOn: String = "", + val validatedCommit: String = "", + val referencesChecked: Boolean = true, ) data class ParameterDoc( @@ -180,13 +187,19 @@ class OperatorDocProcessor( } statusByBackend[backendId] = if (overrides) "implemented" else "inherited" } + val validation = extractDarcValidation(fn) FunctionDoc( name = fn.simpleName.asString(), signature = fn.toSignatureString(), parameters = extractParameters(fn), returnType = extractReturnType(fn), statusByBackend = statusByBackend, - notes = emptyList() + notes = emptyList(), + validated = validation.validated, + validatedBy = validation.by, + validatedOn = validation.on, + validatedCommit = validation.commit, + referencesChecked = validation.referencesChecked, ) } @@ -260,13 +273,56 @@ class OperatorDocProcessor( } private fun createFunctionDoc(function: KSFunctionDeclaration): FunctionDoc { + val validation = extractDarcValidation(function) return FunctionDoc( name = function.simpleName.asString(), signature = function.toSignatureString(), parameters = extractParameters(function), returnType = extractReturnType(function), statusByBackend = deriveStatusByBackend(function), - notes = deriveNotes(function) + notes = deriveNotes(function), + validated = validation.validated, + validatedBy = validation.by, + validatedOn = validation.on, + validatedCommit = validation.commit, + referencesChecked = validation.referencesChecked, + ) + } + + private data class DarcValidation( + val validated: Boolean, + val by: String, + val on: String, + val commit: String, + val referencesChecked: Boolean, + ) + + /** + * Read the `@DarcValidated` annotation off a function, if present. + * Returns a sentinel with `validated = false` when the annotation is + * absent, which the generator renders as the "not validated" badge. + */ + private fun extractDarcValidation(function: KSFunctionDeclaration): DarcValidation { + val annotation = function.annotations.find { + it.shortName.asString() == "DarcValidated" + } ?: return DarcValidation(false, "", "", "", true) + + val by = annotation.arguments.find { it.name?.asString() == "by" } + ?.value?.toString().orEmpty() + val on = annotation.arguments.find { it.name?.asString() == "on" } + ?.value?.toString().orEmpty() + val commit = annotation.arguments.find { it.name?.asString() == "commit" } + ?.value?.toString().orEmpty() + val refsChecked = (annotation.arguments.find { + it.name?.asString() == "referencesChecked" + }?.value as? Boolean) ?: true + + return DarcValidation( + validated = true, + by = by, + on = on, + commit = commit, + referencesChecked = refsChecked, ) } @@ -509,7 +565,21 @@ class OperatorDocProcessor( append("{\"type\": \"${escapeJson(note.type)}\", \"backend\": \"${escapeJson(note.backend)}\", \"content\": \"${escapeJson(note.content)}\"}") if (noteIndex < function.notes.size - 1) append(", ") } - append("]\n") + append("]") + + // DARC validation block. Only emitted when an actual + // @DarcValidated annotation is present, so unannotated + // functions keep the JSON narrow. + if (function.validated) { + append(",\n") + append(" \"validated\": true,\n") + append(" \"validatedBy\": \"${escapeJson(function.validatedBy)}\",\n") + append(" \"validatedOn\": \"${escapeJson(function.validatedOn)}\",\n") + append(" \"validatedCommit\": \"${escapeJson(function.validatedCommit)}\",\n") + append(" \"referencesChecked\": ${function.referencesChecked}\n") + } else { + append("\n") + } append(" }") if (funcIndex < operator.functions.size - 1) append(",") diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/metadata/DocumentationModels.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/metadata/DocumentationModels.kt index 98ec26128..e92079779 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/metadata/DocumentationModels.kt +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/metadata/DocumentationModels.kt @@ -36,7 +36,12 @@ data class FunctionDoc( val parameters: List, val returnType: String, val statusByBackend: Map, - val notes: List + val notes: List, + val validated: Boolean = false, + val validatedBy: String = "", + val validatedOn: String = "", + val validatedCommit: String = "", + val referencesChecked: Boolean = true, ) /** diff --git a/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessorTest.kt b/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessorTest.kt index dcbc6a8b1..07ce9c556 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessorTest.kt +++ b/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessorTest.kt @@ -67,6 +67,86 @@ class OperatorDocProcessorTest { // but the key requirement is that the test compiles and runs } + @Test + fun testDarcValidatedFlowsIntoJson() { + // Two functions annotated with @InProgress so the processor picks + // them up via the annotation-discovery path. Only one of them + // carries @DarcValidated โ€” we expect that one (and only that one) + // to land in operators.json with the validation block. + val sourceCode = """ + package sk.ainet.lang.ops + + @Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) + @Retention(AnnotationRetention.SOURCE) + annotation class InProgress( + vararg val backends: String, + val owner: String = "", + val issue: String = "" + ) + + @Target(AnnotationTarget.FUNCTION) + @Retention(AnnotationRetention.SOURCE) + annotation class DarcValidated( + val by: String, + val on: String, + val commit: String = "", + val referencesChecked: Boolean = true, + ) + + @InProgress("cpu", owner = "ops-team", issue = "GH-1") + @DarcValidated(by = "Reviewer One", on = "2026-05-24") + fun validatedFn(): String = "ok" + + @InProgress("cpu", owner = "ops-team", issue = "GH-2") + fun plainFn(): String = "ok" + """.trimIndent() + + val source = SourceFile.kotlin("sk/ainet/lang/ops/TestDarcOps.kt", sourceCode) + + val compilation = KotlinCompilation().apply { + sources = listOf(source) + configureKsp {} + symbolProcessorProviders = mutableListOf(OperatorDocProcessorProvider()) + inheritClassPath = true + messageOutputStream = System.out + } + + val result = compilation.compile() + val output = result.messages + println("[DEBUG_LOG] Compilation result: ${result.exitCode}") + println("[DEBUG_LOG] Output messages: $output") + + assertTrue(output.contains("Generated operators.json"), + "Processor should generate operators.json") + + // Locate the JSON the processor just wrote. KSP code generator + // outputs resources somewhere under the compilation's working + // dir; the layout is version-dependent, so search rather than + // hard-code a path. + val operatorsJson = compilation.workingDir.walkTopDown() + .firstOrNull { it.isFile && it.name == "operators.json" } + assertTrue(operatorsJson != null && operatorsJson.exists(), + "operators.json should be present in compilation output") + val text = operatorsJson.readText() + println("[DEBUG_LOG] operators.json contents: $text") + + assertTrue(text.contains("\"validatedFn\""), + "validatedFn should be present in JSON") + assertTrue(text.contains("\"validated\": true"), + "validated=true should be emitted for the annotated function") + assertTrue(text.contains("\"validatedBy\": \"Reviewer One\""), + "validatedBy should carry the reviewer identity") + assertTrue(text.contains("\"validatedOn\": \"2026-05-24\""), + "validatedOn should carry the ISO date") + + // The unannotated function must NOT carry a validation block. + // The processor only emits the keys when validated=true, so a + // single occurrence in the file is the expected count. + val validatedKeyCount = Regex("\"validated\":\\s*true").findAll(text).count() + assertTrue(validatedKeyCount == 1, + "Exactly one function should emit validated=true (got $validatedKeyCount)") + } + @Test fun testDslOpAnnotationProcessing() { val sourceCode = """