fix(backend): embed the kernel static archive into published K/N klibs (#941)

.github/workflows/publish.yml

@@ -96,6 +96,31 @@ jobs:
           if-no-files-found: error
           retention-days: 14
 
+      # K/N klib embedding (#941): the linux ELF static archives are produced
+      # only on the ubuntu leg — x64 from the host CMake build above, arm64 via
+      # the -PcrossArm64 cross build — and injected into the publish job so the
+      # linux klibs carry their machine code (the publish host is macOS, whose
+      # CMake build emits Mach-O and must not be embedded into linux klibs).
+      - name: Cross-build aarch64 static archive (Linux only)
+        if: matrix.arch_label == 'linux-x86_64'
+        env:
+          GRADLE_OPTS: -Dorg.gradle.jvmargs=-Xmx4g -Dfile.encoding=UTF-8
+        run: |
+          sudo apt-get update -q && sudo apt-get install -y -q gcc-aarch64-linux-gnu
+          ./gradlew --no-daemon --stacktrace --no-configuration-cache -PcrossArm64=true \
+            :skainet-backends:skainet-backend-native-cpu:buildNativeKernelsArm64
+
+      - name: Upload K/N static archives (Linux only)
+        if: matrix.arch_label == 'linux-x86_64'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: native-static-linux
+          path: |
+            skainet-backends/skainet-backend-native-cpu/build/native/cmake-build/libskainet_kernels.a
+            skainet-backends/skainet-backend-native-cpu/build/native/cmake-build-arm64/libskainet_kernels.a
+          if-no-files-found: error
+          retention-days: 14
+
   publish:
     name: Release build and publish
     needs: build-native
@@ -147,8 +172,27 @@ jobs:
           echo "--- Staged tree ---"
           find "$DEST" -type f
 
+      # K/N klib embedding (#941): resolve the linux ELF static archive dirs
+      # from the ubuntu leg's artifact and hand them to Gradle, so the
+      # linuxX64/linuxArm64 klibs embed real machine code even though the
+      # publish host is macOS. Fail loudly if the archives are missing —
+      # publishing bindings-only klibs is the bug this step exists to prevent.
+      - name: Resolve K/N static archive paths
+        run: |
+          set -euo pipefail
+          BASE="$PWD/native-artifacts/native-static-linux"
+          X64_DIR="$BASE/cmake-build"
+          ARM64_DIR="$BASE/cmake-build-arm64"
+          test -f "$X64_DIR/libskainet_kernels.a" || { echo "Missing linux-x64 static archive" >&2; exit 1; }
+          test -f "$ARM64_DIR/libskainet_kernels.a" || { echo "Missing linux-arm64 static archive" >&2; exit 1; }
+          echo "SKAINET_KERNELS_X64_DIR=$X64_DIR" >> "$GITHUB_ENV"
+          echo "SKAINET_KERNELS_ARM64_DIR=$ARM64_DIR" >> "$GITHUB_ENV"
+
       - name: Publish to MavenCentral
-        run: ./gradlew publish --no-configuration-cache --stacktrace
+        run: |
+          ./gradlew publish --no-configuration-cache --stacktrace \
+            -PskainetKernelsX64Dir="$SKAINET_KERNELS_X64_DIR" \
+            -PskainetKernelsArm64Dir="$SKAINET_KERNELS_ARM64_DIR"
         env:
           ORG_GRADLE_PROJECT_mavenCentralUsername: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
           ORG_GRADLE_PROJECT_mavenCentralPassword: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}

CHANGELOG.md

@@ -4,6 +4,17 @@
 
 ### Fixed
 
+- **Published Kotlin/Native klibs for `skainet-backend-native-cpu` now carry their machine
+  code.** The static kernel archive was attached via project-local `linkerOpts`, which does
+  not travel with a published klib — downstream K/N consumers of the `-linuxx64`/`-linuxarm64`
+  artifacts could not link (unresolved `skainet_*` symbols). The archive is now EMBEDDED into
+  the cinterop klib (`-staticLibrary`/`-libraryPath`), conditional on a correct-architecture
+  ELF archive being available; bindings-only builds (e.g. macOS dev hosts) warn loudly instead
+  of staying silent. The release workflow's ubuntu leg now also cross-builds and uploads both
+  linux static archives, and the macOS publish job injects them, so released klibs embed real
+  code. In-repo K/N test binaries link purely from the embedded klib — the consumer-link
+  scenario is what the test suite now exercises. (#941)
+
 - **`TensorData.copyToFloatArray()` default implementation works for rank >= 2.** It used to
   iterate a single flat index into the vararg `get`, tripping every implementation's
   one-index-per-dimension arity check — a latent trap for any implementation that didn't

skainet-backends/skainet-backend-native-cpu/build.gradle.kts

@@ -12,6 +12,31 @@ val staticArchivePath: String =
 val staticArchiveArm64Path: String =
     layout.buildDirectory.file("native/cmake-build-arm64/libskainet_kernels.a").get().asFile.absolutePath
 
+// --- Archive embedding (#941) ----------------------------------------------
+//
+// The static archive must be EMBEDDED into the cinterop klib (cinterop
+// -staticLibrary/-libraryPath) rather than attached via linkerOpts:
+// linkerOpts applies only to this project's own binaries, so a published
+// klib would carry bindings but no machine code and no downstream K/N
+// consumer could link. Embedding is conditional on a correct-architecture
+// ELF archive being available:
+//   - linuxX64:  a Linux host's CMake build (cmake-build/), or an injected
+//                -PskainetKernelsX64Dir (CI: artifact from the ubuntu leg).
+//   - linuxArm64: the -PcrossArm64 cross build (cmake-build-arm64/), or an
+//                injected -PskainetKernelsArm64Dir.
+// A macOS/Windows host produces host-format archives that must NOT be
+// embedded into a Linux klib; those builds fall back to linkerOpts (their
+// K/N test links are disabled anyway, below) and the cinterop task warns
+// that the produced klib is bindings-only.
+val isLinuxHostForEmbed: Boolean = System.getProperty("os.name").lowercase().contains("linux")
+val crossArm64ForEmbed: Boolean = (findProperty("crossArm64") as String?)?.toBoolean() == true
+val injectedX64Dir: String? = findProperty("skainetKernelsX64Dir") as String?
+val injectedArm64Dir: String? = findProperty("skainetKernelsArm64Dir") as String?
+val x64ArchiveDir: String? = injectedX64Dir
+    ?: if (isLinuxHostForEmbed) layout.buildDirectory.dir("native/cmake-build").get().asFile.absolutePath else null
+val arm64ArchiveDir: String? = injectedArm64Dir
+    ?: if (crossArm64ForEmbed) layout.buildDirectory.dir("native/cmake-build-arm64").get().asFile.absolutePath else null
+
 kotlin {
     explicitApi()
     jvm()
@@ -21,15 +46,23 @@ kotlin {
     // linuxX64 = host (POC / CI-runnable); linuxArm64 = the SL2610 board target
     // (its archive is the aarch64 cross-build with NEON). The JVM consumes the
     // same kernels via FFM instead. Shared K/N code lives in `nativeMain`.
-    fun org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget.wireSkainetKernels(archive: String) {
+    fun org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget.wireSkainetKernels(
+        archiveDir: String?,
+        fallbackArchive: String,
+    ) {
         compilations.getByName("main").cinterops.create("skainetKernels") {
             defFile(project.file("src/nativeInterop/cinterop/skainet_kernels.def"))
             includeDirs(nativeIncludeDir)
+            if (archiveDir != null) {
+                extraOpts("-staticLibrary", "libskainet_kernels.a", "-libraryPath", archiveDir)
+            }
         }
-        binaries.all { linkerOpts(archive) }
+        // Without an embeddable archive the old project-local linking keeps the
+        // in-repo binaries working, but the klib itself carries no machine code.
+        if (archiveDir == null) binaries.all { linkerOpts(fallbackArchive) }
     }
-    linuxX64 { wireSkainetKernels(staticArchivePath) }
-    linuxArm64 { wireSkainetKernels(staticArchiveArm64Path) }
+    linuxX64 { wireSkainetKernels(x64ArchiveDir, staticArchivePath) }
+    linuxArm64 { wireSkainetKernels(arm64ArchiveDir, staticArchiveArm64Path) }
 
     sourceSets {
         val jvmMain by getting {
@@ -146,6 +179,39 @@ tasks.matching { it.name.startsWith("link") && it.name.endsWith("LinuxX64") }.co
     dependsOn(buildNativeKernels)
 }
 
+// With archive embedding (#941) the archive must exist at CINTEROP time, not
+// just link time: the cinterop task copies it into the klib.
+if (x64ArchiveDir != null && injectedX64Dir == null) {
+    tasks.matching { it.name == "cinteropSkainetKernelsLinuxX64" }.configureEach {
+        dependsOn(buildNativeKernels)
+    }
+}
+// Bindings-only builds warn loudly instead of publishing broken klibs silently.
+if (x64ArchiveDir == null) {
+    tasks.matching { it.name == "cinteropSkainetKernelsLinuxX64" }.configureEach {
+        doFirst {
+            logger.warn(
+                "skainet-backend-native-cpu: linuxX64 klib is built WITHOUT the embedded " +
+                    "libskainet_kernels.a (no Linux-ELF archive available on this host). " +
+                    "Downstream consumers of this klib cannot link. Build on Linux or pass " +
+                    "-PskainetKernelsX64Dir=<dir> (#941)."
+            )
+        }
+    }
+}
+if (arm64ArchiveDir == null) {
+    tasks.matching { it.name == "cinteropSkainetKernelsLinuxArm64" }.configureEach {
+        doFirst {
+            logger.warn(
+                "skainet-backend-native-cpu: linuxArm64 klib is built WITHOUT the embedded " +
+                    "libskainet_kernels.a. Downstream consumers of this klib cannot link. " +
+                    "Enable -PcrossArm64=true (aarch64 cross toolchain) or pass " +
+                    "-PskainetKernelsArm64Dir=<dir> (#941)."
+            )
+        }
+    }
+}
+
 // The linuxX64/linuxArm64 K/N *test* binaries link the CMake static archive and
 // execute a Linux ELF. On a non-Linux host the CMake build emits host-format
 // objects (Mach-O on macOS), which ld.lld cannot cross-link into a Linux binary,
@@ -262,6 +328,14 @@ if (crossArm64Enabled) {
         dependsOn(buildNativeKernelsArm64)
     }
 
+    // Embedding (#941): the cross-built archive must exist before the arm64
+    // cinterop copies it into the klib.
+    if (injectedArm64Dir == null) {
+        tasks.matching { it.name == "cinteropSkainetKernelsLinuxArm64" }.configureEach {
+            dependsOn(buildNativeKernelsArm64)
+        }
+    }
+
     // The Kotlin Gradle plugin does not create a run task for non-host K/N test
     // binaries (only linkDebugTestLinuxArm64 / linuxArm64TestBinaries), so wire
     // one explicitly: run test.kexe under qemu-aarch64 (user-mode emulation).

skainet-backends/skainet-backend-native-cpu/src/nativeInterop/cinterop/skainet_kernels.def

@@ -1,8 +1,13 @@
 # Kotlin/Native cinterop binding for the hand-written C matmul kernels
-# (skainet_kernels.h). Generates Kotlin bindings for skainet_q5k_matmul etc.;
-# the static archive libskainet_kernels.a (built by CMake) is linked into the
-# consuming K/N binary via linkerOpts (see build.gradle.kts). includeDirs for
-# the header are supplied from the Gradle cinterop block.
+# (skainet_kernels.h). Generates Kotlin bindings for skainet_q5k_matmul etc.
+#
+# The static archive libskainet_kernels.a (built by CMake) is EMBEDDED into
+# the klib via cinterop `-staticLibrary`/`-libraryPath`, passed per target
+# from build.gradle.kts (the archive paths are machine-specific, so they do
+# not belong in this file). Embedding — rather than project-local linkerOpts —
+# is what lets downstream consumers of the published klib link without
+# supplying the archive themselves (#941). includeDirs for the header are
+# supplied from the Gradle cinterop block.
 headers = skainet_kernels.h
 headerFilter = skainet_kernels.h
 package = sk.ainet.kernels.cinterop