feat(memory): PlatformStorage — common door to platform storage kinds; Kotlin/Native malloc/mmap; JS/Wasm heap fallback (SKEEP-003 P2, S1.1c)

skainet-lang/skainet-lang-core/api/jvm/skainet-lang-core.api

@@ -799,6 +799,30 @@ public final class sk/ainet/lang/memory/Owner$Owned : sk/ainet/lang/memory/Owner
 	public fun toString ()Ljava/lang/String;
 }
 
+public final class sk/ainet/lang/memory/PlatformStorage {
+	public static final field INSTANCE Lsk/ainet/lang/memory/PlatformStorage;
+	public final fun allocate (JLsk/ainet/lang/tensor/storage/MemoryDomain;Lsk/ainet/lang/memory/ScopeKind;Lsk/ainet/lang/tensor/TensorId;Lsk/ainet/lang/memory/trace/TraceSink;)Lsk/ainet/lang/memory/Storage;
+	public static synthetic fun allocate$default (Lsk/ainet/lang/memory/PlatformStorage;JLsk/ainet/lang/tensor/storage/MemoryDomain;Lsk/ainet/lang/memory/ScopeKind;Lsk/ainet/lang/tensor/TensorId;Lsk/ainet/lang/memory/trace/TraceSink;ILjava/lang/Object;)Lsk/ainet/lang/memory/Storage;
+	public final fun getInfo ()Lsk/ainet/lang/memory/PlatformStorageInfo;
+	public final fun getSupportsMappedFiles ()Z
+	public final fun mapFile (Ljava/lang/String;JJLsk/ainet/lang/memory/ScopeKind;Lsk/ainet/lang/tensor/TensorId;Lsk/ainet/lang/memory/trace/TraceSink;)Lsk/ainet/lang/memory/Storage;
+	public static synthetic fun mapFile$default (Lsk/ainet/lang/memory/PlatformStorage;Ljava/lang/String;JJLsk/ainet/lang/memory/ScopeKind;Lsk/ainet/lang/tensor/TensorId;Lsk/ainet/lang/memory/trace/TraceSink;ILjava/lang/Object;)Lsk/ainet/lang/memory/Storage;
+	public final fun supports (Lsk/ainet/lang/tensor/storage/MemoryDomain;)Z
+}
+
+public final class sk/ainet/lang/memory/PlatformStorageInfo {
+	public fun <init> (Ljava/lang/String;Ljava/lang/String;)V
+	public final fun component1 ()Ljava/lang/String;
+	public final fun component2 ()Ljava/lang/String;
+	public final fun copy (Ljava/lang/String;Ljava/lang/String;)Lsk/ainet/lang/memory/PlatformStorageInfo;
+	public static synthetic fun copy$default (Lsk/ainet/lang/memory/PlatformStorageInfo;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Lsk/ainet/lang/memory/PlatformStorageInfo;
+	public fun equals (Ljava/lang/Object;)Z
+	public final fun getMapped ()Ljava/lang/String;
+	public final fun getOffHeap ()Ljava/lang/String;
+	public fun hashCode ()I
+	public fun toString ()Ljava/lang/String;
+}
+
 public final class sk/ainet/lang/memory/ScopeKind : java/lang/Enum {
 	public static final field AMBIENT Lsk/ainet/lang/memory/ScopeKind;
 	public static final field FORWARD Lsk/ainet/lang/memory/ScopeKind;

skainet-lang/skainet-lang-core/src/androidMain/kotlin/sk/ainet/lang/memory/PlatformStorage.android.kt

@@ -0,0 +1,29 @@
+package sk.ainet.lang.memory
+
+import sk.ainet.lang.memory.trace.TraceSink
+import sk.ainet.lang.tensor.TensorId
+import sk.ainet.lang.tensor.storage.MemoryDomain
+import java.nio.file.Path
+
+/** Android: off-heap = direct `ByteBuffer` (outside the ART heap limit, #922), mapped = `MappedByteBuffer` (SKEEP-002 / #921). */
+@ExperimentalMemoryApi
+public actual object PlatformStorage {
+    public actual fun supports(domain: MemoryDomain): Boolean = domain == MemoryDomain.HOST_HEAP || domain == MemoryDomain.HOST_OFFHEAP || domain == MemoryDomain.MMAP_FILE
+    public actual val supportsMappedFiles: Boolean get() = true
+
+    public actual fun allocate(bytes: Long, domain: MemoryDomain, scope: ScopeKind, origin: TensorId?, sink: TraceSink): Storage = when (domain) {
+        MemoryDomain.HOST_OFFHEAP, MemoryDomain.HOST_PINNED, MemoryDomain.UNIFIED -> DirectBufferStorage.allocate(checkedInt(bytes), scope, origin, sink)
+        MemoryDomain.HOST_HEAP -> Storage.Heap.bytes(checkedInt(bytes), scope, origin, sink)
+        MemoryDomain.MMAP_FILE, MemoryDomain.DEVICE_LOCAL -> throw IllegalArgumentException("$domain is not allocatable; use mapFile / a device backend")
+    }
+
+    public actual fun mapFile(path: String, fileOffset: Long, length: Long, scope: ScopeKind, origin: TensorId?, sink: TraceSink): Storage =
+        MappedBufferStorage.map(Path.of(path), fileOffset, length, scope, origin, sink)
+
+    public val info: PlatformStorageInfo = PlatformStorageInfo("direct ByteBuffer", "FileChannel.map → MappedByteBuffer")
+}
+
+internal fun checkedInt(bytes: Long): Int {
+    require(bytes in 0..Int.MAX_VALUE.toLong()) { "buffer storage is limited to 2 GB per buffer, requested $bytes bytes" }
+    return bytes.toInt()
+}

skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/memory/PlatformStorage.kt

@@ -0,0 +1,40 @@
+package sk.ainet.lang.memory
+
+import sk.ainet.lang.memory.trace.NoopTraceSink
+import sk.ainet.lang.memory.trace.TraceSink
+import sk.ainet.lang.tensor.TensorId
+import sk.ainet.lang.tensor.storage.MemoryDomain
+
+/**
+ * The platform's storage kinds behind one common door (SKEEP-003 §4.8): *where bytes live is a
+ * policy decision made by the planner; how long they live is a scope decision; neither is made by
+ * a layer, a loader or a kernel.* Callers ask for a [MemoryDomain]; the platform binds it to what
+ * it has — `MemorySegment` / `FileChannel.map` on the JVM, direct `ByteBuffer` / `MappedByteBuffer`
+ * on Android, `malloc` / `mmap` on Kotlin/Native, and the heap on JS/Wasm (no off-heap, no mmap:
+ * a request for those resolves to [Storage.Heap] and [PlatformStorage.supports] says so, so the
+ * planner can note the fallback).
+ */
+@ExperimentalMemoryApi
+public expect object PlatformStorage {
+    /** Whether this target can honour [domain] natively (false = [allocate] falls back to the heap). */
+    public fun supports(domain: MemoryDomain): Boolean
+
+    /** Whether this target can map files ([mapFile] throws [UnsupportedOperationException] otherwise). */
+    public val supportsMappedFiles: Boolean
+
+    /**
+     * Allocate [bytes] zeroed bytes in [domain] (or the closest the platform has), owned by a scope
+     * of kind [scope]. `HOST_HEAP` → [Storage.Heap.bytes]; `HOST_OFFHEAP` → the platform off-heap kind
+     * or the heap fallback; other domains are not allocatable here.
+     */
+    public fun allocate(bytes: Long, domain: MemoryDomain = MemoryDomain.HOST_HEAP, scope: ScopeKind = ScopeKind.AMBIENT, origin: TensorId? = null, sink: TraceSink = NoopTraceSink): Storage
+
+    /** Map `[fileOffset, fileOffset + length)` of the file at [path] read-only into `MODEL`-scoped storage. */
+    public fun mapFile(path: String, fileOffset: Long, length: Long, scope: ScopeKind = ScopeKind.MODEL, origin: TensorId? = null, sink: TraceSink = NoopTraceSink): Storage
+}
+
+/** The kinds a target binds, for diagnostics (`describe()`, the planner's notes). */
+@ExperimentalMemoryApi
+public data class PlatformStorageInfo(val offHeap: String, val mapped: String) {
+    override fun toString(): String = "OffHeap=$offHeap · Mapped=$mapped"
+}

skainet-lang/skainet-lang-core/src/commonTest/kotlin/sk/ainet/lang/memory/PlatformStorageTest.kt

@@ -0,0 +1,56 @@
+package sk.ainet.lang.memory
+
+import sk.ainet.lang.memory.trace.RecordingTraceSink
+import sk.ainet.lang.memory.trace.TraceEvent
+import sk.ainet.lang.tensor.storage.MemoryDomain
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
+import kotlin.test.assertFalse
+import kotlin.test.assertIs
+import kotlin.test.assertTrue
+
+/** SKEEP-003 §4.8: one common door to the platform's storage kinds; heap everywhere, off-heap where the target has it. */
+@OptIn(ExperimentalMemoryApi::class)
+class PlatformStorageTest {
+
+    @Test
+    fun heapIsSupportedEverywhereAndAllocatesHeapStorage() {
+        assertTrue(PlatformStorage.supports(MemoryDomain.HOST_HEAP))
+        val sink = RecordingTraceSink()
+        val s = PlatformStorage.allocate(64, MemoryDomain.HOST_HEAP, ScopeKind.FORWARD, sink = sink)
+        assertIs<Storage.Heap>(s); assertEquals(64L, s.sizeBytes); assertEquals(ScopeKind.FORWARD, s.scope); assertEquals(MemoryDomain.HOST_HEAP, s.domain)
+        assertEquals(64L, assertIs<TraceEvent.Allocation>(sink.events().single()).bytes)
+        s.close(); assertFalse(s.isAlive)
+    }
+
+    @Test
+    fun offHeapRequestYieldsOffHeapOrTheHeapFallback() {
+        val s = PlatformStorage.allocate(128, MemoryDomain.HOST_OFFHEAP, ScopeKind.MODEL)
+        if (PlatformStorage.supports(MemoryDomain.HOST_OFFHEAP)) {
+            assertIs<Storage.OffHeap>(s); assertEquals(MemoryDomain.HOST_OFFHEAP, s.domain)
+        } else {
+            assertIs<Storage.Heap>(s); assertEquals(MemoryDomain.HOST_HEAP, s.domain) // JS/Wasm
+        }
+        assertEquals(128L, s.sizeBytes); assertEquals(ScopeKind.MODEL, s.scope); assertTrue(s.isMutable)
+        val v = s.slice(32, 64); assertEquals(64L, v.sizeBytes); assertIs<Owner.Alias>(v.owner)
+        s.close(); assertFalse(v.isAlive)
+        assertFailsWith<StorageClosedException> { s.checkAlive() }
+    }
+
+    @Test
+    fun deviceAndFileDomainsAreNotAllocatable() {
+        assertFailsWith<IllegalArgumentException> { PlatformStorage.allocate(1, MemoryDomain.DEVICE_LOCAL) }
+        assertFailsWith<IllegalArgumentException> { PlatformStorage.allocate(1, MemoryDomain.MMAP_FILE) }
+        assertFalse(PlatformStorage.supports(MemoryDomain.DEVICE_LOCAL))
+    }
+
+    @Test
+    fun mappedFilesAreEitherSupportedOrClearlyUnsupported() {
+        if (!PlatformStorage.supportsMappedFiles) {
+            assertFailsWith<UnsupportedOperationException> { PlatformStorage.mapFile("/nonexistent", 0, 1) }
+        } else {
+            assertTrue(PlatformStorage.supports(MemoryDomain.MMAP_FILE))
+        }
+    }
+}

skainet-lang/skainet-lang-core/src/jsMain/kotlin/sk/ainet/lang/memory/PlatformStorage.web.kt

@@ -0,0 +1,29 @@
+package sk.ainet.lang.memory
+
+import sk.ainet.lang.memory.trace.TraceSink
+import sk.ainet.lang.tensor.TensorId
+import sk.ainet.lang.tensor.storage.MemoryDomain
+
+/**
+ * JS / Wasm: linear memory only — no off-heap, no mmap (SKEEP-003 §4.8.4). Off-heap requests resolve
+ * to the heap (the planner records the fallback); files arrive through fetch/range requests into
+ * heap slabs, so [mapFile] is unsupported here.
+ */
+@ExperimentalMemoryApi
+public actual object PlatformStorage {
+    public actual fun supports(domain: MemoryDomain): Boolean = domain == MemoryDomain.HOST_HEAP
+    public actual val supportsMappedFiles: Boolean get() = false
+
+    public actual fun allocate(bytes: Long, domain: MemoryDomain, scope: ScopeKind, origin: TensorId?, sink: TraceSink): Storage {
+        require(bytes in 0..Int.MAX_VALUE.toLong()) { "heap storage is limited to 2 GB per array, requested $bytes bytes" }
+        return when (domain) {
+            MemoryDomain.MMAP_FILE, MemoryDomain.DEVICE_LOCAL -> throw IllegalArgumentException("$domain is not allocatable on this target")
+            else -> Storage.Heap.bytes(bytes.toInt(), scope, origin, sink) // HOST_OFFHEAP / PINNED / UNIFIED fall back to the heap
+        }
+    }
+
+    public actual fun mapFile(path: String, fileOffset: Long, length: Long, scope: ScopeKind, origin: TensorId?, sink: TraceSink): Storage =
+        throw UnsupportedOperationException("memory-mapped files are not available on this target (use the suspend RandomAccessSource into heap slabs)")
+
+    public val info: PlatformStorageInfo = PlatformStorageInfo("heap (fallback)", "none")
+}

skainet-lang/skainet-lang-core/src/jvmMain/kotlin/sk/ainet/lang/memory/PlatformStorage.jvm.kt

@@ -0,0 +1,29 @@
+package sk.ainet.lang.memory
+
+import sk.ainet.lang.memory.trace.TraceSink
+import sk.ainet.lang.tensor.TensorId
+import sk.ainet.lang.tensor.storage.MemoryDomain
+import java.nio.file.Path
+
+/** JVM: off-heap = FFM `MemorySegment` ([SegmentStorage]), mapped = `FileChannel.map` ([MappedFileStorage]). */
+@ExperimentalMemoryApi
+public actual object PlatformStorage {
+    public actual fun supports(domain: MemoryDomain): Boolean = domain == MemoryDomain.HOST_HEAP || domain == MemoryDomain.HOST_OFFHEAP || domain == MemoryDomain.MMAP_FILE
+    public actual val supportsMappedFiles: Boolean get() = true
+
+    public actual fun allocate(bytes: Long, domain: MemoryDomain, scope: ScopeKind, origin: TensorId?, sink: TraceSink): Storage = when (domain) {
+        MemoryDomain.HOST_OFFHEAP, MemoryDomain.HOST_PINNED, MemoryDomain.UNIFIED -> SegmentStorage.allocate(bytes, scope, origin = origin, sink = sink)
+        MemoryDomain.HOST_HEAP -> Storage.Heap.bytes(checkedInt(bytes), scope, origin, sink)
+        MemoryDomain.MMAP_FILE, MemoryDomain.DEVICE_LOCAL -> throw IllegalArgumentException("$domain is not allocatable; use mapFile / a device backend")
+    }
+
+    public actual fun mapFile(path: String, fileOffset: Long, length: Long, scope: ScopeKind, origin: TensorId?, sink: TraceSink): Storage =
+        MappedFileStorage.map(Path.of(path), fileOffset, length, scope, origin, sink)
+
+    public val info: PlatformStorageInfo = PlatformStorageInfo("MemorySegment (FFM)", "FileChannel.map → MemorySegment")
+}
+
+internal fun checkedInt(bytes: Long): Int {
+    require(bytes in 0..Int.MAX_VALUE.toLong()) { "heap storage is limited to 2 GB per array, requested $bytes bytes" }
+    return bytes.toInt()
+}

skainet-lang/skainet-lang-core/src/jvmTest/kotlin/sk/ainet/lang/memory/PlatformStorageJvmTest.kt

@@ -0,0 +1,21 @@
+package sk.ainet.lang.memory
+
+import sk.ainet.lang.tensor.storage.MemoryDomain
+import java.nio.file.Files
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertIs
+
+@OptIn(ExperimentalMemoryApi::class)
+class PlatformStorageJvmTest {
+    @Test
+    fun jvmBindsSegmentsAndMappedFiles() {
+        assertIs<SegmentStorage>(PlatformStorage.allocate(16, MemoryDomain.HOST_OFFHEAP))
+        assertIs<Storage.Heap>(PlatformStorage.allocate(16, MemoryDomain.HOST_HEAP))
+        val f = Files.createTempFile("skainet-ps", ".bin"); f.toFile().deleteOnExit(); Files.write(f, ByteArray(64) { it.toByte() })
+        val m = PlatformStorage.mapFile(f.toString(), 8, 16)
+        assertIs<MappedFileStorage>(m); assertEquals(16L, m.sizeBytes); assertEquals(8.toByte(), m.segment().get(java.lang.foreign.ValueLayout.JAVA_BYTE, 0))
+        m.close()
+        assertEquals("OffHeap=MemorySegment (FFM) · Mapped=FileChannel.map → MemorySegment", PlatformStorage.info.toString())
+    }
+}

skainet-lang/skainet-lang-core/src/nativeMain/kotlin/sk/ainet/lang/memory/NativeStorage.kt

@@ -0,0 +1,121 @@
+@file:OptIn(kotlinx.cinterop.ExperimentalForeignApi::class, kotlinx.cinterop.UnsafeNumber::class)
+
+package sk.ainet.lang.memory
+
+import kotlinx.cinterop.COpaquePointer
+import kotlinx.cinterop.convert
+import kotlinx.cinterop.reinterpret
+import kotlinx.cinterop.CPointer
+import kotlinx.cinterop.ByteVar
+import sk.ainet.lang.memory.trace.NoopTraceSink
+import sk.ainet.lang.memory.trace.TraceEvent
+import sk.ainet.lang.memory.trace.TraceSink
+import sk.ainet.lang.tensor.TensorId
+import sk.ainet.lang.tensor.storage.MemoryDomain
+import platform.posix.MAP_FAILED
+import platform.posix.MAP_PRIVATE
+import platform.posix.O_RDONLY
+import platform.posix.PROT_READ
+import platform.posix.close
+import platform.posix.free
+import platform.posix.malloc
+import platform.posix.memset
+import platform.posix.mmap
+import platform.posix.munmap
+import platform.posix.open
+
+/**
+ * Kotlin/Native binding of [Storage.OffHeap]: `malloc`ed bytes (zeroed), freed on [close]
+ * (SKEEP-003 §4.8.3). Alignment: `malloc` gives 16 bytes on every supported platform, which is what
+ * NEON wants; 64-byte alignment for AMX-class paths is a follow-up with `posix_memalign`.
+ */
+@ExperimentalMemoryApi
+public class NativeMallocStorage private constructor(
+    override val id: StorageId,
+    private val ptr: CPointer<ByteVar>,
+    override val sizeBytes: Long,
+    override val owner: Owner,
+    override val debugOrigin: TensorId?,
+    override val sink: TraceSink,
+    private val mutable: Boolean,
+) : Storage.OffHeap() {
+    override val isMutable: Boolean get() = (owner as? Owner.Alias)?.parent?.isMutable ?: mutable
+
+    /** Raw pointer + byte offset is what a C kernel receives; throws after close. */
+    public fun pointer(): CPointer<ByteVar> { checkAlive(); return ptr }
+
+    override fun slice(offsetBytes: Long, lengthBytes: Long): NativeMallocStorage {
+        checkAlive()
+        require(offsetBytes >= 0 && lengthBytes >= 0 && offsetBytes + lengthBytes <= sizeBytes) { "slice [$offsetBytes, ${offsetBytes + lengthBytes}) outside $sizeBytes bytes" }
+        return NativeMallocStorage(StorageId.next(), (ptr.rawValue + offsetBytes).toLong().toCPointer(), lengthBytes, Owner.Alias(this), debugOrigin, sink, mutable)
+    }
+
+    override fun onClose() { if (owner is Owner.Owned) free(ptr) }
+
+    public companion object {
+        public fun allocate(bytes: Long, scope: ScopeKind = ScopeKind.AMBIENT, origin: TensorId? = null, sink: TraceSink = NoopTraceSink): NativeMallocStorage {
+            require(bytes >= 0) { "bytes must be >= 0" }
+            val p = malloc(maxOf(bytes, 1L).convert()) ?: throw IllegalStateException("malloc($bytes) failed")
+            memset(p, 0, maxOf(bytes, 1L).convert())
+            val s = NativeMallocStorage(StorageId.next(), p.reinterpret(), bytes, Owner.Owned(scope), origin, sink, true)
+            if (sink.isEnabled) sink.emit(TraceEvent.Allocation(s.id.value, scope, bytes, origin))
+            return s
+        }
+
+        /** Borrow a caller's buffer — never freed by us. */
+        public fun borrow(pointer: CPointer<ByteVar>, bytes: Long, mutable: Boolean = true, origin: TensorId? = null, sink: TraceSink = NoopTraceSink): NativeMallocStorage =
+            NativeMallocStorage(StorageId.next(), pointer, bytes, Owner.Borrowed(pointer), origin, sink, mutable)
+    }
+}
+
+/** Kotlin/Native binding of [Storage.Mapped]: `mmap(2)` of a file region, read-only, `munmap` on close. */
+@ExperimentalMemoryApi
+public class NativeMappedStorage private constructor(
+    override val id: StorageId,
+    public val path: String,
+    public val fileOffset: Long,
+    private val base: COpaquePointer,
+    private val ptr: CPointer<ByteVar>,
+    override val sizeBytes: Long,
+    private val mappedLength: Long,
+    override val owner: Owner,
+    override val debugOrigin: TensorId?,
+    override val sink: TraceSink,
+) : Storage.Mapped() {
+    override val isMutable: Boolean get() = false
+
+    public fun pointer(): CPointer<ByteVar> { checkAlive(); return ptr }
+
+    override fun slice(offsetBytes: Long, lengthBytes: Long): NativeMappedStorage {
+        checkAlive()
+        require(offsetBytes >= 0 && lengthBytes >= 0 && offsetBytes + lengthBytes <= sizeBytes) { "slice [$offsetBytes, ${offsetBytes + lengthBytes}) outside $sizeBytes bytes" }
+        return NativeMappedStorage(StorageId.next(), path, fileOffset + offsetBytes, base, (ptr.rawValue + offsetBytes).toLong().toCPointer(), lengthBytes, 0L, Owner.Alias(this), debugOrigin, sink)
+    }
+
+    override fun onClose() { if (owner is Owner.Owned) munmap(base, mappedLength.convert()) }
+
+    public companion object {
+        private const val PAGE: Long = 4096L
+
+        public fun map(path: String, fileOffset: Long, length: Long, scope: ScopeKind = ScopeKind.MODEL, origin: TensorId? = null, sink: TraceSink = NoopTraceSink): NativeMappedStorage {
+            require(fileOffset >= 0 && length >= 0) { "offset/length must be >= 0" }
+            val fd = open(path, O_RDONLY)
+            require(fd >= 0) { "cannot open $path" }
+            try {
+                val pageStart = fileOffset - (fileOffset % PAGE)
+                val delta = fileOffset - pageStart
+                val mapLen = maxOf(length + delta, 1L)
+                val p = mmap(null, mapLen.convert(), PROT_READ, MAP_PRIVATE, fd, pageStart.convert())
+                require(p != null && p != MAP_FAILED) { "mmap($path, $fileOffset, $length) failed" }
+                val s = NativeMappedStorage(StorageId.next(), path, fileOffset, p, (p.rawValue + delta).toLong().toCPointer(), length, mapLen, Owner.Owned(scope), origin, sink)
+                if (sink.isEnabled) sink.emit(TraceEvent.Allocation(s.id.value, scope, length, origin, site = path))
+                return s
+            } finally {
+                close(fd)
+            }
+        }
+    }
+}
+
+private fun Long.toCPointer(): CPointer<ByteVar> = kotlinx.cinterop.interpretCPointer<ByteVar>(kotlinx.cinterop.nativeNullPtr + this)
+    ?: throw IllegalStateException("null pointer")