diff --git a/scripts/pr-gate.sh b/scripts/pr-gate.sh new file mode 100755 index 000000000..616b88670 --- /dev/null +++ b/scripts/pr-gate.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Local PR gate for SKaiNET: runs the same test legs as CI (.github/workflows/build.yml, +# java-tests.yml) plus the binary-compatibility check, so a PR is opened only after +# everything CI will run has passed locally. Usage: +# +# scripts/pr-gate.sh # full gate +# scripts/pr-gate.sh --bench # full gate + StorageBenchmarks and JMH microbenchmarks +# scripts/pr-gate.sh --quick # JVM leg + apiCheck only (iterate fast, then run the full gate) +# +# Set JAVA_HOME to a JDK 25 (CI uses 25; the build requires >= 21). +set -euo pipefail +cd "$(dirname "$0")/.." + +GRADLE=(./gradlew --no-daemon --stacktrace -Dorg.gradle.caching=true -Dorg.gradle.configuration-cache=true) +mode="${1:-full}" + +# Karma's ChromeHeadless (jsBrowserTest / wasmJsBrowserTest) needs a Chrome/Chromium binary. +# GitHub runners ship one; locally, point CHROME_BIN at whatever is installed. +if [[ -z "${CHROME_BIN:-}" ]]; then + for c in google-chrome google-chrome-stable chromium chromium-browser; do + if command -v "$c" >/dev/null 2>&1; then export CHROME_BIN="$(command -v "$c")"; break; fi + done +fi +echo "pr-gate: JAVA_HOME=${JAVA_HOME:-} CHROME_BIN=${CHROME_BIN:-}" + +step() { echo; echo "=== pr-gate: $* ==="; } + +step "JVM tests" +"${GRADLE[@]}" jvmTest + +step "binary-compatibility check (apiCheck; run 'apiDump' and commit the dumps if the change is additive)" +"${GRADLE[@]}" apiCheck + +if [[ "$mode" == "--quick" ]]; then + echo; echo "pr-gate: quick mode done — run the full gate before opening the PR."; exit 0 +fi + +step "JS / Wasm tests" +"${GRADLE[@]}" verifyNpmPins jsTest wasmJsTest wasmWasiTest + +step "Kotlin/Native linuxX64 tests" +"${GRADLE[@]}" linuxX64Test + +step "assemble" +"${GRADLE[@]}" assemble + +step "Java consumer API tests" +"${GRADLE[@]}" :skainet-test:skainet-test-java:test + +if [[ "$mode" == "--bench" ]]; then + step "benchmarks (compare against the committed baseline before/after)" + "${GRADLE[@]}" :skainet-lang:skainet-lang-core:jvmBenchmark + "${GRADLE[@]}" :skainet-backends:benchmarks:jvm-cpu-jmh:jmh +fi + +echo; echo "pr-gate: all legs passed." diff --git a/skainet-lang/skainet-lang-core/api/jvm/skainet-lang-core.api b/skainet-lang/skainet-lang-core/api/jvm/skainet-lang-core.api index 97b8cffdc..a04ca4d63 100644 --- a/skainet-lang/skainet-lang-core/api/jvm/skainet-lang-core.api +++ b/skainet-lang/skainet-lang-core/api/jvm/skainet-lang-core.api @@ -5110,6 +5110,10 @@ public final class sk/ainet/lang/tensor/storage/CopySourceStat { public fun toString ()Ljava/lang/String; } +public final class sk/ainet/lang/tensor/storage/DTypeBridge { + public static final fun toLogicalDType (Lsk/ainet/lang/types/DType;)Lsk/ainet/lang/tensor/storage/LogicalDType; +} + public final class sk/ainet/lang/tensor/storage/DefaultBufferResolver : sk/ainet/lang/tensor/storage/BufferResolver { public fun ()V public fun (Lkotlin/jvm/functions/Function1;)V @@ -5292,6 +5296,7 @@ public final class sk/ainet/lang/tensor/storage/LogicalDType : java/lang/Enum { public final fun getSizeInBytes ()I public final fun isFloatingPoint ()Z public final fun isSigned ()Z + public final fun toDType ()Lsk/ainet/lang/types/DType; public static fun valueOf (Ljava/lang/String;)Lsk/ainet/lang/tensor/storage/LogicalDType; public static fun values ()[Lsk/ainet/lang/tensor/storage/LogicalDType; } @@ -5446,6 +5451,7 @@ public final class sk/ainet/lang/tensor/storage/StorageMemoryReport { public static synthetic fun copy$default (Lsk/ainet/lang/tensor/storage/StorageMemoryReport;Lsk/ainet/lang/tensor/Shape;Lsk/ainet/lang/tensor/storage/LogicalDType;Lsk/ainet/lang/tensor/storage/TensorEncoding;Lsk/ainet/lang/tensor/storage/Ownership;Lsk/ainet/lang/tensor/storage/Placement;JJZZZILjava/lang/Object;)Lsk/ainet/lang/tensor/storage/StorageMemoryReport; public fun equals (Ljava/lang/Object;)Z public final fun getCompressionRatio ()D + public final fun getDtype ()Lsk/ainet/lang/types/DType; public final fun getEncoding ()Lsk/ainet/lang/tensor/storage/TensorEncoding; public final fun getLogicalBytes ()J public final fun getLogicalType ()Lsk/ainet/lang/tensor/storage/LogicalDType; @@ -5471,6 +5477,7 @@ public final class sk/ainet/lang/tensor/storage/StorageSpec { public final fun copy (Lsk/ainet/lang/tensor/storage/LogicalDType;Lsk/ainet/lang/tensor/storage/TensorEncoding;Lsk/ainet/lang/tensor/storage/Ownership;Lsk/ainet/lang/tensor/storage/Placement;)Lsk/ainet/lang/tensor/storage/StorageSpec; public static synthetic fun copy$default (Lsk/ainet/lang/tensor/storage/StorageSpec;Lsk/ainet/lang/tensor/storage/LogicalDType;Lsk/ainet/lang/tensor/storage/TensorEncoding;Lsk/ainet/lang/tensor/storage/Ownership;Lsk/ainet/lang/tensor/storage/Placement;ILjava/lang/Object;)Lsk/ainet/lang/tensor/storage/StorageSpec; public fun equals (Ljava/lang/Object;)Z + public final fun getDtype ()Lsk/ainet/lang/types/DType; public final fun getEncoding ()Lsk/ainet/lang/tensor/storage/TensorEncoding; public final fun getLogicalType ()Lsk/ainet/lang/tensor/storage/LogicalDType; public final fun getOwnership ()Lsk/ainet/lang/tensor/storage/Ownership; @@ -5666,6 +5673,7 @@ public final class sk/ainet/lang/tensor/storage/TensorStorage { public fun equals (Ljava/lang/Object;)Z public final fun getBuffer ()Lsk/ainet/lang/tensor/storage/BufferHandle; public final fun getByteOffset ()J + public final fun getDtype ()Lsk/ainet/lang/types/DType; public final fun getElementCount ()J public final fun getEncoding ()Lsk/ainet/lang/tensor/storage/TensorEncoding; public final fun getLogicalBytes ()J diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/LogicalDType.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/LogicalDType.kt index c42be6f06..76262a5c6 100644 --- a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/LogicalDType.kt +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/LogicalDType.kt @@ -45,7 +45,39 @@ public enum class LogicalDType( public val sizeInBytes: Int get() = (sizeInBits + 7) / 8 + /** + * The [DType] this logical type corresponds to — the inverse of [fromDType]. + * + * Bridge half 1 of 2 (SKEEP-003 Phase 0, decision #13): [LogicalDType] and [DType] are + * bijective (14 ↔ 14) and will merge into one sealed `DType` that carries its `KClass` + * witness; until then this is the single sanctioned way to go from a storage descriptor's + * logical type to the `DType` the tensor DSL uses. Total — every constant maps to exactly + * one `DType` object — and exhaustive by construction (no `else` branch). + * + * @see sk.ainet.lang.tensor.storage.toLogicalDType + */ + public fun toDType(): DType = when (this) { + TERNARY -> Ternary + INT4 -> Int4 + INT8 -> Int8 + INT16 -> Int16 + INT32 -> Int32 + INT64 -> Int64 + UINT8 -> UInt8 + UINT16 -> UInt16 + UINT32 -> UInt32 + UINT64 -> UInt64 + FLOAT16 -> FP16 + BFLOAT16 -> BF16 + FLOAT32 -> FP32 + FLOAT64 -> FP64 + } + public companion object { + /** + * The [LogicalDType] for a [DType]. Inverse of [toDType]; prefer the extension + * [sk.ainet.lang.tensor.storage.toLogicalDType] at call sites. + */ public fun fromDType(dtype: DType): LogicalDType = when (dtype) { is Ternary -> TERNARY is Int4 -> INT4 diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/LogicalDTypeBridge.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/LogicalDTypeBridge.kt new file mode 100644 index 000000000..c93b97975 --- /dev/null +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/LogicalDTypeBridge.kt @@ -0,0 +1,22 @@ +@file:JvmName("DTypeBridge") + +package sk.ainet.lang.tensor.storage + +import sk.ainet.lang.types.DType +import kotlin.jvm.JvmName + +/** + * The [LogicalDType] describing this [DType] — bridge half 2 of 2 (SKEEP-003 Phase 0, + * decision #13). + * + * Lives in the storage package (not in `sk.ainet.lang.types`) so the type package stays a + * leaf; the mapping is total and bijective with [LogicalDType.toDType]: + * + * ``` + * for (l in LogicalDType.entries) check(l.toDType().toLogicalDType() == l) + * ``` + * + * When `LogicalDType` is deprecated (decision #13, separate slice) its `ReplaceWith` targets + * point at `DType` directly and this extension becomes a no-op shim. + */ +public fun DType.toLogicalDType(): LogicalDType = LogicalDType.fromDType(this) diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/StorageMemoryReport.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/StorageMemoryReport.kt index 07963c520..3ea850edf 100644 --- a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/StorageMemoryReport.kt +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/StorageMemoryReport.kt @@ -20,6 +20,9 @@ public data class StorageMemoryReport( val isAlias: Boolean, val isMutable: Boolean ) { + /** The [sk.ainet.lang.types.DType] of [logicalType] (SKEEP-003 Phase 0 bridge). */ + val dtype: sk.ainet.lang.types.DType get() = logicalType.toDType() + /** Compression ratio: logical / physical. >1 means the encoding is smaller than dense. */ val compressionRatio: Double get() = if (physicalBytes > 0) logicalBytes.toDouble() / physicalBytes else 1.0 diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/StorageSpec.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/StorageSpec.kt index 8f4951315..f2a40c629 100644 --- a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/StorageSpec.kt +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/StorageSpec.kt @@ -18,22 +18,31 @@ public data class StorageSpec( val ownership: Ownership = Ownership.OWNED, val placement: Placement = Placement.CPU_HEAP ) { + /** The [DType] of [logicalType] (SKEEP-003 Phase 0 bridge; see [LogicalDType.toDType]). */ + val dtype: DType get() = logicalType.toDType() + public companion object { /** Build a default spec from a legacy DType (dense, owned, CPU heap). */ - public fun fromDType(dtype: DType): StorageSpec = StorageSpec( - logicalType = LogicalDType.fromDType(dtype), - encoding = TensorEncoding.Dense(LogicalDType.fromDType(dtype).sizeInBytes), - ownership = Ownership.OWNED, - placement = Placement.CPU_HEAP - ) + public fun fromDType(dtype: DType): StorageSpec { + val logical = dtype.toLogicalDType() + return StorageSpec( + logicalType = logical, + encoding = TensorEncoding.Dense(logical.sizeInBytes), + ownership = Ownership.OWNED, + placement = Placement.CPU_HEAP + ) + } /** Spec for borrowed dense data. */ - public fun borrowed(dtype: DType): StorageSpec = StorageSpec( - logicalType = LogicalDType.fromDType(dtype), - encoding = TensorEncoding.Dense(LogicalDType.fromDType(dtype).sizeInBytes), - ownership = Ownership.BORROWED, - placement = Placement.CPU_HEAP - ) + public fun borrowed(dtype: DType): StorageSpec { + val logical = dtype.toLogicalDType() + return StorageSpec( + logicalType = logical, + encoding = TensorEncoding.Dense(logical.sizeInBytes), + ownership = Ownership.BORROWED, + placement = Placement.CPU_HEAP + ) + } /** Spec for Q4_K packed data. */ public fun q4k(placement: Placement = Placement.CPU_HEAP): StorageSpec = StorageSpec( @@ -52,11 +61,14 @@ public data class StorageSpec( ) /** Spec for file-backed weights. */ - public fun mmapWeights(dtype: DType): StorageSpec = StorageSpec( - logicalType = LogicalDType.fromDType(dtype), - encoding = TensorEncoding.Dense(LogicalDType.fromDType(dtype).sizeInBytes), - ownership = Ownership.FILE_BACKED, - placement = Placement.MMAP_WEIGHTS - ) + public fun mmapWeights(dtype: DType): StorageSpec { + val logical = dtype.toLogicalDType() + return StorageSpec( + logicalType = logical, + encoding = TensorEncoding.Dense(logical.sizeInBytes), + ownership = Ownership.FILE_BACKED, + placement = Placement.MMAP_WEIGHTS + ) + } } } diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/TensorStorage.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/TensorStorage.kt index d50a4e77e..4572d3a37 100644 --- a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/TensorStorage.kt +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/storage/TensorStorage.kt @@ -29,6 +29,9 @@ public data class TensorStorage( val strides: LongArray? = null, val isContiguous: Boolean = true ) { + /** The [sk.ainet.lang.types.DType] of [logicalType] (SKEEP-003 Phase 0 bridge). */ + val dtype: sk.ainet.lang.types.DType get() = logicalType.toDType() + /** Number of logical elements in this tensor. */ val elementCount: Long get() = shape.volume.toLong() diff --git a/skainet-lang/skainet-lang-core/src/commonTest/kotlin/sk/ainet/lang/tensor/storage/LogicalDTypeBridgeTest.kt b/skainet-lang/skainet-lang-core/src/commonTest/kotlin/sk/ainet/lang/tensor/storage/LogicalDTypeBridgeTest.kt new file mode 100644 index 000000000..70b09b388 --- /dev/null +++ b/skainet-lang/skainet-lang-core/src/commonTest/kotlin/sk/ainet/lang/tensor/storage/LogicalDTypeBridgeTest.kt @@ -0,0 +1,110 @@ +package sk.ainet.lang.tensor.storage + +import sk.ainet.lang.tensor.Shape +import sk.ainet.lang.types.BF16 +import sk.ainet.lang.types.DType +import sk.ainet.lang.types.FP16 +import sk.ainet.lang.types.FP32 +import sk.ainet.lang.types.FP64 +import sk.ainet.lang.types.Int16 +import sk.ainet.lang.types.Int32 +import sk.ainet.lang.types.Int4 +import sk.ainet.lang.types.Int64 +import sk.ainet.lang.types.Int8 +import sk.ainet.lang.types.Ternary +import sk.ainet.lang.types.UInt16 +import sk.ainet.lang.types.UInt32 +import sk.ainet.lang.types.UInt64 +import sk.ainet.lang.types.UInt8 +import sk.ainet.lang.types.isFloatingPoint +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertSame + +/** + * SKEEP-003 Phase 0, decision #13: the two-way `LogicalDType` <-> `DType` bridge must be total + * and bijective so that `LogicalDType` can later be merged into `DType` without a semantic gap. + */ +class LogicalDTypeBridgeTest { + + private val expectedPairs: List> = listOf( + LogicalDType.TERNARY to Ternary, + LogicalDType.INT4 to Int4, + LogicalDType.INT8 to Int8, + LogicalDType.INT16 to Int16, + LogicalDType.INT32 to Int32, + LogicalDType.INT64 to Int64, + LogicalDType.UINT8 to UInt8, + LogicalDType.UINT16 to UInt16, + LogicalDType.UINT32 to UInt32, + LogicalDType.UINT64 to UInt64, + LogicalDType.FLOAT16 to FP16, + LogicalDType.BFLOAT16 to BF16, + LogicalDType.FLOAT32 to FP32, + LogicalDType.FLOAT64 to FP64, + ) + + @Test + fun bothTypeSystemsHaveFourteenMembers() { + assertEquals(14, LogicalDType.entries.size) + assertEquals(14, DType.getAllTypes().size) + assertEquals(14, expectedPairs.size) + } + + @Test + fun explicitPairTable() { + for ((logical, dtype) in expectedPairs) { + assertSame(dtype, logical.toDType(), "toDType of $logical") + assertEquals(logical, dtype.toLogicalDType(), "toLogicalDType of ${dtype.name}") + } + } + + @Test + fun logicalToDTypeAndBackIsIdentity() { + for (logical in LogicalDType.entries) { + assertEquals(logical, logical.toDType().toLogicalDType(), "round trip of $logical") + } + } + + @Test + fun dtypeToLogicalAndBackIsIdentity() { + for (dtype in DType.getAllTypes().values) { + assertSame(dtype, dtype.toLogicalDType().toDType(), "round trip of ${dtype.name}") + } + } + + @Test + fun mappingIsBijective() { + val images = LogicalDType.entries.map { it.toDType() }.toSet() + assertEquals(LogicalDType.entries.size, images.size, "toDType must be injective") + val preImages = DType.getAllTypes().values.map { it.toLogicalDType() }.toSet() + assertEquals(DType.getAllTypes().size, preImages.size, "toLogicalDType must be injective") + } + + @Test + fun widthAndFloatnessAgreeAcrossTheBridge() { + for (logical in LogicalDType.entries) { + val dtype = logical.toDType() + assertEquals(logical.sizeInBits, dtype.sizeInBits, "sizeInBits of $logical") + assertEquals(logical.isFloatingPoint, dtype.isFloatingPoint(), "isFloatingPoint of $logical") + } + } + + @Test + fun descriptorsExposeTheDType() { + assertSame(FP16, StorageSpec.fromDType(FP16).dtype) + assertSame(FP32, StorageSpec.q4k().dtype) + assertSame(FP32, StorageSpec.q80().dtype) + assertSame(BF16, StorageSpec.borrowed(BF16).dtype) + assertSame(Int8, StorageSpec.mmapWeights(Int8).dtype) + + val storage = TensorStorage( + shape = Shape(2, 3), + logicalType = LogicalDType.FLOAT32, + encoding = TensorEncoding.Dense(4), + buffer = BufferHandle.Owned(ByteArray(24)), + ) + assertSame(FP32, storage.dtype) + assertSame(FP32, storage.memoryReport().dtype) + } +}