Fix nightly fuzz runtime verification

.github/workflows/nightly-fuzz.yml

@@ -47,7 +47,10 @@ env:
   WASM3_STABLE_VERSION: 0.5.0
   WASM3_STABLE_REF: v0.5.0
   WASM3_STABLE_SHA256: b778dd72ee2251f4fe9e2666ee3fe1c26f06f517c3ffce572416db067546536c
+  # The source revision and the version printed by `wasm3 --version` are
+  # independent identities; both are pinned and verified below.
   WASM3_MAIN_REF: d77cd814aa0bc68cb1df917580a6304d34cfb30b
+  WASM3_MAIN_VERSION: 0.5.2
   WASM3_MAIN_SHA256: c053196f4076b0649ead668c37f55673350aa2b6437afcf5c2acf43248dbf3c8
   RAW_SEED: "0x4B5741534D"
   SNAPSHOT_SEED: "0x534E415053484F54"
@@ -192,7 +195,7 @@ jobs:
             echo "wasm-tools $WASM_TOOLS_VERSION"
             echo "wasmtime $WASMTIME_VERSION"
             echo "wasm3-stable $WASM3_STABLE_VERSION (ref $WASM3_STABLE_REF, source SHA-256 pinned)"
-            echo "wasm3-main $WASM3_MAIN_REF (source SHA-256 pinned)"
+            echo "wasm3-main $WASM3_MAIN_VERSION (ref $WASM3_MAIN_REF, source SHA-256 pinned)"
             echo "smith-seed $SMITH_SEED"
             echo "smith-modules $SMITH_MODULES"
             echo "raw-seed $RAW_SEED"
@@ -232,7 +235,7 @@ jobs:
             -Pkwasm.fuzz.wasm3="$GITHUB_WORKSPACE/.tools/wasm3-stable/wasm3" \
             -Pkwasm.fuzz.wasm3Version="$WASM3_STABLE_VERSION" \
             -Pkwasm.fuzz.wasm3Secondary="$GITHUB_WORKSPACE/.tools/wasm3-main/wasm3" \
-            -Pkwasm.fuzz.wasm3SecondaryVersion="$WASM3_MAIN_REF" \
+            -Pkwasm.fuzz.wasm3SecondaryVersion="$WASM3_MAIN_VERSION" \
             -Pkwasm.fuzz.artifactsDir="$GITHUB_WORKSPACE/artifacts/fuzz-evidence" \
             -Pkwasm.fuzz.maxModules="$SMITH_MODULES" \
             -Pkwasm.fuzz.maxInvocationsPerModule=4 \
@@ -255,7 +258,7 @@ jobs:
             echo "- wasm-tools: \`$WASM_TOOLS_VERSION\` (archive SHA-256 pinned)"
             echo "- wasmtime: \`$WASMTIME_VERSION\` (archive SHA-256 pinned)"
             echo "- wasm3 stable: \`$WASM3_STABLE_VERSION\` (built from source \`$WASM3_STABLE_REF\`, source SHA-256 pinned)"
-            echo "- wasm3 main: \`$WASM3_MAIN_REF\` (built from source, source SHA-256 pinned)"
+            echo "- wasm3 main: \`$WASM3_MAIN_VERSION\` (built from \`$WASM3_MAIN_REF\`, source SHA-256 pinned)"
             echo "- Raw decoder: \`$RAW_ITERATIONS\` cases from seed \`$RAW_SEED\`"
             echo "- Snapshot mutations: \`$SNAPSHOT_MUTATIONS\` from seed \`$SNAPSHOT_SEED\`"
             echo "- Snapshot continuation properties: \`$SNAPSHOT_PROPERTIES\`"

wasm-tck/build.gradle.kts

@@ -758,70 +758,68 @@ val nightlyFuzz = tasks.register<JavaExec>("nightlyFuzz") {
     // match a prior local run; evidence is a result, not a reusable build cache.
     outputs.upToDateWhen { false }
 
-    doFirst {
-        val configuredArguments = buildList {
-            addAll(listOf("--raw-seed", fuzzRawSeed.get()))
-            addAll(listOf("--raw-iterations", fuzzRawIterations.get()))
-            addAll(listOf("--raw-max-bytes", fuzzRawMaximumBytes.get()))
-            addAll(listOf("--snapshot-seed", fuzzSnapshotSeed.get()))
-            addAll(
-                listOf(
-                    "--snapshot-mutation-iterations",
-                    fuzzSnapshotMutationIterations.get(),
-                ),
-            )
-            addAll(
-                listOf(
-                    "--snapshot-property-iterations",
-                    fuzzSnapshotPropertyIterations.get(),
-                ),
-            )
-            addAll(listOf("--wasmtime", fuzzWasmtime.get()))
-            addAll(listOf("--artifacts", fuzzArtifacts.get().absolutePath))
-            addAll(listOf("--max-modules", fuzzMaximumModules.get()))
-            addAll(
-                listOf(
-                    "--max-invocations-per-module",
-                    fuzzMaximumInvocations.get(),
-                ),
-            )
-            addAll(listOf("--max-module-bytes", fuzzMaximumModuleBytes.get()))
-            addAll(listOf("--process-timeout-ms", fuzzProcessTimeout.get()))
-            addAll(listOf("--execution-fuel", fuzzExecutionFuel.get()))
-            addAll(
-                listOf(
-                    "--max-runtime-memory-bytes",
-                    fuzzMaximumRuntimeMemory.get(),
-                ),
-            )
-            addAll(listOf("--max-table-elements", fuzzMaximumTableElements.get()))
-            addAll(listOf("--minimization-attempts", fuzzMinimizationAttempts.get()))
-            fuzzCorpus.orNull?.let { corpusPath ->
-                addAll(listOf("--corpus", rootProject.file(corpusPath).absolutePath))
-            }
-            fuzzWasmtimeVersion.orNull?.let { version ->
-                addAll(listOf("--expected-wasmtime-version", version))
-            }
-            // wasm3 is an optional oracle; emit its flags only when configured.
-            fuzzWasm3.orNull?.let { executable ->
-                addAll(listOf("--wasm3", executable))
-                fuzzWasm3Version.orNull?.let { version ->
-                    addAll(listOf("--expected-wasm3-version", version))
-                }
-            }
-            fuzzWasm3Secondary.orNull?.let { executable ->
-                addAll(listOf("--wasm3-secondary", executable))
-                fuzzWasm3SecondaryVersion.orNull?.let { version ->
-                    addAll(listOf("--expected-wasm3-secondary-version", version))
-                }
-            }
-            if (fuzzRequireDifferential.get().toBooleanStrict()) {
-                add("--require-differential")
+    val configuredArguments = buildList {
+        addAll(listOf("--raw-seed", fuzzRawSeed.get()))
+        addAll(listOf("--raw-iterations", fuzzRawIterations.get()))
+        addAll(listOf("--raw-max-bytes", fuzzRawMaximumBytes.get()))
+        addAll(listOf("--snapshot-seed", fuzzSnapshotSeed.get()))
+        addAll(
+            listOf(
+                "--snapshot-mutation-iterations",
+                fuzzSnapshotMutationIterations.get(),
+            ),
+        )
+        addAll(
+            listOf(
+                "--snapshot-property-iterations",
+                fuzzSnapshotPropertyIterations.get(),
+            ),
+        )
+        addAll(listOf("--wasmtime", fuzzWasmtime.get()))
+        addAll(listOf("--artifacts", fuzzArtifacts.get().absolutePath))
+        addAll(listOf("--max-modules", fuzzMaximumModules.get()))
+        addAll(
+            listOf(
+                "--max-invocations-per-module",
+                fuzzMaximumInvocations.get(),
+            ),
+        )
+        addAll(listOf("--max-module-bytes", fuzzMaximumModuleBytes.get()))
+        addAll(listOf("--process-timeout-ms", fuzzProcessTimeout.get()))
+        addAll(listOf("--execution-fuel", fuzzExecutionFuel.get()))
+        addAll(
+            listOf(
+                "--max-runtime-memory-bytes",
+                fuzzMaximumRuntimeMemory.get(),
+            ),
+        )
+        addAll(listOf("--max-table-elements", fuzzMaximumTableElements.get()))
+        addAll(listOf("--minimization-attempts", fuzzMinimizationAttempts.get()))
+        fuzzCorpus.orNull?.let { corpusPath ->
+            addAll(listOf("--corpus", rootProject.file(corpusPath).absolutePath))
+        }
+        fuzzWasmtimeVersion.orNull?.let { version ->
+            addAll(listOf("--expected-wasmtime-version", version))
+        }
+        // wasm3 is an optional oracle; emit its flags only when configured.
+        fuzzWasm3.orNull?.let { executable ->
+            addAll(listOf("--wasm3", executable))
+            fuzzWasm3Version.orNull?.let { version ->
+                addAll(listOf("--expected-wasm3-version", version))
             }
-            if (fuzzAllowNoCallableExports.get().toBooleanStrict()) {
-                add("--allow-no-callable-exports")
+        }
+        fuzzWasm3Secondary.orNull?.let { executable ->
+            addAll(listOf("--wasm3-secondary", executable))
+            fuzzWasm3SecondaryVersion.orNull?.let { version ->
+                addAll(listOf("--expected-wasm3-secondary-version", version))
             }
         }
-        setArgs(configuredArguments)
+        if (fuzzRequireDifferential.get().toBooleanStrict()) {
+            add("--require-differential")
+        }
+        if (fuzzAllowNoCallableExports.get().toBooleanStrict()) {
+            add("--allow-no-callable-exports")
+        }
     }
+    setArgs(configuredArguments)
 }

wasm-tck/src/jvmMain/kotlin/io/heapy/kwasm/tck/NightlyFuzzMain.kt

@@ -38,6 +38,7 @@ import kotlinx.coroutines.withTimeout
 
 private const val DEFAULT_RAW_SEED: ULong = 0x4B5741534DUL
 private const val DEFAULT_SNAPSHOT_SEED: ULong = 0x534E415053484F54UL
+private val wasm3ReportedVersion = Regex("""[0-9]+\.[0-9]+\.[0-9]+""")
 
 /**
  * Dedicated JVM entry point for the scheduled robustness gate.
@@ -152,6 +153,16 @@ internal data class NightlyFuzzConfiguration(
         require(wasm3SecondaryExecutable == null || !expectedWasm3SecondaryVersion.isNullOrBlank()) {
             "--wasm3-secondary requires --expected-wasm3-secondary-version"
         }
+        expectedWasm3Version?.let { version ->
+            require(wasm3ReportedVersion.matches(version)) {
+                "--expected-wasm3-version must be the x.y.z version reported by wasm3 --version"
+            }
+        }
+        expectedWasm3SecondaryVersion?.let { version ->
+            require(wasm3ReportedVersion.matches(version)) {
+                "--expected-wasm3-secondary-version must be the x.y.z version reported by wasm3 --version"
+            }
+        }
         require(!requireDifferential || corpusDirectory != null) {
             "--require-differential requires --corpus"
         }
@@ -667,6 +678,14 @@ internal class Wasm3DifferentialEngine(
 
 internal object Wasm3OutputParser {
     private val version = Regex("""^\s*wasm3\s+v?([0-9]+\.[0-9]+\.[0-9]+)(?:\s.*)?$""", RegexOption.IGNORE_CASE)
+    // The differential gate validates each generated module with pinned
+    // wasmtime first. These diagnostics therefore identify wasm3 capability
+    // gaps, not malformed corpus entries or kwasm divergences.
+    private val unsupportedFeatures = listOf(
+        "element table index must be zero for mvp" to
+            "wasm3-unsupported-non-mvp-element-table",
+        "restricted opcode" to "wasm3-unsupported-restricted-opcode",
+    )
 
     fun parseVersion(output: CapturedProcessOutput): String {
         requireInfrastructureCompletion(output, "wasm3 --version")
@@ -731,6 +750,8 @@ internal object Wasm3OutputParser {
         }
 
         val diagnostic = (output.stderr + "\n" + output.stdout).lowercase()
+        unsupportedFeatures.firstOrNull { (message) -> message in diagnostic }
+            ?.let { (_, reason) -> return DifferentialResult.Abstained(reason) }
         canonicalWasm3Trap(diagnostic)?.let { return DifferentialResult.Trapped(it) }
         val phase =
             when {

wasm-tck/src/jvmTest/kotlin/io/heapy/kwasm/tck/NightlyFuzzTest.kt

@@ -32,7 +32,7 @@ class NightlyFuzzTest {
                 "--wasm3", "/tools/wasm3-stable/wasm3",
                 "--expected-wasm3-version", "0.5.0",
                 "--wasm3-secondary", "/tools/wasm3-main/wasm3",
-                "--expected-wasm3-secondary-version", "d77cd814",
+                "--expected-wasm3-secondary-version", "0.5.2",
                 "--artifacts", "evidence",
                 "--max-modules", "23",
                 "--max-invocations-per-module", "2",
@@ -59,7 +59,7 @@ class NightlyFuzzTest {
         assertEquals("/tools/wasm3-stable/wasm3", configured.wasm3Executable)
         assertEquals("0.5.0", configured.expectedWasm3Version)
         assertEquals("/tools/wasm3-main/wasm3", configured.wasm3SecondaryExecutable)
-        assertEquals("d77cd814", configured.expectedWasm3SecondaryVersion)
+        assertEquals("0.5.2", configured.expectedWasm3SecondaryVersion)
         assertEquals(Path.of("evidence"), configured.artifactDirectory)
         assertEquals(23, configured.maximumModules)
         assertEquals(2, configured.maximumInvocationsPerModule)
@@ -95,6 +95,21 @@ class NightlyFuzzTest {
         }
     }
 
+    @Test
+    fun rejectsSourceRevisionAsWasm3ReportedVersion() {
+        assertFailsWith<IllegalArgumentException> {
+            NightlyFuzzConfiguration.parse(
+                listOf(
+                    "--wasm3", "/tools/wasm3-stable/wasm3",
+                    "--expected-wasm3-version", "0.5.0",
+                    "--wasm3-secondary", "/tools/wasm3-main/wasm3",
+                    "--expected-wasm3-secondary-version",
+                    "d77cd814aa0bc68cb1df917580a6304d34cfb30b",
+                ),
+            )
+        }
+    }
+
     @Test
     fun parsesWasmtimeScalarOutputIntoStableTypedValues() {
         val parsed = WasmtimeOutputParser.parseInvocation(
@@ -232,6 +247,38 @@ class NightlyFuzzTest {
         )
     }
 
+    @Test
+    fun wasm3AbstainsWhenModuleNeedsNonMvpElementTable() {
+        assertEquals(
+            DifferentialResult.Abstained("wasm3-unsupported-non-mvp-element-table"),
+            Wasm3OutputParser.parseInvocation(
+                failedWasm3(
+                    """
+                    Error: [Fatal] repl_load: element table index must be zero for MVP
+                    Error: element table index must be zero for MVP
+                    """.trimIndent(),
+                ),
+                listOf("i32"),
+            ),
+        )
+    }
+
+    @Test
+    fun wasm3AbstainsWhenModuleContainsRestrictedOpcode() {
+        assertEquals(
+            DifferentialResult.Abstained("wasm3-unsupported-restricted-opcode"),
+            Wasm3OutputParser.parseInvocation(
+                failedWasm3(
+                    """
+                    Error: [Fatal] repl_load: restricted opcode
+                    Error: restricted opcode
+                    """.trimIndent(),
+                ),
+                listOf("i64"),
+            ),
+        )
+    }
+
     @Test
     fun normalizesWasm3TrapsViaCanonicalWasm3TrapTable() {
         // `[trap] unreachable executed` lacks the `wasm trap` substring that