diff --git a/.github/workflows/nightly-fuzz.yml b/.github/workflows/nightly-fuzz.yml index de42a27..285a1cf 100644 --- a/.github/workflows/nightly-fuzz.yml +++ b/.github/workflows/nightly-fuzz.yml @@ -47,7 +47,10 @@ env: WASM3_STABLE_VERSION: 0.5.0 WASM3_STABLE_REF: v0.5.0 WASM3_STABLE_SHA256: b778dd72ee2251f4fe9e2666ee3fe1c26f06f517c3ffce572416db067546536c + # The source revision and the version printed by `wasm3 --version` are + # independent identities; both are pinned and verified below. WASM3_MAIN_REF: d77cd814aa0bc68cb1df917580a6304d34cfb30b + WASM3_MAIN_VERSION: 0.5.2 WASM3_MAIN_SHA256: c053196f4076b0649ead668c37f55673350aa2b6437afcf5c2acf43248dbf3c8 RAW_SEED: "0x4B5741534D" SNAPSHOT_SEED: "0x534E415053484F54" @@ -192,7 +195,7 @@ jobs: echo "wasm-tools $WASM_TOOLS_VERSION" echo "wasmtime $WASMTIME_VERSION" echo "wasm3-stable $WASM3_STABLE_VERSION (ref $WASM3_STABLE_REF, source SHA-256 pinned)" - echo "wasm3-main $WASM3_MAIN_REF (source SHA-256 pinned)" + echo "wasm3-main $WASM3_MAIN_VERSION (ref $WASM3_MAIN_REF, source SHA-256 pinned)" echo "smith-seed $SMITH_SEED" echo "smith-modules $SMITH_MODULES" echo "raw-seed $RAW_SEED" @@ -232,7 +235,7 @@ jobs: -Pkwasm.fuzz.wasm3="$GITHUB_WORKSPACE/.tools/wasm3-stable/wasm3" \ -Pkwasm.fuzz.wasm3Version="$WASM3_STABLE_VERSION" \ -Pkwasm.fuzz.wasm3Secondary="$GITHUB_WORKSPACE/.tools/wasm3-main/wasm3" \ - -Pkwasm.fuzz.wasm3SecondaryVersion="$WASM3_MAIN_REF" \ + -Pkwasm.fuzz.wasm3SecondaryVersion="$WASM3_MAIN_VERSION" \ -Pkwasm.fuzz.artifactsDir="$GITHUB_WORKSPACE/artifacts/fuzz-evidence" \ -Pkwasm.fuzz.maxModules="$SMITH_MODULES" \ -Pkwasm.fuzz.maxInvocationsPerModule=4 \ @@ -255,7 +258,7 @@ jobs: echo "- wasm-tools: \`$WASM_TOOLS_VERSION\` (archive SHA-256 pinned)" echo "- wasmtime: \`$WASMTIME_VERSION\` (archive SHA-256 pinned)" echo "- wasm3 stable: \`$WASM3_STABLE_VERSION\` (built from source \`$WASM3_STABLE_REF\`, source SHA-256 pinned)" - echo "- wasm3 main: \`$WASM3_MAIN_REF\` (built from source, source SHA-256 pinned)" + echo "- wasm3 main: \`$WASM3_MAIN_VERSION\` (built from \`$WASM3_MAIN_REF\`, source SHA-256 pinned)" echo "- Raw decoder: \`$RAW_ITERATIONS\` cases from seed \`$RAW_SEED\`" echo "- Snapshot mutations: \`$SNAPSHOT_MUTATIONS\` from seed \`$SNAPSHOT_SEED\`" echo "- Snapshot continuation properties: \`$SNAPSHOT_PROPERTIES\`" diff --git a/wasm-tck/build.gradle.kts b/wasm-tck/build.gradle.kts index 0b27ffd..c8784d0 100644 --- a/wasm-tck/build.gradle.kts +++ b/wasm-tck/build.gradle.kts @@ -758,70 +758,68 @@ val nightlyFuzz = tasks.register("nightlyFuzz") { // match a prior local run; evidence is a result, not a reusable build cache. outputs.upToDateWhen { false } - doFirst { - val configuredArguments = buildList { - addAll(listOf("--raw-seed", fuzzRawSeed.get())) - addAll(listOf("--raw-iterations", fuzzRawIterations.get())) - addAll(listOf("--raw-max-bytes", fuzzRawMaximumBytes.get())) - addAll(listOf("--snapshot-seed", fuzzSnapshotSeed.get())) - addAll( - listOf( - "--snapshot-mutation-iterations", - fuzzSnapshotMutationIterations.get(), - ), - ) - addAll( - listOf( - "--snapshot-property-iterations", - fuzzSnapshotPropertyIterations.get(), - ), - ) - addAll(listOf("--wasmtime", fuzzWasmtime.get())) - addAll(listOf("--artifacts", fuzzArtifacts.get().absolutePath)) - addAll(listOf("--max-modules", fuzzMaximumModules.get())) - addAll( - listOf( - "--max-invocations-per-module", - fuzzMaximumInvocations.get(), - ), - ) - addAll(listOf("--max-module-bytes", fuzzMaximumModuleBytes.get())) - addAll(listOf("--process-timeout-ms", fuzzProcessTimeout.get())) - addAll(listOf("--execution-fuel", fuzzExecutionFuel.get())) - addAll( - listOf( - "--max-runtime-memory-bytes", - fuzzMaximumRuntimeMemory.get(), - ), - ) - addAll(listOf("--max-table-elements", fuzzMaximumTableElements.get())) - addAll(listOf("--minimization-attempts", fuzzMinimizationAttempts.get())) - fuzzCorpus.orNull?.let { corpusPath -> - addAll(listOf("--corpus", rootProject.file(corpusPath).absolutePath)) - } - fuzzWasmtimeVersion.orNull?.let { version -> - addAll(listOf("--expected-wasmtime-version", version)) - } - // wasm3 is an optional oracle; emit its flags only when configured. - fuzzWasm3.orNull?.let { executable -> - addAll(listOf("--wasm3", executable)) - fuzzWasm3Version.orNull?.let { version -> - addAll(listOf("--expected-wasm3-version", version)) - } - } - fuzzWasm3Secondary.orNull?.let { executable -> - addAll(listOf("--wasm3-secondary", executable)) - fuzzWasm3SecondaryVersion.orNull?.let { version -> - addAll(listOf("--expected-wasm3-secondary-version", version)) - } - } - if (fuzzRequireDifferential.get().toBooleanStrict()) { - add("--require-differential") + val configuredArguments = buildList { + addAll(listOf("--raw-seed", fuzzRawSeed.get())) + addAll(listOf("--raw-iterations", fuzzRawIterations.get())) + addAll(listOf("--raw-max-bytes", fuzzRawMaximumBytes.get())) + addAll(listOf("--snapshot-seed", fuzzSnapshotSeed.get())) + addAll( + listOf( + "--snapshot-mutation-iterations", + fuzzSnapshotMutationIterations.get(), + ), + ) + addAll( + listOf( + "--snapshot-property-iterations", + fuzzSnapshotPropertyIterations.get(), + ), + ) + addAll(listOf("--wasmtime", fuzzWasmtime.get())) + addAll(listOf("--artifacts", fuzzArtifacts.get().absolutePath)) + addAll(listOf("--max-modules", fuzzMaximumModules.get())) + addAll( + listOf( + "--max-invocations-per-module", + fuzzMaximumInvocations.get(), + ), + ) + addAll(listOf("--max-module-bytes", fuzzMaximumModuleBytes.get())) + addAll(listOf("--process-timeout-ms", fuzzProcessTimeout.get())) + addAll(listOf("--execution-fuel", fuzzExecutionFuel.get())) + addAll( + listOf( + "--max-runtime-memory-bytes", + fuzzMaximumRuntimeMemory.get(), + ), + ) + addAll(listOf("--max-table-elements", fuzzMaximumTableElements.get())) + addAll(listOf("--minimization-attempts", fuzzMinimizationAttempts.get())) + fuzzCorpus.orNull?.let { corpusPath -> + addAll(listOf("--corpus", rootProject.file(corpusPath).absolutePath)) + } + fuzzWasmtimeVersion.orNull?.let { version -> + addAll(listOf("--expected-wasmtime-version", version)) + } + // wasm3 is an optional oracle; emit its flags only when configured. + fuzzWasm3.orNull?.let { executable -> + addAll(listOf("--wasm3", executable)) + fuzzWasm3Version.orNull?.let { version -> + addAll(listOf("--expected-wasm3-version", version)) } - if (fuzzAllowNoCallableExports.get().toBooleanStrict()) { - add("--allow-no-callable-exports") + } + fuzzWasm3Secondary.orNull?.let { executable -> + addAll(listOf("--wasm3-secondary", executable)) + fuzzWasm3SecondaryVersion.orNull?.let { version -> + addAll(listOf("--expected-wasm3-secondary-version", version)) } } - setArgs(configuredArguments) + if (fuzzRequireDifferential.get().toBooleanStrict()) { + add("--require-differential") + } + if (fuzzAllowNoCallableExports.get().toBooleanStrict()) { + add("--allow-no-callable-exports") + } } + setArgs(configuredArguments) } diff --git a/wasm-tck/src/jvmMain/kotlin/io/heapy/kwasm/tck/NightlyFuzzMain.kt b/wasm-tck/src/jvmMain/kotlin/io/heapy/kwasm/tck/NightlyFuzzMain.kt index af812c6..d83472a 100644 --- a/wasm-tck/src/jvmMain/kotlin/io/heapy/kwasm/tck/NightlyFuzzMain.kt +++ b/wasm-tck/src/jvmMain/kotlin/io/heapy/kwasm/tck/NightlyFuzzMain.kt @@ -38,6 +38,7 @@ import kotlinx.coroutines.withTimeout private const val DEFAULT_RAW_SEED: ULong = 0x4B5741534DUL private const val DEFAULT_SNAPSHOT_SEED: ULong = 0x534E415053484F54UL +private val wasm3ReportedVersion = Regex("""[0-9]+\.[0-9]+\.[0-9]+""") /** * Dedicated JVM entry point for the scheduled robustness gate. @@ -152,6 +153,16 @@ internal data class NightlyFuzzConfiguration( require(wasm3SecondaryExecutable == null || !expectedWasm3SecondaryVersion.isNullOrBlank()) { "--wasm3-secondary requires --expected-wasm3-secondary-version" } + expectedWasm3Version?.let { version -> + require(wasm3ReportedVersion.matches(version)) { + "--expected-wasm3-version must be the x.y.z version reported by wasm3 --version" + } + } + expectedWasm3SecondaryVersion?.let { version -> + require(wasm3ReportedVersion.matches(version)) { + "--expected-wasm3-secondary-version must be the x.y.z version reported by wasm3 --version" + } + } require(!requireDifferential || corpusDirectory != null) { "--require-differential requires --corpus" } @@ -667,6 +678,14 @@ internal class Wasm3DifferentialEngine( internal object Wasm3OutputParser { private val version = Regex("""^\s*wasm3\s+v?([0-9]+\.[0-9]+\.[0-9]+)(?:\s.*)?$""", RegexOption.IGNORE_CASE) + // The differential gate validates each generated module with pinned + // wasmtime first. These diagnostics therefore identify wasm3 capability + // gaps, not malformed corpus entries or kwasm divergences. + private val unsupportedFeatures = listOf( + "element table index must be zero for mvp" to + "wasm3-unsupported-non-mvp-element-table", + "restricted opcode" to "wasm3-unsupported-restricted-opcode", + ) fun parseVersion(output: CapturedProcessOutput): String { requireInfrastructureCompletion(output, "wasm3 --version") @@ -731,6 +750,8 @@ internal object Wasm3OutputParser { } val diagnostic = (output.stderr + "\n" + output.stdout).lowercase() + unsupportedFeatures.firstOrNull { (message) -> message in diagnostic } + ?.let { (_, reason) -> return DifferentialResult.Abstained(reason) } canonicalWasm3Trap(diagnostic)?.let { return DifferentialResult.Trapped(it) } val phase = when { diff --git a/wasm-tck/src/jvmTest/kotlin/io/heapy/kwasm/tck/NightlyFuzzTest.kt b/wasm-tck/src/jvmTest/kotlin/io/heapy/kwasm/tck/NightlyFuzzTest.kt index 68638bd..cc8b18e 100644 --- a/wasm-tck/src/jvmTest/kotlin/io/heapy/kwasm/tck/NightlyFuzzTest.kt +++ b/wasm-tck/src/jvmTest/kotlin/io/heapy/kwasm/tck/NightlyFuzzTest.kt @@ -32,7 +32,7 @@ class NightlyFuzzTest { "--wasm3", "/tools/wasm3-stable/wasm3", "--expected-wasm3-version", "0.5.0", "--wasm3-secondary", "/tools/wasm3-main/wasm3", - "--expected-wasm3-secondary-version", "d77cd814", + "--expected-wasm3-secondary-version", "0.5.2", "--artifacts", "evidence", "--max-modules", "23", "--max-invocations-per-module", "2", @@ -59,7 +59,7 @@ class NightlyFuzzTest { assertEquals("/tools/wasm3-stable/wasm3", configured.wasm3Executable) assertEquals("0.5.0", configured.expectedWasm3Version) assertEquals("/tools/wasm3-main/wasm3", configured.wasm3SecondaryExecutable) - assertEquals("d77cd814", configured.expectedWasm3SecondaryVersion) + assertEquals("0.5.2", configured.expectedWasm3SecondaryVersion) assertEquals(Path.of("evidence"), configured.artifactDirectory) assertEquals(23, configured.maximumModules) assertEquals(2, configured.maximumInvocationsPerModule) @@ -95,6 +95,21 @@ class NightlyFuzzTest { } } + @Test + fun rejectsSourceRevisionAsWasm3ReportedVersion() { + assertFailsWith { + NightlyFuzzConfiguration.parse( + listOf( + "--wasm3", "/tools/wasm3-stable/wasm3", + "--expected-wasm3-version", "0.5.0", + "--wasm3-secondary", "/tools/wasm3-main/wasm3", + "--expected-wasm3-secondary-version", + "d77cd814aa0bc68cb1df917580a6304d34cfb30b", + ), + ) + } + } + @Test fun parsesWasmtimeScalarOutputIntoStableTypedValues() { val parsed = WasmtimeOutputParser.parseInvocation( @@ -232,6 +247,38 @@ class NightlyFuzzTest { ) } + @Test + fun wasm3AbstainsWhenModuleNeedsNonMvpElementTable() { + assertEquals( + DifferentialResult.Abstained("wasm3-unsupported-non-mvp-element-table"), + Wasm3OutputParser.parseInvocation( + failedWasm3( + """ + Error: [Fatal] repl_load: element table index must be zero for MVP + Error: element table index must be zero for MVP + """.trimIndent(), + ), + listOf("i32"), + ), + ) + } + + @Test + fun wasm3AbstainsWhenModuleContainsRestrictedOpcode() { + assertEquals( + DifferentialResult.Abstained("wasm3-unsupported-restricted-opcode"), + Wasm3OutputParser.parseInvocation( + failedWasm3( + """ + Error: [Fatal] repl_load: restricted opcode + Error: restricted opcode + """.trimIndent(), + ), + listOf("i64"), + ), + ) + } + @Test fun normalizesWasm3TrapsViaCanonicalWasm3TrapTable() { // `[trap] unreachable executed` lacks the `wasm trap` substring that