From 714f634912fe96926a20d4797c058060fefd21e6 Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Mon, 20 Jul 2026 13:15:59 -0700 Subject: [PATCH 1/4] environ: track which tables can be mutated after instantiation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a serialized `mutated_tables` set on `Module` populated during translation: imported and exported tables are pre-marked (the embedder, or another module importing the export, can write them through the public API), then one decode pass over the code section marks the destination of every table-writing instruction — table.set, table.fill, table.grow, table.copy (destination only), table.init, and the shared-everything-threads atomic table writes. No module containing the atomic forms can currently be compiled (the cranelift translator rejects that proposal), but the scan decodes every operator, so matching them now keeps the analysis from going silently stale if the proposal is implemented. Active element segments are initial state, not mutation; elem.drop writes no table. The pass is skipped when a module has no tables (10 of the 18 modules in the benchmark corpus that motivated this) or when every table is pre-marked (e.g. Emscripten output, which exports its function table). Bodies are scanned with the `VisitOperator` API; with the new `parallel` feature (wired into wasmtime's `parallel-compilation`) they run on the rayon pool, and the serial fallback stops early once every table is marked. Forcing the walk over an 833 KiB Emscripten sqlite3 module measures ~4 ms serial / ~0.6 ms parallel; in a default build that module takes the skip path instead. Consumers arrive separately, starting with a constant table bound for tables that can never grow. `Module::table_is_immutable` documents the contract they may rely on; because they use it for correctness, a false positive here is a soundness bug, not a missed optimization. Bodies are unvalidated when the scan runs, so out-of-range destination indices are dropped rather than sizing any per-table structure from unvalidated input (the module is rejected later by real validation). --- Cargo.lock | 1 + crates/environ/Cargo.toml | 2 + crates/environ/src/compile/module_environ.rs | 208 +++++++++++ crates/environ/src/module.rs | 55 ++- crates/environ/tests/table_mutability.rs | 350 +++++++++++++++++++ crates/wasmtime/Cargo.toml | 2 +- 6 files changed, 616 insertions(+), 2 deletions(-) create mode 100644 crates/environ/tests/table_mutability.rs diff --git a/Cargo.lock b/Cargo.lock index 68340eb8fe1f..de538337a6b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4769,6 +4769,7 @@ dependencies = [ "log", "object 0.39.0", "postcard", + "rayon", "rustc-demangle", "semver", "serde", diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index aecdb45fab85..3bc5e96f6955 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -26,6 +26,7 @@ cranelift-bitset = { workspace = true, features = ['enable-serde'] } cranelift-bforest = { workspace = true } hashbrown = { workspace = true, features = ["default-hasher"] } wasmparser = { workspace = true, features = ['validate', 'serde', 'features'] } +rayon = { workspace = true, optional = true } indexmap = { workspace = true, features = ["serde"] } serde = { workspace = true } serde_derive = { workspace = true } @@ -83,6 +84,7 @@ compile = [ "dep:wasmprinter", ] stack-switching = [] +parallel = ["dep:rayon", "std"] threads = ['std'] wmemcheck = ['std'] std = [ diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs index 1c7961323aed..479a32726f5e 100644 --- a/crates/environ/src/compile/module_environ.rs +++ b/crates/environ/src/compile/module_environ.rs @@ -349,6 +349,8 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { self.translate_payload(payload?)?; } + analyze_table_mutability(&mut self.result)?; + Ok(self.result) } @@ -1582,3 +1584,209 @@ impl ModuleTranslation<'_> { self.module.startup = ModuleStartup::IfMemoriesNeedInit(ty); } } + +/// Populate `Module::mutated_tables`: record every table whose contents or +/// size can change after instantiation. +/// +/// Imported and exported tables are pre-marked up front — the embedder, or +/// another module importing the export, can mutate them through the public +/// API without any instruction in this module executing. Each defined +/// non-exported table is then marked if any instruction in the code +/// section can write to it (`table.set`, `table.fill`, `table.grow`, +/// `table.copy` as the destination, `table.init`, or a +/// shared-everything-threads atomic table write). Active element segments +/// are part of a table's initial state, not a mutation, and `elem.drop` +/// drops a passive segment without writing to any table. +/// +/// Cost: one extra decode pass over the code section. It is skipped +/// entirely when the module has no tables or when every table is already +/// pre-marked (e.g. the single funcref table is exported, as Emscripten +/// output typically is). With the `parallel` feature the bodies are +/// scanned on the rayon thread pool; serially otherwise, stopping early +/// once every table is marked. As a data point, forcing the walk over an +/// 833 KiB Emscripten-built sqlite3 module measures ~4 ms serially on an +/// Apple M4 (about 70% of a serial `validate_all` of the same bytes) and +/// ~0.6 ms with `parallel`; in a default build that module never pays +/// either cost, since its exported table takes the skip path. +fn analyze_table_mutability(translation: &mut ModuleTranslation<'_>) -> Result<()> { + let num_tables = translation.module.tables.len(); + if num_tables == 0 { + return Ok(()); + } + + for i in 0..translation.module.num_imported_tables { + translation + .module + .mark_table_mutated(TableIndex::from_u32(i as u32)); + } + let exported: Vec = translation + .module + .exports + .values() + .filter_map(|entity| match entity { + EntityIndex::Table(index) => Some(*index), + _ => None, + }) + .collect(); + for index in exported { + translation.module.mark_table_mutated(index); + } + + let all_marked = |module: &Module| { + (0..num_tables as u32).all(|i| !module.table_is_immutable(TableIndex::from_u32(i))) + }; + if all_marked(&translation.module) { + return Ok(()); + } + + let module = &mut translation.module; + let bodies = &translation.function_body_inputs; + + #[cfg(feature = "parallel")] + { + use rayon::prelude::*; + let bodies: Vec<_> = bodies.values().collect(); + let found = bodies + .into_par_iter() + .map(|body_data| table_mutations_in_body(&body_data.body, num_tables)) + .collect::>>()?; + for index in found.into_iter().flatten() { + module.mark_table_mutated(index); + } + } + + #[cfg(not(feature = "parallel"))] + { + for (_, body_data) in bodies.iter() { + for index in table_mutations_in_body(&body_data.body, num_tables)? { + module.mark_table_mutated(index); + } + if all_marked(module) { + break; + } + } + } + + Ok(()) +} + +/// Decode one function body and return the tables its instructions can +/// write to, bounded by `num_tables`. +fn table_mutations_in_body(body: &FunctionBody<'_>, num_tables: usize) -> Result> { + let mut scan = TableMutationScan { + num_tables, + found: Vec::new(), + }; + let mut reader = body.get_operators_reader()?; + while !reader.eof() { + reader.visit_operator(&mut scan)?; + } + Ok(scan.found) +} + +/// Operator visitor for `table_mutations_in_body`: records the destination +/// table of every table-writing operator and ignores everything else. +/// +/// Bodies have not been validated yet when this runs, so a decoded table +/// index can be out of range. Indices at or above `num_tables` are dropped +/// here — validation rejects such a module later — rather than sizing any +/// per-table structure from unvalidated input. +struct TableMutationScan { + num_tables: usize, + found: Vec, +} + +impl TableMutationScan { + fn mark(&mut self, table: u32) { + if (table as usize) < self.num_tables { + self.found.push(TableIndex::from_u32(table)); + } + } +} + +macro_rules! table_mutation_scan { + ($(@$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident ($($ann:tt)*))*) => { + $(table_mutation_scan!(@one $visit $({ $($arg: $argty),* })?);)* + }; + + // The operators that can write to a table mark their destination. + // `table.copy` writes only `dst_table`; its source is read-only, and + // wrongly marking it would forbid downstream optimizations on a + // read-only table. `table.atomic.get` is likewise a read. The + // remaining atomic table operators write (`rmw.cmpxchg` conditionally, + // which still counts as a write here). This scan decodes them like any + // other operator; no module containing them can currently be compiled + // (the cranelift translator rejects shared-everything-threads with + // `wasm_unsupported!`), but matching them now keeps this analysis + // sound if that proposal is ever implemented. + (@one visit_table_set { $($arg:ident: $argty:ty),* }) => { + fn visit_table_set(&mut self, table: u32) { + self.mark(table); + } + }; + (@one visit_table_fill { $($arg:ident: $argty:ty),* }) => { + fn visit_table_fill(&mut self, table: u32) { + self.mark(table); + } + }; + (@one visit_table_grow { $($arg:ident: $argty:ty),* }) => { + fn visit_table_grow(&mut self, table: u32) { + self.mark(table); + } + }; + (@one visit_table_init { $($arg:ident: $argty:ty),* }) => { + fn visit_table_init(&mut self, _elem_index: u32, table: u32) { + self.mark(table); + } + }; + (@one visit_table_copy { $($arg:ident: $argty:ty),* }) => { + fn visit_table_copy(&mut self, dst_table: u32, _src_table: u32) { + self.mark(dst_table); + } + }; + (@one visit_table_atomic_set { $($arg:ident: $argty:ty),* }) => { + fn visit_table_atomic_set(&mut self, _ordering: wasmparser::Ordering, table_index: u32) { + self.mark(table_index); + } + }; + (@one visit_table_atomic_rmw_xchg { $($arg:ident: $argty:ty),* }) => { + fn visit_table_atomic_rmw_xchg( + &mut self, + _ordering: wasmparser::Ordering, + table_index: u32, + ) { + self.mark(table_index); + } + }; + (@one visit_table_atomic_rmw_cmpxchg { $($arg:ident: $argty:ty),* }) => { + fn visit_table_atomic_rmw_cmpxchg( + &mut self, + _ordering: wasmparser::Ordering, + table_index: u32, + ) { + self.mark(table_index); + } + }; + + // Every other operator cannot write to a table. + (@one $visit:ident) => { + fn $visit(&mut self) {} + }; + (@one $visit:ident { $($arg:ident: $argty:ty),* }) => { + fn $visit(&mut self, $(_: $argty),*) {} + }; +} + +impl<'a> wasmparser::VisitOperator<'a> for TableMutationScan { + type Output = (); + + fn simd_visitor(&mut self) -> Option<&mut dyn wasmparser::VisitSimdOperator<'a, Output = ()>> { + Some(self) + } + + wasmparser::for_each_visit_operator!(table_mutation_scan); +} + +impl<'a> wasmparser::VisitSimdOperator<'a> for TableMutationScan { + wasmparser::for_each_visit_simd_operator!(table_mutation_scan); +} diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 98e551003d9b..e7e1d51e3472 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -3,7 +3,7 @@ use crate::prelude::*; use crate::*; use core::ops::Range; -use cranelift_entity::{EntityRef, packed_option::ReservedValue}; +use cranelift_entity::{EntityRef, EntitySet, packed_option::ReservedValue}; use serde_derive::{Deserialize, Serialize}; /// A WebAssembly linear memory initializer. @@ -201,6 +201,16 @@ pub struct Module { /// to be infallible as part of [`ModuleTranslation::finalize_table_init`]. pub table_initialization: TryPrimaryMap>, + /// Tables whose contents or size may change after instantiation. + /// + /// Filled in during translation: imported and exported tables are + /// always in this set (the embedder, or another module importing the + /// export, can mutate them through the public API without any + /// instruction in this module), and a defined non-exported table is + /// added when any instruction in the code section can write to it. + /// See [`Module::table_is_immutable`] for the exact contract. + mutated_tables: EntitySet, + /// WebAssembly linear memory initializer. /// /// This will track how memory is initialized, either exclusively via @@ -306,6 +316,7 @@ impl Module { exports: Default::default(), startup: ModuleStartup::None, table_initialization: Default::default(), + mutated_tables: Default::default(), memory_initialization: Default::default(), passive_elements: Default::default(), runtime_data: Default::default(), @@ -382,6 +393,46 @@ impl Module { index.index() < self.num_imported_tables } + /// Records that table `index` may be written to after instantiation. + /// + /// Called during translation: up front for every imported and exported + /// table, and again for each table some instruction in the code + /// section can write to. + #[cfg(feature = "compile")] + pub(crate) fn mark_table_mutated(&mut self, index: TableIndex) { + self.mutated_tables.insert(index); + } + + /// Returns whether table `index` provably retains its + /// instantiation-time contents and size for the lifetime of every + /// instance of this module. + /// + /// True only when all of the following hold: + /// + /// * the table is defined in this module — an imported table's other + /// owners can mutate it out of this module's view; + /// * the table is not exported — the embedder, or any module that + /// imports the export, can mutate it through the public API without + /// any instruction in this module executing; + /// * no instruction in this module's code section can write to it: + /// `table.set`, `table.fill`, `table.grow`, `table.copy` with this + /// table as the destination, `table.init`, or their + /// shared-everything-threads atomic variants. + /// + /// Element segments applied at instantiation are part of the table's + /// *initial* contents, not a mutation. `elem.drop` drops a passive + /// segment without writing to any table, so it does not affect this; + /// a passive segment can only reach a table through `table.init`, + /// which does. + /// + /// Consumers may rely on this for correctness (e.g. eliding + /// `call_indirect` checks), so a false positive here is a soundness + /// bug, not a missed optimization. + #[inline] + pub fn table_is_immutable(&self, index: TableIndex) -> bool { + !self.mutated_tables.contains(index) + } + /// Convert a `DefinedMemoryIndex` into a `MemoryIndex`. #[inline] pub fn memory_index(&self, defined_memory: DefinedMemoryIndex) -> MemoryIndex { @@ -614,6 +665,7 @@ impl TypeTrace for Module { exports: _, startup, table_initialization: _, + mutated_tables: _, memory_initialization: _, passive_elements: _, runtime_data: _, @@ -666,6 +718,7 @@ impl TypeTrace for Module { exports: _, startup, table_initialization: _, + mutated_tables: _, memory_initialization: _, passive_elements: _, runtime_data: _, diff --git a/crates/environ/tests/table_mutability.rs b/crates/environ/tests/table_mutability.rs new file mode 100644 index 000000000000..581665fc2008 --- /dev/null +++ b/crates/environ/tests/table_mutability.rs @@ -0,0 +1,350 @@ +//! Integration tests for the table-mutability analysis behind +//! `Module::table_is_immutable`. +//! +//! The per-table mutability bit is the foundation of the `call_indirect` +//! optimizations in `crates/cranelift/src/func_environ.rs` (bound-load +//! elision on never-grown tables, and the follow-up sig-check / null-check +//! elisions). A false negative here — failing to mark a table as mutated +//! when it actually is — would silently skip required runtime checks. A +//! false positive — marking an immutable table as mutated — is merely a +//! missed optimization. Pin the analysis behaviour with focused +//! module-level tests so any regression surfaces immediately, not after a +//! downstream optimization fires on a now-invalid premise. +//! +//! Test scenario inspiration drawn from comparable bugs in peer +//! interpreters that have shipped fixes for analogous IC-invalidation +//! mistakes: +//! +//! - **Luau** (`LOP_NAMECALL`): inline cache had to be invalidated on +//! `table.insert` / metatable change. Analogous wasm risk: `table.grow` +//! not invalidating an immutability proof, so see `table_grow_marks…`. +//! - **JavaScriptCore** (`ic_table`): inline-cache corruption from missed +//! shape transitions. Analogous risk: over-marking, e.g. `table.copy` +//! wrongly marking the SOURCE table as mutated would forbid downstream +//! optimizations on a perfectly read-only table. See +//! `table_copy_marks_destination_only_not_source`. +//! - **Hermes** (`HiddenClass` cache): property cache misses with +//! `Object.defineProperty`. Analogous risk: `table.init` (passive- +//! segment write at runtime) being treated as a no-op rather than a +//! write. See `table_init_marks_destination`. +//! +//! Lives in `tests/` rather than as a `#[cfg(test)] mod` inside +//! `module_environ.rs` so it builds against the lib as a normal +//! dependency. + +use wasmparser::{Parser, Validator, WasmFeatures}; +use wasmtime_environ::{ + ModuleEnvironment, ModuleTypesBuilder, StaticModuleIndex, TableIndex, Tunables, +}; + +/// Translate `wat` and return the per-table "may be mutated" bits, in +/// table-index order. Helper to keep individual tests short. +fn translate_and_get_mutability(wat: &str) -> Vec { + let bytes = wat::parse_str(wat).expect("WAT parse failed"); + let tunables = Tunables::default_host(); + // WASM2 covers reference-types + bulk-memory, which is what every + // table-mutating opcode below needs (`table.set`, `table.fill`, + // `table.grow`, `table.copy`, `table.init`, `elem.drop`). + let features = WasmFeatures::WASM2; + let mut validator = Validator::new_with_features(features); + let mut types = ModuleTypesBuilder::new(&validator); + let env = ModuleEnvironment::new( + &tunables, + &mut validator, + &mut types, + StaticModuleIndex::from_u32(0), + ); + let parser = Parser::new(0); + let translation = env.translate(parser, &bytes).expect("translate failed"); + let n: u32 = translation.module.tables.len().try_into().unwrap(); + (0..n) + .map(|i| { + !translation + .module + .table_is_immutable(TableIndex::from_u32(i)) + }) + .collect() +} + +/// A table only used as the source of `call_indirect` and `table.get` is +/// provably immutable. (Both ops READ the table; neither writes it.) The +/// table is intentionally NOT exported — exported tables are +/// conservatively pre-marked as mutated (see +/// `exported_tables_are_pre_marked` for the export case) since the host +/// can mutate them via the public wasmtime API. +#[test] +fn read_only_table_is_immutable() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 42) + (elem (i32.const 0) $f $f $f $f) + (func (export "call_zero") (result i32) + i32.const 0 + call_indirect (param) (result i32)) + (func (export "read_zero") (result funcref) + i32.const 0 + table.get 0)) + "#, + ); + assert_eq!(bits, vec![false], "no opcode mutated this table"); +} + +/// Exported tables are always pre-marked as mutated, regardless of +/// whether any opcode in this module touches them. The host can call +/// `Table::set` / `Table::grow` via the public wasmtime API on any +/// exported table, and another module that imports the export can also +/// mutate it. Without this rule, downstream optimizations would +/// happily elide null traps and sig checks on exported tables on the +/// (false) assumption that the table contents are stable. +#[test] +fn exported_tables_are_pre_marked() { + let bits = translate_and_get_mutability( + r#" + (module + (table (export "t") 4 funcref) + (func $f (result i32) i32.const 42) + (elem (i32.const 0) $f $f $f $f)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.set` marks its destination as mutated. +#[test] +fn table_set_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "do_set") + i32.const 1 + ref.func $f + table.set 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.fill` marks its destination as mutated. +#[test] +fn table_fill_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "do_fill") + i32.const 0 + ref.func $f + i32.const 4 + table.fill 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.grow` is treated as mutating — analogous to Luau's NAMECALL IC +/// needing to invalidate on table-shape change. +#[test] +fn table_grow_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func (export "do_grow") (result i32) + ref.null func + i32.const 1 + table.grow 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.copy` marks the DESTINATION but explicitly NOT the source. The +/// source is read-only (its contents aren't changed by the op); marking +/// it as mutated would forbid downstream optimizations from treating it +/// as immutable, which would be incorrect over-conservatism — the JSC +/// `ic_table` analogue. +#[test] +fn table_copy_marks_destination_only_not_source() { + let bits = translate_and_get_mutability( + r#" + (module + (table $dst (export "dst") 4 funcref) + (table $src 4 funcref) + (func $f (result i32) i32.const 0) + (elem (table $src) (i32.const 0) func $f $f $f $f) + (func (export "do_copy") + i32.const 0 ;; dst offset + i32.const 0 ;; src offset + i32.const 4 ;; len + table.copy $dst $src)) + "#, + ); + assert_eq!( + bits, + vec![true, false], + "dst should be mutated, src should remain immutable", + ); +} + +/// `table.init` writes to the destination table from a passive elem +/// segment, so it is treated as mutation (the destination's contents +/// change at runtime). +#[test] +fn table_init_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (elem $e funcref (ref.func $f) (ref.func $f)) + (func (export "do_init") + i32.const 0 ;; dst + i32.const 0 ;; src offset within elem + i32.const 2 ;; len + table.init 0 $e)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `elem.drop` drops a passive element segment but does NOT write to any +/// table — distinct from `table.init` which DOES write. A pessimistic +/// implementation that marked all tables as mutated on `elem.drop` would +/// hand out false positives and shut off optimizations on perfectly- +/// immutable tables. +#[test] +fn elem_drop_does_not_mark_tables() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (elem $e funcref (ref.func $f)) + (func (export "do_drop") + elem.drop $e)) + "#, + ); + assert_eq!(bits, vec![false]); +} + +/// Imported tables are always pre-marked as mutated, regardless of +/// whether any opcode in this module touches them. The importer can +/// mutate the table in ways this module can't see. +#[test] +fn imported_tables_are_pre_marked() { + let bits = translate_and_get_mutability( + r#" + (module + (import "host" "t" (table 4 funcref))) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// A mutation in ONE function correctly marks the table — the analysis +/// has to walk every function body, not just the first. +#[test] +fn mutation_in_any_function_counts() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "innocent") (result i32) + i32.const 0 + call_indirect (param) (result i32)) + (func (export "guilty") + i32.const 0 + ref.func $f + table.set 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// Two tables, one mutated, one not. The analysis tracks per-table — a +/// mutation on one must not leak to the other. +#[test] +fn mutation_isolated_to_target_table() { + let bits = translate_and_get_mutability( + r#" + (module + (table $a 4 funcref) + (table $b 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "mut_a") + i32.const 0 + ref.func $f + table.set $a)) + "#, + ); + assert_eq!( + bits, + vec![true, false], + "$a should be mutated, $b should remain immutable", + ); +} + +/// Translating without any tables at all must not panic, and must produce +/// an empty result rather than e.g. a default-allocated single entry. +#[test] +fn module_with_no_tables_produces_empty_mutability_vec() { + let bits = translate_and_get_mutability( + r#" + (module + (func (export "noop"))) + "#, + ); + assert!(bits.is_empty(), "no tables ⇒ no mutability bits"); +} + +/// The scan runs before function bodies are validated, so a body can name +/// a table index that doesn't exist. The analysis must neither panic nor +/// size any per-table structure from that unvalidated index; the module +/// is rejected later when the body is validated. (Only the code section +/// is malformed here — translation itself succeeds because body +/// validation is deferred to compilation.) +#[test] +fn out_of_range_table_index_in_unvalidated_body_is_ignored() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "bogus") + i32.const 0 + ref.func $f + table.set 99)) + "#, + ); + assert_eq!( + bits, + vec![false], + "an out-of-range destination index marks nothing", + ); +} + +/// When every table is pre-marked (here: one exported, one imported), the +/// result is all-mutated regardless of what the code section contains — +/// this is also the shape where the analysis skips the body walk +/// entirely. +#[test] +fn all_tables_pre_marked_without_code_mutation() { + let bits = translate_and_get_mutability( + r#" + (module + (import "host" "t" (table 4 funcref)) + (table (export "u") 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "innocent") (result i32) + i32.const 0 + call_indirect (param) (result i32))) + "#, + ); + assert_eq!(bits, vec![true, true]); +} diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index c3788854ae4b..db8193582247 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -201,7 +201,7 @@ profiling = [ ] # Enables parallel compilation of WebAssembly code. -parallel-compilation = ["dep:rayon", "std"] +parallel-compilation = ["dep:rayon", "std", "wasmtime-environ/parallel"] # Enables support for automatic cache configuration to be enabled in `Config`. cache = ["dep:wasmtime-cache", "std"] From 664cebca5d374410e978802f8b7dcb68ac35b1ab Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Mon, 20 Jul 2026 13:20:19 -0700 Subject: [PATCH 2/4] cranelift: use a constant table bound when the table can never grow `make_table_base_bound` already treats `min == max` tables as static. Extend that to tables translation proved immutable: their size is `min` forever, so the per-call `current_elements` load disappears and the bounds check compares against a constant. Imported and exported tables never qualify through the new path (the host can grow them); they are pre-marked mutated by the analysis, so only the `min == max` rule can apply to them, as before. The new disas test pins the shape for a `min < max` table that nothing grows. The three regenerated goldens show the same folding kicking in for table-init startup functions: their bound loads and spectre guards constant-fold away since the tables they initialize can't have been resized before startup runs. --- crates/cranelift/src/func_environ.rs | 10 +- .../call-indirect-immutable-static-bound.wat | 129 ++++++++++++++++++ tests/disas/gc/call-indirect-final-type.wat | 116 +++++++--------- tests/disas/startup-elem-active.wat | 44 ++---- tests/disas/startup-table-initial-value.wat | 35 ++--- 5 files changed, 214 insertions(+), 120 deletions(-) create mode 100644 tests/disas/call-indirect-immutable-static-bound.wat diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index e791c092c2ab..9ca331dcac68 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -1652,8 +1652,14 @@ impl FuncEnvironment<'_> { .unwrap(); // A fixed-size table can't be resized, so its base address won't change - // and its base load is `readonly` and `can_move`. - let is_static = Some(table.limits.min) == table.limits.max; + // and its base load is `readonly` and `can_move`. That holds when the + // table's type pins `min == max`, and also when translation proved the + // table can never be mutated or grown: its size is then `min` forever, + // so the bound is a compile-time constant and the per-call bound load + // disappears. (Imported and exported tables are never "immutable" — + // the host can grow them — so they only qualify via `min == max`.) + let is_static = + Some(table.limits.min) == table.limits.max || self.module.table_is_immutable(index); let mut base_flags = ir::MemFlagsData::trusted(); if is_static { base_flags = base_flags.with_readonly().with_can_move(); diff --git a/tests/disas/call-indirect-immutable-static-bound.wat b/tests/disas/call-indirect-immutable-static-bound.wat new file mode 100644 index 000000000000..a58c31d4111d --- /dev/null +++ b/tests/disas/call-indirect-immutable-static-bound.wat @@ -0,0 +1,129 @@ +;;! target = "x86_64" + +;; Table declared with min < max (a "dynamic-declared" table) that is +;; never written to in the module. Without the per-table mutability +;; bit, Cranelift would emit `load.i64 v0+56` per dispatch to fetch +;; the current bound. With it, `make_table_base_bound` lowers to +;; `TableSize::Static` and the bound becomes an immediate. +;; +;; Look for: bounds-check `iconst.i32 16` (the declared min, used as +;; static bound) and NO `load.i64 ... v0+56` for the current_elements +;; field. (`+48` for the funcref base is still loaded — that's the +;; element-data pointer, separate from the bound.) + +(module + ;; min=16, max=64 — distinct, so without our optimization the + ;; bound would be loaded per dispatch from `current_elements`. + (table 16 64 funcref) + + (func $f1 (result i32) i32.const 1) + (func $f2 (result i32) i32.const 2) + (func $f3 (result i32) i32.const 3) + + (func (export "call_it") (param i32) (result i32) + local.get 0 + call_indirect (result i32)) + + (elem (i32.const 0) func $f1 $f2 $f3)) +;; function u0:0(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003f v2 = iconst.i32 1 +;; @0041 jump block1 +;; +;; block1: +;; @0041 return v2 ; v2 = 1 +;; } +;; +;; function u0:1(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0044 v2 = iconst.i32 2 +;; @0046 jump block1 +;; +;; block1: +;; @0046 return v2 ; v2 = 2 +;; } +;; +;; function u0:2(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0049 v2 = iconst.i32 3 +;; @004b jump block1 +;; +;; block1: +;; @004b return v2 ; v2 = 3 +;; } +;; +;; function u0:3(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 671088640 "VMTableDefinition+0x0" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region4 = 40 "VMContext+0x28" +;; region5 = 1677721600 "TypeIdsArray+0x0" +;; region6 = 1610612752 "VMFuncRef+0x10" +;; region7 = 1610612744 "VMFuncRef+0x8" +;; region8 = 1610612760 "VMFuncRef+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx, i64) -> i32 tail +;; sig1 = (i64 vmctx, i32, i64) -> i64 tail +;; fn0 = colocated u805306368:7 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0050 v3 = iconst.i32 16 +;; @0050 v4 = icmp uge v2, v3 ; v3 = 16 +;; @0050 v5 = uextend.i64 v2 +;; @0050 v6 = load.i64 notrap aligned readonly can_move region2 v0+48 +;; @0050 v7 = iconst.i64 3 +;; @0050 v8 = ishl v5, v7 ; v7 = 3 +;; @0050 v9 = iadd v6, v8 +;; @0050 v10 = iconst.i64 0 +;; @0050 v11 = select_spectre_guard v4, v10, v9 ; v10 = 0 +;; @0050 v12 = load.i64 user6 aligned region3 v11 +;; @0050 v13 = iconst.i64 -2 +;; @0050 v14 = band v12, v13 ; v13 = -2 +;; @0050 brif v12, block3(v14), block2 +;; +;; block2 cold: +;; @0050 v16 = iconst.i32 0 +;; @0050 v17 = uextend.i64 v2 +;; @0050 v18 = call fn0(v0, v16, v17) ; v16 = 0 +;; @0050 jump block3(v18) +;; +;; block3(v15: i64): +;; @0050 v19 = load.i64 notrap aligned readonly can_move region4 v0+40 +;; @0050 v20 = load.i32 notrap aligned readonly can_move region5 v19 +;; @0050 v21 = load.i32 user7 aligned readonly region6 v15+16 +;; @0050 v22 = icmp eq v21, v20 +;; @0050 v23 = uextend.i32 v22 +;; @0050 trapz v23, user8 +;; @0050 v24 = load.i64 notrap aligned readonly region7 v15+8 +;; @0050 v25 = load.i64 notrap aligned readonly region8 v15+24 +;; @0050 v26 = call_indirect sig0, v24(v25, v0) +;; @0053 jump block1 +;; +;; block1: +;; @0053 return v26 +;; } diff --git a/tests/disas/gc/call-indirect-final-type.wat b/tests/disas/gc/call-indirect-final-type.wat index 55449f05f049..9bc2dbe79046 100644 --- a/tests/disas/gc/call-indirect-final-type.wat +++ b/tests/disas/gc/call-indirect-final-type.wat @@ -19,13 +19,12 @@ ;; region0 = 8 "VMContext+0x8" ;; region1 = 67108888 "VMStoreContext+0x18" ;; region2 = 671088640 "VMTableDefinition+0x0" -;; region3 = 671088648 "VMTableDefinition+0x8" -;; region4 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" -;; region5 = 40 "VMContext+0x28" -;; region6 = 1677721600 "TypeIdsArray+0x0" -;; region7 = 1610612752 "VMFuncRef+0x10" -;; region8 = 1610612744 "VMFuncRef+0x8" -;; region9 = 1610612760 "VMFuncRef+0x18" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region4 = 40 "VMContext+0x28" +;; region5 = 1677721600 "TypeIdsArray+0x0" +;; region6 = 1610612752 "VMFuncRef+0x10" +;; region7 = 1610612744 "VMFuncRef+0x8" +;; region8 = 1610612760 "VMFuncRef+0x18" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 @@ -35,52 +34,43 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32): -;; @002b v4 = load.i64 notrap aligned region3 v0+56 -;; @002b v8 = load.i64 notrap aligned region2 v0+48 -;; @002b v5 = ireduce.i32 v4 -;; @002b v6 = icmp uge v3, v5 -;; @002b v12 = iconst.i64 0 -;; @002b v7 = uextend.i64 v3 -;; @002b v9 = iconst.i64 3 -;; @002b v10 = ishl v7, v9 ; v9 = 3 -;; @002b v11 = iadd v8, v10 -;; @002b v13 = select_spectre_guard v6, v12, v11 ; v12 = 0 -;; @002b v14 = load.i64 user6 aligned region4 v13 -;; @002b v15 = iconst.i64 -2 -;; @002b v16 = band v14, v15 ; v15 = -2 -;; @002b brif v14, block3(v16), block2 +;; @002b v11 = iconst.i64 0 +;; @002b v13 = load.i64 user6 aligned region3 v11 ; v11 = 0 +;; @002b v14 = iconst.i64 -2 +;; @002b v15 = band v13, v14 ; v14 = -2 +;; @002b brif v13, block3(v15), block2 ;; ;; block2 cold: -;; @002b v18 = iconst.i32 0 -;; @002b v20 = call fn0(v0, v18, v7) ; v18 = 0 -;; @002b jump block3(v20) +;; @002b v4 = iconst.i32 0 +;; @002b v6 = uextend.i64 v3 +;; @002b v19 = call fn0(v0, v4, v6) ; v4 = 0 +;; @002b jump block3(v19) ;; -;; block3(v17: i64): -;; @002b v23 = load.i32 user7 aligned readonly region7 v17+16 -;; @002b v21 = load.i64 notrap aligned readonly can_move region5 v0+40 -;; @002b v22 = load.i32 notrap aligned readonly can_move region6 v21 -;; @002b v24 = icmp eq v23, v22 -;; @002b trapz v24, user8 -;; @002b v26 = load.i64 notrap aligned readonly region8 v17+8 -;; @002b v27 = load.i64 notrap aligned readonly region9 v17+24 -;; @002b v28 = call_indirect sig0, v26(v27, v0, v2) +;; block3(v16: i64): +;; @002b v22 = load.i32 user7 aligned readonly region6 v16+16 +;; @002b v20 = load.i64 notrap aligned readonly can_move region4 v0+40 +;; @002b v21 = load.i32 notrap aligned readonly can_move region5 v20 +;; @002b v23 = icmp eq v22, v21 +;; @002b trapz v23, user8 +;; @002b v25 = load.i64 notrap aligned readonly region7 v16+8 +;; @002b v26 = load.i64 notrap aligned readonly region8 v16+24 +;; @002b v27 = call_indirect sig0, v25(v26, v0, v2) ;; @002e jump block1 ;; ;; block1: -;; @002e return v28 +;; @002e return v27 ;; } ;; ;; function u0:1(i64 vmctx, i64, i32, i32) -> i32 tail { ;; region0 = 8 "VMContext+0x8" ;; region1 = 67108888 "VMStoreContext+0x18" ;; region2 = 671088640 "VMTableDefinition+0x0" -;; region3 = 671088648 "VMTableDefinition+0x8" -;; region4 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" -;; region5 = 40 "VMContext+0x28" -;; region6 = 1677721600 "TypeIdsArray+0x0" -;; region7 = 1610612752 "VMFuncRef+0x10" -;; region8 = 1610612744 "VMFuncRef+0x8" -;; region9 = 1610612760 "VMFuncRef+0x18" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region4 = 40 "VMContext+0x28" +;; region5 = 1677721600 "TypeIdsArray+0x0" +;; region6 = 1610612752 "VMFuncRef+0x10" +;; region7 = 1610612744 "VMFuncRef+0x8" +;; region8 = 1610612760 "VMFuncRef+0x18" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 @@ -90,33 +80,25 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32): -;; @0035 v4 = load.i64 notrap aligned region3 v0+56 -;; @0035 v8 = load.i64 notrap aligned region2 v0+48 -;; @0035 v5 = ireduce.i32 v4 -;; @0035 v6 = icmp uge v3, v5 -;; @0035 v12 = iconst.i64 0 -;; @0035 v7 = uextend.i64 v3 -;; @0035 v9 = iconst.i64 3 -;; @0035 v10 = ishl v7, v9 ; v9 = 3 -;; @0035 v11 = iadd v8, v10 -;; @0035 v13 = select_spectre_guard v6, v12, v11 ; v12 = 0 -;; @0035 v14 = load.i64 user6 aligned region4 v13 -;; @0035 v15 = iconst.i64 -2 -;; @0035 v16 = band v14, v15 ; v15 = -2 -;; @0035 brif v14, block3(v16), block2 +;; @0035 v11 = iconst.i64 0 +;; @0035 v13 = load.i64 user6 aligned region3 v11 ; v11 = 0 +;; @0035 v14 = iconst.i64 -2 +;; @0035 v15 = band v13, v14 ; v14 = -2 +;; @0035 brif v13, block3(v15), block2 ;; ;; block2 cold: -;; @0035 v18 = iconst.i32 0 -;; @0035 v20 = call fn0(v0, v18, v7) ; v18 = 0 -;; @0035 jump block3(v20) +;; @0035 v4 = iconst.i32 0 +;; @0035 v6 = uextend.i64 v3 +;; @0035 v19 = call fn0(v0, v4, v6) ; v4 = 0 +;; @0035 jump block3(v19) ;; -;; block3(v17: i64): -;; @0035 v23 = load.i32 user7 aligned readonly region7 v17+16 -;; @0035 v21 = load.i64 notrap aligned readonly can_move region5 v0+40 -;; @0035 v22 = load.i32 notrap aligned readonly can_move region6 v21 -;; @0035 v24 = icmp eq v23, v22 -;; @0035 trapz v24, user8 -;; @0035 v26 = load.i64 notrap aligned readonly region8 v17+8 -;; @0035 v27 = load.i64 notrap aligned readonly region9 v17+24 -;; @0035 return_call_indirect sig0, v26(v27, v0, v2) +;; block3(v16: i64): +;; @0035 v22 = load.i32 user7 aligned readonly region6 v16+16 +;; @0035 v20 = load.i64 notrap aligned readonly can_move region4 v0+40 +;; @0035 v21 = load.i32 notrap aligned readonly can_move region5 v20 +;; @0035 v23 = icmp eq v22, v21 +;; @0035 trapz v23, user8 +;; @0035 v25 = load.i64 notrap aligned readonly region7 v16+8 +;; @0035 v26 = load.i64 notrap aligned readonly region8 v16+24 +;; @0035 return_call_indirect sig0, v25(v26, v0, v2) ;; } diff --git a/tests/disas/startup-elem-active.wat b/tests/disas/startup-elem-active.wat index 4b802eee4480..5fdf9990a768 100644 --- a/tests/disas/startup-elem-active.wat +++ b/tests/disas/startup-elem-active.wat @@ -47,37 +47,21 @@ ;; ;; function u2415919104:0(i64 vmctx, i64) tail { ;; region0 = 671088640 "VMTableDefinition+0x0" -;; region1 = 671088648 "VMTableDefinition+0x8" -;; region2 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region1 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" ;; ;; block0(v0: i64, v1: i64): -;; v4 = load.i64 notrap aligned region1 v0+56 -;; v5 = ireduce.i32 v4 -;; v6 = uextend.i64 v5 -;; v78 = iconst.i64 4 -;; v84 = icmp ult v6, v78 ; v78 = 4 -;; trapnz v84, user6 -;; v13 = load.i64 notrap aligned region0 v0+48 -;; v95 = iconst.i32 21 -;; v2 = iconst.i32 1 -;; v106 = icmp ule v5, v2 ; v2 = 1 -;; v71 = iconst.i64 0 -;; v17 = iadd v13, v78 ; v78 = 4 -;; v34 = select_spectre_guard v106, v71, v17 ; v71 = 0 -;; store user6 aligned region2 v95, v34 ; v95 = 21 -;; v109 = iconst.i32 23 -;; v115 = iconst.i32 2 -;; v121 = icmp ule v5, v115 ; v115 = 2 -;; v123 = iconst.i64 8 -;; v49 = iadd v13, v123 ; v123 = 8 -;; v51 = select_spectre_guard v121, v71, v49 ; v71 = 0 -;; store user6 aligned region2 v109, v51 ; v109 = 23 -;; v125 = iconst.i32 25 -;; v3 = iconst.i32 3 -;; v136 = icmp ule v5, v3 ; v3 = 3 -;; v138 = iconst.i64 12 -;; v66 = iadd v13, v138 ; v138 = 12 -;; v68 = select_spectre_guard v136, v71, v66 ; v71 = 0 -;; store user6 aligned region2 v125, v68 ; v125 = 25 +;; v96 = iconst.i32 21 +;; v12 = load.i64 notrap aligned readonly can_move region0 v0+48 +;; v75 = iconst.i64 4 +;; v16 = iadd v12, v75 ; v75 = 4 +;; store user6 aligned region1 v96, v16 ; v96 = 21 +;; v113 = iconst.i32 23 +;; v130 = iconst.i64 8 +;; v46 = iadd v12, v130 ; v130 = 8 +;; store user6 aligned region1 v113, v46 ; v113 = 23 +;; v132 = iconst.i32 25 +;; v148 = iconst.i64 12 +;; v62 = iadd v12, v148 ; v148 = 12 +;; store user6 aligned region1 v132, v62 ; v132 = 25 ;; return ;; } diff --git a/tests/disas/startup-table-initial-value.wat b/tests/disas/startup-table-initial-value.wat index 3db39fac9976..3bdc39250ed9 100644 --- a/tests/disas/startup-table-initial-value.wat +++ b/tests/disas/startup-table-initial-value.wat @@ -41,31 +41,24 @@ ;; ;; function u2415919104:0(i64 vmctx, i64) tail { ;; region0 = 671088640 "VMTableDefinition+0x0" -;; region1 = 671088648 "VMTableDefinition+0x8" -;; region2 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region1 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" ;; ;; block0(v0: i64, v1: i64): -;; v9 = load.i64 notrap aligned region1 v0+56 -;; v10 = ireduce.i32 v9 -;; v11 = uextend.i64 v10 -;; v39 = iconst.i64 10 -;; v51 = icmp ult v11, v39 ; v39 = 10 -;; trapnz v51, user6 -;; v18 = load.i64 notrap aligned region0 v0+48 +;; v17 = load.i64 notrap aligned readonly can_move region0 v0+48 ;; v3 = iconst.i32 1 -;; v81 = iconst.i64 36 -;; v83 = iadd v18, v81 ; v81 = 36 -;; v20 = iconst.i64 4 -;; jump block1(v18) +;; v83 = iconst.i64 36 +;; v85 = iadd v17, v83 ; v83 = 36 +;; v19 = iconst.i64 4 +;; jump block1(v17) ;; -;; block1(v29: i64): -;; v86 = iconst.i32 1 -;; store notrap aligned region2 v86, v29 ; v86 = 1 -;; v87 = iadd.i64 v18, v81 ; v81 = 36 -;; v88 = icmp eq v29, v87 -;; v89 = iconst.i64 4 -;; v90 = iadd v29, v89 ; v89 = 4 -;; brif v88, block2, block1(v90) +;; block1(v28: i64): +;; v88 = iconst.i32 1 +;; store notrap aligned region1 v88, v28 ; v88 = 1 +;; v89 = iadd.i64 v17, v83 ; v83 = 36 +;; v90 = icmp eq v28, v89 +;; v91 = iconst.i64 4 +;; v92 = iadd v28, v91 ; v91 = 4 +;; brif v90, block2, block1(v92) ;; ;; block2: ;; return From 4a466539a3a42e06091b639036b6ca657d7b467a Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Mon, 20 Jul 2026 13:21:10 -0700 Subject: [PATCH 3/4] tests: runtime coverage for call_indirect over immutable and mutated tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two wast files pinning runtime behavior for the table shapes the immutable-table optimizations care about, on both sides of the line. Immutable side: mixed-signature table with a null hole, uniform table fully initialized, uniform table with a hole, element segments covering only a prefix, a populated min Date: Mon, 20 Jul 2026 13:30:35 -0700 Subject: [PATCH 4/4] cranelift: elide the call_indirect signature check on uniform immutable tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an untyped funcref table's contents are provably static for the lifetime of every instance and every non-null entry has the signature the call site expects, the runtime signature check can never fail and is dropped the same way as for typed-funcref tables — removing the sig-id load, compare, and trap from every dispatch through the table. Null slots keep their check: may_be_null is inherited from the table type, so a null entry still traps. Module::static_funcref_image owns the whole proof. It refuses tables that are mutable, imported, or exported, tables without a precomputed image — and tables targeted by element segments that finalize_table_init could not fold statically (dynamic offset or expressions-form elements). Those segments are applied at instantiation, after the image, so the image alone does not describe such a table's runtime contents; without this last condition a deferred entry with a different signature would dodge an elided check. The new deferred-segment cases in immutable-table-call-indirect.wast and crates/environ/tests/table_mutability.rs pin exactly that. Uniform-signature tables are rare in large mixed C/C++ modules but common in single-language toolchain output: AssemblyScript and Porffor (which funnels every indirect call through one calling-convention signature by design) both produce them, as do small Rust libraries. Two pre-existing expectations change. readonly-funcrefs.wat — the tracking test for exactly this never-written-funcref-table pattern (#8195) — now shows the signature check gone. indirect-call-no-caching gets a table.set mutator added to its module so it keeps demonstrating the full non-cached dispatch sequence it exists to pin, rather than being silently rewritten by an unrelated optimization. The new disas pair pins both sides directly: the uniform immutable table loses the check; the identical module with one table.set keeps it. --- crates/cranelift/src/func_environ.rs | 39 ++++ crates/environ/src/compile/module_environ.rs | 16 ++ crates/environ/src/module.rs | 50 ++++++ crates/environ/tests/table_mutability.rs | 68 +++++++ .../call-indirect-immutable-elide-sig.wat | 120 +++++++++++++ .../call-indirect-immutable-static-bound.wat | 21 +-- .../call-indirect-mutable-keeps-sigcheck.wat | 166 ++++++++++++++++++ tests/disas/indirect-call-no-caching.wat | 115 ++++++++---- tests/disas/readonly-funcrefs.wat | 18 +- 9 files changed, 548 insertions(+), 65 deletions(-) create mode 100644 tests/disas/call-indirect-immutable-elide-sig.wat create mode 100644 tests/disas/call-indirect-mutable-keeps-sigcheck.wat diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 9ca331dcac68..4f6c8a086c02 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -2167,6 +2167,30 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { .map(Some) } + /// Try to prove that the runtime signature check at a `call_indirect` + /// site through an untyped `funcref` table is redundant: the table's + /// contents are static for the lifetime of every instance + /// ([`Module::static_funcref_image`]) and every non-null entry has + /// the same module-interned signature as the type declared at the + /// call site. Null slots are fine — with the signature check gone, a + /// null entry still traps on the funcref null check. + fn try_elide_sig_check_for_immutable_table( + &self, + table_index: TableIndex, + ty_index: TypeIndex, + ) -> bool { + let module = self.env.module; + let Some(image) = module.static_funcref_image(table_index) else { + return false; + }; + let expected_ty = module.types[ty_index].unwrap_module_type_index(); + image + .iter() + .copied() + .filter(|f| !f.is_reserved_value()) + .all(|f| module.functions[f].signature.unwrap_module_type_index() == expected_ty) + } + fn check_and_load_code_and_callee_vmctx( &mut self, table_index: TableIndex, @@ -2224,6 +2248,21 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { // table of typed functions and that type matches `ty_index`, then // there's no need to perform a typecheck. match table.ref_type.heap_type { + // An untyped `funcref` table ordinarily needs a runtime + // signature check. But when the table's contents are static + // for the lifetime of every instance and every entry has the + // signature this call site expects, the check can never fail + // and is elided the same way as for typed-funcref tables + // below. Null slots are still caught: `may_be_null` is + // inherited from the table type, so the null check remains. + WasmHeapType::Func + if self.try_elide_sig_check_for_immutable_table(table_index, ty_index) => + { + return CheckIndirectCallTypeSignature::StaticMatch { + may_be_null: table.ref_type.nullable, + }; + } + // Functions do not have a statically known type in the table, a // typecheck is required. Fall through to below to perform the // actual typecheck. diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs index 479a32726f5e..972b5ca5ae26 100644 --- a/crates/environ/src/compile/module_environ.rs +++ b/crates/environ/src/compile/module_environ.rs @@ -1561,6 +1561,22 @@ impl ModuleTranslation<'_> { let _ = segments.next(); } self.table_initialization.segments = segments.try_collect().panic_on_oom(); + + // Every segment left over here is applied at instantiation time + // rather than folded into the precomputed image, so the targeted + // tables' runtime contents are not fully described by their + // image. Record that so `Module::static_funcref_image` refuses + // them — an optimization drawing conclusions from the image of + // such a table would miss the deferred writes. + let deferred: Vec = self + .table_initialization + .segments + .iter() + .map(|segment| segment.table_index) + .collect(); + for index in deferred { + self.module.mark_table_has_deferred_segments(index); + } } /// Helper function to ratchet the `startup` function for this module as diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index e7e1d51e3472..7266707bd59c 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -211,6 +211,19 @@ pub struct Module { /// See [`Module::table_is_immutable`] for the exact contract. mutated_tables: EntitySet, + /// Tables that are targeted by element segments deferred to + /// instantiation time. + /// + /// `ModuleTranslation::finalize_table_init` folds element segments + /// into [`Module::table_initialization`] only up to the first segment + /// it cannot apply statically (dynamic offset, expressions-form + /// elements, out-of-bounds or oversized reach); every remaining + /// segment is instead applied when an instance is created. A table in + /// this set therefore has runtime contents that the precomputed image + /// alone does not describe, and nothing may be concluded from its + /// image — see [`Module::static_funcref_image`]. + tables_with_deferred_segments: EntitySet, + /// WebAssembly linear memory initializer. /// /// This will track how memory is initialized, either exclusively via @@ -317,6 +330,7 @@ impl Module { startup: ModuleStartup::None, table_initialization: Default::default(), mutated_tables: Default::default(), + tables_with_deferred_segments: Default::default(), memory_initialization: Default::default(), passive_elements: Default::default(), runtime_data: Default::default(), @@ -403,6 +417,14 @@ impl Module { self.mutated_tables.insert(index); } + /// Records that table `index` is the target of an element segment + /// deferred to instantiation time, so its precomputed image (if any) + /// does not describe its complete runtime contents. + #[cfg(feature = "compile")] + pub(crate) fn mark_table_has_deferred_segments(&mut self, index: TableIndex) { + self.tables_with_deferred_segments.insert(index); + } + /// Returns whether table `index` provably retains its /// instantiation-time contents and size for the lifetime of every /// instance of this module. @@ -433,6 +455,32 @@ impl Module { !self.mutated_tables.contains(index) } + /// Returns the contents of table `index` when they are known to hold + /// for the lifetime of every instance of this module: the table must + /// be provably immutable ([`Module::table_is_immutable`]) and its + /// initial contents precomputed from element segments into + /// [`Module::table_initialization`]. + /// + /// The returned image uses `FuncIndex::reserved_value()` for null + /// slots, and any slot at an index past the end of the image is null. + /// Returns `None` for mutable, imported, or exported tables, for + /// tables without a precomputed image, for an empty image (which + /// carries no information), and for tables targeted by element + /// segments deferred to instantiation time — a deferred segment + /// rewrites slots after the image is applied, so the image alone does + /// not describe such a table's runtime contents. + pub fn static_funcref_image(&self, index: TableIndex) -> Option<&TryVec> { + if !self.table_is_immutable(index) { + return None; + } + if self.tables_with_deferred_segments.contains(index) { + return None; + } + let defined = self.defined_table_index(index)?; + let image = self.table_initialization.get(defined)?; + if image.is_empty() { None } else { Some(image) } + } + /// Convert a `DefinedMemoryIndex` into a `MemoryIndex`. #[inline] pub fn memory_index(&self, defined_memory: DefinedMemoryIndex) -> MemoryIndex { @@ -666,6 +714,7 @@ impl TypeTrace for Module { startup, table_initialization: _, mutated_tables: _, + tables_with_deferred_segments: _, memory_initialization: _, passive_elements: _, runtime_data: _, @@ -719,6 +768,7 @@ impl TypeTrace for Module { startup, table_initialization: _, mutated_tables: _, + tables_with_deferred_segments: _, memory_initialization: _, passive_elements: _, runtime_data: _, diff --git a/crates/environ/tests/table_mutability.rs b/crates/environ/tests/table_mutability.rs index 581665fc2008..70d8b8e6c56e 100644 --- a/crates/environ/tests/table_mutability.rs +++ b/crates/environ/tests/table_mutability.rs @@ -348,3 +348,71 @@ fn all_tables_pre_marked_without_code_mutation() { ); assert_eq!(bits, vec![true, true]); } + +/// Translate + `finalize_table_init`, then return, per table, whether a +/// static funcref image is available (`Module::static_funcref_image`). +fn translate_and_get_static_images(wat: &str) -> Vec { + let bytes = wat::parse_str(wat).expect("WAT parse failed"); + let tunables = Tunables::default_host(); + let features = WasmFeatures::WASM2; + let mut validator = Validator::new_with_features(features); + let mut types = ModuleTypesBuilder::new(&validator); + let env = ModuleEnvironment::new( + &tunables, + &mut validator, + &mut types, + StaticModuleIndex::from_u32(0), + ); + let parser = Parser::new(0); + let mut translation = env.translate(parser, &bytes).expect("translate failed"); + translation.finalize_table_init(&tunables, &mut types); + let n: u32 = translation.module.tables.len().try_into().unwrap(); + (0..n) + .map(|i| { + translation + .module + .static_funcref_image(TableIndex::from_u32(i)) + .is_some() + }) + .collect() +} + +/// A single constant-offset function-list segment folds completely, so +/// the static image is available. +#[test] +fn fully_folded_segments_provide_static_image() { + let images = translate_and_get_static_images( + r#" + (module + (table 4 4 funcref) + (func $f (result i32) i32.const 0) + (elem (i32.const 0) $f $f)) + "#, + ); + assert_eq!(images, vec![true]); +} + +/// An expressions-form segment cannot be folded; it is deferred to +/// instantiation, so the table's runtime contents exceed its +/// compile-time image and no static image may be exposed. Without this +/// guard, an optimization consuming the image would miss the deferred +/// write — the segment here installs a function whose signature differs +/// from the folded prefix. +#[test] +fn deferred_segments_disable_static_image() { + let images = translate_and_get_static_images( + r#" + (module + (table 3 3 funcref) + (func $a (result i32) i32.const 1) + (func $b (result i64) i64.const 2) + (elem (i32.const 0) $a $a) + (elem (i32.const 2) funcref (ref.func $b))) + "#, + ); + assert_eq!( + images, + vec![false], + "a deferred segment must disqualify the static image", + ); +} diff --git a/tests/disas/call-indirect-immutable-elide-sig.wat b/tests/disas/call-indirect-immutable-elide-sig.wat new file mode 100644 index 000000000000..b0abb1bbafc2 --- /dev/null +++ b/tests/disas/call-indirect-immutable-elide-sig.wat @@ -0,0 +1,120 @@ +;;! target = "x86_64" + +;; Immutable funcref table where every elem-segment entry has the same +;; declared type as the call site. No instruction writes to table 0 and +;; it is neither imported nor exported, so `Module::static_funcref_image` +;; is available and every entry resolves to the call site's type. That +;; triggers `try_elide_sig_check_for_immutable_table` → +;; `CheckIndirectCallTypeSignature::StaticMatch`, removing the runtime +;; signature load + compare from the dispatch hot path. +;; +;; Look for the absence of `load.i32 user6 aligned readonly v_+16` (the +;; sig-id load) and the matching `icmp eq / trapz user7` on the call +;; site. Compare with `indirect-call-no-caching.wat` for the +;; non-elided shape. + +(module + (table 10 10 funcref) + + (func $f1 (result i32) i32.const 1) + (func $f2 (result i32) i32.const 2) + (func $f3 (result i32) i32.const 3) + + (func (export "call_it") (param i32) (result i32) + local.get 0 + call_indirect (result i32)) + + (elem (i32.const 0) func $f1 $f2 $f3)) +;; function u0:0(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003f v2 = iconst.i32 1 +;; @0041 jump block1 +;; +;; block1: +;; @0041 return v2 ; v2 = 1 +;; } +;; +;; function u0:1(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0044 v2 = iconst.i32 2 +;; @0046 jump block1 +;; +;; block1: +;; @0046 return v2 ; v2 = 2 +;; } +;; +;; function u0:2(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0049 v2 = iconst.i32 3 +;; @004b jump block1 +;; +;; block1: +;; @004b return v2 ; v2 = 3 +;; } +;; +;; function u0:3(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 671088640 "VMTableDefinition+0x0" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region4 = 1610612744 "VMFuncRef+0x8" +;; region5 = 1610612760 "VMFuncRef+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx, i64) -> i32 tail +;; sig1 = (i64 vmctx, i32, i64) -> i64 tail +;; fn0 = colocated u805306368:7 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0050 v3 = iconst.i32 10 +;; @0050 v4 = icmp uge v2, v3 ; v3 = 10 +;; @0050 v5 = uextend.i64 v2 +;; @0050 v6 = load.i64 notrap aligned readonly can_move region2 v0+48 +;; @0050 v7 = iconst.i64 3 +;; @0050 v8 = ishl v5, v7 ; v7 = 3 +;; @0050 v9 = iadd v6, v8 +;; @0050 v10 = iconst.i64 0 +;; @0050 v11 = select_spectre_guard v4, v10, v9 ; v10 = 0 +;; @0050 v12 = load.i64 user6 aligned region3 v11 +;; @0050 v13 = iconst.i64 -2 +;; @0050 v14 = band v12, v13 ; v13 = -2 +;; @0050 brif v12, block3(v14), block2 +;; +;; block2 cold: +;; @0050 v16 = iconst.i32 0 +;; @0050 v17 = uextend.i64 v2 +;; @0050 v18 = call fn0(v0, v16, v17) ; v16 = 0 +;; @0050 jump block3(v18) +;; +;; block3(v15: i64): +;; @0050 v19 = load.i64 user7 aligned readonly region4 v15+8 +;; @0050 v20 = load.i64 notrap aligned readonly region5 v15+24 +;; @0050 v21 = call_indirect sig0, v19(v20, v0) +;; @0053 jump block1 +;; +;; block1: +;; @0053 return v21 +;; } diff --git a/tests/disas/call-indirect-immutable-static-bound.wat b/tests/disas/call-indirect-immutable-static-bound.wat index a58c31d4111d..51334326496e 100644 --- a/tests/disas/call-indirect-immutable-static-bound.wat +++ b/tests/disas/call-indirect-immutable-static-bound.wat @@ -78,11 +78,8 @@ ;; region1 = 67108888 "VMStoreContext+0x18" ;; region2 = 671088640 "VMTableDefinition+0x0" ;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" -;; region4 = 40 "VMContext+0x28" -;; region5 = 1677721600 "TypeIdsArray+0x0" -;; region6 = 1610612752 "VMFuncRef+0x10" -;; region7 = 1610612744 "VMFuncRef+0x8" -;; region8 = 1610612760 "VMFuncRef+0x18" +;; region4 = 1610612744 "VMFuncRef+0x8" +;; region5 = 1610612760 "VMFuncRef+0x18" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 @@ -113,17 +110,11 @@ ;; @0050 jump block3(v18) ;; ;; block3(v15: i64): -;; @0050 v19 = load.i64 notrap aligned readonly can_move region4 v0+40 -;; @0050 v20 = load.i32 notrap aligned readonly can_move region5 v19 -;; @0050 v21 = load.i32 user7 aligned readonly region6 v15+16 -;; @0050 v22 = icmp eq v21, v20 -;; @0050 v23 = uextend.i32 v22 -;; @0050 trapz v23, user8 -;; @0050 v24 = load.i64 notrap aligned readonly region7 v15+8 -;; @0050 v25 = load.i64 notrap aligned readonly region8 v15+24 -;; @0050 v26 = call_indirect sig0, v24(v25, v0) +;; @0050 v19 = load.i64 user7 aligned readonly region4 v15+8 +;; @0050 v20 = load.i64 notrap aligned readonly region5 v15+24 +;; @0050 v21 = call_indirect sig0, v19(v20, v0) ;; @0053 jump block1 ;; ;; block1: -;; @0053 return v26 +;; @0053 return v21 ;; } diff --git a/tests/disas/call-indirect-mutable-keeps-sigcheck.wat b/tests/disas/call-indirect-mutable-keeps-sigcheck.wat new file mode 100644 index 000000000000..c16cf7d6ab96 --- /dev/null +++ b/tests/disas/call-indirect-mutable-keeps-sigcheck.wat @@ -0,0 +1,166 @@ +;;! target = "x86_64" + +;; Counterpart to `call-indirect-immutable-elide-sig.wat`. Same module +;; shape — same elem segment, same uniform call-site type — but one +;; function writes to the table via `table.set`. That marks the table +;; as mutated and disables sig-check elision. +;; +;; Look for the runtime sig load + compare on the call site: +;; load.i32 user6 aligned readonly v_+16 +;; icmp eq +;; trapz user7 +;; (versus the elided form in the immutable test). + +(module + (table 10 10 funcref) + + (func $f1 (result i32) i32.const 1) + (func $f2 (result i32) i32.const 2) + (func $f3 (result i32) i32.const 3) + + ;; Mutator: this clears the immutability proof for table 0. + (func (export "mutate") (param i32) + local.get 0 + ref.func $f1 + table.set 0) + + (func (export "call_it") (param i32) (result i32) + local.get 0 + call_indirect (result i32)) + + (elem (i32.const 0) func $f1 $f2 $f3)) +;; function u0:0(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @004d v2 = iconst.i32 1 +;; @004f jump block1 +;; +;; block1: +;; @004f return v2 ; v2 = 1 +;; } +;; +;; function u0:1(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0052 v2 = iconst.i32 2 +;; @0054 jump block1 +;; +;; block1: +;; @0054 return v2 ; v2 = 2 +;; } +;; +;; function u0:2(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0057 v2 = iconst.i32 3 +;; @0059 jump block1 +;; +;; block1: +;; @0059 return v2 ; v2 = 3 +;; } +;; +;; function u0:3(i64 vmctx, i64, i32) tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 671088640 "VMTableDefinition+0x0" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx, i32) -> i64 tail +;; fn0 = colocated u805306368:6 sig0 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @005e v3 = iconst.i32 0 +;; @005e v4 = call fn0(v0, v3) ; v3 = 0 +;; @0060 v5 = iconst.i32 10 +;; @0060 v6 = icmp uge v2, v5 ; v5 = 10 +;; @0060 v7 = uextend.i64 v2 +;; @0060 v8 = load.i64 notrap aligned readonly can_move region2 v0+48 +;; @0060 v9 = iconst.i64 3 +;; @0060 v10 = ishl v7, v9 ; v9 = 3 +;; @0060 v11 = iadd v8, v10 +;; @0060 v12 = iconst.i64 0 +;; @0060 v13 = select_spectre_guard v6, v12, v11 ; v12 = 0 +;; @0060 v14 = iconst.i64 1 +;; @0060 v15 = bor v4, v14 ; v14 = 1 +;; @0060 store user6 aligned region3 v15, v13 +;; @0062 jump block1 +;; +;; block1: +;; @0062 return +;; } +;; +;; function u0:4(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 671088640 "VMTableDefinition+0x0" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region4 = 40 "VMContext+0x28" +;; region5 = 1677721600 "TypeIdsArray+0x0" +;; region6 = 1610612752 "VMFuncRef+0x10" +;; region7 = 1610612744 "VMFuncRef+0x8" +;; region8 = 1610612760 "VMFuncRef+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx, i64) -> i32 tail +;; sig1 = (i64 vmctx, i32, i64) -> i64 tail +;; fn0 = colocated u805306368:7 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0067 v3 = iconst.i32 10 +;; @0067 v4 = icmp uge v2, v3 ; v3 = 10 +;; @0067 v5 = uextend.i64 v2 +;; @0067 v6 = load.i64 notrap aligned readonly can_move region2 v0+48 +;; @0067 v7 = iconst.i64 3 +;; @0067 v8 = ishl v5, v7 ; v7 = 3 +;; @0067 v9 = iadd v6, v8 +;; @0067 v10 = iconst.i64 0 +;; @0067 v11 = select_spectre_guard v4, v10, v9 ; v10 = 0 +;; @0067 v12 = load.i64 user6 aligned region3 v11 +;; @0067 v13 = iconst.i64 -2 +;; @0067 v14 = band v12, v13 ; v13 = -2 +;; @0067 brif v12, block3(v14), block2 +;; +;; block2 cold: +;; @0067 v16 = iconst.i32 0 +;; @0067 v17 = uextend.i64 v2 +;; @0067 v18 = call fn0(v0, v16, v17) ; v16 = 0 +;; @0067 jump block3(v18) +;; +;; block3(v15: i64): +;; @0067 v19 = load.i64 notrap aligned readonly can_move region4 v0+40 +;; @0067 v20 = load.i32 notrap aligned readonly can_move region5 v19 +;; @0067 v21 = load.i32 user7 aligned readonly region6 v15+16 +;; @0067 v22 = icmp eq v21, v20 +;; @0067 v23 = uextend.i32 v22 +;; @0067 trapz v23, user8 +;; @0067 v24 = load.i64 notrap aligned readonly region7 v15+8 +;; @0067 v25 = load.i64 notrap aligned readonly region8 v15+24 +;; @0067 v26 = call_indirect sig0, v24(v25, v0) +;; @006a jump block1 +;; +;; block1: +;; @006a return v26 +;; } diff --git a/tests/disas/indirect-call-no-caching.wat b/tests/disas/indirect-call-no-caching.wat index d216ca74abe6..57086c54f914 100644 --- a/tests/disas/indirect-call-no-caching.wat +++ b/tests/disas/indirect-call-no-caching.wat @@ -19,6 +19,14 @@ local.get 0 call_indirect (result i32)) + ;; Writing to the table keeps it out of the immutable-table + ;; signature-check elision, so this test keeps demonstrating the + ;; full non-cached dispatch sequence it was written for. + (func (export "mutate") (param i32) + local.get 0 + ref.func $f1 + table.set 0) + (elem (i32.const 1) func $f1 $f2 $f3)) ;; function u0:0(i64 vmctx, i64) -> i32 tail { ;; region0 = 8 "VMContext+0x8" @@ -29,11 +37,11 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64): -;; @003f v2 = iconst.i32 1 -;; @0041 jump block1 +;; @004d v2 = iconst.i32 1 +;; @004f jump block1 ;; ;; block1: -;; @0041 return v2 ; v2 = 1 +;; @004f return v2 ; v2 = 1 ;; } ;; ;; function u0:1(i64 vmctx, i64) -> i32 tail { @@ -45,11 +53,11 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64): -;; @0044 v2 = iconst.i32 2 -;; @0046 jump block1 +;; @0052 v2 = iconst.i32 2 +;; @0054 jump block1 ;; ;; block1: -;; @0046 return v2 ; v2 = 2 +;; @0054 return v2 ; v2 = 2 ;; } ;; ;; function u0:2(i64 vmctx, i64) -> i32 tail { @@ -61,11 +69,11 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64): -;; @0049 v2 = iconst.i32 3 -;; @004b jump block1 +;; @0057 v2 = iconst.i32 3 +;; @0059 jump block1 ;; ;; block1: -;; @004b return v2 ; v2 = 3 +;; @0059 return v2 ; v2 = 3 ;; } ;; ;; function u0:3(i64 vmctx, i64, i32) -> i32 tail { @@ -87,38 +95,71 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32): -;; @0050 v3 = iconst.i32 10 -;; @0050 v4 = icmp uge v2, v3 ; v3 = 10 -;; @0050 v5 = uextend.i64 v2 -;; @0050 v6 = load.i64 notrap aligned readonly can_move region2 v0+48 -;; @0050 v7 = iconst.i64 3 -;; @0050 v8 = ishl v5, v7 ; v7 = 3 -;; @0050 v9 = iadd v6, v8 -;; @0050 v10 = iconst.i64 0 -;; @0050 v11 = select_spectre_guard v4, v10, v9 ; v10 = 0 -;; @0050 v12 = load.i64 user6 aligned region3 v11 -;; @0050 v13 = iconst.i64 -2 -;; @0050 v14 = band v12, v13 ; v13 = -2 -;; @0050 brif v12, block3(v14), block2 +;; @005e v3 = iconst.i32 10 +;; @005e v4 = icmp uge v2, v3 ; v3 = 10 +;; @005e v5 = uextend.i64 v2 +;; @005e v6 = load.i64 notrap aligned readonly can_move region2 v0+48 +;; @005e v7 = iconst.i64 3 +;; @005e v8 = ishl v5, v7 ; v7 = 3 +;; @005e v9 = iadd v6, v8 +;; @005e v10 = iconst.i64 0 +;; @005e v11 = select_spectre_guard v4, v10, v9 ; v10 = 0 +;; @005e v12 = load.i64 user6 aligned region3 v11 +;; @005e v13 = iconst.i64 -2 +;; @005e v14 = band v12, v13 ; v13 = -2 +;; @005e brif v12, block3(v14), block2 ;; ;; block2 cold: -;; @0050 v16 = iconst.i32 0 -;; @0050 v17 = uextend.i64 v2 -;; @0050 v18 = call fn0(v0, v16, v17) ; v16 = 0 -;; @0050 jump block3(v18) +;; @005e v16 = iconst.i32 0 +;; @005e v17 = uextend.i64 v2 +;; @005e v18 = call fn0(v0, v16, v17) ; v16 = 0 +;; @005e jump block3(v18) ;; ;; block3(v15: i64): -;; @0050 v19 = load.i64 notrap aligned readonly can_move region4 v0+40 -;; @0050 v20 = load.i32 notrap aligned readonly can_move region5 v19 -;; @0050 v21 = load.i32 user7 aligned readonly region6 v15+16 -;; @0050 v22 = icmp eq v21, v20 -;; @0050 v23 = uextend.i32 v22 -;; @0050 trapz v23, user8 -;; @0050 v24 = load.i64 notrap aligned readonly region7 v15+8 -;; @0050 v25 = load.i64 notrap aligned readonly region8 v15+24 -;; @0050 v26 = call_indirect sig0, v24(v25, v0) -;; @0053 jump block1 +;; @005e v19 = load.i64 notrap aligned readonly can_move region4 v0+40 +;; @005e v20 = load.i32 notrap aligned readonly can_move region5 v19 +;; @005e v21 = load.i32 user7 aligned readonly region6 v15+16 +;; @005e v22 = icmp eq v21, v20 +;; @005e v23 = uextend.i32 v22 +;; @005e trapz v23, user8 +;; @005e v24 = load.i64 notrap aligned readonly region7 v15+8 +;; @005e v25 = load.i64 notrap aligned readonly region8 v15+24 +;; @005e v26 = call_indirect sig0, v24(v25, v0) +;; @0061 jump block1 +;; +;; block1: +;; @0061 return v26 +;; } +;; +;; function u0:4(i64 vmctx, i64, i32) tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 671088640 "VMTableDefinition+0x0" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx, i32) -> i64 tail +;; fn0 = colocated u805306368:6 sig0 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0066 v3 = iconst.i32 0 +;; @0066 v4 = call fn0(v0, v3) ; v3 = 0 +;; @0068 v5 = iconst.i32 10 +;; @0068 v6 = icmp uge v2, v5 ; v5 = 10 +;; @0068 v7 = uextend.i64 v2 +;; @0068 v8 = load.i64 notrap aligned readonly can_move region2 v0+48 +;; @0068 v9 = iconst.i64 3 +;; @0068 v10 = ishl v7, v9 ; v9 = 3 +;; @0068 v11 = iadd v8, v10 +;; @0068 v12 = iconst.i64 0 +;; @0068 v13 = select_spectre_guard v6, v12, v11 ; v12 = 0 +;; @0068 v14 = iconst.i64 1 +;; @0068 v15 = bor v4, v14 ; v14 = 1 +;; @0068 store user6 aligned region3 v15, v13 +;; @006a jump block1 ;; ;; block1: -;; @0053 return v26 +;; @006a return ;; } diff --git a/tests/disas/readonly-funcrefs.wat b/tests/disas/readonly-funcrefs.wat index bb9fc978a844..b21bbcbbae47 100644 --- a/tests/disas/readonly-funcrefs.wat +++ b/tests/disas/readonly-funcrefs.wat @@ -38,11 +38,8 @@ ;; region1 = 67108888 "VMStoreContext+0x18" ;; region2 = 671088640 "VMTableDefinition+0x0" ;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" -;; region4 = 40 "VMContext+0x28" -;; region5 = 1677721600 "TypeIdsArray+0x0" -;; region6 = 1610612752 "VMFuncRef+0x10" -;; region7 = 1610612744 "VMFuncRef+0x8" -;; region8 = 1610612760 "VMFuncRef+0x18" +;; region4 = 1610612744 "VMFuncRef+0x8" +;; region5 = 1610612760 "VMFuncRef+0x18" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 @@ -72,14 +69,9 @@ ;; @0031 jump block3(v18) ;; ;; block3(v15: i64): -;; @0031 v21 = load.i32 user7 aligned readonly region6 v15+16 -;; @0031 v19 = load.i64 notrap aligned readonly can_move region4 v0+40 -;; @0031 v20 = load.i32 notrap aligned readonly can_move region5 v19 -;; @0031 v22 = icmp eq v21, v20 -;; @0031 trapz v22, user8 -;; @0031 v24 = load.i64 notrap aligned readonly region7 v15+8 -;; @0031 v25 = load.i64 notrap aligned readonly region8 v15+24 -;; @0031 call_indirect sig0, v24(v25, v0) +;; @0031 v19 = load.i64 user7 aligned readonly region4 v15+8 +;; @0031 v20 = load.i64 notrap aligned readonly region5 v15+24 +;; @0031 call_indirect sig0, v19(v20, v0) ;; @0034 jump block1 ;; ;; block1: