diff --git a/Cargo.lock b/Cargo.lock index 68340eb8fe1f..de538337a6b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4769,6 +4769,7 @@ dependencies = [ "log", "object 0.39.0", "postcard", + "rayon", "rustc-demangle", "semver", "serde", diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index e791c092c2ab..9ca331dcac68 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -1652,8 +1652,14 @@ impl FuncEnvironment<'_> { .unwrap(); // A fixed-size table can't be resized, so its base address won't change - // and its base load is `readonly` and `can_move`. - let is_static = Some(table.limits.min) == table.limits.max; + // and its base load is `readonly` and `can_move`. That holds when the + // table's type pins `min == max`, and also when translation proved the + // table can never be mutated or grown: its size is then `min` forever, + // so the bound is a compile-time constant and the per-call bound load + // disappears. (Imported and exported tables are never "immutable" — + // the host can grow them — so they only qualify via `min == max`.) + let is_static = + Some(table.limits.min) == table.limits.max || self.module.table_is_immutable(index); let mut base_flags = ir::MemFlagsData::trusted(); if is_static { base_flags = base_flags.with_readonly().with_can_move(); diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index aecdb45fab85..3bc5e96f6955 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -26,6 +26,7 @@ cranelift-bitset = { workspace = true, features = ['enable-serde'] } cranelift-bforest = { workspace = true } hashbrown = { workspace = true, features = ["default-hasher"] } wasmparser = { workspace = true, features = ['validate', 'serde', 'features'] } +rayon = { workspace = true, optional = true } indexmap = { workspace = true, features = ["serde"] } serde = { workspace = true } serde_derive = { workspace = true } @@ -83,6 +84,7 @@ compile = [ "dep:wasmprinter", ] stack-switching = [] +parallel = ["dep:rayon", "std"] threads = ['std'] wmemcheck = ['std'] std = [ diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs index 1c7961323aed..479a32726f5e 100644 --- a/crates/environ/src/compile/module_environ.rs +++ b/crates/environ/src/compile/module_environ.rs @@ -349,6 +349,8 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { self.translate_payload(payload?)?; } + analyze_table_mutability(&mut self.result)?; + Ok(self.result) } @@ -1582,3 +1584,209 @@ impl ModuleTranslation<'_> { self.module.startup = ModuleStartup::IfMemoriesNeedInit(ty); } } + +/// Populate `Module::mutated_tables`: record every table whose contents or +/// size can change after instantiation. +/// +/// Imported and exported tables are pre-marked up front — the embedder, or +/// another module importing the export, can mutate them through the public +/// API without any instruction in this module executing. Each defined +/// non-exported table is then marked if any instruction in the code +/// section can write to it (`table.set`, `table.fill`, `table.grow`, +/// `table.copy` as the destination, `table.init`, or a +/// shared-everything-threads atomic table write). Active element segments +/// are part of a table's initial state, not a mutation, and `elem.drop` +/// drops a passive segment without writing to any table. +/// +/// Cost: one extra decode pass over the code section. It is skipped +/// entirely when the module has no tables or when every table is already +/// pre-marked (e.g. the single funcref table is exported, as Emscripten +/// output typically is). With the `parallel` feature the bodies are +/// scanned on the rayon thread pool; serially otherwise, stopping early +/// once every table is marked. As a data point, forcing the walk over an +/// 833 KiB Emscripten-built sqlite3 module measures ~4 ms serially on an +/// Apple M4 (about 70% of a serial `validate_all` of the same bytes) and +/// ~0.6 ms with `parallel`; in a default build that module never pays +/// either cost, since its exported table takes the skip path. +fn analyze_table_mutability(translation: &mut ModuleTranslation<'_>) -> Result<()> { + let num_tables = translation.module.tables.len(); + if num_tables == 0 { + return Ok(()); + } + + for i in 0..translation.module.num_imported_tables { + translation + .module + .mark_table_mutated(TableIndex::from_u32(i as u32)); + } + let exported: Vec = translation + .module + .exports + .values() + .filter_map(|entity| match entity { + EntityIndex::Table(index) => Some(*index), + _ => None, + }) + .collect(); + for index in exported { + translation.module.mark_table_mutated(index); + } + + let all_marked = |module: &Module| { + (0..num_tables as u32).all(|i| !module.table_is_immutable(TableIndex::from_u32(i))) + }; + if all_marked(&translation.module) { + return Ok(()); + } + + let module = &mut translation.module; + let bodies = &translation.function_body_inputs; + + #[cfg(feature = "parallel")] + { + use rayon::prelude::*; + let bodies: Vec<_> = bodies.values().collect(); + let found = bodies + .into_par_iter() + .map(|body_data| table_mutations_in_body(&body_data.body, num_tables)) + .collect::>>()?; + for index in found.into_iter().flatten() { + module.mark_table_mutated(index); + } + } + + #[cfg(not(feature = "parallel"))] + { + for (_, body_data) in bodies.iter() { + for index in table_mutations_in_body(&body_data.body, num_tables)? { + module.mark_table_mutated(index); + } + if all_marked(module) { + break; + } + } + } + + Ok(()) +} + +/// Decode one function body and return the tables its instructions can +/// write to, bounded by `num_tables`. +fn table_mutations_in_body(body: &FunctionBody<'_>, num_tables: usize) -> Result> { + let mut scan = TableMutationScan { + num_tables, + found: Vec::new(), + }; + let mut reader = body.get_operators_reader()?; + while !reader.eof() { + reader.visit_operator(&mut scan)?; + } + Ok(scan.found) +} + +/// Operator visitor for `table_mutations_in_body`: records the destination +/// table of every table-writing operator and ignores everything else. +/// +/// Bodies have not been validated yet when this runs, so a decoded table +/// index can be out of range. Indices at or above `num_tables` are dropped +/// here — validation rejects such a module later — rather than sizing any +/// per-table structure from unvalidated input. +struct TableMutationScan { + num_tables: usize, + found: Vec, +} + +impl TableMutationScan { + fn mark(&mut self, table: u32) { + if (table as usize) < self.num_tables { + self.found.push(TableIndex::from_u32(table)); + } + } +} + +macro_rules! table_mutation_scan { + ($(@$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident ($($ann:tt)*))*) => { + $(table_mutation_scan!(@one $visit $({ $($arg: $argty),* })?);)* + }; + + // The operators that can write to a table mark their destination. + // `table.copy` writes only `dst_table`; its source is read-only, and + // wrongly marking it would forbid downstream optimizations on a + // read-only table. `table.atomic.get` is likewise a read. The + // remaining atomic table operators write (`rmw.cmpxchg` conditionally, + // which still counts as a write here). This scan decodes them like any + // other operator; no module containing them can currently be compiled + // (the cranelift translator rejects shared-everything-threads with + // `wasm_unsupported!`), but matching them now keeps this analysis + // sound if that proposal is ever implemented. + (@one visit_table_set { $($arg:ident: $argty:ty),* }) => { + fn visit_table_set(&mut self, table: u32) { + self.mark(table); + } + }; + (@one visit_table_fill { $($arg:ident: $argty:ty),* }) => { + fn visit_table_fill(&mut self, table: u32) { + self.mark(table); + } + }; + (@one visit_table_grow { $($arg:ident: $argty:ty),* }) => { + fn visit_table_grow(&mut self, table: u32) { + self.mark(table); + } + }; + (@one visit_table_init { $($arg:ident: $argty:ty),* }) => { + fn visit_table_init(&mut self, _elem_index: u32, table: u32) { + self.mark(table); + } + }; + (@one visit_table_copy { $($arg:ident: $argty:ty),* }) => { + fn visit_table_copy(&mut self, dst_table: u32, _src_table: u32) { + self.mark(dst_table); + } + }; + (@one visit_table_atomic_set { $($arg:ident: $argty:ty),* }) => { + fn visit_table_atomic_set(&mut self, _ordering: wasmparser::Ordering, table_index: u32) { + self.mark(table_index); + } + }; + (@one visit_table_atomic_rmw_xchg { $($arg:ident: $argty:ty),* }) => { + fn visit_table_atomic_rmw_xchg( + &mut self, + _ordering: wasmparser::Ordering, + table_index: u32, + ) { + self.mark(table_index); + } + }; + (@one visit_table_atomic_rmw_cmpxchg { $($arg:ident: $argty:ty),* }) => { + fn visit_table_atomic_rmw_cmpxchg( + &mut self, + _ordering: wasmparser::Ordering, + table_index: u32, + ) { + self.mark(table_index); + } + }; + + // Every other operator cannot write to a table. + (@one $visit:ident) => { + fn $visit(&mut self) {} + }; + (@one $visit:ident { $($arg:ident: $argty:ty),* }) => { + fn $visit(&mut self, $(_: $argty),*) {} + }; +} + +impl<'a> wasmparser::VisitOperator<'a> for TableMutationScan { + type Output = (); + + fn simd_visitor(&mut self) -> Option<&mut dyn wasmparser::VisitSimdOperator<'a, Output = ()>> { + Some(self) + } + + wasmparser::for_each_visit_operator!(table_mutation_scan); +} + +impl<'a> wasmparser::VisitSimdOperator<'a> for TableMutationScan { + wasmparser::for_each_visit_simd_operator!(table_mutation_scan); +} diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 98e551003d9b..e7e1d51e3472 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -3,7 +3,7 @@ use crate::prelude::*; use crate::*; use core::ops::Range; -use cranelift_entity::{EntityRef, packed_option::ReservedValue}; +use cranelift_entity::{EntityRef, EntitySet, packed_option::ReservedValue}; use serde_derive::{Deserialize, Serialize}; /// A WebAssembly linear memory initializer. @@ -201,6 +201,16 @@ pub struct Module { /// to be infallible as part of [`ModuleTranslation::finalize_table_init`]. pub table_initialization: TryPrimaryMap>, + /// Tables whose contents or size may change after instantiation. + /// + /// Filled in during translation: imported and exported tables are + /// always in this set (the embedder, or another module importing the + /// export, can mutate them through the public API without any + /// instruction in this module), and a defined non-exported table is + /// added when any instruction in the code section can write to it. + /// See [`Module::table_is_immutable`] for the exact contract. + mutated_tables: EntitySet, + /// WebAssembly linear memory initializer. /// /// This will track how memory is initialized, either exclusively via @@ -306,6 +316,7 @@ impl Module { exports: Default::default(), startup: ModuleStartup::None, table_initialization: Default::default(), + mutated_tables: Default::default(), memory_initialization: Default::default(), passive_elements: Default::default(), runtime_data: Default::default(), @@ -382,6 +393,46 @@ impl Module { index.index() < self.num_imported_tables } + /// Records that table `index` may be written to after instantiation. + /// + /// Called during translation: up front for every imported and exported + /// table, and again for each table some instruction in the code + /// section can write to. + #[cfg(feature = "compile")] + pub(crate) fn mark_table_mutated(&mut self, index: TableIndex) { + self.mutated_tables.insert(index); + } + + /// Returns whether table `index` provably retains its + /// instantiation-time contents and size for the lifetime of every + /// instance of this module. + /// + /// True only when all of the following hold: + /// + /// * the table is defined in this module — an imported table's other + /// owners can mutate it out of this module's view; + /// * the table is not exported — the embedder, or any module that + /// imports the export, can mutate it through the public API without + /// any instruction in this module executing; + /// * no instruction in this module's code section can write to it: + /// `table.set`, `table.fill`, `table.grow`, `table.copy` with this + /// table as the destination, `table.init`, or their + /// shared-everything-threads atomic variants. + /// + /// Element segments applied at instantiation are part of the table's + /// *initial* contents, not a mutation. `elem.drop` drops a passive + /// segment without writing to any table, so it does not affect this; + /// a passive segment can only reach a table through `table.init`, + /// which does. + /// + /// Consumers may rely on this for correctness (e.g. eliding + /// `call_indirect` checks), so a false positive here is a soundness + /// bug, not a missed optimization. + #[inline] + pub fn table_is_immutable(&self, index: TableIndex) -> bool { + !self.mutated_tables.contains(index) + } + /// Convert a `DefinedMemoryIndex` into a `MemoryIndex`. #[inline] pub fn memory_index(&self, defined_memory: DefinedMemoryIndex) -> MemoryIndex { @@ -614,6 +665,7 @@ impl TypeTrace for Module { exports: _, startup, table_initialization: _, + mutated_tables: _, memory_initialization: _, passive_elements: _, runtime_data: _, @@ -666,6 +718,7 @@ impl TypeTrace for Module { exports: _, startup, table_initialization: _, + mutated_tables: _, memory_initialization: _, passive_elements: _, runtime_data: _, diff --git a/crates/environ/tests/table_mutability.rs b/crates/environ/tests/table_mutability.rs new file mode 100644 index 000000000000..581665fc2008 --- /dev/null +++ b/crates/environ/tests/table_mutability.rs @@ -0,0 +1,350 @@ +//! Integration tests for the table-mutability analysis behind +//! `Module::table_is_immutable`. +//! +//! The per-table mutability bit is the foundation of the `call_indirect` +//! optimizations in `crates/cranelift/src/func_environ.rs` (bound-load +//! elision on never-grown tables, and the follow-up sig-check / null-check +//! elisions). A false negative here — failing to mark a table as mutated +//! when it actually is — would silently skip required runtime checks. A +//! false positive — marking an immutable table as mutated — is merely a +//! missed optimization. Pin the analysis behaviour with focused +//! module-level tests so any regression surfaces immediately, not after a +//! downstream optimization fires on a now-invalid premise. +//! +//! Test scenario inspiration drawn from comparable bugs in peer +//! interpreters that have shipped fixes for analogous IC-invalidation +//! mistakes: +//! +//! - **Luau** (`LOP_NAMECALL`): inline cache had to be invalidated on +//! `table.insert` / metatable change. Analogous wasm risk: `table.grow` +//! not invalidating an immutability proof, so see `table_grow_marks…`. +//! - **JavaScriptCore** (`ic_table`): inline-cache corruption from missed +//! shape transitions. Analogous risk: over-marking, e.g. `table.copy` +//! wrongly marking the SOURCE table as mutated would forbid downstream +//! optimizations on a perfectly read-only table. See +//! `table_copy_marks_destination_only_not_source`. +//! - **Hermes** (`HiddenClass` cache): property cache misses with +//! `Object.defineProperty`. Analogous risk: `table.init` (passive- +//! segment write at runtime) being treated as a no-op rather than a +//! write. See `table_init_marks_destination`. +//! +//! Lives in `tests/` rather than as a `#[cfg(test)] mod` inside +//! `module_environ.rs` so it builds against the lib as a normal +//! dependency. + +use wasmparser::{Parser, Validator, WasmFeatures}; +use wasmtime_environ::{ + ModuleEnvironment, ModuleTypesBuilder, StaticModuleIndex, TableIndex, Tunables, +}; + +/// Translate `wat` and return the per-table "may be mutated" bits, in +/// table-index order. Helper to keep individual tests short. +fn translate_and_get_mutability(wat: &str) -> Vec { + let bytes = wat::parse_str(wat).expect("WAT parse failed"); + let tunables = Tunables::default_host(); + // WASM2 covers reference-types + bulk-memory, which is what every + // table-mutating opcode below needs (`table.set`, `table.fill`, + // `table.grow`, `table.copy`, `table.init`, `elem.drop`). + let features = WasmFeatures::WASM2; + let mut validator = Validator::new_with_features(features); + let mut types = ModuleTypesBuilder::new(&validator); + let env = ModuleEnvironment::new( + &tunables, + &mut validator, + &mut types, + StaticModuleIndex::from_u32(0), + ); + let parser = Parser::new(0); + let translation = env.translate(parser, &bytes).expect("translate failed"); + let n: u32 = translation.module.tables.len().try_into().unwrap(); + (0..n) + .map(|i| { + !translation + .module + .table_is_immutable(TableIndex::from_u32(i)) + }) + .collect() +} + +/// A table only used as the source of `call_indirect` and `table.get` is +/// provably immutable. (Both ops READ the table; neither writes it.) The +/// table is intentionally NOT exported — exported tables are +/// conservatively pre-marked as mutated (see +/// `exported_tables_are_pre_marked` for the export case) since the host +/// can mutate them via the public wasmtime API. +#[test] +fn read_only_table_is_immutable() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 42) + (elem (i32.const 0) $f $f $f $f) + (func (export "call_zero") (result i32) + i32.const 0 + call_indirect (param) (result i32)) + (func (export "read_zero") (result funcref) + i32.const 0 + table.get 0)) + "#, + ); + assert_eq!(bits, vec![false], "no opcode mutated this table"); +} + +/// Exported tables are always pre-marked as mutated, regardless of +/// whether any opcode in this module touches them. The host can call +/// `Table::set` / `Table::grow` via the public wasmtime API on any +/// exported table, and another module that imports the export can also +/// mutate it. Without this rule, downstream optimizations would +/// happily elide null traps and sig checks on exported tables on the +/// (false) assumption that the table contents are stable. +#[test] +fn exported_tables_are_pre_marked() { + let bits = translate_and_get_mutability( + r#" + (module + (table (export "t") 4 funcref) + (func $f (result i32) i32.const 42) + (elem (i32.const 0) $f $f $f $f)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.set` marks its destination as mutated. +#[test] +fn table_set_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "do_set") + i32.const 1 + ref.func $f + table.set 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.fill` marks its destination as mutated. +#[test] +fn table_fill_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "do_fill") + i32.const 0 + ref.func $f + i32.const 4 + table.fill 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.grow` is treated as mutating — analogous to Luau's NAMECALL IC +/// needing to invalidate on table-shape change. +#[test] +fn table_grow_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func (export "do_grow") (result i32) + ref.null func + i32.const 1 + table.grow 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.copy` marks the DESTINATION but explicitly NOT the source. The +/// source is read-only (its contents aren't changed by the op); marking +/// it as mutated would forbid downstream optimizations from treating it +/// as immutable, which would be incorrect over-conservatism — the JSC +/// `ic_table` analogue. +#[test] +fn table_copy_marks_destination_only_not_source() { + let bits = translate_and_get_mutability( + r#" + (module + (table $dst (export "dst") 4 funcref) + (table $src 4 funcref) + (func $f (result i32) i32.const 0) + (elem (table $src) (i32.const 0) func $f $f $f $f) + (func (export "do_copy") + i32.const 0 ;; dst offset + i32.const 0 ;; src offset + i32.const 4 ;; len + table.copy $dst $src)) + "#, + ); + assert_eq!( + bits, + vec![true, false], + "dst should be mutated, src should remain immutable", + ); +} + +/// `table.init` writes to the destination table from a passive elem +/// segment, so it is treated as mutation (the destination's contents +/// change at runtime). +#[test] +fn table_init_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (elem $e funcref (ref.func $f) (ref.func $f)) + (func (export "do_init") + i32.const 0 ;; dst + i32.const 0 ;; src offset within elem + i32.const 2 ;; len + table.init 0 $e)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `elem.drop` drops a passive element segment but does NOT write to any +/// table — distinct from `table.init` which DOES write. A pessimistic +/// implementation that marked all tables as mutated on `elem.drop` would +/// hand out false positives and shut off optimizations on perfectly- +/// immutable tables. +#[test] +fn elem_drop_does_not_mark_tables() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (elem $e funcref (ref.func $f)) + (func (export "do_drop") + elem.drop $e)) + "#, + ); + assert_eq!(bits, vec![false]); +} + +/// Imported tables are always pre-marked as mutated, regardless of +/// whether any opcode in this module touches them. The importer can +/// mutate the table in ways this module can't see. +#[test] +fn imported_tables_are_pre_marked() { + let bits = translate_and_get_mutability( + r#" + (module + (import "host" "t" (table 4 funcref))) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// A mutation in ONE function correctly marks the table — the analysis +/// has to walk every function body, not just the first. +#[test] +fn mutation_in_any_function_counts() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "innocent") (result i32) + i32.const 0 + call_indirect (param) (result i32)) + (func (export "guilty") + i32.const 0 + ref.func $f + table.set 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// Two tables, one mutated, one not. The analysis tracks per-table — a +/// mutation on one must not leak to the other. +#[test] +fn mutation_isolated_to_target_table() { + let bits = translate_and_get_mutability( + r#" + (module + (table $a 4 funcref) + (table $b 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "mut_a") + i32.const 0 + ref.func $f + table.set $a)) + "#, + ); + assert_eq!( + bits, + vec![true, false], + "$a should be mutated, $b should remain immutable", + ); +} + +/// Translating without any tables at all must not panic, and must produce +/// an empty result rather than e.g. a default-allocated single entry. +#[test] +fn module_with_no_tables_produces_empty_mutability_vec() { + let bits = translate_and_get_mutability( + r#" + (module + (func (export "noop"))) + "#, + ); + assert!(bits.is_empty(), "no tables ⇒ no mutability bits"); +} + +/// The scan runs before function bodies are validated, so a body can name +/// a table index that doesn't exist. The analysis must neither panic nor +/// size any per-table structure from that unvalidated index; the module +/// is rejected later when the body is validated. (Only the code section +/// is malformed here — translation itself succeeds because body +/// validation is deferred to compilation.) +#[test] +fn out_of_range_table_index_in_unvalidated_body_is_ignored() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "bogus") + i32.const 0 + ref.func $f + table.set 99)) + "#, + ); + assert_eq!( + bits, + vec![false], + "an out-of-range destination index marks nothing", + ); +} + +/// When every table is pre-marked (here: one exported, one imported), the +/// result is all-mutated regardless of what the code section contains — +/// this is also the shape where the analysis skips the body walk +/// entirely. +#[test] +fn all_tables_pre_marked_without_code_mutation() { + let bits = translate_and_get_mutability( + r#" + (module + (import "host" "t" (table 4 funcref)) + (table (export "u") 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "innocent") (result i32) + i32.const 0 + call_indirect (param) (result i32))) + "#, + ); + assert_eq!(bits, vec![true, true]); +} diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index c3788854ae4b..db8193582247 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -201,7 +201,7 @@ profiling = [ ] # Enables parallel compilation of WebAssembly code. -parallel-compilation = ["dep:rayon", "std"] +parallel-compilation = ["dep:rayon", "std", "wasmtime-environ/parallel"] # Enables support for automatic cache configuration to be enabled in `Config`. cache = ["dep:wasmtime-cache", "std"] diff --git a/tests/disas/call-indirect-immutable-static-bound.wat b/tests/disas/call-indirect-immutable-static-bound.wat new file mode 100644 index 000000000000..a58c31d4111d --- /dev/null +++ b/tests/disas/call-indirect-immutable-static-bound.wat @@ -0,0 +1,129 @@ +;;! target = "x86_64" + +;; Table declared with min < max (a "dynamic-declared" table) that is +;; never written to in the module. Without the per-table mutability +;; bit, Cranelift would emit `load.i64 v0+56` per dispatch to fetch +;; the current bound. With it, `make_table_base_bound` lowers to +;; `TableSize::Static` and the bound becomes an immediate. +;; +;; Look for: bounds-check `iconst.i32 16` (the declared min, used as +;; static bound) and NO `load.i64 ... v0+56` for the current_elements +;; field. (`+48` for the funcref base is still loaded — that's the +;; element-data pointer, separate from the bound.) + +(module + ;; min=16, max=64 — distinct, so without our optimization the + ;; bound would be loaded per dispatch from `current_elements`. + (table 16 64 funcref) + + (func $f1 (result i32) i32.const 1) + (func $f2 (result i32) i32.const 2) + (func $f3 (result i32) i32.const 3) + + (func (export "call_it") (param i32) (result i32) + local.get 0 + call_indirect (result i32)) + + (elem (i32.const 0) func $f1 $f2 $f3)) +;; function u0:0(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003f v2 = iconst.i32 1 +;; @0041 jump block1 +;; +;; block1: +;; @0041 return v2 ; v2 = 1 +;; } +;; +;; function u0:1(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0044 v2 = iconst.i32 2 +;; @0046 jump block1 +;; +;; block1: +;; @0046 return v2 ; v2 = 2 +;; } +;; +;; function u0:2(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0049 v2 = iconst.i32 3 +;; @004b jump block1 +;; +;; block1: +;; @004b return v2 ; v2 = 3 +;; } +;; +;; function u0:3(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 671088640 "VMTableDefinition+0x0" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region4 = 40 "VMContext+0x28" +;; region5 = 1677721600 "TypeIdsArray+0x0" +;; region6 = 1610612752 "VMFuncRef+0x10" +;; region7 = 1610612744 "VMFuncRef+0x8" +;; region8 = 1610612760 "VMFuncRef+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx, i64) -> i32 tail +;; sig1 = (i64 vmctx, i32, i64) -> i64 tail +;; fn0 = colocated u805306368:7 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0050 v3 = iconst.i32 16 +;; @0050 v4 = icmp uge v2, v3 ; v3 = 16 +;; @0050 v5 = uextend.i64 v2 +;; @0050 v6 = load.i64 notrap aligned readonly can_move region2 v0+48 +;; @0050 v7 = iconst.i64 3 +;; @0050 v8 = ishl v5, v7 ; v7 = 3 +;; @0050 v9 = iadd v6, v8 +;; @0050 v10 = iconst.i64 0 +;; @0050 v11 = select_spectre_guard v4, v10, v9 ; v10 = 0 +;; @0050 v12 = load.i64 user6 aligned region3 v11 +;; @0050 v13 = iconst.i64 -2 +;; @0050 v14 = band v12, v13 ; v13 = -2 +;; @0050 brif v12, block3(v14), block2 +;; +;; block2 cold: +;; @0050 v16 = iconst.i32 0 +;; @0050 v17 = uextend.i64 v2 +;; @0050 v18 = call fn0(v0, v16, v17) ; v16 = 0 +;; @0050 jump block3(v18) +;; +;; block3(v15: i64): +;; @0050 v19 = load.i64 notrap aligned readonly can_move region4 v0+40 +;; @0050 v20 = load.i32 notrap aligned readonly can_move region5 v19 +;; @0050 v21 = load.i32 user7 aligned readonly region6 v15+16 +;; @0050 v22 = icmp eq v21, v20 +;; @0050 v23 = uextend.i32 v22 +;; @0050 trapz v23, user8 +;; @0050 v24 = load.i64 notrap aligned readonly region7 v15+8 +;; @0050 v25 = load.i64 notrap aligned readonly region8 v15+24 +;; @0050 v26 = call_indirect sig0, v24(v25, v0) +;; @0053 jump block1 +;; +;; block1: +;; @0053 return v26 +;; } diff --git a/tests/disas/gc/call-indirect-final-type.wat b/tests/disas/gc/call-indirect-final-type.wat index 55449f05f049..9bc2dbe79046 100644 --- a/tests/disas/gc/call-indirect-final-type.wat +++ b/tests/disas/gc/call-indirect-final-type.wat @@ -19,13 +19,12 @@ ;; region0 = 8 "VMContext+0x8" ;; region1 = 67108888 "VMStoreContext+0x18" ;; region2 = 671088640 "VMTableDefinition+0x0" -;; region3 = 671088648 "VMTableDefinition+0x8" -;; region4 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" -;; region5 = 40 "VMContext+0x28" -;; region6 = 1677721600 "TypeIdsArray+0x0" -;; region7 = 1610612752 "VMFuncRef+0x10" -;; region8 = 1610612744 "VMFuncRef+0x8" -;; region9 = 1610612760 "VMFuncRef+0x18" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region4 = 40 "VMContext+0x28" +;; region5 = 1677721600 "TypeIdsArray+0x0" +;; region6 = 1610612752 "VMFuncRef+0x10" +;; region7 = 1610612744 "VMFuncRef+0x8" +;; region8 = 1610612760 "VMFuncRef+0x18" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 @@ -35,52 +34,43 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32): -;; @002b v4 = load.i64 notrap aligned region3 v0+56 -;; @002b v8 = load.i64 notrap aligned region2 v0+48 -;; @002b v5 = ireduce.i32 v4 -;; @002b v6 = icmp uge v3, v5 -;; @002b v12 = iconst.i64 0 -;; @002b v7 = uextend.i64 v3 -;; @002b v9 = iconst.i64 3 -;; @002b v10 = ishl v7, v9 ; v9 = 3 -;; @002b v11 = iadd v8, v10 -;; @002b v13 = select_spectre_guard v6, v12, v11 ; v12 = 0 -;; @002b v14 = load.i64 user6 aligned region4 v13 -;; @002b v15 = iconst.i64 -2 -;; @002b v16 = band v14, v15 ; v15 = -2 -;; @002b brif v14, block3(v16), block2 +;; @002b v11 = iconst.i64 0 +;; @002b v13 = load.i64 user6 aligned region3 v11 ; v11 = 0 +;; @002b v14 = iconst.i64 -2 +;; @002b v15 = band v13, v14 ; v14 = -2 +;; @002b brif v13, block3(v15), block2 ;; ;; block2 cold: -;; @002b v18 = iconst.i32 0 -;; @002b v20 = call fn0(v0, v18, v7) ; v18 = 0 -;; @002b jump block3(v20) +;; @002b v4 = iconst.i32 0 +;; @002b v6 = uextend.i64 v3 +;; @002b v19 = call fn0(v0, v4, v6) ; v4 = 0 +;; @002b jump block3(v19) ;; -;; block3(v17: i64): -;; @002b v23 = load.i32 user7 aligned readonly region7 v17+16 -;; @002b v21 = load.i64 notrap aligned readonly can_move region5 v0+40 -;; @002b v22 = load.i32 notrap aligned readonly can_move region6 v21 -;; @002b v24 = icmp eq v23, v22 -;; @002b trapz v24, user8 -;; @002b v26 = load.i64 notrap aligned readonly region8 v17+8 -;; @002b v27 = load.i64 notrap aligned readonly region9 v17+24 -;; @002b v28 = call_indirect sig0, v26(v27, v0, v2) +;; block3(v16: i64): +;; @002b v22 = load.i32 user7 aligned readonly region6 v16+16 +;; @002b v20 = load.i64 notrap aligned readonly can_move region4 v0+40 +;; @002b v21 = load.i32 notrap aligned readonly can_move region5 v20 +;; @002b v23 = icmp eq v22, v21 +;; @002b trapz v23, user8 +;; @002b v25 = load.i64 notrap aligned readonly region7 v16+8 +;; @002b v26 = load.i64 notrap aligned readonly region8 v16+24 +;; @002b v27 = call_indirect sig0, v25(v26, v0, v2) ;; @002e jump block1 ;; ;; block1: -;; @002e return v28 +;; @002e return v27 ;; } ;; ;; function u0:1(i64 vmctx, i64, i32, i32) -> i32 tail { ;; region0 = 8 "VMContext+0x8" ;; region1 = 67108888 "VMStoreContext+0x18" ;; region2 = 671088640 "VMTableDefinition+0x0" -;; region3 = 671088648 "VMTableDefinition+0x8" -;; region4 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" -;; region5 = 40 "VMContext+0x28" -;; region6 = 1677721600 "TypeIdsArray+0x0" -;; region7 = 1610612752 "VMFuncRef+0x10" -;; region8 = 1610612744 "VMFuncRef+0x8" -;; region9 = 1610612760 "VMFuncRef+0x18" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region4 = 40 "VMContext+0x28" +;; region5 = 1677721600 "TypeIdsArray+0x0" +;; region6 = 1610612752 "VMFuncRef+0x10" +;; region7 = 1610612744 "VMFuncRef+0x8" +;; region8 = 1610612760 "VMFuncRef+0x18" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 @@ -90,33 +80,25 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32): -;; @0035 v4 = load.i64 notrap aligned region3 v0+56 -;; @0035 v8 = load.i64 notrap aligned region2 v0+48 -;; @0035 v5 = ireduce.i32 v4 -;; @0035 v6 = icmp uge v3, v5 -;; @0035 v12 = iconst.i64 0 -;; @0035 v7 = uextend.i64 v3 -;; @0035 v9 = iconst.i64 3 -;; @0035 v10 = ishl v7, v9 ; v9 = 3 -;; @0035 v11 = iadd v8, v10 -;; @0035 v13 = select_spectre_guard v6, v12, v11 ; v12 = 0 -;; @0035 v14 = load.i64 user6 aligned region4 v13 -;; @0035 v15 = iconst.i64 -2 -;; @0035 v16 = band v14, v15 ; v15 = -2 -;; @0035 brif v14, block3(v16), block2 +;; @0035 v11 = iconst.i64 0 +;; @0035 v13 = load.i64 user6 aligned region3 v11 ; v11 = 0 +;; @0035 v14 = iconst.i64 -2 +;; @0035 v15 = band v13, v14 ; v14 = -2 +;; @0035 brif v13, block3(v15), block2 ;; ;; block2 cold: -;; @0035 v18 = iconst.i32 0 -;; @0035 v20 = call fn0(v0, v18, v7) ; v18 = 0 -;; @0035 jump block3(v20) +;; @0035 v4 = iconst.i32 0 +;; @0035 v6 = uextend.i64 v3 +;; @0035 v19 = call fn0(v0, v4, v6) ; v4 = 0 +;; @0035 jump block3(v19) ;; -;; block3(v17: i64): -;; @0035 v23 = load.i32 user7 aligned readonly region7 v17+16 -;; @0035 v21 = load.i64 notrap aligned readonly can_move region5 v0+40 -;; @0035 v22 = load.i32 notrap aligned readonly can_move region6 v21 -;; @0035 v24 = icmp eq v23, v22 -;; @0035 trapz v24, user8 -;; @0035 v26 = load.i64 notrap aligned readonly region8 v17+8 -;; @0035 v27 = load.i64 notrap aligned readonly region9 v17+24 -;; @0035 return_call_indirect sig0, v26(v27, v0, v2) +;; block3(v16: i64): +;; @0035 v22 = load.i32 user7 aligned readonly region6 v16+16 +;; @0035 v20 = load.i64 notrap aligned readonly can_move region4 v0+40 +;; @0035 v21 = load.i32 notrap aligned readonly can_move region5 v20 +;; @0035 v23 = icmp eq v22, v21 +;; @0035 trapz v23, user8 +;; @0035 v25 = load.i64 notrap aligned readonly region7 v16+8 +;; @0035 v26 = load.i64 notrap aligned readonly region8 v16+24 +;; @0035 return_call_indirect sig0, v25(v26, v0, v2) ;; } diff --git a/tests/disas/startup-elem-active.wat b/tests/disas/startup-elem-active.wat index 4b802eee4480..5fdf9990a768 100644 --- a/tests/disas/startup-elem-active.wat +++ b/tests/disas/startup-elem-active.wat @@ -47,37 +47,21 @@ ;; ;; function u2415919104:0(i64 vmctx, i64) tail { ;; region0 = 671088640 "VMTableDefinition+0x0" -;; region1 = 671088648 "VMTableDefinition+0x8" -;; region2 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region1 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" ;; ;; block0(v0: i64, v1: i64): -;; v4 = load.i64 notrap aligned region1 v0+56 -;; v5 = ireduce.i32 v4 -;; v6 = uextend.i64 v5 -;; v78 = iconst.i64 4 -;; v84 = icmp ult v6, v78 ; v78 = 4 -;; trapnz v84, user6 -;; v13 = load.i64 notrap aligned region0 v0+48 -;; v95 = iconst.i32 21 -;; v2 = iconst.i32 1 -;; v106 = icmp ule v5, v2 ; v2 = 1 -;; v71 = iconst.i64 0 -;; v17 = iadd v13, v78 ; v78 = 4 -;; v34 = select_spectre_guard v106, v71, v17 ; v71 = 0 -;; store user6 aligned region2 v95, v34 ; v95 = 21 -;; v109 = iconst.i32 23 -;; v115 = iconst.i32 2 -;; v121 = icmp ule v5, v115 ; v115 = 2 -;; v123 = iconst.i64 8 -;; v49 = iadd v13, v123 ; v123 = 8 -;; v51 = select_spectre_guard v121, v71, v49 ; v71 = 0 -;; store user6 aligned region2 v109, v51 ; v109 = 23 -;; v125 = iconst.i32 25 -;; v3 = iconst.i32 3 -;; v136 = icmp ule v5, v3 ; v3 = 3 -;; v138 = iconst.i64 12 -;; v66 = iadd v13, v138 ; v138 = 12 -;; v68 = select_spectre_guard v136, v71, v66 ; v71 = 0 -;; store user6 aligned region2 v125, v68 ; v125 = 25 +;; v96 = iconst.i32 21 +;; v12 = load.i64 notrap aligned readonly can_move region0 v0+48 +;; v75 = iconst.i64 4 +;; v16 = iadd v12, v75 ; v75 = 4 +;; store user6 aligned region1 v96, v16 ; v96 = 21 +;; v113 = iconst.i32 23 +;; v130 = iconst.i64 8 +;; v46 = iadd v12, v130 ; v130 = 8 +;; store user6 aligned region1 v113, v46 ; v113 = 23 +;; v132 = iconst.i32 25 +;; v148 = iconst.i64 12 +;; v62 = iadd v12, v148 ; v148 = 12 +;; store user6 aligned region1 v132, v62 ; v132 = 25 ;; return ;; } diff --git a/tests/disas/startup-table-initial-value.wat b/tests/disas/startup-table-initial-value.wat index 3db39fac9976..3bdc39250ed9 100644 --- a/tests/disas/startup-table-initial-value.wat +++ b/tests/disas/startup-table-initial-value.wat @@ -41,31 +41,24 @@ ;; ;; function u2415919104:0(i64 vmctx, i64) tail { ;; region0 = 671088640 "VMTableDefinition+0x0" -;; region1 = 671088648 "VMTableDefinition+0x8" -;; region2 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region1 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" ;; ;; block0(v0: i64, v1: i64): -;; v9 = load.i64 notrap aligned region1 v0+56 -;; v10 = ireduce.i32 v9 -;; v11 = uextend.i64 v10 -;; v39 = iconst.i64 10 -;; v51 = icmp ult v11, v39 ; v39 = 10 -;; trapnz v51, user6 -;; v18 = load.i64 notrap aligned region0 v0+48 +;; v17 = load.i64 notrap aligned readonly can_move region0 v0+48 ;; v3 = iconst.i32 1 -;; v81 = iconst.i64 36 -;; v83 = iadd v18, v81 ; v81 = 36 -;; v20 = iconst.i64 4 -;; jump block1(v18) +;; v83 = iconst.i64 36 +;; v85 = iadd v17, v83 ; v83 = 36 +;; v19 = iconst.i64 4 +;; jump block1(v17) ;; -;; block1(v29: i64): -;; v86 = iconst.i32 1 -;; store notrap aligned region2 v86, v29 ; v86 = 1 -;; v87 = iadd.i64 v18, v81 ; v81 = 36 -;; v88 = icmp eq v29, v87 -;; v89 = iconst.i64 4 -;; v90 = iadd v29, v89 ; v89 = 4 -;; brif v88, block2, block1(v90) +;; block1(v28: i64): +;; v88 = iconst.i32 1 +;; store notrap aligned region1 v88, v28 ; v88 = 1 +;; v89 = iadd.i64 v17, v83 ; v83 = 36 +;; v90 = icmp eq v28, v89 +;; v91 = iconst.i64 4 +;; v92 = iadd v28, v91 ; v91 = 4 +;; brif v90, block2, block1(v92) ;; ;; block2: ;; return diff --git a/tests/misc_testsuite/immutable-table-call-indirect.wast b/tests/misc_testsuite/immutable-table-call-indirect.wast new file mode 100644 index 000000000000..163f2f0a789e --- /dev/null +++ b/tests/misc_testsuite/immutable-table-call-indirect.wast @@ -0,0 +1,166 @@ +;;! reference_types = true + +;; call_indirect through tables that are never grown, exported, or mutated. +;; Compilation may use a constant bound and elide null/signature checks on +;; these shapes; runtime behavior must be unchanged: in-bounds calls work, +;; and out-of-bounds, null-slot, and signature-mismatch accesses still trap. + +;; Mixed-signature immutable table with a null hole. +(module + (type $i2i (func (param i32) (result i32))) + (type $v2i (func (result i32))) + (table 5 funcref) + (elem (i32.const 0) $add1 $ten $add1) + + (func $add1 (type $i2i) (i32.add (local.get 0) (i32.const 1))) + (func $ten (type $v2i) (i32.const 10)) + + (func (export "call-i2i") (param i32 i32) (result i32) + (call_indirect (type $i2i) (local.get 1) (local.get 0))) + (func (export "call-v2i") (param i32) (result i32) + (call_indirect (type $v2i) (local.get 0)))) + +(assert_return (invoke "call-i2i" (i32.const 0) (i32.const 41)) (i32.const 42)) +(assert_return (invoke "call-i2i" (i32.const 2) (i32.const 7)) (i32.const 8)) +(assert_return (invoke "call-v2i" (i32.const 1)) (i32.const 10)) + +;; Signature mismatch still traps. +(assert_trap (invoke "call-i2i" (i32.const 1) (i32.const 0)) "indirect call type mismatch") +(assert_trap (invoke "call-v2i" (i32.const 0)) "indirect call type mismatch") + +;; Null slots still trap: slot 3 was never initialized. +(assert_trap (invoke "call-i2i" (i32.const 3) (i32.const 0)) "uninitialized element") +(assert_trap (invoke "call-v2i" (i32.const 4)) "uninitialized element") + +;; Out of bounds still traps against the constant bound. +(assert_trap (invoke "call-i2i" (i32.const 5) (i32.const 0)) "undefined element") +(assert_trap (invoke "call-i2i" (i32.const -1) (i32.const 0)) "undefined element") + +;; Uniform-signature immutable table, fully initialized. +(module + (type $v2i (func (result i32))) + (table 3 funcref) + (elem (i32.const 0) $a $b $c) + + (func $a (type $v2i) (i32.const 1)) + (func $b (type $v2i) (i32.const 2)) + (func $c (type $v2i) (i32.const 3)) + + (func (export "call") (param i32) (result i32) + (call_indirect (type $v2i) (local.get 0))) + (func (export "call-wrong-type") (param i32 i32) (result i32) + (call_indirect (param i32) (result i32) (local.get 1) (local.get 0)))) + +(assert_return (invoke "call" (i32.const 0)) (i32.const 1)) +(assert_return (invoke "call" (i32.const 1)) (i32.const 2)) +(assert_return (invoke "call" (i32.const 2)) (i32.const 3)) +(assert_trap (invoke "call" (i32.const 3)) "undefined element") + +;; A caller whose expected type differs from the table's uniform type must +;; still observe the mismatch. +(assert_trap (invoke "call-wrong-type" (i32.const 0) (i32.const 0)) "indirect call type mismatch") + +;; Same shapes through a declared-growable (no max) table never actually +;; grown: an empty never-grown table has no valid index. +(module + (table 0 100 funcref) + (func (export "call-empty") (param i32) + (call_indirect (local.get 0)))) + +(assert_trap (invoke "call-empty" (i32.const 0)) "undefined element") +(assert_trap (invoke "call-empty" (i32.const 99)) "undefined element") + +;; Populated min