Fix panicking overflow when calculating table sizes (#13242)

Return an error instead of panicking in the same manner that OOM is
handled.

Author: alexcrichton

RELEASES.md

@@ -1,3 +1,14 @@
+## 36.0.8
+
+Released 2026-04-30.
+
+### Fixed
+
+* Panic when allocating a table exceeding the size of the host's address space.
+  [GHSA-p8xm-42r7-89xg](https://github.com/bytecodealliance/wasmtime/security/advisories/GHSA-p8xm-42r7-89xg)
+
+--------------------------------------------------------------------------------
+
 ## 36.0.7
 
 Released 2026-04-09.

crates/test-util/src/wast.rs

@@ -432,6 +432,7 @@ impl WastTest {
                 "misc_testsuite/no-panic.wast",
                 "misc_testsuite/simple_ref_is_null.wast",
                 "misc_testsuite/table_grow_with_funcref.wast",
+                "misc_testsuite/memory64/table-too-big.wast",
                 "spec_testsuite/br_table.wast",
                 "spec_testsuite/global.wast",
                 "spec_testsuite/ref_func.wast",

crates/wasmtime/src/runtime/vm/table.rs

@@ -408,7 +408,9 @@ unsafe fn alloc_dynamic_table_elements<T>(len: usize) -> Result<Vec<Option<T>>>
 
     let size = mem::size_of::<Option<T>>();
     let size = size.next_multiple_of(align);
-    let size = size.checked_mul(len).unwrap();
+    let size = size
+        .checked_mul(len)
+        .ok_or_else(|| format_err!("overflow calculating table allocation size"))?;
 
     let layout = Layout::from_size_align(size, align)?;
 
@@ -826,12 +828,15 @@ impl Table {
             // that delta is non-zero and the new size doesn't exceed the
             // maximum mean we can't get here.
             Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => {
+                vec_reserve_resize(elements, new_size)?;
                 elements.resize(new_size, None);
             }
             Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
+                vec_reserve_resize(elements, new_size)?;
                 elements.resize_with(new_size, || None);
             }
             Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => {
+                vec_reserve_resize(elements, new_size)?;
                 elements.resize(new_size, None);
             }
         }
@@ -1230,3 +1235,10 @@ impl Default for Table {
         })
     }
 }
+
+fn vec_reserve_resize<T>(vec: &mut Vec<T>, new_size: usize) -> Result<()> {
+    if vec.len() < new_size {
+        vec.try_reserve(new_size - vec.len())?;
+    }
+    Ok(())
+}

tests/misc_testsuite/memory64/table-too-big.wast

@@ -0,0 +1,17 @@
+;;! memory64 = true
+;;! hogs_memory = true
+;;! reference_types = true
+
+(assert_trap
+  (module (table i64 0x2000_0000_0000_0000 funcref))
+  "overflow calculating table allocation size")
+
+(module
+  (table i64 0 funcref)
+  (func (export "grow") (param i64) (result i64)
+    (table.grow 0 (ref.null func) (local.get 0))
+  )
+)
+
+(assert_trap (invoke "grow" (i64.const 0x2000_0000_0000_0000))
+  "memory allocation failed")