Implement the semi-space copying garbage collector (#13107)

* Implement the semi-space copying garbage collector

This is a classic semi-space copying collector that uses bump allocation and
Cheney-style worklists to avoid an explicit stack for grey (relocated but not
yet scanned) objects outside the GC heap. Forwarding "pointers" (really
`VMGcRef` indices) are stored inline in objects in the old semi-space, which
again avoids explicit data structures outside of the GC heap. Furthermore, an
intrusive linked-list of all `externref`s is maintained to allow for efficiently
sweeping their associated host data after collection (and, once again, avoiding
additional data structures outside the GC heap).

Allocation in compiled Wasm code always happens by calling out to the
`gc_alloc_raw` libcall currently. This is expected to become inline bump
allocation, very similar to the null collector, in a follow-up commit shortly
after this one. It is delayed to make review easier.

Collection is not incremental (in the Wasmtime sense, not the GC literature
sense) yet and the `CopyingCollection::collect_increment` implementation only
has a single increment. This is delayed to make review easier and is also
expected to come shortly in follow-up commits.

I've also added new disas tests for the copying collector and also enabled
running wast tests which need a GC heap with the copying collector.

Finally, fuzz config generation can now generate configurations that enable the
copying collector.

* fix compilation error, add a couple more asserts

* Fix bug where semi-spaces' usable sizes were not equal

* Ensure that we always poison new gc heap capacity in `replace_memory`

* remove dead method

* remove unused import

* review feedback and little clean ups

* fix poisoning for new growth

* Add additional copying collector and GC object tests

* fix compilation after rebase

* actually fix compilation after rebase

* Update disas test output

* Fix grow-vs-collect logic under GC zeal

* remove warning in no-gc build

Author: fitzgen

Cargo.toml

@@ -97,7 +97,7 @@ rustix = { workspace = true, features = ["mm", "process"] }
 
 [dev-dependencies]
 # depend again on wasmtime to activate its default features for tests
-wasmtime = { workspace = true, features = ['default', 'anyhow', 'winch', 'pulley', 'all-arch', 'call-hook', 'memory-protection-keys', 'component-model-async', 'component-model-async-bytes'] }
+wasmtime = { workspace = true, features = ['default', 'anyhow', 'winch', 'pulley', 'all-arch', 'call-hook', 'memory-protection-keys', 'component-model-async', 'component-model-async-bytes', 'gc-copying'] }
 env_logger = { workspace = true }
 log = { workspace = true }
 filecheck = { workspace = true }

crates/cranelift/src/func_environ/gc/enabled.rs

@@ -72,7 +72,7 @@ pub fn gc_compiler(func_env: &mut FuncEnvironment<'_>) -> WasmResult<Box<dyn GcC
 }
 
 #[cfg_attr(
-    not(feature = "gc-drc"),
+    not(any(feature = "gc-drc", feature = "gc-copying")),
     expect(dead_code, reason = "easier to define")
 )]
 fn unbarriered_load_gc_ref(
@@ -90,7 +90,7 @@ fn unbarriered_load_gc_ref(
 }
 
 #[cfg_attr(
-    not(any(feature = "gc-drc", feature = "gc-null")),
+    not(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying")),
     expect(dead_code, reason = "easier to define")
 )]
 fn unbarriered_store_gc_ref(
@@ -105,6 +105,37 @@ fn unbarriered_store_gc_ref(
     Ok(())
 }
 
+/// Emit CLIF to call the `gc_raw_alloc` libcall.
+#[cfg(any(feature = "gc-drc", feature = "gc-copying"))]
+fn emit_gc_raw_alloc(
+    func_env: &mut FuncEnvironment<'_>,
+    builder: &mut FunctionBuilder<'_>,
+    kind: VMGcKind,
+    ty: ModuleInternedTypeIndex,
+    size: ir::Value,
+    align: u32,
+) -> ir::Value {
+    let gc_alloc_raw_builtin = func_env.builtin_functions.gc_alloc_raw(builder.func);
+    let vmctx = func_env.vmctx_val(&mut builder.cursor());
+
+    let kind = builder
+        .ins()
+        .iconst(ir::types::I32, i64::from(kind.as_u32()));
+
+    let ty = func_env.module_interned_to_shared_ty(&mut builder.cursor(), ty);
+
+    assert!(align.is_power_of_two());
+    let align = builder.ins().iconst(ir::types::I32, i64::from(align));
+
+    let call_inst = builder
+        .ins()
+        .call(gc_alloc_raw_builtin, &[vmctx, kind, ty, size, align]);
+
+    let gc_ref = builder.func.dfg.first_result(call_inst);
+    builder.declare_value_needs_stack_map(gc_ref);
+    gc_ref
+}
+
 /// Emit inline CLIF code that asserts an object's `VMGcKind` matches the
 /// expected kind. Only emits code when `cfg(gc_zeal)` is enabled.
 ///
@@ -645,7 +676,7 @@ pub fn translate_array_new_fixed(
 impl ArrayInit<'_> {
     /// Get the length (as an `i32`-typed `ir::Value`) of these array elements.
     #[cfg_attr(
-        not(any(feature = "gc-drc", feature = "gc-null")),
+        not(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying")),
         expect(dead_code, reason = "easier to define")
     )]
     fn len(self, pos: &mut FuncCursor) -> ir::Value {
@@ -660,7 +691,7 @@ impl ArrayInit<'_> {
 
     /// Initialize a newly-allocated array's elements.
     #[cfg_attr(
-        not(any(feature = "gc-drc", feature = "gc-null")),
+        not(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying")),
         expect(dead_code, reason = "easier to define")
     )]
     fn initialize(
@@ -1333,7 +1364,7 @@ fn uextend_i32_to_pointer_type(
 ///
 /// Traps if the size overflows.
 #[cfg_attr(
-    not(any(feature = "gc-drc", feature = "gc-null")),
+    not(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying")),
     expect(dead_code, reason = "easier to define")
 )]
 fn emit_array_size(
@@ -1381,7 +1412,7 @@ fn emit_array_size(
 /// Common helper for struct-field initialization that can be reused across
 /// collectors.
 #[cfg_attr(
-    not(any(feature = "gc-drc", feature = "gc-null")),
+    not(any(feature = "gc-drc", feature = "gc-null", feature = "gc-copying")),
     expect(dead_code, reason = "easier to define")
 )]
 fn initialize_struct_fields(
@@ -1483,7 +1514,7 @@ impl FuncEnvironment<'_> {
     }
 
     /// Get the GC heap's base.
-    #[cfg(any(feature = "gc-null", feature = "gc-drc"))]
+    #[cfg(any(feature = "gc-null", feature = "gc-drc", feature = "gc-copying"))]
     fn get_gc_heap_base(&mut self, builder: &mut FunctionBuilder) -> ir::Value {
         let global = self.get_gc_heap_base_global(&mut builder.func);
         builder.ins().global_value(self.pointer_type(), global)