Add separate GC heap tunables (#13080)

* Add separate GC heap tunables

Before this commit, the GC heap used a memory with identical tunables as regular
Wasm linear memories.

After this commit, the GC heap now uses a memory with its own
tunables (`gc_heap_reservation`, `gc_heap_guard_size`,
`gc_heap_reservation_for_growth`, and `gc_heap_may_move`) so that GC heaps can
be configured independently from linear memories. These new GC heap tunables are
also exposed as CLI flags and `Config` methods. We also generate pseudorandom
values for these tunables in our fuzzing config generator.

The pooling allocator uses the same pool for memories and GC heaps, so we
validate that if you are using GC and the pooling allocator, then the new
tunables must match the memory tunables.

* Fix pooling allocator + GC heap tests

* Addressing review feedback

* cargo fmt

* skip tests that require large memory reservations on 32-bit and qemu

* run miri wast tests with identical flags as compilation

* Fix GC heap tunables defaulting behavior

* Revert "run miri wast tests with identical flags as compilation"

Not necessary after fixing the GC heap tunables defaulting behavior

Author: fitzgen

crates/cli-flags/src/lib.rs

@@ -67,6 +67,19 @@ wasmtime_option_group! {
         /// Size, in bytes, of guard pages for linear memories.
         pub memory_guard_size: Option<u64>,
 
+        /// Do not allow the GC heap to move in the host process's address
+        /// space.
+        pub gc_heap_may_move: Option<bool>,
+
+        /// Initial virtual memory allocation size for the GC heap.
+        pub gc_heap_reservation: Option<u64>,
+
+        /// Bytes to reserve at the end of the GC heap for growth into.
+        pub gc_heap_reservation_for_growth: Option<u64>,
+
+        /// Size, in bytes, of guard pages for the GC heap.
+        pub gc_heap_guard_size: Option<u64>,
+
         /// Indicates whether an unmapped region of memory is placed before all
         /// linear memories.
         pub guard_before_linear_memory: Option<bool>,
@@ -891,6 +904,19 @@ impl CommonOptions {
         if let Some(enable) = self.opts.guard_before_linear_memory {
             config.guard_before_linear_memory(enable);
         }
+
+        if let Some(size) = self.opts.gc_heap_reservation {
+            config.gc_heap_reservation(size);
+        }
+        if let Some(enable) = self.opts.gc_heap_may_move {
+            config.gc_heap_may_move(enable);
+        }
+        if let Some(size) = self.opts.gc_heap_guard_size {
+            config.gc_heap_guard_size(size);
+        }
+        if let Some(size) = self.opts.gc_heap_reservation_for_growth {
+            config.gc_heap_reservation_for_growth(size);
+        }
         if let Some(enable) = self.opts.table_lazy_init {
             config.table_lazy_init(enable);
         }

crates/cranelift/src/bounds_checks.rs

@@ -184,16 +184,17 @@ fn bounds_check_field_access(
         env.heap_access_spectre_mitigation() && clif_memory_traps_enabled;
 
     let host_page_size_log2 = env.target_config().page_size_align_log2;
+    let memory_tunables = heap.memory_tunables(env.tunables());
     let can_use_virtual_memory = heap
         .memory
-        .can_use_virtual_memory(env.tunables(), host_page_size_log2)
+        .can_use_virtual_memory(memory_tunables.tunables(), host_page_size_log2)
         && clif_memory_traps_enabled;
     let can_elide_bounds_check = heap
         .memory
-        .can_elide_bounds_check(env.tunables(), host_page_size_log2)
+        .can_elide_bounds_check(&memory_tunables, host_page_size_log2)
         && clif_memory_traps_enabled;
-    let memory_guard_size = env.tunables().memory_guard_size;
-    let memory_reservation = env.tunables().memory_reservation;
+    let memory_guard_size = memory_tunables.guard_size();
+    let memory_reservation = memory_tunables.reservation();
 
     let offset_and_size = offset_plus_size(offset, access_size);
     let statically_in_bounds = statically_in_bounds(&builder.func, heap, index, offset_and_size);
@@ -348,7 +349,7 @@ fn bounds_check_field_access(
     // factor in the guard pages here.
     if can_use_virtual_memory
         && heap.memory.minimum_byte_size().unwrap_or(u64::MAX) <= memory_reservation
-        && !heap.memory.memory_may_move(env.tunables())
+        && !heap.memory.memory_may_move(&memory_tunables)
         && memory_reservation >= offset_and_size
     {
         let adjusted_bound = memory_reservation.checked_sub(offset_and_size).unwrap();

crates/cranelift/src/func_environ.rs

@@ -4,8 +4,8 @@ pub(crate) mod stack_switching;
 use crate::BuiltinFunctionSignatures;
 use crate::compiler::Compiler;
 use crate::translate::{
-    FuncTranslationStacks, GlobalVariable, Heap, HeapData, StructFieldsVec, TableData, TableSize,
-    TargetEnvironment,
+    FuncTranslationStacks, GlobalVariable, Heap, HeapData, MemoryKind, StructFieldsVec, TableData,
+    TableSize, TargetEnvironment,
 };
 use crate::trap::TranslateTrap;
 use cranelift_codegen::cursor::FuncCursor;
@@ -28,10 +28,10 @@ use wasmtime_core::math::f64_cvt_to_int_bounds;
 use wasmtime_environ::{
     BuiltinFunctionIndex, ComponentPC, DataIndex, DefinedFuncIndex, ElemIndex,
     EngineOrModuleTypeIndex, FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey,
-    GlobalConstValue, GlobalIndex, IndexType, Memory, MemoryIndex, Module, ModuleInternedTypeIndex,
-    ModuleTranslation, ModuleTypesBuilder, PtrSize, Table, TableIndex, TagIndex, Tunables,
-    TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType, WasmHeapTopType,
-    WasmHeapType, WasmRefType, WasmResult, WasmValType,
+    GlobalConstValue, GlobalIndex, IndexType, Memory, MemoryIndex, MemoryTunables, Module,
+    ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PtrSize, Table, TableIndex,
+    TagIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType,
+    WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmValType,
 };
 use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK};
 
@@ -1566,6 +1566,7 @@ impl FuncEnvironment<'_> {
         self.heaps.push(HeapData {
             base,
             bound,
+            kind: MemoryKind::LinearMemory,
             memory,
         })
     }
@@ -1578,9 +1579,10 @@ impl FuncEnvironment<'_> {
         offset: i32,
     ) -> ir::GlobalValue {
         let pointer_type = self.pointer_type();
+        let memory_tunables = MemoryTunables::new(self.tunables, MemoryKind::LinearMemory);
 
         let mut flags = ir::MemFlags::trusted().with_can_move();
-        if !memory.memory_may_move(self.tunables) {
+        if !memory.memory_may_move(&memory_tunables) {
             flags.set_readonly();
         }
 

crates/cranelift/src/func_environ/gc/enabled.rs

@@ -1,7 +1,7 @@
 use super::{ArrayInit, GcCompiler};
 use crate::bounds_checks::BoundsCheck;
 use crate::func_environ::{Extension, FuncEnvironment};
-use crate::translate::{Heap, HeapData, StructFieldsVec, TargetEnvironment};
+use crate::translate::{Heap, HeapData, MemoryKind, StructFieldsVec, TargetEnvironment};
 use crate::trap::TranslateTrap;
 use crate::{Reachability, TRAP_INTERNAL_ASSERT};
 use cranelift_codegen::ir::immediates::Offset32;
@@ -1449,10 +1449,12 @@ impl FuncEnvironment<'_> {
         let offset = self.offsets.ptr.vmstore_context_gc_heap_base();
 
         let mut flags = ir::MemFlags::trusted();
+        let memory_tunables =
+            wasmtime_environ::MemoryTunables::new(self.tunables, MemoryKind::GcHeap);
         if !self
             .tunables
             .gc_heap_memory_type()
-            .memory_may_move(self.tunables)
+            .memory_may_move(&memory_tunables)
         {
             flags.set_readonly();
             flags.set_can_move();
@@ -1512,6 +1514,7 @@ impl FuncEnvironment<'_> {
             base,
             bound,
             memory,
+            kind: MemoryKind::GcHeap,
         });
         self.gc_heap = Some(heap);
         heap

crates/cranelift/src/translate/heap.rs

@@ -4,6 +4,8 @@ use cranelift_codegen::ir::{self, GlobalValue, Type};
 use cranelift_entity::entity_impl;
 use wasmtime_environ::{IndexType, Memory};
 
+pub use wasmtime_environ::MemoryKind;
+
 /// An opaque reference to a [`HeapData`][crate::HeapData].
 ///
 /// While the order is stable, it is arbitrary.
@@ -45,6 +47,9 @@ pub struct HeapData {
 
     /// The type of wasm memory that this heap is operating on.
     pub memory: Memory,
+
+    /// Whether this is a linear memory or a GC heap.
+    pub kind: MemoryKind,
 }
 
 impl HeapData {
@@ -54,4 +59,12 @@ impl HeapData {
             IndexType::I64 => ir::types::I64,
         }
     }
+
+    /// Get the [`MemoryTunables`] for this heap based on its [`MemoryKind`].
+    pub fn memory_tunables<'a>(
+        &self,
+        tunables: &'a wasmtime_environ::Tunables,
+    ) -> wasmtime_environ::MemoryTunables<'a> {
+        wasmtime_environ::MemoryTunables::new(tunables, self.kind)
+    }
 }

crates/cranelift/src/translate/mod.rs

@@ -20,7 +20,7 @@ mod translation_utils;
 
 pub use self::environ::{GlobalVariable, StructFieldsVec, TargetEnvironment};
 pub use self::func_translator::FuncTranslator;
-pub use self::heap::{Heap, HeapData};
+pub use self::heap::{Heap, HeapData, MemoryKind};
 pub use self::stack::FuncTranslationStacks;
 pub use self::table::{TableData, TableSize};
 pub use self::translation_utils::*;

crates/environ/src/tunables.rs

@@ -160,6 +160,29 @@ define_tunables! {
         /// forced and the counter is reset. Only effective when
         /// `cfg(gc_zeal)` is enabled.
         pub gc_zeal_alloc_counter: Option<NonZeroU32>,
+
+        /// Initial size, in bytes, to be allocated for GC heaps.
+        ///
+        /// This is the same as `memory_reservation` but for GC heaps.
+        pub gc_heap_reservation: u64,
+
+        /// The size, in bytes, of the guard page region for GC heaps.
+        ///
+        /// This is the same as `memory_guard_size` but for GC heaps.
+        pub gc_heap_guard_size: u64,
+
+        /// The size, in bytes, to allocate at the end of a relocated GC heap
+        /// for growth.
+        ///
+        /// This is the same as `memory_reservation_for_growth` but for GC
+        /// heaps.
+        pub gc_heap_reservation_for_growth: u64,
+
+        /// Whether or not GC heaps are allowed to be reallocated after initial
+        /// allocation at runtime.
+        ///
+        /// This is the same as `memory_may_move` but for GC heaps.
+        pub gc_heap_may_move: bool,
     }
 
     pub struct ConfigTunables {
@@ -200,6 +223,7 @@ impl Tunables {
         if target.is_pulley() {
             ret.signals_based_traps = false;
             ret.memory_guard_size = 0;
+            ret.gc_heap_guard_size = 0;
         }
         Ok(ret)
     }
@@ -239,6 +263,10 @@ impl Tunables {
             concurrency_support: true,
             recording: false,
             gc_zeal_alloc_counter: None,
+            gc_heap_reservation: 0,
+            gc_heap_guard_size: 0,
+            gc_heap_reservation_for_growth: 0,
+            gc_heap_may_move: true,
         }
     }
 
@@ -253,6 +281,12 @@ impl Tunables {
             memory_reservation_for_growth: 1 << 20, // 1MB
             signals_based_traps: true,
 
+            // GC heaps on 32-bit: conservative defaults similar to linear
+            // memories.
+            gc_heap_reservation: 10 * (1 << 20),
+            gc_heap_guard_size: 0x1_0000,
+            gc_heap_reservation_for_growth: 1 << 20, // 1MB
+
             ..Tunables::default_miri()
         }
     }
@@ -278,6 +312,12 @@ impl Tunables {
             // to avoid memory movement.
             memory_reservation_for_growth: 2 << 30, // 2GB
 
+            // GC heaps on 64-bit: use 4GiB reservation and 32MiB guard pages
+            // to enable bounds check elision, matching linear memory defaults.
+            gc_heap_reservation: 1 << 32,
+            gc_heap_guard_size: 32 << 20,
+            gc_heap_reservation_for_growth: 2 << 30, // 2GB
+
             signals_based_traps: true,
             ..Tunables::default_miri()
         }
@@ -298,6 +338,73 @@ impl Tunables {
     }
 }
 
+/// Whether a heap is backing a linear memory or a GC heap.
+///
+/// This is used by [`MemoryTunables`] to select between the memory tunables and
+/// the GC heap tunables.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub enum MemoryKind {
+    /// A WebAssembly linear memory.
+    LinearMemory,
+    /// A GC heap for garbage-collected objects.
+    GcHeap,
+}
+
+/// A view into a [`Tunables`] that selects the appropriate linear-memory or
+/// GC-heap flavor of each tunable based on a [`MemoryKind`].
+pub struct MemoryTunables<'a> {
+    tunables: &'a Tunables,
+    kind: MemoryKind,
+}
+
+impl<'a> MemoryTunables<'a> {
+    /// Create a new `MemoryTunables` view.
+    pub fn new(tunables: &'a Tunables, kind: MemoryKind) -> Self {
+        Self { tunables, kind }
+    }
+
+    /// The virtual memory reservation for this kind of memory.
+    pub fn reservation(&self) -> u64 {
+        match self.kind {
+            MemoryKind::LinearMemory => self.tunables.memory_reservation,
+            MemoryKind::GcHeap => self.tunables.gc_heap_reservation,
+        }
+    }
+
+    /// The size of the guard page region for this kind of memory.
+    pub fn guard_size(&self) -> u64 {
+        match self.kind {
+            MemoryKind::LinearMemory => self.tunables.memory_guard_size,
+            MemoryKind::GcHeap => self.tunables.gc_heap_guard_size,
+        }
+    }
+
+    /// Extra virtual memory to reserve beyond the initially mapped pages for
+    /// this kind of memory.
+    pub fn reservation_for_growth(&self) -> u64 {
+        match self.kind {
+            MemoryKind::LinearMemory => self.tunables.memory_reservation_for_growth,
+            MemoryKind::GcHeap => self.tunables.gc_heap_reservation_for_growth,
+        }
+    }
+
+    /// Whether this kind of memory's base pointer may be relocated at runtime.
+    pub fn may_move(&self) -> bool {
+        match self.kind {
+            MemoryKind::LinearMemory => self.tunables.memory_may_move,
+            MemoryKind::GcHeap => self.tunables.gc_heap_may_move,
+        }
+    }
+
+    /// Get the underlying tunables.
+    ///
+    /// This is ONLY for accessing tunable fields that DO NOT come in a
+    /// linear-memory flavor and a GC-heap flavor.
+    pub fn tunables(&self) -> &'a Tunables {
+        self.tunables
+    }
+}
+
 /// The garbage collector implementation to use.
 #[derive(Clone, Copy, Hash, Serialize, Deserialize, Debug, PartialEq, Eq)]
 pub enum Collector {