Handle OOM in the rest of Wasmtime's non-component, -async, -compilation APIs (#12858)

* Handle OOM in more places in the public API

A bunch of random places:

* Add: `Trap::try_new` to handle OOM while creating traps
* Use: `TryVec` inside `Func::call_impl_do_call` and `wasm_val_raw_storage` to
  hold the args and rets
* Add: `Instance::try_exports` for iterating over an instance's exports while
  handling OOM
* `Linker:try_get`, like `Linker::get` but handling OOM
* `Linker:try_get_by_import`, like `Linker::get_by_import` but handling OOM
* Use `try_new` to box things in `SharedMemory::new`
* Use `TryVec` instead of `Vec` in our dynamic tables

* Add OOM tests for most of Wasmtime's public API

Excludes component-, async-, and compilation-related APIs.

* address review feedback

* fix test compilation

* fix c-api

Author: fitzgen

crates/c-api/src/linker.rs

@@ -204,11 +204,11 @@ pub unsafe extern "C" fn wasmtime_linker_get(
         Err(_) => return false,
     };
     match linker.get(store, module, name) {
-        Some(which) => {
+        Ok(which) => {
             crate::initialize(item_ptr, which.into());
             true
         }
-        None => false,
+        Err(_) => false,
     }
 }
 

crates/core/src/alloc/vec.rs

@@ -191,6 +191,20 @@ impl<T> TryVec<T> {
         Ok(())
     }
 
+    /// Same as [`std::vec::Vec::resize_with`] but returns an error on
+    /// allocation failure.
+    pub fn resize_with<F>(&mut self, new_len: usize, f: F) -> Result<(), OutOfMemory>
+    where
+        F: FnMut() -> T,
+    {
+        let len = self.len();
+        if new_len > len {
+            self.reserve(new_len - len)?;
+        }
+        self.inner.resize_with(new_len, f);
+        Ok(())
+    }
+
     /// Same as [`std::vec::Vec::retain`].
     pub fn retain<F>(&mut self, f: F)
     where

crates/fuzzing/tests/oom/caller.rs

@@ -0,0 +1,120 @@
+#![cfg(arc_try_new)]
+
+use wasmtime::{Config, Engine, Func, FuncType, Module, Result, Store};
+use wasmtime_fuzzing::oom::OomTest;
+
+#[test]
+fn caller_get_export() -> Result<()> {
+    let module_bytes = {
+        let mut config = Config::new();
+        config.concurrency_support(false);
+        let engine = Engine::new(&config)?;
+        Module::new(
+            &engine,
+            r#"(module
+                (import "" "f" (func))
+                (memory (export "m") 1)
+                (func (export "run") (call 0))
+            )"#,
+        )?
+        .serialize()?
+    };
+    let mut config = Config::new();
+    config.enable_compiler(false);
+    config.concurrency_support(false);
+    let engine = Engine::new(&config)?;
+    let module = unsafe { Module::deserialize(&engine, &module_bytes)? };
+
+    OomTest::new().test(|| {
+        let mut store = Store::try_new(&engine, ())?;
+        let host_func = Func::try_new(
+            &mut store,
+            FuncType::try_new(&engine, [], [])?,
+            |mut caller, _params, _results| {
+                let mem = caller.get_export("m");
+                assert!(mem.is_some());
+                Ok(())
+            },
+        )?;
+        let instance = wasmtime::Instance::new(&mut store, &module, &[host_func.into()])?;
+        let run = instance.get_typed_func::<(), ()>(&mut store, "run")?;
+        run.call(&mut store, ())?;
+        Ok(())
+    })
+}
+
+#[test]
+fn caller_data() -> Result<()> {
+    let module_bytes = {
+        let mut config = Config::new();
+        config.concurrency_support(false);
+        let engine = Engine::new(&config)?;
+        Module::new(
+            &engine,
+            r#"(module
+                (import "" "f" (func))
+                (func (export "run") (call 0))
+            )"#,
+        )?
+        .serialize()?
+    };
+    let mut config = Config::new();
+    config.enable_compiler(false);
+    config.concurrency_support(false);
+    let engine = Engine::new(&config)?;
+    let module = unsafe { Module::deserialize(&engine, &module_bytes)? };
+
+    OomTest::new().test(|| {
+        let mut store = Store::try_new(&engine, 42u32)?;
+        let host_func = Func::try_new(
+            &mut store,
+            FuncType::try_new(&engine, [], [])?,
+            |caller, _params, _results| {
+                assert_eq!(*caller.data(), 42u32);
+                Ok(())
+            },
+        )?;
+        let instance = wasmtime::Instance::new(&mut store, &module, &[host_func.into()])?;
+        let run = instance.get_typed_func::<(), ()>(&mut store, "run")?;
+        run.call(&mut store, ())?;
+        Ok(())
+    })
+}
+
+#[test]
+fn caller_engine() -> Result<()> {
+    let module_bytes = {
+        let mut config = Config::new();
+        config.concurrency_support(false);
+        let engine = Engine::new(&config)?;
+        Module::new(
+            &engine,
+            r#"(module
+                (import "" "f" (func))
+                (func (export "run") (call 0))
+            )"#,
+        )?
+        .serialize()?
+    };
+    let mut config = Config::new();
+    config.enable_compiler(false);
+    config.concurrency_support(false);
+    let engine = Engine::new(&config)?;
+    let module = unsafe { Module::deserialize(&engine, &module_bytes)? };
+
+    OomTest::new().test(|| {
+        let mut store = Store::try_new(&engine, ())?;
+        let host_func = Func::try_new(
+            &mut store,
+            FuncType::try_new(&engine, [], [])?,
+            |caller, _params, _results| {
+                let _engine = caller.engine();
+                Ok(())
+            },
+        )?;
+        let instance = wasmtime::Instance::new(&mut store, &module, &[host_func.into()])?;
+        let run = instance.get_typed_func::<(), ()>(&mut store, "run")?;
+        run.call(&mut store, ())?;
+        Ok(())
+    })
+}

crates/fuzzing/tests/oom/func.rs

@@ -1,6 +1,6 @@
 #![cfg(arc_try_new)]
 
-use wasmtime::{Config, Engine, Func, Result, Store};
+use wasmtime::{Config, Engine, Func, FuncType, Linker, Module, Result, Store, Val, ValType};
 use wasmtime_fuzzing::oom::OomTest;
 
 #[test]
@@ -16,3 +16,99 @@ fn func_new() -> Result<()> {
         Ok(())
     })
 }
+
+#[test]
+fn func_new_with_type() -> Result<()> {
+    let mut config = Config::new();
+    config.enable_compiler(false);
+    config.concurrency_support(false);
+    let engine = Engine::new(&config)?;
+
+    OomTest::new().test(|| {
+        let mut store = Store::try_new(&engine, ())?;
+        let ty = FuncType::try_new(&engine, [ValType::I32], [ValType::I32])?;
+        let _func = Func::try_new(&mut store, ty, |_caller, params, results| {
+            results[0] = params[0].clone();
+            Ok(())
+        })?;
+        Ok(())
+    })
+}
+
+#[test]
+fn func_call() -> Result<()> {
+    let module_bytes = {
+        let mut config = Config::new();
+        config.concurrency_support(false);
+        let engine = Engine::new(&config)?;
+        Module::new(
+            &engine,
+            r#"(module (func (export "id") (param i32) (result i32) (local.get 0)))"#,
+        )?
+        .serialize()?
+    };
+    let mut config = Config::new();
+    config.enable_compiler(false);
+    config.concurrency_support(false);
+    let engine = Engine::new(&config)?;
+    let module = unsafe { Module::deserialize(&engine, &module_bytes)? };
+    let linker = Linker::<()>::new(&engine);
+    let instance_pre = linker.instantiate_pre(&module)?;
+
+    OomTest::new().test(|| {
+        let mut store = Store::try_new(&engine, ())?;
+        let instance = instance_pre.instantiate(&mut store)?;
+        let id = instance.get_func(&mut store, "id").unwrap();
+        let mut results = [Val::I32(0)];
+        id.call(&mut store, &[Val::I32(42)], &mut results)?;
+        assert_eq!(results[0].unwrap_i32(), 42);
+        Ok(())
+    })
+}
+
+#[test]
+fn func_typed() -> Result<()> {
+    let module_bytes = {
+        let mut config = Config::new();
+        config.concurrency_support(false);
+        let engine = Engine::new(&config)?;
+        Module::new(
+            &engine,
+            r#"(module (func (export "id") (param i32) (result i32) (local.get 0)))"#,
+        )?
+        .serialize()?
+    };
+    let mut config = Config::new();
+    config.enable_compiler(false);
+    config.concurrency_support(false);
+    let engine = Engine::new(&config)?;
+    let module = unsafe { Module::deserialize(&engine, &module_bytes)? };
+    let linker = Linker::<()>::new(&engine);
+    let instance_pre = linker.instantiate_pre(&module)?;
+
+    OomTest::new().test(|| {
+        let mut store = Store::try_new(&engine, ())?;
+        let instance = instance_pre.instantiate(&mut store)?;
+        let id = instance.get_typed_func::<i32, i32>(&mut store, "id")?;
+        let result = id.call(&mut store, 42)?;
+        assert_eq!(result, 42);
+        Ok(())
+    })
+}
+
+#[test]
+fn func_ty() -> Result<()> {
+    let mut config = Config::new();
+    config.enable_compiler(false);
+    config.concurrency_support(false);
+    let engine = Engine::new(&config)?;
+
+    OomTest::new().test(|| {
+        let mut store = Store::try_new(&engine, ())?;
+        let func = Func::try_wrap(&mut store, |x: i32| x * 2)?;
+        let ty = func.ty(&store);
+        assert_eq!(ty.params().len(), 1);
+        assert_eq!(ty.results().len(), 1);
+        Ok(())
+    })
+}

crates/fuzzing/tests/oom/global.rs

@@ -1,6 +1,6 @@
 #![cfg(arc_try_new)]
 
-use wasmtime::{Config, Engine, GlobalType, Mutability, Result, Store, Val, ValType};
+use wasmtime::{Config, Engine, Global, GlobalType, Mutability, Result, Store, Val, ValType};
 use wasmtime_fuzzing::oom::OomTest;
 
 #[test]
@@ -17,3 +17,55 @@ fn global_new() -> Result<()> {
         Ok(())
     })
 }
+
+#[test]
+fn global_get() -> Result<()> {
+    let mut config = Config::new();
+    config.enable_compiler(false);
+    config.concurrency_support(false);
+    let engine = Engine::new(&config)?;
+
+    OomTest::new().test(|| {
+        let mut store = Store::try_new(&engine, ())?;
+        let ty = GlobalType::new(ValType::I32, Mutability::Var);
+        let global = Global::new(&mut store, ty, Val::I32(42))?;
+        let val = global.get(&mut store);
+        assert_eq!(val.unwrap_i32(), 42);
+        Ok(())
+    })
+}
+
+#[test]
+fn global_set() -> Result<()> {
+    let mut config = Config::new();
+    config.enable_compiler(false);
+    config.concurrency_support(false);
+    let engine = Engine::new(&config)?;
+
+    OomTest::new().test(|| {
+        let mut store = Store::try_new(&engine, ())?;
+        let ty = GlobalType::new(ValType::I32, Mutability::Var);
+        let global = Global::new(&mut store, ty, Val::I32(42))?;
+        global.set(&mut store, Val::I32(99))?;
+        let val = global.get(&mut store);
+        assert_eq!(val.unwrap_i32(), 99);
+        Ok(())
+    })
+}
+
+#[test]
+fn global_ty() -> Result<()> {
+    let mut config = Config::new();
+    config.enable_compiler(false);
+    config.concurrency_support(false);
+    let engine = Engine::new(&config)?;
+
+    OomTest::new().test(|| {
+        let mut store = Store::try_new(&engine, ())?;
+        let ty = GlobalType::new(ValType::I32, Mutability::Var);
+        let global = Global::new(&mut store, ty, Val::I32(42))?;
+        let ty = global.ty(&store);
+        assert!(ty.content().is_i32());
+        Ok(())
+    })
+}