Handle OOM in Module::deserialize (#12673)

Part of #12069

Author: fitzgen

crates/fuzzing/tests/oom.rs

@@ -508,6 +508,53 @@ fn linker_func_wrap() -> Result<()> {
     })
 }
 
+#[test]
+#[cfg(arc_try_new)]
+fn module_deserialize() -> Result<()> {
+    let module_bytes = {
+        let mut config = Config::new();
+        config.concurrency_support(false);
+        let engine = Engine::new(&config)?;
+        let module = Module::new(
+            &engine,
+            r#"
+                (module
+                    (import "module" "func" (func (param i32) (result i32)))
+
+                    (memory (export "memory") 1)
+                    (data (i32.const 0) "a")
+
+                    (table (export "table") 1 funcref)
+                    (elem (i32.const 0) func 1)
+
+                    (func (export "func") (param i32) (result i32)
+                        (call 0 (local.get 0))
+                    )
+                )
+            "#,
+        )?;
+        module.serialize()?
+    };
+
+    let mut config = Config::new();
+    config.enable_compiler(false);
+    config.concurrency_support(false);
+    let engine = Engine::new(&config)?;
+
+    OomTest::new()
+        // NB: We use `postcard` to deserialize module metadata, and it will
+        // return a `postcard::Error::SerdeDeCustom` when we generate an
+        // `OutOfMemory` error during deserialization. That is then converted
+        // into a `wasmtime::Error`, and in the process we will attempt to box
+        // that into an `Error` trait object. There is no good way to avoid all
+        // this, so just allow allocation attempts after OOM here.
+        .allow_alloc_after_oom(true)
+        .test(|| unsafe {
+            let _ = Module::deserialize(&engine, &module_bytes)?;
+            Ok(())
+        })
+}
+
 #[test]
 #[cfg(arc_try_new)]
 fn store_try_new() -> Result<()> {

crates/wasmtime/src/runtime/instantiate.rs

@@ -55,7 +55,7 @@ impl CompiledModule {
         let mut ret = Self {
             unique_id: CompiledModuleId::new(),
             engine_code,
-            module: Arc::new(info.module),
+            module: try_new::<Arc<_>>(info.module)?,
             meta: info.meta,
             index,
             func_names: info.func_names,

crates/wasmtime/src/runtime/module.rs

@@ -523,8 +523,8 @@ impl Module {
 
         // Package up all our data into an `EngineCode` and delegate to the final
         // step of module compilation.
-        let code = Arc::new(EngineCode::new(code_memory, signatures, types.into()));
-        let index = Arc::new(index);
+        let code = try_new::<Arc<_>>(EngineCode::new(code_memory, signatures, types.into()))?;
+        let index = try_new::<Arc<_>>(index)?;
         Module::from_parts_raw(engine, code, info, index, true)
     }
 
@@ -547,7 +547,7 @@ impl Module {
         let _ = serializable;
 
         Ok(Self {
-            inner: Arc::new(ModuleInner {
+            inner: try_new::<Arc<_>>(ModuleInner {
                 engine: engine.clone(),
                 code,
                 memory_images: OnceLock::new(),
@@ -556,7 +556,7 @@ impl Module {
                 serializable,
                 offsets,
                 checksum,
-            }),
+            })?,
         })
     }