Clear trap pointer on non-trapping instantiation (#13035)

* Clear trap pointer on non-trapping instantiation

* Fix C API instance test formatting

Author: ShoosounS47

crates/c-api/src/instance.rs

@@ -4,6 +4,7 @@ use crate::{
     wasmtime_module_t,
 };
 use std::mem::MaybeUninit;
+use std::ptr;
 use wasmtime::{Instance, InstancePre, Trap};
 
 #[derive(Clone)]
@@ -94,6 +95,7 @@ pub(crate) fn handle_instantiate(
     instance_ptr: &mut Instance,
     trap_ptr: &mut *mut wasm_trap_t,
 ) -> Option<Box<wasmtime_error_t>> {
+    *trap_ptr = ptr::null_mut();
     match instance {
         Ok(i) => {
             *instance_ptr = i;

crates/c-api/tests/instance.cc

@@ -45,3 +45,26 @@ TEST(Instance, Smoke) {
   auto [name, func] = *i.get(store, 0);
   EXPECT_EQ(name, "f");
 }
+
+TEST(Instance, NewClearsTrapPointer) {
+  Engine engine;
+  Store store(engine);
+  auto context = store.context();
+
+  auto ok_module = Module::compile(engine, "(module)").unwrap();
+  wasmtime_instance_t instance;
+  wasm_trap_t *trap = reinterpret_cast<wasm_trap_t *>(1);
+  auto *error = wasmtime_instance_new(context.capi(), ok_module.capi(), nullptr,
+                                      0, &instance, &trap);
+  EXPECT_EQ(error, nullptr);
+  EXPECT_EQ(trap, nullptr);
+
+  auto import_module =
+      Module::compile(engine, "(module (import \"\" \"\" (func)))").unwrap();
+  trap = reinterpret_cast<wasm_trap_t *>(1);
+  error = wasmtime_instance_new(context.capi(), import_module.capi(), nullptr,
+                                0, &instance, &trap);
+  EXPECT_NE(error, nullptr);
+  EXPECT_EQ(trap, nullptr);
+  wasmtime_error_delete(error);
+}