diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index b226284f..ca2e0ea5 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -218,6 +219,27 @@ std::string LongThreadName(const char* exe_name); //! Wrap a socket file descriptor as an async stream, taking ownership of the fd. Stream MakeStream(EventLoop&loop, SocketId socket); +//! Call a KJ function that wraps a socket and takes ownership of it, and close +//! the socket if the function fails to allocate the wrapper object. +//! +//! The socket is closed only on std::bad_alloc, because KJ wrapper objects +//! take ownership of the socket in their first base class constructor, before +//! they register the socket with epoll or kqueue. If registration fails, the +//! KJ object closes the socket itself while it is being destroyed, and +//! closing it again here could close an unrelated descriptor reusing the same +//! number. +template +auto WrapSocket(SocketId socket, Fn&& fn) -> decltype(fn()) +{ + try { + return fn(); + } catch (const std::bad_alloc&) { + // No KJ object owns the socket, so close it while rethrowing. + kj::AutoCloseFd owner{socket}; + throw; + } +} + //! Event loop implementation. //! //! Cap'n Proto threading model is very simple: all I/O operations are @@ -940,8 +962,12 @@ template void ListenConnections(EventLoop& loop, SocketId fd, InitImpl& init, std::optional max_connections = std::nullopt) { loop.sync([&]() { + auto receiver{WrapSocket(fd, [&] { + if (loop.testing_hook_misc) loop.testing_hook_misc("wrap listen socket"); + return loop.m_io_context.lowLevelProvider->wrapListenSocketFd(fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP); + })}; auto listener{std::make_shared( - loop.m_io_context.lowLevelProvider->wrapListenSocketFd(fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP), + kj::mv(receiver), max_connections)}; _Listen(listener, loop, init); }); diff --git a/include/mp/type-context.h b/include/mp/type-context.h index 7cfc7e79..9e5af5dc 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -10,6 +10,8 @@ #include +#include + namespace mp { template requires FieldTypeIs @@ -100,7 +102,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& Mutex cancel_mutex; Lock cancel_lock{cancel_mutex}; server_context.cancel_lock = &cancel_lock; - loop.sync([&] { + auto setup = [&] { // Detect request being canceled before it executes. if (cancel_monitor.m_canceled) { server_context.request_canceled = true; @@ -108,6 +110,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& } // Detect request being canceled while it executes. assert(!cancel_monitor.m_on_cancel); + if (loop.testing_hook_misc) loop.testing_hook_misc("set cancel callback"); cancel_monitor.m_on_cancel = [&loop, &server_context, &cancel_mutex, req]() { MP_LOG(loop, Log::Info) << "IPC server request #" << req << " canceled while executing."; // Lock cancel_mutex here to block the event loop @@ -143,6 +146,18 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // future change accessing other call_context state needs to move // that access to the event loop thread the same way. call_context.getResults(); + }; + // If setup() throws, catch the exception here instead of letting + // sync() rethrow it right away, and rethrow it below after the KJ_DEFER + // cleanup is in place, so the cancellation callback is cleared and the + // self reference is released on the event loop thread. + std::exception_ptr setup_exception; + loop.sync([&] { + try { + setup(); + } catch (...) { + setup_exception = std::current_exception(); + } }); // If an entry was inserted into the request_threads map, @@ -191,6 +206,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& } }); ); + if (setup_exception) std::rethrow_exception(setup_exception); if (server_context.request_canceled) { MP_LOG(loop, Log::Info) << "IPC server request #" << req << " canceled before it could be executed"; } else KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]{ diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 552e3268..b9c54a0c 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include // IWYU pragma: keep @@ -28,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -51,7 +53,10 @@ Stream MakeStream(EventLoop&loop, SocketId socket) { Stream stream; loop.sync([&]() { - stream = loop.m_io_context.lowLevelProvider->wrapSocketFd(socket, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP); + stream = WrapSocket(socket, [&] { + if (loop.testing_hook_misc) loop.testing_hook_misc("wrap socket"); + return loop.m_io_context.lowLevelProvider->wrapSocketFd(socket, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP); + }); }); return stream; } @@ -305,12 +310,9 @@ void EventLoop::loop() if (read_bytes != 1) throw std::logic_error("EventLoop wait_stream closed unexpectedly"); Lock lock(m_mutex); if (m_sync_fn) { - // m_sync_fn throwing is never expected. If it does happen, the caller - // of EventLoop::sync() will return without any indication of failure, - // which will likely cause other bugs. Log the error and continue. - KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() MP_REQUIRES(m_mutex) { Unlock(lock, *m_sync_fn); })) { - MP_LOG(*this, Log::Error) << "EventLoop: m_sync_fn threw: " << kj::str(*exception).cStr(); - } + // m_sync_fn does not throw, because sync() catches exceptions from + // its callback and rethrows them in the calling thread. + Unlock(lock, *m_sync_fn); m_sync_fn = nullptr; m_cv.notify_all(); } else if (done()) { @@ -339,15 +341,28 @@ void EventLoop::sync(kj::FunctionParam fn) fn(); return; } + // Catch any exception fn() throws on the event loop thread and rethrow it + // below, so callers see the same exception they would see if fn() were + // called directly. + std::exception_ptr exception; + auto call_fn = [&] { + try { + fn(); + } catch (...) { + exception = std::current_exception(); + } + }; + kj::FunctionParam sync_fn{call_fn}; Lock lock(m_mutex); EventLoopRef ref(*this, &lock); m_cv.wait(lock.m_lock, [this]() MP_REQUIRES(m_mutex) { return m_sync_fn == nullptr; }); - m_sync_fn = &fn; + m_sync_fn = &sync_fn; Unlock(lock, [&] { char buffer = 0; m_post_writer->write(&buffer, 1); }); - m_cv.wait(lock.m_lock, [this, &fn]() MP_REQUIRES(m_mutex) { return m_sync_fn != &fn; }); + m_cv.wait(lock.m_lock, [this, &sync_fn]() MP_REQUIRES(m_mutex) { return m_sync_fn != &sync_fn; }); + if (exception) std::rethrow_exception(exception); } void EventLoop::startAsyncThread() diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 7d35cf1e..8b20b780 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -6,14 +6,17 @@ #include #include +#include #include #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -26,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -770,5 +774,113 @@ KJ_TEST("Call async IPC method without thread or pool errors correctly") KJ_EXPECT(error_thrown); } +//! Set testing_hook_misc to call fn when the hook is called with the given tag. +static void SetMiscHook(EventLoop& loop, std::string_view tag, std::function fn) +{ + loop.sync([&] { + loop.testing_hook_misc = [tag, fn = std::move(fn)](std::any arg) { + if (const char* const* arg_tag{std::any_cast(&arg)}; arg_tag && *arg_tag == tag) fn(); + }; + }); +} + +//! Return whether the file descriptor is closed. +static bool IsClosed(SocketId fd) +{ + errno = 0; + return fcntl(fd, F_GETFD) == -1 && errno == EBADF; +} + +KJ_TEST("EventLoop::sync rethrows callback exceptions in the calling thread") +{ + TestSetup setup; + EventLoop& loop{*setup.client->m_context.loop}; + EXPECT_EXCEPTION(loop.sync([] { throw std::runtime_error("sync callback failed"); }), "sync callback failed"); + // The event loop keeps running after the exception. + KJ_EXPECT(setup.client->add(1, 2) == 3); +} + +KJ_TEST("Calling IPC method when building the request throws") +{ + // clientInvoke builds the request inside EventLoop::sync. Before sync() + // rethrew exceptions, an exception there was logged and dropped, and the + // calling thread waited forever for a response to a request that was + // never sent. + TestSetup setup; + ProxyClient* foo = setup.client.get(); + EventLoop& loop{*foo->m_context.loop}; + SetMiscHook(loop, "build FooMessage", [] { throw std::runtime_error("build FooMessage failed"); }); + FooMessage message; + message.message = "init"; + EXPECT_EXCEPTION(foo->passMessage(message), "build FooMessage failed"); + + // The same client can make calls after the failure. + SetMiscHook(loop, "", [] {}); + KJ_EXPECT(foo->passMessage(message).message == "init build read call build read"); +} + +KJ_TEST("Socket is closed and the error is reported when wrapping it fails") +{ + // MakeStream and ListenConnections wrap the socket inside EventLoop::sync. + // Before sync() rethrew exceptions, a failure there was logged and + // dropped, so MakeStream returned a null stream and ListenConnections + // returned as if it were listening. Also, if allocating the KJ wrapper + // object failed, nothing closed the socket. + TestSetup setup; + EventLoop& loop{*setup.client->m_context.loop}; + auto sockets{SocketPair()}; + + SetMiscHook(loop, "wrap socket", [] { throw std::bad_alloc{}; }); + bool stream_threw{false}; + try { + (void)MakeStream(loop, sockets[0]); + } catch (const std::bad_alloc&) { + stream_threw = true; + } + KJ_EXPECT(stream_threw); + KJ_EXPECT(IsClosed(sockets[0])); + + SetMiscHook(loop, "wrap listen socket", [] { throw std::bad_alloc{}; }); + FooImplementation foo_impl; + bool listen_threw{false}; + try { + ListenConnections(loop, sockets[1], foo_impl); + } catch (const std::bad_alloc&) { + listen_threw = true; + } + KJ_EXPECT(listen_threw); + KJ_EXPECT(IsClosed(sockets[1])); + SetMiscHook(loop, "", [] {}); +} + +KJ_TEST("Calling async IPC method when setting the cancel callback throws") +{ + // The server installs the cancellation callback for an async method + // inside EventLoop::sync. Before sync() rethrew exceptions, a failure + // there was logged and dropped, and the method ran without the + // cancellation callback. + TestSetup setup; + ProxyClient* foo = setup.client.get(); + foo->initThreadMap(); + bool called{false}; + setup.server->m_impl->m_fn = [&] { called = true; }; + EventLoop& loop{*foo->m_context.loop}; + SetMiscHook(loop, "set cancel callback", [] { throw std::runtime_error("set cancel callback failed"); }); + bool threw{false}; + try { + foo->callFnAsync(); + } catch (const std::runtime_error& e) { + threw = true; + KJ_EXPECT(std::string_view{e.what()}.find("set cancel callback failed") != std::string_view::npos, e.what()); + } + KJ_EXPECT(threw); + KJ_EXPECT(!called); + + // The same client can make calls after the failure. + SetMiscHook(loop, "", [] {}); + foo->callFnAsync(); + KJ_EXPECT(called); +} + } // namespace test } // namespace mp