From 51ec73eb18e56ecd9f28c163b5bde325d8fcc89b Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Tue, 3 Nov 2026 16:37:04 -0500 Subject: [PATCH 1/2] proxy: Rethrow EventLoop::sync() callback exceptions in the calling thread When a thread other than the event loop thread calls EventLoop::sync() and the callback throws, sync() now rethrows the exception in the calling thread, the same as when the callback runs inline on the event loop thread. Before, EventLoop::loop() logged "EventLoop: m_sync_fn threw" and sync() returned as if the callback had succeeded, so callers continued with half-finished setup: - clientInvoke waited forever for a response if building the request threw, because the request was never sent. - MakeStream returned a null stream and ListenConnections returned as if it were listening if wrapping the socket threw, and ConnectStream built a client on a null Connection if creating the Connection threw. - An async server method ran without its cancellation callback if installing the callback threw. The log line is dropped because the exception now reaches the caller, which reports or handles it the way it handles any other exception. The async server method setup in PassField catches the exception inside its sync() callback and rethrows it after the KJ_DEFER cleanup is in place, instead of letting sync() rethrow it right away, so the cancellation callback and the self reference are still cleared on the event loop thread. First reported in https://ipcbugs.ofsky.org/audits/ar-14/ (request build hang), https://ipcbugs.ofsky.org/audits/ar-57/ (socket setup) and https://ipcbugs.ofsky.org/audits/ar-60/ (cancellation callback). This change was written with Claude Opus 5.5 (1M context). --- include/mp/type-context.h | 18 ++++++++++- src/mp/proxy.cpp | 27 +++++++++++----- test/mp/test/test.cpp | 67 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 9 deletions(-) diff --git a/include/mp/type-context.h b/include/mp/type-context.h index 7cfc7e79..9e5af5dc 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -10,6 +10,8 @@ #include +#include + namespace mp { template requires FieldTypeIs @@ -100,7 +102,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& Mutex cancel_mutex; Lock cancel_lock{cancel_mutex}; server_context.cancel_lock = &cancel_lock; - loop.sync([&] { + auto setup = [&] { // Detect request being canceled before it executes. if (cancel_monitor.m_canceled) { server_context.request_canceled = true; @@ -108,6 +110,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& } // Detect request being canceled while it executes. assert(!cancel_monitor.m_on_cancel); + if (loop.testing_hook_misc) loop.testing_hook_misc("set cancel callback"); cancel_monitor.m_on_cancel = [&loop, &server_context, &cancel_mutex, req]() { MP_LOG(loop, Log::Info) << "IPC server request #" << req << " canceled while executing."; // Lock cancel_mutex here to block the event loop @@ -143,6 +146,18 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // future change accessing other call_context state needs to move // that access to the event loop thread the same way. call_context.getResults(); + }; + // If setup() throws, catch the exception here instead of letting + // sync() rethrow it right away, and rethrow it below after the KJ_DEFER + // cleanup is in place, so the cancellation callback is cleared and the + // self reference is released on the event loop thread. + std::exception_ptr setup_exception; + loop.sync([&] { + try { + setup(); + } catch (...) { + setup_exception = std::current_exception(); + } }); // If an entry was inserted into the request_threads map, @@ -191,6 +206,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& } }); ); + if (setup_exception) std::rethrow_exception(setup_exception); if (server_context.request_canceled) { MP_LOG(loop, Log::Info) << "IPC server request #" << req << " canceled before it could be executed"; } else KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]{ diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 552e3268..61e284dc 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -305,12 +306,9 @@ void EventLoop::loop() if (read_bytes != 1) throw std::logic_error("EventLoop wait_stream closed unexpectedly"); Lock lock(m_mutex); if (m_sync_fn) { - // m_sync_fn throwing is never expected. If it does happen, the caller - // of EventLoop::sync() will return without any indication of failure, - // which will likely cause other bugs. Log the error and continue. - KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() MP_REQUIRES(m_mutex) { Unlock(lock, *m_sync_fn); })) { - MP_LOG(*this, Log::Error) << "EventLoop: m_sync_fn threw: " << kj::str(*exception).cStr(); - } + // m_sync_fn does not throw, because sync() catches exceptions from + // its callback and rethrows them in the calling thread. + Unlock(lock, *m_sync_fn); m_sync_fn = nullptr; m_cv.notify_all(); } else if (done()) { @@ -339,15 +337,28 @@ void EventLoop::sync(kj::FunctionParam fn) fn(); return; } + // Catch any exception fn() throws on the event loop thread and rethrow it + // below, so callers see the same exception they would see if fn() were + // called directly. + std::exception_ptr exception; + auto call_fn = [&] { + try { + fn(); + } catch (...) { + exception = std::current_exception(); + } + }; + kj::FunctionParam sync_fn{call_fn}; Lock lock(m_mutex); EventLoopRef ref(*this, &lock); m_cv.wait(lock.m_lock, [this]() MP_REQUIRES(m_mutex) { return m_sync_fn == nullptr; }); - m_sync_fn = &fn; + m_sync_fn = &sync_fn; Unlock(lock, [&] { char buffer = 0; m_post_writer->write(&buffer, 1); }); - m_cv.wait(lock.m_lock, [this, &fn]() MP_REQUIRES(m_mutex) { return m_sync_fn != &fn; }); + m_cv.wait(lock.m_lock, [this, &sync_fn]() MP_REQUIRES(m_mutex) { return m_sync_fn != &sync_fn; }); + if (exception) std::rethrow_exception(exception); } void EventLoop::startAsyncThread() diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 7d35cf1e..82e16b47 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -770,5 +770,72 @@ KJ_TEST("Call async IPC method without thread or pool errors correctly") KJ_EXPECT(error_thrown); } +//! Set testing_hook_misc to call fn when the hook is called with the given tag. +static void SetMiscHook(EventLoop& loop, std::string_view tag, std::function fn) +{ + loop.sync([&] { + loop.testing_hook_misc = [tag, fn = std::move(fn)](std::any arg) { + if (const char* const* arg_tag{std::any_cast(&arg)}; arg_tag && *arg_tag == tag) fn(); + }; + }); +} + +KJ_TEST("EventLoop::sync rethrows callback exceptions in the calling thread") +{ + TestSetup setup; + EventLoop& loop{*setup.client->m_context.loop}; + EXPECT_EXCEPTION(loop.sync([] { throw std::runtime_error("sync callback failed"); }), "sync callback failed"); + // The event loop keeps running after the exception. + KJ_EXPECT(setup.client->add(1, 2) == 3); +} + +KJ_TEST("Calling IPC method when building the request throws") +{ + // clientInvoke builds the request inside EventLoop::sync. Before sync() + // rethrew exceptions, an exception there was logged and dropped, and the + // calling thread waited forever for a response to a request that was + // never sent. + TestSetup setup; + ProxyClient* foo = setup.client.get(); + EventLoop& loop{*foo->m_context.loop}; + SetMiscHook(loop, "build FooMessage", [] { throw std::runtime_error("build FooMessage failed"); }); + FooMessage message; + message.message = "init"; + EXPECT_EXCEPTION(foo->passMessage(message), "build FooMessage failed"); + + // The same client can make calls after the failure. + SetMiscHook(loop, "", [] {}); + KJ_EXPECT(foo->passMessage(message).message == "init build read call build read"); +} + +KJ_TEST("Calling async IPC method when setting the cancel callback throws") +{ + // The server installs the cancellation callback for an async method + // inside EventLoop::sync. Before sync() rethrew exceptions, a failure + // there was logged and dropped, and the method ran without the + // cancellation callback. + TestSetup setup; + ProxyClient* foo = setup.client.get(); + foo->initThreadMap(); + bool called{false}; + setup.server->m_impl->m_fn = [&] { called = true; }; + EventLoop& loop{*foo->m_context.loop}; + SetMiscHook(loop, "set cancel callback", [] { throw std::runtime_error("set cancel callback failed"); }); + bool threw{false}; + try { + foo->callFnAsync(); + } catch (const std::runtime_error& e) { + threw = true; + KJ_EXPECT(std::string_view{e.what()}.find("set cancel callback failed") != std::string_view::npos, e.what()); + } + KJ_EXPECT(threw); + KJ_EXPECT(!called); + + // The same client can make calls after the failure. + SetMiscHook(loop, "", [] {}); + foo->callFnAsync(); + KJ_EXPECT(called); +} + } // namespace test } // namespace mp From b7283cc8cff8badf8c4f001719d91788deeebf0c Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Tue, 3 Nov 2026 16:37:18 -0500 Subject: [PATCH 2/2] proxy: Close the socket when allocating its KJ wrapper fails MakeStream and ListenConnections now close the socket if allocating the KJ object that takes ownership of it throws std::bad_alloc. Before, the socket leaked, because nothing owned it until the KJ object was constructed. Other failures are left to KJ: once its wrapper object starts constructing, it owns the socket and closes it if a later step fails, so closing it here as well could close an unrelated descriptor (see the WrapSocket comment). First reported in https://ipcbugs.ofsky.org/audits/ar-57/. This change was written with Claude Opus 5.5 (1M context). --- include/mp/proxy-io.h | 28 ++++++++++++++++++++++++++- src/mp/proxy.cpp | 6 +++++- test/mp/test/test.cpp | 45 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index b226284f..ca2e0ea5 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -218,6 +219,27 @@ std::string LongThreadName(const char* exe_name); //! Wrap a socket file descriptor as an async stream, taking ownership of the fd. Stream MakeStream(EventLoop&loop, SocketId socket); +//! Call a KJ function that wraps a socket and takes ownership of it, and close +//! the socket if the function fails to allocate the wrapper object. +//! +//! The socket is closed only on std::bad_alloc, because KJ wrapper objects +//! take ownership of the socket in their first base class constructor, before +//! they register the socket with epoll or kqueue. If registration fails, the +//! KJ object closes the socket itself while it is being destroyed, and +//! closing it again here could close an unrelated descriptor reusing the same +//! number. +template +auto WrapSocket(SocketId socket, Fn&& fn) -> decltype(fn()) +{ + try { + return fn(); + } catch (const std::bad_alloc&) { + // No KJ object owns the socket, so close it while rethrowing. + kj::AutoCloseFd owner{socket}; + throw; + } +} + //! Event loop implementation. //! //! Cap'n Proto threading model is very simple: all I/O operations are @@ -940,8 +962,12 @@ template void ListenConnections(EventLoop& loop, SocketId fd, InitImpl& init, std::optional max_connections = std::nullopt) { loop.sync([&]() { + auto receiver{WrapSocket(fd, [&] { + if (loop.testing_hook_misc) loop.testing_hook_misc("wrap listen socket"); + return loop.m_io_context.lowLevelProvider->wrapListenSocketFd(fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP); + })}; auto listener{std::make_shared( - loop.m_io_context.lowLevelProvider->wrapListenSocketFd(fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP), + kj::mv(receiver), max_connections)}; _Listen(listener, loop, init); }); diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 61e284dc..b9c54a0c 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include // IWYU pragma: keep @@ -52,7 +53,10 @@ Stream MakeStream(EventLoop&loop, SocketId socket) { Stream stream; loop.sync([&]() { - stream = loop.m_io_context.lowLevelProvider->wrapSocketFd(socket, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP); + stream = WrapSocket(socket, [&] { + if (loop.testing_hook_misc) loop.testing_hook_misc("wrap socket"); + return loop.m_io_context.lowLevelProvider->wrapSocketFd(socket, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP); + }); }); return stream; } diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 82e16b47..8b20b780 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -6,14 +6,17 @@ #include #include +#include #include #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -26,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -780,6 +784,13 @@ static void SetMiscHook(EventLoop& loop, std::string_view tag, std::functionpassMessage(message).message == "init build read call build read"); } +KJ_TEST("Socket is closed and the error is reported when wrapping it fails") +{ + // MakeStream and ListenConnections wrap the socket inside EventLoop::sync. + // Before sync() rethrew exceptions, a failure there was logged and + // dropped, so MakeStream returned a null stream and ListenConnections + // returned as if it were listening. Also, if allocating the KJ wrapper + // object failed, nothing closed the socket. + TestSetup setup; + EventLoop& loop{*setup.client->m_context.loop}; + auto sockets{SocketPair()}; + + SetMiscHook(loop, "wrap socket", [] { throw std::bad_alloc{}; }); + bool stream_threw{false}; + try { + (void)MakeStream(loop, sockets[0]); + } catch (const std::bad_alloc&) { + stream_threw = true; + } + KJ_EXPECT(stream_threw); + KJ_EXPECT(IsClosed(sockets[0])); + + SetMiscHook(loop, "wrap listen socket", [] { throw std::bad_alloc{}; }); + FooImplementation foo_impl; + bool listen_threw{false}; + try { + ListenConnections(loop, sockets[1], foo_impl); + } catch (const std::bad_alloc&) { + listen_threw = true; + } + KJ_EXPECT(listen_threw); + KJ_EXPECT(IsClosed(sockets[1])); + SetMiscHook(loop, "", [] {}); +} + KJ_TEST("Calling async IPC method when setting the cancel callback throws") { // The server installs the cancellation callback for an async method