From b114f21b5665b90ebea5e2ca711b91f6732c2127 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Wed, 7 Oct 2026 15:04:27 +0200 Subject: [PATCH 1/7] test: share setup for no-op async calls Add TestSetup::initAsyncCalls() to exchange thread maps and install a no-op callFnAsync() handler. Use it in the existing worker-lifetime, disconnect, and simultaneous-call tests without changing their hooks, thread synchronization, or assertions. This prepares reusable setup for the client-exit and worker-exit regression tests in subsequent commits. --- test/mp/test/test.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 0492c9ee..43d32fb2 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -138,6 +138,13 @@ class TestSetup client = client_promise.get_future().get(); } + //! Exchange thread maps and configure a no-op callFnAsync() handler. + void initAsyncCalls() + { + client->initThreadMap(); + server->m_impl->m_fn = [] {}; + } + ~TestSetup() { // Test that client cleanup_fns are executed. @@ -480,8 +487,7 @@ KJ_TEST("Worker thread destroyed before it is initialized") // the worker thread started waiting, causing a SIGSEGV when it did start. TestSetup setup; ProxyClient* foo = setup.client.get(); - foo->initThreadMap(); - setup.server->m_impl->m_fn = [] {}; + setup.initAsyncCalls(); EventLoop& loop = *setup.server->m_context.connection->m_loop; loop.testing_hook_makethread = [&] { @@ -510,8 +516,7 @@ KJ_TEST("Calling async IPC method, with server disconnect racing the call") // calling call_context.getParams(). TestSetup setup; ProxyClient* foo = setup.client.get(); - foo->initThreadMap(); - setup.server->m_impl->m_fn = [] {}; + setup.initAsyncCalls(); EventLoop& loop = *setup.server->m_context.connection->m_loop; loop.testing_hook_async_request_start = [&] { @@ -537,8 +542,7 @@ KJ_TEST("Calling async IPC method, with server disconnect after cleanup") // scope. TestSetup setup; ProxyClient* foo = setup.client.get(); - foo->initThreadMap(); - setup.server->m_impl->m_fn = [] {}; + setup.initAsyncCalls(); EventLoop& loop = *setup.server->m_context.connection->m_loop; loop.testing_hook_async_request_done = [&] { @@ -580,10 +584,9 @@ KJ_TEST("Make simultaneous IPC calls on single remote thread") ProxyClient* foo = setup.client.get(); std::promise signal; - foo->initThreadMap(); + setup.initAsyncCalls(); // Use callFnAsync() to get the client to set up the request_thread // that will be used for the test. - setup.server->m_impl->m_fn = [&] {}; foo->callFnAsync(); ThreadContext& tc{CurrentThread()}; Thread::Client *callback_thread, *request_thread; From ea93dab8d18bd8c3e9ca0b02b0ac2372b8bed0a2 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 28 Sep 2026 18:03:47 +0200 Subject: [PATCH 2/7] proxy: allow move-only async cleanup callbacks Introduce AsyncCleanupList to limit the changes to what the next commit needs, without migrating the other CleanupList users. --- include/mp/proxy-io.h | 4 ++-- include/mp/proxy.h | 2 ++ src/mp/gen.cpp | 1 + src/mp/proxy.cpp | 4 ++-- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index b226284f..9b7f0feb 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -272,7 +272,7 @@ class EventLoop //! Register cleanup function to run on asynchronous worker thread without //! blocking the event loop thread. - void addAsyncCleanup(std::function fn); + void addAsyncCleanup(kj::Function fn); //! Start asynchronous worker thread if necessary. This is only done if //! there are ProxyServerBase::m_impl objects that need to be destroyed @@ -305,7 +305,7 @@ class EventLoop kj::FunctionParam* m_sync_fn MP_GUARDED_BY(m_mutex) = nullptr; //! Callback functions to run on async thread. - std::optional m_async_fns MP_GUARDED_BY(m_mutex); + std::optional m_async_fns MP_GUARDED_BY(m_mutex); //! Socket pair used to post and wait for wakeups to the event loop thread. kj::Own m_wait_stream; diff --git a/include/mp/proxy.h b/include/mp/proxy.h index 2144b571..ca5a77c9 100644 --- a/include/mp/proxy.h +++ b/include/mp/proxy.h @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -33,6 +34,7 @@ template struct ProxyStruct; // IWYU pragma: export //! Mapping from local c++ type to capnp type and traits (specializations are generated by proxy-codegen.cpp). template struct ProxyType; // IWYU pragma: export +using AsyncCleanupList = std::list>; using CleanupList = std::list>; using CleanupIt = typename CleanupList::iterator; diff --git a/src/mp/gen.cpp b/src/mp/gen.cpp index 8d733a91..c955f319 100644 --- a/src/mp/gen.cpp +++ b/src/mp/gen.cpp @@ -344,6 +344,7 @@ static void Generate(kj::StringPtr src_prefix, cpp_types << "#include <" << include_path << ".h> // IWYU pragma: keep\n"; cpp_types << "#include <" << include_path << ".proxy.h>\n"; cpp_types << "#include <" << include_path << ".proxy-types.h> // IWYU pragma: keep\n"; + cpp_types << "#include \n"; cpp_types << "#include <" << PROXY_TYPES << ">\n\n"; cpp_types << "namespace mp {\n"; diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 552e3268..a9181b48 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -231,7 +231,7 @@ void Connection::removeSyncCleanup(CleanupIt it) m_sync_cleanup_fns.erase(it); } -void EventLoop::addAsyncCleanup(std::function fn) +void EventLoop::addAsyncCleanup(kj::Function fn) { const Lock lock(m_mutex); // Add async cleanup callbacks to the back of the list. Unlike the sync @@ -363,7 +363,7 @@ void EventLoop::startAsyncThread() while (m_async_fns) { if (!m_async_fns->empty()) { EventLoopRef ref{*this, &lock}; - const std::function fn = std::move(m_async_fns->front()); + kj::Function fn = std::move(m_async_fns->front()); m_async_fns->pop_front(); Unlock(lock, fn); // Important to relock because of the wait() call below. From 0b12fe7cafe8c23b100ccacd143e1ef29549f3fc Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Wed, 7 Oct 2026 15:39:14 +0200 Subject: [PATCH 3/7] proxy: centralize explicit thread-client release Group the per-connection client maps in ThreadClients with a mutex independent of the thread waiter. Update map users and the existing thread-mapping test to use this state, still stored in ThreadContext. Add ThreadClients::clear() to pin each event loop, extract its clients under the map mutex, and destroy them on that loop after unlocking. Use it during worker teardown so explicit release does not hold the map mutex while waiting for another loop or destroying client objects. Thread-exit cleanup remains synchronous in this preparatory change. --- include/mp/proxy-io.h | 77 +++++++++++++++++++-------------------- include/mp/type-context.h | 12 +++--- src/mp/proxy.cpp | 40 +++++++++++++++++++- test/mp/test/test.cpp | 6 +-- 4 files changed, 85 insertions(+), 50 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index 9b7f0feb..80b8547f 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -91,8 +91,8 @@ struct ProxyClient : public ProxyClientBase //! map. It will also reset m_disconnect_cb so the destructor does not //! access it. In the normal case where there is no sudden disconnect, the //! destructor will unregister m_disconnect_cb so the callback is never run. - //! Since this variable is accessed from multiple threads, accesses should - //! be guarded with the associated Waiter::m_mutex. + //! Registration and removal of this callback run on the associated event + //! loop thread. std::optional m_disconnect_cb; }; @@ -418,13 +418,8 @@ struct Waiter }); } - //! Mutex mainly used internally by waiter class, but also used externally - //! to guard access to related state. Specifically, since the thread_local - //! ThreadContext struct owns a Waiter, the Waiter::m_mutex is used to guard - //! access to other parts of the struct to avoid needing to deal with more - //! mutexes than necessary. This mutex can be held at the same time as - //! EventLoop::m_mutex as long as Waiter::mutex is locked first and - //! EventLoop::m_mutex is locked second. + //! Mutex guarding the waiter's pending function and shutdown signal. + //! May be locked before ThreadClients::mutex or EventLoop::m_mutex. Mutex m_mutex; std::condition_variable m_cv MP_GUARDED_BY(m_mutex); std::optional> m_fn MP_GUARDED_BY(m_mutex); @@ -704,7 +699,7 @@ void ProxyServerBase::invokeDestroy() //! contain multiple if a single thread makes IPC calls over multiple //! connections. A std::optional value type is used to avoid the map needing to //! be locked while ProxyClient objects are constructed, see -//! ThreadContext "Synchronization note" below. +//! ThreadContext::clients synchronization note below. using ConnThreads = std::map>>; using ConnThread = ConnThreads::iterator; @@ -713,6 +708,30 @@ using ConnThread = ConnThreads::iterator; // inserted bool. std::tuple SetThread(GuardedRef threads, Connection* connection, const std::function& make_thread); +//! Per-connection thread clients and their mutex, independent of the waiter. +struct ThreadClients +{ + //! Release all clients on their respective event loops without holding the + //! map mutex while waiting. May race with disconnect callbacks. + void clear(); + + //! May be locked before EventLoop::m_mutex, never after it. + Mutex mutex; + //! Local thread handles passed as Context.callbackThread so the server can + //! call back into the waiting caller, keyed by connection. + ConnThreads callback_threads MP_GUARDED_BY(mutex); + //! Thread handles selecting the server execution thread (Context.thread), + //! keyed by connection: remote workers created by ThreadMap.makeThread, or + //! the caller's callbackThread while this thread handles a server request. + ConnThreads request_threads MP_GUARDED_BY(mutex); + +private: + //! Pin an event loop with remaining clients, or return nullopt if empty. + std::optional pinLoop(const ConnThreads& threads); + //! Extract this loop's clients under the mutex, returning them after unlocking. + ConnThreads extractClients(ConnThreads& threads, const EventLoop& loop); +}; + //! The thread_local ThreadContext struct (see CurrentThread()) provides information //! about individual threads and a way of communicating between them. Because //! it's a thread local struct, each ThreadContext instance is initialized by @@ -748,35 +767,15 @@ struct ThreadContext //! destroyed once for the lifetime of the thread. std::unique_ptr waiter = nullptr; - //! When client is making a request to a server, this is the - //! `callbackThread` argument it passes in the request, used by the server - //! in case it needs to make callbacks into the client that need to execute - //! while the client is waiting. This will be set to a local thread object. - //! - //! Synchronization note: The callback_thread and request_thread maps are - //! only ever accessed internally by this thread's destructor and externally - //! by Cap'n Proto event loop threads. Since it's possible for IPC client - //! threads to make calls over different connections that could have - //! different event loops, these maps are guarded by Waiter::m_mutex in case - //! different event loop threads add or remove map entries simultaneously. - //! However, individual ProxyClient objects in the maps will only be - //! associated with one event loop and guarded by EventLoop::m_mutex. So - //! Waiter::m_mutex does not need to be held while accessing individual - //! ProxyClient instances, and may even need to be released to - //! respect lock order and avoid locking Waiter::m_mutex before - //! EventLoop::m_mutex. - ConnThreads callback_threads MP_GUARDED_BY(waiter->m_mutex); - - //! When client is making a request to a server, this is the `thread` - //! argument it passes in the request, used to control which thread on - //! server will be responsible for executing it. If client call is being - //! made from a local thread, this will be a remote thread object returned - //! by makeThread. If a client call is being made from a thread currently - //! handling a server request, this will be set to the `callbackThread` - //! request thread argument passed in that request. + //! Per-connection thread handles and their mutex. //! - //! Synchronization note: \ref callback_threads note applies here as well. - ConnThreads request_threads MP_GUARDED_BY(waiter->m_mutex); + //! Synchronization note: Maps are accessed by cleanup code and by event loops + //! from potentially different connections, so ThreadClients::mutex guards + //! shared map access. Each client's Cap'n Proto state is accessed only on its + //! own event loop. The mutex must not be held while waiting for an event + //! loop, since disconnect callbacks lock it too. Explicit cleanup extracts + //! clients under the map mutex and destroys them after unlocking it. + ThreadClients clients; //! Whether this thread is a capnp event loop thread. Not really used except //! to assert false if there's an attempt to execute a blocking operation @@ -963,7 +962,7 @@ extern thread_local ThreadContext g_thread_context; // NOLINT(bitcoin-nontrivial //! process boundary and callbacks from the server run on the originating //! client thread. The client-side handles for those dedicated server threads //! (the ProxyClient objects returned by ThreadMap.makeThread, stored -//! per connection in the request_threads / callback_threads maps below) are +//! per connection in the request_threads / callback_threads maps) are //! state that must be keyed implicitly by the calling thread, and must be //! released when the client thread exits so the corresponding server threads //! are freed. A thread_local object is the C++ mechanism that provides both diff --git a/include/mp/type-context.h b/include/mp/type-context.h index 7cfc7e79..931a04ba 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -27,7 +27,7 @@ void CustomBuildField(TypeList<>, // Also store the Thread::Client reference in the callback_threads map so // future calls over this connection can reuse it. auto [callback_thread, _]{SetThread( - GuardedRef{thread_context.waiter->m_mutex, thread_context.callback_threads}, &connection, + GuardedRef{thread_context.clients.mutex, thread_context.clients.callback_threads}, &connection, [&] { return connection.m_threads.add(kj::heap>(connection, thread_context, std::thread{})); })}; // Call remote ThreadMap.makeThread function so server will create a @@ -45,7 +45,7 @@ void CustomBuildField(TypeList<>, return request.send().getResult(); // Nonblocking due to capnp request pipelining. }}; auto [request_thread, _1]{SetThread( - GuardedRef{thread_context.waiter->m_mutex, thread_context.request_threads}, + GuardedRef{thread_context.clients.mutex, thread_context.clients.request_threads}, &connection, make_request_thread)}; auto context = output.init(); @@ -94,7 +94,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // to the same thread already in the map, so there is no // need to update the map. auto& thread_context = CurrentThread(); - auto& request_threads = thread_context.request_threads; + auto& request_threads = thread_context.clients.request_threads; ConnThread request_thread; bool inserted{false}; Mutex cancel_mutex; @@ -129,7 +129,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // cancel_monitor.m_canceled was checked above and this // code is running on the event loop thread. std::tie(request_thread, inserted) = SetThread( - GuardedRef{thread_context.waiter->m_mutex, request_threads}, server.m_context.connection, + GuardedRef{thread_context.clients.mutex, request_threads}, server.m_context.connection, [&] { return Accessor::get(call_context.getParams()).getCallbackThread(); }); // Initialize the request's results struct here on the event loop // thread, so later getResults() calls on the execution thread @@ -180,12 +180,12 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // Look up the thread again without using existing // iterator since entry may no longer be there after // a disconnect. Destroy node after releasing - // Waiter::m_mutex, so the ProxyClient + // ThreadClients::mutex, so the ProxyClient // destructor is able to use EventLoop::mutex // without violating lock order. ConnThreads::node_type removed; { - Lock lock(thread_context.waiter->m_mutex); + Lock lock(thread_context.clients.mutex); removed = request_threads.extract(server.m_context.connection); } } diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index a9181b48..16a2fd49 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -47,6 +48,42 @@ ThreadContext& CurrentThread() return g_thread_context; } +void ThreadClients::clear() +{ + for (ConnThreads* threads : {&request_threads, &callback_threads}) { + while (auto loop_ref = pinLoop(*threads)) { + EventLoop& loop = *loop_ref.value(); + loop.sync([&] { + // A disconnect may have removed entries since pinLoop(). + // Destroy the remaining clients on their loop, after unlocking. + auto removed = extractClients(*threads, loop); + }); + } + } +} + +std::optional ThreadClients::pinLoop(const ConnThreads& threads) +{ + const Lock lock(mutex); + if (threads.empty()) return std::nullopt; + // Pin the loop before a disconnect can erase its client. + return EventLoopRef{*threads.begin()->second->m_context.loop}; +} + +ConnThreads ThreadClients::extractClients(ConnThreads& threads, const EventLoop& loop) +{ + ConnThreads removed; + const Lock lock(mutex); + for (auto it = threads.begin(); it != threads.end();) { + if (it->second->m_context.loop.m_loop == &loop) { + removed.insert(threads.extract(it++)); + } else { + ++it; + } + } + return removed; +} + Stream MakeStream(EventLoop&loop, SocketId socket) { Stream stream; @@ -458,8 +495,7 @@ ProxyServer::~ProxyServer() // below. The maps contain Thread::Client objects that need to be // destroyed from the event loop thread (this thread), which can't // happen if this thread is busy calling join. - m_thread_context.request_threads.clear(); - m_thread_context.callback_threads.clear(); + m_thread_context.clients.clear(); //! Ping waiter. waiter->m_cv.notify_all(); } diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 43d32fb2..d53d2d23 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -591,9 +591,9 @@ KJ_TEST("Make simultaneous IPC calls on single remote thread") ThreadContext& tc{CurrentThread()}; Thread::Client *callback_thread, *request_thread; foo->m_context.loop->sync([&] { - Lock lock(tc.waiter->m_mutex); - callback_thread = &tc.callback_threads.at(foo->m_context.connection)->m_client; - request_thread = &tc.request_threads.at(foo->m_context.connection)->m_client; + Lock lock(tc.clients.mutex); + callback_thread = &tc.clients.callback_threads.at(foo->m_context.connection)->m_client; + request_thread = &tc.clients.request_threads.at(foo->m_context.connection)->m_client; }); // Call callIntFnAsync 3 times with n=100, 200, 300 From 451ae29fdd9adc7cefad758a8e3cf904f3d10c2a Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Wed, 7 Oct 2026 14:54:11 +0200 Subject: [PATCH 4/7] proxy: move thread clients into heap-owned state Allocate ThreadClients separately from ThreadContext without changing its cleanup algorithm or synchronization. Disconnect callbacks can keep referring to the maps and mutex when a later change defers destruction beyond the lifetime of ThreadContext. Update client-state accesses to use the pointer. Thread-exit cleanup remains synchronous in this preparatory change. --- include/mp/proxy-io.h | 10 +++++++--- include/mp/type-context.h | 10 +++++----- src/mp/proxy.cpp | 2 +- test/mp/test/test.cpp | 6 +++--- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index 80b8547f..e431fafd 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -708,7 +708,8 @@ using ConnThread = ConnThreads::iterator; // inserted bool. std::tuple SetThread(GuardedRef threads, Connection* connection, const std::function& make_thread); -//! Per-connection thread clients and their mutex, independent of the waiter. +//! Heap-owned thread clients and their mutex. Disconnect callbacks refer to +//! these maps, so they must outlive ThreadContext when cleanup is deferred. struct ThreadClients { //! Release all clients on their respective event loops without holding the @@ -773,9 +774,12 @@ struct ThreadContext //! from potentially different connections, so ThreadClients::mutex guards //! shared map access. Each client's Cap'n Proto state is accessed only on its //! own event loop. The mutex must not be held while waiting for an event - //! loop, since disconnect callbacks lock it too. Explicit cleanup extracts + //! loop, since disconnect callbacks lock it too. Deferred cleanup extracts //! clients under the map mutex and destroys them after unlocking it. - ThreadClients clients; + //! + //! Heap storage lets the destructor hand cleanup to an async thread without + //! invalidating the map and mutex references captured by SetThread(). + std::unique_ptr clients = std::make_unique(); //! Whether this thread is a capnp event loop thread. Not really used except //! to assert false if there's an attempt to execute a blocking operation diff --git a/include/mp/type-context.h b/include/mp/type-context.h index 931a04ba..050a3097 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -27,7 +27,7 @@ void CustomBuildField(TypeList<>, // Also store the Thread::Client reference in the callback_threads map so // future calls over this connection can reuse it. auto [callback_thread, _]{SetThread( - GuardedRef{thread_context.clients.mutex, thread_context.clients.callback_threads}, &connection, + GuardedRef{thread_context.clients->mutex, thread_context.clients->callback_threads}, &connection, [&] { return connection.m_threads.add(kj::heap>(connection, thread_context, std::thread{})); })}; // Call remote ThreadMap.makeThread function so server will create a @@ -45,7 +45,7 @@ void CustomBuildField(TypeList<>, return request.send().getResult(); // Nonblocking due to capnp request pipelining. }}; auto [request_thread, _1]{SetThread( - GuardedRef{thread_context.clients.mutex, thread_context.clients.request_threads}, + GuardedRef{thread_context.clients->mutex, thread_context.clients->request_threads}, &connection, make_request_thread)}; auto context = output.init(); @@ -94,7 +94,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // to the same thread already in the map, so there is no // need to update the map. auto& thread_context = CurrentThread(); - auto& request_threads = thread_context.clients.request_threads; + auto& request_threads = thread_context.clients->request_threads; ConnThread request_thread; bool inserted{false}; Mutex cancel_mutex; @@ -129,7 +129,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // cancel_monitor.m_canceled was checked above and this // code is running on the event loop thread. std::tie(request_thread, inserted) = SetThread( - GuardedRef{thread_context.clients.mutex, request_threads}, server.m_context.connection, + GuardedRef{thread_context.clients->mutex, request_threads}, server.m_context.connection, [&] { return Accessor::get(call_context.getParams()).getCallbackThread(); }); // Initialize the request's results struct here on the event loop // thread, so later getResults() calls on the execution thread @@ -185,7 +185,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // without violating lock order. ConnThreads::node_type removed; { - Lock lock(thread_context.clients.mutex); + Lock lock(thread_context.clients->mutex); removed = request_threads.extract(server.m_context.connection); } } diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 16a2fd49..98fb7bcb 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -495,7 +495,7 @@ ProxyServer::~ProxyServer() // below. The maps contain Thread::Client objects that need to be // destroyed from the event loop thread (this thread), which can't // happen if this thread is busy calling join. - m_thread_context.clients.clear(); + m_thread_context.clients->clear(); //! Ping waiter. waiter->m_cv.notify_all(); } diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index d53d2d23..d176bd93 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -591,9 +591,9 @@ KJ_TEST("Make simultaneous IPC calls on single remote thread") ThreadContext& tc{CurrentThread()}; Thread::Client *callback_thread, *request_thread; foo->m_context.loop->sync([&] { - Lock lock(tc.clients.mutex); - callback_thread = &tc.clients.callback_threads.at(foo->m_context.connection)->m_client; - request_thread = &tc.clients.request_threads.at(foo->m_context.connection)->m_client; + Lock lock(tc.clients->mutex); + callback_thread = &tc.clients->callback_threads.at(foo->m_context.connection)->m_client; + request_thread = &tc.clients->request_threads.at(foo->m_context.connection)->m_client; }); // Call callIntFnAsync 3 times with n=100, 200, 300 From 9ad7c94b89d1ff4f03e5bb2e7e6d8449aec776c5 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Wed, 7 Oct 2026 14:38:12 +0200 Subject: [PATCH 5/7] proxy: allow async cleanup from other threads Pin the event loop while queuing cleanup. Calls from another thread notify an existing cleanup worker, or wake the event loop to start one. Post only the first wakeup for a pending queue, so exiting threads cannot fill the pipe while the loop is waiting for a worker to start. Keep async-thread creation on the event loop, since callers can hold the Windows loader lock. Test posting move-only callbacks from another thread, both before and after the cleanup worker has started. --- include/mp/proxy-io.h | 5 +++-- src/mp/proxy.cpp | 22 ++++++++++++++++++++-- test/mp/test/test.cpp | 22 ++++++++++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index e431fafd..e5b21093 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -271,11 +271,12 @@ class EventLoop void sync(kj::FunctionParam fn); //! Register cleanup function to run on asynchronous worker thread without - //! blocking the event loop thread. + //! blocking the event loop thread. May be called from any thread while loop() + //! is active. Does not wait for cleanup or for the async thread to start. void addAsyncCleanup(kj::Function fn); //! Start asynchronous worker thread if necessary. This is only done if - //! there are ProxyServerBase::m_impl objects that need to be destroyed + //! there are thread clients or ProxyServerBase::m_impl objects to destroy //! asynchronously, without tying up the event loop thread. This can happen //! when an interface does not declare a destroy() method that would allow //! the client to wait for the destructor to finish and run it on a diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 98fb7bcb..af68d7b6 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -270,7 +270,8 @@ void Connection::removeSyncCleanup(CleanupIt it) void EventLoop::addAsyncCleanup(kj::Function fn) { - const Lock lock(m_mutex); + Lock lock(m_mutex); + EventLoopRef ref{*this, &lock}; // Add async cleanup callbacks to the back of the list. Unlike the sync // cleanup list, this list order is more significant because it determines // the order server objects are destroyed when there is a sudden disconnect, @@ -287,8 +288,24 @@ void EventLoop::addAsyncCleanup(kj::Function fn) // process, otherwise shared pointer counts of the CWallet objects (which // inherit from Chain::Notification) will not be 1 when WalletLoader // destructor runs and it will wait forever for them to be released. + const bool wake_loop = m_async_fns->empty() && !m_async_thread.joinable(); m_async_fns->emplace_back(std::move(fn)); - startAsyncThread(); + if (std::this_thread::get_id() == m_thread_id) { + startAsyncThread(); + } else if (m_async_thread.joinable()) { + m_cv.notify_all(); + } else if (wake_loop) { + // Thread-local destructors can run under the Windows loader lock. + // Wake the event loop to start the async thread instead of starting or + // waiting for a thread here. The reference keeps the loop alive until + // the wakeup is posted, even if cleanup completes in the meantime. + // Only wake on the first queued callback, so exiting threads cannot + // fill the pipe while the loop is waiting for a worker to start. + Unlock(lock, [&] { + char buffer = 0; + m_post_writer->write(&buffer, 1); + }); + } } EventLoop::EventLoop(const char* exe_name, LogOptions log_opts, void* context) @@ -341,6 +358,7 @@ void EventLoop::loop() const size_t read_bytes = wait_stream->read(&buffer, 0, 1).wait(m_io_context.waitScope); if (read_bytes != 1) throw std::logic_error("EventLoop wait_stream closed unexpectedly"); Lock lock(m_mutex); + startAsyncThread(); if (m_sync_fn) { // m_sync_fn throwing is never expected. If it does happen, the caller // of EventLoop::sync() will return without any indication of failure, diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index d176bd93..12f05359 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -476,6 +476,28 @@ KJ_TEST("Calling async IPC method with a remote disconnect while results are bui setup.server_disconnect(); } +KJ_TEST("Async cleanup can be queued from another thread") +{ + // Keep promises alive until setup has joined the async cleanup thread. + std::promise first; + std::promise second; + TestSetup setup; + EventLoop& loop = *setup.client->m_context.loop; + // Ensure loop() is active before posting from this test's thread. + loop.sync([] {}); + + loop.addAsyncCleanup([value = std::make_unique(41), &first] { + first.set_value(*value); + }); + KJ_EXPECT(first.get_future().get() == 41); + + // Exercise notification after the async thread has already been started. + loop.addAsyncCleanup([value = std::make_unique(42), &second] { + second.set_value(*value); + }); + KJ_EXPECT(second.get_future().get() == 42); +} + KJ_TEST("Worker thread destroyed before it is initialized") { // Regression test for bitcoin/bitcoin#34711, bitcoin/bitcoin#34756 where a From 90c1ae7b602c407f56bbd2ff61cc5fc12a7be9c6 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Wed, 7 Oct 2026 15:28:13 +0200 Subject: [PATCH 6/7] proxy: defer thread-local client cleanup Windows ThreadContext destructors can run under the loader lock. Waiting for an event loop from these destructors can deadlock if the loop starts or joins a worker, which needs the loader lock to start or finish exiting. Hand the heap-owned ThreadClients to async cleanup without waiting for an event loop or creating a thread. The maps and mutex remain valid for disconnect callbacks until cleanup releases the clients on their respective event loops. Add portable coverage for client exit before connection closure, exit with a blocked event loop, disconnect after thread exit while deferred cleanup is held, and calls over multiple event loops. --- include/mp/proxy-io.h | 2 + src/mp/proxy.cpp | 18 ++++++++ test/mp/test/test.cpp | 104 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index e5b21093..982a5847 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -749,6 +749,8 @@ struct ThreadClients //! with local and remote ProxyClient objects. struct ThreadContext { + ~ThreadContext(); + //! Identifying string for debug. std::string thread_name; diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index af68d7b6..83147c32 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -84,6 +84,24 @@ ConnThreads ThreadClients::extractClients(ConnThreads& threads, const EventLoop& return removed; } +ThreadContext::~ThreadContext() +{ + // The Windows loader lock may be held here. Queue client cleanup without + // waiting for an event loop or a thread to start or exit. SetThread's + // disconnect callbacks keep referring to the heap-owned maps and mutex. + const Lock lock(clients->mutex); + const ConnThreads* threads = &clients->request_threads; + if (threads->empty()) { + threads = &clients->callback_threads; + } + if (threads->empty()) return; + EventLoop& loop = *threads->begin()->second->m_context.loop; + loop.addAsyncCleanup([clients = std::move(clients)]() mutable { + clients->clear(); + clients.reset(); + }); +} + Stream MakeStream(EventLoop&loop, SocketId socket) { Stream stream; diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 12f05359..a84019ab 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -498,6 +498,110 @@ KJ_TEST("Async cleanup can be queued from another thread") KJ_EXPECT(second.get_future().get() == 42); } +KJ_TEST("Client thread exits before its connection closes") +{ + // Keep the connection open while a caller exits and releases its thread + // handles, then verify another caller can use it. The next test covers + // blocked event loops deterministically without platform-specific timing. + TestSetup setup; + auto* foo = setup.client.get(); + foo->initThreadMap(); + setup.server->m_impl->m_int_fn = [](int arg) { return arg + 1; }; + + // setup.client keeps the connection open while the caller's thread-local + // cleanup releases the remote worker. + std::thread caller{[&] { KJ_EXPECT(foo->callIntFnAsync(41) == 42); }}; + caller.join(); + + // The connection remains usable after the first caller has gone away. + KJ_EXPECT(foo->callIntFnAsync(1) == 2); +} + +KJ_TEST("Client thread exits while the event loop is blocked") +{ + // Model the loader-lock cycle portably: after making a call, let the + // client exit while its event loop is blocked. Thread-local destruction + // must return without waiting for sync(), even if the cleanup thread has + // not been started yet. + std::promise exit_client; + auto exit_client_future = exit_client.get_future(); + std::promise loop_blocked; + std::promise release_loop; + auto release_loop_future = release_loop.get_future(); + std::promise caller_done; + TestSetup setup; + auto* foo = setup.client.get(); + EventLoop& loop = *foo->m_context.loop; + setup.initAsyncCalls(); + + std::thread caller{[&] { + foo->callFnAsync(); + caller_done.set_value(); + exit_client_future.wait(); + }}; + caller_done.get_future().get(); + // Post the blocking task without waiting for it to complete. + loop.sync([&] { + loop.m_task_set->add(kj::evalLater([&] { + loop_blocked.set_value(); + release_loop_future.wait(); + })); + }); + loop_blocked.get_future().get(); + + std::promise caller_exited; + auto caller_exited_future = caller_exited.get_future(); + std::thread joiner{[&] { + caller.join(); + caller_exited.set_value(); + }}; + exit_client.set_value(); + const auto exited = caller_exited_future.wait_for(std::chrono::seconds{5}); + // Always release the loop before checking, so the unfixed code can finish. + release_loop.set_value(); + joiner.join(); + KJ_EXPECT(exited == std::future_status::ready); + // The connection remains usable after deferred client cleanup. + foo->callFnAsync(); +} + +KJ_TEST("Disconnect removes thread clients after their thread exits") +{ + // Hold the async thread so disconnect callbacks run after ThreadContext + // has been destroyed, but before its deferred client cleanup can run. + std::promise cleanup_started; + std::promise release_cleanup; + auto release_cleanup_future = release_cleanup.get_future(); + TestSetup setup{/*client_owns_connection=*/false}; + auto* foo = setup.client.get(); + EventLoop& loop = *foo->m_context.loop; + setup.initAsyncCalls(); + loop.addAsyncCleanup([&] { + cleanup_started.set_value(); + release_cleanup_future.wait(); + }); + cleanup_started.get_future().get(); + std::thread caller{[&] { foo->callFnAsync(); }}; + caller.join(); + setup.client_disconnect(); + setup.server_disconnect(); + release_cleanup.set_value(); +} + +KJ_TEST("Client thread exits after calls over different event loops") +{ + TestSetup first; + TestSetup second; + first.initAsyncCalls(); + second.initAsyncCalls(); + std::thread caller{[&] { + first.client->callFnAsync(); + second.client->callFnAsync(); + }}; + caller.join(); + // Each setup waits for its event loop to exit, including deferred cleanup. +} + KJ_TEST("Worker thread destroyed before it is initialized") { // Regression test for bitcoin/bitcoin#34711, bitcoin/bitcoin#34756 where a From b89d6bd27a4fd94c79f192a7b532f55f089082f7 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Wed, 7 Oct 2026 15:06:20 +0200 Subject: [PATCH 7/7] proxy: join workers off the event loop Signal the worker to stop and queue its join on the existing async cleanup thread. Retain the waiter until the worker has exited, since it still uses the waiter's mutex and condition variable while returning from Waiter::wait(). The event loop remains available for disconnects and other requests during teardown, and still waits for queued cleanup before exiting. Add a worker-exit hook and a portable test holding a stopped worker alive while disconnect completes. This covers event-loop responsiveness separately from the preceding client-exit cleanup fix. --- include/mp/proxy-io.h | 8 ++++---- src/mp/proxy.cpp | 39 +++++++++++++++++++++++---------------- test/mp/test/test.cpp | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 20 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index 982a5847..9aae661d 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -836,10 +836,10 @@ kj::Promise ProxyServer::post(Fn&& fn) result_value.reset(); } result_fulfiller = nullptr; - // Use evalLater to destroy the ProxyServer self - // reference, if it is the last reference, because the - // ProxyServer destructor needs to join the thread, - // which can't happen until this sync() block has exited. + // Defer releasing self to keep the server alive until this + // result-delivery callback returns, keeping shutdown outside + // the callback. This is a lifetime precaution; the asynchronous + // join does not require the deferral. m_loop->m_task_set->add(kj::evalLater([self = kj::mv(self)] {})); }); }); diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 83147c32..33882184 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include // IWYU pragma: keep @@ -513,11 +514,11 @@ ProxyServer::ProxyServer(Connection& connection, ThreadContext& thread_c ProxyServer::~ProxyServer() { if (!m_thread.joinable()) return; - // Stop async thread and wait for it to exit. Need to wait because the - // m_thread handle needs to outlive the thread to avoid "terminate called - // without an active exception" error. An alternative to waiting would be - // detach the thread, but this would introduce nondeterminism which could - // make code harder to debug or extend. + // Worker exit can run thread-local destructors and, on Windows, wait for + // the loader lock. Joining here would block disconnects and other IPC + // during that interval, and could deadlock if a destructor needs the loop. + // Signal the worker, then join it on the async cleanup thread, which holds + // an EventLoopRef until the join completes to keep the loop running. assert(m_thread_context.waiter.get()); std::unique_ptr waiter; { @@ -527,15 +528,18 @@ ProxyServer::~ProxyServer() waiter = std::move(m_thread_context.waiter); //! Assert waiter is idle. This destructor shouldn't be getting called if it is busy. assert(!waiter->m_fn); - // Clear client maps now to avoid deadlock in m_thread.join() call - // below. The maps contain Thread::Client objects that need to be - // destroyed from the event loop thread (this thread), which can't - // happen if this thread is busy calling join. + // Clear client maps before signaling the worker. Releasing clients + // on their event loops avoids doing this during worker TLS teardown, + // where waiting for an event loop can deadlock under the Windows loader lock. m_thread_context.clients->clear(); //! Ping waiter. waiter->m_cv.notify_all(); } - m_thread.join(); + // Retain the waiter until join completes because the worker still uses + // its mutex and condition variable while returning from Waiter::wait(). + m_loop->addAsyncCleanup([thread = std::move(m_thread), waiter = std::move(waiter)]() mutable { + thread.join(); + }); } kj::Promise ProxyServer::getName(GetNameContext context) @@ -580,12 +584,15 @@ kj::Promise ProxyServer::makeThread(MakeThreadContext context) SetOsThreadName("capnp-worker"); CurrentThread().thread_name = ThreadName(loop.m_exe_name) + " (from " + from + ")"; CurrentThread().waiter = std::make_unique(); - Lock lock(CurrentThread().waiter->m_mutex); - thread_context.set_value(&CurrentThread()); - if (loop.testing_hook_makethread_created) loop.testing_hook_makethread_created(); - // Wait for shutdown signal from ProxyServer destructor (signal - // is just waiter getting set to null.) - CurrentThread().waiter->wait(lock, [] { return !CurrentThread().waiter; }); + { + Lock lock(CurrentThread().waiter->m_mutex); + thread_context.set_value(&CurrentThread()); + if (loop.testing_hook_makethread_created) loop.testing_hook_makethread_created(); + // Wait for shutdown signal from ProxyServer destructor (signal + // is just waiter getting set to null.) + CurrentThread().waiter->wait(lock, [] { return !CurrentThread().waiter; }); + } + if (loop.testing_hook_misc) loop.testing_hook_misc("worker thread exit"); }); auto thread_server = kj::heap>(m_connection, *thread_context.get_future().get(), std::move(thread)); auto thread_client = m_connection.m_threads.add(kj::mv(thread_server)); diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index a84019ab..7f9b7e3d 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -602,6 +602,48 @@ KJ_TEST("Client thread exits after calls over different event loops") // Each setup waits for its event loop to exit, including deferred cleanup. } +KJ_TEST("Disconnect completes while a worker exits") +{ + // Keep the worker alive after it stops accepting work, and verify that + // disconnect and event-loop calls can complete before it finishes exiting. + // This complements client-exit coverage: moving client cleanup off the + // exiting thread does not by itself keep the event loop available here. + + // These signals outlive setup, which waits for all worker cleanup. + std::promise worker_stopping; + auto worker_stopping_future = worker_stopping.get_future(); + std::promise release_worker; + auto release_worker_future = release_worker.get_future(); + TestSetup setup; + auto* foo = setup.client.get(); + EventLoop& loop = *foo->m_context.loop; + EventLoopRef loop_ref{loop}; + loop.testing_hook_misc = [&](std::any arg) { + if (const char* const* tag{std::any_cast(&arg)}; + tag && std::string_view{*tag} == "worker thread exit") { + worker_stopping.set_value(); + release_worker_future.wait(); + } + }; + setup.initAsyncCalls(); + foo->callFnAsync(); + + std::promise disconnected; + std::thread disconnect{[&] { + setup.server_disconnect(); + disconnected.set_value(); + }}; + const auto stopping = worker_stopping_future.wait_for(std::chrono::seconds{5}); + const auto complete = disconnected.get_future().wait_for(std::chrono::seconds{5}); + + // Always unblock the worker before checking the results. A synchronous + // join makes disconnect time out, but cleanup can still complete. + release_worker.set_value(); + disconnect.join(); + KJ_EXPECT(stopping == std::future_status::ready); + KJ_EXPECT(complete == std::future_status::ready); +} + KJ_TEST("Worker thread destroyed before it is initialized") { // Regression test for bitcoin/bitcoin#34711, bitcoin/bitcoin#34756 where a