diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index b226284f..9aae661d 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -91,8 +91,8 @@ struct ProxyClient : public ProxyClientBase //! map. It will also reset m_disconnect_cb so the destructor does not //! access it. In the normal case where there is no sudden disconnect, the //! destructor will unregister m_disconnect_cb so the callback is never run. - //! Since this variable is accessed from multiple threads, accesses should - //! be guarded with the associated Waiter::m_mutex. + //! Registration and removal of this callback run on the associated event + //! loop thread. std::optional m_disconnect_cb; }; @@ -271,11 +271,12 @@ class EventLoop void sync(kj::FunctionParam fn); //! Register cleanup function to run on asynchronous worker thread without - //! blocking the event loop thread. - void addAsyncCleanup(std::function fn); + //! blocking the event loop thread. May be called from any thread while loop() + //! is active. Does not wait for cleanup or for the async thread to start. + void addAsyncCleanup(kj::Function fn); //! Start asynchronous worker thread if necessary. This is only done if - //! there are ProxyServerBase::m_impl objects that need to be destroyed + //! there are thread clients or ProxyServerBase::m_impl objects to destroy //! asynchronously, without tying up the event loop thread. This can happen //! when an interface does not declare a destroy() method that would allow //! the client to wait for the destructor to finish and run it on a @@ -305,7 +306,7 @@ class EventLoop kj::FunctionParam* m_sync_fn MP_GUARDED_BY(m_mutex) = nullptr; //! Callback functions to run on async thread. - std::optional m_async_fns MP_GUARDED_BY(m_mutex); + std::optional m_async_fns MP_GUARDED_BY(m_mutex); //! Socket pair used to post and wait for wakeups to the event loop thread. kj::Own m_wait_stream; @@ -418,13 +419,8 @@ struct Waiter }); } - //! Mutex mainly used internally by waiter class, but also used externally - //! to guard access to related state. Specifically, since the thread_local - //! ThreadContext struct owns a Waiter, the Waiter::m_mutex is used to guard - //! access to other parts of the struct to avoid needing to deal with more - //! mutexes than necessary. This mutex can be held at the same time as - //! EventLoop::m_mutex as long as Waiter::mutex is locked first and - //! EventLoop::m_mutex is locked second. + //! Mutex guarding the waiter's pending function and shutdown signal. + //! May be locked before ThreadClients::mutex or EventLoop::m_mutex. Mutex m_mutex; std::condition_variable m_cv MP_GUARDED_BY(m_mutex); std::optional> m_fn MP_GUARDED_BY(m_mutex); @@ -704,7 +700,7 @@ void ProxyServerBase::invokeDestroy() //! contain multiple if a single thread makes IPC calls over multiple //! connections. A std::optional value type is used to avoid the map needing to //! be locked while ProxyClient objects are constructed, see -//! ThreadContext "Synchronization note" below. +//! ThreadContext::clients synchronization note below. using ConnThreads = std::map>>; using ConnThread = ConnThreads::iterator; @@ -713,6 +709,31 @@ using ConnThread = ConnThreads::iterator; // inserted bool. std::tuple SetThread(GuardedRef threads, Connection* connection, const std::function& make_thread); +//! Heap-owned thread clients and their mutex. Disconnect callbacks refer to +//! these maps, so they must outlive ThreadContext when cleanup is deferred. +struct ThreadClients +{ + //! Release all clients on their respective event loops without holding the + //! map mutex while waiting. May race with disconnect callbacks. + void clear(); + + //! May be locked before EventLoop::m_mutex, never after it. + Mutex mutex; + //! Local thread handles passed as Context.callbackThread so the server can + //! call back into the waiting caller, keyed by connection. + ConnThreads callback_threads MP_GUARDED_BY(mutex); + //! Thread handles selecting the server execution thread (Context.thread), + //! keyed by connection: remote workers created by ThreadMap.makeThread, or + //! the caller's callbackThread while this thread handles a server request. + ConnThreads request_threads MP_GUARDED_BY(mutex); + +private: + //! Pin an event loop with remaining clients, or return nullopt if empty. + std::optional pinLoop(const ConnThreads& threads); + //! Extract this loop's clients under the mutex, returning them after unlocking. + ConnThreads extractClients(ConnThreads& threads, const EventLoop& loop); +}; + //! The thread_local ThreadContext struct (see CurrentThread()) provides information //! about individual threads and a way of communicating between them. Because //! it's a thread local struct, each ThreadContext instance is initialized by @@ -728,6 +749,8 @@ std::tuple SetThread(GuardedRef threads, Connecti //! with local and remote ProxyClient objects. struct ThreadContext { + ~ThreadContext(); + //! Identifying string for debug. std::string thread_name; @@ -748,35 +771,18 @@ struct ThreadContext //! destroyed once for the lifetime of the thread. std::unique_ptr waiter = nullptr; - //! When client is making a request to a server, this is the - //! `callbackThread` argument it passes in the request, used by the server - //! in case it needs to make callbacks into the client that need to execute - //! while the client is waiting. This will be set to a local thread object. + //! Per-connection thread handles and their mutex. //! - //! Synchronization note: The callback_thread and request_thread maps are - //! only ever accessed internally by this thread's destructor and externally - //! by Cap'n Proto event loop threads. Since it's possible for IPC client - //! threads to make calls over different connections that could have - //! different event loops, these maps are guarded by Waiter::m_mutex in case - //! different event loop threads add or remove map entries simultaneously. - //! However, individual ProxyClient objects in the maps will only be - //! associated with one event loop and guarded by EventLoop::m_mutex. So - //! Waiter::m_mutex does not need to be held while accessing individual - //! ProxyClient instances, and may even need to be released to - //! respect lock order and avoid locking Waiter::m_mutex before - //! EventLoop::m_mutex. - ConnThreads callback_threads MP_GUARDED_BY(waiter->m_mutex); - - //! When client is making a request to a server, this is the `thread` - //! argument it passes in the request, used to control which thread on - //! server will be responsible for executing it. If client call is being - //! made from a local thread, this will be a remote thread object returned - //! by makeThread. If a client call is being made from a thread currently - //! handling a server request, this will be set to the `callbackThread` - //! request thread argument passed in that request. + //! Synchronization note: Maps are accessed by cleanup code and by event loops + //! from potentially different connections, so ThreadClients::mutex guards + //! shared map access. Each client's Cap'n Proto state is accessed only on its + //! own event loop. The mutex must not be held while waiting for an event + //! loop, since disconnect callbacks lock it too. Deferred cleanup extracts + //! clients under the map mutex and destroys them after unlocking it. //! - //! Synchronization note: \ref callback_threads note applies here as well. - ConnThreads request_threads MP_GUARDED_BY(waiter->m_mutex); + //! Heap storage lets the destructor hand cleanup to an async thread without + //! invalidating the map and mutex references captured by SetThread(). + std::unique_ptr clients = std::make_unique(); //! Whether this thread is a capnp event loop thread. Not really used except //! to assert false if there's an attempt to execute a blocking operation @@ -830,10 +836,10 @@ kj::Promise ProxyServer::post(Fn&& fn) result_value.reset(); } result_fulfiller = nullptr; - // Use evalLater to destroy the ProxyServer self - // reference, if it is the last reference, because the - // ProxyServer destructor needs to join the thread, - // which can't happen until this sync() block has exited. + // Defer releasing self to keep the server alive until this + // result-delivery callback returns, keeping shutdown outside + // the callback. This is a lifetime precaution; the asynchronous + // join does not require the deferral. m_loop->m_task_set->add(kj::evalLater([self = kj::mv(self)] {})); }); }); @@ -963,7 +969,7 @@ extern thread_local ThreadContext g_thread_context; // NOLINT(bitcoin-nontrivial //! process boundary and callbacks from the server run on the originating //! client thread. The client-side handles for those dedicated server threads //! (the ProxyClient objects returned by ThreadMap.makeThread, stored -//! per connection in the request_threads / callback_threads maps below) are +//! per connection in the request_threads / callback_threads maps) are //! state that must be keyed implicitly by the calling thread, and must be //! released when the client thread exits so the corresponding server threads //! are freed. A thread_local object is the C++ mechanism that provides both diff --git a/include/mp/proxy.h b/include/mp/proxy.h index 2144b571..ca5a77c9 100644 --- a/include/mp/proxy.h +++ b/include/mp/proxy.h @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -33,6 +34,7 @@ template struct ProxyStruct; // IWYU pragma: export //! Mapping from local c++ type to capnp type and traits (specializations are generated by proxy-codegen.cpp). template struct ProxyType; // IWYU pragma: export +using AsyncCleanupList = std::list>; using CleanupList = std::list>; using CleanupIt = typename CleanupList::iterator; diff --git a/include/mp/type-context.h b/include/mp/type-context.h index 7cfc7e79..050a3097 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -27,7 +27,7 @@ void CustomBuildField(TypeList<>, // Also store the Thread::Client reference in the callback_threads map so // future calls over this connection can reuse it. auto [callback_thread, _]{SetThread( - GuardedRef{thread_context.waiter->m_mutex, thread_context.callback_threads}, &connection, + GuardedRef{thread_context.clients->mutex, thread_context.clients->callback_threads}, &connection, [&] { return connection.m_threads.add(kj::heap>(connection, thread_context, std::thread{})); })}; // Call remote ThreadMap.makeThread function so server will create a @@ -45,7 +45,7 @@ void CustomBuildField(TypeList<>, return request.send().getResult(); // Nonblocking due to capnp request pipelining. }}; auto [request_thread, _1]{SetThread( - GuardedRef{thread_context.waiter->m_mutex, thread_context.request_threads}, + GuardedRef{thread_context.clients->mutex, thread_context.clients->request_threads}, &connection, make_request_thread)}; auto context = output.init(); @@ -94,7 +94,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // to the same thread already in the map, so there is no // need to update the map. auto& thread_context = CurrentThread(); - auto& request_threads = thread_context.request_threads; + auto& request_threads = thread_context.clients->request_threads; ConnThread request_thread; bool inserted{false}; Mutex cancel_mutex; @@ -129,7 +129,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // cancel_monitor.m_canceled was checked above and this // code is running on the event loop thread. std::tie(request_thread, inserted) = SetThread( - GuardedRef{thread_context.waiter->m_mutex, request_threads}, server.m_context.connection, + GuardedRef{thread_context.clients->mutex, request_threads}, server.m_context.connection, [&] { return Accessor::get(call_context.getParams()).getCallbackThread(); }); // Initialize the request's results struct here on the event loop // thread, so later getResults() calls on the execution thread @@ -180,12 +180,12 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // Look up the thread again without using existing // iterator since entry may no longer be there after // a disconnect. Destroy node after releasing - // Waiter::m_mutex, so the ProxyClient + // ThreadClients::mutex, so the ProxyClient // destructor is able to use EventLoop::mutex // without violating lock order. ConnThreads::node_type removed; { - Lock lock(thread_context.waiter->m_mutex); + Lock lock(thread_context.clients->mutex); removed = request_threads.extract(server.m_context.connection); } } diff --git a/src/mp/gen.cpp b/src/mp/gen.cpp index 8d733a91..c955f319 100644 --- a/src/mp/gen.cpp +++ b/src/mp/gen.cpp @@ -344,6 +344,7 @@ static void Generate(kj::StringPtr src_prefix, cpp_types << "#include <" << include_path << ".h> // IWYU pragma: keep\n"; cpp_types << "#include <" << include_path << ".proxy.h>\n"; cpp_types << "#include <" << include_path << ".proxy-types.h> // IWYU pragma: keep\n"; + cpp_types << "#include \n"; cpp_types << "#include <" << PROXY_TYPES << ">\n\n"; cpp_types << "namespace mp {\n"; diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 552e3268..33882184 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include // IWYU pragma: keep @@ -17,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -47,6 +49,60 @@ ThreadContext& CurrentThread() return g_thread_context; } +void ThreadClients::clear() +{ + for (ConnThreads* threads : {&request_threads, &callback_threads}) { + while (auto loop_ref = pinLoop(*threads)) { + EventLoop& loop = *loop_ref.value(); + loop.sync([&] { + // A disconnect may have removed entries since pinLoop(). + // Destroy the remaining clients on their loop, after unlocking. + auto removed = extractClients(*threads, loop); + }); + } + } +} + +std::optional ThreadClients::pinLoop(const ConnThreads& threads) +{ + const Lock lock(mutex); + if (threads.empty()) return std::nullopt; + // Pin the loop before a disconnect can erase its client. + return EventLoopRef{*threads.begin()->second->m_context.loop}; +} + +ConnThreads ThreadClients::extractClients(ConnThreads& threads, const EventLoop& loop) +{ + ConnThreads removed; + const Lock lock(mutex); + for (auto it = threads.begin(); it != threads.end();) { + if (it->second->m_context.loop.m_loop == &loop) { + removed.insert(threads.extract(it++)); + } else { + ++it; + } + } + return removed; +} + +ThreadContext::~ThreadContext() +{ + // The Windows loader lock may be held here. Queue client cleanup without + // waiting for an event loop or a thread to start or exit. SetThread's + // disconnect callbacks keep referring to the heap-owned maps and mutex. + const Lock lock(clients->mutex); + const ConnThreads* threads = &clients->request_threads; + if (threads->empty()) { + threads = &clients->callback_threads; + } + if (threads->empty()) return; + EventLoop& loop = *threads->begin()->second->m_context.loop; + loop.addAsyncCleanup([clients = std::move(clients)]() mutable { + clients->clear(); + clients.reset(); + }); +} + Stream MakeStream(EventLoop&loop, SocketId socket) { Stream stream; @@ -231,9 +287,10 @@ void Connection::removeSyncCleanup(CleanupIt it) m_sync_cleanup_fns.erase(it); } -void EventLoop::addAsyncCleanup(std::function fn) +void EventLoop::addAsyncCleanup(kj::Function fn) { - const Lock lock(m_mutex); + Lock lock(m_mutex); + EventLoopRef ref{*this, &lock}; // Add async cleanup callbacks to the back of the list. Unlike the sync // cleanup list, this list order is more significant because it determines // the order server objects are destroyed when there is a sudden disconnect, @@ -250,8 +307,24 @@ void EventLoop::addAsyncCleanup(std::function fn) // process, otherwise shared pointer counts of the CWallet objects (which // inherit from Chain::Notification) will not be 1 when WalletLoader // destructor runs and it will wait forever for them to be released. + const bool wake_loop = m_async_fns->empty() && !m_async_thread.joinable(); m_async_fns->emplace_back(std::move(fn)); - startAsyncThread(); + if (std::this_thread::get_id() == m_thread_id) { + startAsyncThread(); + } else if (m_async_thread.joinable()) { + m_cv.notify_all(); + } else if (wake_loop) { + // Thread-local destructors can run under the Windows loader lock. + // Wake the event loop to start the async thread instead of starting or + // waiting for a thread here. The reference keeps the loop alive until + // the wakeup is posted, even if cleanup completes in the meantime. + // Only wake on the first queued callback, so exiting threads cannot + // fill the pipe while the loop is waiting for a worker to start. + Unlock(lock, [&] { + char buffer = 0; + m_post_writer->write(&buffer, 1); + }); + } } EventLoop::EventLoop(const char* exe_name, LogOptions log_opts, void* context) @@ -304,6 +377,7 @@ void EventLoop::loop() const size_t read_bytes = wait_stream->read(&buffer, 0, 1).wait(m_io_context.waitScope); if (read_bytes != 1) throw std::logic_error("EventLoop wait_stream closed unexpectedly"); Lock lock(m_mutex); + startAsyncThread(); if (m_sync_fn) { // m_sync_fn throwing is never expected. If it does happen, the caller // of EventLoop::sync() will return without any indication of failure, @@ -363,7 +437,7 @@ void EventLoop::startAsyncThread() while (m_async_fns) { if (!m_async_fns->empty()) { EventLoopRef ref{*this, &lock}; - const std::function fn = std::move(m_async_fns->front()); + kj::Function fn = std::move(m_async_fns->front()); m_async_fns->pop_front(); Unlock(lock, fn); // Important to relock because of the wait() call below. @@ -440,11 +514,11 @@ ProxyServer::ProxyServer(Connection& connection, ThreadContext& thread_c ProxyServer::~ProxyServer() { if (!m_thread.joinable()) return; - // Stop async thread and wait for it to exit. Need to wait because the - // m_thread handle needs to outlive the thread to avoid "terminate called - // without an active exception" error. An alternative to waiting would be - // detach the thread, but this would introduce nondeterminism which could - // make code harder to debug or extend. + // Worker exit can run thread-local destructors and, on Windows, wait for + // the loader lock. Joining here would block disconnects and other IPC + // during that interval, and could deadlock if a destructor needs the loop. + // Signal the worker, then join it on the async cleanup thread, which holds + // an EventLoopRef until the join completes to keep the loop running. assert(m_thread_context.waiter.get()); std::unique_ptr waiter; { @@ -454,16 +528,18 @@ ProxyServer::~ProxyServer() waiter = std::move(m_thread_context.waiter); //! Assert waiter is idle. This destructor shouldn't be getting called if it is busy. assert(!waiter->m_fn); - // Clear client maps now to avoid deadlock in m_thread.join() call - // below. The maps contain Thread::Client objects that need to be - // destroyed from the event loop thread (this thread), which can't - // happen if this thread is busy calling join. - m_thread_context.request_threads.clear(); - m_thread_context.callback_threads.clear(); + // Clear client maps before signaling the worker. Releasing clients + // on their event loops avoids doing this during worker TLS teardown, + // where waiting for an event loop can deadlock under the Windows loader lock. + m_thread_context.clients->clear(); //! Ping waiter. waiter->m_cv.notify_all(); } - m_thread.join(); + // Retain the waiter until join completes because the worker still uses + // its mutex and condition variable while returning from Waiter::wait(). + m_loop->addAsyncCleanup([thread = std::move(m_thread), waiter = std::move(waiter)]() mutable { + thread.join(); + }); } kj::Promise ProxyServer::getName(GetNameContext context) @@ -508,12 +584,15 @@ kj::Promise ProxyServer::makeThread(MakeThreadContext context) SetOsThreadName("capnp-worker"); CurrentThread().thread_name = ThreadName(loop.m_exe_name) + " (from " + from + ")"; CurrentThread().waiter = std::make_unique(); - Lock lock(CurrentThread().waiter->m_mutex); - thread_context.set_value(&CurrentThread()); - if (loop.testing_hook_makethread_created) loop.testing_hook_makethread_created(); - // Wait for shutdown signal from ProxyServer destructor (signal - // is just waiter getting set to null.) - CurrentThread().waiter->wait(lock, [] { return !CurrentThread().waiter; }); + { + Lock lock(CurrentThread().waiter->m_mutex); + thread_context.set_value(&CurrentThread()); + if (loop.testing_hook_makethread_created) loop.testing_hook_makethread_created(); + // Wait for shutdown signal from ProxyServer destructor (signal + // is just waiter getting set to null.) + CurrentThread().waiter->wait(lock, [] { return !CurrentThread().waiter; }); + } + if (loop.testing_hook_misc) loop.testing_hook_misc("worker thread exit"); }); auto thread_server = kj::heap>(m_connection, *thread_context.get_future().get(), std::move(thread)); auto thread_client = m_connection.m_threads.add(kj::mv(thread_server)); diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 0492c9ee..7f9b7e3d 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -138,6 +138,13 @@ class TestSetup client = client_promise.get_future().get(); } + //! Exchange thread maps and configure a no-op callFnAsync() handler. + void initAsyncCalls() + { + client->initThreadMap(); + server->m_impl->m_fn = [] {}; + } + ~TestSetup() { // Test that client cleanup_fns are executed. @@ -469,6 +476,174 @@ KJ_TEST("Calling async IPC method with a remote disconnect while results are bui setup.server_disconnect(); } +KJ_TEST("Async cleanup can be queued from another thread") +{ + // Keep promises alive until setup has joined the async cleanup thread. + std::promise first; + std::promise second; + TestSetup setup; + EventLoop& loop = *setup.client->m_context.loop; + // Ensure loop() is active before posting from this test's thread. + loop.sync([] {}); + + loop.addAsyncCleanup([value = std::make_unique(41), &first] { + first.set_value(*value); + }); + KJ_EXPECT(first.get_future().get() == 41); + + // Exercise notification after the async thread has already been started. + loop.addAsyncCleanup([value = std::make_unique(42), &second] { + second.set_value(*value); + }); + KJ_EXPECT(second.get_future().get() == 42); +} + +KJ_TEST("Client thread exits before its connection closes") +{ + // Keep the connection open while a caller exits and releases its thread + // handles, then verify another caller can use it. The next test covers + // blocked event loops deterministically without platform-specific timing. + TestSetup setup; + auto* foo = setup.client.get(); + foo->initThreadMap(); + setup.server->m_impl->m_int_fn = [](int arg) { return arg + 1; }; + + // setup.client keeps the connection open while the caller's thread-local + // cleanup releases the remote worker. + std::thread caller{[&] { KJ_EXPECT(foo->callIntFnAsync(41) == 42); }}; + caller.join(); + + // The connection remains usable after the first caller has gone away. + KJ_EXPECT(foo->callIntFnAsync(1) == 2); +} + +KJ_TEST("Client thread exits while the event loop is blocked") +{ + // Model the loader-lock cycle portably: after making a call, let the + // client exit while its event loop is blocked. Thread-local destruction + // must return without waiting for sync(), even if the cleanup thread has + // not been started yet. + std::promise exit_client; + auto exit_client_future = exit_client.get_future(); + std::promise loop_blocked; + std::promise release_loop; + auto release_loop_future = release_loop.get_future(); + std::promise caller_done; + TestSetup setup; + auto* foo = setup.client.get(); + EventLoop& loop = *foo->m_context.loop; + setup.initAsyncCalls(); + + std::thread caller{[&] { + foo->callFnAsync(); + caller_done.set_value(); + exit_client_future.wait(); + }}; + caller_done.get_future().get(); + // Post the blocking task without waiting for it to complete. + loop.sync([&] { + loop.m_task_set->add(kj::evalLater([&] { + loop_blocked.set_value(); + release_loop_future.wait(); + })); + }); + loop_blocked.get_future().get(); + + std::promise caller_exited; + auto caller_exited_future = caller_exited.get_future(); + std::thread joiner{[&] { + caller.join(); + caller_exited.set_value(); + }}; + exit_client.set_value(); + const auto exited = caller_exited_future.wait_for(std::chrono::seconds{5}); + // Always release the loop before checking, so the unfixed code can finish. + release_loop.set_value(); + joiner.join(); + KJ_EXPECT(exited == std::future_status::ready); + // The connection remains usable after deferred client cleanup. + foo->callFnAsync(); +} + +KJ_TEST("Disconnect removes thread clients after their thread exits") +{ + // Hold the async thread so disconnect callbacks run after ThreadContext + // has been destroyed, but before its deferred client cleanup can run. + std::promise cleanup_started; + std::promise release_cleanup; + auto release_cleanup_future = release_cleanup.get_future(); + TestSetup setup{/*client_owns_connection=*/false}; + auto* foo = setup.client.get(); + EventLoop& loop = *foo->m_context.loop; + setup.initAsyncCalls(); + loop.addAsyncCleanup([&] { + cleanup_started.set_value(); + release_cleanup_future.wait(); + }); + cleanup_started.get_future().get(); + std::thread caller{[&] { foo->callFnAsync(); }}; + caller.join(); + setup.client_disconnect(); + setup.server_disconnect(); + release_cleanup.set_value(); +} + +KJ_TEST("Client thread exits after calls over different event loops") +{ + TestSetup first; + TestSetup second; + first.initAsyncCalls(); + second.initAsyncCalls(); + std::thread caller{[&] { + first.client->callFnAsync(); + second.client->callFnAsync(); + }}; + caller.join(); + // Each setup waits for its event loop to exit, including deferred cleanup. +} + +KJ_TEST("Disconnect completes while a worker exits") +{ + // Keep the worker alive after it stops accepting work, and verify that + // disconnect and event-loop calls can complete before it finishes exiting. + // This complements client-exit coverage: moving client cleanup off the + // exiting thread does not by itself keep the event loop available here. + + // These signals outlive setup, which waits for all worker cleanup. + std::promise worker_stopping; + auto worker_stopping_future = worker_stopping.get_future(); + std::promise release_worker; + auto release_worker_future = release_worker.get_future(); + TestSetup setup; + auto* foo = setup.client.get(); + EventLoop& loop = *foo->m_context.loop; + EventLoopRef loop_ref{loop}; + loop.testing_hook_misc = [&](std::any arg) { + if (const char* const* tag{std::any_cast(&arg)}; + tag && std::string_view{*tag} == "worker thread exit") { + worker_stopping.set_value(); + release_worker_future.wait(); + } + }; + setup.initAsyncCalls(); + foo->callFnAsync(); + + std::promise disconnected; + std::thread disconnect{[&] { + setup.server_disconnect(); + disconnected.set_value(); + }}; + const auto stopping = worker_stopping_future.wait_for(std::chrono::seconds{5}); + const auto complete = disconnected.get_future().wait_for(std::chrono::seconds{5}); + + // Always unblock the worker before checking the results. A synchronous + // join makes disconnect time out, but cleanup can still complete. + release_worker.set_value(); + disconnect.join(); + KJ_EXPECT(stopping == std::future_status::ready); + KJ_EXPECT(complete == std::future_status::ready); +} + KJ_TEST("Worker thread destroyed before it is initialized") { // Regression test for bitcoin/bitcoin#34711, bitcoin/bitcoin#34756 where a @@ -480,8 +655,7 @@ KJ_TEST("Worker thread destroyed before it is initialized") // the worker thread started waiting, causing a SIGSEGV when it did start. TestSetup setup; ProxyClient* foo = setup.client.get(); - foo->initThreadMap(); - setup.server->m_impl->m_fn = [] {}; + setup.initAsyncCalls(); EventLoop& loop = *setup.server->m_context.connection->m_loop; loop.testing_hook_makethread = [&] { @@ -510,8 +684,7 @@ KJ_TEST("Calling async IPC method, with server disconnect racing the call") // calling call_context.getParams(). TestSetup setup; ProxyClient* foo = setup.client.get(); - foo->initThreadMap(); - setup.server->m_impl->m_fn = [] {}; + setup.initAsyncCalls(); EventLoop& loop = *setup.server->m_context.connection->m_loop; loop.testing_hook_async_request_start = [&] { @@ -537,8 +710,7 @@ KJ_TEST("Calling async IPC method, with server disconnect after cleanup") // scope. TestSetup setup; ProxyClient* foo = setup.client.get(); - foo->initThreadMap(); - setup.server->m_impl->m_fn = [] {}; + setup.initAsyncCalls(); EventLoop& loop = *setup.server->m_context.connection->m_loop; loop.testing_hook_async_request_done = [&] { @@ -580,17 +752,16 @@ KJ_TEST("Make simultaneous IPC calls on single remote thread") ProxyClient* foo = setup.client.get(); std::promise signal; - foo->initThreadMap(); + setup.initAsyncCalls(); // Use callFnAsync() to get the client to set up the request_thread // that will be used for the test. - setup.server->m_impl->m_fn = [&] {}; foo->callFnAsync(); ThreadContext& tc{CurrentThread()}; Thread::Client *callback_thread, *request_thread; foo->m_context.loop->sync([&] { - Lock lock(tc.waiter->m_mutex); - callback_thread = &tc.callback_threads.at(foo->m_context.connection)->m_client; - request_thread = &tc.request_threads.at(foo->m_context.connection)->m_client; + Lock lock(tc.clients->mutex); + callback_thread = &tc.clients->callback_threads.at(foo->m_context.connection)->m_client; + request_thread = &tc.clients->request_threads.at(foo->m_context.connection)->m_client; }); // Call callIntFnAsync 3 times with n=100, 200, 300