From 2a3c6751ff0687c769e275658c9bc89f8181ac85 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Thu, 3 Sep 2026 15:23:49 -0400 Subject: [PATCH 01/17] doc: Improve disconnect callback comments This is a documentation-only change meant to make upcoming commits easier to understand. This change was written with Claude Opus 4.8 (1M context). --- include/mp/proxy-io.h | 28 +++++++++++++++----------- src/mp/proxy.cpp | 36 ++++++++++++++++++---------------- test/mp/test/connect_tests.cpp | 10 +++++----- test/mp/test/test.cpp | 3 ++- 4 files changed, 43 insertions(+), 34 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index b226284f..a73c25b5 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -456,18 +456,24 @@ class Connection //! destructors of m_impl instances owned by ProxyServer objects). ~Connection() noexcept(false); - //! Register synchronous cleanup function to run on event loop thread (with - //! access to capnp thread local variables) when disconnect() is called. - //! any new i/o. + //! Register a synchronous cleanup function to run on the event loop thread + //! (with access to capnp thread-local variables) when the connection is + //! disconnected -- for either a remote disconnect (the peer closes the + //! connection) or a local one (the connection is torn down on this side). + //! Contrast onDisconnect(), whose handler runs only on a remote + //! disconnect. Returns a handle that can be passed to removeSyncCleanup() + //! to unregister the function before it runs. CleanupIt addSyncCleanup(std::function fn); void removeSyncCleanup(CleanupIt it); - //! Add disconnect handler. + //! Add a remote disconnect handler, run when the peer closes the + //! connection. The handler is canceled if the connection is disconnected + //! locally first (which destroys m_on_disconnect). template void onDisconnect(F&& f) { - // Add disconnect handler to local TaskSet to ensure it is canceled and - // will never run after connection object is destroyed. But when disconnect + // Add the handler to the local TaskSet to ensure it is canceled and + // will never run after the connection object is destroyed. But when the // handler fires, do not call the function f right away, instead add it // to the EventLoop TaskSet to avoid "Promise callback destroyed itself" // error in the typical case where f deletes this Connection object. @@ -478,9 +484,9 @@ class Connection EventLoopRef m_loop; kj::Own m_stream; LoggingErrorHandler m_error_handler{*m_loop}; - //! TaskSet used to cancel the m_network.onDisconnect() handler for remote - //! disconnections, if the connection is closed locally first by deleting - //! this Connection object. + //! TaskSet holding the m_network.onDisconnect() handler for remote + //! disconnections. Reset to cancel the handler if the connection is closed + //! locally first by deleting this Connection object. kj::TaskSet m_on_disconnect{m_error_handler}; ::capnp::TwoPartyVatNetwork m_network; std::optional<::capnp::RpcSystem<::capnp::rpc::twoparty::VatId>> m_rpc_system; @@ -574,8 +580,8 @@ ProxyClientBase::ProxyClientBase(typename Interface::Client cli // Remove disconnect callback on cleanup so it doesn't run and try // to access this object after it's destroyed. This call needs to // run inside loop->sync() on the event loop thread because - // otherwise, if there were an ill-timed disconnect, the - // onDisconnect handler could fire and delete the Connection object + // otherwise, if there were an ill-timed disconnect, the remote + // disconnect handler could fire and delete the Connection object // before the removeSyncCleanup call. if (m_context.connection) m_context.connection->removeSyncCleanup(disconnect_cb); diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 552e3268..983d6ce8 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -111,7 +111,8 @@ Connection::~Connection() noexcept(false) // Connection destructor is always called on the event loop thread. If this // is a local disconnect, it will trigger I/O, so this needs to run on the // event loop thread, and if there was a remote disconnect, this is called - // by an onDisconnect callback directly from the event loop thread. + // by a TwoPartyVatNetwork::onDisconnect callback directly from the event + // loop thread. assert(std::this_thread::get_id() == m_loop->m_thread_id); // Try to cancel any calls that may be executing. @@ -132,13 +133,13 @@ Connection::~Connection() noexcept(false) // // Sending pending data is important if the connection is a socketpair // because when one side of the socketpair is closed, the other side doesn't - // seem to receive any onDisconnect event. So it is important for the other - // side to instead receive Cap'n Proto "release" messages (see `struct - // Release` in capnp/rpc.capnp) from local Client objects being destroyed so - // the remote side can free resources and shut down cleanly. Without this, - // when one side of a socket pair is closed the other side may not receive - // these messages, preventing the remote side from freeing ProxyServer - // resources and shutting down cleanly. + // seem to receive any TwoPartyVatNetwork::onDisconnect event. So it is + // important for the other side to instead receive Cap'n Proto "release" + // messages (see `struct Release` in capnp/rpc.capnp) from local Client + // objects being destroyed so the remote side can free resources and shut + // down cleanly. Without this, when one side of a socket pair is closed the + // other side may not receive these messages, preventing the remote side + // from freeing ProxyServer resources and shutting down cleanly. // Use kj::runCatchingExceptions instead of try/catch because on macOS with // dynamic libraries, kj::Exception typeinfo differs between libcapnp and // the calling binary, so catch (const kj::Exception&) silently fails to @@ -184,17 +185,18 @@ Connection::~Connection() noexcept(false) // connection implementing the Init interface and handling the Init.makeX() calls. // // Either way when a connection is closed, capnp behavior is to call all - // ProxyServer object destructors first, and then trigger an onDisconnect - // callback. + // ProxyServer object destructors first, and then trigger a + // TwoPartyVatNetwork::onDisconnect callback. // - // On incoming side of the connection, the onDisconnect callback is written - // to delete the Connection object from the m_incoming_connections and call - // this destructor which calls Connection::disconnect. + // On incoming side of the connection, the TwoPartyVatNetwork::onDisconnect + // callback is written to delete the Connection object from the + // m_incoming_connections list and call this destructor. // - // On the outgoing side, the Connection object is owned by top level client - // object client, which onDisconnect handler doesn't have ready access to, - // so onDisconnect handler just calls Connection::disconnect directly - // instead. + // On the outgoing side, the Connection object is heap-allocated and owned + // by a top-level ProxyClient object. In this case, the + // TwoPartyVatNetwork::onDisconnect handler deletes the Connection object + // directly, calling this destructor, and loop below sets Connection + // pointers in all associated ProxyClient objects to null. // // Either way disconnect code runs in the event loop thread and called both // on clean and unclean shutdowns. In unclean shutdown case when the diff --git a/test/mp/test/connect_tests.cpp b/test/mp/test/connect_tests.cpp index 4e4f051f..7f522ad6 100644 --- a/test/mp/test/connect_tests.cpp +++ b/test/mp/test/connect_tests.cpp @@ -118,11 +118,11 @@ KJ_TEST("ConnectStream defers disconnect failure to the first IPC request for in } catch (const std::runtime_error& e) { std::string_view reason = e.what(); - // There is a race between the event loop detecting the disconnect - // and foo->add() being called. If the onDisconnect callback fires - // first and nulls m_context.connection, the error is "called after - // disconnect"; if foo->add() submits before the callback fires, - // the error is "interrupted by disconnect". + // There is a race between the event loop detecting the disconnect and + // foo->add() being called. If the TwoPartyVatNetwork::onDisconnect + // callback fires first and nulls m_context.connection, the error is + // "called after disconnect"; if foo->add() submits before the callback + // fires, the error is "interrupted by disconnect". KJ_EXPECT(reason == "IPC client method called after disconnect." || reason == "IPC client method call interrupted by disconnect."); } diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 0492c9ee..377911b7 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -420,7 +420,8 @@ KJ_TEST("Calling async IPC method with a remote disconnect while results are bui // setting request_canceled) and the worker would throw InterruptException // instead of proceeding into getResults(). Keeping it alive matches the // window in the original report, where the worker races with capnp's own - // internal teardown, which runs before any onDisconnect notification. + // internal teardown, which runs before any TwoPartyVatNetwork::onDisconnect + // notification. TestSetup setup{/*client_owns_connection=*/false}; ProxyClient* foo = setup.client.get(); From 5e7dc20b75dce12ad2fe85e54e8d94568aab4318 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Thu, 10 Sep 2026 16:09:00 -0400 Subject: [PATCH 02/17] doc: Improve Waiter/EventLoop lock order documentation Correct the ThreadContext "Synchronization note", which said Waiter::m_mutex must not be locked before EventLoop::m_mutex. That is the reverse of the documented and actual lock order (Waiter::m_mutex first, as ~ProxyServer does). The constraint it was reaching for is the EventLoop blocking rule now documented on Waiter::m_mutex. This change was written with Claude Fable 5.1. --- include/mp/proxy-io.h | 27 +++++++++++++++++++++------ include/mp/type-context.h | 8 ++++---- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index a73c25b5..cca13ee8 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -422,9 +422,23 @@ struct Waiter //! to guard access to related state. Specifically, since the thread_local //! ThreadContext struct owns a Waiter, the Waiter::m_mutex is used to guard //! access to other parts of the struct to avoid needing to deal with more - //! mutexes than necessary. This mutex can be held at the same time as - //! EventLoop::m_mutex as long as Waiter::mutex is locked first and - //! EventLoop::m_mutex is locked second. + //! mutexes than necessary. + //! + //! Lock order: this mutex can be held at the same time as + //! EventLoop::m_mutex as long as Waiter::m_mutex is locked first and + //! EventLoop::m_mutex is locked second. ~ProxyServer locks them + //! in this order on the event loop thread. No code locks them in the + //! reverse order. + //! + //! Blocking rule: a thread other than the event loop thread must not hold + //! this mutex while calling EventLoop::sync() or EventLoop::post(). Those + //! calls block until the event loop thread runs the posted function, and + //! the event loop thread locks Waiter::m_mutex itself (in SetThread and + //! its disconnect callback, Waiter::post, and ~ProxyServer), so + //! holding the mutex across the call could deadlock even though the two + //! mutexes would be locked in the permitted order. This is why + //! ~ThreadContext releases the mutex before destroying ProxyClient + //! objects, whose destructor calls EventLoop::sync(). Mutex m_mutex; std::condition_variable m_cv MP_GUARDED_BY(m_mutex); std::optional> m_fn MP_GUARDED_BY(m_mutex); @@ -768,9 +782,10 @@ struct ThreadContext //! However, individual ProxyClient objects in the maps will only be //! associated with one event loop and guarded by EventLoop::m_mutex. So //! Waiter::m_mutex does not need to be held while accessing individual - //! ProxyClient instances, and may even need to be released to - //! respect lock order and avoid locking Waiter::m_mutex before - //! EventLoop::m_mutex. + //! ProxyClient instances, and must be released before destroying + //! one from a thread other than the event loop thread, because + //! ~ProxyClient calls EventLoop::sync() (see the blocking rule in + //! the Waiter::m_mutex documentation). ConnThreads callback_threads MP_GUARDED_BY(waiter->m_mutex); //! When client is making a request to a server, this is the `thread` diff --git a/include/mp/type-context.h b/include/mp/type-context.h index 7cfc7e79..5f7e05b2 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -179,10 +179,10 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& if (erase_thread) { // Look up the thread again without using existing // iterator since entry may no longer be there after - // a disconnect. Destroy node after releasing - // Waiter::m_mutex, so the ProxyClient - // destructor is able to use EventLoop::mutex - // without violating lock order. + // a disconnect. Destroy the node after releasing + // Waiter::m_mutex, so ~ProxyClient does not + // run with the mutex held (see the SetThread + // disconnect callback). ConnThreads::node_type removed; { Lock lock(thread_context.waiter->m_mutex); From cc1f84e1e0fcbdb4795c698d2630947754cd9b7d Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Thu, 3 Sep 2026 15:30:21 -0400 Subject: [PATCH 03/17] proxy-io: fix listener stuck at capacity after a local disconnect Currently, a ListenConnections listener that reaches its max-connection limit stops accepting new connections permanently if one of its connections is closed locally instead of by a remote disconnect. Closing a connection locally (e.g. erasing it from m_incoming_connections) leaves the listener's active-connection count stuck at the limit, so it never resumes accepting. This happens because the count is decremented by a callback which only fires on a remote disconnects, not local disconnects. Fix by moving the decrement to callback which fires on both local and remote disconnects. Add a regression test that closes a connection locally and checks the listener resumes accepting; it fails before this change (the listener never accepts the waiting client) and passes after. Co-Authored-By: Enoch Azariah This change was written with Claude Opus 4.8 (1M context). --- include/mp/proxy-io.h | 11 +++++++++-- src/mp/proxy.cpp | 3 ++- test/mp/test/listen_tests.cpp | 28 ++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index cca13ee8..22aa4fda 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -905,10 +905,17 @@ void _Serve(EventLoop& loop, kj::Own&& stream, InitImpl& init auto it = loop.m_incoming_connections.begin(); MP_LOG(loop, Log::Info) << "IPC server: socket connected."; if (loop.testing_hook_connected) loop.testing_hook_connected(); - it->onDisconnect([&loop, it, on_disconnect = std::forward(on_disconnect)]() mutable { + // Run on_disconnect (e.g. the listener's active-connection counter + // decrement) on any disconnect. It is registered with addSyncCleanup rather + // than placed in the onDisconnect handler below because that handler + // only fires on a remote disconnect and is canceled when a connection is + // closed locally; if on_disconnect lived there, closing a connection + // locally would leave the listener's slot count stuck and stop it from + // accepting again. + it->addSyncCleanup(std::forward(on_disconnect)); + it->onDisconnect([&loop, it]() mutable { MP_LOG(loop, Log::Info) << "IPC server: socket disconnected."; loop.m_incoming_connections.erase(it); - on_disconnect(); if (loop.testing_hook_disconnected) loop.testing_hook_disconnected(); }); } diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 983d6ce8..5cec2b9d 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -220,7 +220,8 @@ CleanupIt Connection::addSyncCleanup(std::function fn) // order should not be significant because the cleanup callbacks run // synchronously in a single batch when the connection is broken, and they // only reset the connection pointers in the client objects without actually - // deleting the client objects. + // deleting the client objects, or update ListenConnections max_connections + // bookkeeping. return m_sync_cleanup_fns.emplace(m_sync_cleanup_fns.begin(), std::move(fn)); } diff --git a/test/mp/test/listen_tests.cpp b/test/mp/test/listen_tests.cpp index 41fb05aa..810ea677 100644 --- a/test/mp/test/listen_tests.cpp +++ b/test/mp/test/listen_tests.cpp @@ -193,6 +193,34 @@ KJ_TEST("ListenConnections enforces a local connection limit") KJ_EXPECT(client3->client->add(3, 4) == 7); } +KJ_TEST("ListenConnections resumes after a local disconnect") +{ + // A connection closed locally (here by erasing it from the incoming- + // connection list) must still free the listener's slot so the listener + // resumes accepting. The counter decrement runs as an addSyncCleanup cleanup; + // if it were in the onDisconnect handler it would be canceled by the + // local close and the slot would stay stuck. + ListenSetup server(/*max_connections=*/1); + + auto client1 = std::make_unique(server.listener.MakeConnectedSocket()); + server.WaitForConnectedCount(1); + KJ_EXPECT(client1->client->add(1, 2) == 3); + + auto client2 = std::make_unique(server.listener.MakeConnectedSocket()); + (**server.m_loop_ref).sync([] {}); + KJ_EXPECT(server.ConnectedCount() == 1); + + // Close the first connection locally on the event loop thread. + EventLoop& loop{**server.m_loop_ref}; + loop.sync([&] { + KJ_REQUIRE(loop.m_incoming_connections.size() == 1); + loop.m_incoming_connections.pop_front(); + }); + + server.WaitForConnectedCount(2); + KJ_EXPECT(client2->client->add(2, 3) == 5); +} + KJ_TEST("ListenConnections accepts multiple connections") { // With max-connections=2, two clients should be accepted and usable at the From 1570443b558ff91d5e65b47dcc34326cba442b75 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Thu, 3 Sep 2026 16:34:56 -0400 Subject: [PATCH 04/17] proxy-io: fix race deleting a disconnected Connection twice Fix a use-after-free, possible since the destroy_connection option was added in 2019 (c685fa9): a Connection's disconnect handler could run after the Connection had already been destroyed, deleting it a second time and crashing. Reported by enirox001 in https://github.com/bitcoin-core/libmultiprocess/pull/335#discussion_r3821831654 Give each Connection a shared_ptr "alive" token that disconnect handlers hold a weak_ptr to and check before running, so a handler is skipped once its Connection is gone. Having this check also enables the simplifications described below. Previously each Connection kept its disconnect handlers in its own kj::TaskSet, and when the network disconnected it moved a handler onto the shared event loop TaskSet with kj::evalLater. Destroying the Connection destroyed that per-connection TaskSet, canceling a still-pending handler -- but a handler already moved onto the shared TaskSet was no longer canceled and could run after the Connection was gone. (The evalLater step existed only to avoid a "promise callback destroyed itself" error when a handler deletes its own Connection, which the per-connection TaskSet made possible.) With the token doing the cancellation, neither the per-connection TaskSet nor the evalLater step is needed, and both are removed. Co-Authored-By: Enoch Azariah This change was written with Claude Fable 5. --- example/calculator.cpp | 1 - example/printer.cpp | 1 - include/mp/proxy-io.h | 33 ++++++++------- test/mp/test/listen_tests.cpp | 1 + test/mp/test/test.cpp | 80 +++++++++++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 17 deletions(-) diff --git a/example/calculator.cpp b/example/calculator.cpp index 5b9990ea..51d5e587 100644 --- a/example/calculator.cpp +++ b/example/calculator.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include diff --git a/example/printer.cpp b/example/printer.cpp index 2ba3f600..e3d4ce63 100644 --- a/example/printer.cpp +++ b/example/printer.cpp @@ -10,7 +10,6 @@ #include // IWYU pragma: keep #include #include -#include #include #include #include diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index 22aa4fda..c370ab93 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -480,28 +480,31 @@ class Connection CleanupIt addSyncCleanup(std::function fn); void removeSyncCleanup(CleanupIt it); - //! Add a remote disconnect handler, run when the peer closes the - //! connection. The handler is canceled if the connection is disconnected - //! locally first (which destroys m_on_disconnect). + //! Register a handler to run on the event loop thread when the peer + //! disconnects. The handler runs at most once, and only while this + //! Connection is still alive: if the connection is torn down locally before + //! the handler runs, the handler is not called. template void onDisconnect(F&& f) { - // Add the handler to the local TaskSet to ensure it is canceled and - // will never run after the connection object is destroyed. But when the - // handler fires, do not call the function f right away, instead add it - // to the EventLoop TaskSet to avoid "Promise callback destroyed itself" - // error in the typical case where f deletes this Connection object. - m_on_disconnect.add(m_network.onDisconnect().then( - [f = std::forward(f), this]() mutable { m_loop->m_task_set->add(kj::evalLater(kj::mv(f))); })); + // m_network.onDisconnect() fires both on a remote disconnect and on a + // local disconnect (deleting the Connection resets m_rpc_system, which + // drops capnp's last reference to the network and fulfills the + // promise). The m_alive weak_ptr tells the two apart -- it is expired + // only while the Connection is being deleted -- so f is skipped on + // local disconnects. This lets onDisconnect callbacks delete the + // Connection without a double deletion. + m_loop->m_task_set->add(m_network.onDisconnect().then( + [f = std::forward(f), alive = std::weak_ptr(m_alive)]() mutable { + if (!alive.expired()) f(); + })); } EventLoopRef m_loop; kj::Own m_stream; - LoggingErrorHandler m_error_handler{*m_loop}; - //! TaskSet holding the m_network.onDisconnect() handler for remote - //! disconnections. Reset to cancel the handler if the connection is closed - //! locally first by deleting this Connection object. - kj::TaskSet m_on_disconnect{m_error_handler}; + //! Liveness token checked by onDisconnect() callbacks (see there). + //! Could be dropped if Connection lifetime were reference-counted (#336). + std::shared_ptr m_alive{std::make_shared()}; ::capnp::TwoPartyVatNetwork m_network; std::optional<::capnp::RpcSystem<::capnp::rpc::twoparty::VatId>> m_rpc_system; diff --git a/test/mp/test/listen_tests.cpp b/test/mp/test/listen_tests.cpp index 810ea677..8e6b88cd 100644 --- a/test/mp/test/listen_tests.cpp +++ b/test/mp/test/listen_tests.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 377911b7..b21b4f5d 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -736,6 +737,85 @@ KJ_TEST("Async cleanup thread has OS thread name") } #endif // HAVE_PTHREAD_GETNAME_NP +KJ_TEST("onDisconnect handler does not run after the connection is destroyed") +{ + // Regression test for a race condition where an onDisconnect handler + // could fire even after the connection had already been disconnected + // locally. Local disconnects are supposed to preempt onDisconnect + // handlers so they are able to free Connection objects without risking double + // deletions. + // + // To hit the previous race window deterministically without relying on + // socket timing, this test exploits two ordering facts: + // m_network.onDisconnect() hands out branches of a forked promise that fire + // in the order they were added, and kj::evalLater tasks queued during one + // event loop turn run on the next turn in FIFO order. A "destroyer" branch + // is registered before the handler under test, so when the peer disconnects + // the destroyer fires first and queues an evalLater task that destroys the + // Connection ahead of the handler. The handler records whether it ran after + // the Connection was destroyed, reading only test-owned flags (never the + // freed Connection), so a violation is a deterministic wrong result rather + // than a flaky crash. + // + // The Connection must be destroyed from that separate task, not from inside + // an onDisconnect continuation: destroying it there tears down m_network + // while a continuation of m_network's promise is still running, and crashes. + + std::atomic connection_destroyed{false}; + std::atomic handler_ran{false}; + std::atomic handler_ran_after_destroy{false}; + + std::thread thread{[&] { + EventLoop loop("mptest", [](mp::LogMessage log) { + KJ_LOG(INFO, log.level, log.message); + if (log.level == mp::Log::Raise) throw std::runtime_error(log.message); + }); + auto pipe = loop.m_io_context.provider->newTwoWayPipe(); + + // Server-side connection whose onDisconnect handler is under test. + auto server_conn = std::make_unique( + loop, kj::mv(pipe.ends[0]), [&](Connection& connection) { + return ::capnp::Capability::Client(kj::heap>( + std::make_shared(), connection)); + }); + + // Raw peer end of the pipe. Closing it makes the server connection's + // m_network.onDisconnect() resolve, i.e. a real remote disconnect. + kj::Own peer_end = kj::mv(pipe.ends[1]); + + // Destroyer branch, registered BEFORE the handler under test so it + // fires first and its destroy task E_d is queued before the buggy + // bounced handler task E_f. It schedules the destroy on a *separate* + // task, so the connection is not torn down from inside an onDisconnect + // continuation. + loop.m_task_set->add(server_conn->m_network.onDisconnect().then([&] { + loop.m_task_set->add(kj::evalLater([&] { + server_conn.reset(); + connection_destroyed = true; + })); + })); + + // Handler under test. It intentionally does not dereference the + // Connection; it only records whether it ran after the connection was + // destroyed, so the result is deterministic. + server_conn->onDisconnect([&] { + handler_ran = true; + handler_ran_after_destroy = connection_destroyed.load(); + }); + + // Trigger the remote disconnect once the loop is running. + loop.m_task_set->add(kj::evalLater([&] { peer_end = nullptr; })); + + loop.loop(); + }}; + thread.join(); + + // The handler must have run (both versions schedule it), but it must never + // run after the connection was destroyed. + KJ_EXPECT(handler_ran); + KJ_EXPECT(!handler_ran_after_destroy); +} + KJ_TEST("Call async IPC method without thread or pool errors correctly") { TestSetup setup; From 9783fb2eb23d316842eb47bea9a394986304e276 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Fri, 11 Sep 2026 10:22:28 -0400 Subject: [PATCH 05/17] proxy: Fix thread map teardown race causing use-after-free on disconnect Fix a race between a thread exiting after making IPC calls and its connection being destroyed on the event loop thread, which could destroy the same ProxyClient object twice. ~ThreadContext destroyed the thread-local request_threads/callback_threads maps with no locking while the SetThread cleanup callback run by ~Connection erased entries from the same maps. When both ran at once, each side destroyed the entry's ProxyClient, and ~Connection then ran the ProxyClientBase disconnect callback on the freed map node (heap-use-after-free, then a glibc "double free or corruption" abort). Fix by making map entry removal decide which side destroys an entry: ~ThreadContext and the SetThread callback each remove entries under Waiter::m_mutex before destroying them, and a side that finds an entry already gone leaves it to the other. See the code comments for why the entries are destroyed with the mutex released. Add a regression test, "Thread exiting while its connection is destroyed", which uses a new testing_hook_thread_client_destroy hook to interleave the two sides deterministically and fails on every run without the fix. The race is long-standing and reachable on master via connections created by ConnectStream, whose onDisconnect handler deletes the client Connection on the event loop thread when the peer disconnects. This change was written with Claude Fable 5.1. --- include/mp/proxy-io.h | 10 +++++++ src/mp/proxy.cpp | 68 +++++++++++++++++++++++++++++++++---------- test/mp/test/test.cpp | 49 +++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 15 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index c370ab93..ad4ca485 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -367,6 +367,11 @@ class EventLoop //! Hook called on the event loop thread when a client has disconnected. std::function testing_hook_disconnected; + //! Hook called at the start of ~ProxyClient, on whichever thread + //! is destroying the object, with the object being destroyed. Used by + //! tests to control timing during thread map teardown. + std::function*)> testing_hook_thread_client_destroy; + //! Miscellaneous testing hook. Called from various places with an //! argument identifying the call site (typically a string literal), so //! tests can control timing or inject behavior at specific points without @@ -806,6 +811,11 @@ struct ThreadContext //! to assert false if there's an attempt to execute a blocking operation //! which could deadlock the thread. bool loop_thread = false; + + //! Destructor which destroys the thread maps, coordinating with event + //! loop threads that remove entries from them concurrently when + //! connections are broken (see the code comment). + ~ThreadContext(); }; template diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 5cec2b9d..0699fe74 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -397,20 +397,26 @@ std::tuple SetThread(GuardedRef threads, Connecti } if (inserted) { thread->second.emplace(make_thread(), connection, /* destroy_connection= */ false); - thread->second->m_disconnect_cb = connection->addSyncCleanup([threads, thread] { - // Note: it is safe to use the `thread` iterator in this cleanup - // function, because the iterator would only be invalid if the map entry - // was removed, and if the map entry is removed the ProxyClient - // destructor unregisters the cleanup. - - // Connection is being destroyed before thread client is, so reset - // thread client m_disconnect_cb member so thread client destructor does not - // try to unregister this callback after connection is destroyed. - thread->second->m_disconnect_cb.reset(); - - // Remove connection pointer about to be destroyed from the map - const Lock lock(threads.mutex); - threads.ref.erase(thread); + thread->second->m_disconnect_cb = connection->addSyncCleanup([threads, connection] { + // Remove and destroy this connection's map entry, unless the + // thread owning the map is exiting and ~ThreadContext already took + // the entry, in which case that thread destroys it. Look the entry + // up by key rather than capturing the iterator, which would be + // invalid in that case. + ConnThreads::node_type removed; + { + const Lock lock(threads.mutex); + auto it = threads.ref.find(connection); + if (it == threads.ref.end()) return; + // Reset so ~ProxyClient does not try to unregister + // this callback, which is already running. + it->second->m_disconnect_cb.reset(); + removed = threads.ref.extract(it); + } + // The entry is destroyed here with Waiter::m_mutex released, as in + // ~ThreadContext. (Holding it would be safe on the event loop + // thread, where EventLoop::sync() runs directly, but releasing it + // keeps the two consistent.) }); } return {thread, inserted}; @@ -418,6 +424,9 @@ std::tuple SetThread(GuardedRef threads, Connecti ProxyClient::~ProxyClient() { + EventLoop& loop{*m_context.loop}; + if (loop.testing_hook_thread_client_destroy) loop.testing_hook_thread_client_destroy(this); + // If thread is being destroyed before connection is destroyed, remove the // cleanup callback that was registered to handle the connection being // destroyed before the thread being destroyed. @@ -427,13 +436,42 @@ ProxyClient::~ProxyClient() // between this thread trying to remove the callback and the disconnect // handler attempting to call it. m_context.loop->sync([&]() { - if (m_disconnect_cb) { + // Skip if the connection was destroyed while this thread waited + // for the event loop: ~Connection has already run and freed the + // cleanup list m_disconnect_cb points into, and the ProxyClientBase + // disconnect callback has nulled m_context.connection. (The + // SetThread callback resets m_disconnect_cb only when it finds + // this object in the thread map, so if ~ThreadContext took the + // entry first, m_disconnect_cb is still set here.) + if (m_disconnect_cb && m_context.connection) { m_context.connection->removeSyncCleanup(*m_disconnect_cb); } }); } } +ThreadContext::~ThreadContext() +{ + // Server threads created by ProxyServer::makeThread have no + // waiter here: ~ProxyServer moves it away and clears the maps + // before the thread exits. + if (!waiter) return; + + // Take the thread client maps under Waiter::m_mutex, because the SetThread + // disconnect callback removes entries from them concurrently on the event + // loop thread when connections are broken, and whichever side removes an + // entry destroys it. Destroy the maps with the mutex released, because + // ~ProxyClient blocks in EventLoop::sync() and the event loop + // thread locks Waiter::m_mutex itself (see the blocking rule in the + // Waiter::m_mutex documentation). + ConnThreads request, callback; + { + const Lock lock(waiter->m_mutex); + request.swap(request_threads); + callback.swap(callback_threads); + } +} + ProxyServer::ProxyServer(Connection& connection, ThreadContext& thread_context, std::thread&& thread) : m_loop{*connection.m_loop}, m_thread_context(thread_context), m_thread(std::move(thread)) { diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index b21b4f5d..d0327974 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -26,6 +26,7 @@ #include #include #include +#include // IWYU pragma: keep #include #include #include @@ -501,6 +502,54 @@ KJ_TEST("Worker thread destroyed before it is initialized") EXPECT_EXCEPTION(foo->callFnAsync(), "IPC client method call interrupted by disconnect."); } +KJ_TEST("Thread exiting while its connection is destroyed") +{ + // Regression test for a race between a thread exiting after making IPC + // calls and its connection being destroyed on the event loop thread. + // ~ThreadContext on the exiting thread and the SetThread disconnect + // callback run by ~Connection both remove the thread's map entries for + // the connection, and previously nothing synchronized them, so both could + // destroy the same ProxyClient object. + // + // The testing_hook_thread_client_destroy hook, called at the start of + // ~ProxyClient, blocks the exiting thread inside its first map + // entry destructor while the main thread destroys the connection. The + // disconnect callback must leave that entry alone: it resets + // m_disconnect_cb only when it finds the entry in the map and takes over + // destroying it, so the entry's m_disconnect_cb must still be set when + // the exiting thread resumes. + TestSetup setup{/*client_owns_connection=*/false}; + ProxyClient* foo = setup.client.get(); + foo->initThreadMap(); + setup.server->m_impl->m_fn = [] {}; + EventLoop& loop = *foo->m_context.loop; + + std::promise caller_exiting, release_caller; + bool caller_blocked{false}; // caller thread only + bool disconnect_cb_set{false}; // caller thread, read after join + loop.testing_hook_thread_client_destroy = [&](ProxyClient* client) { + // The hook also runs on the event loop thread for entries the + // disconnect callback destroys. Block only the exiting caller thread, + // in the first destructor it runs. + if (std::this_thread::get_id() == loop.m_thread_id || caller_blocked) return; + caller_blocked = true; + caller_exiting.set_value(); + release_caller.get_future().get(); + disconnect_cb_set = client->m_disconnect_cb.has_value(); + }; + + // Make a call taking an mp.Context argument, which adds callback and + // request thread entries for the connection to the caller's thread-local + // ThreadContext maps. They are destroyed when the thread exits. + std::thread caller{[&] { foo->callFnAsync(); }}; + caller_exiting.get_future().get(); + setup.client_disconnect(); + release_caller.set_value(); + caller.join(); + loop.testing_hook_thread_client_destroy = nullptr; + KJ_EXPECT(disconnect_cb_set); +} + KJ_TEST("Calling async IPC method, with server disconnect racing the call") { // Regression test for bitcoin/bitcoin#34777 heap-use-after-free where From 034e0d21b519ef340ae63de7ec30012c7554aefe Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Fri, 11 Sep 2026 14:27:43 -0400 Subject: [PATCH 06/17] proxy-io: add EventLoop::incomingConnections() Add an accessor and a Connections type alias for the EventLoop's list of incoming connections, so future code can be simplified to locate a specific connection without directly accessing the private list or embedding a Connection object itself. This change was written with Claude Sonnet 5. --- include/mp/proxy-io.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index b226284f..c1630b8c 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -290,6 +291,12 @@ class EventLoop //! Check if loop should exit. bool done() const MP_REQUIRES(m_mutex); + //! Type of m_incoming_connections list. + using Connections = std::list; + + //! View of incoming connections yielding Connection& for each entry. + auto incomingConnections() { return std::views::all(m_incoming_connections); } + //! Process name included in thread names so combined debug output from //! multiple processes is easier to understand. const char* m_exe_name; @@ -337,7 +344,7 @@ class EventLoop std::unique_ptr m_task_set; //! List of connections. - std::list m_incoming_connections; + Connections m_incoming_connections; //! Logging options LogOptions m_log_opts; From b8c3dcc2311caf2d1694e3da6b6e6498ed16d79c Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Fri, 31 Jul 2026 15:19:30 -0400 Subject: [PATCH 07/17] proxy-io: add Connection::disconnect() separating teardown from destruction Split connection teardown out of ~Connection into an idempotent disconnect() method, with the destructor delegating to it. For existing callers, this is a behavior-neutral refactor: the same steps run in the same order on destruction. Having a separate disconnect() method allows severing a connection while keeping the Connection object alive, which the next commits use to let shutdown code wait for in-flight server call bodies to finish after a disconnect (bitcoin/bitcoin#35845). Two details are new in the disconnect() method which were not present in the destructor method: - disconnect() expires the m_alive token explicitly, where previously it was expired implicitly by member destruction. This keeps onRemoteDisconnect able to distinguish a local disconnect from a remote one when a connection is severed without destroying the object (see the disconnect() code comment). - disconnect() explicitly releases m_thread_pool and m_thread_map so worker thread teardown happens at disconnect time whether or not the object is destroyed right away. Previously this happened implicitly during member destruction. This change was written with Claude Fable 5. --- include/mp/proxy-io.h | 56 +++++++++++++++++++++++------------ src/mp/proxy.cpp | 68 +++++++++++++++++++++++++++++++++++-------- test/mp/test/test.cpp | 2 +- 3 files changed, 94 insertions(+), 32 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index ad4ca485..5204d275 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -454,27 +454,42 @@ struct Waiter //! on the event loop thread. //! In addition to Cap'n Proto state, it also holds lists of callbacks to run //! when the connection is closed. +//! +//! A Connection may be severed with disconnect() and then kept alive (rather +//! than destroyed) so callers can wait for in-flight server calls to finish. +//! Once disconnect() has run the object holds no transport, RPC system, or +//! worker threads, so the only valid operations on it are destruction and +//! calling disconnect() again (a no-op). Methods that perform I/O or make +//! calls must not be used after disconnect(). class Connection { public: Connection(EventLoop& loop, kj::Own&& stream_) : m_loop(loop), m_stream(kj::mv(stream_)), - m_network(*m_stream, ::capnp::rpc::twoparty::Side::CLIENT, ::capnp::ReaderOptions()), - m_rpc_system(::capnp::makeRpcClient(m_network)) {} + m_network(std::in_place, *m_stream, ::capnp::rpc::twoparty::Side::CLIENT, ::capnp::ReaderOptions()), + m_rpc_system(::capnp::makeRpcClient(*m_network)) {} Connection(EventLoop& loop, kj::Own&& stream_, const std::function<::capnp::Capability::Client(Connection&)>& make_client) : m_loop(loop), m_stream(kj::mv(stream_)), - m_network(*m_stream, ::capnp::rpc::twoparty::Side::SERVER, ::capnp::ReaderOptions()), - m_rpc_system(::capnp::makeRpcServer(m_network, make_client(*this))) {} - - //! Run cleanup functions. Must be called from the event loop thread. First - //! calls synchronous cleanup functions while blocked (to free capnp - //! Capability::Client handles owned by ProxyClient objects), then schedules - //! asynchronous cleanup functions to run in a worker thread (to run - //! destructors of m_impl instances owned by ProxyServer objects). + m_network(std::in_place, *m_stream, ::capnp::rpc::twoparty::Side::SERVER, ::capnp::ReaderOptions()), + m_rpc_system(::capnp::makeRpcServer(*m_network, make_client(*this))) {} + + //! Destroy the connection. Calls disconnect() if it has not been called + //! already. Must be called from the event loop thread. ~Connection() noexcept(false); + //! Sever the connection without destroying this object: close the transport + //! so the peer observes the disconnect, and run the connection's cleanup + //! handlers. Idempotent, and called automatically by the destructor -- + //! calling it directly is only needed to disconnect while keeping the object + //! alive (e.g. to wait for in-flight calls to drain afterward). Must be + //! called from the event loop thread. + //! + //! Cancels the KJ promise of any in-flight call, but a server method body + //! already dispatched to a worker thread runs to completion. + void disconnect(); + //! Register a synchronous cleanup function to run on the event loop thread //! (with access to capnp thread-local variables) when the connection is //! disconnected -- for either a remote disconnect (the peer closes the @@ -492,14 +507,13 @@ class Connection template void onDisconnect(F&& f) { - // m_network.onDisconnect() fires both on a remote disconnect and on a - // local disconnect (deleting the Connection resets m_rpc_system, which - // drops capnp's last reference to the network and fulfills the - // promise). The m_alive weak_ptr tells the two apart -- it is expired - // only while the Connection is being deleted -- so f is skipped on - // local disconnects. This lets onDisconnect callbacks delete the - // Connection without a double deletion. - m_loop->m_task_set->add(m_network.onDisconnect().then( + // m_network->onDisconnect() fires both on a remote disconnect and on a + // local one (disconnecting resets m_rpc_system, which drops capnp's + // last reference to the network and fulfills the promise). The m_alive + // weak_ptr tells the two apart -- disconnect() expires it, so f is + // skipped on local disconnects. This lets onDisconnect callbacks + // delete the Connection without a double deletion. + m_loop->m_task_set->add(m_network->onDisconnect().then( [f = std::forward(f), alive = std::weak_ptr(m_alive)]() mutable { if (!alive.expired()) f(); })); @@ -510,7 +524,11 @@ class Connection //! Liveness token checked by onDisconnect() callbacks (see there). //! Could be dropped if Connection lifetime were reference-counted (#336). std::shared_ptr m_alive{std::make_shared()}; - ::capnp::TwoPartyVatNetwork m_network; + //! Wrapped in std::optional so disconnect() can tear it down (along with + //! the stream) to sever the transport while this object stays alive. + //! Closing the stream is what makes the peer observe the disconnect: it + //! reads EOF and fails its outstanding calls with DISCONNECTED errors. + std::optional<::capnp::TwoPartyVatNetwork> m_network; std::optional<::capnp::RpcSystem<::capnp::rpc::twoparty::VatId>> m_rpc_system; // ThreadMap interface client, used to create a remote server thread when an diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 0699fe74..79655788 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -13,6 +13,7 @@ #include #include #include // IWYU pragma: keep +#include #include #include #include @@ -114,6 +115,28 @@ Connection::~Connection() noexcept(false) // by a TwoPartyVatNetwork::onDisconnect callback directly from the event // loop thread. assert(std::this_thread::get_id() == m_loop->m_thread_id); + disconnect(); +} + +void Connection::disconnect() +{ + // Disconnecting triggers I/O and tears down capnp state, so it must run on + // the event loop thread, like the destructor. + assert(std::this_thread::get_id() == m_loop->m_thread_id); + + // m_network is reset at the end of teardown below, so treat it being null + // as the "already disconnected" state: a second call (including the one + // from the destructor) is a no-op. + if (!m_network) return; + + // Expire m_alive before severing the connection below so onDisconnect + // handlers will not trigger and delete this Connection object. The + // onDisconnect handlers trigger on remote disconnects and automatically + // delete Connection objects. But on local disconnects, they should not + // trigger, because local code that disconnects is responsible for freeing + // Connection objects, and it may want to wait for in-flight calls to finish + // before destroying them. + m_alive.reset(); // Try to cancel any calls that may be executing. m_canceler.cancel("Interrupted by disconnect"); @@ -202,12 +225,32 @@ Connection::~Connection() noexcept(false) // on clean and unclean shutdowns. In unclean shutdown case when the // connection is broken, sync and async cleanup lists will be filled with // callbacks. In the clean shutdown case both lists will be empty. - Lock lock{m_loop->m_mutex}; - while (!m_sync_cleanup_fns.empty()) { - CleanupList fn; - fn.splice(fn.begin(), m_sync_cleanup_fns, m_sync_cleanup_fns.begin()); - Unlock(lock, fn.front()); + { + Lock lock{m_loop->m_mutex}; + while (!m_sync_cleanup_fns.empty()) { + CleanupList fn; + fn.splice(fn.begin(), m_sync_cleanup_fns, m_sync_cleanup_fns.begin()); + Unlock(lock, fn.front()); + } } + + // Release Thread capabilities so idle worker threads are stopped and + // joined at disconnect time, whether or not this object is destroyed right + // away. (A worker thread currently executing a call body is unaffected: + // its ProxyServer object is pinned by the post() call and released + // when the body finishes.) + m_thread_pool.clear(); + m_thread_map = nullptr; + + // Destroy the network and close the stream so the peer observes the + // disconnect, reading EOF and failing its outstanding calls with + // DISCONNECTED errors. This has to be explicit because when disconnect() + // is called without destroying this object, nothing else severs the + // transport: m_rpc_system.reset() above stops reading from the stream but + // does not reliably close it. The network is destroyed first since it + // references the stream. + m_network.reset(); + m_stream = nullptr; } CleanupIt Connection::addSyncCleanup(std::function fn) @@ -436,13 +479,14 @@ ProxyClient::~ProxyClient() // between this thread trying to remove the callback and the disconnect // handler attempting to call it. m_context.loop->sync([&]() { - // Skip if the connection was destroyed while this thread waited - // for the event loop: ~Connection has already run and freed the - // cleanup list m_disconnect_cb points into, and the ProxyClientBase - // disconnect callback has nulled m_context.connection. (The - // SetThread callback resets m_disconnect_cb only when it finds - // this object in the thread map, so if ~ThreadContext took the - // entry first, m_disconnect_cb is still set here.) + // Skip if the connection was disconnected while this thread waited + // for the event loop: Connection::disconnect() has already run and + // freed the cleanup list m_disconnect_cb points into, and the + // ProxyClientBase disconnect callback has nulled + // m_context.connection. (The SetThread callback resets + // m_disconnect_cb only when it finds this object in the thread + // map, so if ~ThreadContext took the entry first, m_disconnect_cb + // is still set here.) if (m_disconnect_cb && m_context.connection) { m_context.connection->removeSyncCleanup(*m_disconnect_cb); } diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index d0327974..f356d065 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -837,7 +837,7 @@ KJ_TEST("onDisconnect handler does not run after the connection is destroyed") // bounced handler task E_f. It schedules the destroy on a *separate* // task, so the connection is not torn down from inside an onDisconnect // continuation. - loop.m_task_set->add(server_conn->m_network.onDisconnect().then([&] { + loop.m_task_set->add(server_conn->m_network->onDisconnect().then([&] { loop.m_task_set->add(kj::evalLater([&] { server_conn.reset(); connection_destroyed = true; From 01808baa1886e4705a39e1df1d8ef05ebe35370b Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Fri, 11 Sep 2026 14:28:51 -0400 Subject: [PATCH 08/17] proxy-io: let ServeStream take ownership of init object Add a _Serve/ServeStream overload accepting the init object as a shared_ptr, so callers can transfer ownership instead of always passing a reference to an object they keep alive themselves. Existing reference-taking callers keep working through a thin overload that wraps the reference in a shared_ptr with an empty deleter. Also return the constructed ProxyServer along with an iterator to its Connection in loop.m_incoming_connections, so callers can look up or erase the connection later without embedding a Connection object themselves. This change was written with Claude Sonnet 5. --- include/mp/proxy-io.h | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index c1630b8c..31d1f041 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -878,15 +878,18 @@ std::unique_ptr> ConnectStream(EventLoop& loop, Strea //! Given stream and init objects, construct a new ProxyServer object that //! handles requests from the stream by calling the init object. Embed the //! ProxyServer in a Connection object that is stored and erased if -//! disconnected. This should be called from the event loop thread. +//! disconnected. This should be called from the event loop thread. Returns +//! the new ProxyServer along with an iterator to its Connection in +//! loop.m_incoming_connections. template -void _Serve(EventLoop& loop, kj::Own&& stream, InitImpl& init, OnDisconnect&& on_disconnect) +std::pair*, EventLoop::Connections::iterator> _Serve( + EventLoop& loop, kj::Own&& stream, std::shared_ptr init, OnDisconnect&& on_disconnect) { + ProxyServer* server = nullptr; loop.m_incoming_connections.emplace_front(loop, kj::mv(stream), [&](Connection& connection) { - // Disable deleter so proxy server object doesn't attempt to delete the - // init implementation when the proxy client is destroyed or - // disconnected. - return kj::heap>(std::shared_ptr(&init, [](InitImpl*){}), connection); + auto proxy_server = kj::heap>(std::move(init), connection); + server = proxy_server; + return capnp::Capability::Client(kj::mv(proxy_server)); }); auto it = loop.m_incoming_connections.begin(); MP_LOG(loop, Log::Info) << "IPC server: socket connected."; @@ -897,6 +900,24 @@ void _Serve(EventLoop& loop, kj::Own&& stream, InitImpl& init on_disconnect(); if (loop.testing_hook_disconnected) loop.testing_hook_disconnected(); }); + return {server, it}; +} + +//! Overload of _Serve that takes a reference to the init object instead of a +//! shared_ptr. ProxyServer objects use shared_ptr's internally to optionally +//! take ownership of interfaces being served, and free them when clients are +//! no longer using them. Some libmultiprocess callers take advantage of this +//! and pass shared_ptr's directly. But other callers that do not give away +//! ownership are not required to use shared_ptr, so this overload converts +//! references they pass into shared_ptr's with empty deleters. This prevents +//! the ProxyServer object from deleting the init object when the client is +//! disconnected. +template +std::pair*, EventLoop::Connections::iterator> _Serve( + EventLoop& loop, kj::Own&& stream, InitImpl& init, OnDisconnect&& on_disconnect) +{ + return _Serve( + loop, kj::mv(stream), std::shared_ptr(&init, [](InitImpl*){}), std::forward(on_disconnect)); } struct Listener @@ -934,11 +955,12 @@ void _Listen(const std::shared_ptr& listener, EventLoop& loop, InitImp } //! Given a stream and an init object, handle requests on the stream by calling -//! methods on the Init object. +//! methods on the Init object. See _Serve for details on the return value. template -void ServeStream(EventLoop& loop, Stream stream, InitImpl& init) +std::pair*, EventLoop::Connections::iterator> ServeStream( + EventLoop& loop, Stream stream, InitImpl&& init) { - _Serve(loop, kj::mv(stream), init, [] {}); + return _Serve(loop, kj::mv(stream), std::forward(init), /*on_disconnect=*/ [] {}); } //! Given listening socket identifier and an init object, handle incoming From 640a07bcfe988ceb057295569d5c91f9db3bf187 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Fri, 31 Jul 2026 15:21:48 -0400 Subject: [PATCH 09/17] proxy-io: add Connection::waitDrained() to wait for in-flight server calls Add a per-connection ServerObjectTracker counting live ProxyServer objects, incremented in the ProxyServerBase constructor and decremented in its destructor, with Connection::waitDrained() blocking until the count reaches zero and Connection::pendingServerObjects() exposing it for logging. Disconnecting a connection cancels the KJ promise of an in-flight call, but a C++ server method body already dispatched to a worker thread runs to completion. Counting live server objects turns Cap'n Proto's object lifetime rules into a usable quiescence signal: a ProxyServer object is not destroyed until its outstanding calls finish (the target capability is kept alive for the duration of a call and pinned by post()/PassField via thisCap()), so after disconnect() the count drains to zero exactly when no server call body is still executing. Waiting for that lets shutdown code avoid freeing application state that a still-running call body dereferences (bitcoin/bitcoin#35845). The tracker is held via shared_ptr by the Connection and by every ProxyServer object because objects kept alive by in-flight calls can outlive the Connection on some teardown paths (see ~ProxyServerBase), and their destructors must decrement state that is still valid. It must be declared before m_rpc_system, whose construction creates the bootstrap server object that registers itself with the tracker. This change was written with Claude Fable 5. --- include/mp/proxy-io.h | 110 +++++++++++++++++++++++++++++++++++++++++- include/mp/proxy.h | 7 +++ 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index 5204d275..7fe00eb5 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -449,6 +449,81 @@ struct Waiter std::optional> m_fn MP_GUARDED_BY(m_mutex); }; +//! Counter tracking the number of live ProxyServer objects associated with a +//! Connection, used to wait for a disconnected connection's server side to +//! become quiescent. +//! +//! This count is non-zero as long as the client is holding any object handles +//! or as long as any server methods are running, even after the client as +//! disconnected or freed all its handles. When this is zero it means no more +//! ProxyServer method calls can originate from the connection and it is safe to +//! free any resources accessed through the connection. +// +//! Why counting live server objects is a valid "no server call body running" +//! signal: a ProxyServer object is reference counted and is not destroyed +//! until its outstanding calls finish. Cap'n Proto keeps the target capability +//! alive for the duration of a call, and the mp.Context PassField overload and +//! ProxyServer::post() additionally pin it (self = thisCap()) until +//! the call body running on a worker thread completes and its result is +//! delivered. So "object destroyed" implies "its call bodies finished", and a +//! connection whose live-object count reached zero after a disconnect has no +//! server code running. This matters because disconnecting only cancels the +//! KJ promise of an in-flight call; it does not interrupt a call body that +//! was already dispatched to a worker thread (see Connection::disconnect). +//! +//! The counter is held via shared_ptr by the Connection and by every +//! ProxyServer object created for the connection, because a ProxyServer +//! object kept alive by an in-flight call can outlive the Connection (see +//! ~ProxyServerBase), and its destructor must decrement state that is still +//! valid. +//! +//! ProxyServer and ProxyServer are separate +//! specializations (not ProxyServerBase instances) and are intentionally not +//! counted: every application method body runs on an interface ProxyServer, +//! which is counted and stays alive for the duration of the body, so counting +//! those is sufficient. +struct ServerObjectTracker +{ + //! Called from the ProxyServerBase constructor (on the event loop thread). + void addServerObject() + { + const Lock lock(m_mutex); + m_count += 1; + } + + //! Called from the ProxyServerBase destructor (on the event loop thread). + void removeServerObject() + { + { + const Lock lock(m_mutex); + assert(m_count > 0); + m_count -= 1; + } + m_cv.notify_all(); + } + + //! Return the current count. May be called from any thread. + size_t pendingServerObjects() const + { + const Lock lock(m_mutex); + return m_count; + } + + //! Block until no server objects remain. Must NOT be called from the event + //! loop thread: in-flight call bodies need the event loop to deliver their + //! results before their server objects are destroyed, so blocking the loop + //! here would deadlock. + void waitDrained() + { + Lock lock(m_mutex); + m_cv.wait(lock.m_lock, [this]() MP_REQUIRES(m_mutex) { return m_count == 0; }); + } + + mutable Mutex m_mutex; + std::condition_variable m_cv; + size_t m_count MP_GUARDED_BY(m_mutex){0}; +}; + //! Object holding network & rpc state associated with either an incoming server //! connection, or an outgoing client connection. It must be created and destroyed //! on the event loop thread. @@ -490,6 +565,12 @@ class Connection //! already dispatched to a worker thread runs to completion. void disconnect(); + //! Return server object tracker allowing clients to block until no + //! ProxyServer objects associated with this connection remain, i.e. until + //! no server call body is still executing (see ServerObjectTracker). + using Tracker = std::shared_ptr; + Tracker tracker() { return m_server_objects; } + //! Register a synchronous cleanup function to run on the event loop thread //! (with access to capnp thread-local variables) when the connection is //! disconnected -- for either a remote disconnect (the peer closes the @@ -529,6 +610,18 @@ class Connection //! Closing the stream is what makes the peer observe the disconnect: it //! reads EOF and fails its outstanding calls with DISCONNECTED errors. std::optional<::capnp::TwoPartyVatNetwork> m_network; + + //! Tracker for live ProxyServer objects associated with this connection, + //! used by waitDrained(). Held via shared_ptr because ProxyServer objects + //! kept alive by in-flight calls can outlive the Connection (see + //! ServerObjectTracker and ~ProxyServerBase). + //! + //! Must be declared before m_rpc_system: constructing m_rpc_system runs + //! the make_client callback, which creates the bootstrap (Init) server + //! object, whose ProxyServerBase constructor registers itself with this + //! tracker. + std::shared_ptr m_server_objects{std::make_shared()}; + std::optional<::capnp::RpcSystem<::capnp::rpc::twoparty::VatId>> m_rpc_system; // ThreadMap interface client, used to create a remote server thread when an @@ -670,8 +763,14 @@ ProxyClientBase::~ProxyClientBase() noexcept template ProxyServerBase::ProxyServerBase(std::shared_ptr impl, Connection& connection) - : m_impl(std::move(impl)), m_context(&connection) + : m_impl(std::move(impl)), m_context(&connection), m_server_objects(connection.m_server_objects) { + // Register this object with the connection's live-object tracker. This + // runs on the event loop thread, so it is ordered before any connection + // teardown (which also runs on the event loop thread): code that + // disconnects the connection and then calls Connection::waitDrained() is + // guaranteed to see this object. + m_server_objects->addServerObject(); MP_LOG(*m_context.loop, Log::Debug) << "Creating " << CxxTypeName(*this) << " " << this; assert(m_impl); } @@ -719,6 +818,15 @@ ProxyServerBase::~ProxyServerBase() } assert(m_context.cleanup_fns.empty()); MP_LOG(*m_context.loop, Log::Debug) << "Destroying " << CxxTypeName(*this) << " " << this; + // Deregister this object from the connection's live-object tracker, + // through the shared m_server_objects handle since m_context.connection + // may be dangling here (see comment above). Done at the end of the + // destructor so a zero count means destruction fully completed. Note that + // any m_impl destruction scheduled through addAsyncCleanup above is NOT + // covered by the tracker: it runs later on the async cleanup thread, so + // Connection::waitDrained() waits for server call bodies, not for + // m_impl destructors. + m_server_objects->removeServerObject(); } //! If the capnp interface defined a special "destroy" method, as described the diff --git a/include/mp/proxy.h b/include/mp/proxy.h index 2144b571..b02abfd8 100644 --- a/include/mp/proxy.h +++ b/include/mp/proxy.h @@ -20,6 +20,7 @@ namespace mp { class Connection; class EventLoop; +struct ServerObjectTracker; //! Mapping from capnp interface type to proxy client implementation (specializations are generated by //! proxy-codegen.cpp). template struct ProxyClient; // IWYU pragma: export @@ -172,6 +173,12 @@ struct ProxyServerBase : public virtual Interface_::Server * wrapped. */ std::shared_ptr m_impl; ProxyContext m_context; + //! Live-object tracker shared with this object's Connection, incremented + //! in the constructor and decremented in the destructor so shutdown code + //! can wait for a disconnected connection's server objects to drain. Held + //! via shared_ptr so it remains valid if this object (kept alive by an + //! in-flight call) outlives the Connection. See ServerObjectTracker. + std::shared_ptr m_server_objects; }; //! Customizable (through template specialization) base class which ProxyServer From e46876545b4d3f98e0b223180d4c06ff8474810d Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Fri, 11 Sep 2026 14:30:15 -0400 Subject: [PATCH 10/17] proxy-io: add destroy_connection parameter to ServeStream and ConnectStream Give ServeStream and ConnectStream a destroy_connection parameter, defaulting to true, so callers can opt out of automatic connection teardown and manage the Connection's lifetime themselves instead. ServeStream gates the internal disconnect handler's list erase on the parameter; ConnectStream just forwards it to the existing ProxyClientBase parameter of the same name. This lets callers that need to keep a connection alive past a disconnect notification (e.g. to let in-flight server calls finish) use these helpers instead of constructing a Connection manually. This change was written with Claude Sonnet 5. --- include/mp/proxy-io.h | 52 ++++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index 31d1f041..dadf6612 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -862,8 +862,13 @@ kj::Promise ProxyServer::post(Fn&& fn) //! If the init interface declares a construct() method, creating the client //! calls it, so this function may block making an IPC call and may throw if //! the call fails. +//! +//! If destroy_connection is false, the returned client does not take +//! ownership of the connection, and the caller is responsible for +//! disconnecting and freeing it (accessible as client->m_context.connection) +//! whenever it is done with it. template -std::unique_ptr> ConnectStream(EventLoop& loop, Stream stream) +std::unique_ptr> ConnectStream(EventLoop& loop, Stream stream, bool destroy_connection = true) { typename InitInterface::Client init_client(nullptr); std::unique_ptr connection; @@ -871,19 +876,26 @@ std::unique_ptr> ConnectStream(EventLoop& loop, Strea connection = std::make_unique(loop, kj::mv(stream)); init_client = connection->m_rpc_system->bootstrap(ServerVatId().vat_id).castAs(); }); - return std::make_unique>( - kj::mv(init_client), connection.release(), /* destroy_connection= */ true); + return std::make_unique>(kj::mv(init_client), connection.release(), destroy_connection); } //! Given stream and init objects, construct a new ProxyServer object that //! handles requests from the stream by calling the init object. Embed the -//! ProxyServer in a Connection object that is stored and erased if -//! disconnected. This should be called from the event loop thread. Returns -//! the new ProxyServer along with an iterator to its Connection in -//! loop.m_incoming_connections. +//! ProxyServer in a Connection object that is stored in +//! loop.m_incoming_connections. This should be called from the event loop +//! thread. Returns the new ProxyServer along with an iterator to its +//! Connection. +//! +//! If destroy_connection is false, the connection is not erased +//! automatically when the peer disconnects, and the caller is responsible +//! for erasing loop.m_incoming_connections at the returned iterator +//! whenever it is done with the connection. template -std::pair*, EventLoop::Connections::iterator> _Serve( - EventLoop& loop, kj::Own&& stream, std::shared_ptr init, OnDisconnect&& on_disconnect) +std::pair*, EventLoop::Connections::iterator> _Serve(EventLoop& loop, + kj::Own&& stream, + std::shared_ptr init, + OnDisconnect&& on_disconnect, + bool destroy_connection = true) { ProxyServer* server = nullptr; loop.m_incoming_connections.emplace_front(loop, kj::mv(stream), [&](Connection& connection) { @@ -894,9 +906,9 @@ std::pair*, EventLoop::Connections::iterator> _Serve( auto it = loop.m_incoming_connections.begin(); MP_LOG(loop, Log::Info) << "IPC server: socket connected."; if (loop.testing_hook_connected) loop.testing_hook_connected(); - it->onDisconnect([&loop, it, on_disconnect = std::forward(on_disconnect)]() mutable { + it->onDisconnect([&loop, it, on_disconnect = std::forward(on_disconnect), destroy_connection]() mutable { MP_LOG(loop, Log::Info) << "IPC server: socket disconnected."; - loop.m_incoming_connections.erase(it); + if (destroy_connection) loop.m_incoming_connections.erase(it); on_disconnect(); if (loop.testing_hook_disconnected) loop.testing_hook_disconnected(); }); @@ -913,11 +925,14 @@ std::pair*, EventLoop::Connections::iterator> _Serve( //! the ProxyServer object from deleting the init object when the client is //! disconnected. template -std::pair*, EventLoop::Connections::iterator> _Serve( - EventLoop& loop, kj::Own&& stream, InitImpl& init, OnDisconnect&& on_disconnect) +std::pair*, EventLoop::Connections::iterator> _Serve(EventLoop& loop, + kj::Own&& stream, + InitImpl& init, + OnDisconnect&& on_disconnect, + bool destroy_connection = true) { - return _Serve( - loop, kj::mv(stream), std::shared_ptr(&init, [](InitImpl*){}), std::forward(on_disconnect)); + return _Serve(loop, kj::mv(stream), std::shared_ptr(&init, [](InitImpl*){}), + std::forward(on_disconnect), destroy_connection); } struct Listener @@ -955,12 +970,13 @@ void _Listen(const std::shared_ptr& listener, EventLoop& loop, InitImp } //! Given a stream and an init object, handle requests on the stream by calling -//! methods on the Init object. See _Serve for details on the return value. +//! methods on the Init object. See _Serve for details on the return value and +//! the destroy_connection parameter. template std::pair*, EventLoop::Connections::iterator> ServeStream( - EventLoop& loop, Stream stream, InitImpl&& init) + EventLoop& loop, Stream stream, InitImpl&& init, bool destroy_connection = true) { - return _Serve(loop, kj::mv(stream), std::forward(init), /*on_disconnect=*/ [] {}); + return _Serve(loop, kj::mv(stream), std::forward(init), /*on_disconnect=*/ [] {}, destroy_connection); } //! Given listening socket identifier and an init object, handle incoming From abdcf054296b7b589cb49b4082a68b2713a9a8d3 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Fri, 31 Jul 2026 15:40:26 -0400 Subject: [PATCH 11/17] test: cover draining in-flight server call after disconnect Add a deterministic mptest regression test for bitcoin/bitcoin#35845: hold a server method body in flight on a worker thread, call Connection::disconnect(), and assert that Connection::waitDrained() blocks until the body finishes and its server object is destroyed. Also covers destroying an already-disconnected connection (~Connection noticing disconnect() has run). This change was written with Claude Fable 5. Co-Authored-By: Enoch Azariah --- include/mp/proxy-io.h | 2 ++ test/mp/test/test.cpp | 75 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index 7fe00eb5..616c1c49 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -516,12 +516,14 @@ struct ServerObjectTracker void waitDrained() { Lock lock(m_mutex); + if (m_count != 0 && testing_hook_wait) testing_hook_wait(); m_cv.wait(lock.m_lock, [this]() MP_REQUIRES(m_mutex) { return m_count == 0; }); } mutable Mutex m_mutex; std::condition_variable m_cv; size_t m_count MP_GUARDED_BY(m_mutex){0}; + std::function testing_hook_wait; }; //! Object holding network & rpc state associated with either an incoming server diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index f356d065..9e1eef7c 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -599,6 +599,81 @@ KJ_TEST("Calling async IPC method, with server disconnect after cleanup") EXPECT_EXCEPTION(foo->callFnAsync(), "IPC client method call interrupted by disconnect."); } +KJ_TEST("Waiting for in-flight server call to finish after disconnect") +{ + // Regression test for bitcoin/bitcoin#35845. Disconnecting a connection + // cancels the KJ promise of an in-flight call, but a C++ server method + // body already dispatched to a worker thread runs to completion. Verify + // that Connection::waitDrained() blocks until such a body finishes and its + // server object is destroyed, so shutdown code can wait for a disconnected + // connection to become quiescent before freeing state the body accesses. + + std::promise body_started, release_body; + TestSetup setup; + ProxyClient* foo = setup.client.get(); + foo->initThreadMap(); + + // A server call body that signals when it starts and then blocks until the + // test releases it, so the in-flight state can be observed + // deterministically. + setup.server->m_impl->m_fn = [&] { + body_started.set_value(); + release_body.get_future().get(); + }; + + // Grab the server Connection object on the event loop thread before + // disconnecting. It stays valid until server_disconnect() destroys it + // below. + Connection* connection{nullptr}; + foo->m_context.loop->sync([&] { connection = setup.server->m_context.connection; }); + + // Invoke the async method on a separate thread so its body blocks there + // while this thread makes assertions. callFnAsync() takes an mp.Context, + // so its body runs on a worker thread via ProxyServer::post(). + std::thread call_thread([&] { + EXPECT_EXCEPTION(foo->callFnAsync(), "IPC client method call interrupted by disconnect."); + }); + body_started.get_future().get(); + + // The FooInterface server object is the connection's only counted server + // object, and its call body is executing. + KJ_EXPECT(connection->tracker()->pendingServerObjects() == 1); + + // Disconnect. This cancels the call's promise (the client above sees the + // disconnect error), but the body is still blocked on the worker thread, + // so its server object must still be alive. + foo->m_context.loop->sync([&] { connection->disconnect(); }); + KJ_EXPECT(connection->tracker()->pendingServerObjects() == 1); + + // A drain must block while the body runs and return only once it + // finishes, which is what Ipc::disconnectIncoming relies on during + // shutdown. + std::promise drain_waiting; + connection->tracker()->testing_hook_wait = [&] { drain_waiting.set_value(); }; + std::atomic drained{false}; + std::thread drain_thread([&] { + connection->tracker()->waitDrained(); + drained = true; + }); + + // Wait until waitDrained() has observed the live server object and is + // about to block, then verify it does not return while the body is blocked. + drain_waiting.get_future().get(); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + KJ_EXPECT(!drained); + + // Let the body finish; the drain should now complete. + release_body.set_value(); + drain_thread.join(); + KJ_EXPECT(drained); + KJ_EXPECT(connection->tracker()->pendingServerObjects() == 0); + call_thread.join(); + + // Destroy the drained connection. (~Connection notices disconnect() has + // already run and does not tear things down twice.) + setup.server_disconnect(); +} + KJ_TEST("Destroying ProxyClient<> with destroy method after peer disconnect") { // Regression test for bitcoin-core/libmultiprocess#219 where From 326ccfc2f87872f8f48e80f52cba7b53ee0f7d6a Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Fri, 11 Sep 2026 14:35:27 -0400 Subject: [PATCH 12/17] test: simplify TestSetup using ServeStream/ConnectStream Replace TestSetup's manual Connection construction with ServeStream/ConnectStream, following the same pattern already used in Bitcoin Core's own IPC test and fuzz code. This drops server_on_disconnect entirely: ServeStream's destroy_connection parameter now controls whether a remote disconnect erases the server Connection, so the only test that needed to suppress that (the mp#348 getResults race test) just constructs TestSetup with server_owns_connection=false instead of overriding a callback afterward. This change was written with Claude Sonnet 5. --- test/mp/test/test.cpp | 73 ++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 43 deletions(-) diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 0492c9ee..3f5f6634 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -25,6 +24,7 @@ #include #include #include +#include // IWYU pragma: keep #include #include #include @@ -75,13 +75,18 @@ static_assert(std::is_integral_v, "MP_MINOR_VERSION * this to be true to simplify shutdown and avoid needing to call * client_disconnect manually, but false allows testing more ProxyClient * behavior and the "IPC client method called after disconnect" code path. + * + * Also accepts an analogous server_owns_connection option controlling + * whether the server Connection is erased automatically when the peer + * disconnects. False is needed by tests that need to keep the server + * Connection alive across a disconnect notification, tearing it down + * explicitly later via server_disconnect() instead. */ class TestSetup { public: std::function server_disconnect; std::function server_disconnect_later; - std::function server_on_disconnect; std::function client_disconnect; std::promise>> client_promise; std::unique_ptr> client; @@ -90,7 +95,7 @@ class TestSetup //! not start until the other members are initialized. std::thread thread; - TestSetup(bool client_owns_connection = true) + TestSetup(bool client_owns_connection = true, bool server_owns_connection = true) : thread{[&] { EventLoop loop("mptest", [](mp::LogMessage log) { // Info logs are not printed by default, but will be shown with `mptest --verbose` @@ -99,36 +104,21 @@ class TestSetup }); auto pipe = loop.m_io_context.provider->newTwoWayPipe(); - auto server_connection = - std::make_unique(loop, kj::mv(pipe.ends[0]), [&](Connection& connection) { - auto server_proxy = kj::heap>( - std::make_shared(), connection); - server = server_proxy; - return capnp::Capability::Client(kj::mv(server_proxy)); - }); - server_disconnect = [&] { loop.sync([&] { server_connection.reset(); }); }; + auto server_result = ServeStream( + loop, kj::mv(pipe.ends[0]), std::make_shared(), server_owns_connection); + server = server_result.first; + EventLoop::Connections::iterator server_it = server_result.second; + server_disconnect = [&] { loop.sync([&] { loop.m_incoming_connections.erase(server_it); }); }; server_disconnect_later = [&] { assert(std::this_thread::get_id() == loop.m_thread_id); - loop.m_task_set->add(kj::evalLater([&] { server_connection.reset(); })); + loop.m_task_set->add(kj::evalLater([&] { loop.m_incoming_connections.erase(server_it); })); }; - // Set handler to destroy the server when the client disconnects. This - // is ignored if server_disconnect() is called instead. Tests can - // assign server_on_disconnect to override the default behavior of - // destroying the server connection as soon as the disconnect is - // detected (in which case they need to destroy it themselves, - // e.g. by calling server_disconnect(), so the event loop can - // exit). - server_on_disconnect = [&] { server_connection.reset(); }; - server_connection->onDisconnect([&] { server_on_disconnect(); }); - - auto client_connection = std::make_unique(loop, kj::mv(pipe.ends[1])); - auto client_proxy = std::make_unique>( - client_connection->m_rpc_system->bootstrap(ServerVatId().vat_id).castAs(), - client_connection.get(), /* destroy_connection= */ client_owns_connection); - if (client_owns_connection) { - (void)client_connection.release(); - } else { - client_disconnect = [&] { loop.sync([&] { client_connection.reset(); }); }; + + auto client_proxy = + ConnectStream(loop, kj::mv(pipe.ends[1]), client_owns_connection); + if (!client_owns_connection) { + Connection* client_connection = client_proxy->m_context.connection; + client_disconnect = [&loop, client_connection] { loop.sync([&] { delete client_connection; }); }; } client_promise.set_value(std::move(client_proxy)); @@ -415,23 +405,20 @@ KJ_TEST("Calling async IPC method with a remote disconnect while results are bui // per-granule shadow history before a late reader comes along. // // The server Connection object is deliberately kept alive during all this - // by overriding server_on_disconnect: destroying it would cancel the - // in-flight request (Connection::~Connection calls m_canceler.cancel(), - // setting request_canceled) and the worker would throw InterruptException - // instead of proceeding into getResults(). Keeping it alive matches the - // window in the original report, where the worker races with capnp's own - // internal teardown, which runs before any onDisconnect notification. - - TestSetup setup{/*client_owns_connection=*/false}; + // by constructing TestSetup with server_owns_connection=false: destroying + // it would cancel the in-flight request (Connection::~Connection calls + // m_canceler.cancel(), setting request_canceled) and the worker would + // throw InterruptException instead of proceeding into getResults(). + // Keeping it alive matches the window in the original report, where the + // worker races with capnp's own internal teardown, which runs before any + // onDisconnect notification. The connection is destroyed explicitly at the + // end of the test instead. + + TestSetup setup{/*client_owns_connection=*/false, /*server_owns_connection=*/false}; ProxyClient* foo = setup.client.get(); KJ_EXPECT(foo->add(1, 2) == 3); foo->initThreadMap(); - // Keep the server Connection object alive when the disconnect is detected - // so the in-flight request is not canceled (see comment above). The - // connection is destroyed at the end of the test instead. - setup.server_on_disconnect = [] {}; - // Signaled by the worker thread when the method body runs, just before it // returns and the worker calls getResults() and serializes the results. std::promise fn_called; From ef84b200924aa8e0c97942856142a22b3f58bbbe Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Fri, 11 Sep 2026 11:55:10 -0400 Subject: [PATCH 13/17] proxy-io: manage Connection lifetime with shared_ptr Make Connection objects shared_ptr-owned, created via a new Connection::make() factory, and have every proxy object share ownership of its connection (ProxyContext::connection becomes a shared_ptr, populated via enable_shared_from_this). A Connection now always outlives its proxy objects and survives disconnect() as an inert husk until its last reference drops. Sharing ownership removes two workarounds that existed only because a Connection could previously be destroyed while proxy objects still referenced it: - ~ProxyServerBase no longer has to avoid dereferencing m_context.connection; the connection is guaranteed to still exist. - The separate m_alive liveness token is dropped. The deferred disconnect handler (renamed onRemoteDisconnect -> afterDisconnect, since it fires on any disconnect and not only remote ones) instead runs off a weak_ptr to the Connection and passes each handler a Connection* -- the live connection, or null if it was already destroyed before the handler ran. Call sites branch on that explicitly rather than the framework silently skipping them; handler bodies are idempotent disconnect() calls, not deletions. Two supporting changes fall out of this and are documented at the code rather than repeated here: connection construction is split into make() and serve() (see their comments), and because the shared references form a cycle that reference-dropping can't break, all teardown now routes through disconnect() -- so every teardown path calls it (see the note on Connection::make). The _Serve disconnect handler's list bookkeeping changed to match (see the comment there). Also drop the test that checked a queued disconnect handler does not run after its Connection is destroyed. It reproduced the m_alive race by constructing a Connection directly and freeing it while the handler was still queued. Both premises are gone: a Connection can only be built through make() now, and handler bodies no longer delete anything, so the double deletion the test guarded against is structurally impossible. afterDisconnect now hands such a handler a null Connection* instead, which its callers handle explicitly, and the ordinary disconnect and drain tests exercise that path. This change was written with Claude Fable 5. --- include/mp/proxy-io.h | 270 ++++++++++++++++++++++------------ include/mp/proxy-types.h | 2 +- include/mp/proxy.h | 6 +- include/mp/type-context.h | 6 +- src/mp/proxy.cpp | 19 +-- test/mp/test/listen_tests.cpp | 16 +- test/mp/test/test.cpp | 130 +++++----------- 7 files changed, 239 insertions(+), 210 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index 4be1de93..4473b3c2 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -292,10 +292,13 @@ class EventLoop bool done() const MP_REQUIRES(m_mutex); //! Type of m_incoming_connections list. - using Connections = std::list; + using Connections = std::list>; //! View of incoming connections yielding Connection& for each entry. - auto incomingConnections() { return std::views::all(m_incoming_connections); } + auto incomingConnections() + { + return m_incoming_connections | std::views::transform([](auto& ptr) -> Connection& { return *ptr; }); + } //! Process name included in thread names so combined debug output from //! multiple processes is easier to understand. @@ -343,7 +346,10 @@ class EventLoop //! Capnp list of pending promises. std::unique_ptr m_task_set; - //! List of connections. + //! List of connections. Holds one shared_ptr reference per connection; + //! proxy objects created for a connection hold additional references (see + //! ProxyContext::connection), so erasing a connection from this list does + //! not necessarily destroy it. Connections m_incoming_connections; //! Logging options @@ -534,8 +540,12 @@ struct ServerObjectTracker }; //! Object holding network & rpc state associated with either an incoming server -//! connection, or an outgoing client connection. It must be created and destroyed -//! on the event loop thread. +//! connection, or an outgoing client connection. It must be created on the +//! event loop thread with the make() factory function, which returns a +//! shared_ptr owner whose custom deleter destroys the object on the event loop +//! thread. Proxy objects created for the connection share ownership of it (see +//! ProxyContext::connection), so the Connection is guaranteed to outlive them +//! and stays valid -- as a disconnected husk -- even after disconnect(). //! In addition to Cap'n Proto state, it also holds lists of callbacks to run //! when the connection is closed. //! @@ -545,19 +555,46 @@ struct ServerObjectTracker //! worker threads, so the only valid operations on it are destruction and //! calling disconnect() again (a no-op). Methods that perform I/O or make //! calls must not be used after disconnect(). -class Connection +class Connection : public std::enable_shared_from_this { public: - Connection(EventLoop& loop, kj::Own&& stream_) - : m_loop(loop), m_stream(kj::mv(stream_)), - m_network(std::in_place, *m_stream, ::capnp::rpc::twoparty::Side::CLIENT, ::capnp::ReaderOptions()), - m_rpc_system(::capnp::makeRpcClient(*m_network)) {} - Connection(EventLoop& loop, - kj::Own&& stream_, - const std::function<::capnp::Capability::Client(Connection&)>& make_client) - : m_loop(loop), m_stream(kj::mv(stream_)), - m_network(std::in_place, *m_stream, ::capnp::rpc::twoparty::Side::SERVER, ::capnp::ReaderOptions()), - m_rpc_system(::capnp::makeRpcServer(*m_network, make_client(*this))) {} + //! Create a Connection with shared ownership. Connection objects must be + //! owned by shared_ptr because proxy objects created for the connection + //! take shared ownership of it in their ProxyContext (via + //! shared_from_this), which requires an existing shared_ptr owner. The + //! custom deleter runs the destructor on the event loop thread (running + //! it directly if the last reference is dropped on the event loop thread, + //! posting to the loop otherwise), which ~Connection requires. The + //! connection's own EventLoopRef keeps the loop running until the deleter + //! has run. + //! + //! Note: dropping shared_ptr references alone will not usually destroy a + //! connected Connection, because connected state holds capability + //! references to server objects which themselves hold references back to + //! the Connection (m_rpc_system exports -> ProxyServer objects -> + //! ProxyContext::connection). disconnect() breaks these cycles, so every + //! code path that tears down a connection must call it; after that, the + //! object is destroyed when the last reference is dropped. + template + static std::shared_ptr make(Args&&... args) + { + return {new Connection(std::forward(args)...), [](Connection* connection) { + EventLoop& loop{*connection->m_loop}; + loop.sync([&] { delete connection; }); + }}; + } + + //! Start serving RPC requests on the connection, handling them with server + //! objects created by the make_client callback. Called (for server-side + //! connections) after make(), not during construction, because the server + //! objects created by the callback take shared ownership of this + //! connection via shared_from_this(), which requires the shared_ptr owner + //! returned by make() to already exist. + void serve(const std::function<::capnp::Capability::Client(Connection&)>& make_client) + { + assert(!m_rpc_system); + m_rpc_system.emplace(::capnp::makeRpcServer(*m_network, make_client(*this))); + } //! Destroy the connection. Calls disconnect() if it has not been called //! already. Must be called from the event loop thread. @@ -574,6 +611,12 @@ class Connection //! already dispatched to a worker thread runs to completion. void disconnect(); + //! True once disconnect() has run (locally, or as part of destruction), + //! i.e. the connection has been severed on this side. False while the + //! connection is still connected, including after a remote disconnect that + //! this side has not yet reacted to. + bool disconnected() const { return !m_network; } + //! Return server object tracker allowing clients to block until no //! ProxyServer objects associated with this connection remain, i.e. until //! no server call body is still executing (see ServerObjectTracker). @@ -584,36 +627,43 @@ class Connection //! (with access to capnp thread-local variables) when the connection is //! disconnected -- for either a remote disconnect (the peer closes the //! connection) or a local one (the connection is torn down on this side). - //! Contrast onDisconnect(), whose handler runs only on a remote - //! disconnect. Returns a handle that can be passed to removeSyncCleanup() - //! to unregister the function before it runs. - CleanupIt addSyncCleanup(std::function fn); - void removeSyncCleanup(CleanupIt it); - - //! Register a handler to run on the event loop thread when the peer - //! disconnects. The handler runs at most once, and only while this - //! Connection is still alive: if the connection is torn down locally before - //! the handler runs, the handler is not called. + //! Runs inside disconnect() as part of teardown. Returns a handle that can + //! be passed to cancelOnDisconnect() to unregister the function before it + //! runs. + CleanupIt onDisconnect(std::function fn); + void cancelOnDisconnect(CleanupIt it); + + //! Register a handler for when the capnp network reports the connection + //! disconnected -- which happens on both a remote disconnect (the peer + //! closes it) and a local disconnect() on this side. The handler runs + //! deferred, on a later event loop turn. + //! + //! It is passed a Connection* that is either the still-live connection (safe + //! to use for the duration of the call) or null if the Connection was + //! already destroyed before the handler ran, which is the case when a + //! connection is torn down and freed locally. Handlers must decide + //! explicitly what to do in each case. + //! + //! Unless the event loop exits early, this callback will always be called + //! after disconnecting. So it is possible to use this callback to free the + //! last reference to the EventLoop, and exit after the last disconnect. template - void onDisconnect(F&& f) + void afterDisconnect(F&& f) { - // m_network->onDisconnect() fires both on a remote disconnect and on a - // local one (disconnecting resets m_rpc_system, which drops capnp's - // last reference to the network and fulfills the promise). The m_alive - // weak_ptr tells the two apart -- disconnect() expires it, so f is - // skipped on local disconnects. This lets onDisconnect callbacks - // delete the Connection without a double deletion. + // m_network->onDisconnect() fires on both remote and local disconnects + // (disconnecting resets m_rpc_system, which drops capnp's last + // reference to the network and fulfills the promise). Lock a weak_ptr to + // this Connection and pass the raw pointer to f: it is null if the + // object was already destroyed, and while non-null the temporary + // shared_ptr from lock() keeps it alive for the duration of the call. m_loop->m_task_set->add(m_network->onDisconnect().then( - [f = std::forward(f), alive = std::weak_ptr(m_alive)]() mutable { - if (!alive.expired()) f(); + [f = std::forward(f), weak = weak_from_this()]() mutable { + f(weak.lock().get()); })); } EventLoopRef m_loop; kj::Own m_stream; - //! Liveness token checked by onDisconnect() callbacks (see there). - //! Could be dropped if Connection lifetime were reference-counted (#336). - std::shared_ptr m_alive{std::make_shared()}; //! Wrapped in std::optional so disconnect() can tear it down (along with //! the stream) to sever the transport while this object stays alive. //! Closing the stream is what makes the peer observe the disconnect: it @@ -621,14 +671,7 @@ class Connection std::optional<::capnp::TwoPartyVatNetwork> m_network; //! Tracker for live ProxyServer objects associated with this connection, - //! used by waitDrained(). Held via shared_ptr because ProxyServer objects - //! kept alive by in-flight calls can outlive the Connection (see - //! ServerObjectTracker and ~ProxyServerBase). - //! - //! Must be declared before m_rpc_system: constructing m_rpc_system runs - //! the make_client callback, which creates the bootstrap (Init) server - //! object, whose ProxyServerBase constructor registers itself with this - //! tracker. + //! used by waitDrained(). std::shared_ptr m_server_objects{std::make_shared()}; std::optional<::capnp::RpcSystem<::capnp::rpc::twoparty::VatId>> m_rpc_system; @@ -657,6 +700,20 @@ class Connection //! will be empty if all ProxyClient are destroyed cleanly before the //! connection is destroyed. CleanupList m_sync_cleanup_fns; + +private: + //! Construct a client-side connection. Private; use make(). + Connection(EventLoop& loop, kj::Own&& stream_) + : Connection(loop, kj::mv(stream_), ::capnp::rpc::twoparty::Side::CLIENT) + { + m_rpc_system.emplace(::capnp::makeRpcClient(*m_network)); + } + //! Construct a connection for the given side without starting the RPC + //! system. Server-side connections start it with serve() after make() + //! returns. Private; use make(). + Connection(EventLoop& loop, kj::Own&& stream_, ::capnp::rpc::twoparty::Side side) + : m_loop(loop), m_stream(kj::mv(stream_)), + m_network(std::in_place, *m_stream, side, ::capnp::ReaderOptions()) {} }; //! Vat id for server side of connection. Required argument to RpcSystem::bootStrap() @@ -683,14 +740,15 @@ ProxyClientBase::ProxyClientBase(typename Interface::Client cli { MP_LOG(*m_context.loop, Log::Debug) << "Creating " << CxxTypeName(*this) << " " << this; - // Handler for the connection getting destroyed before this client object. - auto disconnect_cb = m_context.connection->addSyncCleanup([this]() { + // Handler for the connection getting disconnected before this client + // object is destroyed. + auto disconnect_cb = m_context.connection->onDisconnect([this]() { // Release client capability by move-assigning to temporary. { typename Interface::Client(std::move(m_client)); } Lock lock{m_context.loop->m_mutex}; - m_context.connection = nullptr; + m_context.connection.reset(); }); // Two shutdown sequences are supported: @@ -724,16 +782,23 @@ ProxyClientBase::ProxyClientBase(typename Interface::Client cli // run inside loop->sync() on the event loop thread because // otherwise, if there were an ill-timed disconnect, the remote // disconnect handler could fire and delete the Connection object - // before the removeSyncCleanup call. - if (m_context.connection) m_context.connection->removeSyncCleanup(disconnect_cb); + // before the cancelOnDisconnect call. + if (m_context.connection) m_context.connection->cancelOnDisconnect(disconnect_cb); // Release client capability by move-assigning to temporary. { typename Interface::Client(std::move(m_client)); } - if (destroy_connection) { - delete m_context.connection; - m_context.connection = nullptr; + if (destroy_connection && m_context.connection) { + // Disconnect the connection and drop this object's reference + // while on the event loop thread, destroying the Connection + // here unless other proxy objects still reference it (in + // which case it is destroyed when the last of them is). If the + // connection already disconnected, disconnect_cb above has + // already dropped this object's reference, so m_context.connection + // is null here and there is nothing to do. + m_context.connection->disconnect(); + m_context.connection.reset(); } }); } @@ -749,14 +814,18 @@ ProxyClientBase::ProxyClientBase(typename Interface::Client cli throw; } - // If this client owns the connection, delete the connection on disconnect. + // If this client owns the connection, register a handler to process a + // remote disconnect and shut down the connection. if (destroy_connection) { m_context.loop->sync([&] { EventLoop& loop = *m_context.loop; - Connection* connection = m_context.connection; - connection->onDisconnect([&loop, connection] { + m_context.connection->afterDisconnect([&loop](Connection* conn) { + // If conn is null or disconnected() is true, a disconnect was + // already trigged locally, so there is nothing to do and an + // unexpected network disconnect message should not be logged. + if (!conn || conn->disconnected()) return; MP_LOG(loop, Log::Warning) << "IPC client: unexpected network disconnect."; - delete connection; + conn->disconnect(); }); }); } @@ -787,15 +856,13 @@ ProxyServerBase::ProxyServerBase(std::shared_ptr impl, Co //! ProxyServer destructor, called from the EventLoop thread by Cap'n Proto //! garbage collection code after there are no more references to this object. //! This will typically happen when the corresponding ProxyClient object on the -//! other side of the connection is destroyed. It can also happen earlier if the -//! connection is broken or destroyed. In the latter case this destructor will -//! typically be called inside m_rpc_system.reset() call in the ~Connection -//! destructor while the Connection object still exists. However, because -//! ProxyServer objects are refcounted, and the Connection object could be -//! destroyed while asynchronous IPC calls are still in-flight, it's possible -//! for this destructor to be called after the Connection object no longer -//! exists, so it is NOT valid to dereference the m_context.connection pointer -//! from this function. +//! other side of the connection is destroyed. It can also happen earlier if +//! the connection is broken or disconnected, in which case this destructor is +//! typically called inside the m_rpc_system.reset() call in +//! Connection::disconnect(). If an asynchronous IPC call is still in-flight at +//! disconnect time, Cap'n Proto keeps this object alive and this destructor +//! runs later, when the call body finishes; m_context.connection remains valid +//! even then, because m_context holds shared ownership of the Connection. template ProxyServerBase::~ProxyServerBase() { @@ -1026,62 +1093,79 @@ kj::Promise ProxyServer::post(Fn&& fn) //! the call fails. //! //! If destroy_connection is false, the returned client does not take -//! ownership of the connection, and the caller is responsible for -//! disconnecting and freeing it (accessible as client->m_context.connection) -//! whenever it is done with it. +//! ownership of the connection, and the caller is responsible for calling +//! disconnect() on it (accessible as client->m_context.connection) whenever +//! it is done with it. template std::unique_ptr> ConnectStream(EventLoop& loop, Stream stream, bool destroy_connection = true) { typename InitInterface::Client init_client(nullptr); - std::unique_ptr connection; + std::shared_ptr connection; loop.sync([&] { - connection = std::make_unique(loop, kj::mv(stream)); + connection = Connection::make(loop, kj::mv(stream)); init_client = connection->m_rpc_system->bootstrap(ServerVatId().vat_id).castAs(); }); - return std::make_unique>(kj::mv(init_client), connection.release(), destroy_connection); + // The local `connection` reference is dropped when this function returns; + // the returned ProxyClient keeps the connection alive (see + // ProxyContext::connection) and disconnects it on destruction if + // destroy_connection is true. + return std::make_unique>(kj::mv(init_client), connection.get(), destroy_connection); } //! Given stream and init objects, construct a new ProxyServer object that //! handles requests from the stream by calling the init object. Embed the //! ProxyServer in a Connection object that is stored in //! loop.m_incoming_connections. This should be called from the event loop -//! thread. Returns the new ProxyServer along with an iterator to its +//! thread. Returns the new ProxyServer along with a shared_ptr to its //! Connection. //! -//! If destroy_connection is false, the connection is not erased -//! automatically when the peer disconnects, and the caller is responsible -//! for erasing loop.m_incoming_connections at the returned iterator -//! whenever it is done with the connection. +//! If destroy_connection is false, the connection is not disconnected and +//! removed automatically when the peer disconnects, and the caller is +//! responsible for calling disconnect() and removing it from +//! loop.m_incoming_connections (by value, not iterator -- see the comment +//! below) whenever it is done with the connection. template -std::pair*, EventLoop::Connections::iterator> _Serve(EventLoop& loop, +std::pair*, std::shared_ptr> _Serve(EventLoop& loop, kj::Own&& stream, std::shared_ptr init, OnDisconnect&& on_disconnect, bool destroy_connection = true) { ProxyServer* server = nullptr; - loop.m_incoming_connections.emplace_front(loop, kj::mv(stream), [&](Connection& connection) { + auto connection{Connection::make(loop, kj::mv(stream), ::capnp::rpc::twoparty::Side::SERVER)}; + loop.m_incoming_connections.emplace_front(connection); + connection->serve([&](Connection& connection) { auto proxy_server = kj::heap>(std::move(init), connection); server = proxy_server; return capnp::Capability::Client(kj::mv(proxy_server)); }); - auto it = loop.m_incoming_connections.begin(); MP_LOG(loop, Log::Info) << "IPC server: socket connected."; if (loop.testing_hook_connected) loop.testing_hook_connected(); // Run on_disconnect (e.g. the listener's active-connection counter - // decrement) on any disconnect. It is registered with addSyncCleanup rather - // than placed in the onDisconnect handler below because that handler - // only fires on a remote disconnect and is canceled when a connection is - // closed locally; if on_disconnect lived there, closing a connection - // locally would leave the listener's slot count stuck and stop it from - // accepting again. - it->addSyncCleanup(std::forward(on_disconnect)); - it->onDisconnect([&loop, it, destroy_connection]() mutable { + // decrement) as a sync cleanup, so it runs synchronously on every + // disconnect. The afterDisconnect handler below is deferred, so putting the + // decrement there could leave the listener's slot count stuck on a local + // disconnect() and stop it accepting. + connection->onDisconnect(std::forward(on_disconnect)); + connection->afterDisconnect([&loop, destroy_connection](Connection* conn) { + // If conn is null or disconnected() is true, a disconnect has been + // triggered locally and the code which triggered it is responsible for + // tearing down the Connection object. Otherwise the remote peer + // disconnected, and if destroy_connection is true, the connection is + // still on the incoming-connection list and must be disconnected and + // removed. + if (!conn || conn->disconnected()) return; MP_LOG(loop, Log::Info) << "IPC server: socket disconnected."; - if (destroy_connection) loop.m_incoming_connections.erase(it); + if (destroy_connection) { + // Remove by value, not through a captured iterator: other code may + // have reordered the list between this handler being queued and + // running. + conn->disconnect(); + loop.m_incoming_connections.remove(conn->shared_from_this()); + } if (loop.testing_hook_disconnected) loop.testing_hook_disconnected(); }); - return {server, it}; + return {server, connection}; } //! Overload of _Serve that takes a reference to the init object instead of a @@ -1094,7 +1178,7 @@ std::pair*, EventLoop::Connections::iterator> _Serve( //! the ProxyServer object from deleting the init object when the client is //! disconnected. template -std::pair*, EventLoop::Connections::iterator> _Serve(EventLoop& loop, +std::pair*, std::shared_ptr> _Serve(EventLoop& loop, kj::Own&& stream, InitImpl& init, OnDisconnect&& on_disconnect, @@ -1142,7 +1226,7 @@ void _Listen(const std::shared_ptr& listener, EventLoop& loop, InitImp //! methods on the Init object. See _Serve for details on the return value and //! the destroy_connection parameter. template -std::pair*, EventLoop::Connections::iterator> ServeStream( +std::pair*, std::shared_ptr> ServeStream( EventLoop& loop, Stream stream, InitImpl&& init, bool destroy_connection = true) { return _Serve(loop, kj::mv(stream), std::forward(init), /*on_disconnect=*/ [] {}, destroy_connection); diff --git a/include/mp/proxy-types.h b/include/mp/proxy-types.h index 0c00b825..c6dc3713 100644 --- a/include/mp/proxy-types.h +++ b/include/mp/proxy-types.h @@ -351,7 +351,7 @@ auto PassField(Priority<1>, TypeList, ServerContext& server_context, const auto& params = server_context.call_context.getParams(); const auto& input = Make(params); using Interface = typename Decay::Calls; - auto param = std::make_unique>(input.get(), server_context.proxy_server.m_context.connection, false); + auto param = std::make_unique>(input.get(), server_context.proxy_server.m_context.connection.get(), false); fn.invoke(server_context, std::forward(args)..., *param); } diff --git a/include/mp/proxy.h b/include/mp/proxy.h index b02abfd8..e1ec61cb 100644 --- a/include/mp/proxy.h +++ b/include/mp/proxy.h @@ -70,7 +70,11 @@ class EventLoopRef //! Context data associated with proxy client and server classes. struct ProxyContext { - Connection* connection; + //! Connection this proxy object is associated with. Holding shared + //! ownership (Connection objects are always owned by shared_ptr, see + //! Connection::make) guarantees the Connection outlives this proxy + //! object, so the pointer stays valid even after a disconnect. + std::shared_ptr connection; EventLoopRef loop; CleanupList cleanup_fns; diff --git a/include/mp/type-context.h b/include/mp/type-context.h index 5f7e05b2..503a016a 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -129,7 +129,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // cancel_monitor.m_canceled was checked above and this // code is running on the event loop thread. std::tie(request_thread, inserted) = SetThread( - GuardedRef{thread_context.waiter->m_mutex, request_threads}, server.m_context.connection, + GuardedRef{thread_context.waiter->m_mutex, request_threads}, server.m_context.connection.get(), [&] { return Accessor::get(call_context.getParams()).getCallbackThread(); }); // Initialize the request's results struct here on the event loop // thread, so later getResults() calls on the execution thread @@ -186,7 +186,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& ConnThreads::node_type removed; { Lock lock(thread_context.waiter->m_mutex); - removed = request_threads.extract(server.m_context.connection); + removed = request_threads.extract(server.m_context.connection.get()); } } }); @@ -215,7 +215,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& if (!context_arg.hasThread()) { // No client thread specified — dispatch through the server thread // pool, picking the slot with the smallest in-flight depth. - auto* connection = server.m_context.connection; + auto* connection = server.m_context.connection.get(); auto& pool = connection->m_thread_pool; if (pool.empty()) { MP_LOG(loop, Log::Error) diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 79655788..f69413ef 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -105,7 +105,7 @@ void EventLoopRef::reset(bool relock) MP_NO_TSA } } -ProxyContext::ProxyContext(Connection* connection) : connection(connection), loop{*connection->m_loop} {} +ProxyContext::ProxyContext(Connection* connection) : connection(connection->shared_from_this()), loop{*connection->m_loop} {} Connection::~Connection() noexcept(false) { @@ -129,15 +129,6 @@ void Connection::disconnect() // from the destructor) is a no-op. if (!m_network) return; - // Expire m_alive before severing the connection below so onDisconnect - // handlers will not trigger and delete this Connection object. The - // onDisconnect handlers trigger on remote disconnects and automatically - // delete Connection objects. But on local disconnects, they should not - // trigger, because local code that disconnects is responsible for freeing - // Connection objects, and it may want to wait for in-flight calls to finish - // before destroying them. - m_alive.reset(); - // Try to cancel any calls that may be executing. m_canceler.cancel("Interrupted by disconnect"); @@ -253,7 +244,7 @@ void Connection::disconnect() m_stream = nullptr; } -CleanupIt Connection::addSyncCleanup(std::function fn) +CleanupIt Connection::onDisconnect(std::function fn) { const Lock lock(m_loop->m_mutex); // Add cleanup callbacks to the front of list, so sync cleanup functions run @@ -268,7 +259,7 @@ CleanupIt Connection::addSyncCleanup(std::function fn) return m_sync_cleanup_fns.emplace(m_sync_cleanup_fns.begin(), std::move(fn)); } -void Connection::removeSyncCleanup(CleanupIt it) +void Connection::cancelOnDisconnect(CleanupIt it) { // Require cleanup functions to be removed on the event loop thread to avoid // needing to deal with them being removed in the middle of a disconnect. @@ -440,7 +431,7 @@ std::tuple SetThread(GuardedRef threads, Connecti } if (inserted) { thread->second.emplace(make_thread(), connection, /* destroy_connection= */ false); - thread->second->m_disconnect_cb = connection->addSyncCleanup([threads, connection] { + thread->second->m_disconnect_cb = connection->onDisconnect([threads, connection] { // Remove and destroy this connection's map entry, unless the // thread owning the map is exiting and ~ThreadContext already took // the entry, in which case that thread destroys it. Look the entry @@ -488,7 +479,7 @@ ProxyClient::~ProxyClient() // map, so if ~ThreadContext took the entry first, m_disconnect_cb // is still set here.) if (m_disconnect_cb && m_context.connection) { - m_context.connection->removeSyncCleanup(*m_disconnect_cb); + m_context.connection->cancelOnDisconnect(*m_disconnect_cb); } }); } diff --git a/test/mp/test/listen_tests.cpp b/test/mp/test/listen_tests.cpp index 8e6b88cd..edc65eb9 100644 --- a/test/mp/test/listen_tests.cpp +++ b/test/mp/test/listen_tests.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -196,11 +195,11 @@ KJ_TEST("ListenConnections enforces a local connection limit") KJ_TEST("ListenConnections resumes after a local disconnect") { - // A connection closed locally (here by erasing it from the incoming- - // connection list) must still free the listener's slot so the listener - // resumes accepting. The counter decrement runs as an addSyncCleanup cleanup; - // if it were in the onDisconnect handler it would be canceled by the - // local close and the slot would stay stuck. + // A connection closed locally with disconnect() must still free the + // listener's slot so the listener resumes accepting. The counter decrement + // runs as an onDisconnect cleanup, so it runs synchronously on local + // disconnects too; if it were in the deferred afterDisconnect handler it + // could be handed a null connection or dropped and the slot would stay stuck. ListenSetup server(/*max_connections=*/1); auto client1 = std::make_unique(server.listener.MakeConnectedSocket()); @@ -211,10 +210,13 @@ KJ_TEST("ListenConnections resumes after a local disconnect") (**server.m_loop_ref).sync([] {}); KJ_EXPECT(server.ConnectedCount() == 1); - // Close the first connection locally on the event loop thread. + // Close the first connection locally on the event loop thread. Disconnect + // it before dropping the list's reference: under shared ownership, erasing + // it from the list alone would not tear it down (see Connection::make). EventLoop& loop{**server.m_loop_ref}; loop.sync([&] { KJ_REQUIRE(loop.m_incoming_connections.size() == 1); + loop.m_incoming_connections.front()->disconnect(); loop.m_incoming_connections.pop_front(); }); diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 416d962f..66567b88 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -105,21 +104,49 @@ class TestSetup }); auto pipe = loop.m_io_context.provider->newTwoWayPipe(); - auto server_result = ServeStream( - loop, kj::mv(pipe.ends[0]), std::make_shared(), server_owns_connection); - server = server_result.first; - EventLoop::Connections::iterator server_it = server_result.second; - server_disconnect = [&] { loop.sync([&] { loop.m_incoming_connections.erase(server_it); }); }; + // Weak, not owning: nothing here needs to keep the connection + // alive. When server_owns_connection is true, ServeStream's own + // afterDisconnect handler owns tearing it down on a remote + // disconnect (via loop.m_incoming_connections); when false (or + // when a test disconnects it directly, bypassing + // server_disconnect()), the connection stays alive on that list + // until something else drops the last reference. An owning + // reference here would race both cases: it would leak the + // EventLoop forever in the first case unless dropped in lockstep + // with ServeStream's own handler, and would let this code tear + // the connection down early in the second case, before callers + // that expect it to survive until they call server_disconnect() + // are done with it. + std::weak_ptr server_connection; + { + auto server_result = ServeStream( + loop, kj::mv(pipe.ends[0]), std::make_shared(), server_owns_connection); + server = server_result.first; + server_connection = server_result.second; + } + auto server_close = [&] { + if (auto connection = server_connection.lock()) { + connection->disconnect(); + loop.m_incoming_connections.remove(connection); + } + }; + server_disconnect = [&] { loop.sync(server_close); }; server_disconnect_later = [&] { assert(std::this_thread::get_id() == loop.m_thread_id); - loop.m_task_set->add(kj::evalLater([&] { loop.m_incoming_connections.erase(server_it); })); + loop.m_task_set->add(kj::evalLater([&] { server_close(); })); }; auto client_proxy = ConnectStream(loop, kj::mv(pipe.ends[1]), client_owns_connection); + std::shared_ptr client_connection; if (!client_owns_connection) { - Connection* client_connection = client_proxy->m_context.connection; - client_disconnect = [&loop, client_connection] { loop.sync([&] { delete client_connection; }); }; + client_connection = client_proxy->m_context.connection; + client_disconnect = [&] { loop.sync([&] { + if (client_connection) { + client_connection->disconnect(); + client_connection.reset(); + } + }); }; } client_promise.set_value(std::move(client_proxy)); @@ -610,7 +637,7 @@ KJ_TEST("Waiting for in-flight server call to finish after disconnect") // disconnecting. It stays valid until server_disconnect() destroys it // below. Connection* connection{nullptr}; - foo->m_context.loop->sync([&] { connection = setup.server->m_context.connection; }); + foo->m_context.loop->sync([&] { connection = setup.server->m_context.connection.get(); }); // Invoke the async method on a separate thread so its body blocks there // while this thread makes assertions. callFnAsync() takes an mp.Context, @@ -700,8 +727,8 @@ KJ_TEST("Make simultaneous IPC calls on single remote thread") Thread::Client *callback_thread, *request_thread; foo->m_context.loop->sync([&] { Lock lock(tc.waiter->m_mutex); - callback_thread = &tc.callback_threads.at(foo->m_context.connection)->m_client; - request_thread = &tc.request_threads.at(foo->m_context.connection)->m_client; + callback_thread = &tc.callback_threads.at(foo->m_context.connection.get())->m_client; + request_thread = &tc.request_threads.at(foo->m_context.connection.get())->m_client; }); // Call callIntFnAsync 3 times with n=100, 200, 300 @@ -846,85 +873,6 @@ KJ_TEST("Async cleanup thread has OS thread name") } #endif // HAVE_PTHREAD_GETNAME_NP -KJ_TEST("onDisconnect handler does not run after the connection is destroyed") -{ - // Regression test for a race condition where an onDisconnect handler - // could fire even after the connection had already been disconnected - // locally. Local disconnects are supposed to preempt onDisconnect - // handlers so they are able to free Connection objects without risking double - // deletions. - // - // To hit the previous race window deterministically without relying on - // socket timing, this test exploits two ordering facts: - // m_network.onDisconnect() hands out branches of a forked promise that fire - // in the order they were added, and kj::evalLater tasks queued during one - // event loop turn run on the next turn in FIFO order. A "destroyer" branch - // is registered before the handler under test, so when the peer disconnects - // the destroyer fires first and queues an evalLater task that destroys the - // Connection ahead of the handler. The handler records whether it ran after - // the Connection was destroyed, reading only test-owned flags (never the - // freed Connection), so a violation is a deterministic wrong result rather - // than a flaky crash. - // - // The Connection must be destroyed from that separate task, not from inside - // an onDisconnect continuation: destroying it there tears down m_network - // while a continuation of m_network's promise is still running, and crashes. - - std::atomic connection_destroyed{false}; - std::atomic handler_ran{false}; - std::atomic handler_ran_after_destroy{false}; - - std::thread thread{[&] { - EventLoop loop("mptest", [](mp::LogMessage log) { - KJ_LOG(INFO, log.level, log.message); - if (log.level == mp::Log::Raise) throw std::runtime_error(log.message); - }); - auto pipe = loop.m_io_context.provider->newTwoWayPipe(); - - // Server-side connection whose onDisconnect handler is under test. - auto server_conn = std::make_unique( - loop, kj::mv(pipe.ends[0]), [&](Connection& connection) { - return ::capnp::Capability::Client(kj::heap>( - std::make_shared(), connection)); - }); - - // Raw peer end of the pipe. Closing it makes the server connection's - // m_network.onDisconnect() resolve, i.e. a real remote disconnect. - kj::Own peer_end = kj::mv(pipe.ends[1]); - - // Destroyer branch, registered BEFORE the handler under test so it - // fires first and its destroy task E_d is queued before the buggy - // bounced handler task E_f. It schedules the destroy on a *separate* - // task, so the connection is not torn down from inside an onDisconnect - // continuation. - loop.m_task_set->add(server_conn->m_network->onDisconnect().then([&] { - loop.m_task_set->add(kj::evalLater([&] { - server_conn.reset(); - connection_destroyed = true; - })); - })); - - // Handler under test. It intentionally does not dereference the - // Connection; it only records whether it ran after the connection was - // destroyed, so the result is deterministic. - server_conn->onDisconnect([&] { - handler_ran = true; - handler_ran_after_destroy = connection_destroyed.load(); - }); - - // Trigger the remote disconnect once the loop is running. - loop.m_task_set->add(kj::evalLater([&] { peer_end = nullptr; })); - - loop.loop(); - }}; - thread.join(); - - // The handler must have run (both versions schedule it), but it must never - // run after the connection was destroyed. - KJ_EXPECT(handler_ran); - KJ_EXPECT(!handler_ran_after_destroy); -} - KJ_TEST("Call async IPC method without thread or pool errors correctly") { TestSetup setup; From 7eb79e5eef66c12c2a7b764250182b730bfb05a7 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Fri, 31 Jul 2026 15:56:59 -0400 Subject: [PATCH 14/17] proxy-io: keep client capability handles across disconnect Remove the per-client disconnect tracking from ProxyClientBase: client objects no longer register a cleanup callback with their Connection, and a disconnect no longer eagerly releases their m_client capability handles or nulls their connection pointers. Neither is necessary now that proxy objects share ownership of their Connection. The connection pointer stays valid after a disconnect because the Connection outlives its proxies, and keeping the capability handle is safe: Cap'n Proto's per-connection state is refcounted and outlives the RPC system as long as handles reference it, with calls on handles of a disconnected connection failing cleanly with DISCONNECTED errors. The handle is simply released (on the event loop thread, since capability refcounts are not thread safe) whenever the client object is eventually destroyed, and clientInvoke checks the connection's disconnected() predicate instead of a nulled pointer, throwing the same 'IPC client method called after disconnect' error as before. This deletes the detach machinery from ~ProxyClientBase, including the FIXME'd duplicate-cleanup code path. Connection::onDisconnect (the renamed addSyncCleanup) remains for its one other user, the per-thread connection maps (see SetThread), which the next commit converts. Because the connection pointer is no longer nulled on disconnect, ~ProxyClient can no longer use it to tell whether Connection::disconnect() has already run and freed its m_disconnect_cb cleanup node. It now keys off the connection's disconnected() predicate instead: if the connection is disconnected the node is already gone and must not be passed to cancelOnDisconnect. Without this, ~ProxyClient erased an already-freed list node -- a heap-use-after-free (see the "Waiting for in-flight server call to finish after disconnect" test, which this commit re-enables). That test also has to hold its own shared reference to the server Connection across the disconnect() + waitDrained() sequence, the way Ipc::disconnect Incoming does: under shared ownership the last server proxy (destroyed once the drained body finishes) would otherwise free the Connection while the test is still observing it. This change was written with Claude Fable 5. This change was written with Claude Opus 4.8. --- include/mp/proxy-io.h | 65 +++++++++++++--------------------------- include/mp/proxy-types.h | 7 ++++- src/mp/proxy.cpp | 27 +++++++++++------ test/mp/test/test.cpp | 22 +++++++++----- 4 files changed, 60 insertions(+), 61 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index 4473b3c2..a2f5752d 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -740,68 +740,45 @@ ProxyClientBase::ProxyClientBase(typename Interface::Client cli { MP_LOG(*m_context.loop, Log::Debug) << "Creating " << CxxTypeName(*this) << " " << this; - // Handler for the connection getting disconnected before this client - // object is destroyed. - auto disconnect_cb = m_context.connection->onDisconnect([this]() { - // Release client capability by move-assigning to temporary. - { - typename Interface::Client(std::move(m_client)); - } - Lock lock{m_context.loop->m_mutex}; - m_context.connection.reset(); - }); - - // Two shutdown sequences are supported: - // - // - A normal sequence where client proxy objects are deleted by external - // code that no longer needs them - // - // - A garbage collection sequence where the connection or event loop shuts - // down while external code is still holding client references. - // - // The first case is handled here when m_context.connection is not null. The - // second case is handled by the disconnect_cb function, which sets - // m_context.connection to null so nothing happens here. - m_context.cleanup_fns.emplace_front([this, destroy_connection, disconnect_cb]{ - { + // Cleanup function that runs when this object is destroyed. Unlike server + // objects, client objects are owned by application code, which can keep + // them alive arbitrarily long after a disconnect, but this needs no + // special handling: m_context holds shared ownership of the connection, + // so the Connection object is guaranteed to still exist here, and the + // m_client capability handle is safe to keep across a disconnect (Cap'n + // Proto keeps handles valid after the connection state is torn down; + // using them just fails with DISCONNECTED errors). The handle only needs + // to be released on the event loop thread, done in the sync() call below, + // because capability reference counts are not thread safe. + m_context.cleanup_fns.emplace_front([this, destroy_connection]{ // If the capnp interface defines a destroy method, call it to destroy // the remote object, waiting for it to be deleted server side. If the // capnp interface does not define a destroy method, this will just call // an empty stub defined in the ProxyClientBase class and do nothing. // Exceptions are caught and logged rather than propagated because - // ~ProxyClientBase is noexcept and the peer may be gone by the time - // this runs. + // ~ProxyClientBase is noexcept and the connection may have been + // disconnected. (In that case clientInvoke fails with "IPC client + // method called after disconnect", caught and logged here.) if (kj::runCatchingExceptions([&]{ Sub::destroy(*this); }) != nullptr) { MP_LOG(*m_context.loop, Log::Warning) << "Remote destroy call failed during cleanup. Continuing."; } - // FIXME: Could just invoke removed addCleanup fn here instead of duplicating code m_context.loop->sync([&]() { - // Remove disconnect callback on cleanup so it doesn't run and try - // to access this object after it's destroyed. This call needs to - // run inside loop->sync() on the event loop thread because - // otherwise, if there were an ill-timed disconnect, the remote - // disconnect handler could fire and delete the Connection object - // before the cancelOnDisconnect call. - if (m_context.connection) m_context.connection->cancelOnDisconnect(disconnect_cb); - // Release client capability by move-assigning to temporary. { typename Interface::Client(std::move(m_client)); } - if (destroy_connection && m_context.connection) { - // Disconnect the connection and drop this object's reference - // while on the event loop thread, destroying the Connection - // here unless other proxy objects still reference it (in - // which case it is destroyed when the last of them is). If the - // connection already disconnected, disconnect_cb above has - // already dropped this object's reference, so m_context.connection - // is null here and there is nothing to do. + if (destroy_connection) { + // This client owns the connection: disconnect it and drop this + // object's reference while on the event loop thread, destroying + // the Connection here unless other proxy objects still reference + // it (in which case it is destroyed when the last of them is). + // m_context.connection is still valid here -- this object shares + // ownership of it and nothing resets the pointer on disconnect. m_context.connection->disconnect(); m_context.connection.reset(); } }); - } }); // If construct() fails, run the cleanup functions before rethrowing, // because ~ProxyClientBase will not run for an object whose constructor diff --git a/include/mp/proxy-types.h b/include/mp/proxy-types.h index c6dc3713..0efe4b46 100644 --- a/include/mp/proxy-types.h +++ b/include/mp/proxy-types.h @@ -740,7 +740,12 @@ void clientInvoke(ProxyClient& proxy_client, const GetRequest& get_request, Fiel bool done = false; const char* disconnected = nullptr; proxy_client.m_context.loop->sync([&]() { - if (!proxy_client.m_context.connection) { + // Fail immediately on a disconnected connection instead of trying to + // send a request through it. (The connection object itself is always + // valid, since m_context holds shared ownership of it.) disconnected() + // is only meaningful on the event loop thread, which this sync() + // callback runs on. + if (proxy_client.m_context.connection->disconnected()) { const Lock lock(thread_context.waiter->m_mutex); done = true; disconnected = "IPC client method called after disconnect."; diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index f69413ef..0cb86a85 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -470,15 +470,24 @@ ProxyClient::~ProxyClient() // between this thread trying to remove the callback and the disconnect // handler attempting to call it. m_context.loop->sync([&]() { - // Skip if the connection was disconnected while this thread waited - // for the event loop: Connection::disconnect() has already run and - // freed the cleanup list m_disconnect_cb points into, and the - // ProxyClientBase disconnect callback has nulled - // m_context.connection. (The SetThread callback resets - // m_disconnect_cb only when it finds this object in the thread - // map, so if ~ThreadContext took the entry first, m_disconnect_cb - // is still set here.) - if (m_disconnect_cb && m_context.connection) { + // Check connection->disconnected() in addition to m_disconnect_cb: + // if the connection was disconnected while this thread was waiting + // for the event loop, Connection::disconnect() has already run and + // freed every m_sync_cleanup_fns node, including the one + // m_disconnect_cb points at, so the iterator is dangling and must + // not be passed to cancelOnDisconnect. disconnect() runs to + // completion on the event loop thread without interleaving with + // this posted lambda, so disconnected() is true here exactly when + // the node has already been freed. + // + // m_disconnect_cb can be set here even though disconnect() freed + // the node: the SetThread callback only resets m_disconnect_cb when + // it still finds this object in the thread map (see SetThread); if + // ~ThreadContext extracted the map entry first, the callback + // returns early and leaves m_disconnect_cb set. Before this + // object's connection ownership was made shared, m_context.connection + // was nulled on disconnect and this check keyed off that instead. + if (m_disconnect_cb && !m_context.connection->disconnected()) { m_context.connection->cancelOnDisconnect(*m_disconnect_cb); } }); diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 66567b88..e025b85d 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -633,11 +633,15 @@ KJ_TEST("Waiting for in-flight server call to finish after disconnect") release_body.get_future().get(); }; - // Grab the server Connection object on the event loop thread before - // disconnecting. It stays valid until server_disconnect() destroys it - // below. - Connection* connection{nullptr}; - foo->m_context.loop->sync([&] { connection = setup.server->m_context.connection.get(); }); + // Grab a shared reference to the server Connection on the event loop thread + // before disconnecting, and hold it for the rest of the test. This mirrors + // Ipc::disconnectIncoming, which keeps its own reference alive across the + // disconnect() + waitDrained() sequence: under shared ownership the last + // server proxy (destroyed once the drained body finishes) would otherwise + // free the Connection while the test is still observing it. The reference is + // released on the event loop thread at the end so ~Connection runs there. + std::shared_ptr connection; + foo->m_context.loop->sync([&] { connection = setup.server->m_context.connection; }); // Invoke the async method on a separate thread so its body blocks there // while this thread makes assertions. callFnAsync() takes an mp.Context, @@ -681,8 +685,12 @@ KJ_TEST("Waiting for in-flight server call to finish after disconnect") KJ_EXPECT(connection->tracker()->pendingServerObjects() == 0); call_thread.join(); - // Destroy the drained connection. (~Connection notices disconnect() has - // already run and does not tear things down twice.) + // Drop the test's reference on the event loop thread, mirroring how + // Ipc::disconnectIncoming releases the last reference after draining. This + // destroys the drained connection (~Connection notices disconnect() has + // already run and does not tear things down twice). server_disconnect() is + // then a no-op that just lets the loop exit. + foo->m_context.loop->sync([&] { connection.reset(); }); setup.server_disconnect(); } From 111069ef29dfcda8d47509db3a15cb6046e5e7eb Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Fri, 31 Jul 2026 16:06:21 -0400 Subject: [PATCH 15/17] doc: scope Connection cleanup callbacks to per-thread map entries Update comments to reflect that after the previous commit, the sync cleanup callback list has exactly one remaining purpose: eagerly removing a disconnected connection's ProxyClient entries from the thread_local per-thread connection maps (ThreadContext::request_threads / callback_threads) via callbacks registered by SetThread. Unlike interface clients, these entries cannot simply be left alive across a disconnect: they are owned by other threads that may never touch their maps again, and a surviving entry would hold the disconnected Connection object -- and through its EventLoopRef the event loop -- alive indefinitely, preventing the loop from ever exiting. (Replacing the callbacks with lazy garbage collection in SetThread was tried and hangs mptest for exactly this reason: entries owned by long-lived threads pin the loop after their connection is gone.) So this per-object disconnect tracking is retained by design, now clearly documented as thread-map-specific rather than a general client-object mechanism. This change was written with Claude Fable 5. --- include/mp/proxy-io.h | 33 ++++++++++++------ src/mp/proxy.cpp | 80 +++++++++++++++++-------------------------- 2 files changed, 53 insertions(+), 60 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index a2f5752d..ef498e6b 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -86,14 +86,16 @@ struct ProxyClient : public ProxyClientBase ~ProxyClient(); //! Reference to callback function that is run if there is a sudden - //! disconnect and the Connection object is destroyed before this - //! ProxyClient object. The callback will destroy this object and - //! remove its entry from the thread's request_threads or callback_threads - //! map. It will also reset m_disconnect_cb so the destructor does not - //! access it. In the normal case where there is no sudden disconnect, the - //! destructor will unregister m_disconnect_cb so the callback is never run. - //! Since this variable is accessed from multiple threads, accesses should - //! be guarded with the associated Waiter::m_mutex. + //! disconnect and the Connection is disconnected before this + //! ProxyClient object is destroyed. The callback will destroy this + //! object and remove its entry from the thread's request_threads or + //! callback_threads map (see SetThread, which explains why removal must + //! happen eagerly at disconnect time). It will also reset m_disconnect_cb + //! so the destructor does not access it. In the normal case where there is + //! no sudden disconnect, the destructor will unregister m_disconnect_cb so + //! the callback is never run. Since this variable is accessed from + //! multiple threads, accesses should be guarded with the associated + //! Waiter::m_mutex. std::optional m_disconnect_cb; }; @@ -630,6 +632,14 @@ class Connection : public std::enable_shared_from_this //! Runs inside disconnect() as part of teardown. Returns a handle that can //! be passed to cancelOnDisconnect() to unregister the function before it //! runs. + //! + //! The only current use is by SetThread, which registers callbacks that + //! eagerly remove this connection's entries from per-thread connection maps + //! (ProxyClient objects owned by other threads, see ThreadContext). + //! Eager removal is required because the owning threads may never touch + //! their maps again, and a surviving entry would hold this Connection object + //! -- and through its EventLoopRef the event loop -- alive indefinitely. + //! Other proxy objects need no disconnect notification (see disconnect()). CleanupIt onDisconnect(std::function fn); void cancelOnDisconnect(CleanupIt it); @@ -696,9 +706,10 @@ class Connection : public std::enable_shared_from_this //! still executing at time of disconnection. kj::Canceler m_canceler; - //! Cleanup functions to run if connection is broken unexpectedly. List - //! will be empty if all ProxyClient are destroyed cleanly before the - //! connection is destroyed. + //! Cleanup functions to run when the connection is disconnected, removing + //! this connection's entries from per-thread connection maps. See + //! addSyncCleanup. List will be empty if all ProxyClient objects + //! are destroyed cleanly before the connection is disconnected. CleanupList m_sync_cleanup_fns; private: diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 0cb86a85..0d9cb287 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -173,49 +173,26 @@ void Connection::disconnect() } } - // ProxyClient cleanup handlers are in sync list, and ProxyServer cleanup - // handlers are in the async list. + // Run cleanup functions registered with addSyncCleanup(). These remove + // this connection's ProxyClient entries from per-thread connection + // maps (see SetThread). The removal must happen eagerly here rather than + // whenever the owning threads next touch their maps, because the owning + // threads might never touch them again, and surviving entries would hold + // this Connection object -- and through its EventLoopRef the event loop -- + // alive indefinitely. // - // The ProxyClient cleanup handlers are synchronous because they are fast - // and don't do anything besides release capnp resources and reset state so - // future calls to client methods immediately throw exceptions instead of - // trying to communicate across the socket. The synchronous callbacks set - // ProxyClient capability pointers to null, so new method calls on client - // objects fail without triggering i/o or relying on event loop which may go - // out of scope or trigger obscure capnp i/o errors. - // - // The ProxyServer cleanup handlers call user defined destructors on the server - // object, which can run arbitrary blocking bitcoin code so they have to run - // asynchronously in a different thread. The asynchronous cleanup functions - // intentionally aren't started until after the synchronous cleanup - // functions run, so client objects are fully disconnected before bitcoin - // code in the destructors are run. This way if the bitcoin code tries to - // make client requests the requests will just fail immediately instead of - // sending i/o or accessing the event loop. - // - // The context where Connection objects are destroyed and this destructor is invoked - // is different depending on whether this is an outgoing connection being used - // to make an Init.makeX call() (e.g. Init.makeNode or Init.makeWalletClient) or an incoming - // connection implementing the Init interface and handling the Init.makeX() calls. - // - // Either way when a connection is closed, capnp behavior is to call all - // ProxyServer object destructors first, and then trigger a - // TwoPartyVatNetwork::onDisconnect callback. - // - // On incoming side of the connection, the TwoPartyVatNetwork::onDisconnect - // callback is written to delete the Connection object from the - // m_incoming_connections list and call this destructor. - // - // On the outgoing side, the Connection object is heap-allocated and owned - // by a top-level ProxyClient object. In this case, the - // TwoPartyVatNetwork::onDisconnect handler deletes the Connection object - // directly, calling this destructor, and loop below sets Connection - // pointers in all associated ProxyClient objects to null. - // - // Either way disconnect code runs in the event loop thread and called both - // on clean and unclean shutdowns. In unclean shutdown case when the - // connection is broken, sync and async cleanup lists will be filled with - // callbacks. In the clean shutdown case both lists will be empty. + // Other proxy objects associated with this connection do not need to be + // notified about the disconnect. Interface ProxyClient objects hold shared + // ownership of this Connection object, and their capability handles remain + // safe to hold and release after the disconnect (calls made through them + // just fail with "IPC client method called after disconnect" errors, see + // clientInvoke), so they are simply destroyed whenever the application + // code owning them gets around to it. ProxyServer objects that are not + // kept alive by in-flight calls are destroyed by the m_rpc_system.reset() + // call above; destructors of the m_impl objects they wrap can run + // arbitrary blocking application code, so ~ProxyServerBase schedules those + // on the EventLoop::m_async_fns worker thread instead of running them + // here, where they could deadlock the event loop thread. { Lock lock{m_loop->m_mutex}; while (!m_sync_cleanup_fns.empty()) { @@ -249,13 +226,12 @@ CleanupIt Connection::onDisconnect(std::function fn) const Lock lock(m_loop->m_mutex); // Add cleanup callbacks to the front of list, so sync cleanup functions run // in LIFO order. This is a good approach because sync cleanup functions are - // added as client objects are created, and it is natural to clean up - // objects in the reverse order they were created. In practice, however, - // order should not be significant because the cleanup callbacks run - // synchronously in a single batch when the connection is broken, and they - // only reset the connection pointers in the client objects without actually - // deleting the client objects, or update ListenConnections max_connections - // bookkeeping. + // added as objects are created, and it is natural to clean up objects in + // the reverse order they were created. In practice, however, order should + // not be significant because the cleanup callbacks run synchronously in a + // single batch when the connection is disconnected, and each one acts on + // independent state: removing a map entry (see SetThread) or decrementing + // a listener's active-connection counter (see _Serve). return m_sync_cleanup_fns.emplace(m_sync_cleanup_fns.begin(), std::move(fn)); } @@ -431,6 +407,12 @@ std::tuple SetThread(GuardedRef threads, Connecti } if (inserted) { thread->second.emplace(make_thread(), connection, /* destroy_connection= */ false); + // Register a cleanup callback eagerly removing this entry from the map + // when the connection is disconnected. This cannot be left to the + // thread owning the map, which might never touch the map again; a + // surviving entry would hold the disconnected Connection object -- and + // through its EventLoopRef the event loop -- alive indefinitely. See + // Connection::disconnect. thread->second->m_disconnect_cb = connection->onDisconnect([threads, connection] { // Remove and destroy this connection's map entry, unless the // thread owning the map is exiting and ~ThreadContext already took From 0ef5407ecbf71942845c51798144d460b9679224 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Fri, 31 Jul 2026 16:09:21 -0400 Subject: [PATCH 16/17] proxy-io: make server object tracker a plain Connection member Now that proxy objects hold shared ownership of their Connection, a ProxyServer object kept alive by an in-flight call can no longer outlive the Connection, so ~ProxyServerBase can always reach the tracker through m_context.connection. Drop the shared_ptr indirection that existed to keep the tracker valid past the Connection's death, and the separate tracker handle member on ProxyServerBase. No behavior change; Connection::waitDrained() semantics are identical. This change was written with Claude Fable 5. --- include/mp/proxy-io.h | 37 ++++++++++++++++++------------------- include/mp/proxy.h | 7 ------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index ef498e6b..505c8c10 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -486,11 +486,11 @@ struct Waiter //! KJ promise of an in-flight call; it does not interrupt a call body that //! was already dispatched to a worker thread (see Connection::disconnect). //! -//! The counter is held via shared_ptr by the Connection and by every -//! ProxyServer object created for the connection, because a ProxyServer -//! object kept alive by an in-flight call can outlive the Connection (see -//! ~ProxyServerBase), and its destructor must decrement state that is still -//! valid. +//! The counter can be a plain Connection member because proxy objects hold +//! shared ownership of their Connection (see ProxyContext::connection), so a +//! ProxyServer object kept alive by an in-flight call cannot outlive the +//! Connection, and its destructor can always safely reach the counter through +//! m_context.connection. //! //! ProxyServer and ProxyServer are separate //! specializations (not ProxyServerBase instances) and are intentionally not @@ -557,7 +557,7 @@ struct ServerObjectTracker //! worker threads, so the only valid operations on it are destruction and //! calling disconnect() again (a no-op). Methods that perform I/O or make //! calls must not be used after disconnect(). -class Connection : public std::enable_shared_from_this +class Connection : public ServerObjectTracker, public std::enable_shared_from_this { public: //! Create a Connection with shared ownership. Connection objects must be @@ -623,7 +623,7 @@ class Connection : public std::enable_shared_from_this //! ProxyServer objects associated with this connection remain, i.e. until //! no server call body is still executing (see ServerObjectTracker). using Tracker = std::shared_ptr; - Tracker tracker() { return m_server_objects; } + Tracker tracker() { return shared_from_this(); } //! Register a synchronous cleanup function to run on the event loop thread //! (with access to capnp thread-local variables) when the connection is @@ -682,7 +682,7 @@ class Connection : public std::enable_shared_from_this //! Tracker for live ProxyServer objects associated with this connection, //! used by waitDrained(). - std::shared_ptr m_server_objects{std::make_shared()}; + ServerObjectTracker m_server_objects; std::optional<::capnp::RpcSystem<::capnp::rpc::twoparty::VatId>> m_rpc_system; @@ -829,14 +829,14 @@ ProxyClientBase::~ProxyClientBase() noexcept template ProxyServerBase::ProxyServerBase(std::shared_ptr impl, Connection& connection) - : m_impl(std::move(impl)), m_context(&connection), m_server_objects(connection.m_server_objects) + : m_impl(std::move(impl)), m_context(&connection) { // Register this object with the connection's live-object tracker. This // runs on the event loop thread, so it is ordered before any connection // teardown (which also runs on the event loop thread): code that // disconnects the connection and then calls Connection::waitDrained() is // guaranteed to see this object. - m_server_objects->addServerObject(); + connection.addServerObject(); MP_LOG(*m_context.loop, Log::Debug) << "Creating " << CxxTypeName(*this) << " " << this; assert(m_impl); } @@ -882,15 +882,14 @@ ProxyServerBase::~ProxyServerBase() } assert(m_context.cleanup_fns.empty()); MP_LOG(*m_context.loop, Log::Debug) << "Destroying " << CxxTypeName(*this) << " " << this; - // Deregister this object from the connection's live-object tracker, - // through the shared m_server_objects handle since m_context.connection - // may be dangling here (see comment above). Done at the end of the - // destructor so a zero count means destruction fully completed. Note that - // any m_impl destruction scheduled through addAsyncCleanup above is NOT - // covered by the tracker: it runs later on the async cleanup thread, so - // Connection::waitDrained() waits for server call bodies, not for - // m_impl destructors. - m_server_objects->removeServerObject(); + // Deregister this object from the connection's live-object tracker + // (m_context.connection is always valid here, see comment above). Done at + // the end of the destructor so a zero count means destruction fully + // completed. Note that any m_impl destruction scheduled through + // addAsyncCleanup above is NOT covered by the tracker: it runs later on + // the async cleanup thread, so Connection::waitDrained() waits for server + // call bodies, not for m_impl destructors. + m_context.connection->removeServerObject(); } //! If the capnp interface defined a special "destroy" method, as described the diff --git a/include/mp/proxy.h b/include/mp/proxy.h index e1ec61cb..cac7c189 100644 --- a/include/mp/proxy.h +++ b/include/mp/proxy.h @@ -20,7 +20,6 @@ namespace mp { class Connection; class EventLoop; -struct ServerObjectTracker; //! Mapping from capnp interface type to proxy client implementation (specializations are generated by //! proxy-codegen.cpp). template struct ProxyClient; // IWYU pragma: export @@ -177,12 +176,6 @@ struct ProxyServerBase : public virtual Interface_::Server * wrapped. */ std::shared_ptr m_impl; ProxyContext m_context; - //! Live-object tracker shared with this object's Connection, incremented - //! in the constructor and decremented in the destructor so shutdown code - //! can wait for a disconnected connection's server objects to drain. Held - //! via shared_ptr so it remains valid if this object (kept alive by an - //! in-flight call) outlives the Connection. See ServerObjectTracker. - std::shared_ptr m_server_objects; }; //! Customizable (through template specialization) base class which ProxyServer From 4874ee0d5bb8e5e23e8aa271ee81d8e6e2da2127 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Fri, 14 Aug 2026 11:59:59 -0400 Subject: [PATCH 17/17] ci: Check out bitcoin/bitcoin PR #35932 instead of master Needed because changing m_incoming_connections type is an API change. --- .github/workflows/bitcoin-core-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bitcoin-core-ci.yml b/.github/workflows/bitcoin-core-ci.yml index 6aa1aa58..f68a4ef4 100644 --- a/.github/workflows/bitcoin-core-ci.yml +++ b/.github/workflows/bitcoin-core-ci.yml @@ -18,7 +18,8 @@ concurrency: env: BITCOIN_REPO: bitcoin/bitcoin - BITCOIN_CORE_REF: refs/heads/master + # Temporary: use PR #35932 until it merges; revert to refs/heads/master after + BITCOIN_CORE_REF: refs/pull/35932/merge LLVM_VERSION: 23 LIBCXX_DIR: /tmp/libcxx-build/