From 174419a5dd173e778f5cdd678d4ecfb7df422215 Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:05:56 -0300 Subject: [PATCH] loop: replace pipe-based sync call scheduling with kj::Executor::executeSync Before this commit, sync calls originated from external threads were coordinated over a pipe: - External threads called `EventLoop::sync()`, which stored a function in `m_sync_fn` and wrote a byte to the pipe using `m_post_writer`. - `EventLoop::loop()` waited on the read end of the pipe. When a byte arrived, it woke up and executed the function stored in `m_sync_fn`, if present. - If `m_sync_fn` was not set, it then checked the `done()` predicate to determine whether to stop the loop. `EventLoopRef::reset()` could also write to the pipe if the last reference was destroyed. KJ already provides a [cross-thread communication mechanism](https://github.com/capnproto/capnproto/blob/v0.9.2/kjdoc/tour.md#threads), which this commit takes advantage of. Additionally, get rid of the pipe-related members and replace them with a promise-fulfiller pair. The pair wakes `EventLoop::loop()` so it can exit and is now triggered only by `EventLoopRef::reset()`. With `m_post_writer` gone, the comments and conditional locking in `EventLoopRef::reset()` no longer hold. Drop them and simplify the function body. `reset()` now calls `fulfill()` with `m_mutex` held, so the `relock` parameter is removed since it is not used anymore. `sync()` no longer holds its own `EventLoopRef` while the call is pending, and `loop()` now returns the first time `done()` is true. Document what callers must do for both to be safe in the `loop()` and `sync()` declarations. Also document that `ListenConnections()` callers need to hold an `EventLoopRef` for as long as connections should be accepted. This is a purely internal improvement and does not change the mp::EventLoop API. --- include/mp/proxy-io.h | 46 ++++++++++++--------- include/mp/proxy.h | 2 +- src/mp/proxy.cpp | 95 +++++++++---------------------------------- 3 files changed, 47 insertions(+), 96 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index d772be5b..c4633cca 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -263,12 +263,17 @@ class EventLoop ~EventLoop(); //! Run event loop. Does not return until shutdown. This should only be - //! called once from the m_thread_id thread. This will block until - //! the m_num_refs reference count is 0. + //! called once from the m_thread_id thread. This will block until the + //! m_num_refs reference count drops to 0 and there are no pending async + //! cleanup functions. This only happens once and loop() returns right + //! after, so new EventLoopRef instances should only be created by code + //! that already holds a reference, or is kept alive by one. void loop(); //! Run function on event loop thread. Does not return until function completes. - //! Must be called while the loop() function is active. + //! Must be called while the loop() function is active, and the caller must + //! prevent loop() from returning until sync() returns, normally by holding + //! an EventLoopRef. void sync(kj::FunctionParam fn); //! Register cleanup function to run on asynchronous worker thread without @@ -308,35 +313,34 @@ class EventLoop //! method has not been called. std::thread m_async_thread; - //! Callback function to run on event loop thread during sync() call. - kj::FunctionParam* m_sync_fn MP_GUARDED_BY(m_mutex) = nullptr; - //! Callback functions to run on async thread. std::optional m_async_fns MP_GUARDED_BY(m_mutex); - //! Socket pair used to post and wait for wakeups to the event loop thread. - kj::Own m_wait_stream; - kj::Own m_post_stream; - - //! Synchronous writer used to write to m_post_stream. - kj::Own m_post_writer; - //! Number of EventLoopRef instances referencing this event loop. This is a //! sum of the number of client and server objects (Connection, ProxyClient, - //! ProxyServer) using the loop, plus temporary references held while - //! posting functions to the loop, plus any references held by external code - //! to keep the loop running. The loop() method exits when this count drops - //! to 0 (and m_async_fns is empty). + //! ProxyServer) using the loop, plus a temporary reference held by the + //! async thread while it runs each cleanup function, plus any references + //! held by external code to keep the loop running. The loop() method exits + //! when this count drops to 0 (and m_async_fns is empty). int m_num_refs MP_GUARDED_BY(m_mutex) = 0; - //! Mutex and condition variable used to post tasks to event loop and async - //! thread. + //! Mutex guarding the reference count and async cleanup list. Mutex m_mutex; + //! Condition variable used to wake up the async thread when a cleanup + //! function is added or when loop() returns. std::condition_variable m_cv; //! Capnp IO context. kj::AsyncIoContext m_io_context; + //! Executor for the event loop thread. sync() uses it to run functions on + //! the event loop thread when called from other threads. + const kj::Executor& m_executor; + + //! Promise that loop() waits on before returning. EventLoopRef::reset() + //! fulfills it when done() becomes true. + kj::PromiseCrossThreadFulfillerPair m_stop_promise_fulfiller_pair; + //! Capnp error handler. Needs to outlive m_task_set. LoggingErrorHandler m_error_handler{*this}; @@ -994,6 +998,10 @@ std::pair*, EventLoop::Connections::iterator> ServeSt //! Given listening socket identifier and an init object, handle incoming //! connections and requests by calling methods on the Init object. +//! +//! The listener does not hold an EventLoopRef, so the caller needs to hold +//! one for as long as connections should be accepted. Otherwise the event +//! loop exits after the last accepted connection is closed. template void ListenConnections(EventLoop& loop, SocketId fd, InitImpl& init, std::optional max_connections = std::nullopt) { diff --git a/include/mp/proxy.h b/include/mp/proxy.h index eef6962b..ff257741 100644 --- a/include/mp/proxy.h +++ b/include/mp/proxy.h @@ -60,7 +60,7 @@ class EventLoopRef ~EventLoopRef() { reset(); } EventLoop& operator*() const { assert(m_loop); return *m_loop; } EventLoop* operator->() const { assert(m_loop); return m_loop; } - void reset(bool relock=false); + void reset(); EventLoop* m_loop{nullptr}; Lock* m_lock{nullptr}; diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 552e3268..3c84221f 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -24,10 +24,10 @@ #include #include #include -#include #include #include #include +#include #include #include #include @@ -69,10 +69,7 @@ EventLoopRef::EventLoopRef(EventLoop& loop, Lock* lock) : m_loop(&loop), m_lock( m_loop->m_num_refs += 1; } -// Due to the conditionals in this function, MP_NO_TSA is required to avoid -// error "error: mutex 'loop_lock' is not held on every path through here -// [-Wthread-safety-analysis]" -void EventLoopRef::reset(bool relock) MP_NO_TSA +void EventLoopRef::reset() { if (auto* loop{m_loop}) { m_loop = nullptr; @@ -81,25 +78,11 @@ void EventLoopRef::reset(bool relock) MP_NO_TSA assert(loop->m_num_refs > 0); loop->m_num_refs -= 1; if (loop->done()) { - loop->m_cv.notify_all(); - // Capture loop->m_post_writer pointer before releasing the lock. - // The pointer can't actually change before the write() call below, - // but copying it with the lock held instead of accessing it - // directly below prevents TSAN false positives because TSAN is - // unaware of socketpair write() synchronization and might falsely - // report the pointer being used in this thread and assigned in the - // other thread without synchronization between. - kj::OutputStream* post_writer{loop->m_post_writer.get()}; - loop_lock->unlock(); - char buffer = 0; - // It safe to access post_writer here because the loop can't - // exit until this write takes place. See "Intentionally do not - // break..." comment in EventLoop::loop - post_writer->write(&buffer, 1); - // By default, do not try to relock `loop_lock` after writing, - // because the event loop could wake up and destroy itself and the - // mutex might no longer exist. - if (relock) loop_lock->lock(); + // Wake up EventLoop::loop() so it can exit. This is called with + // m_mutex held, and loop() locks m_mutex before returning, so the + // loop can't exit and destroy the fulfiller while fulfill() is + // running. Do not unlock m_mutex before this call. + loop->m_stop_promise_fulfiller_pair.fulfiller->fulfill(); } } } @@ -257,28 +240,17 @@ void EventLoop::addAsyncCleanup(std::function fn) EventLoop::EventLoop(const char* exe_name, LogOptions log_opts, void* context) : m_exe_name(exe_name), m_io_context(kj::setupAsyncIo()), + m_executor(kj::getCurrentThreadExecutor()), + m_stop_promise_fulfiller_pair(kj::newPromiseAndCrossThreadFulfiller()), m_task_set(new kj::TaskSet(m_error_handler)), m_log_opts(std::move(log_opts)), - m_context(context) -{ - auto pipe = m_io_context.provider->newTwoWayPipe(); - m_wait_stream = kj::mv(pipe.ends[0]); - m_post_stream = kj::mv(pipe.ends[1]); - KJ_IF_MAYBE(fd, m_post_stream->getFd()) { - m_post_writer = kj::heap(*fd); - } else { - throw std::logic_error("Could not get file descriptor for new pipe."); - } -} + m_context(context) {} EventLoop::~EventLoop() { if (m_async_thread.joinable()) m_async_thread.join(); const Lock lock(m_mutex); - KJ_ASSERT(m_sync_fn == nullptr); KJ_ASSERT(!m_async_fns); - KJ_ASSERT(!m_wait_stream); - KJ_ASSERT(!m_post_stream); KJ_ASSERT(m_num_refs == 0); // Spin event loop. wait for any promises triggered by RPC shutdown. @@ -298,37 +270,13 @@ void EventLoop::loop() m_async_fns.emplace(); } - kj::Own& wait_stream{m_wait_stream}; - char buffer = 0; - for (;;) { - const size_t read_bytes = wait_stream->read(&buffer, 0, 1).wait(m_io_context.waitScope); - if (read_bytes != 1) throw std::logic_error("EventLoop wait_stream closed unexpectedly"); - Lock lock(m_mutex); - if (m_sync_fn) { - // m_sync_fn throwing is never expected. If it does happen, the caller - // of EventLoop::sync() will return without any indication of failure, - // which will likely cause other bugs. Log the error and continue. - KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() MP_REQUIRES(m_mutex) { Unlock(lock, *m_sync_fn); })) { - MP_LOG(*this, Log::Error) << "EventLoop: m_sync_fn threw: " << kj::str(*exception).cStr(); - } - m_sync_fn = nullptr; - m_cv.notify_all(); - } else if (done()) { - // Intentionally do not break if m_sync_fn was set, even if done() - // would return true, to ensure that the sync() m_post_writer->write() - // call always succeeds and the loop does not exit between the time - // that the done condition is set and the write call is made. - break; - } - } + // Block execution here until all references are dropped + m_stop_promise_fulfiller_pair.promise.wait(m_io_context.waitScope); + MP_LOG(*this, Log::Info) << "EventLoop::loop done, cancelling event listeners."; m_task_set.reset(); MP_LOG(*this, Log::Info) << "EventLoop::loop bye."; - wait_stream = nullptr; const Lock lock(m_mutex); - m_post_writer = nullptr; - m_wait_stream = nullptr; - m_post_stream = nullptr; m_async_fns.reset(); m_cv.notify_all(); } @@ -339,15 +287,12 @@ void EventLoop::sync(kj::FunctionParam fn) fn(); return; } - Lock lock(m_mutex); - EventLoopRef ref(*this, &lock); - m_cv.wait(lock.m_lock, [this]() MP_REQUIRES(m_mutex) { return m_sync_fn == nullptr; }); - m_sync_fn = &fn; - Unlock(lock, [&] { - char buffer = 0; - m_post_writer->write(&buffer, 1); - }); - m_cv.wait(lock.m_lock, [this, &fn]() MP_REQUIRES(m_mutex) { return m_sync_fn != &fn; }); + // fn throwing is never expected. If it does happen, the caller of + // EventLoop::sync() will return without any indication of failure, which + // will likely cause other bugs. Log the error and continue. + KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() { m_executor.executeSync(fn); })) { + MP_LOG(*this, Log::Error) << "EventLoop: m_executor.executeSync threw: " << kj::str(*exception).cStr(); + } } void EventLoop::startAsyncThread() @@ -366,8 +311,6 @@ void EventLoop::startAsyncThread() const std::function fn = std::move(m_async_fns->front()); m_async_fns->pop_front(); Unlock(lock, fn); - // Important to relock because of the wait() call below. - ref.reset(/*relock=*/true); // Continue without waiting in case there are more async_fns continue; }