From fa2c56ec27f21eac9a102afadd49a073f74504f2 Mon Sep 17 00:00:00 2001 From: marco <*~=`'#}+{/-|&$^_@721217.xyz> Date: Thu, 11 Jun 2026 07:35:52 +0200 Subject: [PATCH 01/81] ci: Bump channel to nixos-26.05 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffaca1ca..7d6e572e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,7 +131,7 @@ jobs: runs-on: ubuntu-latest env: - NIXPKGS_CHANNEL: nixos-25.05 + NIXPKGS_CHANNEL: nixos-26.05 NIX_EXTRA_CONFIG: | keep-env-derivations = true keep-outputs = true From ce865a9ba86e603e6eb69aba53f791203b80fcb2 Mon Sep 17 00:00:00 2001 From: marco <*~=`'#}+{/-|&$^_@721217.xyz> Date: Tue, 14 Jul 2026 11:05:39 +0200 Subject: [PATCH 02/81] refactor: Directly use value in CustomBuildField This avoids a std::span ctor. Recommended in https://github.com/bitcoin-core/libmultiprocess/pull/305#pullrequestreview-4689778284 --- include/mp/type-data.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/mp/type-data.h b/include/mp/type-data.h index 09c93df9..9110be91 100644 --- a/include/mp/type-data.h +++ b/include/mp/type-data.h @@ -32,9 +32,8 @@ template void CustomBuildField(TypeList, Priority<2>, InvokeContext& invoke_context, Value&& value, Output&& output) requires (std::is_same_v && IsByteSpan) { - auto data = std::span{value}; - auto result = output.init(data.size()); - std::ranges::copy(data, result.begin()); + auto result = output.init(value.size()); + std::ranges::copy(value, result.begin()); } template From 17eab90b526e3fb713acbda1ec01321f2b9f2e84 Mon Sep 17 00:00:00 2001 From: marco <*~=`'#}+{/-|&$^_@721217.xyz> Date: Tue, 14 Jul 2026 11:06:41 +0200 Subject: [PATCH 03/81] test: Fix typo in listen_tests.cpp Recommended by the LLM in https://github.com/bitcoin/bitcoin/pull/35684#issuecomment-4916571942 --- test/mp/test/listen_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mp/test/listen_tests.cpp b/test/mp/test/listen_tests.cpp index 1506eaa4..24ca1652 100644 --- a/test/mp/test/listen_tests.cpp +++ b/test/mp/test/listen_tests.cpp @@ -225,7 +225,7 @@ KJ_TEST("ListenConnections enforces a local connection limit") { // With max-connections=1, the second socket can connect to the kernel // backlog, but ListenConnections should not accept or serve it until the - // first accepts clients disconnects. + // first accepted client disconnects. ListenSetup server(/*max_connections=*/1); From 8550ee6a31726832218f7f3cc2e473603a5eb1d2 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 20 Jul 2026 13:47:25 +0200 Subject: [PATCH 04/81] util, refactor: Add ChildFail helper for post-fork child errors Factor out the write-message-then-_exit(126) pattern used when a syscall fails in the post-fork child of SpawnProcess, and document its constraints (async-signal-safe calls only, message must not allocate) in one place. The execvp failure path is intentionally left alone: replacing perror() there would lose the errno string, and improving it is already covered by the existing TODO about reporting errors to the parent via a pipe. Co-Authored-By: Claude Fable 5 --- src/mp/util.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/mp/util.cpp b/src/mp/util.cpp index 42cda9ce..0be367fc 100644 --- a/src/mp/util.cpp +++ b/src/mp/util.cpp @@ -60,6 +60,18 @@ size_t MaxFd() } } +//! Report an error and exit from the post-fork child of a multi-threaded +//! process, where only async-signal-safe calls (like write and _exit) are +//! allowed. Accepting only a reference to a char array (in practice a string +//! literal) ensures no allocation is needed at the call site. +template +[[noreturn]] void ChildFail(const char (&msg)[N]) noexcept +{ + const ssize_t written = ::write(STDERR_FILENO, msg, N - 1); + (void)written; + _exit(126); +} + } // namespace std::string ThreadName(const char* exe_name) @@ -147,10 +159,7 @@ std::tuple SpawnProcess(SpawnConnectInfoToArgsFn&& connect_ (void)close(fds[1]); throw std::system_error(errno, std::system_category(), "close"); } - static constexpr char msg[] = "SpawnProcess(child): close(fds[1]) failed\n"; - const ssize_t writeResult = ::write(STDERR_FILENO, msg, sizeof(msg) - 1); - (void)writeResult; - _exit(126); + ChildFail("SpawnProcess(child): close(fds[1]) failed\n"); } if (!pid) { From 1e0c7ff9a51aedf9ddbda13dc3d8f03428da1c87 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 20 Jul 2026 13:49:29 +0200 Subject: [PATCH 05/81] util: Clear FD_CLOEXEC in child instead of parent before fork SpawnProcess cleared FD_CLOEXEC on the child's socket in the parent before forking. Between that fcntl() and fork(), a concurrent fork+exec on another thread would snapshot the descriptor with the close-on-exec flag already cleared, leaking it into an unrelated child process where it survives exec. A leaked duplicate of the socket also prevents the parent from seeing EOF when the spawned process exits. Instead, keep both descriptors close-on-exec in the parent for their entire lifetime and have the intended child clear the flag on its own copy of the descriptor between fork() and exec(). Co-Authored-By: Claude Fable 5 --- src/mp/util.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/mp/util.cpp b/src/mp/util.cpp index 0be367fc..f524ab23 100644 --- a/src/mp/util.cpp +++ b/src/mp/util.cpp @@ -142,11 +142,6 @@ std::tuple SpawnProcess(SpawnConnectInfoToArgsFn&& connect_ const std::vector args{connect_info_to_args(std::to_string(fds[0]))}; const std::vector argv{MakeArgv(args)}; - // Clear FD_CLOEXEC on fds[0] before forking so it survives exec in the child. - int fds0_flags; - KJ_SYSCALL(fds0_flags = fcntl(fds[0], F_GETFD)); - KJ_SYSCALL(fcntl(fds[0], F_SETFD, fds0_flags & ~FD_CLOEXEC)); - ProcessId pid = fork(); if (pid == -1) { throw std::system_error(errno, std::system_category(), "fork"); @@ -172,6 +167,16 @@ std::tuple SpawnProcess(SpawnConnectInfoToArgsFn&& connect_ } } + // Clear FD_CLOEXEC on socket 0 so it survives exec. Doing this here + // rather than in the parent before forking avoids a window where a + // concurrent fork in another thread would leak the descriptor into an + // unrelated child process (which would not clear it). + // fcntl is async-signal-safe. + const int fds0_flags = fcntl(fds[0], F_GETFD); + if (fds0_flags == -1 || fcntl(fds[0], F_SETFD, fds0_flags & ~FD_CLOEXEC) == -1) { + ChildFail("SpawnProcess(child): clearing FD_CLOEXEC failed\n"); + } + execvp(argv[0], argv.data()); // NOTE: perror() is not async-signal-safe; calling it here in a // post-fork child may deadlock in multithreaded parents. From 140d9ba6ff78a18d3480ed5d8a1fc76e0a68b9ed Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:50:41 -0300 Subject: [PATCH 06/81] test: allow custom log handler in `ListenSetup` Add `DefaultLogHandler` and a `log_handler` parameter so tests can provide their own handler and observe log output more easily. --- test/mp/test/listen_tests.cpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/mp/test/listen_tests.cpp b/test/mp/test/listen_tests.cpp index 1506eaa4..8356c8e2 100644 --- a/test/mp/test/listen_tests.cpp +++ b/test/mp/test/listen_tests.cpp @@ -129,18 +129,23 @@ class ClientSetup std::thread thread; }; +//! Default server event loop log handler, throws so tests can assert on errors. +void DefaultLogHandler(mp::LogMessage log) +{ + KJ_LOG(INFO, log.level, log.message); + if (log.level == mp::Log::Raise) throw std::runtime_error(log.message); +} + //! Runs a server EventLoop on its own thread, starts ListenConnections() on a //! UnixListener socket, and records connection/disconnection counts through //! EventLoop test hooks class ListenSetup { public: - explicit ListenSetup(std::optional max_connections = std::nullopt) - : thread([this, max_connections] { - EventLoop loop("mptest-server", [](mp::LogMessage log) { - KJ_LOG(INFO, log.level, log.message); - if (log.level == mp::Log::Raise) throw std::runtime_error(log.message); - }); + explicit ListenSetup(std::optional max_connections = std::nullopt, + mp::LogFn log_handler = DefaultLogHandler) + : thread([this, max_connections, log_handler] { + EventLoop loop("mptest-server", log_handler); loop.testing_hook_disconnected = [&] { Lock lock(counter_mutex); ++disconnected_count; From a26a08496b8bd9c43999e3636f9f53ec9022d124 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Tue, 21 Jul 2026 11:15:14 -0400 Subject: [PATCH 07/81] cmake: Fix mpgen capnp tool path for vcpkg/Windows builds Error seen (MSVC native build): mpgen error: mp\util.cpp:332: failed: CreateProcessA(...): #2 The system cannot find the file specified. Problem: mpgen constructs capnp tool paths as capnp_PREFIX + "/bin/capnp" and capnp_PREFIX + "/bin/capnpc-c++". capnp_PREFIX is derived from the capnproto include directory parent, so on a vcpkg install it resolves to something like vcpkg_installed/x64-windows-release. But vcpkg installs executables under tools/capnproto/, not bin/, so the constructed path does not exist and CreateProcessA fails with ERROR_FILE_NOT_FOUND. Fix: pass the CAPNP_EXECUTABLE and CAPNPC_CXX_EXECUTABLE cmake variables from CapnProtoConfig.cmake as compile definitions to mpgen, and use them in gen.cpp instead of constructing paths from capnp_PREFIX. Using these cmake variables rather than reading the CapnProto::capnp_tool imported target IMPORTED_LOCATION directly is important: on Debian/Ubuntu, IMPORTED_LOCATION is set to a non-existent path due to a multiarch cmake packaging bug in libcapnp-dev (see https://github.com/bitcoin-core/libmultiprocess/issues/328). The cmake variables avoid this because CapnProtoConfig.cmake hardcodes the correct paths on Debian/Ubuntu, and on other platforms (including vcpkg) the variables hold generator expressions that target_compile_definitions evaluates correctly at generation time. Co-Authored-By: Claude Sonnet 4.6 --- CMakeLists.txt | 10 ++++++++++ src/mp/gen.cpp | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6354b0ee..f6f2662d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -209,6 +209,16 @@ target_link_libraries(mpgen PRIVATE CapnProto::capnp-rpc) target_link_libraries(mpgen PRIVATE CapnProto::capnpc) target_link_libraries(mpgen PRIVATE CapnProto::kj) target_link_libraries(mpgen PRIVATE Threads::Threads) +# Use CAPNP_EXECUTABLE/CAPNPC_CXX_EXECUTABLE cmake variables set by +# CapnProtoConfig.cmake rather than reading the CapnProto::capnp_tool +# IMPORTED_LOCATION directly. On Debian/Ubuntu, IMPORTED_LOCATION is broken +# by a multiarch cmake packaging bug (see +# https://github.com/bitcoin-core/libmultiprocess/issues/328), but +# CapnProtoConfig.cmake works around it by hardcoding the correct paths. On +# other platforms the variables hold generator expressions that +# target_compile_definitions evaluates correctly at generation time. +target_compile_definitions(mpgen PRIVATE CAPNP_EXECUTABLE=\"${CAPNP_EXECUTABLE}\") +target_compile_definitions(mpgen PRIVATE CAPNPC_CXX_EXECUTABLE=\"${CAPNPC_CXX_EXECUTABLE}\") set_target_properties(mpgen PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) set_target_properties(mpgen PROPERTIES diff --git a/src/mp/gen.cpp b/src/mp/gen.cpp index 435ba441..7733eba6 100644 --- a/src/mp/gen.cpp +++ b/src/mp/gen.cpp @@ -272,7 +272,7 @@ static void Generate(kj::StringPtr src_prefix, if (p != std::string::npos) include_base.erase(p); std::vector args; - args.emplace_back(capnp_PREFIX "/bin/capnp"); + args.emplace_back(CAPNP_EXECUTABLE); args.emplace_back("compile"); args.emplace_back("--src-prefix="); args.back().append(src_prefix.cStr(), src_prefix.size()); @@ -280,11 +280,11 @@ static void Generate(kj::StringPtr src_prefix, args.emplace_back("--import-path="); args.back().append(import_path.cStr(), import_path.size()); } - args.emplace_back("--output=" capnp_PREFIX "/bin/capnpc-c++"); + args.emplace_back("--output=" CAPNPC_CXX_EXECUTABLE); args.emplace_back(src_file); const int status = mp::WaitProcess(mp::StartProcess(args)); if (status) { - throw std::runtime_error("Invoking " capnp_PREFIX "/bin/capnp failed"); + throw std::runtime_error("Invoking " CAPNP_EXECUTABLE " failed"); } const capnp::SchemaParser parser; From eea9c64f6e1682f7ad7c4a0e7ffbcc8c61619b21 Mon Sep 17 00:00:00 2001 From: ViniciusCestarii Date: Wed, 22 Jul 2026 11:49:35 -0300 Subject: [PATCH 08/81] cmake: Fix stale codegen when mpgen binary changes --- cmake/TargetCapnpSources.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/TargetCapnpSources.cmake b/cmake/TargetCapnpSources.cmake index f04a9657..2334e260 100644 --- a/cmake/TargetCapnpSources.cmake +++ b/cmake/TargetCapnpSources.cmake @@ -83,7 +83,7 @@ function(target_capnp_sources target include_prefix) add_custom_command( OUTPUT ${capnp_file}.c++ ${capnp_file}.h ${capnp_file}.proxy-client.c++ ${capnp_file}.proxy-types.h ${capnp_file}.proxy-server.c++ ${capnp_file}.proxy-types.c++ ${capnp_file}.proxy.h COMMAND ${MPGEN_BINARY} ${CMAKE_CURRENT_SOURCE_DIR} ${include_prefix} ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_file} ${TCS_IMPORT_PATHS} ${mp_include_dir} - DEPENDS ${capnp_file} + DEPENDS ${capnp_file} ${MPGEN_BINARY} VERBATIM ) # Skip linting for capnp-generated files but keep it for mpgen-generated ones From 86b48105018e2908748dc69d726b6e5543cc0d65 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Wed, 29 Jul 2026 13:43:33 +0200 Subject: [PATCH 09/81] refactor: access ThreadContext through CurrentThread() Introduce CurrentThread() as an accessor for the existing namespace-scope thread_local object and migrate internal callers to it. This does not change storage duration, initialization, destruction, the exported TLS symbol, or its lint suppressions. Co-authored-by: Ryan Ofsky --- include/mp/proxy-io.h | 22 +++++++++++++++++++++- include/mp/proxy-types.h | 14 +++++++------- include/mp/type-context.h | 2 +- src/mp/proxy.cpp | 33 +++++++++++++++++++-------------- test/mp/test/test.cpp | 4 ++-- 5 files changed, 50 insertions(+), 25 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index c2a8d433..f15965bb 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -694,7 +694,7 @@ using ConnThread = ConnThreads::iterator; // inserted bool. std::tuple SetThread(GuardedRef threads, Connection* connection, const std::function& make_thread); -//! The thread_local ThreadContext g_thread_context struct provides information +//! The thread_local ThreadContext struct (see CurrentThread()) provides information //! about individual threads and a way of communicating between them. Because //! it's a thread local struct, each ThreadContext instance is initialized by //! the thread that owns it. @@ -934,6 +934,26 @@ extern thread_local ThreadContext g_thread_context; // NOLINT(bitcoin-nontrivial // cannot be thread_local" which should not be a problem on modern platforms, and // could lead to a small memory leak at worst on older ones. +//! Return the current thread's ThreadContext. +//! +//! Why per-thread state is needed at all: libmultiprocess has no control over +//! which threads the C++ application uses to call ProxyClient methods after +//! the proxy objects are returned to it. The thread-mapping model (see +//! "Thread Mapping" in doc/design.md) gives each application thread making +//! IPC calls a dedicated server-side thread that executes its requests, so +//! thread-local state and recursive mutexes work as expected across the +//! process boundary and callbacks from the server run on the originating +//! client thread. The client-side handles for those dedicated server threads +//! (the ProxyClient objects returned by ThreadMap.makeThread, stored +//! per connection in the request_threads / callback_threads maps below) are +//! state that must be keyed implicitly by the calling thread, and must be +//! released when the client thread exits so the corresponding server threads +//! are freed. A thread_local object is the C++ mechanism that provides both +//! of these: per-thread storage plus a destructor that runs at thread exit +//! (the C equivalent would be a pthread key destructor). This is why +//! ThreadContext is thread_local and why its destructor is nontrivial. +ThreadContext& CurrentThread(); + } // namespace mp #endif // MP_PROXY_IO_H diff --git a/include/mp/proxy-types.h b/include/mp/proxy-types.h index f1127da5..ed458209 100644 --- a/include/mp/proxy-types.h +++ b/include/mp/proxy-types.h @@ -691,9 +691,9 @@ void serverDestroy(Server& server) template void clientInvoke(ProxyClient& proxy_client, const GetRequest& get_request, FieldObjs&&... fields) { - if (!g_thread_context.waiter) { - assert(g_thread_context.thread_name.empty()); - g_thread_context.thread_name = ThreadName(proxy_client.m_context.loop->m_exe_name); + if (!CurrentThread().waiter) { + assert(CurrentThread().thread_name.empty()); + CurrentThread().thread_name = ThreadName(proxy_client.m_context.loop->m_exe_name); // If next assert triggers, it means clientInvoke is being called from // the capnp event loop thread. This can happen when a ProxyServer // method implementation that runs synchronously on the event loop @@ -702,13 +702,13 @@ void clientInvoke(ProxyClient& proxy_client, const GetRequest& get_request, Fiel // run asynchronously off the event loop thread. This is easy to fix by // just adding a 'context :Proxy.Context' argument to the capnp method // declaration so the server method runs in a dedicated thread. - assert(!g_thread_context.loop_thread); - g_thread_context.waiter = std::make_unique(); + assert(!CurrentThread().loop_thread); + CurrentThread().waiter = std::make_unique(); MP_LOGPLAIN(*proxy_client.m_context.loop, Log::Info) - << "{" << g_thread_context.thread_name + << "{" << CurrentThread().thread_name << "} IPC client first request from current thread, constructing waiter"; } - ThreadContext& thread_context{g_thread_context}; + ThreadContext& thread_context{CurrentThread()}; std::optional invoke_context; // Must outlive waiter->wait() call below std::exception_ptr exception; std::string kj_exception; diff --git a/include/mp/type-context.h b/include/mp/type-context.h index b63959f4..54007207 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -93,7 +93,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // call. In this case, the callbackThread value should point // to the same thread already in the map, so there is no // need to update the map. - auto& thread_context = g_thread_context; + auto& thread_context = CurrentThread(); auto& request_threads = thread_context.request_threads; ConnThread request_thread; bool inserted{false}; diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index a0a7aba0..4c7f7666 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -42,6 +42,11 @@ namespace mp { thread_local ThreadContext g_thread_context; // NOLINT(bitcoin-nontrivial-threadlocal) +ThreadContext& CurrentThread() +{ + return g_thread_context; +} + Stream MakeStream(EventLoop&loop, SocketId socket) { Stream stream; @@ -283,9 +288,9 @@ EventLoop::~EventLoop() void EventLoop::loop() { - assert(!g_thread_context.loop_thread); - g_thread_context.loop_thread = true; - KJ_DEFER(g_thread_context.loop_thread = false); + assert(!CurrentThread().loop_thread); + CurrentThread().loop_thread = true; + KJ_DEFER(CurrentThread().loop_thread = false); { const Lock lock(m_mutex); @@ -479,11 +484,11 @@ kj::Promise ProxyServer::makePool(MakePoolContext context) const std::string thread_name = "pool/" + std::to_string(i); std::promise thread_context; std::thread thread([&loop, &thread_context, thread_name]() { - g_thread_context.thread_name = ThreadName(loop.m_exe_name) + " (" + thread_name + ")"; - g_thread_context.waiter = std::make_unique(); - Lock lock(g_thread_context.waiter->m_mutex); - thread_context.set_value(&g_thread_context); - g_thread_context.waiter->wait(lock, [] { return !g_thread_context.waiter; }); + CurrentThread().thread_name = ThreadName(loop.m_exe_name) + " (" + thread_name + ")"; + CurrentThread().waiter = std::make_unique(); + Lock lock(CurrentThread().waiter->m_mutex); + thread_context.set_value(&CurrentThread()); + CurrentThread().waiter->wait(lock, [] { return !CurrentThread().waiter; }); }); auto thread_server = kj::heap>(m_connection, *thread_context.get_future().get(), std::move(thread)); m_connection.m_thread_pool.push_back({m_connection.m_threads.add(kj::mv(thread_server))}); @@ -498,14 +503,14 @@ kj::Promise ProxyServer::makeThread(MakeThreadContext context) const std::string from = context.getParams().getName(); std::promise thread_context; std::thread thread([&loop, &thread_context, from]() { - g_thread_context.thread_name = ThreadName(loop.m_exe_name) + " (from " + from + ")"; - g_thread_context.waiter = std::make_unique(); - Lock lock(g_thread_context.waiter->m_mutex); - thread_context.set_value(&g_thread_context); + CurrentThread().thread_name = ThreadName(loop.m_exe_name) + " (from " + from + ")"; + CurrentThread().waiter = std::make_unique(); + Lock lock(CurrentThread().waiter->m_mutex); + thread_context.set_value(&CurrentThread()); if (loop.testing_hook_makethread_created) loop.testing_hook_makethread_created(); // Wait for shutdown signal from ProxyServer destructor (signal // is just waiter getting set to null.) - g_thread_context.waiter->wait(lock, [] { return !g_thread_context.waiter; }); + CurrentThread().waiter->wait(lock, [] { return !CurrentThread().waiter; }); }); auto thread_server = kj::heap>(m_connection, *thread_context.get_future().get(), std::move(thread)); auto thread_client = m_connection.m_threads.add(kj::mv(thread_server)); @@ -517,7 +522,7 @@ std::atomic server_reqs{0}; std::string LongThreadName(const char* exe_name) { - return g_thread_context.thread_name.empty() ? ThreadName(exe_name) : g_thread_context.thread_name; + return CurrentThread().thread_name.empty() ? ThreadName(exe_name) : CurrentThread().thread_name; } kj::StringPtr KJ_STRINGIFY(Log v) diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 7a0940c8..f5f35437 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -462,7 +462,7 @@ KJ_TEST("Make simultaneous IPC calls on single remote thread") // that will be used for the test. setup.server->m_impl->m_fn = [&] {}; foo->callFnAsync(); - ThreadContext& tc{g_thread_context}; + ThreadContext& tc{CurrentThread()}; Thread::Client *callback_thread, *request_thread; foo->m_context.loop->sync([&] { Lock lock(tc.waiter->m_mutex); @@ -516,7 +516,7 @@ KJ_TEST("Call async IPC method dispatched to pool thread") foo->initThreadMap(); setup.server->m_impl->m_int_fn = [](int n) { return n * 2; }; - ThreadContext& tc{g_thread_context}; + ThreadContext& tc{CurrentThread()}; std::atomic running{3}; std::promise pool_ready; foo->m_context.loop->sync([&] { From 66298c737f43b12009e25396383aeb98b6c8164b Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:54:52 +0100 Subject: [PATCH 10/81] ci: Switch back to Bitcoin Core's master branch bitcoin/bitcoin#35084 has now been merged. --- .github/workflows/bitcoin-core-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/bitcoin-core-ci.yml b/.github/workflows/bitcoin-core-ci.yml index 83dff816..f14d5682 100644 --- a/.github/workflows/bitcoin-core-ci.yml +++ b/.github/workflows/bitcoin-core-ci.yml @@ -18,8 +18,7 @@ concurrency: env: BITCOIN_REPO: bitcoin/bitcoin - # Temporary: use PR #35084 until it merges; revert to refs/heads/master after - BITCOIN_CORE_REF: refs/pull/35084/merge + BITCOIN_CORE_REF: refs/heads/master LLVM_VERSION: 22 LIBCXX_DIR: /tmp/libcxx-build/ From 3a997e113cf781222022d2a61ce15e05efd43cdd Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Thu, 30 Jul 2026 11:18:16 -0300 Subject: [PATCH 11/81] Fix startup race in mpexample Since #274, connection setup has been split into two calls: `MakeStream` followed by `ConnectStream`. Between these calls, the event loop's reference count can temporarily drop to zero, allowing `EventLoop::loop()` to exit before `ConnectStream` posts its work, crashing the example on startup. Keep an `EventLoopRef` alive in main() so the loop stays running while it is in use. --- example/example.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/example/example.cpp b/example/example.cpp index c6b32f0d..e4d159ab 100644 --- a/example/example.cpp +++ b/example/example.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -55,6 +56,7 @@ int main(int argc, char** argv) loop.loop(); }); mp::EventLoop* loop = promise.get_future().get(); + mp::EventLoopRef loop_ref{*loop}; auto [printer_init, printer_pid] = Spawn(*loop, argv[0], "mpprinter"); auto [calc_init, calc_pid] = Spawn(*loop, argv[0], "mpcalculator"); @@ -71,6 +73,7 @@ int main(int argc, char** argv) mp::WaitProcess(calc_pid); printer_init.reset(); mp::WaitProcess(printer_pid); + loop_ref.reset(); loop_thread.join(); std::cout << "Bye!" << std::endl; return 0; From 36c6c63520e2f23a67c21c21598adc1e2672b9cf Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:39:27 -0300 Subject: [PATCH 12/81] doc: Document reference-counted EventLoop lifetime `EventLoop::loop()` exits when the last `EventLoopRef` is released, so code making multiple calls against the loop needs to hold its own reference. State this in the `EventLoopRef` comment and usage.md. --- doc/usage.md | 6 ++++++ include/mp/proxy.h | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/usage.md b/doc/usage.md index df103e9f..6b294ce1 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -12,6 +12,12 @@ The `*.capnp` data definition files are consumed by the _libmultiprocess_ code g The `ProxyServer` objects help translate IPC requests from a socket to method calls on a local object. The `ProxyServer` objects are just used internally by the `mp::ServeStream(loop, socket, wrapped_object)` and `mp::ListenConnections(loop, socket, wrapped_object[, max_connections])` functions, and aren't exposed externally. The `ProxyClient` classes are exposed, and returned from the `mp::ConnectStream(loop, socket)` function and meant to be used directly. The classes implement methods described in `.capnp` definitions, and whenever any method is called, a request with the method arguments is sent over the associated IPC connection, and the corresponding `wrapped_object` method on the other end of the connection is called, with the `ProxyClient` method blocking until it returns and forwarding back any return value to the `ProxyClient` method caller. +## Event loop + +Making connections and IPC calls requires an `mp::EventLoop` instance running in its own dedicated thread. The event loop is reference-counted, and `EventLoop::loop()` returns when the last `mp::EventLoopRef` is released. Therefore, any code that needs to use the loop across multiple calls should maintain its own `EventLoopRef`. Connection objects and proxy objects automatically hold a reference for as long as they exist. + +For a typical setup, see [example.cpp](../example/example.cpp). + ## Example A simple interface description can be found at [test/mp/test/foo.capnp](../test/mp/test/foo.capnp), implementation in [test/mp/test/foo.h](../test/mp/test/foo.h), and usage in [test/mp/test/test.cpp](../test/mp/test/test.cpp). diff --git a/include/mp/proxy.h b/include/mp/proxy.h index 8d63e9aa..2144b571 100644 --- a/include/mp/proxy.h +++ b/include/mp/proxy.h @@ -44,7 +44,9 @@ inline void CleanupRun(CleanupList& fns) { } } -//! Event loop smart pointer automatically managing m_num_refs. +//! Event loop smart pointer automatically managing m_num_refs. Hold one +//! reference to keep the loop running, once the last reference is released +//! EventLoop::loop() will exit. //! If a lock pointer argument is passed, the specified lock will be used, //! otherwise EventLoop::m_mutex will be locked when needed. class EventLoopRef From 496fb84e69f8dbf8fd9e2d29a03b44e1c73db3d2 Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:28:58 -0300 Subject: [PATCH 13/81] test: cover immediate client disconnects for `ListenConnections` Connect to the listening socket and `close()` the connection right away, note that this is racy and the connection might be accepted normally if the `close()` syscall doesn't win the race. Additionally, a custom log handler is added to catch "Uncaught exception in daemonized task." logs, generated by a Cap'n Proto bug that shows up on macOS, so CI does not fail on them. --- test/mp/test/listen_tests.cpp | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/mp/test/listen_tests.cpp b/test/mp/test/listen_tests.cpp index 8356c8e2..d284f103 100644 --- a/test/mp/test/listen_tests.cpp +++ b/test/mp/test/listen_tests.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -294,6 +295,43 @@ KJ_TEST("ListenConnections accepts multiple connections") KJ_EXPECT(client3->client->add(3, 4) == 7); } +KJ_TEST("ListenConnections handles a client that disconnects before being accepted") +{ + Mutex mutex; + bool accept_error = false; + ListenSetup server(std::nullopt, [&](mp::LogMessage log) { + // On macOS, accept() can fail if the peer closes the connection before it is accepted. + // The event loop then reports this as an uncaught task exception. We catch and ignore + // this specific error here so that the corresponding CI job does not fail. + // + // This is a Cap'n Proto bug, a fix is available in the v2 branch at: https://github.com/capnproto/capnproto/commit/7df5bd078f389ded313479981bd0ae06cbcdfe1b#diff-ec577ad66535f58f6d7396ea51d3e56c0065308aa8fb02751cd6a8cfaa67252fR1358-R1372 + if (log.level == mp::Log::Error && log.message.find("Uncaught exception in daemonized task.") != std::string::npos) { + Lock lock(mutex); + accept_error = true; + } + DefaultLogHandler(log); + }); + + // This is racy, if the close does not happen before accept(), + // the connection is accepted normally. + int fd = server.listener.MakeConnectedSocket(); + KJ_SYSCALL(close(fd)); + + // Wait for the connection to either be accepted and disconnected, or fail + // to be accepted and log the error above. + const auto deadline = std::chrono::steady_clock::now() + FAILURE_TIMEOUT; + auto accept_error_seen = [&] { Lock lock(mutex); return accept_error; }; + while (!accept_error_seen() && server.DisconnectedCount() < 1 && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + +#ifndef __APPLE__ + // No error is expected on platforms other than macOS. + KJ_REQUIRE(!accept_error_seen()); +#endif + KJ_EXPECT(server.DisconnectedCount() == (accept_error_seen() ? 0 : 1)); +} + } // namespace } // namespace test } // namespace mp From 45f6255975da63da58cbd0675c3ffe44b5c38db3 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 29 Jul 2026 20:20:31 -0400 Subject: [PATCH 14/81] type-number: exclude bool from the integral overload Exclude bool from the integral BuildPrimitive overload (bool is integral per std::is_integral_v but not a standard integer per std::cmp_*); add a dedicated bool overload that static_asserts the capnp LocalType is also bool, so a schema mismatch (e.g. Float64 used for a bool parameter) produces a clear error pointing at the schema. This change is also needed so the next commit can tighten the integer range changes in the BuildPrimitive integer overload. Co-Authored-By: Claude Sonnet 4.6 --- include/mp/type-number.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/include/mp/type-number.h b/include/mp/type-number.h index 380d70c9..6df13a04 100644 --- a/include/mp/type-number.h +++ b/include/mp/type-number.h @@ -21,7 +21,7 @@ LocalType BuildPrimitive(InvokeContext& invoke_context, } template - requires std::is_integral_v + requires (std::is_integral_v && !std::is_same_v, bool>) LocalType BuildPrimitive(InvokeContext& invoke_context, const Value& value, TypeList) @@ -33,6 +33,14 @@ LocalType BuildPrimitive(InvokeContext& invoke_context, return value; } +template +LocalType BuildPrimitive(InvokeContext& invoke_context, bool value, TypeList) +{ + static_assert(std::is_same_v, + "capnp field type should be Bool for bool parameters. Fix the .capnp schema."); + return value; +} + template requires std::is_floating_point_v LocalType BuildPrimitive(InvokeContext& invoke_context, From 45b685c3f58397b41c0d34915cc95850b78ba211 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Thu, 9 Jul 2026 10:07:34 -0400 Subject: [PATCH 15/81] type-number, type-chrono: Fix static assert signed/unsigned comparisons Use std::cmp_less_equal/std::cmp_greater_equal (C++20) in the range static_asserts to avoid signed/unsigned comparison pitfall: when one side is unsigned and the other signed, the usual arithmetic conversions silently convert the signed lowest() to a huge positive number, making the assert pass when it should fire. Also fix the identical pattern in type-number.h BuildPrimitive. Problem was reported by ViniciusCestarii in https://github.com/bitcoin-core/libmultiprocess/pull/303#discussion_r3551873101 Co-Authored-By: Claude Sonnet 4.6 --- include/mp/type-chrono.h | 4 ++-- include/mp/type-number.h | 7 ++++--- include/mp/util.h | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/include/mp/type-chrono.h b/include/mp/type-chrono.h index d7154986..da1f22fc 100644 --- a/include/mp/type-chrono.h +++ b/include/mp/type-chrono.h @@ -16,9 +16,9 @@ template void CustomBuildField(TypeList>, Priority<1>, InvokeContext& invoke_context, Value&& value, Output&& output) { - static_assert(std::numeric_limits::lowest() <= std::numeric_limits::lowest(), + static_assert(safe_less_equal(std::numeric_limits::lowest(), std::numeric_limits::lowest()), "capnp type does not have enough range to hold lowest std::chrono::duration value"); - static_assert(std::numeric_limits::max() >= std::numeric_limits::max(), + static_assert(safe_greater_equal(std::numeric_limits::max(), std::numeric_limits::max()), "capnp type does not have enough range to hold highest std::chrono::duration value"); output.set(value.count()); } diff --git a/include/mp/type-number.h b/include/mp/type-number.h index 6df13a04..7ad21e3a 100644 --- a/include/mp/type-number.h +++ b/include/mp/type-number.h @@ -6,6 +6,7 @@ #define MP_PROXY_TYPE_NUMBER_H #include +#include namespace mp { template @@ -16,7 +17,7 @@ LocalType BuildPrimitive(InvokeContext& invoke_context, { using E = std::make_unsigned_t>; using T = std::make_unsigned_t; - static_assert(std::numeric_limits::max() >= std::numeric_limits::max(), "mismatched integral/enum types"); + static_assert(std::cmp_greater_equal(std::numeric_limits::max(), std::numeric_limits::max()), "mismatched integral/enum types"); return static_cast(value); } @@ -27,9 +28,9 @@ LocalType BuildPrimitive(InvokeContext& invoke_context, TypeList) { static_assert( - std::numeric_limits::lowest() <= std::numeric_limits::lowest(), "mismatched integral types"); + std::cmp_less_equal(std::numeric_limits::lowest(), std::numeric_limits::lowest()), "mismatched integral types"); static_assert( - std::numeric_limits::max() >= std::numeric_limits::max(), "mismatched integral types"); + std::cmp_greater_equal(std::numeric_limits::max(), std::numeric_limits::max()), "mismatched integral types"); return value; } diff --git a/include/mp/util.h b/include/mp/util.h index f647d5d7..eac655be 100644 --- a/include/mp/util.h +++ b/include/mp/util.h @@ -33,6 +33,21 @@ namespace mp { //! Generic utility functions used by capnp code. +// std::cmp_less_equal/cmp_greater_equal only accept integer types (C++20 ยง[utility.intcmp]). +// These wrappers fall back to regular comparison for floating-point types. +template +constexpr bool safe_less_equal(A a, B b) +{ + if constexpr (std::is_floating_point_v || std::is_floating_point_v) return a <= b; + else return std::cmp_less_equal(a, b); +} +template +constexpr bool safe_greater_equal(A a, B b) +{ + if constexpr (std::is_floating_point_v || std::is_floating_point_v) return a >= b; + else return std::cmp_greater_equal(a, b); +} + //! Type holding a list of types. //! //! Example: From 7a72df02e2d46f953c4c97662c3e109aa3874ea9 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Sat, 6 Jun 2026 18:27:03 -0400 Subject: [PATCH 16/81] type-chrono: Add CustomBuildField/CustomReadField overloads for std::chrono::time_point Needed by bitcoin/bitcoin#34882 which uses NodeClock::time_point in the Node stats struct (m_last_send, m_last_recv, m_ping_start), requiring IPC serialization support for time_point types. Co-Authored-By: Claude Sonnet 4.6 --- include/mp/type-chrono.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/include/mp/type-chrono.h b/include/mp/type-chrono.h index da1f22fc..94b0f1f4 100644 --- a/include/mp/type-chrono.h +++ b/include/mp/type-chrono.h @@ -29,6 +29,29 @@ decltype(auto) CustomReadField(TypeList>, Pri { return read_dest.construct(input.get()); } + +//! Overload CustomBuildField and CustomReadField to serialize +//! std::chrono::time_point parameters and return values as integer tick counts. +//! The capnp field type must be an integer type with enough range to hold the +//! time_since_epoch() count for the given Duration (e.g. Int64 for nanoseconds). +template +void CustomBuildField(TypeList>, Priority<1>, InvokeContext& invoke_context, + Value&& value, Output&& output) +{ + using Rep = typename Duration::rep; + static_assert(safe_less_equal(std::numeric_limits::lowest(), std::numeric_limits::lowest()), + "capnp type does not have enough range to hold lowest std::chrono::time_point value"); + static_assert(safe_greater_equal(std::numeric_limits::max(), std::numeric_limits::max()), + "capnp type does not have enough range to hold highest std::chrono::time_point value"); + output.set(value.time_since_epoch().count()); +} + +template +decltype(auto) CustomReadField(TypeList>, Priority<1>, + InvokeContext& invoke_context, Input&& input, ReadDest&& read_dest) +{ + return read_dest.construct(std::chrono::time_point{Duration{input.get()}}); +} } // namespace mp #endif // MP_PROXY_TYPE_CHRONO_H From 26452e02d75e8a2a33752ef181f7ffc96ca968f0 Mon Sep 17 00:00:00 2001 From: marco <*~=`'#}+{/-|&$^_@721217.xyz> Date: Tue, 14 Jul 2026 11:08:48 +0200 Subject: [PATCH 17/81] refactor: memcpy -> std::ranges::copy I think there is no issue of passing a nullptr to memcpy here, but it makes sense to use the std-lib copy for consistency. Recommended in https://github.com/bitcoin-core/libmultiprocess/pull/305#pullrequestreview-4689778284 Also, add a size sanity check when reading a char array. --- include/mp/type-char.h | 12 +++++++++--- include/mp/type-string.h | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/include/mp/type-char.h b/include/mp/type-char.h index b51ffeb1..f4023334 100644 --- a/include/mp/type-char.h +++ b/include/mp/type-char.h @@ -7,6 +7,10 @@ #include +#include +#include +#include + namespace mp { template void CustomBuildField(TypeList, @@ -16,7 +20,7 @@ void CustomBuildField(TypeList, Output&& output) { auto result = output.init(size); - memcpy(result.begin(), value, size); + std::ranges::copy(value, result.begin()); } template @@ -26,9 +30,11 @@ decltype(auto) CustomReadField(TypeList, Input&& input, ReadDest&& read_dest) { + auto data = input.get(); + if (data.size() != size) throw std::range_error("unexpected field size"); + return read_dest.update([&](auto& value) { - auto data = input.get(); - memcpy(value, data.begin(), size); + std::ranges::copy(input.get(), std::ranges::begin(value)); }); } } // namespace mp diff --git a/include/mp/type-string.h b/include/mp/type-string.h index d4c3383b..d81d59ec 100644 --- a/include/mp/type-string.h +++ b/include/mp/type-string.h @@ -7,6 +7,8 @@ #include +#include + namespace mp { template void CustomBuildField(TypeList, @@ -16,7 +18,7 @@ void CustomBuildField(TypeList, Output&& output) { auto result = output.init(value.size()); - memcpy(result.begin(), value.data(), value.size()); + std::ranges::copy(value, result.begin()); } template From a494b764de5fddd527e6859f915c3b80c88f4969 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:49:45 +0100 Subject: [PATCH 18/81] cmake: Remove `QUIET` option from `find_package(CapnProto ...)` Cap'n Proto's Autotools-installed package config forwards the `QUIET` option to its internal `find_package(PkgConfig)` call and, when `QUIET` is given, suppresses its own diagnostics. As a result, a missing `PkgConfig` (for example, with the system `capnproto` package on NetBSD) was indistinguishable from `capnproto` itself being absent. Without `QUIET`, the actual cause can be diagnosed: ``` -- Could NOT find PkgConfig (missing: PKG_CONFIG_EXECUTABLE) CMake Warning at /usr/pkg/lib/cmake/CapnProto/CapnProtoTargets.cmake:40 (message): pkg-config cannot be found Call Stack (most recent call first): /usr/pkg/lib/cmake/CapnProto/CapnProtoConfig.cmake:102 (include) CMakeLists.txt:16 (find_package) CMake Warning at CMakeLists.txt:16 (find_package): Found package configuration file: /usr/pkg/lib/cmake/CapnProto/CapnProtoConfig.cmake but it set CapnProto_FOUND to FALSE so package "CapnProto" is considered to be NOT FOUND. CMake Error at CMakeLists.txt:18 (message): Cap'n Proto is required but was not found. To resolve, choose one of the following: - Install Cap'n Proto (version 1.0+ recommended) - For Bitcoin Core compilation build with -DENABLE_IPC=OFF to disable multiprocess support -- Configuring incomplete, errors occurred! ``` --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6354b0ee..6e15e79e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ endif() include("cmake/compat_find.cmake") find_package(Threads REQUIRED) -find_package(CapnProto 0.9 QUIET NO_MODULE) +find_package(CapnProto 0.9 NO_MODULE) if(NOT CapnProto_FOUND) message(FATAL_ERROR "Cap'n Proto is required but was not found.\n" From fa47449afe1405df661364ff22911892beb79715 Mon Sep 17 00:00:00 2001 From: marco <*~=`'#}+{/-|&$^_@721217.xyz> Date: Sat, 1 Aug 2026 09:09:11 +0200 Subject: [PATCH 19/81] cmake: avoid unnecessary capnp-rpc dependency for mpgen Instead, use only the required kj-async. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6354b0ee..62009d7d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,9 +205,9 @@ add_executable(Libmultiprocess::mpgen ALIAS mpgen) target_include_directories(mpgen PRIVATE $) target_include_directories(mpgen PUBLIC $ $) target_link_libraries(mpgen PRIVATE CapnProto::capnp) -target_link_libraries(mpgen PRIVATE CapnProto::capnp-rpc) target_link_libraries(mpgen PRIVATE CapnProto::capnpc) target_link_libraries(mpgen PRIVATE CapnProto::kj) +target_link_libraries(mpgen PRIVATE CapnProto::kj-async) target_link_libraries(mpgen PRIVATE Threads::Threads) set_target_properties(mpgen PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) From edf6343562450eec0a212d10d96fb9e39192933f Mon Sep 17 00:00:00 2001 From: marco <*~=`'#}+{/-|&$^_@721217.xyz> Date: Sun, 2 Aug 2026 12:16:49 +0200 Subject: [PATCH 20/81] ci: Compile with minimum supported g++-11 in olddeps --- ci/configs/olddeps.bash | 2 +- shell.nix | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/configs/olddeps.bash b/ci/configs/olddeps.bash index 1a363b1b..a82c5eae 100644 --- a/ci/configs/olddeps.bash +++ b/ci/configs/olddeps.bash @@ -1,5 +1,5 @@ CI_DESC="CI job using old Cap'n Proto and cmake versions" CI_DIR=build-olddeps export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds" -NIX_ARGS=(--argstr capnprotoVersion "0.9.2" --argstr cmakeVersion "3.12.4") +NIX_ARGS=(--argstr capnprotoVersion "0.9.2" --argstr cmakeVersion "3.12.4" --argstr gccVersion "11") BUILD_ARGS=(-k) diff --git a/shell.nix b/shell.nix index 1a4614e3..064c8847 100644 --- a/shell.nix +++ b/shell.nix @@ -5,6 +5,7 @@ , capnprotoVersion ? null , capnprotoSanitizers ? null # Optional sanitizers to build cap'n proto with , cmakeVersion ? null +, gccVersion ? null , libcxxSanitizers ? null # Optional LLVM_USE_SANITIZER value to use for libc++, see https://llvm.org/docs/CMake.html }: @@ -59,6 +60,7 @@ let cmakeHashes = { "3.12.4" = "sha256-UlVYS/0EPrcXViz/iULUcvHA5GecSUHYS6raqbKOMZQ="; }; + gcc = if gccVersion == null then null else builtins.getAttr ("gcc" + gccVersion) pkgs; cmakeBuild = if cmakeVersion == null then pkgs.cmake else (pkgs.cmake.overrideAttrs (old: { version = cmakeVersion; src = pkgs.fetchurl { @@ -75,11 +77,14 @@ in crossPkgs.mkShell { cmakeBuild include-what-you-use ninja - ] ++ lib.optionals (!minimal) [ + ] ++ lib.optional (gcc != null) gcc ++ lib.optionals (!minimal) [ clang clang-tools ]; + CC = if gcc == null then null else "${gcc}/bin/gcc"; + CXX = if gcc == null then null else "${gcc}/bin/g++"; + # Tell IWYU where its libc++ mapping lives IWYU_MAPPING_FILE = if enableLibcxx then "${llvm.libcxx.dev}/include/c++/v1/libcxx.imp" else null; } From 7402affd0ce7370fb6a82efb8268f3f168408519 Mon Sep 17 00:00:00 2001 From: marco <*~=`'#}+{/-|&$^_@721217.xyz> Date: Sun, 2 Aug 2026 12:34:14 +0200 Subject: [PATCH 21/81] ci: Pin oldeps config to older nixpkgs channel to compile older cmake with older gcc --- ci/configs/olddeps.bash | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/configs/olddeps.bash b/ci/configs/olddeps.bash index 95f44128..cbba7508 100644 --- a/ci/configs/olddeps.bash +++ b/ci/configs/olddeps.bash @@ -1,5 +1,8 @@ CI_DESC="CI job using old Cap'n Proto and cmake versions" CI_DIR=build-olddeps +# Pin olddeps to an older Nixpkgs channel, since compiling the old CMake +# requires an older GCC. +NIXPKGS_CHANNEL=nixos-25.05 export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds" NIX_ARGS=(--argstr capnprotoVersion "0.7.1" --argstr cmakeVersion "3.12.4") BUILD_ARGS=(-k) From 788f17a850942448f84e543aeefda3ac6c577327 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sun, 2 Aug 2026 20:23:28 +0100 Subject: [PATCH 22/81] Remove code for Cap'n Proto versions before 0.9 The removed code has been dead since db4f9a3d7395d435c484f5735a787f0798080c52, which bumped the minimum required Cap'n Proto version to 0.9. --- CMakeLists.txt | 4 +--- cmake/compat_config.cmake | 42 --------------------------------------- shell.nix | 4 ---- 3 files changed, 1 insertion(+), 49 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6354b0ee..8b5b840b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,9 +35,7 @@ set(CAPNPROTO_CLANG_INCOMPATIBLE FALSE) # https://github.com/capnproto/capnproto/blob/master/security-advisories/2022-11-30-0-pointer-list-bounds.md # https://capnproto.org/news/2022-11-30-CVE-2022-46149-security-advisory.html # https://dwrensha.github.io/capnproto-rust/2022/11/30/out_of_bounds_memory_access_bug.html -if(CapnProto_VERSION STREQUAL "0.7.0" - OR CapnProto_VERSION STREQUAL "0.8.0" - OR CapnProto_VERSION STREQUAL "0.9.0" +if(CapnProto_VERSION STREQUAL "0.9.0" OR CapnProto_VERSION STREQUAL "0.9.1" OR CapnProto_VERSION STREQUAL "0.10.0" OR CapnProto_VERSION STREQUAL "0.10.1" diff --git a/cmake/compat_config.cmake b/cmake/compat_config.cmake index f9d3004f..32a9d6dd 100644 --- a/cmake/compat_config.cmake +++ b/cmake/compat_config.cmake @@ -15,45 +15,3 @@ endif() if (NOT DEFINED CAPNPC_OUTPUT_DIR) set(CAPNPC_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}") endif() - -# CMake target definitions for backwards compatibility with Ubuntu bionic -# capnproto 0.6.1 package (https://packages.ubuntu.com/bionic/libcapnp-dev) -# https://github.com/bitcoin-core/libmultiprocess/issues/27 - -if (NOT DEFINED CAPNP_LIB_CAPNPC AND DEFINED CAPNP_LIB_CAPNP-RPC) - string(REPLACE "-rpc" "c" CAPNP_LIB_CAPNPC "${CAPNP_LIB_CAPNP-RPC}") -endif() - -if (NOT DEFINED CapnProto_capnpc_IMPORTED_LOCATION AND DEFINED CapnProto_capnp-rpc_IMPORTED_LOCATION) - string(REPLACE "-rpc" "c" CapnProto_capnpc_IMPORTED_LOCATION "${CapnProto_capnp-rpc_IMPORTED_LOCATION}") -endif() - -if (NOT TARGET CapnProto::capnp AND DEFINED CAPNP_LIB_CAPNP) - add_library(CapnProto::capnp SHARED IMPORTED) - set_target_properties(CapnProto::capnp PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_CAPNP}") -endif() - -if (NOT TARGET CapnProto::capnpc AND DEFINED CAPNP_LIB_CAPNPC) - add_library(CapnProto::capnpc SHARED IMPORTED) - set_target_properties(CapnProto::capnpc PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_CAPNPC}") -endif() - -if (NOT TARGET CapnProto::capnpc AND DEFINED CapnProto_capnpc_IMPORTED_LOCATION) - add_library(CapnProto::capnpc SHARED IMPORTED) - set_target_properties(CapnProto::capnpc PROPERTIES IMPORTED_LOCATION "${CapnProto_capnpc_IMPORTED_LOCATION}") -endif() - -if (NOT TARGET CapnProto::capnp-rpc AND DEFINED CAPNP_LIB_CAPNP-RPC) - add_library(CapnProto::capnp-rpc SHARED IMPORTED) - set_target_properties(CapnProto::capnp-rpc PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_CAPNP-RPC}") -endif() - -if (NOT TARGET CapnProto::kj AND DEFINED CAPNP_LIB_KJ) - add_library(CapnProto::kj SHARED IMPORTED) - set_target_properties(CapnProto::kj PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_KJ}") -endif() - -if (NOT TARGET CapnProto::kj-async AND DEFINED CAPNP_LIB_KJ-ASYNC) - add_library(CapnProto::kj-async SHARED IMPORTED) - set_target_properties(CapnProto::kj-async PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_KJ-ASYNC}") -endif() diff --git a/shell.nix b/shell.nix index 1a4614e3..e04518f3 100644 --- a/shell.nix +++ b/shell.nix @@ -17,10 +17,6 @@ let }; }; capnprotoHashes = { - "0.7.0" = "sha256-Y/7dUOQPDHjniuKNRw3j8dG1NI9f/aRWpf8V0WzV9k8="; - "0.7.1" = "sha256-3cBpVmpvCXyqPUXDp12vCFCk32ZXWpkdOliNH37UwWE="; - "0.8.0" = "sha256-rfiqN83begjJ9eYjtr21/tk1GJBjmeVfa3C3dZBJ93w="; - "0.8.1" = "sha256-OZqNVYdyszro5rIe+w6YN00g6y8U/1b8dKYc214q/2o="; "0.9.0" = "sha256-yhbDcWUe6jp5PbIXzn5EoKabXiWN8lnS08hyfxUgEQ0="; "0.9.2" = "sha256-BspWOPZcP5nCTvmsDE62Zutox+aY5pw42d6hpH3v4cM="; "0.10.0" = "sha256-++F4l54OMTDnJ+FO3kV/Y/VLobKVRk461dopanuU3IQ="; From bd508311b5617bd8d31b875d8781392f2a6ad7be Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sun, 2 Aug 2026 20:49:05 +0100 Subject: [PATCH 23/81] refactor: Drop stray semicolons after function definitions --- include/mp/proxy-types.h | 2 +- include/mp/type-function.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mp/proxy-types.h b/include/mp/proxy-types.h index ed458209..b790dd39 100644 --- a/include/mp/proxy-types.h +++ b/include/mp/proxy-types.h @@ -627,7 +627,7 @@ template auto PassField(Priority<2>, Args&&... args) -> decltype(CustomPassField(std::forward(args)...)) { return CustomPassField(std::forward(args)...); -}; +} template struct ServerField : Parent diff --git a/include/mp/type-function.h b/include/mp/type-function.h index 951e0885..fade9b42 100644 --- a/include/mp/type-function.h +++ b/include/mp/type-function.h @@ -61,7 +61,7 @@ decltype(auto) CustomReadField(TypeList> types, return read_dest.construct(ProxyCallFn{std::move(client)}); } return read_dest.construct(); -}; +} } // namespace mp #endif // MP_PROXY_TYPE_FUNCTION_H From 36f74002775352e7d69748f8ad5ffecad5278ead Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sun, 2 Aug 2026 20:49:13 +0100 Subject: [PATCH 24/81] ci: Drop NetBSD release 9.4 NetBSD 9.x will become unsupported once release 11.0 is released. --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffaca1ca..8aa6bfe7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,8 +17,6 @@ jobs: # Test all supported releases. # See https://www.netbsd.org/releases/. include: - - release: 9.4 - capnproto-cppflags: 'CPPFLAGS="-DKJ_NO_EXCEPTIONS=0 -DKJ_USE_KQUEUE=0"' - release: 10.1 capnproto-cppflags: 'CPPFLAGS="-DKJ_NO_EXCEPTIONS=0"' steps: From 2d33b14fb0e7ad79ee7ad527c119066a0ecd480e Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sun, 2 Aug 2026 20:49:25 +0100 Subject: [PATCH 25/81] ci: Switch to default compiler on NetBSD 9.4 Also document a workaround for a bug in Cap'n Proto affecting compilers prior to GCC 14. --- .github/workflows/ci.yml | 7 ++++++- ci/configs/netbsd.bash | 8 +------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8aa6bfe7..4119f99a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,8 +17,13 @@ jobs: # Test all supported releases. # See https://www.netbsd.org/releases/. include: + # Due to a bug in Cap'n Proto, GCC compilers older + # than 14.x require `-DKJ_NO_EXCEPTIONS=0` when + # processing `capnproto/c++/src/kj/common.h`. + # See https://github.com/capnproto/capnproto/pull/2756/changes/28fa1a6753514d8de56875ec2222fa0e1c3a04f7. - release: 10.1 capnproto-cppflags: 'CPPFLAGS="-DKJ_NO_EXCEPTIONS=0"' + env_vars: 'CXXFLAGS="-DKJ_NO_EXCEPTIONS=0"' steps: - uses: actions/checkout@v6 @@ -61,7 +66,7 @@ jobs: - name: Run CI script run: | cd ${{ github.workspace }} - CI_CONFIG="ci/configs/netbsd.bash" bash ci/scripts/ci.sh + CI_CONFIG="ci/configs/netbsd.bash" ${{ matrix.env_vars }} bash ci/scripts/ci.sh build-openbsd: runs-on: ubuntu-latest diff --git a/ci/configs/netbsd.bash b/ci/configs/netbsd.bash index c3e7d258..203b6623 100644 --- a/ci/configs/netbsd.bash +++ b/ci/configs/netbsd.bash @@ -1,11 +1,5 @@ CI_DESC="CI config for NetBSD" CI_DIR=build-netbsd -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" -# Hardcode GCC 14, since default GCC versions installed by NetBSD are older -# and may not be compatible with libmultiprocess. GCC 14 was chosen because -# it's the latest compiler available on all versions of NetBSD that we test. -# Note that the GCC version specified here must match the version specified -# in pkg_add in ci.yml. -export CXX="/usr/pkg/gcc14/bin/g++" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter ${CXXFLAGS:-}" CMAKE_ARGS=(-G Ninja) BUILD_ARGS=(-k 0) From a9e70dbe775ab628dd6610c7e173bf0d350036f8 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sun, 2 Aug 2026 20:49:34 +0100 Subject: [PATCH 26/81] ci: Add NetBSD release 11.0 --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4119f99a..529db751 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,10 @@ jobs: # than 14.x require `-DKJ_NO_EXCEPTIONS=0` when # processing `capnproto/c++/src/kj/common.h`. # See https://github.com/capnproto/capnproto/pull/2756/changes/28fa1a6753514d8de56875ec2222fa0e1c3a04f7. - - release: 10.1 + - release: '10.1' + capnproto-cppflags: 'CPPFLAGS="-DKJ_NO_EXCEPTIONS=0"' + env_vars: 'CXXFLAGS="-DKJ_NO_EXCEPTIONS=0"' + - release: '11.0' capnproto-cppflags: 'CPPFLAGS="-DKJ_NO_EXCEPTIONS=0"' env_vars: 'CXXFLAGS="-DKJ_NO_EXCEPTIONS=0"' steps: From 4a56c1837a781fbd859c4a3ef91fdd9e213b42a7 Mon Sep 17 00:00:00 2001 From: ViniciusCestarii Date: Thu, 30 Jul 2026 11:44:03 -0300 Subject: [PATCH 27/81] util: report back child error to parent and throw --- src/mp/util.cpp | 137 ++++++++++++++++++++++++++++++++--- test/mp/test/spawn_tests.cpp | 18 +++++ 2 files changed, 144 insertions(+), 11 deletions(-) diff --git a/src/mp/util.cpp b/src/mp/util.cpp index f524ab23..86417a80 100644 --- a/src/mp/util.cpp +++ b/src/mp/util.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -72,6 +73,92 @@ template _exit(126); } +enum class SpawnErrorOp +{ + CLOSE, + EXECVP, + READ, + FCNTL, +}; + +struct SpawnError { + SpawnErrorOp which; + int err; +}; + +const char* SpawnErrorName(SpawnErrorOp which) +{ + switch (which) { + case SpawnErrorOp::CLOSE: return "close"; + case SpawnErrorOp::EXECVP: return "execvp"; + case SpawnErrorOp::READ: return "read"; + case SpawnErrorOp::FCNTL: return "fcntl"; + } + return "unknown"; +} + +// Read the child's error report. Returns nullopt on success, or a SpawnError to +// throw on failure. Success is a clean EOF: read() returns 0 with nothing +// buffered because the child's write end was closed by a successful exec (via +// FD_CLOEXEC). A fully-read struct is the failure the child reported. A read() +// error, or an EOF partway through the struct (the child died mid-report), is +// surfaced as a SpawnErrorOp::READ failure rather than being mistaken for +// success. A single read() is not guaranteed to return all sizeof(SpawnError) +// bytes (the socket is SOCK_STREAM, which has no message boundaries) and may be +// interrupted by a signal, so loop until the whole struct is read. +// +// Note that a clean EOF is also what happens if the child is killed before it +// reaches exec, so this function reports success in that case. There is no good +// way to distinguish the two here, but callers will still see the failure when +// they call WaitProcess and get the child's exit status. +std::optional ReadSpawnResult(int fd) +{ + SpawnError error{}; + char* buf = reinterpret_cast(&error); + size_t remaining = sizeof(error); + while (remaining > 0) { + const ssize_t n = ::read(fd, buf, remaining); + if (n < 0) { + if (errno == EINTR) continue; + return SpawnError{.which = SpawnErrorOp::READ, .err = errno}; + } + if (n == 0) { + if (remaining == sizeof(error)) return std::nullopt; // clean EOF: success + return SpawnError{.which = SpawnErrorOp::READ, .err = EPROTO}; // torn report + } + buf += n; + remaining -= static_cast(n); + } + return error; +} + +// Write the whole SpawnError to fd, retrying short writes and EINTR so the +// parent never sees a torn struct. Runs in the post-fork child, so it must stay +// async-signal-safe: it only calls write() and does not allocate or throw. This +// is best-effort -- if the write cannot complete there is nothing useful the +// child can do, so it stops and lets the caller _exit(). +void WriteSpawnError(int fd, const SpawnError& error) +{ + const char* buf = reinterpret_cast(&error); + size_t remaining = sizeof(error); + while (remaining > 0) { + const ssize_t n = ::write(fd, buf, remaining); + if (n < 0) { + if (errno == EINTR) continue; + break; + } + buf += n; + remaining -= static_cast(n); + } + if (remaining > 0) { + // The parent's read end is gone (e.g. the parent exited before the + // child could report), so the structured error can't be delivered. + // Leave a breadcrumb on stderr and exit. The exit code is irrelevant + // here since no live parent remains to wait on it. + ChildFail("SpawnProcess(child): failed and could not report error to parent\n"); + } +} + } // namespace std::string ThreadName(const char* exe_name) @@ -132,6 +219,9 @@ std::string LogEscape(const kj::StringTree& string, size_t max_size) std::tuple SpawnProcess(SpawnConnectInfoToArgsFn&& connect_info_to_args) { auto fds{SocketPair()}; + // Only used for the child to report errors back to the parent, so a one-way + // pipe would be sufficient, but reuse the existing SocketPair() helper. + auto error_fds{SocketPair()}; // Evaluate the callback and build the argv array before forking. // @@ -144,25 +234,35 @@ std::tuple SpawnProcess(SpawnConnectInfoToArgsFn&& connect_ ProcessId pid = fork(); if (pid == -1) { - throw std::system_error(errno, std::system_category(), "fork"); + const int err = errno; + (void)close(fds[0]); + (void)close(fds[1]); + (void)close(error_fds[0]); + (void)close(error_fds[1]); + throw std::system_error(err, std::system_category(), "fork"); } // Parent process closes the descriptor for socket 0, child closes the // descriptor for socket 1. On failure, the parent throws, but the child // must _exit(126) (post-fork child must not throw). if (close(fds[pid ? 0 : 1]) != 0) { + const int err = errno; if (pid) { (void)close(fds[1]); - throw std::system_error(errno, std::system_category(), "close"); + (void)close(error_fds[0]); + (void)close(error_fds[1]); + throw std::system_error(err, std::system_category(), "close"); } - ChildFail("SpawnProcess(child): close(fds[1]) failed\n"); + WriteSpawnError(error_fds[0], {.which = SpawnErrorOp::CLOSE, .err = err}); + _exit(126); } if (!pid) { // Child process must close all potentially open descriptors, except - // socket 0. Do not throw, allocate, or do non-fork-safe work here. + // socket 0 and the error-reporting socket 0. Do not throw, allocate, or + // do non-fork-safe work here. const int maxFd = MaxFd(); for (int fd = 3; fd < maxFd; ++fd) { - if (fd != fds[0]) { + if (fd != fds[0] && fd != error_fds[0]) { close(fd); } } @@ -174,17 +274,32 @@ std::tuple SpawnProcess(SpawnConnectInfoToArgsFn&& connect_ // fcntl is async-signal-safe. const int fds0_flags = fcntl(fds[0], F_GETFD); if (fds0_flags == -1 || fcntl(fds[0], F_SETFD, fds0_flags & ~FD_CLOEXEC) == -1) { - ChildFail("SpawnProcess(child): clearing FD_CLOEXEC failed\n"); + const int err = errno; + WriteSpawnError(error_fds[0], {.which = SpawnErrorOp::FCNTL, .err = err}); + _exit(126); } execvp(argv[0], argv.data()); - // NOTE: perror() is not async-signal-safe; calling it here in a - // post-fork child may deadlock in multithreaded parents. - // TODO: Report errors to the parent via a pipe (e.g. write errno) - // so callers can get diagnostics without relying on perror(). - perror("execvp failed"); + + const int err = errno; + WriteSpawnError(error_fds[0], {.which = SpawnErrorOp::EXECVP, .err = err}); _exit(127); } + + // Close the parent's copy of the child's write end. + if (close(error_fds[0]) != 0) { + const int err = errno; + (void)close(error_fds[1]); + (void)close(fds[1]); + throw std::system_error(err, std::system_category(), "close"); + } + + const std::optional error{ReadSpawnResult(error_fds[1])}; + (void)close(error_fds[1]); + if (error) { + (void)close(fds[1]); + throw std::system_error(error->err, std::system_category(), SpawnErrorName(error->which)); + } return {pid, fds[1]}; } diff --git a/test/mp/test/spawn_tests.cpp b/test/mp/test/spawn_tests.cpp index bdc9ef54..5f991275 100644 --- a/test/mp/test/spawn_tests.cpp +++ b/test/mp/test/spawn_tests.cpp @@ -4,8 +4,11 @@ #include +#include +#include #include +#include #include #include #include @@ -13,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -113,5 +118,18 @@ KJ_TEST("SpawnProcess does not run callback in child") KJ_EXPECT(exited, "Timeout waiting for child process to exit"); KJ_EXPECT(WIFEXITED(status) && WEXITSTATUS(status) == 0); } + +KJ_TEST("SpawnProcess throws on execvp failure") +{ + try { + SpawnProcess([&](SpawnConnectInfo) -> std::vector { + return {"/nonexistent/binary"}; + }); + KJ_EXPECT(false, "expected SpawnProcess to throw"); + } catch (const std::system_error& e) { + KJ_EXPECT(e.code().value() == ENOENT); + KJ_EXPECT(std::string_view{e.what()}.find("execvp") != std::string_view::npos); + } +} } // namespace test } // namespace mp From 3f05b11624ce02e35b52e7451b2c7092b9b48a7c Mon Sep 17 00:00:00 2001 From: ViniciusCestarii Date: Mon, 3 Aug 2026 16:57:46 -0300 Subject: [PATCH 28/81] util: kill and reap child on SpawnProcess error --- src/mp/util.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/mp/util.cpp b/src/mp/util.cpp index 86417a80..96bd344e 100644 --- a/src/mp/util.cpp +++ b/src/mp/util.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -159,6 +160,16 @@ void WriteSpawnError(int fd, const SpawnError& error) } } +// Get rid of a child process the parent is abandoning because SpawnProcess is +// about to throw, so it is not left behind as a zombie. The child may still be +// alive, so kill it first: waiting without that could block for as long as the +// spawned program runs. +void KillAndReapChild(ProcessId pid) +{ + (void)::kill(pid, SIGKILL); + while (::waitpid(pid, /*status=*/nullptr, /*options=*/0) == -1 && errno == EINTR) {} +} + } // namespace std::string ThreadName(const char* exe_name) @@ -250,6 +261,7 @@ std::tuple SpawnProcess(SpawnConnectInfoToArgsFn&& connect_ (void)close(fds[1]); (void)close(error_fds[0]); (void)close(error_fds[1]); + KillAndReapChild(pid); throw std::system_error(err, std::system_category(), "close"); } WriteSpawnError(error_fds[0], {.which = SpawnErrorOp::CLOSE, .err = err}); @@ -291,6 +303,7 @@ std::tuple SpawnProcess(SpawnConnectInfoToArgsFn&& connect_ const int err = errno; (void)close(error_fds[1]); (void)close(fds[1]); + KillAndReapChild(pid); throw std::system_error(err, std::system_category(), "close"); } @@ -298,6 +311,7 @@ std::tuple SpawnProcess(SpawnConnectInfoToArgsFn&& connect_ (void)close(error_fds[1]); if (error) { (void)close(fds[1]); + KillAndReapChild(pid); throw std::system_error(error->err, std::system_category(), SpawnErrorName(error->which)); } return {pid, fds[1]}; From ce51d737255df9e420d5a3dc09e1615b5d12f2ff Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Mon, 20 Jul 2026 12:19:27 -0400 Subject: [PATCH 29/81] ci: Set CMAKE_BUILD_PARALLEL_LEVEL to enable parallism in build jobs by default --- ci/scripts/ci.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/ci.sh b/ci/scripts/ci.sh index d989e9f4..0022fd28 100755 --- a/ci/scripts/ci.sh +++ b/ci/scripts/ci.sh @@ -24,6 +24,7 @@ ver_ge() { [ "$(printf '%s\n' "$2" "$1" | sort -V | head -n1)" = "$2" ]; } src_dir=$PWD mkdir -p "$CI_DIR" cd "$CI_DIR" +export CMAKE_BUILD_PARALLEL_LEVEL="$(nproc)" cmake "$src_dir" "${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}" if ver_ge "$cmake_ver" "3.15"; then cmake --build . -t "${BUILD_TARGETS[@]}" -- "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" From 62f25af06c39636d5e09d85753d3e958f264eec1 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 1 Jul 2026 18:23:58 -0400 Subject: [PATCH 30/81] test: check ReadList return value ReadList was originally declared void and did not return the result of read_dest.update(), silently discarding the return value. When callers pass a ReadDestTemp or other ReadDest whose update() returns a constructed value, that value was lost. Add a test to demonstrate the bug: extend FooCustom with a v3 :List(Int32) / std::vector field and use the assignment pattern value.v3 = ReadField(..., ReadDestTemp>()) inside CustomReadField. This fails to compile if ReadList returns void and succeeds with the fix. Co-Authored-By: Claude Sonnet 4.6 --- test/mp/test/foo-types.h | 2 ++ test/mp/test/foo.capnp | 1 + test/mp/test/foo.h | 1 + test/mp/test/test.cpp | 2 ++ 4 files changed, 6 insertions(+) diff --git a/test/mp/test/foo-types.h b/test/mp/test/foo-types.h index b96eabfc..1bc6c523 100644 --- a/test/mp/test/foo-types.h +++ b/test/mp/test/foo-types.h @@ -46,6 +46,7 @@ void CustomBuildField(TypeList, Priority<1>, InvokeContext& invoke_co { BuildField(TypeList(), invoke_context, output, value.v1); output.setV2(value.v2); + BuildField(TypeList>(), invoke_context, output, value.v3); } template @@ -55,6 +56,7 @@ decltype(auto) CustomReadField(TypeList, Priority<1>, InvokeContext& return read_dest.update([&](FooCustom& value) { value.v1 = ReadField(TypeList(), invoke_context, mp::Make(custom.getV1()), ReadDestTemp()); value.v2 = custom.getV2(); + value.v3 = ReadField(TypeList>(), invoke_context, mp::Make(custom.getV3()), ReadDestTemp>()); }); } diff --git a/test/mp/test/foo.capnp b/test/mp/test/foo.capnp index 9e6213fd..26890ea5 100644 --- a/test/mp/test/foo.capnp +++ b/test/mp/test/foo.capnp @@ -66,6 +66,7 @@ struct FooStruct $Proxy.wrap("mp::test::FooStruct") { struct FooCustom $Proxy.wrap("mp::test::FooCustom") { v1 @0 :Text; v2 @1 :Int32; + v3 @2 :List(Int32); } struct FooEmpty $Proxy.wrap("mp::test::FooEmpty") { diff --git a/test/mp/test/foo.h b/test/mp/test/foo.h index 779c8db1..a8fd9805 100644 --- a/test/mp/test/foo.h +++ b/test/mp/test/foo.h @@ -34,6 +34,7 @@ struct FooCustom { std::string v1; int v2; + std::vector v3; }; struct FooEmpty diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index f5f35437..715b6d91 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -233,9 +233,11 @@ KJ_TEST("Call FooInterface methods") FooCustom custom_in; custom_in.v1 = "v1"; custom_in.v2 = 5; + custom_in.v3 = {10, 20, 30}; FooCustom custom_out = foo->passCustom(custom_in); KJ_EXPECT(custom_in.v1 == custom_out.v1); KJ_EXPECT(custom_in.v2 == custom_out.v2); + KJ_EXPECT(custom_in.v3 == custom_out.v3); foo->passEmpty(FooEmpty{}); From 060c1a50d03dc06cff29d5d3bdf7a9d1b2b49335 Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:02:53 -0300 Subject: [PATCH 31/81] Extract `UnixListener` class to a dedicated file A later commit will consume the `UnixListener` class in another test file, so move it to a shared one. --- test/mp/test/listen_tests.cpp | 65 +-------------------------- test/mp/test/unixlistener.h | 84 +++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 64 deletions(-) create mode 100644 test/mp/test/unixlistener.h diff --git a/test/mp/test/listen_tests.cpp b/test/mp/test/listen_tests.cpp index a9d4dca2..b9b0bfc2 100644 --- a/test/mp/test/listen_tests.cpp +++ b/test/mp/test/listen_tests.cpp @@ -2,15 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include -#include +#include "unixlistener.h" #include #include #include #include #include -#include #include #include #include @@ -27,8 +25,6 @@ #include #include #include -#include -#include #include #include @@ -38,65 +34,6 @@ namespace { constexpr auto FAILURE_TIMEOUT = std::chrono::seconds{30}; -//! Owns a temporary Unix-domain listening socket used by ListenSetup. Tests call -//! Connect() to create client socket FDs and release() to transfer the listening -//! FD to ListenConnections(). -class UnixListener -{ -public: - UnixListener() - { - std::string dir_template = (std::filesystem::temp_directory_path() / "mptest-listener-XXXXXX").string(); - char* dir = mkdtemp(dir_template.data()); - KJ_REQUIRE(dir != nullptr); - m_dir = dir; - m_path = m_dir + "/socket"; - - m_fd = socket(AF_UNIX, SOCK_STREAM, 0); - KJ_REQUIRE(m_fd >= 0); - - sockaddr_un addr{}; - addr.sun_family = AF_UNIX; - KJ_REQUIRE(m_path.size() < sizeof(addr.sun_path)); - std::strncpy(addr.sun_path, m_path.c_str(), sizeof(addr.sun_path) - 1); - KJ_REQUIRE(bind(m_fd, reinterpret_cast(&addr), sizeof(addr)) == 0); - KJ_REQUIRE(listen(m_fd, SOMAXCONN) == 0); - } - - ~UnixListener() - { - if (m_fd >= 0) close(m_fd); - if (!m_path.empty()) unlink(m_path.c_str()); - if (!m_dir.empty()) rmdir(m_dir.c_str()); - } - - int release() - { - assert(m_fd >= 0); - int fd = m_fd; - m_fd = -1; - return fd; - } - - int MakeConnectedSocket() const - { - int fd = socket(AF_UNIX, SOCK_STREAM, 0); - KJ_REQUIRE(fd >= 0); - - sockaddr_un addr{}; - addr.sun_family = AF_UNIX; - KJ_REQUIRE(m_path.size() < sizeof(addr.sun_path)); - std::strncpy(addr.sun_path, m_path.c_str(), sizeof(addr.sun_path) - 1); - KJ_REQUIRE(connect(fd, reinterpret_cast(&addr), sizeof(addr)) == 0); - return fd; - } - -private: - int m_fd{-1}; - std::string m_dir; - std::string m_path; -}; - //! Runs a client EventLoop on its own thread and connects one socket FD to the //! server. The constructed ProxyClient can be used by the test thread to make //! calls over that connection. diff --git a/test/mp/test/unixlistener.h b/test/mp/test/unixlistener.h new file mode 100644 index 00000000..8d77a41c --- /dev/null +++ b/test/mp/test/unixlistener.h @@ -0,0 +1,84 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_TEST_UNIXLISTENER_H +#define MP_TEST_UNIXLISTENER_H + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace mp { +namespace test { + +//! Owns a temporary Unix-domain listening socket used by ListenSetup. Tests call +//! Connect() to create client socket FDs and release() to transfer the listening +//! FD to ListenConnections(). +class UnixListener +{ +public: + UnixListener() + { + std::string dir_template = (std::filesystem::temp_directory_path() / "mptest-listener-XXXXXX").string(); + char* dir = mkdtemp(dir_template.data()); + KJ_REQUIRE(dir != nullptr); + m_dir = dir; + m_path = m_dir + "/socket"; + + m_fd = socket(AF_UNIX, SOCK_STREAM, 0); + KJ_REQUIRE(m_fd >= 0); + + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + KJ_REQUIRE(m_path.size() < sizeof(addr.sun_path)); + std::strncpy(addr.sun_path, m_path.c_str(), sizeof(addr.sun_path) - 1); + KJ_REQUIRE(bind(m_fd, reinterpret_cast(&addr), sizeof(addr)) == 0); + KJ_REQUIRE(listen(m_fd, SOMAXCONN) == 0); + } + + ~UnixListener() + { + if (m_fd >= 0) close(m_fd); + if (!m_path.empty()) unlink(m_path.c_str()); + if (!m_dir.empty()) rmdir(m_dir.c_str()); + } + + int release() + { + assert(m_fd >= 0); + int fd = m_fd; + m_fd = -1; + return fd; + } + + int MakeConnectedSocket() const + { + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + KJ_REQUIRE(fd >= 0); + + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + KJ_REQUIRE(m_path.size() < sizeof(addr.sun_path)); + std::strncpy(addr.sun_path, m_path.c_str(), sizeof(addr.sun_path) - 1); + KJ_REQUIRE(connect(fd, reinterpret_cast(&addr), sizeof(addr)) == 0); + return fd; + } + +private: + int m_fd{-1}; + std::string m_dir; + std::string m_path; +}; + +} // namespace test +} // namespace mp + +#endif // MP_TEST_UNIXLISTENER_H From 231361ae5afd4e5892c38d762921c2d9971db837 Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:30:28 -0300 Subject: [PATCH 32/81] Correct stale UnixListener doc comment --- test/mp/test/unixlistener.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/mp/test/unixlistener.h b/test/mp/test/unixlistener.h index 8d77a41c..a743a475 100644 --- a/test/mp/test/unixlistener.h +++ b/test/mp/test/unixlistener.h @@ -19,9 +19,9 @@ namespace mp { namespace test { -//! Owns a temporary Unix-domain listening socket used by ListenSetup. Tests call -//! Connect() to create client socket FDs and release() to transfer the listening -//! FD to ListenConnections(). +//! Owns a temporary Unix-domain listening socket. Tests call +//! MakeConnectedSocket() to create client socket FDs and release() to transfer +//! ownership of the listening FD. class UnixListener { public: From 44d191420c6fdc83d2c1da159a409c4c18151cbe Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Tue, 23 Jun 2026 22:49:13 -0300 Subject: [PATCH 33/81] Add test coverage for ConnectStream This commit introduces a new test file `connect_tests.cpp` for testing the `ConnectStream` function. It also adds a `FooInit` test interface declaring a `construct()` method, which `ConnectStream` calls implicitly when present. --- test/CMakeLists.txt | 1 + test/mp/test/connect_tests.cpp | 171 +++++++++++++++++++++++++++++++++ test/mp/test/foo.capnp | 4 + test/mp/test/foo.h | 4 + 4 files changed, 180 insertions(+) create mode 100644 test/mp/test/connect_tests.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 13246293..fd0aa023 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -28,6 +28,7 @@ if(BUILD_TESTING AND TARGET CapnProto::kj-test) mp/test/foo.h mp/test/listen_tests.cpp mp/test/spawn_tests.cpp + mp/test/connect_tests.cpp mp/test/test.cpp ) include(${PROJECT_SOURCE_DIR}/cmake/TargetCapnpSources.cmake) diff --git a/test/mp/test/connect_tests.cpp b/test/mp/test/connect_tests.cpp new file mode 100644 index 00000000..c3a3a363 --- /dev/null +++ b/test/mp/test/connect_tests.cpp @@ -0,0 +1,171 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "unixlistener.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include // IWYU pragma: keep +#include +#include +#include +#include +#include +#include +#include + +namespace mp { +namespace test { +namespace { + +//! Default event loop log handler used by tests, throws so the calling code +//! can assert on errors. +void DefaultLogHandler(mp::LogMessage log) +{ + if (log.level == mp::Log::Raise) + throw std::runtime_error(log.message); +} + +class TestSetup +{ +public: + int client_fd; + int server_fd; + + mp::EventLoop* loop; + std::optional loop_ref; + //! Thread variable should be after other struct members so the thread does + //! not start until the other members are initialized. + std::thread loop_thread; + + TestSetup(mp::LogFn log_handler = DefaultLogHandler) + : TestSetup( + [](int fds[2]) { + KJ_REQUIRE(socketpair(AF_UNIX, SOCK_STREAM, 0, fds) != -1); + }, + log_handler) {} + + TestSetup(const std::function& init_sockets, + mp::LogFn log_handler = DefaultLogHandler) + { + std::promise loop_promise; + loop_thread = std::thread([&, log_handler] { + mp::EventLoop loop("mptest-connect", log_handler); + loop_promise.set_value(&loop); + loop.loop(); + }); + loop = loop_promise.get_future().get(); + loop_ref.emplace(*loop); + + // Initialize and store sockets + int fds[2] = {-1, -1}; + init_sockets(fds); + + client_fd = fds[0]; + server_fd = fds[1]; + } + + ~TestSetup() + { + loop_ref.reset(); + loop_thread.join(); + } +}; + +KJ_TEST("ConnectStream connects to a socket serving a valid init interface") +{ + TestSetup setup; + + std::thread server_thread([&setup]() { + mp::EventLoop server_loop("mptest-valid-server", DefaultLogHandler); + std::unique_ptr init = std::make_unique(); + ServeStream(server_loop, MakeStream(server_loop, setup.server_fd), *init); + server_loop.loop(); + }); + + auto init = ConnectStream(*setup.loop, MakeStream(*setup.loop, setup.client_fd)); + + init.reset(); + server_thread.join(); + KJ_EXPECT(true); +} + +KJ_TEST("ConnectStream throws when the socket disconnects after receiving data") +{ + TestSetup setup; + + std::thread server_thread([&setup]() { + char buf[128]; + + ssize_t bytes_received = + recv(setup.server_fd, buf, sizeof(buf), 0); + + if (bytes_received > 0) { + close(setup.server_fd); + } + }); + + try { + auto init = ConnectStream(*setup.loop, MakeStream(*setup.loop, setup.client_fd)); + + if (server_thread.joinable()) server_thread.join(); + KJ_EXPECT(false); + } catch (const std::runtime_error& e) { + if (server_thread.joinable()) server_thread.join(); + + std::string_view reason = e.what(); + KJ_EXPECT(reason == "IPC client method called after disconnect."); + } +} + +KJ_TEST("ConnectStream throws when a connection accepted from a listener disconnects after receiving data") +{ + UnixListener listener; + + TestSetup setup([&listener](int fds[2]) { + fds[0] = listener.MakeConnectedSocket(); // client_fd + fds[1] = listener.release(); // server_fd + }); + + std::thread server_thread([&setup]() { + char buf[128]; + + int connection_fd = accept(setup.server_fd, nullptr, nullptr); + + if (connection_fd >= 0) { + ssize_t bytes_received = + recv(connection_fd, buf, sizeof(buf), 0); + + if (bytes_received > 0) { + close(connection_fd); + } + } + close(setup.server_fd); + }); + + try { + auto init = ConnectStream(*setup.loop, MakeStream(*setup.loop, setup.client_fd)); + + if (server_thread.joinable()) server_thread.join(); + KJ_EXPECT(false); + } catch (const std::runtime_error& e) { + if (server_thread.joinable()) server_thread.join(); + + std::string_view reason = e.what(); + KJ_EXPECT(reason == "IPC client method called after disconnect."); + } +} + +} // namespace +} // namespace test +} // namespace mp diff --git a/test/mp/test/foo.capnp b/test/mp/test/foo.capnp index 9e6213fd..40acce30 100644 --- a/test/mp/test/foo.capnp +++ b/test/mp/test/foo.capnp @@ -39,6 +39,10 @@ interface FooInterface $Proxy.wrap("mp::test::FooImplementation") { passDataPointers @22 (arg :List(Data)) -> (result :List(Data)); } +interface FooInit $Proxy.wrap("mp::test::FooInit") { + construct @0 () -> (); +} + interface FooCallback $Proxy.wrap("mp::test::FooCallback") { destroy @0 (context :Proxy.Context) -> (); call @1 (context :Proxy.Context, arg :Int32) -> (result :Int32); diff --git a/test/mp/test/foo.h b/test/mp/test/foo.h index 779c8db1..eaac23f2 100644 --- a/test/mp/test/foo.h +++ b/test/mp/test/foo.h @@ -66,6 +66,10 @@ class ExtendedCallback : public FooCallback virtual int callExtended(int arg) = 0; }; +class FooInit +{ +}; + class FooImplementation { public: From bb473690c97ceed78a482d6a97b480cec4a63190 Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Fri, 10 Jul 2026 01:00:43 -0300 Subject: [PATCH 34/81] Fix error handling when creating clients Two bugs that have always been present: 1. `ConnectStream` registered the `onDisconnect` handler that deletes the `Connection` before the `ProxyClient` object owning it was created, so an early disconnect could delete the `Connection` while the client constructor was reading it (#308). 2. A `construct()` method failing during client construction caused a `Connection` leak. Normally, cleanup happens in the destructor but a constructor that throws leaves no object behind, so it never runs, resulting in an event loop ref preventing `EventLoop::loop()` from exiting (#309). Fix the first by making `ProxyClientBase` responsible for the connection when `destroy_connection` is true, registering the delete-on-disconnect handler at the end of its constructor. Fix the second by running the cleanup functions before rethrowing. Fixes #308 Fixes #309 --- include/mp/proxy-io.h | 32 ++++++++++++--- test/mp/test/connect_tests.cpp | 72 +++++++++++++++++++++++++++++++++- test/mp/test/test.cpp | 11 +++++- 3 files changed, 106 insertions(+), 9 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index f15965bb..cda9064d 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -592,7 +592,28 @@ ProxyClientBase::ProxyClientBase(typename Interface::Client cli }); } }); - Sub::construct(*this); + // If construct() fails, run the cleanup functions before rethrowing, + // because ~ProxyClientBase will not run for an object whose constructor + // threw, and the connection would otherwise be leaked. + try { + Sub::construct(*this); + } catch (...) { + MP_LOG(*m_context.loop, Log::Debug) << "Cleaning up " << CxxTypeName(*this) << " " << this << " after construct() failure"; + CleanupRun(m_context.cleanup_fns); + throw; + } + + // If this client owns the connection, delete the connection on disconnect. + if (destroy_connection) { + m_context.loop->sync([&] { + EventLoop& loop = *m_context.loop; + Connection* connection = m_context.connection; + connection->onDisconnect([&loop, connection] { + MP_LOG(loop, Log::Warning) << "IPC client: unexpected network disconnect."; + delete connection; + }); + }); + } } template @@ -832,6 +853,10 @@ kj::Promise ProxyServer::post(Fn&& fn) //! Given a stream, make a new ProxyClient object to send requests over it. //! Also create a new Connection object embedded in the client that is freed //! when the client is closed. +//! +//! If the init interface declares a construct() method, creating the client +//! calls it, so this function may block making an IPC call and may throw if +//! the call fails. template std::unique_ptr> ConnectStream(EventLoop& loop, Stream stream) { @@ -840,11 +865,6 @@ std::unique_ptr> ConnectStream(EventLoop& loop, Strea loop.sync([&] { connection = std::make_unique(loop, kj::mv(stream)); init_client = connection->m_rpc_system->bootstrap(ServerVatId().vat_id).castAs(); - Connection* connection_ptr = connection.get(); - connection->onDisconnect([&loop, connection_ptr] { - MP_LOG(loop, Log::Warning) << "IPC client: unexpected network disconnect."; - delete connection_ptr; - }); }); return std::make_unique>( kj::mv(init_client), connection.release(), /* destroy_connection= */ true); diff --git a/test/mp/test/connect_tests.cpp b/test/mp/test/connect_tests.cpp index c3a3a363..73b1a396 100644 --- a/test/mp/test/connect_tests.cpp +++ b/test/mp/test/connect_tests.cpp @@ -15,12 +15,16 @@ #include #include +#include +#include #include // IWYU pragma: keep #include #include #include +#include #include #include +#include #include #include @@ -28,6 +32,8 @@ namespace mp { namespace test { namespace { +constexpr auto FAILURE_TIMEOUT = std::chrono::seconds{30}; + //! Default event loop log handler used by tests, throws so the calling code //! can assert on errors. void DefaultLogHandler(mp::LogMessage log) @@ -100,6 +106,68 @@ KJ_TEST("ConnectStream connects to a socket serving a valid init interface") KJ_EXPECT(true); } +KJ_TEST("ConnectStream throws when the socket is already disconnected") +{ + TestSetup setup; + + close(setup.server_fd); + + try { + auto init = ConnectStream(*setup.loop, MakeStream(*setup.loop, setup.client_fd)); + + KJ_EXPECT(false); + } catch (const std::runtime_error& e) { + std::string_view reason = e.what(); + + KJ_EXPECT(reason == "IPC client method call interrupted by disconnect."); + } +} + +KJ_TEST("ConnectStream defers disconnect failure to the first IPC request for interfaces without construct()") +{ + TestSetup setup; + + close(setup.server_fd); + + // Without a construct() method no IPC call is made during client + // creation, so ConnectStream succeeds even though the peer is gone. + auto foo = ConnectStream(*setup.loop, MakeStream(*setup.loop, setup.client_fd)); + + try { + foo->add(1, 2); + KJ_EXPECT(false); + } catch (const std::runtime_error& e) { + std::string_view reason = e.what(); + + KJ_EXPECT(reason == "IPC client method called after disconnect."); + } +} + +KJ_TEST("ConnectStream handles a disconnect when no client calls are made") +{ + std::mutex mutex; + std::condition_variable cv; + bool warned = false; + + TestSetup setup([&](mp::LogMessage log) { + if (log.level == mp::Log::Warning && log.message.find("unexpected network disconnect") != std::string::npos) { + const std::lock_guard lock(mutex); + warned = true; + cv.notify_all(); + } + DefaultLogHandler(log); + }); + + close(setup.server_fd); + + auto foo = ConnectStream(*setup.loop, MakeStream(*setup.loop, setup.client_fd)); + + // The disconnect handler registered by ProxyClientBase should run and + // delete the connection even when no calls are ever made. + std::unique_lock lock(mutex); + KJ_EXPECT(cv.wait_for(lock, FAILURE_TIMEOUT, [&] { return warned; })); +} + KJ_TEST("ConnectStream throws when the socket disconnects after receiving data") { TestSetup setup; @@ -124,7 +192,7 @@ KJ_TEST("ConnectStream throws when the socket disconnects after receiving data") if (server_thread.joinable()) server_thread.join(); std::string_view reason = e.what(); - KJ_EXPECT(reason == "IPC client method called after disconnect."); + KJ_EXPECT(reason == "IPC client method call interrupted by disconnect."); } } @@ -162,7 +230,7 @@ KJ_TEST("ConnectStream throws when a connection accepted from a listener disconn if (server_thread.joinable()) server_thread.join(); std::string_view reason = e.what(); - KJ_EXPECT(reason == "IPC client method called after disconnect."); + KJ_EXPECT(reason == "IPC client method call interrupted by disconnect."); } } diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index f5f35437..6ada5bf1 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -288,7 +288,16 @@ KJ_TEST("Calling IPC method after server connection is closed") KJ_EXPECT(foo->add(1, 2) == 3); setup.server_disconnect(); - EXPECT_EXCEPTION(foo->add(1, 2), "IPC client method call interrupted by disconnect."); + try { + foo->add(1, 2); + KJ_EXPECT(false); + } catch (const std::runtime_error& e) { + std::string_view reason{e.what()}; + + // The disconnect handler may delete the connection before the + // call is processed or while the call is in flight, both errors are possible. + KJ_EXPECT(reason == "IPC client method called after disconnect." || reason == "IPC client method call interrupted by disconnect."); + } } KJ_TEST("Calling IPC method and disconnecting during the call") From fae9a637e35df7020c5f42677e9f089754a95d2c Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:20:47 -0300 Subject: [PATCH 35/81] example: Remove unused kj/async.h include Flagged by IWYU. --- example/example.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/example/example.cpp b/example/example.cpp index e4d159ab..2ea13861 100644 --- a/example/example.cpp +++ b/example/example.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include From 49834b2609e3e4dca1b9cba36ee152552e221039 Mon Sep 17 00:00:00 2001 From: fanquake Date: Tue, 4 Aug 2026 11:02:46 +0100 Subject: [PATCH 36/81] ci: add -Wextra-semi to default config --- ci/configs/default.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/configs/default.bash b/ci/configs/default.bash index 56231228..50c55956 100644 --- a/ci/configs/default.bash +++ b/ci/configs/default.bash @@ -1,5 +1,5 @@ CI_DESC="CI job using default libraries and tools, and running IWYU" CI_DIR=build-default -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wextra-semi" CMAKE_ARGS=(-DMP_ENABLE_IWYU=ON) BUILD_ARGS=(-k) From 648a18589c45c953a498af40f2f3a384c407e18b Mon Sep 17 00:00:00 2001 From: ViniciusCestarii Date: Fri, 31 Jul 2026 10:29:22 -0300 Subject: [PATCH 37/81] proxy: Name threads spawned by the event loop --- cmake/pthread_checks.cmake | 8 ++++++++ include/mp/config.h.in | 1 + include/mp/util.h | 3 +++ src/mp/proxy.cpp | 5 ++++- src/mp/util.cpp | 22 ++++++++++++++++++++++ 5 files changed, 38 insertions(+), 1 deletion(-) diff --git a/cmake/pthread_checks.cmake b/cmake/pthread_checks.cmake index 87711416..1208391b 100644 --- a/cmake/pthread_checks.cmake +++ b/cmake/pthread_checks.cmake @@ -19,6 +19,14 @@ check_cxx_source_compiles(" }" HAVE_PTHREAD_GETNAME_NP) +check_cxx_source_compiles(" + #include + int main(int argc, char** argv) + { + return pthread_setname_np(pthread_self(), \"%s\", (void*)\"name\"); + }" + HAVE_PTHREAD_SETNAME_NP_3ARG) + check_cxx_source_compiles(" #include #include diff --git a/include/mp/config.h.in b/include/mp/config.h.in index 9d3c6240..b4ca6ded 100644 --- a/include/mp/config.h.in +++ b/include/mp/config.h.in @@ -10,6 +10,7 @@ #cmakedefine HAVE_KJ_FILESYSTEM #cmakedefine HAVE_PTHREAD_GETNAME_NP @HAVE_PTHREAD_GETNAME_NP@ +#cmakedefine HAVE_PTHREAD_SETNAME_NP_3ARG @HAVE_PTHREAD_SETNAME_NP_3ARG@ #cmakedefine HAVE_PTHREAD_THREADID_NP @HAVE_PTHREAD_THREADID_NP@ #cmakedefine HAVE_PTHREAD_GETTHREADID_NP @HAVE_PTHREAD_GETTHREADID_NP@ diff --git a/include/mp/util.h b/include/mp/util.h index f647d5d7..293ca67c 100644 --- a/include/mp/util.h +++ b/include/mp/util.h @@ -264,6 +264,9 @@ decltype(auto) TryFinally(Fn&& fn, After&& after) } } +//! Set the OS-level name of the current thread +void SetOsThreadName(const char* name); + //! Format current thread name as "{exe_name}-{$pid}/{thread_name}-{$tid}". std::string ThreadName(const char* exe_name); diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 4c7f7666..eb2aee0c 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -358,6 +358,7 @@ void EventLoop::startAsyncThread() m_cv.notify_all(); } else if (!m_async_fns->empty()) { m_async_thread = std::thread([this] { + SetOsThreadName("capnp-async"); Lock lock(m_mutex); while (m_async_fns) { if (!m_async_fns->empty()) { @@ -483,7 +484,8 @@ kj::Promise ProxyServer::makePool(MakePoolContext context) for (uint32_t i = 0; i < count; ++i) { const std::string thread_name = "pool/" + std::to_string(i); std::promise thread_context; - std::thread thread([&loop, &thread_context, thread_name]() { + std::thread thread([&loop, &thread_context, thread_name, i]() { + SetOsThreadName(("capnp-pool-" + std::to_string(i)).c_str()); CurrentThread().thread_name = ThreadName(loop.m_exe_name) + " (" + thread_name + ")"; CurrentThread().waiter = std::make_unique(); Lock lock(CurrentThread().waiter->m_mutex); @@ -503,6 +505,7 @@ kj::Promise ProxyServer::makeThread(MakeThreadContext context) const std::string from = context.getParams().getName(); std::promise thread_context; std::thread thread([&loop, &thread_context, from]() { + SetOsThreadName("capnp-worker"); CurrentThread().thread_name = ThreadName(loop.m_exe_name) + " (from " + from + ")"; CurrentThread().waiter = std::make_unique(); Lock lock(CurrentThread().waiter->m_mutex); diff --git a/src/mp/util.cpp b/src/mp/util.cpp index f524ab23..7d81ab38 100644 --- a/src/mp/util.cpp +++ b/src/mp/util.cpp @@ -33,6 +33,10 @@ #include #endif // HAVE_PTHREAD_GETTHREADID_NP +#if __has_include() +#include +#endif + extern "C" char **environ; // NOLINT(readability-redundant-declaration) namespace mp { @@ -74,6 +78,24 @@ template } // namespace +// Copied from https://github.com/bitcoin/bitcoin/blob/d3e40af2597/src/util/threadnames.cpp#L21-L36 +void SetOsThreadName(const char* name) +{ +#if defined(PR_SET_NAME) + // Only the first 15 characters are used (16 - NUL terminator) + ::prctl(PR_SET_NAME, name, 0, 0, 0); +#elif defined(HAVE_PTHREAD_SETNAME_NP_3ARG) + pthread_setname_np(pthread_self(), "%s", const_cast(name)); +#elif defined(HAVE_PTHREAD_GETTHREADID_NP) + pthread_set_name_np(pthread_self(), name); +#elif defined(__APPLE__) + pthread_setname_np(name); +#else + // Prevent warnings for unused parameters... + (void)name; +#endif +} + std::string ThreadName(const char* exe_name) { char thread_name[16] = {0}; From 9f25ffca5b0680db2c7f87cc5ab3a17aeadb815c Mon Sep 17 00:00:00 2001 From: ViniciusCestarii Date: Thu, 6 Aug 2026 14:36:08 -0300 Subject: [PATCH 38/81] test: Cover OS thread names for worker, pool, and async threads --- test/mp/test/test.cpp | 63 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index f5f35437..45c9ffe3 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -550,6 +551,68 @@ KJ_TEST("Call async IPC method dispatched to pool thread") } } +#ifdef HAVE_PTHREAD_GETNAME_NP +KJ_TEST("Worker thread has OS thread name") +{ + TestSetup setup; + ProxyClient* foo = setup.client.get(); + foo->initThreadMap(); + + std::promise thread_name; + setup.server->m_impl->m_fn = [&] { thread_name.set_value(ThreadName("")); }; + foo->callFnAsync(); + + const std::string name{thread_name.get_future().get()}; + KJ_EXPECT(name.find("/capnp-worker-") != std::string::npos, name); +} + +KJ_TEST("Pool thread has OS thread name") +{ + TestSetup setup; + ProxyClient* foo = setup.client.get(); + foo->initThreadMap(); + + std::promise thread_name; + setup.server->m_impl->m_fn = [&] { thread_name.set_value(ThreadName("")); }; + + std::promise pool_ready; + foo->m_context.loop->sync([&] { + auto pool_req = foo->m_context.connection->m_thread_map.makePoolRequest(); + pool_req.setCount(1); + foo->m_context.loop->m_task_set->add( + pool_req.send().then([&](auto&&) { pool_ready.set_value(); })); + }); + pool_ready.get_future().get(); + + std::promise done; + foo->m_context.loop->sync([&] { + auto request{foo->m_client.callFnAsyncRequest()}; + foo->m_context.loop->m_task_set->add( + request.send().then([&](auto&&) { done.set_value(); })); + }); + // Wait for the reply before returning, so the connection is not torn down + // while the request is still in flight. + done.get_future().get(); + + const std::string name{thread_name.get_future().get()}; + KJ_EXPECT(name.find("/capnp-pool-0-") != std::string::npos, name); +} + +KJ_TEST("Async cleanup thread has OS thread name") +{ + std::promise thread_name; + { + TestSetup setup; + // FooInterface has no destroy method, so the server ProxyServer runs + // its cleanup functions on the async thread when it is destroyed. + setup.server->m_context.cleanup_fns.emplace_front( + [&] { thread_name.set_value(ThreadName("")); }); + } + const std::string name{thread_name.get_future().get()}; + KJ_EXPECT(name.find("/capnp-async-") != std::string::npos, name); +} +#endif // HAVE_PTHREAD_GETNAME_NP + KJ_TEST("Call async IPC method without thread or pool errors correctly") { TestSetup setup; From 7e94790b08ad3bd19facf808427073371c28f016 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:36:57 +0100 Subject: [PATCH 39/81] ci: Update `capnproto` prerequisites on NetBSD --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a203ae9..a8e5b752 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,9 +50,9 @@ jobs: pkgsrc/devel/libtool-base \ pkgsrc/devel/pkgconf \ pkgsrc/devel/zlib \ - `# gcc15 is referenced here because the pkgsrc framework requires lang/gcc15/version.mk to exist` \ + `# gcc16 is referenced here because the pkgsrc framework requires lang/gcc16/version.mk to exist` \ `# during the "make install" step below, even though we compile our project with gcc14.` \ - pkgsrc/lang/gcc15 \ + pkgsrc/lang/gcc16 \ pkgsrc/mk \ pkgsrc/pkgtools \ pkgsrc/security/openssl \ From 636aaff576bf9129999597dcc76398ccd6a6ba6c Mon Sep 17 00:00:00 2001 From: fanquake Date: Mon, 10 Aug 2026 11:49:38 +0100 Subject: [PATCH 40/81] refactor: add missing [[noreturn]] attributes These will be used downstream, see https://github.com/bitcoin/bitcoin/pull/35911. --- include/mp/proxy-types.h | 2 +- test/mp/test/foo.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mp/proxy-types.h b/include/mp/proxy-types.h index b790dd39..02e8eefb 100644 --- a/include/mp/proxy-types.h +++ b/include/mp/proxy-types.h @@ -220,7 +220,7 @@ void ThrowField(TypeList, InvokeContext& invoke_context, Input&& inpu { ReadField( TypeList(), invoke_context, input, ReadDestEmplace(TypeList(), - [](auto&& ...args) -> const LocalType& { throw LocalType{std::forward(args)...}; })); + [] [[noreturn]] (auto&& ...args) -> const LocalType& { throw LocalType{std::forward(args)...}; })); } //! Special case for generic std::exception. It's an abstract type so it can't diff --git a/test/mp/test/foo.h b/test/mp/test/foo.h index bc1a22ac..01566c58 100644 --- a/test/mp/test/foo.h +++ b/test/mp/test/foo.h @@ -79,7 +79,7 @@ class FooImplementation void addInOut(int x, int& sum) { sum += x; } int mapSize(const std::map& map) { return map.size(); } FooStruct pass(FooStruct foo) { return foo; } - void raise(FooStruct foo) { throw foo; } + [[noreturn]] void raise(FooStruct foo) { throw foo; } void initThreadMap() {} int callback(FooCallback& callback, int arg) { return callback.call(arg); } int callbackUnique(std::unique_ptr callback, int arg) { return callback->call(arg); } From a779a09764c2d24d3a6a54e54126b5623384a020 Mon Sep 17 00:00:00 2001 From: fanquake Date: Wed, 12 Aug 2026 11:22:39 +0100 Subject: [PATCH 41/81] ci: add -Wmissing-noreturn Suppress `-Wc++23-lambda-attributes` warnings in effected jobs. ```bash /libmultiprocess/include/mp/proxy-types.h:223:16: warning: an attribute specifier sequence in this position is a C++23 extension [-Wc++23-lambda-attributes] 223 | [] [[noreturn]] (auto&& ...args) -> const LocalType& { throw LocalType{std::forward(args)...}; })); | ^ 1 warning generated. ``` --- ci/configs/default.bash | 2 +- ci/configs/freebsd.bash | 2 +- ci/configs/llvm.bash | 2 +- ci/configs/macos.bash | 2 +- ci/configs/openbsd.bash | 2 +- ci/configs/sanitize.bash | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/configs/default.bash b/ci/configs/default.bash index 50c55956..94dde74b 100644 --- a/ci/configs/default.bash +++ b/ci/configs/default.bash @@ -1,5 +1,5 @@ CI_DESC="CI job using default libraries and tools, and running IWYU" CI_DIR=build-default -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wextra-semi" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wextra-semi -Wmissing-noreturn" CMAKE_ARGS=(-DMP_ENABLE_IWYU=ON) BUILD_ARGS=(-k) diff --git a/ci/configs/freebsd.bash b/ci/configs/freebsd.bash index 017f2186..096e2649 100644 --- a/ci/configs/freebsd.bash +++ b/ci/configs/freebsd.bash @@ -1,5 +1,5 @@ CI_DESC="CI config for FreeBSD" CI_DIR=build-freebsd -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-c++23-lambda-attributes" CMAKE_ARGS=(-G Ninja) BUILD_ARGS=(-k 0) diff --git a/ci/configs/llvm.bash b/ci/configs/llvm.bash index ba34cbf4..7289208e 100644 --- a/ci/configs/llvm.bash +++ b/ci/configs/llvm.bash @@ -2,7 +2,7 @@ CI_DESC="CI job using LLVM-based libraries and tools (clang, libc++, clang-tidy, CI_DIR=build-llvm NIX_ARGS=(--arg enableLibcxx true) export CXX=clang++ -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wthread-safety -Wno-unused-parameter" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wthread-safety -Wno-unused-parameter -Wno-c++23-lambda-attributes" CMAKE_ARGS=( -G Ninja -DMP_ENABLE_CLANG_TIDY=ON diff --git a/ci/configs/macos.bash b/ci/configs/macos.bash index 895d795f..4c3a0fea 100644 --- a/ci/configs/macos.bash +++ b/ci/configs/macos.bash @@ -1,5 +1,5 @@ CI_DESC="CI config for macOS" CI_DIR=build-macos -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-c++23-lambda-attributes" CMAKE_ARGS=(-G Ninja -DMP_ENABLE_CLANG_TIDY=ON) BUILD_ARGS=(-k 0) diff --git a/ci/configs/openbsd.bash b/ci/configs/openbsd.bash index a404e2bc..944f8d7a 100644 --- a/ci/configs/openbsd.bash +++ b/ci/configs/openbsd.bash @@ -1,5 +1,5 @@ CI_DESC="CI config for OpenBSD" CI_DIR=build-openbsd -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-c++23-lambda-attributes" CMAKE_ARGS=(-G Ninja) BUILD_ARGS=(-k 0) diff --git a/ci/configs/sanitize.bash b/ci/configs/sanitize.bash index 0375fd3a..e582dc5d 100644 --- a/ci/configs/sanitize.bash +++ b/ci/configs/sanitize.bash @@ -2,7 +2,7 @@ CI_DESC="CI job running ThreadSanitizer" CI_DIR=build-sanitize NIX_ARGS=(--arg enableLibcxx true --argstr libcxxSanitizers "Thread" --argstr capnprotoSanitizers "thread") export CXX=clang++ -export CXXFLAGS="-ggdb -Werror -Wall -Wextra -Wpedantic -Wthread-safety -Wno-unused-parameter -fsanitize=thread" +export CXXFLAGS="-ggdb -Werror -Wall -Wextra -Wpedantic -Wthread-safety -Wno-unused-parameter -fsanitize=thread -Wno-c++23-lambda-attributes" CMAKE_ARGS=() BUILD_ARGS=(-k -j4) BUILD_TARGETS=(mptest) From b3b134eed8b4e7d8178b8867977d214579ed9a7a Mon Sep 17 00:00:00 2001 From: fanquake Date: Wed, 12 Aug 2026 11:14:20 +0100 Subject: [PATCH 42/81] ci: add -Wextra-semi to llvm config --- ci/configs/llvm.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/configs/llvm.bash b/ci/configs/llvm.bash index 7289208e..af3b9719 100644 --- a/ci/configs/llvm.bash +++ b/ci/configs/llvm.bash @@ -2,7 +2,7 @@ CI_DESC="CI job using LLVM-based libraries and tools (clang, libc++, clang-tidy, CI_DIR=build-llvm NIX_ARGS=(--arg enableLibcxx true) export CXX=clang++ -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wthread-safety -Wno-unused-parameter -Wno-c++23-lambda-attributes" +export CXXFLAGS="-Werror -Wall -Wextra -Wextra-semi -Wpedantic -Wthread-safety -Wno-unused-parameter -Wno-c++23-lambda-attributes" CMAKE_ARGS=( -G Ninja -DMP_ENABLE_CLANG_TIDY=ON From 137a6e4e03973b6fdc7d2fdf399f3fb37c2c9e1c Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 12 Aug 2026 15:24:31 -0400 Subject: [PATCH 43/81] test: fix race in connect_tests disconnect-deferred-failure test The "ConnectStream defers disconnect failure" test was flaky: there is a race between the event loop detecting the disconnect and foo->add() being called. If the onDisconnect callback fires first and nulls m_context.connection, the error is "called after disconnect"; if foo->add() submits before the callback fires, the error is "interrupted by disconnect". Accept both, with a comment explaining the race. Co-Authored-By: Claude Sonnet 4.6 --- test/mp/test/connect_tests.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/mp/test/connect_tests.cpp b/test/mp/test/connect_tests.cpp index 73b1a396..40fe08e0 100644 --- a/test/mp/test/connect_tests.cpp +++ b/test/mp/test/connect_tests.cpp @@ -139,7 +139,13 @@ KJ_TEST("ConnectStream defers disconnect failure to the first IPC request for in } catch (const std::runtime_error& e) { std::string_view reason = e.what(); - KJ_EXPECT(reason == "IPC client method called after disconnect."); + // There is a race between the event loop detecting the disconnect + // and foo->add() being called. If the onDisconnect callback fires + // first and nulls m_context.connection, the error is "called after + // disconnect"; if foo->add() submits before the callback fires, + // the error is "interrupted by disconnect". + KJ_EXPECT(reason == "IPC client method called after disconnect." || + reason == "IPC client method call interrupted by disconnect."); } } From cc260f2526ff13221f03a6fdec571b55c0429952 Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:01:21 -0300 Subject: [PATCH 44/81] test: replace capnp fix link with upstream PR --- test/mp/test/listen_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mp/test/listen_tests.cpp b/test/mp/test/listen_tests.cpp index b9b0bfc2..0f8706ae 100644 --- a/test/mp/test/listen_tests.cpp +++ b/test/mp/test/listen_tests.cpp @@ -241,7 +241,7 @@ KJ_TEST("ListenConnections handles a client that disconnects before being accept // The event loop then reports this as an uncaught task exception. We catch and ignore // this specific error here so that the corresponding CI job does not fail. // - // This is a Cap'n Proto bug, a fix is available in the v2 branch at: https://github.com/capnproto/capnproto/commit/7df5bd078f389ded313479981bd0ae06cbcdfe1b#diff-ec577ad66535f58f6d7396ea51d3e56c0065308aa8fb02751cd6a8cfaa67252fR1358-R1372 + // This is a Cap'n Proto bug, fixed by https://github.com/capnproto/capnproto/pull/2748 if (log.level == mp::Log::Error && log.message.find("Uncaught exception in daemonized task.") != std::string::npos) { Lock lock(mutex); accept_error = true; From 70467c5a727ba50789865b0323f313167d2e500b Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:30:41 -0300 Subject: [PATCH 45/81] test: add m_ prefix to TestSetup members in connect tests --- test/mp/test/connect_tests.cpp | 52 +++++++++++++++++----------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/test/mp/test/connect_tests.cpp b/test/mp/test/connect_tests.cpp index 73b1a396..f64f7368 100644 --- a/test/mp/test/connect_tests.cpp +++ b/test/mp/test/connect_tests.cpp @@ -45,14 +45,14 @@ void DefaultLogHandler(mp::LogMessage log) class TestSetup { public: - int client_fd; - int server_fd; + int m_client_fd; + int m_server_fd; - mp::EventLoop* loop; - std::optional loop_ref; + mp::EventLoop* m_loop; + std::optional m_loop_ref; //! Thread variable should be after other struct members so the thread does //! not start until the other members are initialized. - std::thread loop_thread; + std::thread m_loop_thread; TestSetup(mp::LogFn log_handler = DefaultLogHandler) : TestSetup( @@ -65,26 +65,26 @@ class TestSetup mp::LogFn log_handler = DefaultLogHandler) { std::promise loop_promise; - loop_thread = std::thread([&, log_handler] { + m_loop_thread = std::thread([&, log_handler] { mp::EventLoop loop("mptest-connect", log_handler); loop_promise.set_value(&loop); loop.loop(); }); - loop = loop_promise.get_future().get(); - loop_ref.emplace(*loop); + m_loop = loop_promise.get_future().get(); + m_loop_ref.emplace(*m_loop); // Initialize and store sockets int fds[2] = {-1, -1}; init_sockets(fds); - client_fd = fds[0]; - server_fd = fds[1]; + m_client_fd = fds[0]; + m_server_fd = fds[1]; } ~TestSetup() { - loop_ref.reset(); - loop_thread.join(); + m_loop_ref.reset(); + m_loop_thread.join(); } }; @@ -95,11 +95,11 @@ KJ_TEST("ConnectStream connects to a socket serving a valid init interface") std::thread server_thread([&setup]() { mp::EventLoop server_loop("mptest-valid-server", DefaultLogHandler); std::unique_ptr init = std::make_unique(); - ServeStream(server_loop, MakeStream(server_loop, setup.server_fd), *init); + ServeStream(server_loop, MakeStream(server_loop, setup.m_server_fd), *init); server_loop.loop(); }); - auto init = ConnectStream(*setup.loop, MakeStream(*setup.loop, setup.client_fd)); + auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, setup.m_client_fd)); init.reset(); server_thread.join(); @@ -110,10 +110,10 @@ KJ_TEST("ConnectStream throws when the socket is already disconnected") { TestSetup setup; - close(setup.server_fd); + close(setup.m_server_fd); try { - auto init = ConnectStream(*setup.loop, MakeStream(*setup.loop, setup.client_fd)); + auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, setup.m_client_fd)); KJ_EXPECT(false); } catch (const std::runtime_error& e) { @@ -127,11 +127,11 @@ KJ_TEST("ConnectStream defers disconnect failure to the first IPC request for in { TestSetup setup; - close(setup.server_fd); + close(setup.m_server_fd); // Without a construct() method no IPC call is made during client // creation, so ConnectStream succeeds even though the peer is gone. - auto foo = ConnectStream(*setup.loop, MakeStream(*setup.loop, setup.client_fd)); + auto foo = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, setup.m_client_fd)); try { foo->add(1, 2); @@ -158,9 +158,9 @@ KJ_TEST("ConnectStream handles a disconnect when no client calls are made") DefaultLogHandler(log); }); - close(setup.server_fd); + close(setup.m_server_fd); - auto foo = ConnectStream(*setup.loop, MakeStream(*setup.loop, setup.client_fd)); + auto foo = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, setup.m_client_fd)); // The disconnect handler registered by ProxyClientBase should run and // delete the connection even when no calls are ever made. @@ -176,15 +176,15 @@ KJ_TEST("ConnectStream throws when the socket disconnects after receiving data") char buf[128]; ssize_t bytes_received = - recv(setup.server_fd, buf, sizeof(buf), 0); + recv(setup.m_server_fd, buf, sizeof(buf), 0); if (bytes_received > 0) { - close(setup.server_fd); + close(setup.m_server_fd); } }); try { - auto init = ConnectStream(*setup.loop, MakeStream(*setup.loop, setup.client_fd)); + auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, setup.m_client_fd)); if (server_thread.joinable()) server_thread.join(); KJ_EXPECT(false); @@ -208,7 +208,7 @@ KJ_TEST("ConnectStream throws when a connection accepted from a listener disconn std::thread server_thread([&setup]() { char buf[128]; - int connection_fd = accept(setup.server_fd, nullptr, nullptr); + int connection_fd = accept(setup.m_server_fd, nullptr, nullptr); if (connection_fd >= 0) { ssize_t bytes_received = @@ -218,11 +218,11 @@ KJ_TEST("ConnectStream throws when a connection accepted from a listener disconn close(connection_fd); } } - close(setup.server_fd); + close(setup.m_server_fd); }); try { - auto init = ConnectStream(*setup.loop, MakeStream(*setup.loop, setup.client_fd)); + auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, setup.m_client_fd)); if (server_thread.joinable()) server_thread.join(); KJ_EXPECT(false); From b54a1633085b9ab7254d04f0efbead459f1c23f6 Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:16:20 -0300 Subject: [PATCH 46/81] test: drop TestSetup socket members in connect tests Use the existing `SocketPair()` helper directly in each test instead of the `init_sockets` callback and fd members, which nothing else in the class used. --- test/mp/test/connect_tests.cpp | 71 ++++++++++++++-------------------- 1 file changed, 28 insertions(+), 43 deletions(-) diff --git a/test/mp/test/connect_tests.cpp b/test/mp/test/connect_tests.cpp index f64f7368..cd70f953 100644 --- a/test/mp/test/connect_tests.cpp +++ b/test/mp/test/connect_tests.cpp @@ -7,18 +7,18 @@ #include #include #include -#include #include +#include #include #include +#include #include -#include #include +#include #include #include #include // IWYU pragma: keep -#include #include #include #include @@ -45,9 +45,6 @@ void DefaultLogHandler(mp::LogMessage log) class TestSetup { public: - int m_client_fd; - int m_server_fd; - mp::EventLoop* m_loop; std::optional m_loop_ref; //! Thread variable should be after other struct members so the thread does @@ -55,14 +52,6 @@ class TestSetup std::thread m_loop_thread; TestSetup(mp::LogFn log_handler = DefaultLogHandler) - : TestSetup( - [](int fds[2]) { - KJ_REQUIRE(socketpair(AF_UNIX, SOCK_STREAM, 0, fds) != -1); - }, - log_handler) {} - - TestSetup(const std::function& init_sockets, - mp::LogFn log_handler = DefaultLogHandler) { std::promise loop_promise; m_loop_thread = std::thread([&, log_handler] { @@ -72,13 +61,6 @@ class TestSetup }); m_loop = loop_promise.get_future().get(); m_loop_ref.emplace(*m_loop); - - // Initialize and store sockets - int fds[2] = {-1, -1}; - init_sockets(fds); - - m_client_fd = fds[0]; - m_server_fd = fds[1]; } ~TestSetup() @@ -91,15 +73,16 @@ class TestSetup KJ_TEST("ConnectStream connects to a socket serving a valid init interface") { TestSetup setup; + auto [client_fd, server_fd] = SocketPair(); - std::thread server_thread([&setup]() { + std::thread server_thread([&]() { mp::EventLoop server_loop("mptest-valid-server", DefaultLogHandler); std::unique_ptr init = std::make_unique(); - ServeStream(server_loop, MakeStream(server_loop, setup.m_server_fd), *init); + ServeStream(server_loop, MakeStream(server_loop, server_fd), *init); server_loop.loop(); }); - auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, setup.m_client_fd)); + auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); init.reset(); server_thread.join(); @@ -109,11 +92,12 @@ KJ_TEST("ConnectStream connects to a socket serving a valid init interface") KJ_TEST("ConnectStream throws when the socket is already disconnected") { TestSetup setup; + auto [client_fd, server_fd] = SocketPair(); - close(setup.m_server_fd); + close(server_fd); try { - auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, setup.m_client_fd)); + auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); KJ_EXPECT(false); } catch (const std::runtime_error& e) { @@ -126,12 +110,13 @@ KJ_TEST("ConnectStream throws when the socket is already disconnected") KJ_TEST("ConnectStream defers disconnect failure to the first IPC request for interfaces without construct()") { TestSetup setup; + auto [client_fd, server_fd] = SocketPair(); - close(setup.m_server_fd); + close(server_fd); // Without a construct() method no IPC call is made during client // creation, so ConnectStream succeeds even though the peer is gone. - auto foo = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, setup.m_client_fd)); + auto foo = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); try { foo->add(1, 2); @@ -157,10 +142,11 @@ KJ_TEST("ConnectStream handles a disconnect when no client calls are made") } DefaultLogHandler(log); }); + auto [client_fd, server_fd] = SocketPair(); - close(setup.m_server_fd); + close(server_fd); - auto foo = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, setup.m_client_fd)); + auto foo = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); // The disconnect handler registered by ProxyClientBase should run and // delete the connection even when no calls are ever made. @@ -171,20 +157,21 @@ KJ_TEST("ConnectStream handles a disconnect when no client calls are made") KJ_TEST("ConnectStream throws when the socket disconnects after receiving data") { TestSetup setup; + auto [client_fd, server_fd] = SocketPair(); - std::thread server_thread([&setup]() { + std::thread server_thread([&]() { char buf[128]; ssize_t bytes_received = - recv(setup.m_server_fd, buf, sizeof(buf), 0); + recv(server_fd, buf, sizeof(buf), 0); if (bytes_received > 0) { - close(setup.m_server_fd); + close(server_fd); } }); try { - auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, setup.m_client_fd)); + auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); if (server_thread.joinable()) server_thread.join(); KJ_EXPECT(false); @@ -199,16 +186,14 @@ KJ_TEST("ConnectStream throws when the socket disconnects after receiving data") KJ_TEST("ConnectStream throws when a connection accepted from a listener disconnects after receiving data") { UnixListener listener; + TestSetup setup; + int client_fd = listener.MakeConnectedSocket(); + int server_fd = listener.release(); - TestSetup setup([&listener](int fds[2]) { - fds[0] = listener.MakeConnectedSocket(); // client_fd - fds[1] = listener.release(); // server_fd - }); - - std::thread server_thread([&setup]() { + std::thread server_thread([&]() { char buf[128]; - int connection_fd = accept(setup.m_server_fd, nullptr, nullptr); + int connection_fd = accept(server_fd, nullptr, nullptr); if (connection_fd >= 0) { ssize_t bytes_received = @@ -218,11 +203,11 @@ KJ_TEST("ConnectStream throws when a connection accepted from a listener disconn close(connection_fd); } } - close(setup.m_server_fd); + close(server_fd); }); try { - auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, setup.m_client_fd)); + auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); if (server_thread.joinable()) server_thread.join(); KJ_EXPECT(false); From 038d33eb31eb0d5742a27caeb0c5b244f0f794a1 Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:28:01 -0300 Subject: [PATCH 47/81] test: share DefaultLogHandler between test files Move it to a common.h file. The connect_tests copy previously didn't log messages, now it does through the shared version. --- test/mp/test/common.h | 27 +++++++++++++++++++++++++++ test/mp/test/connect_tests.cpp | 9 +-------- test/mp/test/listen_tests.cpp | 14 ++------------ 3 files changed, 30 insertions(+), 20 deletions(-) create mode 100644 test/mp/test/common.h diff --git a/test/mp/test/common.h b/test/mp/test/common.h new file mode 100644 index 00000000..527d4a2a --- /dev/null +++ b/test/mp/test/common.h @@ -0,0 +1,27 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_TEST_COMMON_H +#define MP_TEST_COMMON_H + +#include +#include + +#include + +namespace mp { +namespace test { + +//! Default event loop log handler used by tests. Logs all messages and throws +//! on errors so calling code can assert on them. +inline void DefaultLogHandler(LogMessage log) +{ + KJ_LOG(INFO, log.level, log.message); + if (log.level == Log::Raise) throw std::runtime_error(log.message); +} + +} // namespace test +} // namespace mp + +#endif // MP_TEST_COMMON_H diff --git a/test/mp/test/connect_tests.cpp b/test/mp/test/connect_tests.cpp index cd70f953..f3062741 100644 --- a/test/mp/test/connect_tests.cpp +++ b/test/mp/test/connect_tests.cpp @@ -1,6 +1,7 @@ // Copyright (c) The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "common.h" #include "unixlistener.h" #include #include @@ -34,14 +35,6 @@ namespace { constexpr auto FAILURE_TIMEOUT = std::chrono::seconds{30}; -//! Default event loop log handler used by tests, throws so the calling code -//! can assert on errors. -void DefaultLogHandler(mp::LogMessage log) -{ - if (log.level == mp::Log::Raise) - throw std::runtime_error(log.message); -} - class TestSetup { public: diff --git a/test/mp/test/listen_tests.cpp b/test/mp/test/listen_tests.cpp index 0f8706ae..8d06cb35 100644 --- a/test/mp/test/listen_tests.cpp +++ b/test/mp/test/listen_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "common.h" #include "unixlistener.h" #include #include @@ -23,7 +24,6 @@ #include #include // IWYU pragma: keep #include -#include #include #include #include @@ -42,10 +42,7 @@ class ClientSetup public: explicit ClientSetup(int fd) : thread([this, fd] { - EventLoop loop("mptest-client", [](mp::LogMessage log) { - KJ_LOG(INFO, log.level, log.message); - if (log.level == mp::Log::Raise) throw std::runtime_error(log.message); - }); + EventLoop loop("mptest-client", DefaultLogHandler); client_promise.set_value(ConnectStream(loop, MakeStream(loop, fd))); loop.loop(); }) @@ -67,13 +64,6 @@ class ClientSetup std::thread thread; }; -//! Default server event loop log handler, throws so tests can assert on errors. -void DefaultLogHandler(mp::LogMessage log) -{ - KJ_LOG(INFO, log.level, log.message); - if (log.level == mp::Log::Raise) throw std::runtime_error(log.message); -} - //! Runs a server EventLoop on its own thread, starts ListenConnections() on a //! UnixListener socket, and records connection/disconnection counts through //! EventLoop test hooks From 44bc4630bc18bb03977a3616bf979fb2b3192f54 Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:35:23 -0300 Subject: [PATCH 48/81] test: drop mp:: prefixes in connect tests Unnecessary since the tests are inside the mp namespace. --- test/mp/test/connect_tests.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/mp/test/connect_tests.cpp b/test/mp/test/connect_tests.cpp index f3062741..83d248f5 100644 --- a/test/mp/test/connect_tests.cpp +++ b/test/mp/test/connect_tests.cpp @@ -38,17 +38,17 @@ constexpr auto FAILURE_TIMEOUT = std::chrono::seconds{30}; class TestSetup { public: - mp::EventLoop* m_loop; - std::optional m_loop_ref; + EventLoop* m_loop; + std::optional m_loop_ref; //! Thread variable should be after other struct members so the thread does //! not start until the other members are initialized. std::thread m_loop_thread; - TestSetup(mp::LogFn log_handler = DefaultLogHandler) + TestSetup(LogFn log_handler = DefaultLogHandler) { - std::promise loop_promise; + std::promise loop_promise; m_loop_thread = std::thread([&, log_handler] { - mp::EventLoop loop("mptest-connect", log_handler); + EventLoop loop("mptest-connect", log_handler); loop_promise.set_value(&loop); loop.loop(); }); @@ -69,7 +69,7 @@ KJ_TEST("ConnectStream connects to a socket serving a valid init interface") auto [client_fd, server_fd] = SocketPair(); std::thread server_thread([&]() { - mp::EventLoop server_loop("mptest-valid-server", DefaultLogHandler); + EventLoop server_loop("mptest-valid-server", DefaultLogHandler); std::unique_ptr init = std::make_unique(); ServeStream(server_loop, MakeStream(server_loop, server_fd), *init); server_loop.loop(); @@ -127,8 +127,8 @@ KJ_TEST("ConnectStream handles a disconnect when no client calls are made") std::condition_variable cv; bool warned = false; - TestSetup setup([&](mp::LogMessage log) { - if (log.level == mp::Log::Warning && log.message.find("unexpected network disconnect") != std::string::npos) { + TestSetup setup([&](LogMessage log) { + if (log.level == Log::Warning && log.message.find("unexpected network disconnect") != std::string::npos) { const std::lock_guard lock(mutex); warned = true; cv.notify_all(); From 113f1d4d28702349a7269eef2557d8a56ff3354e Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:39:45 -0300 Subject: [PATCH 49/81] test: join server thread unconditionally in connect tests --- test/mp/test/connect_tests.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test/mp/test/connect_tests.cpp b/test/mp/test/connect_tests.cpp index 83d248f5..2d77d8f4 100644 --- a/test/mp/test/connect_tests.cpp +++ b/test/mp/test/connect_tests.cpp @@ -166,14 +166,12 @@ KJ_TEST("ConnectStream throws when the socket disconnects after receiving data") try { auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); - if (server_thread.joinable()) server_thread.join(); KJ_EXPECT(false); } catch (const std::runtime_error& e) { - if (server_thread.joinable()) server_thread.join(); - std::string_view reason = e.what(); KJ_EXPECT(reason == "IPC client method call interrupted by disconnect."); } + server_thread.join(); } KJ_TEST("ConnectStream throws when a connection accepted from a listener disconnects after receiving data") @@ -202,14 +200,12 @@ KJ_TEST("ConnectStream throws when a connection accepted from a listener disconn try { auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); - if (server_thread.joinable()) server_thread.join(); KJ_EXPECT(false); } catch (const std::runtime_error& e) { - if (server_thread.joinable()) server_thread.join(); - std::string_view reason = e.what(); KJ_EXPECT(reason == "IPC client method call interrupted by disconnect."); } + server_thread.join(); } } // namespace From 7eb741e6359afe94c2a8c90cf829bf6144cb20f0 Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:40:46 -0300 Subject: [PATCH 50/81] test: drop unnecessary KJ_EXPECT(true) --- test/mp/test/connect_tests.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/test/mp/test/connect_tests.cpp b/test/mp/test/connect_tests.cpp index 2d77d8f4..5ff88c1f 100644 --- a/test/mp/test/connect_tests.cpp +++ b/test/mp/test/connect_tests.cpp @@ -79,7 +79,6 @@ KJ_TEST("ConnectStream connects to a socket serving a valid init interface") init.reset(); server_thread.join(); - KJ_EXPECT(true); } KJ_TEST("ConnectStream throws when the socket is already disconnected") From b9c36c617518c0026f3b8c35014415b7437f6b8a Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:53:21 -0300 Subject: [PATCH 51/81] test: close sockets unconditionally and check errors with KJ_SYSCALL The conditional served no purpose and leaked the descriptor when `recv()` failed or returned zero. --- test/mp/test/connect_tests.cpp | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/test/mp/test/connect_tests.cpp b/test/mp/test/connect_tests.cpp index 5ff88c1f..b029e448 100644 --- a/test/mp/test/connect_tests.cpp +++ b/test/mp/test/connect_tests.cpp @@ -86,7 +86,7 @@ KJ_TEST("ConnectStream throws when the socket is already disconnected") TestSetup setup; auto [client_fd, server_fd] = SocketPair(); - close(server_fd); + KJ_SYSCALL(close(server_fd)); try { auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); @@ -104,7 +104,7 @@ KJ_TEST("ConnectStream defers disconnect failure to the first IPC request for in TestSetup setup; auto [client_fd, server_fd] = SocketPair(); - close(server_fd); + KJ_SYSCALL(close(server_fd)); // Without a construct() method no IPC call is made during client // creation, so ConnectStream succeeds even though the peer is gone. @@ -136,7 +136,7 @@ KJ_TEST("ConnectStream handles a disconnect when no client calls are made") }); auto [client_fd, server_fd] = SocketPair(); - close(server_fd); + KJ_SYSCALL(close(server_fd)); auto foo = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); @@ -154,12 +154,8 @@ KJ_TEST("ConnectStream throws when the socket disconnects after receiving data") std::thread server_thread([&]() { char buf[128]; - ssize_t bytes_received = - recv(server_fd, buf, sizeof(buf), 0); - - if (bytes_received > 0) { - close(server_fd); - } + recv(server_fd, buf, sizeof(buf), 0); + KJ_SYSCALL(close(server_fd)); }); try { @@ -186,14 +182,10 @@ KJ_TEST("ConnectStream throws when a connection accepted from a listener disconn int connection_fd = accept(server_fd, nullptr, nullptr); if (connection_fd >= 0) { - ssize_t bytes_received = - recv(connection_fd, buf, sizeof(buf), 0); - - if (bytes_received > 0) { - close(connection_fd); - } + recv(connection_fd, buf, sizeof(buf), 0); + KJ_SYSCALL(close(connection_fd)); } - close(server_fd); + KJ_SYSCALL(close(server_fd)); }); try { From c39c7850c663d1915ae2478dc53f2c7b690504db Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Thu, 13 Aug 2026 00:05:31 -0300 Subject: [PATCH 52/81] doc: note construct() call in valid init interface test --- test/mp/test/connect_tests.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/mp/test/connect_tests.cpp b/test/mp/test/connect_tests.cpp index b029e448..cad55b1d 100644 --- a/test/mp/test/connect_tests.cpp +++ b/test/mp/test/connect_tests.cpp @@ -75,6 +75,8 @@ KJ_TEST("ConnectStream connects to a socket serving a valid init interface") server_loop.loop(); }); + // FooInit has a `construct()` method, so this connects to the server and + // sends an IPC request that must complete successfully. auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); init.reset(); From d6f8588d1ab8195c72a24359d0e67f9f6c0f11b2 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 8 Jul 2026 13:03:00 -0400 Subject: [PATCH 53/81] proxy: fix BuildList to use non-const iteration for interface types BuildList turns a C++ container into a capnp list. It looped over the container with `for (const auto& elem : value)`, so every element was const. That breaks when the elements are interface pointers: to send one to the capnp server the proxy code has to take ownership away from the unique_ptr by calling unique_ptr::release(), and that cannot be called on a const object. So the code failed to compile, and returning a list of interface pointers stopped working after #277. Fix this by looping with a plain iterator and passing each element to BuildField as non-const, only moving elements out of the container when the container itself is a temporary. See the code comment for details. In Bitcoin Core, interfaces::Node::listExternalSigners() returns a vector> over IPC and hits this code path. Add a `listBars` method to the test FooInterface that returns a vector>, plus a test that calls it over IPC, so this is covered and does not break again. Co-Authored-by: Sjors Provoost Co-Authored-By: Claude Opus 4.8 --- include/mp/proxy-types.h | 25 ++++++++++++++++++++++--- include/mp/type-map.h | 2 ++ src/mp/gen.cpp | 1 + test/mp/test/foo-types.h | 1 + test/mp/test/foo.capnp | 6 ++++++ test/mp/test/foo.h | 24 ++++++++++++++++++++++++ test/mp/test/test.cpp | 10 ++++++++++ 7 files changed, 66 insertions(+), 3 deletions(-) diff --git a/include/mp/proxy-types.h b/include/mp/proxy-types.h index 02e8eefb..3a5dd3d8 100644 --- a/include/mp/proxy-types.h +++ b/include/mp/proxy-types.h @@ -285,9 +285,28 @@ void BuildList(TypeList, InvokeContext& invoke_context, Output&& outp { auto list = output.init(value.size()); size_t i = 0; - for (const auto& elem : value) { - BuildField(TypeList(), invoke_context, ListOutput(list, i), elem); - ++i; + // Iterate with an explicit iterator rather than a range-for loop so the + // value category of `*it` is passed through to BuildField unchanged. This + // matters for two reasons: + // + // - Elements must be passed as non-const so that BuildField can move out of + // them, e.g. calling unique_ptr::release() to transfer ownership of an + // interface pointer to the capnp server. + // + // - For proxy containers like std::vector, `*it` is a prvalue proxy + // object rather than a reference. A range-for loop would bind it to a + // named variable and demote it to an lvalue; passing `*it` directly + // preserves the prvalue-ness. + // + // Only move out of elements when `value` itself is an rvalue container that + // is about to be destroyed. When it is an lvalue reference owned by the + // caller, pass elements as lvalues so BuildField does not move from them. + for (auto it = value.begin(); it != value.end(); ++it, ++i) { + if constexpr (std::is_lvalue_reference_v) { + BuildField(TypeList(), invoke_context, ListOutput(list, i), *it); + } else { + BuildField(TypeList(), invoke_context, ListOutput(list, i), std::move(*it)); + } } } diff --git a/include/mp/type-map.h b/include/mp/type-map.h index e00a7813..ccfefb87 100644 --- a/include/mp/type-map.h +++ b/include/mp/type-map.h @@ -9,6 +9,8 @@ #include #include +#include + namespace mp { template void CustomBuildField(TypeList>, diff --git a/src/mp/gen.cpp b/src/mp/gen.cpp index 7733eba6..8d733a91 100644 --- a/src/mp/gen.cpp +++ b/src/mp/gen.cpp @@ -329,6 +329,7 @@ static void Generate(kj::StringPtr src_prefix, cpp_client << "#include \n"; cpp_client << "#include \n"; cpp_client << "#include \n"; + cpp_client << "#include \n"; cpp_client << "#include \n"; cpp_client << "#include \n"; cpp_client << "#include \n"; diff --git a/test/mp/test/foo-types.h b/test/mp/test/foo-types.h index 1bc6c523..ee12f490 100644 --- a/test/mp/test/foo-types.h +++ b/test/mp/test/foo-types.h @@ -39,6 +39,7 @@ struct ExtendedCallback; // IWYU pragma: export struct FooCallback; // IWYU pragma: export struct FooFn; // IWYU pragma: export struct FooInterface; // IWYU pragma: export +struct BarInterface; // IWYU pragma: export } // namespace messages template diff --git a/test/mp/test/foo.capnp b/test/mp/test/foo.capnp index 18df85aa..11bfd9c5 100644 --- a/test/mp/test/foo.capnp +++ b/test/mp/test/foo.capnp @@ -37,6 +37,7 @@ interface FooInterface $Proxy.wrap("mp::test::FooImplementation") { callFnAsync @18 (context :Proxy.Context) -> (); callIntFnAsync @21 (context :Proxy.Context, arg :Int32) -> (result :Int32); passDataPointers @22 (arg :List(Data)) -> (result :List(Data)); + listBars @25 (context :Proxy.Context, n :Int32) -> (result :List(BarInterface)); } interface FooInit $Proxy.wrap("mp::test::FooInit") { @@ -52,6 +53,11 @@ interface ExtendedCallback extends(FooCallback) $Proxy.wrap("mp::test::ExtendedC callExtended @0 (context :Proxy.Context, arg :Int32) -> (result :Int32); } +interface BarInterface $Proxy.wrap("mp::test::Bar") { + destroy @0 (context :Proxy.Context) -> (); + value @1 (context :Proxy.Context) -> (result :Int32); +} + interface FooFn $Proxy.wrap("ProxyCallback>") { destroy @0 (context :Proxy.Context) -> (); call @1 (context :Proxy.Context) -> (result :Int32); diff --git a/test/mp/test/foo.h b/test/mp/test/foo.h index 01566c58..8fc34eee 100644 --- a/test/mp/test/foo.h +++ b/test/mp/test/foo.h @@ -71,6 +71,23 @@ class FooInit { }; +//! A second, arbitrary interface used to test returning a +//! list of interface objects. +class Bar +{ +public: + virtual ~Bar() = default; + virtual int value() = 0; +}; + +//! Concrete Bar that returns a fixed value, used by listBars tests. +class SimpleBar : public Bar +{ +public: + explicit SimpleBar(int value) : m_value(value) {} + int value() override { return m_value; } + int m_value; +}; class FooImplementation { public: @@ -96,6 +113,13 @@ class FooImplementation double passDouble(double value) { return value; } int passFn(std::function fn) { return fn(); } std::vector passDataPointers(std::vector values) { return values; } + std::vector> listBars(int n) + { + std::vector> result; + result.reserve(n); + for (int i = 0; i < n; ++i) result.push_back(std::make_unique(i)); + return result; + } std::shared_ptr m_callback; void callFn() { assert(m_fn); m_fn(); } void callFnAsync() { assert(m_fn); m_fn(); } diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 7af1365e..400aff73 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -272,6 +272,16 @@ KJ_TEST("Call FooInterface methods") KJ_REQUIRE(data_out[0] != nullptr); KJ_EXPECT(*data_out[0] == *data_in[0]); KJ_EXPECT(!data_out[1]); + + // Test returning vector> from server. This exercises + // BuildList with interface element types, which requires non-const iteration + // so unique_ptr::release() can transfer ownership to the proxy server. + std::vector> bars{foo->listBars(3)}; + KJ_REQUIRE(bars.size() == 3u); + for (int i = 0; i < 3; ++i) { + KJ_REQUIRE(bars[i] != nullptr); + KJ_EXPECT(bars[i]->value() == i); + } } KJ_TEST("Call IPC method after client connection is closed") From b3fc922ee4a663811e0048556b788b4ca473d781 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Tue, 9 Sep 2025 14:13:59 -0400 Subject: [PATCH 54/81] ci: add newdeps job testing newest versions of cmake and capnproto --- .github/workflows/ci.yml | 4 ++-- ci/README.md | 1 + ci/configs/newdeps.bash | 6 ++++++ ci/scripts/ci.sh | 17 +++++++++++++++++ shell.nix | 7 ++++++- 5 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 ci/configs/newdeps.bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8e5b752..5ad4597f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,7 +106,7 @@ jobs: uses: vmactions/freebsd-vm@v1 with: prepare: | - pkg install -y cmake ninja bash capnproto + pkg install -y cmake ninja bash capnproto git sync: 'rsync' copyback: false @@ -148,7 +148,7 @@ jobs: strategy: fail-fast: false matrix: - config: [default, llvm, gnu32, sanitize, olddeps] + config: [default, llvm, gnu32, sanitize, olddeps, newdeps] name: build โ€ข ${{ matrix.config }} diff --git a/ci/README.md b/ci/README.md index fef1c022..843b61e8 100644 --- a/ci/README.md +++ b/ci/README.md @@ -21,6 +21,7 @@ CI_CONFIG=ci/configs/llvm.bash ci/scripts/run.sh CI_CONFIG=ci/configs/gnu32.bash ci/scripts/run.sh CI_CONFIG=ci/configs/sanitize.bash ci/scripts/run.sh CI_CONFIG=ci/configs/olddeps.bash ci/scripts/run.sh +CI_CONFIG=ci/configs/newdeps.bash ci/scripts/run.sh ``` By default CI jobs will reuse their build directories. `CI_CLEAN=1` can be specified to delete them before running instead. diff --git a/ci/configs/newdeps.bash b/ci/configs/newdeps.bash new file mode 100644 index 00000000..e05913a6 --- /dev/null +++ b/ci/configs/newdeps.bash @@ -0,0 +1,6 @@ +CI_DESC="CI job using newest Cap'n Proto and cmake versions" +CI_DIR=build-newdeps +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds" +CAPNP_CHECKOUT=master +NIX_ARGS=(--argstr capnprotoVersion "none" --argstr cmakeVersion "4.1.1") +BUILD_ARGS=(-k) diff --git a/ci/scripts/ci.sh b/ci/scripts/ci.sh index 0022fd28..b4bd8fb4 100755 --- a/ci/scripts/ci.sh +++ b/ci/scripts/ci.sh @@ -21,10 +21,27 @@ cmake --version cmake_ver=$(cmake --version | awk '/version/{print $3; exit}') ver_ge() { [ "$(printf '%s\n' "$2" "$1" | sort -V | head -n1)" = "$2" ]; } +# If CAPNP_CHECKOUT was requested, clone and install requested Cap'n Proto branch or tag +capnp_prefix= +if [ -n "${CAPNP_CHECKOUT-}" ]; then + capnp_prefix="$PWD/capnp-install" + [ -e "capnp" ] || git clone -b "${CAPNP_CHECKOUT}" "https://github.com/capnproto/capnproto" capnp + mkdir -p capnp/build + ( + cd capnp/build + git --no-pager log -1 || true + CXXFLAGS="-std=c++20" cmake .. "-DCMAKE_INSTALL_PREFIX=${capnp_prefix}" -DBUILD_TESTING=OFF -DWITH_OPENSSL=OFF -DWITH_ZLIB=OFF + cmake --build . + cmake --install . + ) + export CMAKE_PREFIX_PATH="${capnp_prefix}:${CMAKE_PREFIX_PATH-}" +fi + src_dir=$PWD mkdir -p "$CI_DIR" cd "$CI_DIR" export CMAKE_BUILD_PARALLEL_LEVEL="$(nproc)" +git --no-pager log -1 || true cmake "$src_dir" "${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}" if ver_ge "$cmake_ver" "3.15"; then cmake --build . -t "${BUILD_TARGETS[@]}" -- "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" diff --git a/shell.nix b/shell.nix index 2d115fea..17789f31 100644 --- a/shell.nix +++ b/shell.nix @@ -55,6 +55,7 @@ let clang-tools = llvm.clang-tools.override { inherit enableLibcxx; }; cmakeHashes = { "3.12.4" = "sha256-UlVYS/0EPrcXViz/iULUcvHA5GecSUHYS6raqbKOMZQ="; + "4.1.1" = "sha256-sp9vGXM6oiS3djUHoQikJ+1Ixojh+vIrKcROHDBUkoI="; }; gcc = if gccVersion == null then null else builtins.getAttr ("gcc" + gccVersion) pkgs; cmakeBuild = if cmakeVersion == null then pkgs.cmake else (pkgs.cmake.overrideAttrs (old: { @@ -66,11 +67,12 @@ let patches = []; })).override { isMinimalBuild = true; }; in crossPkgs.mkShell { - buildInputs = [ + buildInputs = lib.optionals (capnprotoVersion != "none") [ capnproto ]; nativeBuildInputs = with pkgs; [ cmakeBuild + git include-what-you-use ninja ] ++ lib.optional (gcc != null) gcc ++ lib.optionals (!minimal) [ @@ -83,4 +85,7 @@ in crossPkgs.mkShell { # Tell IWYU where its libc++ mapping lives IWYU_MAPPING_FILE = if enableLibcxx then "${llvm.libcxx.dev}/include/c++/v1/libcxx.imp" else null; + + # Avoid "SSL certificate problem: unable to get local issuer certificate" error during git clone in ci/scripts/ci.sh + NIX_SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; } From 2448d282ccdbbe739d097463c070a1a89dc21cd2 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 29 Jul 2026 20:20:52 -0400 Subject: [PATCH 55/81] cmake: add type-unordered-set.h and version.h to public headers This ensures all headers are properly installed, and is useful for building bitcoin core with WITH_EXTERNAL_LIBMULTIPROCESS. --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bf50018a..ff8b1282 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -174,9 +174,11 @@ set(MP_PUBLIC_HEADERS include/mp/type-struct.h include/mp/type-threadmap.h include/mp/type-tuple.h + include/mp/type-unordered-set.h include/mp/type-vector.h include/mp/type-void.h - include/mp/util.h) + include/mp/util.h + include/mp/version.h) add_library(multiprocess STATIC ${MP_PROXY_SRCS} ${MP_PUBLIC_HEADERS} From 2f4be9ec6ea9226f1eeff7a22e635fc80c742244 Mon Sep 17 00:00:00 2001 From: ViniciusCestarii Date: Fri, 14 Aug 2026 17:08:09 -0300 Subject: [PATCH 56/81] refactor: Remove trailing whitespace --- src/mp/util.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mp/util.cpp b/src/mp/util.cpp index 8de287db..e1eae4dc 100644 --- a/src/mp/util.cpp +++ b/src/mp/util.cpp @@ -166,7 +166,7 @@ void WriteSpawnError(int fd, const SpawnError& error) // Get rid of a child process the parent is abandoning because SpawnProcess is // about to throw, so it is not left behind as a zombie. The child may still be -// alive, so kill it first: waiting without that could block for as long as the +// alive, so kill it first: waiting without that could block for as long as the // spawned program runs. void KillAndReapChild(ProcessId pid) { From 9b136782af109206a7beeb73e23d99486f25400e Mon Sep 17 00:00:00 2001 From: ViniciusCestarii Date: Fri, 14 Aug 2026 17:08:33 -0300 Subject: [PATCH 57/81] ci: Add -Wtrailing-whitespace to default config --- ci/configs/default.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/configs/default.bash b/ci/configs/default.bash index 94dde74b..f36f4e49 100644 --- a/ci/configs/default.bash +++ b/ci/configs/default.bash @@ -1,5 +1,5 @@ CI_DESC="CI job using default libraries and tools, and running IWYU" CI_DIR=build-default -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wextra-semi -Wmissing-noreturn" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wextra-semi -Wmissing-noreturn -Wtrailing-whitespace" CMAKE_ARGS=(-DMP_ENABLE_IWYU=ON) BUILD_ARGS=(-k) From 914dc839f279269d11bd575f61dc1c09c5c2008e Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 19 Aug 2026 13:04:35 -0400 Subject: [PATCH 58/81] proxy: fix data race between server request threads and disconnect handling ThreadSanitizer reported a data race between a server thread executing an async request and the event loop thread handling an abrupt remote disconnect (https://github.com/bitcoin-core/libmultiprocess/issues/348): the server thread called call_context.getResults(), which reads Cap'n Proto connection state, while the event loop thread overwrote that state. In addition to the general undefined behavior, the race has one interleaving with a concrete failure: a server thread can dereference a null pointer and crash the process, meaning a client that disconnects mid-call can take down the server. RpcConnectionState::disconnect() (capnp/rpc.c++) runs on the event loop thread and tears down the connection in two steps, moving the live connection out of the RpcConnectionState::connection field (nulling the stored pointer) and then flipping the field to its disconnected state. A server thread calling getResults() between the two steps passes the is() check but then dereferences the nulled pointer. The other interleavings are harmless: reading the field before both writes builds results into an outgoing message that is simply never sent, and reading it after both writes takes the normal disconnected code path, which builds results into a message detached from the connection. There is no use-after-free, since the objects involved stay alive through reference counts and the existing cancellation handshake. The underlying problem is that connection state may only be accessed on the event loop thread, and nothing lets libmultiprocess order server thread accesses against the disconnect teardown: - The teardown happens with no warning. The Connection::onDisconnect promise used to clean up after disconnects only fires after capnp has finished tearing down the connection and shutting down the stream. - The in-flight request is not canceled first. With capnp's allowCancellation feature off (the default), LocalClient::callInternal (capnp/capability.c++) detaches a fork of the call promise, so capnp's teardown does not destroy the promise chain that would trigger the CancelMonitor cancellation handshake in PassField. Enabling allowCancellation would not help either: disconnect() would then cancel in-flight requests as part of its teardown, but only after the connection field has already been overwritten, so the cancellation handshake still could not order server thread reads against those writes, only narrow the window. So no mutex or flag in libmultiprocess can help; the only options are making server threads stop reading connection state, or patching capnp. Fortunately, only the first getResults() call on a request reads connection state, to decide whether to allocate the results struct inside a real outgoing message or in a message detached from the connection (RpcCallContext::getResults in capnp/rpc.c++). The response it allocates is cached, and later getResults() calls return it without reading connection state. So fix the race by initializing the results struct on the event loop thread, in the existing loop.sync() call that runs before a request executes. The getResults() calls that later run on the server thread just return the cached response and never touch connection state. The cost is that if the method throws, the preallocated results message is wasted (error returns are built separately), the same tradeoff capnp itself makes with its internal "force initialization of response" getResults calls. Co-Authored-By: Claude Fable 5 --- include/mp/proxy-io.h | 7 +++ include/mp/type-context.h | 12 +++++ test/mp/test/foo-types.h | 3 ++ test/mp/test/foo.capnp | 1 + test/mp/test/foo.h | 1 + test/mp/test/test.cpp | 104 +++++++++++++++++++++++++++++++++++++- 6 files changed, 126 insertions(+), 2 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index cda9064d..49b0611a 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -12,6 +12,7 @@ #include +#include #include #include #include @@ -374,6 +375,12 @@ class EventLoop //! Hook called on the event loop thread when a client has disconnected. std::function testing_hook_disconnected; + + //! Miscellaneous testing hook. Called from various places with an + //! argument identifying the call site (typically a string literal), so + //! tests can control timing or inject behavior at specific points without + //! requiring a dedicated hook for each one. + std::function testing_hook_misc; }; //! Single element task queue used to handle recursive capnp calls. (If the diff --git a/include/mp/type-context.h b/include/mp/type-context.h index 54007207..7cfc7e79 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -131,6 +131,18 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& std::tie(request_thread, inserted) = SetThread( GuardedRef{thread_context.waiter->m_mutex, request_threads}, server.m_context.connection, [&] { return Accessor::get(call_context.getParams()).getCallbackThread(); }); + // Initialize the request's results struct here on the event loop + // thread, so later getResults() calls on the execution thread + // return the response capnp caches on first use instead of + // reading Cap'n Proto connection state, which is unsafe off the + // event loop thread because RpcConnectionState::disconnect() + // overwrites it there without synchronization on an abrupt remote + // disconnect (bitcoin-core/libmultiprocess#348). After this call, + // the only call_context state accessed by the execution thread is + // the params reader and the results struct allocated here; any + // future change accessing other call_context state needs to move + // that access to the event loop thread the same way. + call_context.getResults(); }); // If an entry was inserted into the request_threads map, diff --git a/test/mp/test/foo-types.h b/test/mp/test/foo-types.h index ee12f490..85c36349 100644 --- a/test/mp/test/foo-types.h +++ b/test/mp/test/foo-types.h @@ -9,6 +9,7 @@ #include // IWYU pragma: begin_exports +#include #include #include #include @@ -75,6 +76,8 @@ inline void CustomBuildMessage(InvokeContext& invoke_context, const test::FooMessage& src, test::messages::FooMessage::Builder&& builder) { + const auto& hook{invoke_context.connection.m_loop->testing_hook_misc}; + if (hook) hook("build FooMessage"); builder.setMessage(src.message + " build"); } diff --git a/test/mp/test/foo.capnp b/test/mp/test/foo.capnp index 11bfd9c5..392719ec 100644 --- a/test/mp/test/foo.capnp +++ b/test/mp/test/foo.capnp @@ -38,6 +38,7 @@ interface FooInterface $Proxy.wrap("mp::test::FooImplementation") { callIntFnAsync @21 (context :Proxy.Context, arg :Int32) -> (result :Int32); passDataPointers @22 (arg :List(Data)) -> (result :List(Data)); listBars @25 (context :Proxy.Context, n :Int32) -> (result :List(BarInterface)); + callMessageAsync @26 (context :Proxy.Context) -> (result :FooMessage); } interface FooInit $Proxy.wrap("mp::test::FooInit") { diff --git a/test/mp/test/foo.h b/test/mp/test/foo.h index 8fc34eee..e8782708 100644 --- a/test/mp/test/foo.h +++ b/test/mp/test/foo.h @@ -124,6 +124,7 @@ class FooImplementation void callFn() { assert(m_fn); m_fn(); } void callFnAsync() { assert(m_fn); m_fn(); } int callIntFnAsync(int arg) { assert(m_int_fn); return m_int_fn(arg); } + FooMessage callMessageAsync() { assert(m_fn); m_fn(); return {}; } std::function m_fn; std::function m_int_fn; }; diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 400aff73..7f465a33 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -79,6 +80,7 @@ class TestSetup public: std::function server_disconnect; std::function server_disconnect_later; + std::function server_on_disconnect; std::function client_disconnect; std::promise>> client_promise; std::unique_ptr> client; @@ -109,8 +111,14 @@ class TestSetup loop.m_task_set->add(kj::evalLater([&] { server_connection.reset(); })); }; // Set handler to destroy the server when the client disconnects. This - // is ignored if server_disconnect() is called instead. - server_connection->onDisconnect([&] { server_connection.reset(); }); + // is ignored if server_disconnect() is called instead. Tests can + // assign server_on_disconnect to override the default behavior of + // destroying the server connection as soon as the disconnect is + // detected (in which case they need to destroy it themselves, + // e.g. by calling server_disconnect(), so the event loop can + // exit). + server_on_disconnect = [&] { server_connection.reset(); }; + server_connection->onDisconnect([&] { server_on_disconnect(); }); auto client_connection = std::make_unique(loop, kj::mv(pipe.ends[1])); auto client_proxy = std::make_unique>( @@ -368,6 +376,98 @@ KJ_TEST("Calling IPC method, disconnecting and blocking during the call") signal.set_value(); } +KJ_TEST("Calling async IPC method with a remote disconnect while results are built") +{ + // Regression test for bitcoin-core/libmultiprocess#348, a data race + // reported by ThreadSanitizer where a server worker thread called + // call_context.getResults() while the event loop thread was tearing down + // Cap'n Proto connection state (capnp::_::RpcConnectionState::disconnect() + // overwriting the RpcConnectionState::connection field) after an abrupt + // remote disconnect. + // + // The test makes an async IPC call (callMessageAsync) whose method body + // just signals the main thread. When the worker thread returns from the + // method body, it calls call_context.getResults(), reading the connection + // state, and starts serializing the FooMessage result, where the + // testing_hook_misc hook set below makes it sleep. Meanwhile the main + // thread destroys the client connection, and the event loop thread + // processes the resulting EOF, running RpcConnectionState::disconnect() + // and overwriting the connection state the worker thread just read, with + // no synchronization between the two accesses. + // + // Two details are essential for ThreadSanitizer to detect the race: + // + // - There must be no synchronization between the worker thread's + // getResults() call and the event loop thread's disconnect processing. + // The worker signals the main thread *before* getResults() and sleeps + // (sleeping creates no happens-before edge) across the disconnect, so + // the racing accesses are not ordered by any of the test's own + // synchronization. + // + // - The worker thread's read should come *before* the event loop thread's + // write, close in time. In the write-then-read order (e.g. method + // returning long after the disconnect), the race exists too, but + // ThreadSanitizer usually misses it: right after disconnect() writes the + // connection field, the loop thread's teardown destructors re-read it + // many times (~ImportClient etc. check connection.is() to + // decide whether to send messages), evicting the write from the + // per-granule shadow history before a late reader comes along. + // + // The server Connection object is deliberately kept alive during all this + // by overriding server_on_disconnect: destroying it would cancel the + // in-flight request (Connection::~Connection calls m_canceler.cancel(), + // setting request_canceled) and the worker would throw InterruptException + // instead of proceeding into getResults(). Keeping it alive matches the + // window in the original report, where the worker races with capnp's own + // internal teardown, which runs before any onDisconnect notification. + + TestSetup setup{/*client_owns_connection=*/false}; + ProxyClient* foo = setup.client.get(); + KJ_EXPECT(foo->add(1, 2) == 3); + foo->initThreadMap(); + + // Keep the server Connection object alive when the disconnect is detected + // so the in-flight request is not canceled (see comment above). The + // connection is destroyed at the end of the test instead. + setup.server_on_disconnect = [] {}; + + // Signaled by the worker thread when the method body runs, just before it + // returns and the worker calls getResults() and serializes the results. + std::promise fn_called; + setup.server->m_impl->m_fn = [&] { fn_called.set_value(); }; + + // Keep the worker thread inside the results-building step, without + // synchronizing, while the event loop thread processes the disconnect. + EventLoop& loop = *setup.server->m_context.connection->m_loop; + loop.testing_hook_misc = [](std::any arg) { + if (const char* const* tag{std::any_cast(&arg)}; + tag && std::string_view{*tag} == "build FooMessage") { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + }; + + // Signaled by the worker thread when it is completely done with the + // request, including serializing the results. + std::promise request_done; + loop.testing_hook_async_request_done = [&] { request_done.set_value(); }; + + // Make the IPC call from a separate thread so this thread can trigger the + // client disconnect while the call is executing. + std::thread caller{[&] { + EXPECT_EXCEPTION(foo->callMessageAsync(), "IPC client method call interrupted by disconnect."); + }}; + + fn_called.get_future().get(); + setup.client_disconnect(); + caller.join(); + + // Wait for the worker thread to finish the request, then tear down the + // server connection that was deliberately kept alive above, so the event + // loop is able to exit. + request_done.get_future().get(); + setup.server_disconnect(); +} + KJ_TEST("Worker thread destroyed before it is initialized") { // Regression test for bitcoin/bitcoin#34711, bitcoin/bitcoin#34756 where a From fe1b8339f00dc9a5985b0638407ee1d82b451e1c Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Thu, 20 Aug 2026 12:03:48 -0400 Subject: [PATCH 59/81] ci: do not ignore NIXPKGS_CHANNEL in local ci runs Currently, olddeps ci job specifies nixos-25.05 channel, but this is ignored in local ci runs, because the NIXPKGS_CHANNEL is passed as the cachiex nix path instead of being passed directly to nix develop. Fix this and also rename the NIXPKGS_CHANNEL environment variable to CI_NIXPKGS_CHANNEL because CI configuration variables are supposed to have the CI_ prefix. The NIXPKGS_CHANNEL looks misleadingly like an upstream configuration variable when it's just a local setting. --- .github/workflows/ci.yml | 3 +-- ci/configs/olddeps.bash | 2 +- ci/scripts/config.sh | 4 ++-- ci/scripts/run.sh | 10 ++++++---- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8e5b752..1155a420 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,7 +137,7 @@ jobs: runs-on: ubuntu-latest env: - NIXPKGS_CHANNEL: nixos-26.05 + CI_NIXPKGS_CHANNEL: nixos-26.05 NIX_EXTRA_CONFIG: | keep-env-derivations = true keep-outputs = true @@ -164,7 +164,6 @@ jobs: - name: Install Nix uses: cachix/install-nix-action@v31 # 2025-05-27, from https://github.com/cachix/install-nix-action/tags with: - nix_path: nixpkgs=https://github.com/NixOS/nixpkgs/archive/${{ steps.config.outputs.nixpkgs_rev }}.tar.gz # Act executes inside an unprivileged container (Docker or Podman), # so KVM support isn't available. enable_kvm: "${{ github.actor != 'nektos/act' }}" diff --git a/ci/configs/olddeps.bash b/ci/configs/olddeps.bash index 151a300c..76082c7a 100644 --- a/ci/configs/olddeps.bash +++ b/ci/configs/olddeps.bash @@ -2,7 +2,7 @@ CI_DESC="CI job using old Cap'n Proto and cmake versions" CI_DIR=build-olddeps # Pin olddeps to an older Nixpkgs channel, since compiling the old CMake # requires an older GCC. -NIXPKGS_CHANNEL=nixos-25.05 +CI_NIXPKGS_CHANNEL=nixos-25.05 export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds" NIX_ARGS=(--argstr capnprotoVersion "0.9.2" --argstr cmakeVersion "3.12.4" --argstr gccVersion "11") BUILD_ARGS=(-k) diff --git a/ci/scripts/config.sh b/ci/scripts/config.sh index 7fe8118d..887e245b 100755 --- a/ci/scripts/config.sh +++ b/ci/scripts/config.sh @@ -17,8 +17,8 @@ source "${SCRIPT_DIR}/ci_helpers.sh" [ "${CI_CONFIG+x}" ] && source "$CI_CONFIG" # Resolve the nixpkgs channel to a specific revision for use in cache keys. -if [[ -n "${NIXPKGS_CHANNEL:-}" ]]; then - rev="$(curl --fail --location --silent --show-error "https://channels.nixos.org/${NIXPKGS_CHANNEL}/git-revision")" +if [[ -n "${CI_NIXPKGS_CHANNEL:-}" ]]; then + rev="$(curl --fail --location --silent --show-error "https://channels.nixos.org/${CI_NIXPKGS_CHANNEL}/git-revision")" test -n "${rev}" write_output_var nixpkgs_rev "${rev}" fi diff --git a/ci/scripts/run.sh b/ci/scripts/run.sh index 04ac8596..aadeed11 100755 --- a/ci/scripts/run.sh +++ b/ci/scripts/run.sh @@ -10,14 +10,16 @@ set -o errexit -o nounset -o pipefail -o xtrace [ "${CI_CONFIG+x}" ] && source "$CI_CONFIG" -nix develop --ignore-environment --keep CI_CONFIG --keep CI_CLEAN "${NIX_ARGS[@]+"${NIX_ARGS[@]}"}" -f shell.nix --command ci/scripts/ci.sh +nix_args=("${NIX_ARGS[@]+${NIX_ARGS[@]}}") +if [[ -n "${CI_NIXPKGS_CHANNEL:-}" ]]; then + nix_args+=(-I "nixpkgs=channel:${CI_NIXPKGS_CHANNEL}") +fi +nix develop "${nix_args[@]}" --ignore-environment --keep CI_CONFIG --keep CI_CLEAN -f shell.nix --command ci/scripts/ci.sh # Create a GC root for the shell closure so the cache-nix-action save step # does not garbage-collect it. if [ -n "${CI_CACHE_NIX_STORE-}" ]; then - nix-build shell.nix \ - -o "$CI_DIR/gcroot" \ - "${NIX_ARGS[@]+"${NIX_ARGS[@]}"}" + nix-build shell.nix "${nix_args[@]}" -o "$CI_DIR/gcroot" # Verify the closure is complete so the cache-nix-action save step has # everything it needs. nix-store --query --requisites "$CI_DIR/gcroot" >/dev/null From 7c73cceda9b4149b706dd667f70cbea381d6364a Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Thu, 20 Aug 2026 11:42:22 -0400 Subject: [PATCH 60/81] ci: rename CI-internal variables to use CI_ prefix --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1155a420..9c6db241 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,10 +138,10 @@ jobs: env: CI_NIXPKGS_CHANNEL: nixos-26.05 - NIX_EXTRA_CONFIG: | + CI_NIX_CONFIG: | keep-env-derivations = true keep-outputs = true - NIX_EXTRA_CONFIG_ACT: | + CI_NIX_CONFIG_ACT: | sandbox = false filter-syscalls = false @@ -168,8 +168,8 @@ jobs: # so KVM support isn't available. enable_kvm: "${{ github.actor != 'nektos/act' }}" extra_nix_config: | - ${{ env.NIX_EXTRA_CONFIG }} - ${{ github.actor == 'nektos/act' && env.NIX_EXTRA_CONFIG_ACT || '' }} + ${{ env.CI_NIX_CONFIG }} + ${{ github.actor == 'nektos/act' && env.CI_NIX_CONFIG_ACT || '' }} - name: Cache Nix store if: steps.config.outputs.cache_nix_store == 'true' From a7ff9d5da2290c1edc08460262c909f31cff938a Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 10 Sep 2025 07:06:21 -0400 Subject: [PATCH 61/81] ci: add cmake debug output Co-Authored-By: ViniciusCestarii <124843824+ViniciusCestarii@users.noreply.github.com> Co-Authored-By: hebasto <32963518+hebasto@users.noreply.github.com> Co-Authored-By: Claude Sonnet 4.6 --- ci/scripts/ci.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ci/scripts/ci.sh b/ci/scripts/ci.sh index b4bd8fb4..a91a5e71 100755 --- a/ci/scripts/ci.sh +++ b/ci/scripts/ci.sh @@ -42,7 +42,22 @@ mkdir -p "$CI_DIR" cd "$CI_DIR" export CMAKE_BUILD_PARALLEL_LEVEL="$(nproc)" git --no-pager log -1 || true -cmake "$src_dir" "${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}" +cmake_args=("${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}") +if ! cmake "$src_dir" "${cmake_args[@]}"; then + # If cmake failed, try it again with debug options. + # Could add --trace / --trace-expand here too but they are very verbose. + cmake_args+=(--debug-output --debug-trycompile) + if ver_ge "$cmake_ver" "3.16"; then cmake_args+=(--log-level=DEBUG); fi + if ver_ge "$cmake_ver" "3.17"; then cmake_args+=(--debug-find); fi + cmake "$src_dir" "${cmake_args[@]}" || : "cmake exited with $?" + if ver_ge "$cmake_ver" "3.26"; then + cat CMakeFiles/CMakeConfigureLog.yaml || true + else + cat CMakeFiles/CMakeError.log CMakeFiles/CMakeOutput.log || true + fi + find . -ls || true + false +fi if ver_ge "$cmake_ver" "3.15"; then cmake --build . -t "${BUILD_TARGETS[@]}" -- "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" else From fa30e2093c7574f2fb8fe8f2eecd550927a5ae47 Mon Sep 17 00:00:00 2001 From: marco <*~=`'#}+{/-|&$^_@721217.xyz> Date: Wed, 26 Aug 2026 11:12:21 +0200 Subject: [PATCH 62/81] ci: Remove hard-coded -j4 from sanitize config Not needed after CMAKE_BUILD_PARALLEL_LEVEL is set to nproc. --- ci/configs/sanitize.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/configs/sanitize.bash b/ci/configs/sanitize.bash index e582dc5d..907f3295 100644 --- a/ci/configs/sanitize.bash +++ b/ci/configs/sanitize.bash @@ -4,5 +4,5 @@ NIX_ARGS=(--arg enableLibcxx true --argstr libcxxSanitizers "Thread" --argstr ca export CXX=clang++ export CXXFLAGS="-ggdb -Werror -Wall -Wextra -Wpedantic -Wthread-safety -Wno-unused-parameter -fsanitize=thread -Wno-c++23-lambda-attributes" CMAKE_ARGS=() -BUILD_ARGS=(-k -j4) +BUILD_ARGS=(-k) BUILD_TARGETS=(mptest) From 81f824b025fd3fe3e151d8c7eb80fcf2d88d4290 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:26:44 +0100 Subject: [PATCH 63/81] doc: Update Cap'n Proto version to match minimum --- CMakeLists.txt | 2 +- doc/install.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ff8b1282..c2bfcb41 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,7 +64,7 @@ if(CAPNPROTO_CVE_AFFECTED OR CAPNPROTO_CLANG_INCOMPATIBLE) string(APPEND RESOLUTION_OPTIONS " - Upgrade to Cap'n Proto version 1.0 or newer (recommended)\n") if(CAPNPROTO_CVE_AFFECTED AND NOT CAPNPROTO_CLANG_INCOMPATIBLE) - string(APPEND RESOLUTION_OPTIONS " - Upgrade to a patched minor version (0.7.1, 0.8.1, 0.9.2, 0.10.3, or later)\n") + string(APPEND RESOLUTION_OPTIONS " - Upgrade to a patched minor version (0.9.2, 0.10.3, or later)\n") elseif(CAPNPROTO_CLANG_INCOMPATIBLE AND NOT CAPNPROTO_CVE_AFFECTED) string(APPEND RESOLUTION_OPTIONS " - Use GCC instead of Clang\n") endif() diff --git a/doc/install.md b/doc/install.md index ac2f7f6a..015cfd19 100644 --- a/doc/install.md +++ b/doc/install.md @@ -1,6 +1,6 @@ # libmultiprocess Installation -Installation currently requires Cap'n Proto 0.7 or higher: +Installation currently requires Cap'n Proto 0.9 or higher: ```sh apt install libcapnp-dev capnproto From fa101113b5097f6fabc7eedf2799f76e4503fb5d Mon Sep 17 00:00:00 2001 From: marco <*~=`'#}+{/-|&$^_@721217.xyz> Date: Wed, 26 Aug 2026 14:09:32 +0200 Subject: [PATCH 64/81] refactor: Enable readability-container-contains --- .clang-tidy | 1 + test/mp/test/test.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index 9a2afcc5..b1b6561e 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -23,6 +23,7 @@ performance-*, -performance-noexcept-move-constructor, -performance-unnecessary-value-param, readability-const-return-type, +readability-container-contains, readability-redundant-declaration, readability-redundant-string-init, clang-analyzer-core.*, diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 7f465a33..5ecb7cc4 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -181,7 +181,7 @@ KJ_TEST("Call FooInterface methods") } KJ_EXPECT(in.unordered_set_int.size() == out.unordered_set_int.size()); for (const auto& elem : in.unordered_set_int) { - KJ_EXPECT(out.unordered_set_int.count(elem) == 1); + KJ_EXPECT(out.unordered_set_int.contains(elem)); } KJ_EXPECT(in.vector_bool.size() == out.vector_bool.size()); for (size_t i = 0; i < in.vector_bool.size(); ++i) { From cc3675280bfa73e145fd05acc4b1743763aa6ecc Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 26 Aug 2026 09:35:19 -0400 Subject: [PATCH 65/81] ci: bump cmake version to 4.3.4 in newdeps job Update cmake from 4.1.1 to 4.3.4, including the corresponding sha256 hash in shell.nix. Suggested-by: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Reference: https://github.com/bitcoin-core/libmultiprocess/pull/212#pullrequestreview-5002523847 Co-Authored-By: Claude Sonnet 4.6 --- ci/configs/newdeps.bash | 2 +- shell.nix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/configs/newdeps.bash b/ci/configs/newdeps.bash index e05913a6..e6e5fe1d 100644 --- a/ci/configs/newdeps.bash +++ b/ci/configs/newdeps.bash @@ -2,5 +2,5 @@ CI_DESC="CI job using newest Cap'n Proto and cmake versions" CI_DIR=build-newdeps export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds" CAPNP_CHECKOUT=master -NIX_ARGS=(--argstr capnprotoVersion "none" --argstr cmakeVersion "4.1.1") +NIX_ARGS=(--argstr capnprotoVersion "none" --argstr cmakeVersion "4.3.4") BUILD_ARGS=(-k) diff --git a/shell.nix b/shell.nix index 17789f31..1fbfda51 100644 --- a/shell.nix +++ b/shell.nix @@ -56,6 +56,7 @@ let cmakeHashes = { "3.12.4" = "sha256-UlVYS/0EPrcXViz/iULUcvHA5GecSUHYS6raqbKOMZQ="; "4.1.1" = "sha256-sp9vGXM6oiS3djUHoQikJ+1Ixojh+vIrKcROHDBUkoI="; + "4.3.4" = "sha256-/e/4l7nrSddkU58rHtxut+FEDfMlZ4qXwZeEmekxrdo="; }; gcc = if gccVersion == null then null else builtins.getAttr ("gcc" + gccVersion) pkgs; cmakeBuild = if cmakeVersion == null then pkgs.cmake else (pkgs.cmake.overrideAttrs (old: { From 766867fb326a8cc11aa4c0d6142288609b9bce72 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Tue, 25 Aug 2026 17:05:11 -0400 Subject: [PATCH 66/81] ci: clarify CAPNP_CHECKOUT=master is the v1.x release branch capnproto's default branch switched to v2, so "master" is non-obvious. Suggested-by: MarcoFalke Reference: https://github.com/bitcoin-core/libmultiprocess/pull/212#discussion_r3827402335 Co-Authored-By: Claude Sonnet 4.6 --- ci/configs/newdeps.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/configs/newdeps.bash b/ci/configs/newdeps.bash index e6e5fe1d..a86f9dee 100644 --- a/ci/configs/newdeps.bash +++ b/ci/configs/newdeps.bash @@ -1,6 +1,6 @@ CI_DESC="CI job using newest Cap'n Proto and cmake versions" CI_DIR=build-newdeps export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds" -CAPNP_CHECKOUT=master +CAPNP_CHECKOUT=master # This is the v1.x release branch NIX_ARGS=(--argstr capnprotoVersion "none" --argstr cmakeVersion "4.3.4") BUILD_ARGS=(-k) From 2330fbe81e255996710e3e258cb6a5ea94cfb250 Mon Sep 17 00:00:00 2001 From: ViniciusCestarii Date: Sun, 16 Aug 2026 11:13:46 -0300 Subject: [PATCH 67/81] refactor: replace EventLoop::post() with sync() taking kj::FunctionParam kj::FunctionParam only stores a pointer to the caller's callable instead of taking ownership of it. This avoids a heap allocation per call, and is safe because EventLoop:sync() is synchronous, so the callable is guaranteed not to go out of scope before the event loop thread is done running it. It also means the callable does not need to be copyable or movable. --- include/mp/proxy-io.h | 19 +++++-------------- src/mp/proxy.cpp | 6 +++--- test/mp/test/listen_tests.cpp | 1 + test/mp/test/test.cpp | 1 + 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index cda9064d..b56f7d08 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -233,8 +233,8 @@ Stream MakeStream(EventLoop&loop, SocketId socket); //! assumes it will only be accessed from one thread. So all this code needs to //! actually run on one thread, and the EventLoop::loop() method is the entry point for //! this thread. ProxyClient and ProxyServer objects that use other threads and -//! need to perform I/O operations post to this thread using EventLoop::post() -//! and EventLoop::sync() methods. +//! need to perform I/O operations post to this thread using the +//! EventLoop::sync() method. //! //! Specifically, because ProxyClient methods can be called from arbitrary //! threads, and ProxyServer methods can run on arbitrary threads, ProxyClient @@ -267,16 +267,7 @@ class EventLoop //! Run function on event loop thread. Does not return until function completes. //! Must be called while the loop() function is active. - void post(kj::Function fn); - - //! Wrapper around EventLoop::post that takes advantage of the - //! fact that callable will not go out of scope to avoid requirement that it - //! be copyable. - template - void sync(Callable&& callable) - { - post(std::forward(callable)); - } + void sync(kj::FunctionParam fn); //! Register cleanup function to run on asynchronous worker thread without //! blocking the event loop thread. @@ -309,8 +300,8 @@ class EventLoop //! method has not been called. std::thread m_async_thread; - //! Callback function to run on event loop thread during post() or sync() call. - kj::Function* m_post_fn MP_GUARDED_BY(m_mutex) = nullptr; + //! Callback function to run on event loop thread during sync() call. + kj::FunctionParam* m_post_fn MP_GUARDED_BY(m_mutex) = nullptr; //! Callback functions to run on async thread. std::optional m_async_fns MP_GUARDED_BY(m_mutex); diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index eb2aee0c..ae05ca6d 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -306,7 +306,7 @@ void EventLoop::loop() Lock lock(m_mutex); if (m_post_fn) { // m_post_fn throwing is never expected. If it does happen, the caller - // of EventLoop::post() will return without any indication of failure, + // of EventLoop::sync() will return without any indication of failure, // which will likely cause other bugs. Log the error and continue. KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() MP_REQUIRES(m_mutex) { Unlock(lock, *m_post_fn); })) { MP_LOG(*this, Log::Error) << "EventLoop: m_post_fn threw: " << kj::str(*exception).cStr(); @@ -315,7 +315,7 @@ void EventLoop::loop() m_cv.notify_all(); } else if (done()) { // Intentionally do not break if m_post_fn was set, even if done() - // would return true, to ensure that the post() m_post_writer->write() + // would return true, to ensure that the sync() m_post_writer->write() // call always succeeds and the loop does not exit between the time // that the done condition is set and the write call is made. break; @@ -333,7 +333,7 @@ void EventLoop::loop() m_cv.notify_all(); } -void EventLoop::post(kj::Function fn) +void EventLoop::sync(kj::FunctionParam fn) { if (std::this_thread::get_id() == m_thread_id) { fn(); diff --git a/test/mp/test/listen_tests.cpp b/test/mp/test/listen_tests.cpp index 8d06cb35..41fb05aa 100644 --- a/test/mp/test/listen_tests.cpp +++ b/test/mp/test/listen_tests.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 400aff73..5e1b010e 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include From 5c49666a1d8860028b6ed04b3aead64d06979d6d Mon Sep 17 00:00:00 2001 From: ViniciusCestarii Date: Sun, 16 Aug 2026 11:54:05 -0300 Subject: [PATCH 68/81] refactor: rename EventLoop::m_post_fn to m_sync_fn --- include/mp/proxy-io.h | 2 +- src/mp/proxy.cpp | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index b56f7d08..d716b8ec 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -301,7 +301,7 @@ class EventLoop std::thread m_async_thread; //! Callback function to run on event loop thread during sync() call. - kj::FunctionParam* m_post_fn MP_GUARDED_BY(m_mutex) = nullptr; + kj::FunctionParam* m_sync_fn MP_GUARDED_BY(m_mutex) = nullptr; //! Callback functions to run on async thread. std::optional m_async_fns MP_GUARDED_BY(m_mutex); diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index ae05ca6d..552e3268 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -275,7 +275,7 @@ EventLoop::~EventLoop() { if (m_async_thread.joinable()) m_async_thread.join(); const Lock lock(m_mutex); - KJ_ASSERT(m_post_fn == nullptr); + KJ_ASSERT(m_sync_fn == nullptr); KJ_ASSERT(!m_async_fns); KJ_ASSERT(!m_wait_stream); KJ_ASSERT(!m_post_stream); @@ -304,17 +304,17 @@ void EventLoop::loop() const size_t read_bytes = wait_stream->read(&buffer, 0, 1).wait(m_io_context.waitScope); if (read_bytes != 1) throw std::logic_error("EventLoop wait_stream closed unexpectedly"); Lock lock(m_mutex); - if (m_post_fn) { - // m_post_fn throwing is never expected. If it does happen, the caller + if (m_sync_fn) { + // m_sync_fn throwing is never expected. If it does happen, the caller // of EventLoop::sync() will return without any indication of failure, // which will likely cause other bugs. Log the error and continue. - KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() MP_REQUIRES(m_mutex) { Unlock(lock, *m_post_fn); })) { - MP_LOG(*this, Log::Error) << "EventLoop: m_post_fn threw: " << kj::str(*exception).cStr(); + KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() MP_REQUIRES(m_mutex) { Unlock(lock, *m_sync_fn); })) { + MP_LOG(*this, Log::Error) << "EventLoop: m_sync_fn threw: " << kj::str(*exception).cStr(); } - m_post_fn = nullptr; + m_sync_fn = nullptr; m_cv.notify_all(); } else if (done()) { - // Intentionally do not break if m_post_fn was set, even if done() + // Intentionally do not break if m_sync_fn was set, even if done() // would return true, to ensure that the sync() m_post_writer->write() // call always succeeds and the loop does not exit between the time // that the done condition is set and the write call is made. @@ -341,13 +341,13 @@ void EventLoop::sync(kj::FunctionParam fn) } Lock lock(m_mutex); EventLoopRef ref(*this, &lock); - m_cv.wait(lock.m_lock, [this]() MP_REQUIRES(m_mutex) { return m_post_fn == nullptr; }); - m_post_fn = &fn; + m_cv.wait(lock.m_lock, [this]() MP_REQUIRES(m_mutex) { return m_sync_fn == nullptr; }); + m_sync_fn = &fn; Unlock(lock, [&] { char buffer = 0; m_post_writer->write(&buffer, 1); }); - m_cv.wait(lock.m_lock, [this, &fn]() MP_REQUIRES(m_mutex) { return m_post_fn != &fn; }); + m_cv.wait(lock.m_lock, [this, &fn]() MP_REQUIRES(m_mutex) { return m_sync_fn != &fn; }); } void EventLoop::startAsyncThread() From fa1db7a9e3ed4b42c84c38cbc795fd1818829778 Mon Sep 17 00:00:00 2001 From: marco <*~=`'#}+{/-|&$^_@721217.xyz> Date: Tue, 1 Sep 2026 15:33:43 +0200 Subject: [PATCH 69/81] pull latest .clang-tidy from downstream --- .clang-tidy | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index b1b6561e..16dafd03 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -6,8 +6,12 @@ bugprone-string-constructor, bugprone-use-after-move, bugprone-lambda-function-name, bugprone-unhandled-self-assignment, -misc-unused-using-decls, +bugprone-unused-return-value, +fuchsia-header-anon-namespaces, +misc-definitions-in-headers, misc-no-recursion, +misc-unused-using-decls, +modernize-avoid-bind, modernize-deprecated-headers, modernize-use-default-member-init, modernize-use-emplace, @@ -22,6 +26,7 @@ performance-*, -performance-no-int-to-ptr, -performance-noexcept-move-constructor, -performance-unnecessary-value-param, +readability-avoid-const-params-in-decls, readability-const-return-type, readability-container-contains, readability-redundant-declaration, @@ -36,6 +41,10 @@ CheckOptions: - key: modernize-deprecated-headers.CheckHeaderFile value: false - key: performance-move-const-arg.CheckTriviallyCopyableMove - value: false + value: false # Disabled, to allow the bugprone-use-after-move rule on trivially copyable types, to catch logic bugs - key: bugprone-unhandled-self-assignment.WarnOnlyIfThisHasSuspiciousField value: false + - key: bugprone-unused-return-value.CheckedReturnTypes + value: '^::std::error_code$;^::std::error_condition$;^::std::errc$;^::std::expected$;^::util::Result$;^::util::Expected$' + - key: bugprone-unused-return-value.AllowCastToVoid + value: true # Can be removed with C++26 once the _ placeholder exists. From fb4ac7eb8c833c47fb5dee4b63c38394efa9a660 Mon Sep 17 00:00:00 2001 From: fanquake Date: Wed, 9 Sep 2026 11:57:39 +0100 Subject: [PATCH 70/81] ci: use LLVM 23 in Bitcoin Core CI --- .github/workflows/bitcoin-core-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bitcoin-core-ci.yml b/.github/workflows/bitcoin-core-ci.yml index f14d5682..6aa1aa58 100644 --- a/.github/workflows/bitcoin-core-ci.yml +++ b/.github/workflows/bitcoin-core-ci.yml @@ -19,7 +19,7 @@ concurrency: env: BITCOIN_REPO: bitcoin/bitcoin BITCOIN_CORE_REF: refs/heads/master - LLVM_VERSION: 22 + LLVM_VERSION: 23 LIBCXX_DIR: /tmp/libcxx-build/ jobs: @@ -37,13 +37,13 @@ jobs: functional_test_runs: 20 nproc_multiplier: 2 functional_timeout_factor: 40 - runner: ubuntu-24.04 + runner: ubuntu-26.04 apt-llvm: true packages: >- ccache - clang-22 - llvm-22 - libclang-rt-22-dev + clang-23 + llvm-23 + libclang-rt-23-dev libboost-dev libsqlite3-dev libcapnp-dev From 4a1f400f8b7259db748b311c1441d375e68d76ca Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Thu, 10 Sep 2026 09:37:11 -0400 Subject: [PATCH 71/81] util: Error out at compile time if KJ_NO_EXCEPTIONS is set libmultiprocess requires capnp/kj to be built with exception support: mp::serverInvoke (proxy-types.h) relies on capnp's RPC layer catching exceptions thrown by server methods and converting them into remote-exception replies for the client, which capnp only does when KJ_NO_EXCEPTIONS is unset. If it's set, that conversion code (kj::getCaughtExceptionAsKj) is compiled out of capnp/kj, and an exception thrown by a server method escapes uncaught instead, crashing the process. This was hit in practice on NetBSD (bitcoin/bitcoin#36058): capnp's kj/common.h exception-support autodetection is fooled there, because NetBSD's defines a dummy __has_feature(x)=0 stub for compilers (GCC<14) that don't natively support __has_feature. capnp reads that as "no exceptions" even though the compiler has exceptions enabled, so NetBSD's prebuilt capnproto package ends up linked without the exception-catching code libmultiprocess needs. Fixed upstream in capnproto/capnproto#2756, but not yet in any capnproto release or in NetBSD's pkgsrc package. Fail the build immediately with an explanatory #error instead of letting this surface later as a runtime crash on the affected platform. Co-Authored-By: Claude Sonnet 5 --- include/mp/proxy-types.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/mp/proxy-types.h b/include/mp/proxy-types.h index 3a5dd3d8..0c00b825 100644 --- a/include/mp/proxy-types.h +++ b/include/mp/proxy-types.h @@ -13,6 +13,12 @@ #include #include +// KJ exception support is required, or exceptions thrown in serverInvoke below go uncaught +// and crash the process instead of being returned to the client as errors. +#if KJ_NO_EXCEPTIONS +#error "KJ_NO_EXCEPTIONS=1 is set but libmultiprocess requires support for exceptions. Please check build settings. On NetBSD you may also need to set -DKJ_NO_EXCEPTIONS=0 -DKJ_NO_RTTI=0 explicitly (https://github.com/capnproto/capnproto/pull/2756)" +#endif + namespace mp { template From fa5633467afb99ddb850f39dafb96cae3642cd75 Mon Sep 17 00:00:00 2001 From: marco <*~=`'#}+{/-|&$^_@721217.xyz> Date: Wed, 16 Sep 2026 13:32:05 +0200 Subject: [PATCH 72/81] ci: Compile with minimum supported g++-10 in olddeps --- ci/configs/olddeps.bash | 6 +++--- ci/scripts/ci.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/configs/olddeps.bash b/ci/configs/olddeps.bash index 76082c7a..1d9ea6fc 100644 --- a/ci/configs/olddeps.bash +++ b/ci/configs/olddeps.bash @@ -1,8 +1,8 @@ CI_DESC="CI job using old Cap'n Proto and cmake versions" CI_DIR=build-olddeps -# Pin olddeps to an older Nixpkgs channel, since compiling the old CMake -# requires an older GCC. +# Pin olddeps to an older Nixpkgs channel, to be able to test an older GCC. CI_NIXPKGS_CHANNEL=nixos-25.05 export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds" -NIX_ARGS=(--argstr capnprotoVersion "0.9.2" --argstr cmakeVersion "3.12.4" --argstr gccVersion "11") +CAPNP_CHECKOUT=v0.9.2 # Use a checkout to compile it with the selected GCC +NIX_ARGS=(--argstr capnprotoVersion "none" --argstr cmakeVersion "3.12.4" --argstr gccVersion "10") BUILD_ARGS=(-k) diff --git a/ci/scripts/ci.sh b/ci/scripts/ci.sh index a91a5e71..8016d921 100755 --- a/ci/scripts/ci.sh +++ b/ci/scripts/ci.sh @@ -32,7 +32,7 @@ if [ -n "${CAPNP_CHECKOUT-}" ]; then git --no-pager log -1 || true CXXFLAGS="-std=c++20" cmake .. "-DCMAKE_INSTALL_PREFIX=${capnp_prefix}" -DBUILD_TESTING=OFF -DWITH_OPENSSL=OFF -DWITH_ZLIB=OFF cmake --build . - cmake --install . + cmake --build . --target install # Replacement for modern "cmake --install .". ) export CMAKE_PREFIX_PATH="${capnp_prefix}:${CMAKE_PREFIX_PATH-}" fi From 56cfdaa884e710a2ddf6d9636141c36269a034c1 Mon Sep 17 00:00:00 2001 From: mutatrum Date: Thu, 17 Sep 2026 15:15:51 +0200 Subject: [PATCH 73/81] Change CapnProto library linkage for mpgen When building against static Cap'n Proto libraries, `mpgen` compilation fails at link time as `libcapnpc.a` depends on symbols defined in `libcapnp.a`. Flipping the order fixes the undefined references. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c2bfcb41..4eb0e694 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,8 +204,8 @@ add_executable(mpgen src/mp/gen.cpp $) add_executable(Libmultiprocess::mpgen ALIAS mpgen) target_include_directories(mpgen PRIVATE $) target_include_directories(mpgen PUBLIC $ $) -target_link_libraries(mpgen PRIVATE CapnProto::capnp) target_link_libraries(mpgen PRIVATE CapnProto::capnpc) +target_link_libraries(mpgen PRIVATE CapnProto::capnp) target_link_libraries(mpgen PRIVATE CapnProto::kj) target_link_libraries(mpgen PRIVATE CapnProto::kj-async) target_link_libraries(mpgen PRIVATE Threads::Threads) From 701449c1657e04ac9ca248fbecfa2b1583a4ec95 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 30 Sep 2026 20:29:33 -0400 Subject: [PATCH 74/81] util: Make SocketError inline constexpr Declare SocketError as an inline variable so the header does not define a separate internal-linkage copy in every translation unit that includes it. Bitcoin Core's build now enables -Wunused-const-variable (bitcoin/bitcoin#36275), which with GCC warns about unused const variables defined in headers and fails the build with -Werror in every file that includes mp/util.h without using SocketError. Bitcoin Core made the same change to its own SocketError definition in bitcoin/bitcoin#35852. This change was written with Claude Opus 5.5 (1M context). --- include/mp/util.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mp/util.h b/include/mp/util.h index 30742014..adc7cfbb 100644 --- a/include/mp/util.h +++ b/include/mp/util.h @@ -293,7 +293,7 @@ using Stream = kj::Own; using ProcessId = int; using SocketId = int; -constexpr SocketId SocketError{-1}; +inline constexpr SocketId SocketError{-1}; //! Information about parent process passed to child process as a command-line //! argument. On unix this is the child socket fd number formatted as a string. From a8c64c7faa5511e77350c64a941f75fdcfa91b18 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 30 Sep 2026 20:31:04 -0400 Subject: [PATCH 75/81] ci: Enable -Wunused-const-variable in GCC jobs Bitcoin Core builds the libmultiprocess subtree with -Wunused-const-variable (bitcoin/bitcoin#36275). With GCC this warns about unused const variables defined in headers, so a header-level constexpr that is not inline breaks the Bitcoin Core build. Enable it in the GCC-based CI configs to catch this here first. Clang's -Wunused-const-variable does not warn about variables defined in headers, so it is not added to the clang configs. This change was written with Claude Opus 5.5 (1M context). --- ci/configs/default.bash | 2 +- ci/configs/gnu32.bash | 2 +- ci/configs/newdeps.bash | 2 +- ci/configs/olddeps.bash | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/configs/default.bash b/ci/configs/default.bash index f36f4e49..acb38e15 100644 --- a/ci/configs/default.bash +++ b/ci/configs/default.bash @@ -1,5 +1,5 @@ CI_DESC="CI job using default libraries and tools, and running IWYU" CI_DIR=build-default -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wextra-semi -Wmissing-noreturn -Wtrailing-whitespace" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wunused-const-variable -Wno-unused-parameter -Wextra-semi -Wmissing-noreturn -Wtrailing-whitespace" CMAKE_ARGS=(-DMP_ENABLE_IWYU=ON) BUILD_ARGS=(-k) diff --git a/ci/configs/gnu32.bash b/ci/configs/gnu32.bash index 04d78a24..3d63360f 100644 --- a/ci/configs/gnu32.bash +++ b/ci/configs/gnu32.bash @@ -6,6 +6,6 @@ NIX_ARGS=( --arg minimal true --arg crossPkgs 'import { crossSystem = { config = "i686-unknown-linux-gnu"; }; }' ) -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wunused-const-variable -Wno-unused-parameter" CMAKE_ARGS=(-G Ninja) BUILD_ARGS=(-k 0) diff --git a/ci/configs/newdeps.bash b/ci/configs/newdeps.bash index a86f9dee..c86a833d 100644 --- a/ci/configs/newdeps.bash +++ b/ci/configs/newdeps.bash @@ -1,6 +1,6 @@ CI_DESC="CI job using newest Cap'n Proto and cmake versions" CI_DIR=build-newdeps -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wunused-const-variable -Wno-unused-parameter -Wno-error=array-bounds" CAPNP_CHECKOUT=master # This is the v1.x release branch NIX_ARGS=(--argstr capnprotoVersion "none" --argstr cmakeVersion "4.3.4") BUILD_ARGS=(-k) diff --git a/ci/configs/olddeps.bash b/ci/configs/olddeps.bash index 1d9ea6fc..81d084fc 100644 --- a/ci/configs/olddeps.bash +++ b/ci/configs/olddeps.bash @@ -2,7 +2,7 @@ CI_DESC="CI job using old Cap'n Proto and cmake versions" CI_DIR=build-olddeps # Pin olddeps to an older Nixpkgs channel, to be able to test an older GCC. CI_NIXPKGS_CHANNEL=nixos-25.05 -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wunused-const-variable -Wno-unused-parameter -Wno-error=array-bounds" CAPNP_CHECKOUT=v0.9.2 # Use a checkout to compile it with the selected GCC NIX_ARGS=(--argstr capnprotoVersion "none" --argstr cmakeVersion "3.12.4" --argstr gccVersion "10") BUILD_ARGS=(-k) From debd42955ea76e88f6066335eab6fdce1bea8c49 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Thu, 1 Oct 2026 19:07:59 -0400 Subject: [PATCH 76/81] proxy-types: Avoid -Wundef error when KJ_NO_EXCEPTIONS is not defined Check that KJ_NO_EXCEPTIONS is defined before testing its value. Bitcoin Core builds the libmultiprocess subtree with -Wundef, so the plain `#if KJ_NO_EXCEPTIONS` check added in #364 fails the build with -Werror whenever the macro is not defined, which is the normal case. This change was written with Claude Opus 5.5 (1M context). --- include/mp/proxy-types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mp/proxy-types.h b/include/mp/proxy-types.h index 0c00b825..9822c121 100644 --- a/include/mp/proxy-types.h +++ b/include/mp/proxy-types.h @@ -15,7 +15,7 @@ // KJ exception support is required, or exceptions thrown in serverInvoke below go uncaught // and crash the process instead of being returned to the client as errors. -#if KJ_NO_EXCEPTIONS +#if defined(KJ_NO_EXCEPTIONS) && KJ_NO_EXCEPTIONS #error "KJ_NO_EXCEPTIONS=1 is set but libmultiprocess requires support for exceptions. Please check build settings. On NetBSD you may also need to set -DKJ_NO_EXCEPTIONS=0 -DKJ_NO_RTTI=0 explicitly (https://github.com/capnproto/capnproto/pull/2756)" #endif From 3577d23f616ec6e8d601d14452a5793f9d386708 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Thu, 1 Oct 2026 19:08:25 -0400 Subject: [PATCH 77/81] ci: Enable -Wundef in default and llvm jobs Bitcoin Core builds the libmultiprocess subtree with -Wundef, so a preprocessor check on an undefined macro breaks the Bitcoin Core build. Enable it in one GCC and one clang config to catch this here first. This change was written with Claude Opus 5.5 (1M context). --- ci/configs/default.bash | 2 +- ci/configs/llvm.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/configs/default.bash b/ci/configs/default.bash index acb38e15..492992c6 100644 --- a/ci/configs/default.bash +++ b/ci/configs/default.bash @@ -1,5 +1,5 @@ CI_DESC="CI job using default libraries and tools, and running IWYU" CI_DIR=build-default -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wunused-const-variable -Wno-unused-parameter -Wextra-semi -Wmissing-noreturn -Wtrailing-whitespace" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wundef -Wunused-const-variable -Wno-unused-parameter -Wextra-semi -Wmissing-noreturn -Wtrailing-whitespace" CMAKE_ARGS=(-DMP_ENABLE_IWYU=ON) BUILD_ARGS=(-k) diff --git a/ci/configs/llvm.bash b/ci/configs/llvm.bash index af3b9719..4257db83 100644 --- a/ci/configs/llvm.bash +++ b/ci/configs/llvm.bash @@ -2,7 +2,7 @@ CI_DESC="CI job using LLVM-based libraries and tools (clang, libc++, clang-tidy, CI_DIR=build-llvm NIX_ARGS=(--arg enableLibcxx true) export CXX=clang++ -export CXXFLAGS="-Werror -Wall -Wextra -Wextra-semi -Wpedantic -Wthread-safety -Wno-unused-parameter -Wno-c++23-lambda-attributes" +export CXXFLAGS="-Werror -Wall -Wextra -Wextra-semi -Wpedantic -Wundef -Wthread-safety -Wno-unused-parameter -Wno-c++23-lambda-attributes" CMAKE_ARGS=( -G Ninja -DMP_ENABLE_CLANG_TIDY=ON From 92127a984a47113a918720fadfa6b7e6fc1387e1 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Thu, 1 Oct 2026 22:01:32 -0400 Subject: [PATCH 78/81] cmake: Pass -w to include-what-you-use Stop the compiler warning flags in CXXFLAGS from turning clang warnings into errors in the IWYU pass. IWYU's clang frontend receives the same flags as the real compiler, so in the GCC CI jobs (-Werror -Wpedantic) every KJ_TEST reports "'__COUNTER__' is a C2y extension" and the GCC-only -Wtrailing-whitespace is an unknown option. Once a file has enough tests to reach clang's 20-error limit (test/mp/test/test.cpp currently has 16), clang gives up, IWYU exits 1 without analyzing the file, and the build fails with no diagnostics. Real IWYU findings are still reported with -w. This change was written with Claude Opus 5.5 (1M context). --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4eb0e694..6d9fd74f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,7 +96,10 @@ if(MP_ENABLE_IWYU) if(NOT IWYU_EXECUTABLE) message(FATAL_ERROR "MP_ENABLE_IWYU is ON but include-what-you-use was not found.") endif() - set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE "${IWYU_EXECUTABLE};-Xiwyu;--error") + # Pass -w so CXXFLAGS meant for the compiler (-Werror -Wpedantic, GCC-only -W + # options) do not turn clang warnings into errors in the IWYU pass. Clang + # stops after 19 errors, and IWYU then exits nonzero without checking includes. + set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE "${IWYU_EXECUTABLE};-Xiwyu;--error;-w") if(DEFINED ENV{IWYU_MAPPING_FILE}) list(APPEND CMAKE_CXX_INCLUDE_WHAT_YOU_USE "-Xiwyu" "--mapping_file=$ENV{IWYU_MAPPING_FILE}") endif() From f0a14754ef4f922c8e391c201f3d4329cb69ee0e Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Thu, 1 Oct 2026 22:01:45 -0400 Subject: [PATCH 79/81] cmake: Show include-what-you-use diagnostics when it fails Run include-what-you-use through a wrapper script that makes CMake print its output when it fails without suggesting includes. CMake only relays IWYU output containing "should add these lines:" or "should remove these lines:", so a clang error in the IWYU pass (like the too-many-errors abort fixed in the previous commit) failed the build with a bare "Error 1" and nothing to debug. This change was written with Claude Opus 5.5 (1M context). --- CMakeLists.txt | 4 +++- cmake/iwyu-wrapper.sh | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100755 cmake/iwyu-wrapper.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d9fd74f..3462f3e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,7 +99,9 @@ if(MP_ENABLE_IWYU) # Pass -w so CXXFLAGS meant for the compiler (-Werror -Wpedantic, GCC-only -W # options) do not turn clang warnings into errors in the IWYU pass. Clang # stops after 19 errors, and IWYU then exits nonzero without checking includes. - set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE "${IWYU_EXECUTABLE};-Xiwyu;--error;-w") + # Run IWYU through a wrapper so its diagnostics are shown when it fails + # without suggesting includes (see cmake/iwyu-wrapper.sh). + set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE "${CMAKE_CURRENT_SOURCE_DIR}/cmake/iwyu-wrapper.sh;${IWYU_EXECUTABLE};-Xiwyu;--error;-w") if(DEFINED ENV{IWYU_MAPPING_FILE}) list(APPEND CMAKE_CXX_INCLUDE_WHAT_YOU_USE "-Xiwyu" "--mapping_file=$ENV{IWYU_MAPPING_FILE}") endif() diff --git a/cmake/iwyu-wrapper.sh b/cmake/iwyu-wrapper.sh new file mode 100755 index 00000000..fa2fd1a0 --- /dev/null +++ b/cmake/iwyu-wrapper.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Run include-what-you-use for CMAKE_CXX_INCLUDE_WHAT_YOU_USE and make sure its +# output is shown when it fails. Usage: iwyu-wrapper.sh IWYU [ARGS...] +# +# "cmake -E __run_co_compile" only prints IWYU output that contains "should add +# these lines:" or "should remove these lines:" (HandleIWYU in CMake's +# Source/cmcmd.cxx). If IWYU fails for another reason, such as a clang error, +# the build stops with a bare "Error 1". In that case this script appends a line +# containing the marker text, so CMake prints the clang diagnostics too. +# +# This works around https://gitlab.kitware.com/cmake/cmake/-/issues/28020 and +# can be removed once CMake prints IWYU output whenever IWYU fails. +set -o nounset +iwyu=$1 +shift +out=$("$iwyu" "$@" 2>&1) +ret=$? +printf '%s\n' "$out" >&2 +if [ "$ret" -ne 0 ] && ! grep -q -e 'should add these lines:' -e 'should remove these lines:' <<<"$out"; then + printf 'include-what-you-use failed with exit status %s before suggesting includes (marker for CMake: "should add these lines:"); see the diagnostics above.\n' "$ret" >&2 +fi +exit "$ret" From 04a0465d67a58040013c74dbb92e0f7a0572cc0e Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:59:51 -0300 Subject: [PATCH 80/81] ci: Compile capnproto with GCC in gnu32 job In the gnu32 job, capnproto (libkj) is compiled with Clang while libmultiprocess is compiled with GCC, but they handle empty structs differently on i686. Clang omits them, while GCC reserves a stack slot for them. This causes a segfault in calls to functions like `kj::str(kj::Exception)`, which internally pass an empty `Stringifier` object. This commit fixes the issue by compiling capnproto with the same toolchain as libmultiprocess (GCC) when i686 is detected in shell.nix. --- shell.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/shell.nix b/shell.nix index 1fbfda51..01cf415c 100644 --- a/shell.nix +++ b/shell.nix @@ -50,7 +50,12 @@ let "-g" ]; }; - })).override (lib.optionalAttrs enableLibcxx { clangStdenv = llvm.libcxxStdenv; }); + })).override (lib.optionalAttrs enableLibcxx { clangStdenv = llvm.libcxxStdenv; } + # On i686, GCC and clang handle empty structs differently, causing an + # ABI mismatch and segfaulting in calls to functions like + # kj::str(kj::Exception). Prevent this by compiling capnproto with the + # same toolchain as libmultiprocess. + // lib.optionalAttrs crossPkgs.stdenv.hostPlatform.isx86_32 { clangStdenv = crossPkgs.stdenv; }); clang = if enableLibcxx then llvm.libcxxClang else llvm.clang; clang-tools = llvm.clang-tools.override { inherit enableLibcxx; }; cmakeHashes = { From 0e96139da1c6b56fa8f73171bc982f5fd23d81da Mon Sep 17 00:00:00 2001 From: xyzconstant <263061129+xyzconstant@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:58:15 -0300 Subject: [PATCH 81/81] proxy: Reject destroy() during active calls and calls after destroy() A non-libmultiprocess client can send requests after destroy(), causing a null m_impl pointer to be dereferenced and triggering a SIGSEGV. It can also call destroy() while another request on the same object is active, freeing m_impl while that request is using it. Track requests queued or executing on worker threads with m_active_calls. Reject destroy() while other requests are active, and reject any requests received after destruction. Fixes #378 --- doc/design.md | 2 +- include/mp/proxy-io.h | 15 +++++- include/mp/proxy-types.h | 4 ++ include/mp/proxy.h | 6 +++ include/mp/type-context.h | 5 +- test/mp/test/test.cpp | 109 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 138 insertions(+), 3 deletions(-) diff --git a/doc/design.md b/doc/design.md index 319ee3de..0eb4ccfc 100644 --- a/doc/design.md +++ b/doc/design.md @@ -134,7 +134,7 @@ sequenceDiagram SF1->>serverInvoke: return ``` -Destroy methods are a special case: a capnp interface can declare a `destroy` method that is handled by `ServerDestroy` instead of `ServerCall`. Rather than dispatching through the `ServerField`/`ServerRet`/`ServerCall` chain to a C++ interface method, `ServerDestroy` calls `invokeDestroy()` on the `ProxyServer`, which resets `m_impl` and runs any registered cleanup functions, giving the client a way to synchronously destroy the wrapped object on the server side. +Destroy methods are a special case: a capnp interface can declare a `destroy` method that is handled by `ServerDestroy` instead of `ServerCall`. Rather than dispatching through the `ServerField`/`ServerRet`/`ServerCall` chain to a C++ interface method, `ServerDestroy` calls `invokeDestroy()` on the `ProxyServer`, which resets `m_impl` and runs any registered cleanup functions, giving the client a way to synchronously destroy the wrapped object on the server side. The destroy request fails if other requests on the object are running, and requests sent after it fail. Clients that use Cap'n Proto directly should drop the capability instead. ## Advanced Features diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index b226284f..c7d08673 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -692,10 +692,23 @@ ProxyServerBase::~ProxyServerBase() //! Context.thread parameter was passed, this method will run on the worker //! thread specified by the client. Otherwise it will run on the EventLoop //! thread, like other server methods without an assigned thread. +//! +//! Non-libmultiprocess clients can send destroy while other requests are +//! running, so it is rejected in that case. template void ProxyServerBase::invokeDestroy() { - m_impl.reset(); + // A destroy request with an mp.Context runs on a worker thread and is + // counted in m_active_calls itself. + const int self_count{std::this_thread::get_id() == m_context.loop->m_thread_id ? 0 : 1}; + // Move m_impl out on the event loop thread, where serverInvoke checks it, + // and destroy it after returning to this thread. + std::shared_ptr impl; + m_context.loop->sync([&] { + if (m_active_calls == self_count) impl = std::move(m_impl); + }); + if (!impl) throw std::runtime_error("destroy called while other calls are in progress"); + impl.reset(); CleanupRun(m_context.cleanup_fns); } diff --git a/include/mp/proxy-types.h b/include/mp/proxy-types.h index 9822c121..2a2ce593 100644 --- a/include/mp/proxy-types.h +++ b/include/mp/proxy-types.h @@ -834,6 +834,10 @@ kj::Promise serverInvoke(Server& server, CallContext& call_context, Fn fn) MP_LOG(loop, Log::Trace) << "request data: " << LogEscape(params.toString(), server.m_context.loop->m_log_opts.max_chars); + if (!server.m_impl) { + MP_LOG(loop, Log::Error) << "IPC server error request #" << req << ", method called after destroy"; + throw std::runtime_error("method called after destroy"); + } try { using ServerContext = ServerInvokeContext; using ArgList = typename ProxyClientMethodTraits::Params; diff --git a/include/mp/proxy.h b/include/mp/proxy.h index 2144b571..eef6962b 100644 --- a/include/mp/proxy.h +++ b/include/mp/proxy.h @@ -172,6 +172,12 @@ struct ProxyServerBase : public virtual Interface_::Server * wrapped. */ std::shared_ptr m_impl; ProxyContext m_context; + //! Number of requests queued or executing on worker threads. + //! Incremented in the mp.Context PassField overload before being posted and + //! decremented when completed or dropped. Requests executing on the + //! event-loop thread are excluded because they run to completion without + //! allowing another request to run. + int m_active_calls{0}; }; //! Customizable (through template specialization) base class which ProxyServer diff --git a/include/mp/type-context.h b/include/mp/type-context.h index 7cfc7e79..2407b29a 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -64,6 +64,8 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& auto& server = server_context.proxy_server; EventLoop& loop = *server.m_context.loop; int req = server_context.req; + ++server.m_active_calls; + auto active_call = std::shared_ptr{nullptr, [&server](void*) { --server.m_active_calls; }}; // Keep a reference to the ProxyServer instance by assigning it to the self // variable. ProxyServer instances are reference-counted and if the client // drops its reference and the IPC call is canceled, this variable keeps the @@ -71,7 +73,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // needs to be destroyed on the event loop thread so it is freed in a sync() // call below. auto self = server.thisCap(); - auto invoke = [self = kj::mv(self), call_context = kj::mv(server_context.call_context), &server, &loop, req, fn, args...](CancelMonitor& cancel_monitor) mutable { + auto invoke = [self = kj::mv(self), active_call = kj::mv(active_call), call_context = kj::mv(server_context.call_context), &server, &loop, req, fn, args...](CancelMonitor& cancel_monitor) mutable { MP_LOG(loop, Log::Debug) << "IPC server executing request #" << req; if (loop.testing_hook_async_request_start) loop.testing_hook_async_request_start(); KJ_DEFER(if (loop.testing_hook_async_request_done) loop.testing_hook_async_request_done()); @@ -176,6 +178,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // scope when it happens. cancel_monitor.m_on_cancel = nullptr; auto self_dispose{kj::mv(self)}; + auto active_call_dispose{kj::mv(active_call)}; if (erase_thread) { // Look up the thread again without using existing // iterator since entry may no longer be there after diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 0492c9ee..212d6ab4 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -762,5 +762,114 @@ KJ_TEST("Call async IPC method without thread or pool errors correctly") KJ_EXPECT(error_thrown); } +KJ_TEST("Calling IPC method after destroy and destroy during a call") +{ + // Regression test for bitcoin-core/libmultiprocess#378 where a + // non-libmultiprocess client could: + // + // 1. Call a method after destroy, making the server call it through a + // null m_impl pointer. + // 2. Call destroy while another call on the same object was running, + // making the server free m_impl under that call. + // + // This test builds requests directly through Cap'n Proto, mimicking the + // behavior of a buggy client. + TestSetup setup; + ProxyClient* foo = setup.client.get(); + EventLoop& loop = *foo->m_context.loop; + foo->initThreadMap(); + + // Create two pool threads, so the destroy() and value() calls run on + // different threads. + std::promise pool_ready; + loop.sync([&] { + auto pool_req = foo->m_context.connection->m_thread_map.makePoolRequest(); + pool_req.setCount(2); + loop.m_task_set->add(pool_req.send().then([&](auto&&) { pool_ready.set_value(); })); + }); + pool_ready.get_future().get(); + + // Create two Bar objects, one for each case below. BarInterface declares a + // destroy method, unlike FooInterface. + std::optional bar1; + std::optional bar2; + std::promise bars_ready; + loop.sync([&] { + auto request{foo->m_client.listBarsRequest()}; + request.initContext(); + request.setN(2); + loop.m_task_set->add(request.send().then([&](auto&& results) { + bar1.emplace(results.getResult()[0]); + bar2.emplace(results.getResult()[1]); + bars_ready.set_value(); + })); + }); + bars_ready.get_future().get(); + + //////////////////////////////////////////////////////////////////////////// + // Case 1: method called after destroy. + //////////////////////////////////////////////////////////////////////////// + + std::promise destroyed; + loop.sync([&] { + auto request{bar1->destroyRequest()}; + request.initContext(); + loop.m_task_set->add(request.send().then([&](auto&&) { destroyed.set_value(); })); + }); + destroyed.get_future().get(); + + // Calls after destroy are rejected. + std::promise value_error; + loop.sync([&] { + auto request{bar1->valueRequest()}; + request.initContext(); + loop.m_task_set->add(request.send().then( + [&](auto&&) { value_error.set_value(""); }, + [&](kj::Exception&& e) { value_error.set_value(e.getDescription().cStr()); })); + }); + KJ_EXPECT(value_error.get_future().get().find("method called after destroy") != std::string::npos); + + // Release bar1. + loop.sync([&] { bar1.reset(); }); + + //////////////////////////////////////////////////////////////////////////// + // Case 2: destroy called while another call is running. + //////////////////////////////////////////////////////////////////////////// + + // Block the next async request, the value() call, before it calls m_impl. + std::promise value_started; + std::promise value_release; + std::atomic blocked{false}; + loop.testing_hook_async_request_start = [&] { + if (blocked.exchange(true)) return; + value_started.set_value(); + value_release.get_future().wait(); + }; + std::promise value; + loop.sync([&] { + auto request{bar2->valueRequest()}; + request.initContext(); + loop.m_task_set->add(request.send().then([&](auto&& results) { value.set_value(results.getResult()); })); + }); + value_started.get_future().get(); + + // Destroy fails if value() request is being executed. + std::promise destroy_error; + loop.sync([&] { + auto request{bar2->destroyRequest()}; + request.initContext(); + loop.m_task_set->add(request.send().then( + [&](auto&&) { destroy_error.set_value(""); }, + [&](kj::Exception&& e) { destroy_error.set_value(e.getDescription().cStr()); })); + }); + KJ_EXPECT(destroy_error.get_future().get().find("destroy called while other calls are in progress") != std::string::npos); + + value_release.set_value(); + KJ_EXPECT(value.get_future().get() == 1); + + // Release bar2. + loop.sync([&] { bar2.reset(); }); +} + } // namespace test } // namespace mp