diff --git a/.clang-tidy b/.clang-tidy index 9a2afcc5..16dafd03 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -6,8 +6,12 @@ bugprone-string-constructor, bugprone-use-after-move, bugprone-lambda-function-name, bugprone-unhandled-self-assignment, -misc-unused-using-decls, +bugprone-unused-return-value, +fuchsia-header-anon-namespaces, +misc-definitions-in-headers, misc-no-recursion, +misc-unused-using-decls, +modernize-avoid-bind, modernize-deprecated-headers, modernize-use-default-member-init, modernize-use-emplace, @@ -22,7 +26,9 @@ performance-*, -performance-no-int-to-ptr, -performance-noexcept-move-constructor, -performance-unnecessary-value-param, +readability-avoid-const-params-in-decls, readability-const-return-type, +readability-container-contains, readability-redundant-declaration, readability-redundant-string-init, clang-analyzer-core.*, @@ -35,6 +41,10 @@ CheckOptions: - key: modernize-deprecated-headers.CheckHeaderFile value: false - key: performance-move-const-arg.CheckTriviallyCopyableMove - value: false + value: false # Disabled, to allow the bugprone-use-after-move rule on trivially copyable types, to catch logic bugs - key: bugprone-unhandled-self-assignment.WarnOnlyIfThisHasSuspiciousField value: false + - key: bugprone-unused-return-value.CheckedReturnTypes + value: '^::std::error_code$;^::std::error_condition$;^::std::errc$;^::std::expected$;^::util::Result$;^::util::Expected$' + - key: bugprone-unused-return-value.AllowCastToVoid + value: true # Can be removed with C++26 once the _ placeholder exists. diff --git a/.github/workflows/bitcoin-core-ci.yml b/.github/workflows/bitcoin-core-ci.yml index 83dff816..6aa1aa58 100644 --- a/.github/workflows/bitcoin-core-ci.yml +++ b/.github/workflows/bitcoin-core-ci.yml @@ -18,9 +18,8 @@ concurrency: env: BITCOIN_REPO: bitcoin/bitcoin - # Temporary: use PR #35084 until it merges; revert to refs/heads/master after - BITCOIN_CORE_REF: refs/pull/35084/merge - LLVM_VERSION: 22 + BITCOIN_CORE_REF: refs/heads/master + LLVM_VERSION: 23 LIBCXX_DIR: /tmp/libcxx-build/ jobs: @@ -38,13 +37,13 @@ jobs: functional_test_runs: 20 nproc_multiplier: 2 functional_timeout_factor: 40 - runner: ubuntu-24.04 + runner: ubuntu-26.04 apt-llvm: true packages: >- ccache - clang-22 - llvm-22 - libclang-rt-22-dev + clang-23 + llvm-23 + libclang-rt-23-dev libboost-dev libsqlite3-dev libcapnp-dev diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffaca1ca..9925dedb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,10 +17,16 @@ jobs: # Test all supported releases. # See https://www.netbsd.org/releases/. include: - - release: 9.4 - capnproto-cppflags: 'CPPFLAGS="-DKJ_NO_EXCEPTIONS=0 -DKJ_USE_KQUEUE=0"' - - release: 10.1 + # Due to a bug in Cap'n Proto, GCC compilers older + # than 14.x require `-DKJ_NO_EXCEPTIONS=0` when + # processing `capnproto/c++/src/kj/common.h`. + # See https://github.com/capnproto/capnproto/pull/2756/changes/28fa1a6753514d8de56875ec2222fa0e1c3a04f7. + - release: '10.1' capnproto-cppflags: 'CPPFLAGS="-DKJ_NO_EXCEPTIONS=0"' + env_vars: 'CXXFLAGS="-DKJ_NO_EXCEPTIONS=0"' + - release: '11.0' + capnproto-cppflags: 'CPPFLAGS="-DKJ_NO_EXCEPTIONS=0"' + env_vars: 'CXXFLAGS="-DKJ_NO_EXCEPTIONS=0"' steps: - uses: actions/checkout@v6 @@ -44,9 +50,9 @@ jobs: pkgsrc/devel/libtool-base \ pkgsrc/devel/pkgconf \ pkgsrc/devel/zlib \ - `# gcc15 is referenced here because the pkgsrc framework requires lang/gcc15/version.mk to exist` \ + `# gcc16 is referenced here because the pkgsrc framework requires lang/gcc16/version.mk to exist` \ `# during the "make install" step below, even though we compile our project with gcc14.` \ - pkgsrc/lang/gcc15 \ + pkgsrc/lang/gcc16 \ pkgsrc/mk \ pkgsrc/pkgtools \ pkgsrc/security/openssl \ @@ -63,7 +69,7 @@ jobs: - name: Run CI script run: | cd ${{ github.workspace }} - CI_CONFIG="ci/configs/netbsd.bash" bash ci/scripts/ci.sh + CI_CONFIG="ci/configs/netbsd.bash" ${{ matrix.env_vars }} bash ci/scripts/ci.sh build-openbsd: runs-on: ubuntu-latest @@ -100,7 +106,7 @@ jobs: uses: vmactions/freebsd-vm@v1 with: prepare: | - pkg install -y cmake ninja bash capnproto + pkg install -y cmake ninja bash capnproto git sync: 'rsync' copyback: false @@ -131,18 +137,18 @@ jobs: runs-on: ubuntu-latest env: - NIXPKGS_CHANNEL: nixos-25.05 - NIX_EXTRA_CONFIG: | + CI_NIXPKGS_CHANNEL: nixos-26.05 + CI_NIX_CONFIG: | keep-env-derivations = true keep-outputs = true - NIX_EXTRA_CONFIG_ACT: | + CI_NIX_CONFIG_ACT: | sandbox = false filter-syscalls = false strategy: fail-fast: false matrix: - config: [default, llvm, gnu32, sanitize, olddeps] + config: [default, llvm, gnu32, sanitize, olddeps, newdeps] name: build • ${{ matrix.config }} @@ -158,13 +164,12 @@ jobs: - name: Install Nix uses: cachix/install-nix-action@v31 # 2025-05-27, from https://github.com/cachix/install-nix-action/tags with: - nix_path: nixpkgs=https://github.com/NixOS/nixpkgs/archive/${{ steps.config.outputs.nixpkgs_rev }}.tar.gz # Act executes inside an unprivileged container (Docker or Podman), # so KVM support isn't available. enable_kvm: "${{ github.actor != 'nektos/act' }}" extra_nix_config: | - ${{ env.NIX_EXTRA_CONFIG }} - ${{ github.actor == 'nektos/act' && env.NIX_EXTRA_CONFIG_ACT || '' }} + ${{ env.CI_NIX_CONFIG }} + ${{ github.actor == 'nektos/act' && env.CI_NIX_CONFIG_ACT || '' }} - name: Cache Nix store if: steps.config.outputs.cache_nix_store == 'true' diff --git a/CMakeLists.txt b/CMakeLists.txt index 6354b0ee..3462f3e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ endif() include("cmake/compat_find.cmake") find_package(Threads REQUIRED) -find_package(CapnProto 0.9 QUIET NO_MODULE) +find_package(CapnProto 0.9 NO_MODULE) if(NOT CapnProto_FOUND) message(FATAL_ERROR "Cap'n Proto is required but was not found.\n" @@ -35,9 +35,7 @@ set(CAPNPROTO_CLANG_INCOMPATIBLE FALSE) # https://github.com/capnproto/capnproto/blob/master/security-advisories/2022-11-30-0-pointer-list-bounds.md # https://capnproto.org/news/2022-11-30-CVE-2022-46149-security-advisory.html # https://dwrensha.github.io/capnproto-rust/2022/11/30/out_of_bounds_memory_access_bug.html -if(CapnProto_VERSION STREQUAL "0.7.0" - OR CapnProto_VERSION STREQUAL "0.8.0" - OR CapnProto_VERSION STREQUAL "0.9.0" +if(CapnProto_VERSION STREQUAL "0.9.0" OR CapnProto_VERSION STREQUAL "0.9.1" OR CapnProto_VERSION STREQUAL "0.10.0" OR CapnProto_VERSION STREQUAL "0.10.1" @@ -66,7 +64,7 @@ if(CAPNPROTO_CVE_AFFECTED OR CAPNPROTO_CLANG_INCOMPATIBLE) string(APPEND RESOLUTION_OPTIONS " - Upgrade to Cap'n Proto version 1.0 or newer (recommended)\n") if(CAPNPROTO_CVE_AFFECTED AND NOT CAPNPROTO_CLANG_INCOMPATIBLE) - string(APPEND RESOLUTION_OPTIONS " - Upgrade to a patched minor version (0.7.1, 0.8.1, 0.9.2, 0.10.3, or later)\n") + string(APPEND RESOLUTION_OPTIONS " - Upgrade to a patched minor version (0.9.2, 0.10.3, or later)\n") elseif(CAPNPROTO_CLANG_INCOMPATIBLE AND NOT CAPNPROTO_CVE_AFFECTED) string(APPEND RESOLUTION_OPTIONS " - Use GCC instead of Clang\n") endif() @@ -98,7 +96,12 @@ if(MP_ENABLE_IWYU) if(NOT IWYU_EXECUTABLE) message(FATAL_ERROR "MP_ENABLE_IWYU is ON but include-what-you-use was not found.") endif() - set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE "${IWYU_EXECUTABLE};-Xiwyu;--error") + # Pass -w so CXXFLAGS meant for the compiler (-Werror -Wpedantic, GCC-only -W + # options) do not turn clang warnings into errors in the IWYU pass. Clang + # stops after 19 errors, and IWYU then exits nonzero without checking includes. + # Run IWYU through a wrapper so its diagnostics are shown when it fails + # without suggesting includes (see cmake/iwyu-wrapper.sh). + set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE "${CMAKE_CURRENT_SOURCE_DIR}/cmake/iwyu-wrapper.sh;${IWYU_EXECUTABLE};-Xiwyu;--error;-w") if(DEFINED ENV{IWYU_MAPPING_FILE}) list(APPEND CMAKE_CXX_INCLUDE_WHAT_YOU_USE "-Xiwyu" "--mapping_file=$ENV{IWYU_MAPPING_FILE}") endif() @@ -176,9 +179,11 @@ set(MP_PUBLIC_HEADERS include/mp/type-struct.h include/mp/type-threadmap.h include/mp/type-tuple.h + include/mp/type-unordered-set.h include/mp/type-vector.h include/mp/type-void.h - include/mp/util.h) + include/mp/util.h + include/mp/version.h) add_library(multiprocess STATIC ${MP_PROXY_SRCS} ${MP_PUBLIC_HEADERS} @@ -204,11 +209,21 @@ add_executable(mpgen src/mp/gen.cpp $) add_executable(Libmultiprocess::mpgen ALIAS mpgen) target_include_directories(mpgen PRIVATE $) target_include_directories(mpgen PUBLIC $ $) -target_link_libraries(mpgen PRIVATE CapnProto::capnp) -target_link_libraries(mpgen PRIVATE CapnProto::capnp-rpc) target_link_libraries(mpgen PRIVATE CapnProto::capnpc) +target_link_libraries(mpgen PRIVATE CapnProto::capnp) target_link_libraries(mpgen PRIVATE CapnProto::kj) +target_link_libraries(mpgen PRIVATE CapnProto::kj-async) target_link_libraries(mpgen PRIVATE Threads::Threads) +# Use CAPNP_EXECUTABLE/CAPNPC_CXX_EXECUTABLE cmake variables set by +# CapnProtoConfig.cmake rather than reading the CapnProto::capnp_tool +# IMPORTED_LOCATION directly. On Debian/Ubuntu, IMPORTED_LOCATION is broken +# by a multiarch cmake packaging bug (see +# https://github.com/bitcoin-core/libmultiprocess/issues/328), but +# CapnProtoConfig.cmake works around it by hardcoding the correct paths. On +# other platforms the variables hold generator expressions that +# target_compile_definitions evaluates correctly at generation time. +target_compile_definitions(mpgen PRIVATE CAPNP_EXECUTABLE=\"${CAPNP_EXECUTABLE}\") +target_compile_definitions(mpgen PRIVATE CAPNPC_CXX_EXECUTABLE=\"${CAPNPC_CXX_EXECUTABLE}\") set_target_properties(mpgen PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) set_target_properties(mpgen PROPERTIES diff --git a/ci/README.md b/ci/README.md index fef1c022..843b61e8 100644 --- a/ci/README.md +++ b/ci/README.md @@ -21,6 +21,7 @@ CI_CONFIG=ci/configs/llvm.bash ci/scripts/run.sh CI_CONFIG=ci/configs/gnu32.bash ci/scripts/run.sh CI_CONFIG=ci/configs/sanitize.bash ci/scripts/run.sh CI_CONFIG=ci/configs/olddeps.bash ci/scripts/run.sh +CI_CONFIG=ci/configs/newdeps.bash ci/scripts/run.sh ``` By default CI jobs will reuse their build directories. `CI_CLEAN=1` can be specified to delete them before running instead. diff --git a/ci/configs/default.bash b/ci/configs/default.bash index 56231228..492992c6 100644 --- a/ci/configs/default.bash +++ b/ci/configs/default.bash @@ -1,5 +1,5 @@ CI_DESC="CI job using default libraries and tools, and running IWYU" CI_DIR=build-default -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wundef -Wunused-const-variable -Wno-unused-parameter -Wextra-semi -Wmissing-noreturn -Wtrailing-whitespace" CMAKE_ARGS=(-DMP_ENABLE_IWYU=ON) BUILD_ARGS=(-k) diff --git a/ci/configs/freebsd.bash b/ci/configs/freebsd.bash index 017f2186..096e2649 100644 --- a/ci/configs/freebsd.bash +++ b/ci/configs/freebsd.bash @@ -1,5 +1,5 @@ CI_DESC="CI config for FreeBSD" CI_DIR=build-freebsd -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-c++23-lambda-attributes" CMAKE_ARGS=(-G Ninja) BUILD_ARGS=(-k 0) diff --git a/ci/configs/gnu32.bash b/ci/configs/gnu32.bash index 04d78a24..3d63360f 100644 --- a/ci/configs/gnu32.bash +++ b/ci/configs/gnu32.bash @@ -6,6 +6,6 @@ NIX_ARGS=( --arg minimal true --arg crossPkgs 'import { crossSystem = { config = "i686-unknown-linux-gnu"; }; }' ) -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wunused-const-variable -Wno-unused-parameter" CMAKE_ARGS=(-G Ninja) BUILD_ARGS=(-k 0) diff --git a/ci/configs/llvm.bash b/ci/configs/llvm.bash index ba34cbf4..4257db83 100644 --- a/ci/configs/llvm.bash +++ b/ci/configs/llvm.bash @@ -2,7 +2,7 @@ CI_DESC="CI job using LLVM-based libraries and tools (clang, libc++, clang-tidy, CI_DIR=build-llvm NIX_ARGS=(--arg enableLibcxx true) export CXX=clang++ -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wthread-safety -Wno-unused-parameter" +export CXXFLAGS="-Werror -Wall -Wextra -Wextra-semi -Wpedantic -Wundef -Wthread-safety -Wno-unused-parameter -Wno-c++23-lambda-attributes" CMAKE_ARGS=( -G Ninja -DMP_ENABLE_CLANG_TIDY=ON diff --git a/ci/configs/macos.bash b/ci/configs/macos.bash index 895d795f..4c3a0fea 100644 --- a/ci/configs/macos.bash +++ b/ci/configs/macos.bash @@ -1,5 +1,5 @@ CI_DESC="CI config for macOS" CI_DIR=build-macos -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-c++23-lambda-attributes" CMAKE_ARGS=(-G Ninja -DMP_ENABLE_CLANG_TIDY=ON) BUILD_ARGS=(-k 0) diff --git a/ci/configs/netbsd.bash b/ci/configs/netbsd.bash index c3e7d258..203b6623 100644 --- a/ci/configs/netbsd.bash +++ b/ci/configs/netbsd.bash @@ -1,11 +1,5 @@ CI_DESC="CI config for NetBSD" CI_DIR=build-netbsd -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" -# Hardcode GCC 14, since default GCC versions installed by NetBSD are older -# and may not be compatible with libmultiprocess. GCC 14 was chosen because -# it's the latest compiler available on all versions of NetBSD that we test. -# Note that the GCC version specified here must match the version specified -# in pkg_add in ci.yml. -export CXX="/usr/pkg/gcc14/bin/g++" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter ${CXXFLAGS:-}" CMAKE_ARGS=(-G Ninja) BUILD_ARGS=(-k 0) diff --git a/ci/configs/newdeps.bash b/ci/configs/newdeps.bash new file mode 100644 index 00000000..c86a833d --- /dev/null +++ b/ci/configs/newdeps.bash @@ -0,0 +1,6 @@ +CI_DESC="CI job using newest Cap'n Proto and cmake versions" +CI_DIR=build-newdeps +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wunused-const-variable -Wno-unused-parameter -Wno-error=array-bounds" +CAPNP_CHECKOUT=master # This is the v1.x release branch +NIX_ARGS=(--argstr capnprotoVersion "none" --argstr cmakeVersion "4.3.4") +BUILD_ARGS=(-k) diff --git a/ci/configs/olddeps.bash b/ci/configs/olddeps.bash index 1a363b1b..81d084fc 100644 --- a/ci/configs/olddeps.bash +++ b/ci/configs/olddeps.bash @@ -1,5 +1,8 @@ CI_DESC="CI job using old Cap'n Proto and cmake versions" CI_DIR=build-olddeps -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds" -NIX_ARGS=(--argstr capnprotoVersion "0.9.2" --argstr cmakeVersion "3.12.4") +# Pin olddeps to an older Nixpkgs channel, to be able to test an older GCC. +CI_NIXPKGS_CHANNEL=nixos-25.05 +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wunused-const-variable -Wno-unused-parameter -Wno-error=array-bounds" +CAPNP_CHECKOUT=v0.9.2 # Use a checkout to compile it with the selected GCC +NIX_ARGS=(--argstr capnprotoVersion "none" --argstr cmakeVersion "3.12.4" --argstr gccVersion "10") BUILD_ARGS=(-k) diff --git a/ci/configs/openbsd.bash b/ci/configs/openbsd.bash index a404e2bc..944f8d7a 100644 --- a/ci/configs/openbsd.bash +++ b/ci/configs/openbsd.bash @@ -1,5 +1,5 @@ CI_DESC="CI config for OpenBSD" CI_DIR=build-openbsd -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" +export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-c++23-lambda-attributes" CMAKE_ARGS=(-G Ninja) BUILD_ARGS=(-k 0) diff --git a/ci/configs/sanitize.bash b/ci/configs/sanitize.bash index 0375fd3a..907f3295 100644 --- a/ci/configs/sanitize.bash +++ b/ci/configs/sanitize.bash @@ -2,7 +2,7 @@ CI_DESC="CI job running ThreadSanitizer" CI_DIR=build-sanitize NIX_ARGS=(--arg enableLibcxx true --argstr libcxxSanitizers "Thread" --argstr capnprotoSanitizers "thread") export CXX=clang++ -export CXXFLAGS="-ggdb -Werror -Wall -Wextra -Wpedantic -Wthread-safety -Wno-unused-parameter -fsanitize=thread" +export CXXFLAGS="-ggdb -Werror -Wall -Wextra -Wpedantic -Wthread-safety -Wno-unused-parameter -fsanitize=thread -Wno-c++23-lambda-attributes" CMAKE_ARGS=() -BUILD_ARGS=(-k -j4) +BUILD_ARGS=(-k) BUILD_TARGETS=(mptest) diff --git a/ci/scripts/ci.sh b/ci/scripts/ci.sh index d989e9f4..8016d921 100755 --- a/ci/scripts/ci.sh +++ b/ci/scripts/ci.sh @@ -21,10 +21,43 @@ cmake --version cmake_ver=$(cmake --version | awk '/version/{print $3; exit}') ver_ge() { [ "$(printf '%s\n' "$2" "$1" | sort -V | head -n1)" = "$2" ]; } +# If CAPNP_CHECKOUT was requested, clone and install requested Cap'n Proto branch or tag +capnp_prefix= +if [ -n "${CAPNP_CHECKOUT-}" ]; then + capnp_prefix="$PWD/capnp-install" + [ -e "capnp" ] || git clone -b "${CAPNP_CHECKOUT}" "https://github.com/capnproto/capnproto" capnp + mkdir -p capnp/build + ( + cd capnp/build + git --no-pager log -1 || true + CXXFLAGS="-std=c++20" cmake .. "-DCMAKE_INSTALL_PREFIX=${capnp_prefix}" -DBUILD_TESTING=OFF -DWITH_OPENSSL=OFF -DWITH_ZLIB=OFF + cmake --build . + cmake --build . --target install # Replacement for modern "cmake --install .". + ) + export CMAKE_PREFIX_PATH="${capnp_prefix}:${CMAKE_PREFIX_PATH-}" +fi + src_dir=$PWD mkdir -p "$CI_DIR" cd "$CI_DIR" -cmake "$src_dir" "${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}" +export CMAKE_BUILD_PARALLEL_LEVEL="$(nproc)" +git --no-pager log -1 || true +cmake_args=("${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}") +if ! cmake "$src_dir" "${cmake_args[@]}"; then + # If cmake failed, try it again with debug options. + # Could add --trace / --trace-expand here too but they are very verbose. + cmake_args+=(--debug-output --debug-trycompile) + if ver_ge "$cmake_ver" "3.16"; then cmake_args+=(--log-level=DEBUG); fi + if ver_ge "$cmake_ver" "3.17"; then cmake_args+=(--debug-find); fi + cmake "$src_dir" "${cmake_args[@]}" || : "cmake exited with $?" + if ver_ge "$cmake_ver" "3.26"; then + cat CMakeFiles/CMakeConfigureLog.yaml || true + else + cat CMakeFiles/CMakeError.log CMakeFiles/CMakeOutput.log || true + fi + find . -ls || true + false +fi if ver_ge "$cmake_ver" "3.15"; then cmake --build . -t "${BUILD_TARGETS[@]}" -- "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" else diff --git a/ci/scripts/config.sh b/ci/scripts/config.sh index 7fe8118d..887e245b 100755 --- a/ci/scripts/config.sh +++ b/ci/scripts/config.sh @@ -17,8 +17,8 @@ source "${SCRIPT_DIR}/ci_helpers.sh" [ "${CI_CONFIG+x}" ] && source "$CI_CONFIG" # Resolve the nixpkgs channel to a specific revision for use in cache keys. -if [[ -n "${NIXPKGS_CHANNEL:-}" ]]; then - rev="$(curl --fail --location --silent --show-error "https://channels.nixos.org/${NIXPKGS_CHANNEL}/git-revision")" +if [[ -n "${CI_NIXPKGS_CHANNEL:-}" ]]; then + rev="$(curl --fail --location --silent --show-error "https://channels.nixos.org/${CI_NIXPKGS_CHANNEL}/git-revision")" test -n "${rev}" write_output_var nixpkgs_rev "${rev}" fi diff --git a/ci/scripts/run.sh b/ci/scripts/run.sh index 04ac8596..aadeed11 100755 --- a/ci/scripts/run.sh +++ b/ci/scripts/run.sh @@ -10,14 +10,16 @@ set -o errexit -o nounset -o pipefail -o xtrace [ "${CI_CONFIG+x}" ] && source "$CI_CONFIG" -nix develop --ignore-environment --keep CI_CONFIG --keep CI_CLEAN "${NIX_ARGS[@]+"${NIX_ARGS[@]}"}" -f shell.nix --command ci/scripts/ci.sh +nix_args=("${NIX_ARGS[@]+${NIX_ARGS[@]}}") +if [[ -n "${CI_NIXPKGS_CHANNEL:-}" ]]; then + nix_args+=(-I "nixpkgs=channel:${CI_NIXPKGS_CHANNEL}") +fi +nix develop "${nix_args[@]}" --ignore-environment --keep CI_CONFIG --keep CI_CLEAN -f shell.nix --command ci/scripts/ci.sh # Create a GC root for the shell closure so the cache-nix-action save step # does not garbage-collect it. if [ -n "${CI_CACHE_NIX_STORE-}" ]; then - nix-build shell.nix \ - -o "$CI_DIR/gcroot" \ - "${NIX_ARGS[@]+"${NIX_ARGS[@]}"}" + nix-build shell.nix "${nix_args[@]}" -o "$CI_DIR/gcroot" # Verify the closure is complete so the cache-nix-action save step has # everything it needs. nix-store --query --requisites "$CI_DIR/gcroot" >/dev/null diff --git a/cmake/TargetCapnpSources.cmake b/cmake/TargetCapnpSources.cmake index f04a9657..2334e260 100644 --- a/cmake/TargetCapnpSources.cmake +++ b/cmake/TargetCapnpSources.cmake @@ -83,7 +83,7 @@ function(target_capnp_sources target include_prefix) add_custom_command( OUTPUT ${capnp_file}.c++ ${capnp_file}.h ${capnp_file}.proxy-client.c++ ${capnp_file}.proxy-types.h ${capnp_file}.proxy-server.c++ ${capnp_file}.proxy-types.c++ ${capnp_file}.proxy.h COMMAND ${MPGEN_BINARY} ${CMAKE_CURRENT_SOURCE_DIR} ${include_prefix} ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_file} ${TCS_IMPORT_PATHS} ${mp_include_dir} - DEPENDS ${capnp_file} + DEPENDS ${capnp_file} ${MPGEN_BINARY} VERBATIM ) # Skip linting for capnp-generated files but keep it for mpgen-generated ones diff --git a/cmake/compat_config.cmake b/cmake/compat_config.cmake index f9d3004f..32a9d6dd 100644 --- a/cmake/compat_config.cmake +++ b/cmake/compat_config.cmake @@ -15,45 +15,3 @@ endif() if (NOT DEFINED CAPNPC_OUTPUT_DIR) set(CAPNPC_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}") endif() - -# CMake target definitions for backwards compatibility with Ubuntu bionic -# capnproto 0.6.1 package (https://packages.ubuntu.com/bionic/libcapnp-dev) -# https://github.com/bitcoin-core/libmultiprocess/issues/27 - -if (NOT DEFINED CAPNP_LIB_CAPNPC AND DEFINED CAPNP_LIB_CAPNP-RPC) - string(REPLACE "-rpc" "c" CAPNP_LIB_CAPNPC "${CAPNP_LIB_CAPNP-RPC}") -endif() - -if (NOT DEFINED CapnProto_capnpc_IMPORTED_LOCATION AND DEFINED CapnProto_capnp-rpc_IMPORTED_LOCATION) - string(REPLACE "-rpc" "c" CapnProto_capnpc_IMPORTED_LOCATION "${CapnProto_capnp-rpc_IMPORTED_LOCATION}") -endif() - -if (NOT TARGET CapnProto::capnp AND DEFINED CAPNP_LIB_CAPNP) - add_library(CapnProto::capnp SHARED IMPORTED) - set_target_properties(CapnProto::capnp PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_CAPNP}") -endif() - -if (NOT TARGET CapnProto::capnpc AND DEFINED CAPNP_LIB_CAPNPC) - add_library(CapnProto::capnpc SHARED IMPORTED) - set_target_properties(CapnProto::capnpc PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_CAPNPC}") -endif() - -if (NOT TARGET CapnProto::capnpc AND DEFINED CapnProto_capnpc_IMPORTED_LOCATION) - add_library(CapnProto::capnpc SHARED IMPORTED) - set_target_properties(CapnProto::capnpc PROPERTIES IMPORTED_LOCATION "${CapnProto_capnpc_IMPORTED_LOCATION}") -endif() - -if (NOT TARGET CapnProto::capnp-rpc AND DEFINED CAPNP_LIB_CAPNP-RPC) - add_library(CapnProto::capnp-rpc SHARED IMPORTED) - set_target_properties(CapnProto::capnp-rpc PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_CAPNP-RPC}") -endif() - -if (NOT TARGET CapnProto::kj AND DEFINED CAPNP_LIB_KJ) - add_library(CapnProto::kj SHARED IMPORTED) - set_target_properties(CapnProto::kj PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_KJ}") -endif() - -if (NOT TARGET CapnProto::kj-async AND DEFINED CAPNP_LIB_KJ-ASYNC) - add_library(CapnProto::kj-async SHARED IMPORTED) - set_target_properties(CapnProto::kj-async PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_KJ-ASYNC}") -endif() diff --git a/cmake/iwyu-wrapper.sh b/cmake/iwyu-wrapper.sh new file mode 100755 index 00000000..fa2fd1a0 --- /dev/null +++ b/cmake/iwyu-wrapper.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Run include-what-you-use for CMAKE_CXX_INCLUDE_WHAT_YOU_USE and make sure its +# output is shown when it fails. Usage: iwyu-wrapper.sh IWYU [ARGS...] +# +# "cmake -E __run_co_compile" only prints IWYU output that contains "should add +# these lines:" or "should remove these lines:" (HandleIWYU in CMake's +# Source/cmcmd.cxx). If IWYU fails for another reason, such as a clang error, +# the build stops with a bare "Error 1". In that case this script appends a line +# containing the marker text, so CMake prints the clang diagnostics too. +# +# This works around https://gitlab.kitware.com/cmake/cmake/-/issues/28020 and +# can be removed once CMake prints IWYU output whenever IWYU fails. +set -o nounset +iwyu=$1 +shift +out=$("$iwyu" "$@" 2>&1) +ret=$? +printf '%s\n' "$out" >&2 +if [ "$ret" -ne 0 ] && ! grep -q -e 'should add these lines:' -e 'should remove these lines:' <<<"$out"; then + printf 'include-what-you-use failed with exit status %s before suggesting includes (marker for CMake: "should add these lines:"); see the diagnostics above.\n' "$ret" >&2 +fi +exit "$ret" diff --git a/cmake/pthread_checks.cmake b/cmake/pthread_checks.cmake index 87711416..1208391b 100644 --- a/cmake/pthread_checks.cmake +++ b/cmake/pthread_checks.cmake @@ -19,6 +19,14 @@ check_cxx_source_compiles(" }" HAVE_PTHREAD_GETNAME_NP) +check_cxx_source_compiles(" + #include + int main(int argc, char** argv) + { + return pthread_setname_np(pthread_self(), \"%s\", (void*)\"name\"); + }" + HAVE_PTHREAD_SETNAME_NP_3ARG) + check_cxx_source_compiles(" #include #include diff --git a/doc/design.md b/doc/design.md index 319ee3de..0eb4ccfc 100644 --- a/doc/design.md +++ b/doc/design.md @@ -134,7 +134,7 @@ sequenceDiagram SF1->>serverInvoke: return ``` -Destroy methods are a special case: a capnp interface can declare a `destroy` method that is handled by `ServerDestroy` instead of `ServerCall`. Rather than dispatching through the `ServerField`/`ServerRet`/`ServerCall` chain to a C++ interface method, `ServerDestroy` calls `invokeDestroy()` on the `ProxyServer`, which resets `m_impl` and runs any registered cleanup functions, giving the client a way to synchronously destroy the wrapped object on the server side. +Destroy methods are a special case: a capnp interface can declare a `destroy` method that is handled by `ServerDestroy` instead of `ServerCall`. Rather than dispatching through the `ServerField`/`ServerRet`/`ServerCall` chain to a C++ interface method, `ServerDestroy` calls `invokeDestroy()` on the `ProxyServer`, which resets `m_impl` and runs any registered cleanup functions, giving the client a way to synchronously destroy the wrapped object on the server side. The destroy request fails if other requests on the object are running, and requests sent after it fail. Clients that use Cap'n Proto directly should drop the capability instead. ## Advanced Features diff --git a/doc/install.md b/doc/install.md index ac2f7f6a..015cfd19 100644 --- a/doc/install.md +++ b/doc/install.md @@ -1,6 +1,6 @@ # libmultiprocess Installation -Installation currently requires Cap'n Proto 0.7 or higher: +Installation currently requires Cap'n Proto 0.9 or higher: ```sh apt install libcapnp-dev capnproto diff --git a/doc/usage.md b/doc/usage.md index df103e9f..6b294ce1 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -12,6 +12,12 @@ The `*.capnp` data definition files are consumed by the _libmultiprocess_ code g The `ProxyServer` objects help translate IPC requests from a socket to method calls on a local object. The `ProxyServer` objects are just used internally by the `mp::ServeStream(loop, socket, wrapped_object)` and `mp::ListenConnections(loop, socket, wrapped_object[, max_connections])` functions, and aren't exposed externally. The `ProxyClient` classes are exposed, and returned from the `mp::ConnectStream(loop, socket)` function and meant to be used directly. The classes implement methods described in `.capnp` definitions, and whenever any method is called, a request with the method arguments is sent over the associated IPC connection, and the corresponding `wrapped_object` method on the other end of the connection is called, with the `ProxyClient` method blocking until it returns and forwarding back any return value to the `ProxyClient` method caller. +## Event loop + +Making connections and IPC calls requires an `mp::EventLoop` instance running in its own dedicated thread. The event loop is reference-counted, and `EventLoop::loop()` returns when the last `mp::EventLoopRef` is released. Therefore, any code that needs to use the loop across multiple calls should maintain its own `EventLoopRef`. Connection objects and proxy objects automatically hold a reference for as long as they exist. + +For a typical setup, see [example.cpp](../example/example.cpp). + ## Example A simple interface description can be found at [test/mp/test/foo.capnp](../test/mp/test/foo.capnp), implementation in [test/mp/test/foo.h](../test/mp/test/foo.h), and usage in [test/mp/test/test.cpp](../test/mp/test/test.cpp). diff --git a/example/example.cpp b/example/example.cpp index c6b32f0d..2ea13861 100644 --- a/example/example.cpp +++ b/example/example.cpp @@ -10,9 +10,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -55,6 +55,7 @@ int main(int argc, char** argv) loop.loop(); }); mp::EventLoop* loop = promise.get_future().get(); + mp::EventLoopRef loop_ref{*loop}; auto [printer_init, printer_pid] = Spawn(*loop, argv[0], "mpprinter"); auto [calc_init, calc_pid] = Spawn(*loop, argv[0], "mpcalculator"); @@ -71,6 +72,7 @@ int main(int argc, char** argv) mp::WaitProcess(calc_pid); printer_init.reset(); mp::WaitProcess(printer_pid); + loop_ref.reset(); loop_thread.join(); std::cout << "Bye!" << std::endl; return 0; diff --git a/include/mp/config.h.in b/include/mp/config.h.in index 9d3c6240..b4ca6ded 100644 --- a/include/mp/config.h.in +++ b/include/mp/config.h.in @@ -10,6 +10,7 @@ #cmakedefine HAVE_KJ_FILESYSTEM #cmakedefine HAVE_PTHREAD_GETNAME_NP @HAVE_PTHREAD_GETNAME_NP@ +#cmakedefine HAVE_PTHREAD_SETNAME_NP_3ARG @HAVE_PTHREAD_SETNAME_NP_3ARG@ #cmakedefine HAVE_PTHREAD_THREADID_NP @HAVE_PTHREAD_THREADID_NP@ #cmakedefine HAVE_PTHREAD_GETTHREADID_NP @HAVE_PTHREAD_GETTHREADID_NP@ diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index c2a8d433..c7d08673 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -12,6 +12,7 @@ #include +#include #include #include #include @@ -233,8 +234,8 @@ Stream MakeStream(EventLoop&loop, SocketId socket); //! assumes it will only be accessed from one thread. So all this code needs to //! actually run on one thread, and the EventLoop::loop() method is the entry point for //! this thread. ProxyClient and ProxyServer objects that use other threads and -//! need to perform I/O operations post to this thread using EventLoop::post() -//! and EventLoop::sync() methods. +//! need to perform I/O operations post to this thread using the +//! EventLoop::sync() method. //! //! Specifically, because ProxyClient methods can be called from arbitrary //! threads, and ProxyServer methods can run on arbitrary threads, ProxyClient @@ -267,16 +268,7 @@ class EventLoop //! Run function on event loop thread. Does not return until function completes. //! Must be called while the loop() function is active. - void post(kj::Function fn); - - //! Wrapper around EventLoop::post that takes advantage of the - //! fact that callable will not go out of scope to avoid requirement that it - //! be copyable. - template - void sync(Callable&& callable) - { - post(std::forward(callable)); - } + void sync(kj::FunctionParam fn); //! Register cleanup function to run on asynchronous worker thread without //! blocking the event loop thread. @@ -309,8 +301,8 @@ class EventLoop //! method has not been called. std::thread m_async_thread; - //! Callback function to run on event loop thread during post() or sync() call. - kj::Function* m_post_fn MP_GUARDED_BY(m_mutex) = nullptr; + //! Callback function to run on event loop thread during sync() call. + kj::FunctionParam* m_sync_fn MP_GUARDED_BY(m_mutex) = nullptr; //! Callback functions to run on async thread. std::optional m_async_fns MP_GUARDED_BY(m_mutex); @@ -374,6 +366,12 @@ class EventLoop //! Hook called on the event loop thread when a client has disconnected. std::function testing_hook_disconnected; + + //! Miscellaneous testing hook. Called from various places with an + //! argument identifying the call site (typically a string literal), so + //! tests can control timing or inject behavior at specific points without + //! requiring a dedicated hook for each one. + std::function testing_hook_misc; }; //! Single element task queue used to handle recursive capnp calls. (If the @@ -592,7 +590,28 @@ ProxyClientBase::ProxyClientBase(typename Interface::Client cli }); } }); - Sub::construct(*this); + // If construct() fails, run the cleanup functions before rethrowing, + // because ~ProxyClientBase will not run for an object whose constructor + // threw, and the connection would otherwise be leaked. + try { + Sub::construct(*this); + } catch (...) { + MP_LOG(*m_context.loop, Log::Debug) << "Cleaning up " << CxxTypeName(*this) << " " << this << " after construct() failure"; + CleanupRun(m_context.cleanup_fns); + throw; + } + + // If this client owns the connection, delete the connection on disconnect. + if (destroy_connection) { + m_context.loop->sync([&] { + EventLoop& loop = *m_context.loop; + Connection* connection = m_context.connection; + connection->onDisconnect([&loop, connection] { + MP_LOG(loop, Log::Warning) << "IPC client: unexpected network disconnect."; + delete connection; + }); + }); + } } template @@ -673,10 +692,23 @@ ProxyServerBase::~ProxyServerBase() //! Context.thread parameter was passed, this method will run on the worker //! thread specified by the client. Otherwise it will run on the EventLoop //! thread, like other server methods without an assigned thread. +//! +//! Non-libmultiprocess clients can send destroy while other requests are +//! running, so it is rejected in that case. template void ProxyServerBase::invokeDestroy() { - m_impl.reset(); + // A destroy request with an mp.Context runs on a worker thread and is + // counted in m_active_calls itself. + const int self_count{std::this_thread::get_id() == m_context.loop->m_thread_id ? 0 : 1}; + // Move m_impl out on the event loop thread, where serverInvoke checks it, + // and destroy it after returning to this thread. + std::shared_ptr impl; + m_context.loop->sync([&] { + if (m_active_calls == self_count) impl = std::move(m_impl); + }); + if (!impl) throw std::runtime_error("destroy called while other calls are in progress"); + impl.reset(); CleanupRun(m_context.cleanup_fns); } @@ -694,7 +726,7 @@ using ConnThread = ConnThreads::iterator; // inserted bool. std::tuple SetThread(GuardedRef threads, Connection* connection, const std::function& make_thread); -//! The thread_local ThreadContext g_thread_context struct provides information +//! The thread_local ThreadContext struct (see CurrentThread()) provides information //! about individual threads and a way of communicating between them. Because //! it's a thread local struct, each ThreadContext instance is initialized by //! the thread that owns it. @@ -832,6 +864,10 @@ kj::Promise ProxyServer::post(Fn&& fn) //! Given a stream, make a new ProxyClient object to send requests over it. //! Also create a new Connection object embedded in the client that is freed //! when the client is closed. +//! +//! If the init interface declares a construct() method, creating the client +//! calls it, so this function may block making an IPC call and may throw if +//! the call fails. template std::unique_ptr> ConnectStream(EventLoop& loop, Stream stream) { @@ -840,11 +876,6 @@ std::unique_ptr> ConnectStream(EventLoop& loop, Strea loop.sync([&] { connection = std::make_unique(loop, kj::mv(stream)); init_client = connection->m_rpc_system->bootstrap(ServerVatId().vat_id).castAs(); - Connection* connection_ptr = connection.get(); - connection->onDisconnect([&loop, connection_ptr] { - MP_LOG(loop, Log::Warning) << "IPC client: unexpected network disconnect."; - delete connection_ptr; - }); }); return std::make_unique>( kj::mv(init_client), connection.release(), /* destroy_connection= */ true); @@ -934,6 +965,26 @@ extern thread_local ThreadContext g_thread_context; // NOLINT(bitcoin-nontrivial // cannot be thread_local" which should not be a problem on modern platforms, and // could lead to a small memory leak at worst on older ones. +//! Return the current thread's ThreadContext. +//! +//! Why per-thread state is needed at all: libmultiprocess has no control over +//! which threads the C++ application uses to call ProxyClient methods after +//! the proxy objects are returned to it. The thread-mapping model (see +//! "Thread Mapping" in doc/design.md) gives each application thread making +//! IPC calls a dedicated server-side thread that executes its requests, so +//! thread-local state and recursive mutexes work as expected across the +//! process boundary and callbacks from the server run on the originating +//! client thread. The client-side handles for those dedicated server threads +//! (the ProxyClient objects returned by ThreadMap.makeThread, stored +//! per connection in the request_threads / callback_threads maps below) are +//! state that must be keyed implicitly by the calling thread, and must be +//! released when the client thread exits so the corresponding server threads +//! are freed. A thread_local object is the C++ mechanism that provides both +//! of these: per-thread storage plus a destructor that runs at thread exit +//! (the C equivalent would be a pthread key destructor). This is why +//! ThreadContext is thread_local and why its destructor is nontrivial. +ThreadContext& CurrentThread(); + } // namespace mp #endif // MP_PROXY_IO_H diff --git a/include/mp/proxy-types.h b/include/mp/proxy-types.h index f1127da5..2a2ce593 100644 --- a/include/mp/proxy-types.h +++ b/include/mp/proxy-types.h @@ -13,6 +13,12 @@ #include #include +// KJ exception support is required, or exceptions thrown in serverInvoke below go uncaught +// and crash the process instead of being returned to the client as errors. +#if defined(KJ_NO_EXCEPTIONS) && KJ_NO_EXCEPTIONS +#error "KJ_NO_EXCEPTIONS=1 is set but libmultiprocess requires support for exceptions. Please check build settings. On NetBSD you may also need to set -DKJ_NO_EXCEPTIONS=0 -DKJ_NO_RTTI=0 explicitly (https://github.com/capnproto/capnproto/pull/2756)" +#endif + namespace mp { template @@ -220,7 +226,7 @@ void ThrowField(TypeList, InvokeContext& invoke_context, Input&& inpu { ReadField( TypeList(), invoke_context, input, ReadDestEmplace(TypeList(), - [](auto&& ...args) -> const LocalType& { throw LocalType{std::forward(args)...}; })); + [] [[noreturn]] (auto&& ...args) -> const LocalType& { throw LocalType{std::forward(args)...}; })); } //! Special case for generic std::exception. It's an abstract type so it can't @@ -285,9 +291,28 @@ void BuildList(TypeList, InvokeContext& invoke_context, Output&& outp { auto list = output.init(value.size()); size_t i = 0; - for (const auto& elem : value) { - BuildField(TypeList(), invoke_context, ListOutput(list, i), elem); - ++i; + // Iterate with an explicit iterator rather than a range-for loop so the + // value category of `*it` is passed through to BuildField unchanged. This + // matters for two reasons: + // + // - Elements must be passed as non-const so that BuildField can move out of + // them, e.g. calling unique_ptr::release() to transfer ownership of an + // interface pointer to the capnp server. + // + // - For proxy containers like std::vector, `*it` is a prvalue proxy + // object rather than a reference. A range-for loop would bind it to a + // named variable and demote it to an lvalue; passing `*it` directly + // preserves the prvalue-ness. + // + // Only move out of elements when `value` itself is an rvalue container that + // is about to be destroyed. When it is an lvalue reference owned by the + // caller, pass elements as lvalues so BuildField does not move from them. + for (auto it = value.begin(); it != value.end(); ++it, ++i) { + if constexpr (std::is_lvalue_reference_v) { + BuildField(TypeList(), invoke_context, ListOutput(list, i), *it); + } else { + BuildField(TypeList(), invoke_context, ListOutput(list, i), std::move(*it)); + } } } @@ -627,7 +652,7 @@ template auto PassField(Priority<2>, Args&&... args) -> decltype(CustomPassField(std::forward(args)...)) { return CustomPassField(std::forward(args)...); -}; +} template struct ServerField : Parent @@ -691,9 +716,9 @@ void serverDestroy(Server& server) template void clientInvoke(ProxyClient& proxy_client, const GetRequest& get_request, FieldObjs&&... fields) { - if (!g_thread_context.waiter) { - assert(g_thread_context.thread_name.empty()); - g_thread_context.thread_name = ThreadName(proxy_client.m_context.loop->m_exe_name); + if (!CurrentThread().waiter) { + assert(CurrentThread().thread_name.empty()); + CurrentThread().thread_name = ThreadName(proxy_client.m_context.loop->m_exe_name); // If next assert triggers, it means clientInvoke is being called from // the capnp event loop thread. This can happen when a ProxyServer // method implementation that runs synchronously on the event loop @@ -702,13 +727,13 @@ void clientInvoke(ProxyClient& proxy_client, const GetRequest& get_request, Fiel // run asynchronously off the event loop thread. This is easy to fix by // just adding a 'context :Proxy.Context' argument to the capnp method // declaration so the server method runs in a dedicated thread. - assert(!g_thread_context.loop_thread); - g_thread_context.waiter = std::make_unique(); + assert(!CurrentThread().loop_thread); + CurrentThread().waiter = std::make_unique(); MP_LOGPLAIN(*proxy_client.m_context.loop, Log::Info) - << "{" << g_thread_context.thread_name + << "{" << CurrentThread().thread_name << "} IPC client first request from current thread, constructing waiter"; } - ThreadContext& thread_context{g_thread_context}; + ThreadContext& thread_context{CurrentThread()}; std::optional invoke_context; // Must outlive waiter->wait() call below std::exception_ptr exception; std::string kj_exception; @@ -809,6 +834,10 @@ kj::Promise serverInvoke(Server& server, CallContext& call_context, Fn fn) MP_LOG(loop, Log::Trace) << "request data: " << LogEscape(params.toString(), server.m_context.loop->m_log_opts.max_chars); + if (!server.m_impl) { + MP_LOG(loop, Log::Error) << "IPC server error request #" << req << ", method called after destroy"; + throw std::runtime_error("method called after destroy"); + } try { using ServerContext = ServerInvokeContext; using ArgList = typename ProxyClientMethodTraits::Params; diff --git a/include/mp/proxy.h b/include/mp/proxy.h index 8d63e9aa..eef6962b 100644 --- a/include/mp/proxy.h +++ b/include/mp/proxy.h @@ -44,7 +44,9 @@ inline void CleanupRun(CleanupList& fns) { } } -//! Event loop smart pointer automatically managing m_num_refs. +//! Event loop smart pointer automatically managing m_num_refs. Hold one +//! reference to keep the loop running, once the last reference is released +//! EventLoop::loop() will exit. //! If a lock pointer argument is passed, the specified lock will be used, //! otherwise EventLoop::m_mutex will be locked when needed. class EventLoopRef @@ -170,6 +172,12 @@ struct ProxyServerBase : public virtual Interface_::Server * wrapped. */ std::shared_ptr m_impl; ProxyContext m_context; + //! Number of requests queued or executing on worker threads. + //! Incremented in the mp.Context PassField overload before being posted and + //! decremented when completed or dropped. Requests executing on the + //! event-loop thread are excluded because they run to completion without + //! allowing another request to run. + int m_active_calls{0}; }; //! Customizable (through template specialization) base class which ProxyServer diff --git a/include/mp/type-char.h b/include/mp/type-char.h index b51ffeb1..f4023334 100644 --- a/include/mp/type-char.h +++ b/include/mp/type-char.h @@ -7,6 +7,10 @@ #include +#include +#include +#include + namespace mp { template void CustomBuildField(TypeList, @@ -16,7 +20,7 @@ void CustomBuildField(TypeList, Output&& output) { auto result = output.init(size); - memcpy(result.begin(), value, size); + std::ranges::copy(value, result.begin()); } template @@ -26,9 +30,11 @@ decltype(auto) CustomReadField(TypeList, Input&& input, ReadDest&& read_dest) { + auto data = input.get(); + if (data.size() != size) throw std::range_error("unexpected field size"); + return read_dest.update([&](auto& value) { - auto data = input.get(); - memcpy(value, data.begin(), size); + std::ranges::copy(input.get(), std::ranges::begin(value)); }); } } // namespace mp diff --git a/include/mp/type-chrono.h b/include/mp/type-chrono.h index d7154986..94b0f1f4 100644 --- a/include/mp/type-chrono.h +++ b/include/mp/type-chrono.h @@ -16,9 +16,9 @@ template void CustomBuildField(TypeList>, Priority<1>, InvokeContext& invoke_context, Value&& value, Output&& output) { - static_assert(std::numeric_limits::lowest() <= std::numeric_limits::lowest(), + static_assert(safe_less_equal(std::numeric_limits::lowest(), std::numeric_limits::lowest()), "capnp type does not have enough range to hold lowest std::chrono::duration value"); - static_assert(std::numeric_limits::max() >= std::numeric_limits::max(), + static_assert(safe_greater_equal(std::numeric_limits::max(), std::numeric_limits::max()), "capnp type does not have enough range to hold highest std::chrono::duration value"); output.set(value.count()); } @@ -29,6 +29,29 @@ decltype(auto) CustomReadField(TypeList>, Pri { return read_dest.construct(input.get()); } + +//! Overload CustomBuildField and CustomReadField to serialize +//! std::chrono::time_point parameters and return values as integer tick counts. +//! The capnp field type must be an integer type with enough range to hold the +//! time_since_epoch() count for the given Duration (e.g. Int64 for nanoseconds). +template +void CustomBuildField(TypeList>, Priority<1>, InvokeContext& invoke_context, + Value&& value, Output&& output) +{ + using Rep = typename Duration::rep; + static_assert(safe_less_equal(std::numeric_limits::lowest(), std::numeric_limits::lowest()), + "capnp type does not have enough range to hold lowest std::chrono::time_point value"); + static_assert(safe_greater_equal(std::numeric_limits::max(), std::numeric_limits::max()), + "capnp type does not have enough range to hold highest std::chrono::time_point value"); + output.set(value.time_since_epoch().count()); +} + +template +decltype(auto) CustomReadField(TypeList>, Priority<1>, + InvokeContext& invoke_context, Input&& input, ReadDest&& read_dest) +{ + return read_dest.construct(std::chrono::time_point{Duration{input.get()}}); +} } // namespace mp #endif // MP_PROXY_TYPE_CHRONO_H diff --git a/include/mp/type-context.h b/include/mp/type-context.h index b63959f4..2407b29a 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -64,6 +64,8 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& auto& server = server_context.proxy_server; EventLoop& loop = *server.m_context.loop; int req = server_context.req; + ++server.m_active_calls; + auto active_call = std::shared_ptr{nullptr, [&server](void*) { --server.m_active_calls; }}; // Keep a reference to the ProxyServer instance by assigning it to the self // variable. ProxyServer instances are reference-counted and if the client // drops its reference and the IPC call is canceled, this variable keeps the @@ -71,7 +73,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // needs to be destroyed on the event loop thread so it is freed in a sync() // call below. auto self = server.thisCap(); - auto invoke = [self = kj::mv(self), call_context = kj::mv(server_context.call_context), &server, &loop, req, fn, args...](CancelMonitor& cancel_monitor) mutable { + auto invoke = [self = kj::mv(self), active_call = kj::mv(active_call), call_context = kj::mv(server_context.call_context), &server, &loop, req, fn, args...](CancelMonitor& cancel_monitor) mutable { MP_LOG(loop, Log::Debug) << "IPC server executing request #" << req; if (loop.testing_hook_async_request_start) loop.testing_hook_async_request_start(); KJ_DEFER(if (loop.testing_hook_async_request_done) loop.testing_hook_async_request_done()); @@ -93,7 +95,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // call. In this case, the callbackThread value should point // to the same thread already in the map, so there is no // need to update the map. - auto& thread_context = g_thread_context; + auto& thread_context = CurrentThread(); auto& request_threads = thread_context.request_threads; ConnThread request_thread; bool inserted{false}; @@ -131,6 +133,18 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& std::tie(request_thread, inserted) = SetThread( GuardedRef{thread_context.waiter->m_mutex, request_threads}, server.m_context.connection, [&] { return Accessor::get(call_context.getParams()).getCallbackThread(); }); + // Initialize the request's results struct here on the event loop + // thread, so later getResults() calls on the execution thread + // return the response capnp caches on first use instead of + // reading Cap'n Proto connection state, which is unsafe off the + // event loop thread because RpcConnectionState::disconnect() + // overwrites it there without synchronization on an abrupt remote + // disconnect (bitcoin-core/libmultiprocess#348). After this call, + // the only call_context state accessed by the execution thread is + // the params reader and the results struct allocated here; any + // future change accessing other call_context state needs to move + // that access to the event loop thread the same way. + call_context.getResults(); }); // If an entry was inserted into the request_threads map, @@ -164,6 +178,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // scope when it happens. cancel_monitor.m_on_cancel = nullptr; auto self_dispose{kj::mv(self)}; + auto active_call_dispose{kj::mv(active_call)}; if (erase_thread) { // Look up the thread again without using existing // iterator since entry may no longer be there after diff --git a/include/mp/type-data.h b/include/mp/type-data.h index 09c93df9..9110be91 100644 --- a/include/mp/type-data.h +++ b/include/mp/type-data.h @@ -32,9 +32,8 @@ template void CustomBuildField(TypeList, Priority<2>, InvokeContext& invoke_context, Value&& value, Output&& output) requires (std::is_same_v && IsByteSpan) { - auto data = std::span{value}; - auto result = output.init(data.size()); - std::ranges::copy(data, result.begin()); + auto result = output.init(value.size()); + std::ranges::copy(value, result.begin()); } template diff --git a/include/mp/type-function.h b/include/mp/type-function.h index 951e0885..fade9b42 100644 --- a/include/mp/type-function.h +++ b/include/mp/type-function.h @@ -61,7 +61,7 @@ decltype(auto) CustomReadField(TypeList> types, return read_dest.construct(ProxyCallFn{std::move(client)}); } return read_dest.construct(); -}; +} } // namespace mp #endif // MP_PROXY_TYPE_FUNCTION_H diff --git a/include/mp/type-map.h b/include/mp/type-map.h index e00a7813..ccfefb87 100644 --- a/include/mp/type-map.h +++ b/include/mp/type-map.h @@ -9,6 +9,8 @@ #include #include +#include + namespace mp { template void CustomBuildField(TypeList>, diff --git a/include/mp/type-number.h b/include/mp/type-number.h index 380d70c9..7ad21e3a 100644 --- a/include/mp/type-number.h +++ b/include/mp/type-number.h @@ -6,6 +6,7 @@ #define MP_PROXY_TYPE_NUMBER_H #include +#include namespace mp { template @@ -16,20 +17,28 @@ LocalType BuildPrimitive(InvokeContext& invoke_context, { using E = std::make_unsigned_t>; using T = std::make_unsigned_t; - static_assert(std::numeric_limits::max() >= std::numeric_limits::max(), "mismatched integral/enum types"); + static_assert(std::cmp_greater_equal(std::numeric_limits::max(), std::numeric_limits::max()), "mismatched integral/enum types"); return static_cast(value); } template - requires std::is_integral_v + requires (std::is_integral_v && !std::is_same_v, bool>) LocalType BuildPrimitive(InvokeContext& invoke_context, const Value& value, TypeList) { static_assert( - std::numeric_limits::lowest() <= std::numeric_limits::lowest(), "mismatched integral types"); + std::cmp_less_equal(std::numeric_limits::lowest(), std::numeric_limits::lowest()), "mismatched integral types"); static_assert( - std::numeric_limits::max() >= std::numeric_limits::max(), "mismatched integral types"); + std::cmp_greater_equal(std::numeric_limits::max(), std::numeric_limits::max()), "mismatched integral types"); + return value; +} + +template +LocalType BuildPrimitive(InvokeContext& invoke_context, bool value, TypeList) +{ + static_assert(std::is_same_v, + "capnp field type should be Bool for bool parameters. Fix the .capnp schema."); return value; } diff --git a/include/mp/type-string.h b/include/mp/type-string.h index d4c3383b..d81d59ec 100644 --- a/include/mp/type-string.h +++ b/include/mp/type-string.h @@ -7,6 +7,8 @@ #include +#include + namespace mp { template void CustomBuildField(TypeList, @@ -16,7 +18,7 @@ void CustomBuildField(TypeList, Output&& output) { auto result = output.init(value.size()); - memcpy(result.begin(), value.data(), value.size()); + std::ranges::copy(value, result.begin()); } template diff --git a/include/mp/util.h b/include/mp/util.h index f647d5d7..adc7cfbb 100644 --- a/include/mp/util.h +++ b/include/mp/util.h @@ -33,6 +33,21 @@ namespace mp { //! Generic utility functions used by capnp code. +// std::cmp_less_equal/cmp_greater_equal only accept integer types (C++20 §[utility.intcmp]). +// These wrappers fall back to regular comparison for floating-point types. +template +constexpr bool safe_less_equal(A a, B b) +{ + if constexpr (std::is_floating_point_v || std::is_floating_point_v) return a <= b; + else return std::cmp_less_equal(a, b); +} +template +constexpr bool safe_greater_equal(A a, B b) +{ + if constexpr (std::is_floating_point_v || std::is_floating_point_v) return a >= b; + else return std::cmp_greater_equal(a, b); +} + //! Type holding a list of types. //! //! Example: @@ -264,6 +279,9 @@ decltype(auto) TryFinally(Fn&& fn, After&& after) } } +//! Set the OS-level name of the current thread +void SetOsThreadName(const char* name); + //! Format current thread name as "{exe_name}-{$pid}/{thread_name}-{$tid}". std::string ThreadName(const char* exe_name); @@ -275,7 +293,7 @@ using Stream = kj::Own; using ProcessId = int; using SocketId = int; -constexpr SocketId SocketError{-1}; +inline constexpr SocketId SocketError{-1}; //! Information about parent process passed to child process as a command-line //! argument. On unix this is the child socket fd number formatted as a string. diff --git a/shell.nix b/shell.nix index 1a4614e3..01cf415c 100644 --- a/shell.nix +++ b/shell.nix @@ -5,6 +5,7 @@ , capnprotoVersion ? null , capnprotoSanitizers ? null # Optional sanitizers to build cap'n proto with , cmakeVersion ? null +, gccVersion ? null , libcxxSanitizers ? null # Optional LLVM_USE_SANITIZER value to use for libc++, see https://llvm.org/docs/CMake.html }: @@ -17,10 +18,6 @@ let }; }; capnprotoHashes = { - "0.7.0" = "sha256-Y/7dUOQPDHjniuKNRw3j8dG1NI9f/aRWpf8V0WzV9k8="; - "0.7.1" = "sha256-3cBpVmpvCXyqPUXDp12vCFCk32ZXWpkdOliNH37UwWE="; - "0.8.0" = "sha256-rfiqN83begjJ9eYjtr21/tk1GJBjmeVfa3C3dZBJ93w="; - "0.8.1" = "sha256-OZqNVYdyszro5rIe+w6YN00g6y8U/1b8dKYc214q/2o="; "0.9.0" = "sha256-yhbDcWUe6jp5PbIXzn5EoKabXiWN8lnS08hyfxUgEQ0="; "0.9.2" = "sha256-BspWOPZcP5nCTvmsDE62Zutox+aY5pw42d6hpH3v4cM="; "0.10.0" = "sha256-++F4l54OMTDnJ+FO3kV/Y/VLobKVRk461dopanuU3IQ="; @@ -53,12 +50,20 @@ let "-g" ]; }; - })).override (lib.optionalAttrs enableLibcxx { clangStdenv = llvm.libcxxStdenv; }); + })).override (lib.optionalAttrs enableLibcxx { clangStdenv = llvm.libcxxStdenv; } + # On i686, GCC and clang handle empty structs differently, causing an + # ABI mismatch and segfaulting in calls to functions like + # kj::str(kj::Exception). Prevent this by compiling capnproto with the + # same toolchain as libmultiprocess. + // lib.optionalAttrs crossPkgs.stdenv.hostPlatform.isx86_32 { clangStdenv = crossPkgs.stdenv; }); clang = if enableLibcxx then llvm.libcxxClang else llvm.clang; clang-tools = llvm.clang-tools.override { inherit enableLibcxx; }; cmakeHashes = { "3.12.4" = "sha256-UlVYS/0EPrcXViz/iULUcvHA5GecSUHYS6raqbKOMZQ="; + "4.1.1" = "sha256-sp9vGXM6oiS3djUHoQikJ+1Ixojh+vIrKcROHDBUkoI="; + "4.3.4" = "sha256-/e/4l7nrSddkU58rHtxut+FEDfMlZ4qXwZeEmekxrdo="; }; + gcc = if gccVersion == null then null else builtins.getAttr ("gcc" + gccVersion) pkgs; cmakeBuild = if cmakeVersion == null then pkgs.cmake else (pkgs.cmake.overrideAttrs (old: { version = cmakeVersion; src = pkgs.fetchurl { @@ -68,18 +73,25 @@ let patches = []; })).override { isMinimalBuild = true; }; in crossPkgs.mkShell { - buildInputs = [ + buildInputs = lib.optionals (capnprotoVersion != "none") [ capnproto ]; nativeBuildInputs = with pkgs; [ cmakeBuild + git include-what-you-use ninja - ] ++ lib.optionals (!minimal) [ + ] ++ lib.optional (gcc != null) gcc ++ lib.optionals (!minimal) [ clang clang-tools ]; + CC = if gcc == null then null else "${gcc}/bin/gcc"; + CXX = if gcc == null then null else "${gcc}/bin/g++"; + # Tell IWYU where its libc++ mapping lives IWYU_MAPPING_FILE = if enableLibcxx then "${llvm.libcxx.dev}/include/c++/v1/libcxx.imp" else null; + + # Avoid "SSL certificate problem: unable to get local issuer certificate" error during git clone in ci/scripts/ci.sh + NIX_SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; } diff --git a/src/mp/gen.cpp b/src/mp/gen.cpp index 435ba441..8d733a91 100644 --- a/src/mp/gen.cpp +++ b/src/mp/gen.cpp @@ -272,7 +272,7 @@ static void Generate(kj::StringPtr src_prefix, if (p != std::string::npos) include_base.erase(p); std::vector args; - args.emplace_back(capnp_PREFIX "/bin/capnp"); + args.emplace_back(CAPNP_EXECUTABLE); args.emplace_back("compile"); args.emplace_back("--src-prefix="); args.back().append(src_prefix.cStr(), src_prefix.size()); @@ -280,11 +280,11 @@ static void Generate(kj::StringPtr src_prefix, args.emplace_back("--import-path="); args.back().append(import_path.cStr(), import_path.size()); } - args.emplace_back("--output=" capnp_PREFIX "/bin/capnpc-c++"); + args.emplace_back("--output=" CAPNPC_CXX_EXECUTABLE); args.emplace_back(src_file); const int status = mp::WaitProcess(mp::StartProcess(args)); if (status) { - throw std::runtime_error("Invoking " capnp_PREFIX "/bin/capnp failed"); + throw std::runtime_error("Invoking " CAPNP_EXECUTABLE " failed"); } const capnp::SchemaParser parser; @@ -329,6 +329,7 @@ static void Generate(kj::StringPtr src_prefix, cpp_client << "#include \n"; cpp_client << "#include \n"; cpp_client << "#include \n"; + cpp_client << "#include \n"; cpp_client << "#include \n"; cpp_client << "#include \n"; cpp_client << "#include \n"; diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index a0a7aba0..552e3268 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -42,6 +42,11 @@ namespace mp { thread_local ThreadContext g_thread_context; // NOLINT(bitcoin-nontrivial-threadlocal) +ThreadContext& CurrentThread() +{ + return g_thread_context; +} + Stream MakeStream(EventLoop&loop, SocketId socket) { Stream stream; @@ -270,7 +275,7 @@ EventLoop::~EventLoop() { if (m_async_thread.joinable()) m_async_thread.join(); const Lock lock(m_mutex); - KJ_ASSERT(m_post_fn == nullptr); + KJ_ASSERT(m_sync_fn == nullptr); KJ_ASSERT(!m_async_fns); KJ_ASSERT(!m_wait_stream); KJ_ASSERT(!m_post_stream); @@ -283,9 +288,9 @@ EventLoop::~EventLoop() void EventLoop::loop() { - assert(!g_thread_context.loop_thread); - g_thread_context.loop_thread = true; - KJ_DEFER(g_thread_context.loop_thread = false); + assert(!CurrentThread().loop_thread); + CurrentThread().loop_thread = true; + KJ_DEFER(CurrentThread().loop_thread = false); { const Lock lock(m_mutex); @@ -299,18 +304,18 @@ void EventLoop::loop() const size_t read_bytes = wait_stream->read(&buffer, 0, 1).wait(m_io_context.waitScope); if (read_bytes != 1) throw std::logic_error("EventLoop wait_stream closed unexpectedly"); Lock lock(m_mutex); - if (m_post_fn) { - // m_post_fn throwing is never expected. If it does happen, the caller - // of EventLoop::post() will return without any indication of failure, + if (m_sync_fn) { + // m_sync_fn throwing is never expected. If it does happen, the caller + // of EventLoop::sync() will return without any indication of failure, // which will likely cause other bugs. Log the error and continue. - KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() MP_REQUIRES(m_mutex) { Unlock(lock, *m_post_fn); })) { - MP_LOG(*this, Log::Error) << "EventLoop: m_post_fn threw: " << kj::str(*exception).cStr(); + KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() MP_REQUIRES(m_mutex) { Unlock(lock, *m_sync_fn); })) { + MP_LOG(*this, Log::Error) << "EventLoop: m_sync_fn threw: " << kj::str(*exception).cStr(); } - m_post_fn = nullptr; + m_sync_fn = nullptr; m_cv.notify_all(); } else if (done()) { - // Intentionally do not break if m_post_fn was set, even if done() - // would return true, to ensure that the post() m_post_writer->write() + // Intentionally do not break if m_sync_fn was set, even if done() + // would return true, to ensure that the sync() m_post_writer->write() // call always succeeds and the loop does not exit between the time // that the done condition is set and the write call is made. break; @@ -328,7 +333,7 @@ void EventLoop::loop() m_cv.notify_all(); } -void EventLoop::post(kj::Function fn) +void EventLoop::sync(kj::FunctionParam fn) { if (std::this_thread::get_id() == m_thread_id) { fn(); @@ -336,13 +341,13 @@ void EventLoop::post(kj::Function fn) } Lock lock(m_mutex); EventLoopRef ref(*this, &lock); - m_cv.wait(lock.m_lock, [this]() MP_REQUIRES(m_mutex) { return m_post_fn == nullptr; }); - m_post_fn = &fn; + m_cv.wait(lock.m_lock, [this]() MP_REQUIRES(m_mutex) { return m_sync_fn == nullptr; }); + m_sync_fn = &fn; Unlock(lock, [&] { char buffer = 0; m_post_writer->write(&buffer, 1); }); - m_cv.wait(lock.m_lock, [this, &fn]() MP_REQUIRES(m_mutex) { return m_post_fn != &fn; }); + m_cv.wait(lock.m_lock, [this, &fn]() MP_REQUIRES(m_mutex) { return m_sync_fn != &fn; }); } void EventLoop::startAsyncThread() @@ -353,6 +358,7 @@ void EventLoop::startAsyncThread() m_cv.notify_all(); } else if (!m_async_fns->empty()) { m_async_thread = std::thread([this] { + SetOsThreadName("capnp-async"); Lock lock(m_mutex); while (m_async_fns) { if (!m_async_fns->empty()) { @@ -478,12 +484,13 @@ kj::Promise ProxyServer::makePool(MakePoolContext context) for (uint32_t i = 0; i < count; ++i) { const std::string thread_name = "pool/" + std::to_string(i); std::promise thread_context; - std::thread thread([&loop, &thread_context, thread_name]() { - g_thread_context.thread_name = ThreadName(loop.m_exe_name) + " (" + thread_name + ")"; - g_thread_context.waiter = std::make_unique(); - Lock lock(g_thread_context.waiter->m_mutex); - thread_context.set_value(&g_thread_context); - g_thread_context.waiter->wait(lock, [] { return !g_thread_context.waiter; }); + std::thread thread([&loop, &thread_context, thread_name, i]() { + SetOsThreadName(("capnp-pool-" + std::to_string(i)).c_str()); + CurrentThread().thread_name = ThreadName(loop.m_exe_name) + " (" + thread_name + ")"; + CurrentThread().waiter = std::make_unique(); + Lock lock(CurrentThread().waiter->m_mutex); + thread_context.set_value(&CurrentThread()); + CurrentThread().waiter->wait(lock, [] { return !CurrentThread().waiter; }); }); auto thread_server = kj::heap>(m_connection, *thread_context.get_future().get(), std::move(thread)); m_connection.m_thread_pool.push_back({m_connection.m_threads.add(kj::mv(thread_server))}); @@ -498,14 +505,15 @@ kj::Promise ProxyServer::makeThread(MakeThreadContext context) const std::string from = context.getParams().getName(); std::promise thread_context; std::thread thread([&loop, &thread_context, from]() { - g_thread_context.thread_name = ThreadName(loop.m_exe_name) + " (from " + from + ")"; - g_thread_context.waiter = std::make_unique(); - Lock lock(g_thread_context.waiter->m_mutex); - thread_context.set_value(&g_thread_context); + SetOsThreadName("capnp-worker"); + CurrentThread().thread_name = ThreadName(loop.m_exe_name) + " (from " + from + ")"; + CurrentThread().waiter = std::make_unique(); + Lock lock(CurrentThread().waiter->m_mutex); + thread_context.set_value(&CurrentThread()); if (loop.testing_hook_makethread_created) loop.testing_hook_makethread_created(); // Wait for shutdown signal from ProxyServer destructor (signal // is just waiter getting set to null.) - g_thread_context.waiter->wait(lock, [] { return !g_thread_context.waiter; }); + CurrentThread().waiter->wait(lock, [] { return !CurrentThread().waiter; }); }); auto thread_server = kj::heap>(m_connection, *thread_context.get_future().get(), std::move(thread)); auto thread_client = m_connection.m_threads.add(kj::mv(thread_server)); @@ -517,7 +525,7 @@ std::atomic server_reqs{0}; std::string LongThreadName(const char* exe_name) { - return g_thread_context.thread_name.empty() ? ThreadName(exe_name) : g_thread_context.thread_name; + return CurrentThread().thread_name.empty() ? ThreadName(exe_name) : CurrentThread().thread_name; } kj::StringPtr KJ_STRINGIFY(Log v) diff --git a/src/mp/util.cpp b/src/mp/util.cpp index 42cda9ce..e1eae4dc 100644 --- a/src/mp/util.cpp +++ b/src/mp/util.cpp @@ -11,7 +11,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -33,6 +35,10 @@ #include #endif // HAVE_PTHREAD_GETTHREADID_NP +#if __has_include() +#include +#endif + extern "C" char **environ; // NOLINT(readability-redundant-declaration) namespace mp { @@ -60,8 +66,134 @@ size_t MaxFd() } } +//! Report an error and exit from the post-fork child of a multi-threaded +//! process, where only async-signal-safe calls (like write and _exit) are +//! allowed. Accepting only a reference to a char array (in practice a string +//! literal) ensures no allocation is needed at the call site. +template +[[noreturn]] void ChildFail(const char (&msg)[N]) noexcept +{ + const ssize_t written = ::write(STDERR_FILENO, msg, N - 1); + (void)written; + _exit(126); +} + +enum class SpawnErrorOp +{ + CLOSE, + EXECVP, + READ, + FCNTL, +}; + +struct SpawnError { + SpawnErrorOp which; + int err; +}; + +const char* SpawnErrorName(SpawnErrorOp which) +{ + switch (which) { + case SpawnErrorOp::CLOSE: return "close"; + case SpawnErrorOp::EXECVP: return "execvp"; + case SpawnErrorOp::READ: return "read"; + case SpawnErrorOp::FCNTL: return "fcntl"; + } + return "unknown"; +} + +// Read the child's error report. Returns nullopt on success, or a SpawnError to +// throw on failure. Success is a clean EOF: read() returns 0 with nothing +// buffered because the child's write end was closed by a successful exec (via +// FD_CLOEXEC). A fully-read struct is the failure the child reported. A read() +// error, or an EOF partway through the struct (the child died mid-report), is +// surfaced as a SpawnErrorOp::READ failure rather than being mistaken for +// success. A single read() is not guaranteed to return all sizeof(SpawnError) +// bytes (the socket is SOCK_STREAM, which has no message boundaries) and may be +// interrupted by a signal, so loop until the whole struct is read. +// +// Note that a clean EOF is also what happens if the child is killed before it +// reaches exec, so this function reports success in that case. There is no good +// way to distinguish the two here, but callers will still see the failure when +// they call WaitProcess and get the child's exit status. +std::optional ReadSpawnResult(int fd) +{ + SpawnError error{}; + char* buf = reinterpret_cast(&error); + size_t remaining = sizeof(error); + while (remaining > 0) { + const ssize_t n = ::read(fd, buf, remaining); + if (n < 0) { + if (errno == EINTR) continue; + return SpawnError{.which = SpawnErrorOp::READ, .err = errno}; + } + if (n == 0) { + if (remaining == sizeof(error)) return std::nullopt; // clean EOF: success + return SpawnError{.which = SpawnErrorOp::READ, .err = EPROTO}; // torn report + } + buf += n; + remaining -= static_cast(n); + } + return error; +} + +// Write the whole SpawnError to fd, retrying short writes and EINTR so the +// parent never sees a torn struct. Runs in the post-fork child, so it must stay +// async-signal-safe: it only calls write() and does not allocate or throw. This +// is best-effort -- if the write cannot complete there is nothing useful the +// child can do, so it stops and lets the caller _exit(). +void WriteSpawnError(int fd, const SpawnError& error) +{ + const char* buf = reinterpret_cast(&error); + size_t remaining = sizeof(error); + while (remaining > 0) { + const ssize_t n = ::write(fd, buf, remaining); + if (n < 0) { + if (errno == EINTR) continue; + break; + } + buf += n; + remaining -= static_cast(n); + } + if (remaining > 0) { + // The parent's read end is gone (e.g. the parent exited before the + // child could report), so the structured error can't be delivered. + // Leave a breadcrumb on stderr and exit. The exit code is irrelevant + // here since no live parent remains to wait on it. + ChildFail("SpawnProcess(child): failed and could not report error to parent\n"); + } +} + +// Get rid of a child process the parent is abandoning because SpawnProcess is +// about to throw, so it is not left behind as a zombie. The child may still be +// alive, so kill it first: waiting without that could block for as long as the +// spawned program runs. +void KillAndReapChild(ProcessId pid) +{ + (void)::kill(pid, SIGKILL); + while (::waitpid(pid, /*status=*/nullptr, /*options=*/0) == -1 && errno == EINTR) {} +} + } // namespace +// Copied from https://github.com/bitcoin/bitcoin/blob/d3e40af2597/src/util/threadnames.cpp#L21-L36 +void SetOsThreadName(const char* name) +{ +#if defined(PR_SET_NAME) + // Only the first 15 characters are used (16 - NUL terminator) + ::prctl(PR_SET_NAME, name, 0, 0, 0); +#elif defined(HAVE_PTHREAD_SETNAME_NP_3ARG) + pthread_setname_np(pthread_self(), "%s", const_cast(name)); +#elif defined(HAVE_PTHREAD_GETTHREADID_NP) + pthread_set_name_np(pthread_self(), name); +#elif defined(__APPLE__) + pthread_setname_np(name); +#else + // Prevent warnings for unused parameters... + (void)name; +#endif +} + std::string ThreadName(const char* exe_name) { char thread_name[16] = {0}; @@ -120,6 +252,9 @@ std::string LogEscape(const kj::StringTree& string, size_t max_size) std::tuple SpawnProcess(SpawnConnectInfoToArgsFn&& connect_info_to_args) { auto fds{SocketPair()}; + // Only used for the child to report errors back to the parent, so a one-way + // pipe would be sufficient, but reuse the existing SocketPair() helper. + auto error_fds{SocketPair()}; // Evaluate the callback and build the argv array before forking. // @@ -130,47 +265,77 @@ std::tuple SpawnProcess(SpawnConnectInfoToArgsFn&& connect_ const std::vector args{connect_info_to_args(std::to_string(fds[0]))}; const std::vector argv{MakeArgv(args)}; - // Clear FD_CLOEXEC on fds[0] before forking so it survives exec in the child. - int fds0_flags; - KJ_SYSCALL(fds0_flags = fcntl(fds[0], F_GETFD)); - KJ_SYSCALL(fcntl(fds[0], F_SETFD, fds0_flags & ~FD_CLOEXEC)); - ProcessId pid = fork(); if (pid == -1) { - throw std::system_error(errno, std::system_category(), "fork"); + const int err = errno; + (void)close(fds[0]); + (void)close(fds[1]); + (void)close(error_fds[0]); + (void)close(error_fds[1]); + throw std::system_error(err, std::system_category(), "fork"); } // Parent process closes the descriptor for socket 0, child closes the // descriptor for socket 1. On failure, the parent throws, but the child // must _exit(126) (post-fork child must not throw). if (close(fds[pid ? 0 : 1]) != 0) { + const int err = errno; if (pid) { (void)close(fds[1]); - throw std::system_error(errno, std::system_category(), "close"); + (void)close(error_fds[0]); + (void)close(error_fds[1]); + KillAndReapChild(pid); + throw std::system_error(err, std::system_category(), "close"); } - static constexpr char msg[] = "SpawnProcess(child): close(fds[1]) failed\n"; - const ssize_t writeResult = ::write(STDERR_FILENO, msg, sizeof(msg) - 1); - (void)writeResult; + WriteSpawnError(error_fds[0], {.which = SpawnErrorOp::CLOSE, .err = err}); _exit(126); } if (!pid) { // Child process must close all potentially open descriptors, except - // socket 0. Do not throw, allocate, or do non-fork-safe work here. + // socket 0 and the error-reporting socket 0. Do not throw, allocate, or + // do non-fork-safe work here. const int maxFd = MaxFd(); for (int fd = 3; fd < maxFd; ++fd) { - if (fd != fds[0]) { + if (fd != fds[0] && fd != error_fds[0]) { close(fd); } } + // Clear FD_CLOEXEC on socket 0 so it survives exec. Doing this here + // rather than in the parent before forking avoids a window where a + // concurrent fork in another thread would leak the descriptor into an + // unrelated child process (which would not clear it). + // fcntl is async-signal-safe. + const int fds0_flags = fcntl(fds[0], F_GETFD); + if (fds0_flags == -1 || fcntl(fds[0], F_SETFD, fds0_flags & ~FD_CLOEXEC) == -1) { + const int err = errno; + WriteSpawnError(error_fds[0], {.which = SpawnErrorOp::FCNTL, .err = err}); + _exit(126); + } + execvp(argv[0], argv.data()); - // NOTE: perror() is not async-signal-safe; calling it here in a - // post-fork child may deadlock in multithreaded parents. - // TODO: Report errors to the parent via a pipe (e.g. write errno) - // so callers can get diagnostics without relying on perror(). - perror("execvp failed"); + + const int err = errno; + WriteSpawnError(error_fds[0], {.which = SpawnErrorOp::EXECVP, .err = err}); _exit(127); } + + // Close the parent's copy of the child's write end. + if (close(error_fds[0]) != 0) { + const int err = errno; + (void)close(error_fds[1]); + (void)close(fds[1]); + KillAndReapChild(pid); + throw std::system_error(err, std::system_category(), "close"); + } + + const std::optional error{ReadSpawnResult(error_fds[1])}; + (void)close(error_fds[1]); + if (error) { + (void)close(fds[1]); + KillAndReapChild(pid); + throw std::system_error(error->err, std::system_category(), SpawnErrorName(error->which)); + } return {pid, fds[1]}; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 13246293..fd0aa023 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -28,6 +28,7 @@ if(BUILD_TESTING AND TARGET CapnProto::kj-test) mp/test/foo.h mp/test/listen_tests.cpp mp/test/spawn_tests.cpp + mp/test/connect_tests.cpp mp/test/test.cpp ) include(${PROJECT_SOURCE_DIR}/cmake/TargetCapnpSources.cmake) diff --git a/test/mp/test/common.h b/test/mp/test/common.h new file mode 100644 index 00000000..527d4a2a --- /dev/null +++ b/test/mp/test/common.h @@ -0,0 +1,27 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_TEST_COMMON_H +#define MP_TEST_COMMON_H + +#include +#include + +#include + +namespace mp { +namespace test { + +//! Default event loop log handler used by tests. Logs all messages and throws +//! on errors so calling code can assert on them. +inline void DefaultLogHandler(LogMessage log) +{ + KJ_LOG(INFO, log.level, log.message); + if (log.level == Log::Raise) throw std::runtime_error(log.message); +} + +} // namespace test +} // namespace mp + +#endif // MP_TEST_COMMON_H diff --git a/test/mp/test/connect_tests.cpp b/test/mp/test/connect_tests.cpp new file mode 100644 index 00000000..4e4f051f --- /dev/null +++ b/test/mp/test/connect_tests.cpp @@ -0,0 +1,212 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "common.h" +#include "unixlistener.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mp { +namespace test { +namespace { + +constexpr auto FAILURE_TIMEOUT = std::chrono::seconds{30}; + +class TestSetup +{ +public: + EventLoop* m_loop; + std::optional m_loop_ref; + //! Thread variable should be after other struct members so the thread does + //! not start until the other members are initialized. + std::thread m_loop_thread; + + TestSetup(LogFn log_handler = DefaultLogHandler) + { + std::promise loop_promise; + m_loop_thread = std::thread([&, log_handler] { + EventLoop loop("mptest-connect", log_handler); + loop_promise.set_value(&loop); + loop.loop(); + }); + m_loop = loop_promise.get_future().get(); + m_loop_ref.emplace(*m_loop); + } + + ~TestSetup() + { + m_loop_ref.reset(); + m_loop_thread.join(); + } +}; + +KJ_TEST("ConnectStream connects to a socket serving a valid init interface") +{ + TestSetup setup; + auto [client_fd, server_fd] = SocketPair(); + + std::thread server_thread([&]() { + EventLoop server_loop("mptest-valid-server", DefaultLogHandler); + std::unique_ptr init = std::make_unique(); + ServeStream(server_loop, MakeStream(server_loop, server_fd), *init); + server_loop.loop(); + }); + + // FooInit has a `construct()` method, so this connects to the server and + // sends an IPC request that must complete successfully. + auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); + + init.reset(); + server_thread.join(); +} + +KJ_TEST("ConnectStream throws when the socket is already disconnected") +{ + TestSetup setup; + auto [client_fd, server_fd] = SocketPair(); + + KJ_SYSCALL(close(server_fd)); + + try { + auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); + + KJ_EXPECT(false); + } catch (const std::runtime_error& e) { + std::string_view reason = e.what(); + + KJ_EXPECT(reason == "IPC client method call interrupted by disconnect."); + } +} + +KJ_TEST("ConnectStream defers disconnect failure to the first IPC request for interfaces without construct()") +{ + TestSetup setup; + auto [client_fd, server_fd] = SocketPair(); + + KJ_SYSCALL(close(server_fd)); + + // Without a construct() method no IPC call is made during client + // creation, so ConnectStream succeeds even though the peer is gone. + auto foo = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); + + try { + foo->add(1, 2); + KJ_EXPECT(false); + } catch (const std::runtime_error& e) { + std::string_view reason = e.what(); + + // There is a race between the event loop detecting the disconnect + // and foo->add() being called. If the onDisconnect callback fires + // first and nulls m_context.connection, the error is "called after + // disconnect"; if foo->add() submits before the callback fires, + // the error is "interrupted by disconnect". + KJ_EXPECT(reason == "IPC client method called after disconnect." || + reason == "IPC client method call interrupted by disconnect."); + } +} + +KJ_TEST("ConnectStream handles a disconnect when no client calls are made") +{ + std::mutex mutex; + std::condition_variable cv; + bool warned = false; + + TestSetup setup([&](LogMessage log) { + if (log.level == Log::Warning && log.message.find("unexpected network disconnect") != std::string::npos) { + const std::lock_guard lock(mutex); + warned = true; + cv.notify_all(); + } + DefaultLogHandler(log); + }); + auto [client_fd, server_fd] = SocketPair(); + + KJ_SYSCALL(close(server_fd)); + + auto foo = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); + + // The disconnect handler registered by ProxyClientBase should run and + // delete the connection even when no calls are ever made. + std::unique_lock lock(mutex); + KJ_EXPECT(cv.wait_for(lock, FAILURE_TIMEOUT, [&] { return warned; })); +} + +KJ_TEST("ConnectStream throws when the socket disconnects after receiving data") +{ + TestSetup setup; + auto [client_fd, server_fd] = SocketPair(); + + std::thread server_thread([&]() { + char buf[128]; + + recv(server_fd, buf, sizeof(buf), 0); + KJ_SYSCALL(close(server_fd)); + }); + + try { + auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); + + KJ_EXPECT(false); + } catch (const std::runtime_error& e) { + std::string_view reason = e.what(); + KJ_EXPECT(reason == "IPC client method call interrupted by disconnect."); + } + server_thread.join(); +} + +KJ_TEST("ConnectStream throws when a connection accepted from a listener disconnects after receiving data") +{ + UnixListener listener; + TestSetup setup; + int client_fd = listener.MakeConnectedSocket(); + int server_fd = listener.release(); + + std::thread server_thread([&]() { + char buf[128]; + + int connection_fd = accept(server_fd, nullptr, nullptr); + + if (connection_fd >= 0) { + recv(connection_fd, buf, sizeof(buf), 0); + KJ_SYSCALL(close(connection_fd)); + } + KJ_SYSCALL(close(server_fd)); + }); + + try { + auto init = ConnectStream(*setup.m_loop, MakeStream(*setup.m_loop, client_fd)); + + KJ_EXPECT(false); + } catch (const std::runtime_error& e) { + std::string_view reason = e.what(); + KJ_EXPECT(reason == "IPC client method call interrupted by disconnect."); + } + server_thread.join(); +} + +} // namespace +} // namespace test +} // namespace mp diff --git a/test/mp/test/foo-types.h b/test/mp/test/foo-types.h index b96eabfc..85c36349 100644 --- a/test/mp/test/foo-types.h +++ b/test/mp/test/foo-types.h @@ -9,6 +9,7 @@ #include // IWYU pragma: begin_exports +#include #include #include #include @@ -39,6 +40,7 @@ struct ExtendedCallback; // IWYU pragma: export struct FooCallback; // IWYU pragma: export struct FooFn; // IWYU pragma: export struct FooInterface; // IWYU pragma: export +struct BarInterface; // IWYU pragma: export } // namespace messages template @@ -46,6 +48,7 @@ void CustomBuildField(TypeList, Priority<1>, InvokeContext& invoke_co { BuildField(TypeList(), invoke_context, output, value.v1); output.setV2(value.v2); + BuildField(TypeList>(), invoke_context, output, value.v3); } template @@ -55,6 +58,7 @@ decltype(auto) CustomReadField(TypeList, Priority<1>, InvokeContext& return read_dest.update([&](FooCustom& value) { value.v1 = ReadField(TypeList(), invoke_context, mp::Make(custom.getV1()), ReadDestTemp()); value.v2 = custom.getV2(); + value.v3 = ReadField(TypeList>(), invoke_context, mp::Make(custom.getV3()), ReadDestTemp>()); }); } @@ -72,6 +76,8 @@ inline void CustomBuildMessage(InvokeContext& invoke_context, const test::FooMessage& src, test::messages::FooMessage::Builder&& builder) { + const auto& hook{invoke_context.connection.m_loop->testing_hook_misc}; + if (hook) hook("build FooMessage"); builder.setMessage(src.message + " build"); } diff --git a/test/mp/test/foo.capnp b/test/mp/test/foo.capnp index 9e6213fd..392719ec 100644 --- a/test/mp/test/foo.capnp +++ b/test/mp/test/foo.capnp @@ -37,6 +37,12 @@ interface FooInterface $Proxy.wrap("mp::test::FooImplementation") { callFnAsync @18 (context :Proxy.Context) -> (); callIntFnAsync @21 (context :Proxy.Context, arg :Int32) -> (result :Int32); passDataPointers @22 (arg :List(Data)) -> (result :List(Data)); + listBars @25 (context :Proxy.Context, n :Int32) -> (result :List(BarInterface)); + callMessageAsync @26 (context :Proxy.Context) -> (result :FooMessage); +} + +interface FooInit $Proxy.wrap("mp::test::FooInit") { + construct @0 () -> (); } interface FooCallback $Proxy.wrap("mp::test::FooCallback") { @@ -48,6 +54,11 @@ interface ExtendedCallback extends(FooCallback) $Proxy.wrap("mp::test::ExtendedC callExtended @0 (context :Proxy.Context, arg :Int32) -> (result :Int32); } +interface BarInterface $Proxy.wrap("mp::test::Bar") { + destroy @0 (context :Proxy.Context) -> (); + value @1 (context :Proxy.Context) -> (result :Int32); +} + interface FooFn $Proxy.wrap("ProxyCallback>") { destroy @0 (context :Proxy.Context) -> (); call @1 (context :Proxy.Context) -> (result :Int32); @@ -66,6 +77,7 @@ struct FooStruct $Proxy.wrap("mp::test::FooStruct") { struct FooCustom $Proxy.wrap("mp::test::FooCustom") { v1 @0 :Text; v2 @1 :Int32; + v3 @2 :List(Int32); } struct FooEmpty $Proxy.wrap("mp::test::FooEmpty") { diff --git a/test/mp/test/foo.h b/test/mp/test/foo.h index 779c8db1..e8782708 100644 --- a/test/mp/test/foo.h +++ b/test/mp/test/foo.h @@ -34,6 +34,7 @@ struct FooCustom { std::string v1; int v2; + std::vector v3; }; struct FooEmpty @@ -66,6 +67,27 @@ class ExtendedCallback : public FooCallback virtual int callExtended(int arg) = 0; }; +class FooInit +{ +}; + +//! A second, arbitrary interface used to test returning a +//! list of interface objects. +class Bar +{ +public: + virtual ~Bar() = default; + virtual int value() = 0; +}; + +//! Concrete Bar that returns a fixed value, used by listBars tests. +class SimpleBar : public Bar +{ +public: + explicit SimpleBar(int value) : m_value(value) {} + int value() override { return m_value; } + int m_value; +}; class FooImplementation { public: @@ -74,7 +96,7 @@ class FooImplementation void addInOut(int x, int& sum) { sum += x; } int mapSize(const std::map& map) { return map.size(); } FooStruct pass(FooStruct foo) { return foo; } - void raise(FooStruct foo) { throw foo; } + [[noreturn]] void raise(FooStruct foo) { throw foo; } void initThreadMap() {} int callback(FooCallback& callback, int arg) { return callback.call(arg); } int callbackUnique(std::unique_ptr callback, int arg) { return callback->call(arg); } @@ -91,10 +113,18 @@ class FooImplementation double passDouble(double value) { return value; } int passFn(std::function fn) { return fn(); } std::vector passDataPointers(std::vector values) { return values; } + std::vector> listBars(int n) + { + std::vector> result; + result.reserve(n); + for (int i = 0; i < n; ++i) result.push_back(std::make_unique(i)); + return result; + } std::shared_ptr m_callback; void callFn() { assert(m_fn); m_fn(); } void callFnAsync() { assert(m_fn); m_fn(); } int callIntFnAsync(int arg) { assert(m_int_fn); return m_int_fn(arg); } + FooMessage callMessageAsync() { assert(m_fn); m_fn(); return {}; } std::function m_fn; std::function m_int_fn; }; diff --git a/test/mp/test/listen_tests.cpp b/test/mp/test/listen_tests.cpp index 1506eaa4..41fb05aa 100644 --- a/test/mp/test/listen_tests.cpp +++ b/test/mp/test/listen_tests.cpp @@ -2,20 +2,21 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include -#include +#include "common.h" +#include "unixlistener.h" #include #include #include +#include #include -#include #include #include #include #include #include #include +#include #include #include #include @@ -24,10 +25,7 @@ #include #include // IWYU pragma: keep #include -#include #include -#include -#include #include #include @@ -37,65 +35,6 @@ namespace { constexpr auto FAILURE_TIMEOUT = std::chrono::seconds{30}; -//! Owns a temporary Unix-domain listening socket used by ListenSetup. Tests call -//! Connect() to create client socket FDs and release() to transfer the listening -//! FD to ListenConnections(). -class UnixListener -{ -public: - UnixListener() - { - std::string dir_template = (std::filesystem::temp_directory_path() / "mptest-listener-XXXXXX").string(); - char* dir = mkdtemp(dir_template.data()); - KJ_REQUIRE(dir != nullptr); - m_dir = dir; - m_path = m_dir + "/socket"; - - m_fd = socket(AF_UNIX, SOCK_STREAM, 0); - KJ_REQUIRE(m_fd >= 0); - - sockaddr_un addr{}; - addr.sun_family = AF_UNIX; - KJ_REQUIRE(m_path.size() < sizeof(addr.sun_path)); - std::strncpy(addr.sun_path, m_path.c_str(), sizeof(addr.sun_path) - 1); - KJ_REQUIRE(bind(m_fd, reinterpret_cast(&addr), sizeof(addr)) == 0); - KJ_REQUIRE(listen(m_fd, SOMAXCONN) == 0); - } - - ~UnixListener() - { - if (m_fd >= 0) close(m_fd); - if (!m_path.empty()) unlink(m_path.c_str()); - if (!m_dir.empty()) rmdir(m_dir.c_str()); - } - - int release() - { - assert(m_fd >= 0); - int fd = m_fd; - m_fd = -1; - return fd; - } - - int MakeConnectedSocket() const - { - int fd = socket(AF_UNIX, SOCK_STREAM, 0); - KJ_REQUIRE(fd >= 0); - - sockaddr_un addr{}; - addr.sun_family = AF_UNIX; - KJ_REQUIRE(m_path.size() < sizeof(addr.sun_path)); - std::strncpy(addr.sun_path, m_path.c_str(), sizeof(addr.sun_path) - 1); - KJ_REQUIRE(connect(fd, reinterpret_cast(&addr), sizeof(addr)) == 0); - return fd; - } - -private: - int m_fd{-1}; - std::string m_dir; - std::string m_path; -}; - //! Runs a client EventLoop on its own thread and connects one socket FD to the //! server. The constructed ProxyClient can be used by the test thread to make //! calls over that connection. @@ -104,10 +43,7 @@ class ClientSetup public: explicit ClientSetup(int fd) : thread([this, fd] { - EventLoop loop("mptest-client", [](mp::LogMessage log) { - KJ_LOG(INFO, log.level, log.message); - if (log.level == mp::Log::Raise) throw std::runtime_error(log.message); - }); + EventLoop loop("mptest-client", DefaultLogHandler); client_promise.set_value(ConnectStream(loop, MakeStream(loop, fd))); loop.loop(); }) @@ -135,12 +71,10 @@ class ClientSetup class ListenSetup { public: - explicit ListenSetup(std::optional max_connections = std::nullopt) - : thread([this, max_connections] { - EventLoop loop("mptest-server", [](mp::LogMessage log) { - KJ_LOG(INFO, log.level, log.message); - if (log.level == mp::Log::Raise) throw std::runtime_error(log.message); - }); + explicit ListenSetup(std::optional max_connections = std::nullopt, + mp::LogFn log_handler = DefaultLogHandler) + : thread([this, max_connections, log_handler] { + EventLoop loop("mptest-server", log_handler); loop.testing_hook_disconnected = [&] { Lock lock(counter_mutex); ++disconnected_count; @@ -225,7 +159,7 @@ KJ_TEST("ListenConnections enforces a local connection limit") { // With max-connections=1, the second socket can connect to the kernel // backlog, but ListenConnections should not accept or serve it until the - // first accepts clients disconnects. + // first accepted client disconnects. ListenSetup server(/*max_connections=*/1); @@ -289,6 +223,43 @@ KJ_TEST("ListenConnections accepts multiple connections") KJ_EXPECT(client3->client->add(3, 4) == 7); } +KJ_TEST("ListenConnections handles a client that disconnects before being accepted") +{ + Mutex mutex; + bool accept_error = false; + ListenSetup server(std::nullopt, [&](mp::LogMessage log) { + // On macOS, accept() can fail if the peer closes the connection before it is accepted. + // The event loop then reports this as an uncaught task exception. We catch and ignore + // this specific error here so that the corresponding CI job does not fail. + // + // This is a Cap'n Proto bug, fixed by https://github.com/capnproto/capnproto/pull/2748 + if (log.level == mp::Log::Error && log.message.find("Uncaught exception in daemonized task.") != std::string::npos) { + Lock lock(mutex); + accept_error = true; + } + DefaultLogHandler(log); + }); + + // This is racy, if the close does not happen before accept(), + // the connection is accepted normally. + int fd = server.listener.MakeConnectedSocket(); + KJ_SYSCALL(close(fd)); + + // Wait for the connection to either be accepted and disconnected, or fail + // to be accepted and log the error above. + const auto deadline = std::chrono::steady_clock::now() + FAILURE_TIMEOUT; + auto accept_error_seen = [&] { Lock lock(mutex); return accept_error; }; + while (!accept_error_seen() && server.DisconnectedCount() < 1 && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + +#ifndef __APPLE__ + // No error is expected on platforms other than macOS. + KJ_REQUIRE(!accept_error_seen()); +#endif + KJ_EXPECT(server.DisconnectedCount() == (accept_error_seen() ? 0 : 1)); +} + } // namespace } // namespace test } // namespace mp diff --git a/test/mp/test/spawn_tests.cpp b/test/mp/test/spawn_tests.cpp index bdc9ef54..5f991275 100644 --- a/test/mp/test/spawn_tests.cpp +++ b/test/mp/test/spawn_tests.cpp @@ -4,8 +4,11 @@ #include +#include +#include #include +#include #include #include #include @@ -13,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -113,5 +118,18 @@ KJ_TEST("SpawnProcess does not run callback in child") KJ_EXPECT(exited, "Timeout waiting for child process to exit"); KJ_EXPECT(WIFEXITED(status) && WEXITSTATUS(status) == 0); } + +KJ_TEST("SpawnProcess throws on execvp failure") +{ + try { + SpawnProcess([&](SpawnConnectInfo) -> std::vector { + return {"/nonexistent/binary"}; + }); + KJ_EXPECT(false, "expected SpawnProcess to throw"); + } catch (const std::system_error& e) { + KJ_EXPECT(e.code().value() == ENOENT); + KJ_EXPECT(std::string_view{e.what()}.find("execvp") != std::string_view::npos); + } +} } // namespace test } // namespace mp diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 7a0940c8..212d6ab4 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -20,11 +21,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -78,6 +81,7 @@ class TestSetup public: std::function server_disconnect; std::function server_disconnect_later; + std::function server_on_disconnect; std::function client_disconnect; std::promise>> client_promise; std::unique_ptr> client; @@ -108,8 +112,14 @@ class TestSetup loop.m_task_set->add(kj::evalLater([&] { server_connection.reset(); })); }; // Set handler to destroy the server when the client disconnects. This - // is ignored if server_disconnect() is called instead. - server_connection->onDisconnect([&] { server_connection.reset(); }); + // is ignored if server_disconnect() is called instead. Tests can + // assign server_on_disconnect to override the default behavior of + // destroying the server connection as soon as the disconnect is + // detected (in which case they need to destroy it themselves, + // e.g. by calling server_disconnect(), so the event loop can + // exit). + server_on_disconnect = [&] { server_connection.reset(); }; + server_connection->onDisconnect([&] { server_on_disconnect(); }); auto client_connection = std::make_unique(loop, kj::mv(pipe.ends[1])); auto client_proxy = std::make_unique>( @@ -172,7 +182,7 @@ KJ_TEST("Call FooInterface methods") } KJ_EXPECT(in.unordered_set_int.size() == out.unordered_set_int.size()); for (const auto& elem : in.unordered_set_int) { - KJ_EXPECT(out.unordered_set_int.count(elem) == 1); + KJ_EXPECT(out.unordered_set_int.contains(elem)); } KJ_EXPECT(in.vector_bool.size() == out.vector_bool.size()); for (size_t i = 0; i < in.vector_bool.size(); ++i) { @@ -233,9 +243,11 @@ KJ_TEST("Call FooInterface methods") FooCustom custom_in; custom_in.v1 = "v1"; custom_in.v2 = 5; + custom_in.v3 = {10, 20, 30}; FooCustom custom_out = foo->passCustom(custom_in); KJ_EXPECT(custom_in.v1 == custom_out.v1); KJ_EXPECT(custom_in.v2 == custom_out.v2); + KJ_EXPECT(custom_in.v3 == custom_out.v3); foo->passEmpty(FooEmpty{}); @@ -269,6 +281,16 @@ KJ_TEST("Call FooInterface methods") KJ_REQUIRE(data_out[0] != nullptr); KJ_EXPECT(*data_out[0] == *data_in[0]); KJ_EXPECT(!data_out[1]); + + // Test returning vector> from server. This exercises + // BuildList with interface element types, which requires non-const iteration + // so unique_ptr::release() can transfer ownership to the proxy server. + std::vector> bars{foo->listBars(3)}; + KJ_REQUIRE(bars.size() == 3u); + for (int i = 0; i < 3; ++i) { + KJ_REQUIRE(bars[i] != nullptr); + KJ_EXPECT(bars[i]->value() == i); + } } KJ_TEST("Call IPC method after client connection is closed") @@ -288,7 +310,16 @@ KJ_TEST("Calling IPC method after server connection is closed") KJ_EXPECT(foo->add(1, 2) == 3); setup.server_disconnect(); - EXPECT_EXCEPTION(foo->add(1, 2), "IPC client method call interrupted by disconnect."); + try { + foo->add(1, 2); + KJ_EXPECT(false); + } catch (const std::runtime_error& e) { + std::string_view reason{e.what()}; + + // The disconnect handler may delete the connection before the + // call is processed or while the call is in flight, both errors are possible. + KJ_EXPECT(reason == "IPC client method called after disconnect." || reason == "IPC client method call interrupted by disconnect."); + } } KJ_TEST("Calling IPC method and disconnecting during the call") @@ -346,6 +377,98 @@ KJ_TEST("Calling IPC method, disconnecting and blocking during the call") signal.set_value(); } +KJ_TEST("Calling async IPC method with a remote disconnect while results are built") +{ + // Regression test for bitcoin-core/libmultiprocess#348, a data race + // reported by ThreadSanitizer where a server worker thread called + // call_context.getResults() while the event loop thread was tearing down + // Cap'n Proto connection state (capnp::_::RpcConnectionState::disconnect() + // overwriting the RpcConnectionState::connection field) after an abrupt + // remote disconnect. + // + // The test makes an async IPC call (callMessageAsync) whose method body + // just signals the main thread. When the worker thread returns from the + // method body, it calls call_context.getResults(), reading the connection + // state, and starts serializing the FooMessage result, where the + // testing_hook_misc hook set below makes it sleep. Meanwhile the main + // thread destroys the client connection, and the event loop thread + // processes the resulting EOF, running RpcConnectionState::disconnect() + // and overwriting the connection state the worker thread just read, with + // no synchronization between the two accesses. + // + // Two details are essential for ThreadSanitizer to detect the race: + // + // - There must be no synchronization between the worker thread's + // getResults() call and the event loop thread's disconnect processing. + // The worker signals the main thread *before* getResults() and sleeps + // (sleeping creates no happens-before edge) across the disconnect, so + // the racing accesses are not ordered by any of the test's own + // synchronization. + // + // - The worker thread's read should come *before* the event loop thread's + // write, close in time. In the write-then-read order (e.g. method + // returning long after the disconnect), the race exists too, but + // ThreadSanitizer usually misses it: right after disconnect() writes the + // connection field, the loop thread's teardown destructors re-read it + // many times (~ImportClient etc. check connection.is() to + // decide whether to send messages), evicting the write from the + // per-granule shadow history before a late reader comes along. + // + // The server Connection object is deliberately kept alive during all this + // by overriding server_on_disconnect: destroying it would cancel the + // in-flight request (Connection::~Connection calls m_canceler.cancel(), + // setting request_canceled) and the worker would throw InterruptException + // instead of proceeding into getResults(). Keeping it alive matches the + // window in the original report, where the worker races with capnp's own + // internal teardown, which runs before any onDisconnect notification. + + TestSetup setup{/*client_owns_connection=*/false}; + ProxyClient* foo = setup.client.get(); + KJ_EXPECT(foo->add(1, 2) == 3); + foo->initThreadMap(); + + // Keep the server Connection object alive when the disconnect is detected + // so the in-flight request is not canceled (see comment above). The + // connection is destroyed at the end of the test instead. + setup.server_on_disconnect = [] {}; + + // Signaled by the worker thread when the method body runs, just before it + // returns and the worker calls getResults() and serializes the results. + std::promise fn_called; + setup.server->m_impl->m_fn = [&] { fn_called.set_value(); }; + + // Keep the worker thread inside the results-building step, without + // synchronizing, while the event loop thread processes the disconnect. + EventLoop& loop = *setup.server->m_context.connection->m_loop; + loop.testing_hook_misc = [](std::any arg) { + if (const char* const* tag{std::any_cast(&arg)}; + tag && std::string_view{*tag} == "build FooMessage") { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + }; + + // Signaled by the worker thread when it is completely done with the + // request, including serializing the results. + std::promise request_done; + loop.testing_hook_async_request_done = [&] { request_done.set_value(); }; + + // Make the IPC call from a separate thread so this thread can trigger the + // client disconnect while the call is executing. + std::thread caller{[&] { + EXPECT_EXCEPTION(foo->callMessageAsync(), "IPC client method call interrupted by disconnect."); + }}; + + fn_called.get_future().get(); + setup.client_disconnect(); + caller.join(); + + // Wait for the worker thread to finish the request, then tear down the + // server connection that was deliberately kept alive above, so the event + // loop is able to exit. + request_done.get_future().get(); + setup.server_disconnect(); +} + KJ_TEST("Worker thread destroyed before it is initialized") { // Regression test for bitcoin/bitcoin#34711, bitcoin/bitcoin#34756 where a @@ -462,7 +585,7 @@ KJ_TEST("Make simultaneous IPC calls on single remote thread") // that will be used for the test. setup.server->m_impl->m_fn = [&] {}; foo->callFnAsync(); - ThreadContext& tc{g_thread_context}; + ThreadContext& tc{CurrentThread()}; Thread::Client *callback_thread, *request_thread; foo->m_context.loop->sync([&] { Lock lock(tc.waiter->m_mutex); @@ -516,7 +639,7 @@ KJ_TEST("Call async IPC method dispatched to pool thread") foo->initThreadMap(); setup.server->m_impl->m_int_fn = [](int n) { return n * 2; }; - ThreadContext& tc{g_thread_context}; + ThreadContext& tc{CurrentThread()}; std::atomic running{3}; std::promise pool_ready; foo->m_context.loop->sync([&] { @@ -550,6 +673,68 @@ KJ_TEST("Call async IPC method dispatched to pool thread") } } +#ifdef HAVE_PTHREAD_GETNAME_NP +KJ_TEST("Worker thread has OS thread name") +{ + TestSetup setup; + ProxyClient* foo = setup.client.get(); + foo->initThreadMap(); + + std::promise thread_name; + setup.server->m_impl->m_fn = [&] { thread_name.set_value(ThreadName("")); }; + foo->callFnAsync(); + + const std::string name{thread_name.get_future().get()}; + KJ_EXPECT(name.find("/capnp-worker-") != std::string::npos, name); +} + +KJ_TEST("Pool thread has OS thread name") +{ + TestSetup setup; + ProxyClient* foo = setup.client.get(); + foo->initThreadMap(); + + std::promise thread_name; + setup.server->m_impl->m_fn = [&] { thread_name.set_value(ThreadName("")); }; + + std::promise pool_ready; + foo->m_context.loop->sync([&] { + auto pool_req = foo->m_context.connection->m_thread_map.makePoolRequest(); + pool_req.setCount(1); + foo->m_context.loop->m_task_set->add( + pool_req.send().then([&](auto&&) { pool_ready.set_value(); })); + }); + pool_ready.get_future().get(); + + std::promise done; + foo->m_context.loop->sync([&] { + auto request{foo->m_client.callFnAsyncRequest()}; + foo->m_context.loop->m_task_set->add( + request.send().then([&](auto&&) { done.set_value(); })); + }); + // Wait for the reply before returning, so the connection is not torn down + // while the request is still in flight. + done.get_future().get(); + + const std::string name{thread_name.get_future().get()}; + KJ_EXPECT(name.find("/capnp-pool-0-") != std::string::npos, name); +} + +KJ_TEST("Async cleanup thread has OS thread name") +{ + std::promise thread_name; + { + TestSetup setup; + // FooInterface has no destroy method, so the server ProxyServer runs + // its cleanup functions on the async thread when it is destroyed. + setup.server->m_context.cleanup_fns.emplace_front( + [&] { thread_name.set_value(ThreadName("")); }); + } + const std::string name{thread_name.get_future().get()}; + KJ_EXPECT(name.find("/capnp-async-") != std::string::npos, name); +} +#endif // HAVE_PTHREAD_GETNAME_NP + KJ_TEST("Call async IPC method without thread or pool errors correctly") { TestSetup setup; @@ -577,5 +762,114 @@ KJ_TEST("Call async IPC method without thread or pool errors correctly") KJ_EXPECT(error_thrown); } +KJ_TEST("Calling IPC method after destroy and destroy during a call") +{ + // Regression test for bitcoin-core/libmultiprocess#378 where a + // non-libmultiprocess client could: + // + // 1. Call a method after destroy, making the server call it through a + // null m_impl pointer. + // 2. Call destroy while another call on the same object was running, + // making the server free m_impl under that call. + // + // This test builds requests directly through Cap'n Proto, mimicking the + // behavior of a buggy client. + TestSetup setup; + ProxyClient* foo = setup.client.get(); + EventLoop& loop = *foo->m_context.loop; + foo->initThreadMap(); + + // Create two pool threads, so the destroy() and value() calls run on + // different threads. + std::promise pool_ready; + loop.sync([&] { + auto pool_req = foo->m_context.connection->m_thread_map.makePoolRequest(); + pool_req.setCount(2); + loop.m_task_set->add(pool_req.send().then([&](auto&&) { pool_ready.set_value(); })); + }); + pool_ready.get_future().get(); + + // Create two Bar objects, one for each case below. BarInterface declares a + // destroy method, unlike FooInterface. + std::optional bar1; + std::optional bar2; + std::promise bars_ready; + loop.sync([&] { + auto request{foo->m_client.listBarsRequest()}; + request.initContext(); + request.setN(2); + loop.m_task_set->add(request.send().then([&](auto&& results) { + bar1.emplace(results.getResult()[0]); + bar2.emplace(results.getResult()[1]); + bars_ready.set_value(); + })); + }); + bars_ready.get_future().get(); + + //////////////////////////////////////////////////////////////////////////// + // Case 1: method called after destroy. + //////////////////////////////////////////////////////////////////////////// + + std::promise destroyed; + loop.sync([&] { + auto request{bar1->destroyRequest()}; + request.initContext(); + loop.m_task_set->add(request.send().then([&](auto&&) { destroyed.set_value(); })); + }); + destroyed.get_future().get(); + + // Calls after destroy are rejected. + std::promise value_error; + loop.sync([&] { + auto request{bar1->valueRequest()}; + request.initContext(); + loop.m_task_set->add(request.send().then( + [&](auto&&) { value_error.set_value(""); }, + [&](kj::Exception&& e) { value_error.set_value(e.getDescription().cStr()); })); + }); + KJ_EXPECT(value_error.get_future().get().find("method called after destroy") != std::string::npos); + + // Release bar1. + loop.sync([&] { bar1.reset(); }); + + //////////////////////////////////////////////////////////////////////////// + // Case 2: destroy called while another call is running. + //////////////////////////////////////////////////////////////////////////// + + // Block the next async request, the value() call, before it calls m_impl. + std::promise value_started; + std::promise value_release; + std::atomic blocked{false}; + loop.testing_hook_async_request_start = [&] { + if (blocked.exchange(true)) return; + value_started.set_value(); + value_release.get_future().wait(); + }; + std::promise value; + loop.sync([&] { + auto request{bar2->valueRequest()}; + request.initContext(); + loop.m_task_set->add(request.send().then([&](auto&& results) { value.set_value(results.getResult()); })); + }); + value_started.get_future().get(); + + // Destroy fails if value() request is being executed. + std::promise destroy_error; + loop.sync([&] { + auto request{bar2->destroyRequest()}; + request.initContext(); + loop.m_task_set->add(request.send().then( + [&](auto&&) { destroy_error.set_value(""); }, + [&](kj::Exception&& e) { destroy_error.set_value(e.getDescription().cStr()); })); + }); + KJ_EXPECT(destroy_error.get_future().get().find("destroy called while other calls are in progress") != std::string::npos); + + value_release.set_value(); + KJ_EXPECT(value.get_future().get() == 1); + + // Release bar2. + loop.sync([&] { bar2.reset(); }); +} + } // namespace test } // namespace mp diff --git a/test/mp/test/unixlistener.h b/test/mp/test/unixlistener.h new file mode 100644 index 00000000..a743a475 --- /dev/null +++ b/test/mp/test/unixlistener.h @@ -0,0 +1,84 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_TEST_UNIXLISTENER_H +#define MP_TEST_UNIXLISTENER_H + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace mp { +namespace test { + +//! Owns a temporary Unix-domain listening socket. Tests call +//! MakeConnectedSocket() to create client socket FDs and release() to transfer +//! ownership of the listening FD. +class UnixListener +{ +public: + UnixListener() + { + std::string dir_template = (std::filesystem::temp_directory_path() / "mptest-listener-XXXXXX").string(); + char* dir = mkdtemp(dir_template.data()); + KJ_REQUIRE(dir != nullptr); + m_dir = dir; + m_path = m_dir + "/socket"; + + m_fd = socket(AF_UNIX, SOCK_STREAM, 0); + KJ_REQUIRE(m_fd >= 0); + + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + KJ_REQUIRE(m_path.size() < sizeof(addr.sun_path)); + std::strncpy(addr.sun_path, m_path.c_str(), sizeof(addr.sun_path) - 1); + KJ_REQUIRE(bind(m_fd, reinterpret_cast(&addr), sizeof(addr)) == 0); + KJ_REQUIRE(listen(m_fd, SOMAXCONN) == 0); + } + + ~UnixListener() + { + if (m_fd >= 0) close(m_fd); + if (!m_path.empty()) unlink(m_path.c_str()); + if (!m_dir.empty()) rmdir(m_dir.c_str()); + } + + int release() + { + assert(m_fd >= 0); + int fd = m_fd; + m_fd = -1; + return fd; + } + + int MakeConnectedSocket() const + { + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + KJ_REQUIRE(fd >= 0); + + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + KJ_REQUIRE(m_path.size() < sizeof(addr.sun_path)); + std::strncpy(addr.sun_path, m_path.c_str(), sizeof(addr.sun_path) - 1); + KJ_REQUIRE(connect(fd, reinterpret_cast(&addr), sizeof(addr)) == 0); + return fd; + } + +private: + int m_fd{-1}; + std::string m_dir; + std::string m_path; +}; + +} // namespace test +} // namespace mp + +#endif // MP_TEST_UNIXLISTENER_H