diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ce4aca735453..f9505740f66b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -342,7 +342,6 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL kernel/context.cpp kernel/cs_main.cpp kernel/disconnected_transactions.cpp - kernel/mempool_persist.cpp kernel/mempool_removal_reason.cpp instantsend/db.cpp instantsend/instantsend.cpp @@ -394,6 +393,7 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL node/interfaces.cpp node/kernel_notifications.cpp node/mempool_args.cpp + node/mempool_persist.cpp node/mempool_persist_args.cpp node/miner.cpp node/minisketchwrapper.cpp diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt index 8ce707bd35e7..b3dc500f0c9f 100644 --- a/src/bench/CMakeLists.txt +++ b/src/bench/CMakeLists.txt @@ -24,6 +24,7 @@ add_executable(bench_bitcoin ccoins_caching.cpp chacha20.cpp checkblock.cpp + checkblockindex.cpp checkqueue.cpp disconnected_transactions.cpp crypto_hash.cpp @@ -41,11 +42,13 @@ add_executable(bench_bitcoin mempool_stress.cpp merkle_root.cpp net_processing.cpp + parse_hex.cpp peer_eviction.cpp poly1305.cpp pool.cpp pow_hash.cpp prevector.cpp + readblock.cpp rollingbloom.cpp rpc_blockchain.cpp rpc_mempool.cpp diff --git a/src/bench/checkblockindex.cpp b/src/bench/checkblockindex.cpp new file mode 100644 index 000000000000..e8a848dbd409 --- /dev/null +++ b/src/bench/checkblockindex.cpp @@ -0,0 +1,20 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include + +static void CheckBlockIndex(benchmark::Bench& bench) +{ + auto testing_setup{MakeNoLogFileContext()}; + // Mine some more blocks + testing_setup->mineBlocks(1000); + bench.run([&] { + testing_setup->m_node.chainman->CheckBlockIndex(); + }); +} + + +BENCHMARK(CheckBlockIndex, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/parse_hex.cpp b/src/bench/parse_hex.cpp new file mode 100644 index 000000000000..db3ead043c81 --- /dev/null +++ b/src/bench/parse_hex.cpp @@ -0,0 +1,36 @@ +// Copyright (c) 2024- The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include + +std::string generateHexString(size_t length) { + const auto hex_digits = "0123456789ABCDEF"; + FastRandomContext rng(/*fDeterministic=*/true); + + std::string data; + while (data.size() < length) { + auto digit = hex_digits[rng.randbits(4)]; + data.push_back(digit); + } + return data; +} + +static void HexParse(benchmark::Bench& bench) +{ + auto data = generateHexString(130); // Generates 678B0EDA0A1FD30904D5A65E3568DB82DB2D918B0AD8DEA18A63FECCB877D07CAD1495C7157584D877420EF38B8DA473A6348B4F51811AC13C786B962BEE5668F9 by default + + bench.batch(data.size()).unit("base16").run([&] { + auto result = TryParseHex(data); + assert(result != std::nullopt); // make sure we're measuring the successful case + ankerl::nanobench::doNotOptimizeAway(result); + }); +} + +BENCHMARK(HexParse, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/readblock.cpp b/src/bench/readblock.cpp new file mode 100644 index 000000000000..af4df56f5d4d --- /dev/null +++ b/src/bench/readblock.cpp @@ -0,0 +1,38 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include + +#include +#include +#include +#include +#include +#include + +static FlatFilePos WriteBlockToDisk(ChainstateManager& chainman) +{ + DataStream stream{benchmark::data::block813851}; + CBlock block; + stream >> block; + + return chainman.m_blockman.SaveBlockToDisk(block, 0, nullptr); +} + +static void ReadBlockFromDiskTest(benchmark::Bench& bench) +{ + const auto testing_setup{MakeNoLogFileContext(ChainType::MAIN)}; + ChainstateManager& chainman{*testing_setup->m_node.chainman}; + + CBlock block; + const auto pos{WriteBlockToDisk(chainman)}; + + bench.run([&] { + const auto success{chainman.m_blockman.ReadBlockFromDisk(block, pos)}; + assert(success); + }); +} + +BENCHMARK(ReadBlockFromDiskTest, benchmark::PriorityLevel::HIGH); diff --git a/src/init.cpp b/src/init.cpp index 2936d9e06fd9..ca94d87b176b 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -12,7 +12,6 @@ #include #include -#include #include #include @@ -63,6 +62,7 @@ #include #include #include +#include #include #include #include @@ -173,8 +173,6 @@ using kernel::CoinStatsHashType; using kernel::DEFAULT_STOPAFTERBLOCKIMPORT; -using kernel::DumpMempool; -using kernel::LoadMempool; using kernel::ValidationCacheSizes; using node::ApplyArgsManOptions; @@ -183,6 +181,8 @@ using node::CacheSizes; using node::CalculateCacheSizes; using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINTPRIORITY; +using node::DumpMempool; +using node::LoadMempool; using node::g_indexes_ready_to_sync; using node::KernelNotifications; using node::LoadChainstate; @@ -821,7 +821,7 @@ void SetupServerArgs(ArgsManager& argsman) hidden_args.emplace_back("-zmqpubsequencehwm="); #endif - argsman.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, and occasionally. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, chainstate, and other validation data structures every operations. Use 0 to disable. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkblocks=", strprintf("How many blocks to check at startup (default: %u, 0 = all)", DEFAULT_CHECKBLOCKS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checklevel=", strprintf("How thorough the block verification of -checkblocks is: %s (0-4, default: %u)", Join(CHECKLEVEL_DOC, ", "), DEFAULT_CHECKLEVEL), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkaddrman=", strprintf("Run addrman consistency checks every operations. Use 0 to disable. (default: %u)", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); diff --git a/src/ipc/capnp/common-types.h b/src/ipc/capnp/common-types.h new file mode 100644 index 000000000000..4de906997ce9 --- /dev/null +++ b/src/ipc/capnp/common-types.h @@ -0,0 +1,108 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_CAPNP_COMMON_TYPES_H +#define BITCOIN_IPC_CAPNP_COMMON_TYPES_H + +#include +#include +#include + +#include +#include +#include +#include + +namespace ipc { +namespace capnp { +//! Use SFINAE to define Serializeable trait which is true if type T has a +//! Serialize(stream) method, false otherwise. +template +struct Serializable { +private: + template + static std::true_type test(decltype(std::declval().Serialize(std::declval()))*); + template + static std::false_type test(...); + +public: + static constexpr bool value = decltype(test(nullptr))::value; +}; + +//! Use SFINAE to define Unserializeable trait which is true if type T has +//! an Unserialize(stream) method, false otherwise. +template +struct Unserializable { +private: + template + static std::true_type test(decltype(std::declval().Unserialize(std::declval()))*); + template + static std::false_type test(...); + +public: + static constexpr bool value = decltype(test(nullptr))::value; +}; +} // namespace capnp +} // namespace ipc + +//! Functions to serialize / deserialize common bitcoin types. +namespace mp { +//! Overload multiprocess library's CustomBuildField hook to allow any +//! serializable object to be stored in a capnproto Data field or passed to a +//! canproto interface. Use Priority<1> so this hook has medium priority, and +//! higher priority hooks could take precedence over this one. +template +void CustomBuildField( + TypeList, Priority<1>, InvokeContext& invoke_context, Value&& value, Output&& output, + // Enable if serializeable and if LocalType is not cv or reference + // qualified. If LocalType is cv or reference qualified, it is important to + // fall back to lower-priority Priority<0> implementation of this function + // that strips cv references, to prevent this CustomBuildField overload from + // taking precedence over more narrow overloads for specific LocalTypes. + std::enable_if_t::value && + std::is_same_v>>>* enable = nullptr) +{ + DataStream stream; + value.Serialize(stream); + auto result = output.init(stream.size()); + memcpy(result.begin(), stream.data(), stream.size()); +} + +//! Overload multiprocess library's CustomReadField hook to allow any object +//! with an Unserialize method to be read from a capnproto Data field or +//! returned from canproto interface. Use Priority<1> so this hook has medium +//! priority, and higher priority hooks could take precedence over this one. +template +decltype(auto) +CustomReadField(TypeList, Priority<1>, InvokeContext& invoke_context, Input&& input, ReadDest&& read_dest, + std::enable_if_t::value>* enable = nullptr) +{ + return read_dest.update([&](auto& value) { + if (!input.has()) return; + auto data = input.get(); + SpanReader stream{SER_NETWORK, CLIENT_VERSION, {data.begin(), data.end()}}; + value.Unserialize(stream); + }); +} + +template +void CustomBuildField(TypeList, Priority<1>, InvokeContext& invoke_context, Value&& value, Output&& output) +{ + std::string str = value.write(); + auto result = output.init(str.size()); + memcpy(result.begin(), str.data(), str.size()); +} + +template +decltype(auto) CustomReadField(TypeList, Priority<1>, InvokeContext& invoke_context, Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + auto data = input.get(); + value.read(std::string_view{data.begin(), data.size()}); + }); +} +} // namespace mp + +#endif // BITCOIN_IPC_CAPNP_COMMON_TYPES_H diff --git a/src/kernel/CMakeLists.txt b/src/kernel/CMakeLists.txt index 04ad20d008dc..fd082829f2f7 100644 --- a/src/kernel/CMakeLists.txt +++ b/src/kernel/CMakeLists.txt @@ -59,7 +59,6 @@ add_library(bitcoinkernel STATIC EXCLUDE_FROM_ALL context.cpp cs_main.cpp disconnected_transactions.cpp - mempool_persist.cpp mempool_removal_reason.cpp ../key.cpp ../key_io.cpp diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index 7fc875f26a30..75b10560e2c2 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -36,7 +36,7 @@ namespace kernel { struct ChainstateManagerOpts { const CChainParams& chainparams; fs::path datadir; - std::optional check_block_index{}; + std::optional check_block_index{}; bool checkpoints_enabled{DEFAULT_CHECKPOINTS_ENABLED}; //! If set, it will override the minimum work we will assume exists on some valid chain. std::optional minimum_chain_work{}; diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp index 40d1c9540157..de242ffb503b 100644 --- a/src/node/chainstatemanager_args.cpp +++ b/src/node/chainstatemanager_args.cpp @@ -22,7 +22,10 @@ namespace node { std::optional ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) { - if (auto value{args.GetBoolArg("-checkblockindex")}) opts.check_block_index = *value; + if (auto value{args.GetIntArg("-checkblockindex")}) { + // Interpret bare -checkblockindex argument as 1 instead of 0. + opts.check_block_index = args.GetArg("-checkblockindex")->empty() ? 1 : *value; + } if (auto value{args.GetBoolArg("-checkpoints")}) opts.checkpoints_enabled = *value; diff --git a/src/kernel/mempool_persist.cpp b/src/node/mempool_persist.cpp similarity index 98% rename from src/kernel/mempool_persist.cpp rename to src/node/mempool_persist.cpp index 2e9d26a87472..c7c0e5fcc164 100644 --- a/src/kernel/mempool_persist.cpp +++ b/src/node/mempool_persist.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include #include #include @@ -32,7 +32,7 @@ using fsbridge::FopenFn; -namespace kernel { +namespace node { static const uint64_t MEMPOOL_DUMP_VERSION = 1; @@ -203,4 +203,4 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock return true; } -} // namespace kernel +} // namespace node diff --git a/src/kernel/mempool_persist.h b/src/node/mempool_persist.h similarity index 84% rename from src/kernel/mempool_persist.h rename to src/node/mempool_persist.h index e124a8eadf8f..7c5754a90c96 100644 --- a/src/kernel/mempool_persist.h +++ b/src/node/mempool_persist.h @@ -2,15 +2,15 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_KERNEL_MEMPOOL_PERSIST_H -#define BITCOIN_KERNEL_MEMPOOL_PERSIST_H +#ifndef BITCOIN_NODE_MEMPOOL_PERSIST_H +#define BITCOIN_NODE_MEMPOOL_PERSIST_H #include class Chainstate; class CTxMemPool; -namespace kernel { +namespace node { /** Dump the mempool to a file. */ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, @@ -28,7 +28,7 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, ImportMempoolOptions&& opts); -} // namespace kernel +} // namespace node -#endif // BITCOIN_KERNEL_MEMPOOL_PERSIST_H +#endif // BITCOIN_NODE_MEMPOOL_PERSIST_H diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index dfe293434a0e..f2979ef68b37 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include @@ -31,7 +31,7 @@ #include -using kernel::DumpMempool; +using node::DumpMempool; using node::DEFAULT_MAX_RAW_TX_FEE_RATE; using node::MempoolPath; @@ -774,13 +774,13 @@ static RPCHelpMan importmempool() const UniValue& use_current_time{request.params[1]["use_current_time"]}; const UniValue& apply_fee_delta{request.params[1]["apply_fee_delta_priority"]}; const UniValue& apply_unbroadcast{request.params[1]["apply_unbroadcast_set"]}; - kernel::ImportMempoolOptions opts{ + node::ImportMempoolOptions opts{ .use_current_time = use_current_time.isNull() ? true : use_current_time.get_bool(), .apply_fee_delta_priority = apply_fee_delta.isNull() ? false : apply_fee_delta.get_bool(), .apply_unbroadcast_set = apply_unbroadcast.isNull() ? false : apply_unbroadcast.get_bool(), }; - if (!kernel::LoadMempool(mempool, load_path, chainstate, std::move(opts))) { + if (!node::LoadMempool(mempool, load_path, chainstate, std::move(opts))) { throw JSONRPCError(RPC_MISC_ERROR, "Unable to import mempool file, see debug.log for details."); } diff --git a/src/test/.gitignore b/src/test/.gitignore new file mode 100644 index 000000000000..036df1430c56 --- /dev/null +++ b/src/test/.gitignore @@ -0,0 +1,2 @@ +# capnp generated files +*.capnp.* diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 5d495f2984a3..ccbc7c648c15 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -215,10 +215,27 @@ if(ENABLE_WALLET) add_subdirectory(${PROJECT_SOURCE_DIR}/src/wallet/test wallet) endif() -# NOTE: Upstream also builds an IPC test library here from ipc_test.cpp, -# ipc_test.capnp and ipc_tests.cpp. Dash has not backported those sources -# (bitcoin#28921), and referencing them makes -DWITH_MULTIPROCESS=ON fail -# during generation. Restore this block along with the test sources. +if(WITH_MULTIPROCESS) + add_library(bitcoin_ipc_test STATIC EXCLUDE_FROM_ALL + ipc_test.cpp + ) + + target_capnp_sources(bitcoin_ipc_test ${PROJECT_SOURCE_DIR} + ipc_test.capnp + ) + + target_link_libraries(bitcoin_ipc_test + PRIVATE + core_interface + univalue + ) + + target_sources(test_bitcoin + PRIVATE + ipc_tests.cpp + ) + target_link_libraries(test_bitcoin bitcoin_ipc_test) +endif() function(add_boost_test source_file) if(NOT EXISTS ${source_file}) diff --git a/src/test/fuzz/CMakeLists.txt b/src/test/fuzz/CMakeLists.txt index 7baca6d0431e..2ffb195a4cb7 100644 --- a/src/test/fuzz/CMakeLists.txt +++ b/src/test/fuzz/CMakeLists.txt @@ -17,6 +17,7 @@ add_executable(fuzz bitset.cpp block.cpp block_header.cpp + block_index.cpp blockfilter.cpp bloom_filter.cpp buffered_file.cpp @@ -29,6 +30,7 @@ add_executable(fuzz crypto_aes256.cpp crypto_aes256cbc.cpp crypto_chacha20.cpp + crypto_chacha20poly1305.cpp crypto_common.cpp crypto_diff_fuzz_chacha20.cpp crypto_hkdf_hmac_sha256_l32.cpp diff --git a/src/test/fuzz/block_index.cpp b/src/test/fuzz/block_index.cpp new file mode 100644 index 000000000000..6056e6f87da7 --- /dev/null +++ b/src/test/fuzz/block_index.cpp @@ -0,0 +1,133 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +const BasicTestingSetup* g_setup; + +// Hardcoded block hash and nBits to make sure the blocks we store pass the pow check. +uint256 g_block_hash; + +bool operator==(const CBlockFileInfo& a, const CBlockFileInfo& b) +{ + return a.nBlocks == b.nBlocks && + a.nSize == b.nSize && + a.nUndoSize == b.nUndoSize && + a.nHeightFirst == b.nHeightFirst && + a.nHeightLast == b.nHeightLast && + a.nTimeFirst == b.nTimeFirst && + a.nTimeLast == b.nTimeLast; +} + +CBlockHeader ConsumeBlockHeader(FuzzedDataProvider& provider) +{ + CBlockHeader header; + header.nVersion = provider.ConsumeIntegral(); + header.hashPrevBlock = g_block_hash; + header.hashMerkleRoot = g_block_hash; + header.nTime = provider.ConsumeIntegral(); + header.nBits = Params().GenesisBlock().nBits; + header.nNonce = provider.ConsumeIntegral(); + return header; +} + +} // namespace + +void init_block_index() +{ + static const auto testing_setup = MakeNoLogFileContext<>(ChainType::MAIN); + g_setup = testing_setup.get(); + g_block_hash = Params().GenesisBlock().GetHash(); +} + +FUZZ_TARGET(block_index, .init = init_block_index) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + auto block_index = CBlockTreeDB(DBParams{ + .path = "", // Memory only. + .cache_bytes = 1 << 20, // 1MB. + .memory_only = true, + }); + + // Generate a number of block files to be stored in the index. + int files_count = fuzzed_data_provider.ConsumeIntegralInRange(1, 100); + std::vector> files; + files.reserve(files_count); + std::vector> files_info; + files_info.reserve(files_count); + for (int i = 0; i < files_count; i++) { + if (auto file_info = ConsumeDeserializable(fuzzed_data_provider)) { + files.push_back(std::make_unique(std::move(*file_info))); + files_info.emplace_back(i, files.back().get()); + } else { + return; + } + } + + // Generate a number of block headers to be stored in the index. + int blocks_count = fuzzed_data_provider.ConsumeIntegralInRange(files_count * 10, files_count * 100); + std::vector> blocks; + blocks.reserve(blocks_count); + std::vector blocks_info; + blocks_info.reserve(blocks_count); + for (int i = 0; i < blocks_count; i++) { + CBlockHeader header{ConsumeBlockHeader(fuzzed_data_provider)}; + blocks.push_back(std::make_unique(std::move(header))); + blocks.back()->phashBlock = &g_block_hash; + blocks_info.push_back(blocks.back().get()); + } + + // Store these files and blocks in the block index. It should not fail. + assert(block_index.WriteBatchSync(files_info, files_count - 1, blocks_info)); + + // We should be able to read every block file info we stored. Its value should correspond to + // what we stored above. + CBlockFileInfo info; + for (const auto& [n, file_info]: files_info) { + assert(block_index.ReadBlockFileInfo(n, info)); + assert(info == *file_info); + } + + // We should be able to read the last block file number. Its value should be consistent. + int last_block_file; + assert(block_index.ReadLastBlockFile(last_block_file)); + assert(last_block_file == files_count - 1); + + // We should be able to flip and read the reindexing flag. + bool reindexing; + block_index.WriteReindexing(true); + block_index.ReadReindexing(reindexing); + assert(reindexing); + block_index.WriteReindexing(false); + block_index.ReadReindexing(reindexing); + assert(!reindexing); + + // We should be able to set and read the value of any random flag. + const std::string flag_name = fuzzed_data_provider.ConsumeRandomLengthString(100); + bool flag_value; + block_index.WriteFlag(flag_name, true); + block_index.ReadFlag(flag_name, flag_value); + assert(flag_value); + block_index.WriteFlag(flag_name, false); + block_index.ReadFlag(flag_name, flag_value); + assert(!flag_value); + + // We should be able to load everything we've previously stored. Note to assert on the + // return value we need to make sure all blocks pass the pow check. + const auto params{Params().GetConsensus()}; + const auto inserter = [&](const uint256&) { + return blocks.back().get(); + }; + WITH_LOCK(::cs_main, assert(block_index.LoadBlockIndexGuts(params, inserter))); +} diff --git a/src/test/fuzz/crypto_chacha20poly1305.cpp b/src/test/fuzz/crypto_chacha20poly1305.cpp new file mode 100644 index 000000000000..9eb3f5271cc4 --- /dev/null +++ b/src/test/fuzz/crypto_chacha20poly1305.cpp @@ -0,0 +1,200 @@ +// Copyright (c) 2020-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +constexpr static inline void crypt_till_rekey(FSChaCha20Poly1305& aead, int rekey_interval, bool encrypt) +{ + for (int i = 0; i < rekey_interval; ++i) { + std::byte dummy_tag[FSChaCha20Poly1305::EXPANSION] = {{}}; + if (encrypt) { + aead.Encrypt(Span{dummy_tag}.first(0), Span{dummy_tag}.first(0), dummy_tag); + } else { + aead.Decrypt(dummy_tag, Span{dummy_tag}.first(0), Span{dummy_tag}.first(0)); + } + } +} + +FUZZ_TARGET(crypto_aeadchacha20poly1305) +{ + FuzzedDataProvider provider{buffer.data(), buffer.size()}; + + auto key = provider.ConsumeBytes(32); + key.resize(32); + AEADChaCha20Poly1305 aead(key); + + // Initialize RNG deterministically, to generate contents and AAD. We assume that there are no + // (potentially buggy) edge cases triggered by specific values of contents/AAD, so we can avoid + // reading the actual data for those from the fuzzer input (which would need large amounts of + // data). + FastRandomContext rng{ConsumeUInt256(provider)}; + + LIMITED_WHILE(provider.ConsumeBool(), 10000) + { + // Mode: + // - Bit 0: whether to use single-plain Encrypt/Decrypt; otherwise use a split at prefix. + // - Bit 2: whether this ciphertext will be corrupted (making it the last sent one) + // - Bit 3-4: controls the maximum aad length (max 511 bytes) + // - Bit 5-7: controls the maximum content length (max 16383 bytes, for performance reasons) + unsigned mode = provider.ConsumeIntegral(); + bool use_splits = mode & 1; + bool damage = mode & 4; + unsigned aad_length_bits = 3 * ((mode >> 3) & 3); + unsigned aad_length = provider.ConsumeIntegralInRange(0, (1 << aad_length_bits) - 1); + unsigned length_bits = 2 * ((mode >> 5) & 7); + unsigned length = provider.ConsumeIntegralInRange(0, (1 << length_bits) - 1); + // Generate aad and content. + auto aad = rng.randbytes(aad_length); + auto plain = rng.randbytes(length); + std::vector cipher(length + AEADChaCha20Poly1305::EXPANSION); + // Generate nonce + AEADChaCha20Poly1305::Nonce96 nonce = {(uint32_t)rng(), rng()}; + + if (use_splits && length > 0) { + size_t split_index = provider.ConsumeIntegralInRange(1, length); + aead.Encrypt(Span{plain}.first(split_index), Span{plain}.subspan(split_index), aad, nonce, cipher); + } else { + aead.Encrypt(plain, aad, nonce, cipher); + } + + // Test Keystream output + std::vector keystream(length); + aead.Keystream(nonce, keystream); + for (size_t i = 0; i < length; ++i) { + assert((plain[i] ^ keystream[i]) == cipher[i]); + } + + std::vector decrypted_contents(length); + bool ok{false}; + + // damage the key + unsigned key_position = provider.ConsumeIntegralInRange(0, 31); + std::byte damage_val{(uint8_t)(1U << (key_position & 7))}; + std::vector bad_key = key; + bad_key[key_position] ^= damage_val; + + AEADChaCha20Poly1305 bad_aead(bad_key); + ok = bad_aead.Decrypt(cipher, aad, nonce, decrypted_contents); + assert(!ok); + + // Optionally damage 1 bit in either the cipher (corresponding to a change in transit) + // or the aad (to make sure that decryption will fail if the AAD mismatches). + if (damage) { + unsigned damage_bit = provider.ConsumeIntegralInRange(0, (cipher.size() + aad.size()) * 8U - 1U); + unsigned damage_pos = damage_bit >> 3; + std::byte damage_val{(uint8_t)(1U << (damage_bit & 7))}; + if (damage_pos >= cipher.size()) { + aad[damage_pos - cipher.size()] ^= damage_val; + } else { + cipher[damage_pos] ^= damage_val; + } + } + + if (use_splits && length > 0) { + size_t split_index = provider.ConsumeIntegralInRange(1, length); + ok = aead.Decrypt(cipher, aad, nonce, Span{decrypted_contents}.first(split_index), Span{decrypted_contents}.subspan(split_index)); + } else { + ok = aead.Decrypt(cipher, aad, nonce, decrypted_contents); + } + + // Decryption *must* fail if the packet was damaged, and succeed if it wasn't. + assert(!ok == damage); + if (!ok) break; + assert(decrypted_contents == plain); + } +} + +FUZZ_TARGET(crypto_fschacha20poly1305) +{ + FuzzedDataProvider provider{buffer.data(), buffer.size()}; + + uint32_t rekey_interval = provider.ConsumeIntegralInRange(32, 512); + auto key = provider.ConsumeBytes(32); + key.resize(32); + FSChaCha20Poly1305 enc_aead(key, rekey_interval); + FSChaCha20Poly1305 dec_aead(key, rekey_interval); + + // Initialize RNG deterministically, to generate contents and AAD. We assume that there are no + // (potentially buggy) edge cases triggered by specific values of contents/AAD, so we can avoid + // reading the actual data for those from the fuzzer input (which would need large amounts of + // data). + FastRandomContext rng{ConsumeUInt256(provider)}; + + LIMITED_WHILE(provider.ConsumeBool(), 10000) + { + // Mode: + // - Bit 0: whether to use single-plain Encrypt/Decrypt; otherwise use a split at prefix. + // - Bit 2: whether this ciphertext will be corrupted (making it the last sent one) + // - Bit 3-4: controls the maximum aad length (max 511 bytes) + // - Bit 5-7: controls the maximum content length (max 16383 bytes, for performance reasons) + unsigned mode = provider.ConsumeIntegral(); + bool use_splits = mode & 1; + bool damage = mode & 4; + unsigned aad_length_bits = 3 * ((mode >> 3) & 3); + unsigned aad_length = provider.ConsumeIntegralInRange(0, (1 << aad_length_bits) - 1); + unsigned length_bits = 2 * ((mode >> 5) & 7); + unsigned length = provider.ConsumeIntegralInRange(0, (1 << length_bits) - 1); + // Generate aad and content. + auto aad = rng.randbytes(aad_length); + auto plain = rng.randbytes(length); + std::vector cipher(length + FSChaCha20Poly1305::EXPANSION); + + crypt_till_rekey(enc_aead, rekey_interval, true); + if (use_splits && length > 0) { + size_t split_index = provider.ConsumeIntegralInRange(1, length); + enc_aead.Encrypt(Span{plain}.first(split_index), Span{plain}.subspan(split_index), aad, cipher); + } else { + enc_aead.Encrypt(plain, aad, cipher); + } + + std::vector decrypted_contents(length); + bool ok{false}; + + // damage the key + unsigned key_position = provider.ConsumeIntegralInRange(0, 31); + std::byte damage_val{(uint8_t)(1U << (key_position & 7))}; + std::vector bad_key = key; + bad_key[key_position] ^= damage_val; + + FSChaCha20Poly1305 bad_fs_aead(bad_key, rekey_interval); + crypt_till_rekey(bad_fs_aead, rekey_interval, false); + ok = bad_fs_aead.Decrypt(cipher, aad, decrypted_contents); + assert(!ok); + + // Optionally damage 1 bit in either the cipher (corresponding to a change in transit) + // or the aad (to make sure that decryption will fail if the AAD mismatches). + if (damage) { + unsigned damage_bit = provider.ConsumeIntegralInRange(0, (cipher.size() + aad.size()) * 8U - 1U); + unsigned damage_pos = damage_bit >> 3; + std::byte damage_val{(uint8_t)(1U << (damage_bit & 7))}; + if (damage_pos >= cipher.size()) { + aad[damage_pos - cipher.size()] ^= damage_val; + } else { + cipher[damage_pos] ^= damage_val; + } + } + + crypt_till_rekey(dec_aead, rekey_interval, false); + if (use_splits && length > 0) { + size_t split_index = provider.ConsumeIntegralInRange(1, length); + ok = dec_aead.Decrypt(cipher, aad, Span{decrypted_contents}.first(split_index), Span{decrypted_contents}.subspan(split_index)); + } else { + ok = dec_aead.Decrypt(cipher, aad, decrypted_contents); + } + + // Decryption *must* fail if the packet was damaged, and succeed if it wasn't. + assert(!ok == damage); + if (!ok) break; + assert(decrypted_contents == plain); + } +} diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index 5d020b4d5938..79981b35bce0 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include #include #include @@ -19,8 +19,8 @@ #include #include -using kernel::DumpMempool; -using kernel::LoadMempool; +using node::DumpMempool; +using node::LoadMempool; using node::MempoolPath; diff --git a/src/test/ipc_test.capnp b/src/test/ipc_test.capnp new file mode 100644 index 000000000000..55a3dc268393 --- /dev/null +++ b/src/test/ipc_test.capnp @@ -0,0 +1,18 @@ +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xd71b0fc8727fdf83; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("gen"); + +using Proxy = import "/mp/proxy.capnp"; +$Proxy.include("test/ipc_test.h"); +$Proxy.includeTypes("ipc/capnp/common-types.h"); + +interface FooInterface $Proxy.wrap("FooImplementation") { + add @0 (a :Int32, b :Int32) -> (result :Int32); + passOutPoint @1 (arg :Data) -> (result :Data); + passUniValue @2 (arg :Text) -> (result :Text); +} diff --git a/src/test/ipc_test.cpp b/src/test/ipc_test.cpp new file mode 100644 index 000000000000..33d2cfb0681f --- /dev/null +++ b/src/test/ipc_test.cpp @@ -0,0 +1,67 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +//! Unit test that tests execution of IPC calls without actually creating a +//! separate process. This test is primarily intended to verify behavior of type +//! conversion code that converts C++ objects to Cap'n Proto messages and vice +//! versa. +//! +//! The test creates a thread which creates a FooImplementation object (defined +//! in ipc_test.h) and a two-way pipe accepting IPC requests which call methods +//! on the object through FooInterface (defined in ipc_test.capnp). +void IpcTest() +{ + // Setup: create FooImplemention object and listen for FooInterface requests + std::promise>> foo_promise; + std::function disconnect_client; + std::thread thread([&]() { + mp::EventLoop loop("IpcTest", [](bool raise, const std::string& log) { LogPrintf("LOG%i: %s\n", raise, log); }); + auto pipe = loop.m_io_context.provider->newTwoWayPipe(); + + auto connection_client = std::make_unique(loop, kj::mv(pipe.ends[0])); + auto foo_client = std::make_unique>( + connection_client->m_rpc_system->bootstrap(mp::ServerVatId().vat_id).castAs(), + connection_client.get(), /* destroy_connection= */ false); + foo_promise.set_value(std::move(foo_client)); + disconnect_client = [&] { loop.sync([&] { connection_client.reset(); }); }; + + auto connection_server = std::make_unique(loop, kj::mv(pipe.ends[1]), [&](mp::Connection& connection) { + auto foo_server = kj::heap>(std::make_shared(), connection); + return capnp::Capability::Client(kj::mv(foo_server)); + }); + connection_server->onDisconnect([&] { connection_server.reset(); }); + loop.loop(); + }); + std::unique_ptr> foo{foo_promise.get_future().get()}; + + // Test: make sure arguments were sent and return value is received + BOOST_CHECK_EQUAL(foo->add(1, 2), 3); + + COutPoint txout1{uint256{100}, 200}; + COutPoint txout2{foo->passOutPoint(txout1)}; + BOOST_CHECK(txout1 == txout2); + + UniValue uni1{UniValue::VOBJ}; + uni1.pushKV("i", 1); + uni1.pushKV("s", "two"); + UniValue uni2{foo->passUniValue(uni1)}; + BOOST_CHECK_EQUAL(uni1.write(), uni2.write()); + + // Test cleanup: disconnect pipe and join thread + disconnect_client(); + thread.join(); +} diff --git a/src/test/ipc_test.h b/src/test/ipc_test.h new file mode 100644 index 000000000000..bcfcc2125c68 --- /dev/null +++ b/src/test/ipc_test.h @@ -0,0 +1,21 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_IPC_TEST_H +#define BITCOIN_TEST_IPC_TEST_H + +#include +#include + +class FooImplementation +{ +public: + int add(int a, int b) { return a + b; } + COutPoint passOutPoint(COutPoint o) { return o; } + UniValue passUniValue(UniValue v) { return v; } +}; + +void IpcTest(); + +#endif // BITCOIN_TEST_IPC_TEST_H diff --git a/src/test/ipc_tests.cpp b/src/test/ipc_tests.cpp new file mode 100644 index 000000000000..6e144b0f418b --- /dev/null +++ b/src/test/ipc_tests.cpp @@ -0,0 +1,13 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include + +BOOST_AUTO_TEST_SUITE(ipc_tests) +BOOST_AUTO_TEST_CASE(ipc_tests) +{ + IpcTest(); +} +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 35407c665f43..288c74fc80b0 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -284,7 +284,7 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, const std::vecto const ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .datadir = m_args.GetDataDirNet(), - .check_block_index = true, + .check_block_index = 1, .superblock_status = [&sbman = *m_node.sbman](const CDeterministicMNList& mn_list, int height) { return sbman.GetStatus(mn_list, height); }, .mn_sync = m_node.mn_sync.get(), .notifications = *m_node.notifications, diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 6fc6c30bb400..7b5ded2975b5 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -81,6 +81,8 @@ template std::optional> TryParseHex(std::string_view str) { std::vector vch; + vch.reserve(str.size() / 2); // two hex characters form a single byte + auto it = str.begin(); while (it != str.end()) { if (IsSpace(*it)) { diff --git a/src/validation.cpp b/src/validation.cpp index 5f893abbd353..8f0ea0c27a43 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -10,9 +10,6 @@ #include -#include -#include - #include #include #include @@ -30,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -5834,6 +5832,14 @@ void ChainstateManager::LoadExternalBlockFile( } } +bool ChainstateManager::ShouldCheckBlockIndex() const +{ + // Assert to verify Flatten() has been called. + if (!*Assert(m_options.check_block_index)) return false; + if (GetRand(*m_options.check_block_index) >= 1) return false; + return true; +} + void ChainstateManager::CheckBlockIndex() { if (!ShouldCheckBlockIndex()) { @@ -5850,19 +5856,28 @@ void ChainstateManager::CheckBlockIndex() return; } - // Build forward-pointing map of the entire block tree. + // Build forward-pointing data structure for the entire block tree. + // For performance reasons, indexes of the best header chain are stored in a vector (within CChain). + // All remaining blocks are stored in a multimap. + // The best header chain can differ from the active chain: E.g. its entries may belong to blocks that + // are not yet validated. + CChain best_hdr_chain; + assert(m_best_header); + best_hdr_chain.SetTip(*m_best_header); + std::multimap forward; for (auto& [_, block_index] : m_blockman.m_block_index) { - forward.emplace(block_index.pprev, &block_index); + // Only save indexes in forward that are not part of the best header chain. + if (!best_hdr_chain.Contains(&block_index)) { + // Only genesis, which must be part of the best header chain, can have a nullptr parent. + assert(block_index.pprev); + forward.emplace(block_index.pprev, &block_index); + } } + assert(forward.size() + best_hdr_chain.Height() + 1 == m_blockman.m_block_index.size()); - assert(forward.size() == m_blockman.m_block_index.size()); - - std::pair::iterator,std::multimap::iterator> rangeGenesis = forward.equal_range(nullptr); - CBlockIndex *pindex = rangeGenesis.first->second; - rangeGenesis.first++; - assert(rangeGenesis.first == rangeGenesis.second); // There is only one index entry with parent nullptr. - + CBlockIndex* pindex = best_hdr_chain[0]; + assert(pindex); // Iterate over the entire block tree, using depth-first search. // Along the way, remember whether there are blocks on the path from genesis // block being explored which are the first to have certain properties. @@ -6035,13 +6050,20 @@ void ChainstateManager::CheckBlockIndex() // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow // End: actual consistency checks. - // Try descending into the first subnode. + + // Try descending into the first subnode. Always process forks first and the best header chain after. std::pair::iterator,std::multimap::iterator> range = forward.equal_range(pindex); if (range.first != range.second) { - // A subnode was found. + // A subnode not part of the best header chain was found. pindex = range.first->second; nHeight++; continue; + } else if (best_hdr_chain.Contains(pindex)) { + // Descend further into best header chain. + nHeight++; + pindex = best_hdr_chain[nHeight]; + if (!pindex) break; // we are finished, since the best header chain is always processed last + continue; } // This is a leaf node. // Move upwards until we reach a node of which we have not yet visited the last child. @@ -6068,9 +6090,15 @@ void ChainstateManager::CheckBlockIndex() // Proceed to the next one. rangePar.first++; if (rangePar.first != rangePar.second) { - // Move to the sibling. + // Move to a sibling not part of the best header chain. pindex = rangePar.first->second; break; + } else if (pindexPar == best_hdr_chain[nHeight - 1]) { + // Move to pindex's sibling on the best-chain, if it has one. + pindex = best_hdr_chain[nHeight]; + // There will not be a next block if (and only if) parent block is the best header. + assert((pindex == nullptr) == (pindexPar == best_hdr_chain.Tip())); + break; } else { // Move up further. pindex = pindexPar; @@ -6080,8 +6108,8 @@ void ChainstateManager::CheckBlockIndex() } } - // Check that we actually traversed the entire map. - assert(nNodes == forward.size()); + // Check that we actually traversed the entire block index. + assert(nNodes == forward.size() + best_hdr_chain.Height() + 1); } std::string Chainstate::ToString() diff --git a/src/validation.h b/src/validation.h index 1b914f00a593..ae08f5ef99bd 100644 --- a/src/validation.h +++ b/src/validation.h @@ -965,7 +965,7 @@ class ChainstateManager const CChainParams& GetParams() const { return m_options.chainparams; } const Consensus::Params& GetConsensus() const { return m_options.chainparams.GetConsensus(); } - bool ShouldCheckBlockIndex() const { return *Assert(m_options.check_block_index); } + bool ShouldCheckBlockIndex() const; const arith_uint256& MinimumChainWork() const { return *Assert(m_options.minimum_chain_work); } const uint256& AssumedValidBlock() const { return *Assert(m_options.assumed_valid_block); } kernel::Notifications& GetNotifications() const { return m_options.notifications; }; diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py index a8912268946b..d1b543603106 100755 --- a/test/lint/lint-circular-dependencies.py +++ b/test/lint/lint-circular-dependencies.py @@ -20,7 +20,6 @@ "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel", "wallet/wallet -> wallet/walletdb -> wallet/wallet", "kernel/coinstats -> validation -> kernel/coinstats", - "kernel/mempool_persist -> validation -> kernel/mempool_persist", # Temporary, removed in followup https://github.com/bitcoin/bitcoin/pull/24230 "index/base -> node/context -> net_processing -> index/blockfilterindex -> index/base",