From b9322fcba246445808626286dfaf7d1401b28754 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Tue, 24 Oct 2023 10:13:55 +0100 Subject: [PATCH 01/11] Merge bitcoin/bitcoin#28660: test: enable reindex readonly test on *BSD 5a0688a20d88a9641c02436abbd7b49e227f1a37 test: enable reindex readonly test on *BSD and macOS as root (Matthew Zipkin) Pull request description: see https://github.com/bitcoin/bitcoin/pull/27850#discussion_r1349505585 OpenBSD and FreeBSD don't have `chattr` but they do have `chflags`, use that method to make the block file immutable for the reindex_readonly test. Written and tested on a VPS running FreeBSD: ``` FreeBSD freebsd-13-1 13.2-RELEASE-p4 FreeBSD 13.2-RELEASE-p4 GENERIC amd64 ``` ACKs for top commit: maflcko: re-cr-lgtm-ACK 5a0688a20d88a9641c02436abbd7b49e227f1a37 jonatack: ACK 5a0688a20d88a9641c02436abbd7b49e227f1a37 tested on macOS only theStack: ACK 5a0688a20d88a9641c02436abbd7b49e227f1a37 Tree-SHA512: 8c88d282d09c00355d22c4c504b779f60e420327a5e07bcf80fa77b97fefcb04952af9ceaf439d9033a0a2448cb26a02663fe6bddcd4a74792857cfbaf1c5162 Co-authored-by: fanquake --- test/functional/feature_reindex_readonly.py | 43 +++++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/test/functional/feature_reindex_readonly.py b/test/functional/feature_reindex_readonly.py index 26531f472bf8..dd99c3c4fa31 100755 --- a/test/functional/feature_reindex_readonly.py +++ b/test/functional/feature_reindex_readonly.py @@ -7,7 +7,6 @@ """ import os -import platform import stat import subprocess from test_framework.test_framework import BitcoinTestFramework @@ -34,11 +33,13 @@ def reindex_readonly(self): filename = self.nodes[0].chain_path / "blocks" / "blk00000.dat" filename.chmod(stat.S_IREAD) - used_chattr = False - if platform.system() == "Linux": + undo_immutable = lambda: None + # Linux + try: + subprocess.run(['chattr'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) try: subprocess.run(['chattr', '+i', filename], capture_output=True, check=True) - used_chattr = True + undo_immutable = lambda: subprocess.check_call(['chattr', '-i', filename]) self.log.info("Made file immutable with chattr") except subprocess.CalledProcessError as e: self.log.warning(str(e)) @@ -50,15 +51,33 @@ def reindex_readonly(self): self.log.warning("Return early on Linux under root, because chattr failed.") self.log.warning("This should only happen due to missing capabilities in a container.") self.log.warning("Make sure to --cap-add LINUX_IMMUTABLE if you want to run this test.") - return - - self.log.debug("Attempt to restart and reindex the node with the unwritable block file") - with self.nodes[0].assert_debug_log(expected_msgs=['FlushStateToDisk', 'failed to open file'], unexpected_msgs=[]): - self.nodes[0].assert_start_raises_init_error(extra_args=['-reindex', '-fastprune'], - expected_msg="Error: A fatal internal error occurred, see debug.log for details") + undo_immutable = False + except Exception: + # macOS, and *BSD + try: + subprocess.run(['chflags'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + subprocess.run(['chflags', 'uchg', filename], capture_output=True, check=True) + undo_immutable = lambda: subprocess.check_call(['chflags', 'nouchg', filename]) + self.log.info("Made file immutable with chflags") + except subprocess.CalledProcessError as e: + self.log.warning(str(e)) + if e.stdout: + self.log.warning(f"stdout: {e.stdout}") + if e.stderr: + self.log.warning(f"stderr: {e.stderr}") + if os.getuid() == 0: + self.log.warning("Return early on BSD under root, because chflags failed.") + undo_immutable = False + except Exception: + pass - if used_chattr: - subprocess.check_call(['chattr', '-i', filename]) + if undo_immutable: + self.log.info("Attempt to restart and reindex the node with the unwritable block file") + with self.nodes[0].assert_debug_log(expected_msgs=['FlushStateToDisk', 'failed to open file'], unexpected_msgs=[]): + self.nodes[0].assert_start_raises_init_error(extra_args=['-reindex', '-fastprune'], + expected_msg="Error: A fatal internal error occurred, see debug.log for details") + undo_immutable() filename.chmod(0o777) From be6b03b4608adae5776b9fb6baaac31d33aaf6d5 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Thu, 14 Sep 2023 10:39:10 +0100 Subject: [PATCH 02/11] Merge bitcoin/bitcoin#28423: kernel: Remove protocol.h/netaddress.h/compat.h from kernel headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit d5067651991f3e6daf456ba13c7036ddc4545352 [refactor] Remove compat.h from kernel headers (TheCharlatan) 36193af47c8dcff53e59498c416b85b59e0d0f91 [refactor] Remove netaddress.h from kernel headers (TheCharlatan) 2b08c55f01996e0b05763f05eac50b83ba9d5a8e [refactor] Add CChainParams member to CConnman (TheCharlatan) f0d1d8b35c3aa9f2f923f74e3dbbf1e5ece4cd2f [refactor] Add missing includes for next commit (TheCharlatan) 534b314a7401d44f51aabd4565f97be9ee411740 kernel: Move MessageStartChars to its own file (TheCharlatan) 9be330b654cfbd792620295f3867f592059d6a7a [refactor] Define MessageStartChars as std::array (TheCharlatan) 37e2b011136ca1cf00dfb9e575d12f0d035a6a2c [refactor] Allow std::array in serialize.h (MarcoFalke) Pull request description: This removes the non-consensus critical `protocol.h` and `netaddress.h` headers from the kernel headers. With this patch, they are no longer required to include in order to use the libbitcoinkernel library. This also allows for the removal of the `compat.h` header from the kernel headers. As an added future benefit it also reduces the number of of kernel headers that include the platform specific `bitcoin-config.h`. For those interested, the currently required kernel headers can be inspected visually with the [sourcetrail](https://github.com/CoatiSoftware/Sourcetrail) tool by looking at the required includes of `bitcoin-chainstate.cpp`. --- This is part of the [libbitcoinkernel project](https://github.com/bitcoin/bitcoin/issues/27587), namely its stage 1 step 3: Decouple most non-consensus headers from libbitcoinkernel. ACKs for top commit: stickies-v: re-ACK d506765 hebasto: ACK d5067651991f3e6daf456ba13c7036ddc4545352. ajtowns: utACK d5067651991f3e6daf456ba13c7036ddc4545352 MarcoFalke: lgtm ACK d5067651991f3e6daf456ba13c7036ddc4545352 🍛 Tree-SHA512: 6f90ea510a302c2927e84d16900e89997c39b8ff3ce9d4effeb8a134bd29cc52bd9e81e51aaa11f7496bad00025b78a58b88c5a9e0bb3f4ebbe9a76309215fb7 Co-authored-by: fanquake --- src/Makefile.am | 1 + src/addrdb.cpp | 4 +-- src/bench/block_assemble.cpp | 1 + src/bench/duplicate_inputs.cpp | 1 + src/bench/mempool_stress.cpp | 1 + src/bitcoin-chainstate.cpp | 1 + src/bitcoin-util.cpp | 1 + src/flat-database.h | 4 +-- src/init.cpp | 2 +- src/kernel/chainparams.cpp | 1 + src/kernel/chainparams.h | 16 +++-------- src/kernel/messagestartchars.h | 13 +++++++++ src/key_io.cpp | 1 + src/net.cpp | 37 +++++++++++++++++--------- src/net.h | 11 ++++++-- src/node/blockstorage.cpp | 7 ++--- src/node/blockstorage.h | 8 +++--- src/protocol.cpp | 2 +- src/protocol.h | 10 +++---- src/random.cpp | 1 + src/randomenv.cpp | 1 + src/test/denialofservice_tests.cpp | 8 +++--- src/test/evo_mnauth_tests.cpp | 3 ++- src/test/fuzz/connman.cpp | 1 + src/test/fuzz/descriptor_parse.cpp | 1 + src/test/fuzz/net.cpp | 2 +- src/test/net_peer_connection_tests.cpp | 2 +- src/test/net_tests.cpp | 2 +- src/test/txvalidationcache_tests.cpp | 2 ++ src/test/util/setup_common.cpp | 2 +- src/util/time.h | 2 -- src/validation.cpp | 5 ++-- src/wallet/db.cpp | 2 +- src/wallet/sqlite.cpp | 4 +-- 34 files changed, 98 insertions(+), 62 deletions(-) create mode 100644 src/kernel/messagestartchars.h diff --git a/src/Makefile.am b/src/Makefile.am index 7069ea92ed85..06df7b96dafb 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -300,6 +300,7 @@ BITCOIN_CORE_H = \ kernel/mempool_options.h \ kernel/mempool_persist.h \ kernel/mempool_removal_reason.h \ + kernel/messagestartchars.h \ kernel/notifications_interface.h \ kernel/validation_cache_sizes.h \ key.h \ diff --git a/src/addrdb.cpp b/src/addrdb.cpp index a1a536ea2b4f..8bafa5763c89 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -98,10 +98,10 @@ void DeserializeDB(Stream& stream, Data&& data, bool fCheckSum = true) { HashVerifier verifier{stream}; // de-serialize file header (network specific magic number) and .. - unsigned char pchMsgTmp[4]; + MessageStartChars pchMsgTmp; verifier >> pchMsgTmp; // ... verify the network matches ours - if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) { + if (pchMsgTmp != Params().MessageStart()) { throw std::runtime_error{"Invalid network magic number"}; } diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index c193f5e7801e..10271bf52909 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -7,6 +7,7 @@ #include #include