From d3910c6336959a91fb4453bdf9f26b9c7f20a3f4 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Sat, 14 Nov 2026 19:13:54 +0000 Subject: [PATCH 01/40] test: make two functional test checks able to fail p2p_instantsend.py asserted that the key "confirmation" was absent from getrawtransaction, but the key is "confirmations", so the check that a transaction without a lock is not mined before it is 10 minutes old never ran. It would have failed: the isolated node's mocktime lagged behind the bumps made while waiting for the lock, so the transaction was already old enough. Sync the isolated node's clock and assert the early block leaves the transaction in the mempool. The rejection of the doublespend block was checked with waitforblock timeouts of 10 ms and 1 ms, before the block could arrive; wait for it to be listed as invalid instead. feature_masternode_params.py only logged the masternode's services and indexes and asserted getblockcount() >= 0, and looked the block filter index up under the wrong name, so its "regular node has no filter index" check could not fail either. Assert the service bit and the index. Co-Authored-By: Claude Fable 5.2 Co-authored-by: pasta --- test/functional/feature_masternode_params.py | 10 ++----- test/functional/p2p_instantsend.py | 29 ++++++++++---------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/test/functional/feature_masternode_params.py b/test/functional/feature_masternode_params.py index a674a902e849..09da856265b1 100755 --- a/test/functional/feature_masternode_params.py +++ b/test/functional/feature_masternode_params.py @@ -16,7 +16,7 @@ NODE_COMPACT_FILTERS = (1 << 6) # Constants -BASIC_FILTER_INDEX = 'basic filter index' +BASIC_FILTER_INDEX = 'basic block filter index' class MasternodeParamsTest(BitcoinTestFramework): @@ -50,15 +50,11 @@ def run_test(self): # Masternode should have peerblockfilters enabled services = int(node1.getnetworkinfo()['localservices'], 16) - self.log.info(f"Masternode services: {hex(services)}, has COMPACT_FILTERS: {services & NODE_COMPACT_FILTERS != 0}") + assert services & NODE_COMPACT_FILTERS # Check blockfilterindex index_info = node1.getindexinfo() - self.log.info(f"Masternode indexes: {list(index_info.keys())}") - - # For now, just check that the node started successfully with masternode key - # The actual filter enabling might require the node to be fully synced - assert node1.getblockcount() >= 0 # Basic check that node is running + assert BASIC_FILTER_INDEX in index_info self.log.info("Test that masternode refuses settings that silently skip its duties") self.stop_node(1) diff --git a/test/functional/p2p_instantsend.py b/test/functional/p2p_instantsend.py index 7ae8fa353889..1c7dfbd4db51 100755 --- a/test/functional/p2p_instantsend.py +++ b/test/functional/p2p_instantsend.py @@ -81,27 +81,28 @@ def test_block_doublespend(self): connected_nodes = self.nodes.copy() del connected_nodes[self.isolated_idx] self.wait_for_instantlock(is_id, nodes=connected_nodes) - # send doublespend transaction to isolated node + # send doublespend transaction to isolated node, whose clock missed the bumps made while waiting for the lock + isolated.setmocktime(self.mocktime) dblspnd_txid = isolated.sendrawtransaction(dblspnd_tx['hex']) - # generate block on isolated node with doublespend transaction + # a transaction without a lock is not mined until it is 10 minutes old self.bump_mocktime(599) - wrong_early_block = self.generate(isolated, 1, sync_fun=self.no_op)[0] - assert not "confirmation" in isolated.getrawtransaction(dblspnd_txid, 1) - isolated.invalidateblock(wrong_early_block) + early_block = self.generate(isolated, 1, sync_fun=self.no_op)[0] + assert dblspnd_txid not in isolated.getblock(early_block)["tx"] + assert dblspnd_txid in isolated.getrawmempool() + isolated.invalidateblock(early_block) + # generate block on isolated node with doublespend transaction self.bump_mocktime(1) wrong_block = self.generate(isolated, 1, sync_fun=self.no_op)[0] assert_equal(isolated.getrawtransaction(dblspnd_txid, 1)["confirmations"], 1) # connect isolated block to network self.reconnect_isolated_node(self.isolated_idx, 0) - # check doublespend block is rejected by other nodes - timeout = 10 - for idx, node in enumerate(self.nodes): - if idx == self.isolated_idx: - continue - res = node.waitforblock(wrong_block, timeout) - assert res['hash'] != wrong_block - # wait for long time only for first node - timeout = 1 + # check doublespend block is rejected + self.wait_until(lambda: any(tip["hash"] == wrong_block and tip["status"] == "invalid" + for tip in self.nodes[0].getchaintips())) + self.sync_blocks([n for n in self.nodes if n is not isolated]) + for node in self.nodes: + if node is not isolated: + assert node.getbestblockhash() != wrong_block # send coins back to the controller node without waiting for confirmations receiver.sendtoaddress(self.nodes[0].getnewaddress(), 0.9, "", "", True) assert_equal(receiver.getwalletinfo()["balance"], 0) From 3786f8f64a04662038b93f7237fca839ad16842a Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Sun, 15 Nov 2026 05:20:18 +0000 Subject: [PATCH 02/40] test: make index and EHF checks observe the behavior they name feature_addressindex.py and feature_spentindex.py toggled the index off and on at genesis, before any block the index could miss. The toggle now covers a spend mined while the index is disabled, and the index has to catch up when it is turned back on. feature_mnehf.py asserted that a payload it had just filled in still contained those values, and polled activation one block at a time. The payload check is gone. Activation is mined across the BIP9 windows and the started, locked_in and active states are checked. feature_masternode_payout_shares.py asserted that v24 was active after a loop that only exits once it is. Co-Authored-By: Claude Fable 5.2 Co-authored-by: Cursor Co-authored-by: pasta --- test/functional/feature_addressindex.py | 13 ++------ .../feature_masternode_payout_shares.py | 1 - test/functional/feature_mnehf.py | 17 +++++----- test/functional/feature_spentindex.py | 31 +++++++++---------- 4 files changed, 24 insertions(+), 38 deletions(-) diff --git a/test/functional/feature_addressindex.py b/test/functional/feature_addressindex.py index 176083b3cf21..df68acb433db 100755 --- a/test/functional/feature_addressindex.py +++ b/test/functional/feature_addressindex.py @@ -49,15 +49,6 @@ def setup_network(self): self.import_deterministic_coinbase_privkeys() def run_test(self): - self.log.info("Test that settings can be disabled and re-enabled...") - self.restart_node(1, ["-addressindex=0"]) - self.connect_nodes(0, 1) - self.sync_all() - # AddressIndex is now async, so it can be enabled without -reindex - self.restart_node(1, ["-addressindex"]) - self.connect_nodes(0, 1) - self.sync_all() - self.log.info("Mining blocks...") mining_address = self.nodes[0].getnewaddress() self.generatetoaddress(self.nodes[0], 105, mining_address) @@ -361,10 +352,12 @@ def test_reorg_while_disabled(self): self.restart_node(3, ["-addressindex=0"]) node.invalidateblock(orphan_block) - self.generatetoaddress(node, 2, getnewdestination()[2], sync_fun=self.no_op) + new_address = getnewdestination()[2] + self.generatetoaddress(node, 2, new_address, sync_fun=self.no_op) self.restart_node(3, ["-addressindex"]) self.wait_index_synced(node) + assert_greater_than(node.getaddressbalance(new_address)["received"], 0) assert_equal(node.getaddressbalance(orphan_address)["received"], 0) assert_equal(node.getaddresstxids(orphan_address), []) diff --git a/test/functional/feature_masternode_payout_shares.py b/test/functional/feature_masternode_payout_shares.py index b3976c21f06a..0a541e3f2eed 100755 --- a/test/functional/feature_masternode_payout_shares.py +++ b/test/functional/feature_masternode_payout_shares.py @@ -30,7 +30,6 @@ def activate_v24(self): while not softfork_active(self.nodes[0], "v24"): self.bump_mocktime(50) self.generate(self.nodes[0], 50, sync_fun=self.no_op) - assert softfork_active(self.nodes[0], "v24") def build_block(self, node, tamper=None): """Build a block extending the tip whose coinbase pays exactly the template's diff --git a/test/functional/feature_mnehf.py b/test/functional/feature_mnehf.py index 851607a8e5bb..bc9581e6b7e0 100755 --- a/test/functional/feature_mnehf.py +++ b/test/functional/feature_mnehf.py @@ -145,13 +145,8 @@ def run_test(self): ehf_tx = self.create_mnehf(28, pubkey) ehf_unknown_tx = self.create_mnehf(27, pubkey) - self.log.info("Checking deserialization of CMnEhf by python's code") mnehf_payload = CMnEhf() mnehf_payload.deserialize(BytesIO(ehf_tx.vExtraPayload)) - assert_equal(mnehf_payload.version, 1) - assert_equal(mnehf_payload.versionBit, 28) - self.log.info("Checking correctness of requestId and quorumHash") - assert_equal(mnehf_payload.quorumHash, int(self.mninfo[0].get_node(self).quorum("selectquorum", 100, 'a0eee872d7d3170dd20d5c5e8380c92b3aa887da5f63d8033289fafa35a90691')["quorumHash"], 16)) ehf_tx_sent = self.send_tx(ehf_tx) self.log.info(f"ehf tx: {ehf_tx_sent}") @@ -243,11 +238,13 @@ def run_test(self): self.mine_quorum_single_member() self.check_fork('defined') - def check_ehf_activated(self): - self.bump_mocktime(1) - self.generate(self.nodes[1], 1) - return get_bip9_details(self.nodes[0], 'testdummy')['status'] == 'active' - self.wait_until(lambda: check_ehf_activated(self), sleep=1) + self.wait_until(lambda: len(self.ehf_signals_in_mempool(self.nodes[1], 28)) == 1) + self.generate(self.nodes[1], 4, sync_fun=self.sync_blocks) + self.check_fork('started') + self.generate(self.nodes[1], 4, sync_fun=self.sync_blocks) + self.check_fork('locked_in') + self.generate(self.nodes[1], 4, sync_fun=self.sync_blocks) + self.check_fork('active') if __name__ == '__main__': MnehfTest().main() diff --git a/test/functional/feature_spentindex.py b/test/functional/feature_spentindex.py index 9413e47023b6..00f862932676 100755 --- a/test/functional/feature_spentindex.py +++ b/test/functional/feature_spentindex.py @@ -29,7 +29,7 @@ def add_options(self, parser): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 4 + self.num_nodes = 3 def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -39,25 +39,14 @@ def setup_network(self): # Nodes 0/1 are "wallet" nodes self.start_node(0) self.start_node(1, ["-spentindex"]) - # Nodes 2/3 are used for testing - self.start_node(2, ["-spentindex"]) - self.start_node(3, ["-spentindex", "-txindex"]) + # Node 2 is used for testing + self.start_node(2, ["-spentindex", "-txindex"]) self.connect_nodes(0, 1) self.connect_nodes(0, 2) - self.connect_nodes(0, 3) self.sync_all() self.import_deterministic_coinbase_privkeys() def run_test(self): - self.log.info("Test that settings can be disabled and re-enabled...") - self.restart_node(1, ["-spentindex=0"]) - self.connect_nodes(0, 1) - self.sync_all() - # SpentIndex is now async, so it can be enabled without -reindex - self.restart_node(1, ["-spentindex"]) - self.connect_nodes(0, 1) - self.sync_all() - self.log.info("Mining blocks...") self.generate(self.nodes[0], 105) @@ -80,8 +69,16 @@ def run_test(self): tx.rehash() signed_tx = self.nodes[0].signrawtransactionwithwallet(tx.serialize().hex()) + self.log.info("Test that the index catches up without -reindex on a spend mined while it was disabled") + self.restart_node(1, ["-spentindex=0"]) + self.connect_nodes(0, 1) txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], 0) self.generate(self.nodes[0], 1) + # SpentIndex is async, so it can be enabled without -reindex + self.restart_node(1, ["-spentindex"]) + self.connect_nodes(0, 1) + self.sync_all() + self.wait_until(lambda: self.nodes[1].getindexinfo()["spentindex"] == {"synced": True, "best_block_height": 106}) self.log.info("Testing getspentinfo method...") @@ -94,13 +91,13 @@ def run_test(self): self.log.info("Testing getrawtransaction method...") # Check that verbose raw transaction includes spent info - txVerbose = self.nodes[3].getrawtransaction(unspent[0]["txid"], 1) + txVerbose = self.nodes[2].getrawtransaction(unspent[0]["txid"], 1) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentTxId"], txid) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentIndex"], 0) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentHeight"], 106) # Check that verbose raw transaction includes input values - txVerbose2 = self.nodes[3].getrawtransaction(txid, 1) + txVerbose2 = self.nodes[2].getrawtransaction(txid, 1) assert_equal(txVerbose2["vin"][0]["value"], Decimal(unspent[0]["amount"])) assert_equal(txVerbose2["vin"][0]["valueSat"] - tx_fee_sat, amount) @@ -134,7 +131,7 @@ def run_test(self): # Check the database index self.generate(self.nodes[0], 1) - txVerbose4 = self.nodes[3].getrawtransaction(txid2, 1) + txVerbose4 = self.nodes[2].getrawtransaction(txid2, 1) assert_equal(txVerbose4["vin"][0]["address"], address2) assert_equal(txVerbose4["vin"][0]["value"], Decimal(unspent[0]["amount"]) - tx_fee) assert_equal(txVerbose4["vin"][0]["valueSat"], amount) From 37a9ec71a4e932706a6ca78030812716d9e4d695 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Sun, 15 Nov 2026 05:36:02 +0000 Subject: [PATCH 03/40] test: require the address index to catch both blocks mined while disabled A received total greater than zero still passed if only one of the two coinbases was indexed. Compare it with the sum paid to the address by both blocks. Co-authored-by: Cursor Co-authored-by: pasta --- test/functional/feature_addressindex.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/functional/feature_addressindex.py b/test/functional/feature_addressindex.py index df68acb433db..42c31105d1fb 100755 --- a/test/functional/feature_addressindex.py +++ b/test/functional/feature_addressindex.py @@ -353,11 +353,16 @@ def test_reorg_while_disabled(self): self.restart_node(3, ["-addressindex=0"]) node.invalidateblock(orphan_block) new_address = getnewdestination()[2] - self.generatetoaddress(node, 2, new_address, sync_fun=self.no_op) + block_hashes = self.generatetoaddress(node, 2, new_address, sync_fun=self.no_op) + # Greater-than-zero would pass if only one of the two blocks was indexed. + paid = sum(vout["valueSat"] + for block_hash in block_hashes + for vout in node.getblock(block_hash, 2)["tx"][0]["vout"] + if vout["scriptPubKey"].get("address") == new_address) self.restart_node(3, ["-addressindex"]) self.wait_index_synced(node) - assert_greater_than(node.getaddressbalance(new_address)["received"], 0) + assert_equal(node.getaddressbalance(new_address)["received"], paid) assert_equal(node.getaddressbalance(orphan_address)["received"], 0) assert_equal(node.getaddresstxids(orphan_address), []) From 990a0f8a6c0895a1c0433d387cf44ac8eac11dcc Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Sun, 15 Nov 2026 06:52:15 +0000 Subject: [PATCH 04/40] test: activate the payout-shares v24 deployment with activate_by_name Setup already mines past height 100, so the old loop was a no-op and the helper's pending-fork check would fail. The minimum activation height now stays above that premine. Co-authored-by: Cursor Co-authored-by: pasta --- test/functional/feature_masternode_payout_shares.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/functional/feature_masternode_payout_shares.py b/test/functional/feature_masternode_payout_shares.py index 0a541e3f2eed..8360d7391365 100755 --- a/test/functional/feature_masternode_payout_shares.py +++ b/test/functional/feature_masternode_payout_shares.py @@ -8,9 +8,11 @@ from test_framework.messages import CTxOut, tx_from_hex from test_framework.script import CScript from test_framework.test_framework import DashTestFramework, MasternodeInfo, p2p_port -from test_framework.util import assert_equal, softfork_active +from test_framework.util import assert_equal -V24_ACTIVATION_THRESHOLD = 100 +# setup_nodes mines through coinbase maturity (height ~109) before run_test. +# The deployment has to still be pending so activate_by_name can bring v24 in. +V24_ACTIVATION_THRESHOLD = 200 def payout_address_rewards(payouts): @@ -27,9 +29,7 @@ def set_test_params(self): ]]) def activate_v24(self): - while not softfork_active(self.nodes[0], "v24"): - self.bump_mocktime(50) - self.generate(self.nodes[0], 50, sync_fun=self.no_op) + self.activate_by_name("v24") def build_block(self, node, tamper=None): """Build a block extending the tip whose coinbase pays exactly the template's From 852654b8f92de17c336a5fd286bfd3947ada1bb9 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Tue, 5 Sep 2023 11:40:21 +0300 Subject: [PATCH 05/40] Merge bitcoin/bitcoin#28291: rpc: removed StrFormatInternalBug quote delimitation 6e8f6468cbf1320b70cf01333002a31b44cb7c33 removed StrFormatInternalBug quote delimitation (Reese Russell) Pull request description: This PR rectifies an unnecessary set of quotes delimiting the contents of ```StrFormatInternalBug```. This is a follow up to MarcoFalke https://github.com/bitcoin/bitcoin/pull/28123#discussion_r1297191493. The method of action was to remove the escaped quotes that were a part of strprintf. A single functional test case was modified to reflect the new output format. ```STR_INTERNAL_BUG``` was applied to https://github.com/bitcoin/bitcoin/pull/28123 in ```std::string RPCArg::ToString(const bool oneline)``` in ```rpc/util.cpp``` The results can be seen below. Previously ![image](https://github.com/bitcoin/bitcoin/assets/3104223/53f9ea59-317f-4c62-9fc1-04255eeb4641) This PR ![image](https://github.com/bitcoin/bitcoin/assets/3104223/5c6a3110-f1f3-4b3c-8e8a-9c8f1c3176e7) Additional context can be found here. https://github.com/bitcoin/bitcoin/pull/28123#discussion_r1271871716 Thank you. ACKs for top commit: MarcoFalke: review ACK 6e8f6468cbf1320b70cf01333002a31b44cb7c33 stickies-v: ACK 6e8f6468cbf1320b70cf01333002a31b44cb7c33 Tree-SHA512: 35317e31a527630495b566407e37db9941dab7f81cfaeb1ea3309683c48e4273284645ad615f73e646a137b4f2ae35933603e9182a7dbdd22cac98d038c491dc Co-authored-by: fanquake --- src/rpc/util.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 7ac6af392fb9..226cb108b4fc 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -1147,10 +1147,9 @@ std::string RPCArg::ToString(const bool oneline) const if (oneline && !m_opts.oneline_description.empty()) { if (m_opts.oneline_description[0] == '\"' && m_type != Type::STR_HEX && m_type != Type::STR && gArgs.GetBoolArg("-rpcdoccheck", DEFAULT_RPC_DOC_CHECK)) { throw std::runtime_error{ - strprintf("Internal bug detected: non-string RPC arg \"%s\" quotes oneline_description:\n%s\n%s %s\nPlease report this issue here: %s\n", - m_names, m_opts.oneline_description, - PACKAGE_NAME, FormatFullVersion(), - PACKAGE_BUGREPORT)}; + STR_INTERNAL_BUG(strprintf("non-string RPC arg \"%s\" quotes oneline_description:\n%s", + m_names, m_opts.oneline_description) + )}; } return m_opts.oneline_description; } From 2de09c848814937f0549fb35848a02a429e03fab Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Fri, 6 Nov 2026 22:32:25 +0700 Subject: [PATCH 06/40] refactor: always serialize CDeterministicMN::nType CDeterministicMN gated nType on the stream version: it was written when the version was CLIENT_VERSION (taken to mean "disk") or at least DMN_TYPE_PROTO_VERSION (taken to mean "network"), and defaulted to MnType::Regular otherwise. The network branch is dead: nothing sends CDeterministicMN over P2P, peers get CSimplifiedMNListEntry instead. So nType is always written and read, and the on-disk bytes do not change. Dropping the gate removes the last dependency of CDeterministicMN on the stream version, which the versionless DataStream used by dbwrapper after the following backports does not provide. --- src/evo/deterministicmns.h | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/evo/deterministicmns.h b/src/evo/deterministicmns.h index ae49a23ac95f..64d4292b7f2a 100644 --- a/src/evo/deterministicmns.h +++ b/src/evo/deterministicmns.h @@ -73,14 +73,7 @@ class CDeterministicMN READWRITE(obj.collateralOutpoint); READWRITE(obj.nOperatorReward); READWRITE(obj.pdmnState); - // We can't know if we are serialising for the Disk or for the Network here (s.GetType() is not accessible) - // Therefore if s.GetVersion() == CLIENT_VERSION -> Then we know we are serialising for the Disk - // Otherwise, we can safely check with protocol versioning logic so we won't break old clients - if (s.GetVersion() == CLIENT_VERSION || s.GetVersion() >= DMN_TYPE_PROTO_VERSION) { - READWRITE(obj.nType); - } else { - SER_READ(obj, obj.nType = MnType::Regular); - } + READWRITE(obj.nType); } [[nodiscard]] bool HasInternalId() const { return internalId != std::numeric_limits::max(); } From 93e10bc3bd2a6baba35c6f0570bf2b9130c576c2 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Sat, 9 Sep 2023 11:16:31 +0100 Subject: [PATCH 07/40] Merge bitcoin/bitcoin#28428: Hard-code version number value for CBlockLocator and CDiskBlockIndex e73d2a8018def940afadb5d699b18f39e882c1fc refactor: remove clientversion include from dbwrapper.h (Cory Fields) 4240a082b81d8ceb7615b1b4ca0d2857382f317b refactor: Use DataStream now that version/type are unused (Cory Fields) f15f790618d328abd207d55e6291229eb2a8643f Remove version/hashing options from CBlockLocator/CDiskBlockIndex (Cory Fields) Pull request description: This is also a much simpler replacement for #28327. There are version fields in `CBlockLocator` and `CDiskBlockIndex` that have always been written but discarded when read. I intended to convert them to use SerParams as introduced by #25284, which [ended up looking like this](https://github.com/theuni/bitcoin/commit/3e3af451652322c92e8e41cf918e69d608ec7c77). However because we don't currently have any definition of what a hash value would mean for either one of those, and we've never assigned the version field any meaning, I think it's better to just not worry about them. If we ever need to assign meaning in the future, we can introduce `SerParams` as was done for `CAddress`. As for the dummy values chosen: `CDiskBlockIndex::DUMMY_VERSION` was easy as the highest ever client version, and I don't expect any objection there. `CBlockLocator::DUMMY_VERSION` is hard-coded to the higest _PROTOCOL_ version ever used. This is to avoid a sudden bump that would be visible on the network if CLIENT_VERSION were used instead. In the future, if we ever need to use the value, we can discard anything in the CLIENT_VERSION range (for a few years as needed), as it's quite a bit higher. While reviewing, I suggest looking at the throwaway `SerParams` commit above as it shows where the call-sites are. I believe that should be enough to convince one's self that hashing is never used. ACKs for top commit: TheCharlatan: Re-ACK e73d2a8018def940afadb5d699b18f39e882c1fc ajtowns: reACK e73d2a8018def940afadb5d699b18f39e882c1fc Tree-SHA512: 45b0dd7c2e918493e2ee92a8e35320ad17991cb8908cb811150a96c5fd584ce177c775baeeb8675a602c90b9ba9203b8cefc0a2a0c6a71078b1d9c2b41e1f3ba Dash adaptations: - CDiskBlockIndex::DUMMY_VERSION is 240100, Dash's current CLIENT_VERSION (10000 * 24 + 100 * 1 + 0 from _CLIENT_VERSION_MAJOR/MINOR/BUILD in configure.ac, per src/clientversion.h), instead of upstream's 259900. - CBlockLocator::DUMMY_VERSION is 70242, Dash's highest PROTOCOL_VERSION (src/version.h), instead of upstream's 70016. - Neither type is hashed with SER_GETHASH anywhere in Dash, so dropping that branch changes nothing. - The CDBWrapper::ReadDataStream() overloads, which upstream does not have, now fill a DataStream as well, and so do their callers in evo/deterministicmns.cpp, llmq/dkgsessionmgr.cpp, llmq/quorums.cpp, llmq/quorumsman.cpp and llmq/signing.cpp. None of them needs a stream version or type: they decode BLS version wrappers, CRecoveredSig, recovered-sig markers, raw bytes and the legacy CDeterministicMNListDiff format. - dbwrapper.h keeps including clientversion.h: Dash's CDBTransaction still estimates memory usage with ::GetSerializeSize(v, CLIENT_VERSION) and CDBWrapper::CompactRange() still builds SER_DISK/CLIENT_VERSION key streams. The upstream include additions to the files that use CLIENT_VERSION directly are kept. Co-authored-by: fanquake --- src/chain.h | 12 ++++++++++-- src/dbwrapper.h | 10 +++++----- src/evo/deterministicmns.cpp | 2 +- src/index/blockfilterindex.cpp | 1 + src/index/txindex.cpp | 1 + src/init.cpp | 1 + src/llmq/dkgsessionmgr.cpp | 2 +- src/llmq/quorums.cpp | 2 +- src/llmq/quorumsman.cpp | 2 +- src/llmq/signing.cpp | 4 ++-- src/primitives/block.h | 14 +++++++++++--- src/validation.cpp | 1 + 12 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/chain.h b/src/chain.h index a67a88641e9f..7b82ea1c8762 100644 --- a/src/chain.h +++ b/src/chain.h @@ -372,6 +372,14 @@ const CBlockIndex* LastCommonAncestor(const CBlockIndex* pa, const CBlockIndex* /** Used to marshal pointers into hashes for db storage. */ class CDiskBlockIndex : public CBlockIndex { + /** Historically CBlockLocator's version field has been written to disk + * streams as the client version, but the value has never been used. + * + * Hard-code to the highest client version ever written. + * SerParams can be used if the field requires any meaning in the future. + **/ + static constexpr int DUMMY_VERSION = 240100; + public: uint256 hash; uint256 hashPrev; @@ -391,8 +399,8 @@ class CDiskBlockIndex : public CBlockIndex SERIALIZE_METHODS(CDiskBlockIndex, obj) { LOCK(::cs_main); - int _nVersion = s.GetVersion(); - if (!(s.GetType() & SER_GETHASH)) READWRITE(VARINT_MODE(_nVersion, VarIntMode::NONNEGATIVE_SIGNED)); + int _nVersion = DUMMY_VERSION; + READWRITE(VARINT_MODE(_nVersion, VarIntMode::NONNEGATIVE_SIGNED)); READWRITE(VARINT_MODE(obj.nHeight, VarIntMode::NONNEGATIVE_SIGNED)); READWRITE(VARINT(obj.nStatus)); diff --git a/src/dbwrapper.h b/src/dbwrapper.h index affdb583438a..8c3364f1887c 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -199,7 +199,7 @@ class CDBIterator template bool GetValue(V& value) { try { - CDataStream ssValue{GetValueImpl(), SER_DISK, CLIENT_VERSION}; + DataStream ssValue{GetValueImpl()}; ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); ssValue >> value; } catch (const std::exception&) { @@ -252,7 +252,7 @@ class CDBWrapper CDBWrapper& operator=(const CDBWrapper&) = delete; template - bool ReadDataStream(const K& key, CDataStream& ssValue) const + bool ReadDataStream(const K& key, DataStream& ssValue) const { DataStream ssKey{}; ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); @@ -260,13 +260,13 @@ class CDBWrapper return ReadDataStream(ssKey, ssValue); } - bool ReadDataStream(const DataStream& ssKey, CDataStream& ssValue) const + bool ReadDataStream(const DataStream& ssKey, DataStream& ssValue) const { std::optional strValue{ReadImpl(ssKey)}; if (!strValue) { return false; } - CDataStream ssValueTmp{MakeByteSpan(*strValue), SER_DISK, CLIENT_VERSION}; + DataStream ssValueTmp{MakeByteSpan(*strValue)}; ssValueTmp.Xor(obfuscate_key); ssValue = std::move(ssValueTmp); return true; @@ -284,7 +284,7 @@ class CDBWrapper template bool Read(const DataStream& ssKey, V& value) const { - CDataStream ssValue(SER_DISK, CLIENT_VERSION); + DataStream ssValue{}; if (!ReadDataStream(ssKey, ssValue)) { return false; } diff --git a/src/evo/deterministicmns.cpp b/src/evo/deterministicmns.cpp index dbba411101f9..c9871c5409c7 100644 --- a/src/evo/deterministicmns.cpp +++ b/src/evo/deterministicmns.cpp @@ -1075,7 +1075,7 @@ bool CDeterministicMNManager::MigrateLegacyDiffs(const CBlockIndex* const tip_in } // Use legacy-aware deserialization for DB_LIST_DIFF_LEGACY entries - CDataStream s(SER_DISK, CLIENT_VERSION); + DataStream s{}; if (!m_evoDb.GetRawDB().ReadDataStream(key, s)) { break; } diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index d2b03a992734..d41b8fe21799 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -4,6 +4,7 @@ #include +#include #include #include #include diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 67caf35574d1..e6b7c6358fae 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -4,6 +4,7 @@ #include +#include #include #include #include diff --git a/src/init.cpp b/src/init.cpp index 0974e003054b..dc2a1cb95296 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include diff --git a/src/llmq/dkgsessionmgr.cpp b/src/llmq/dkgsessionmgr.cpp index b53026a0ca3d..8886bdac2aaa 100644 --- a/src/llmq/dkgsessionmgr.cpp +++ b/src/llmq/dkgsessionmgr.cpp @@ -170,7 +170,7 @@ bool CDKGSessionManager::GetVerifiedContributions(Consensus::LLMQType llmqType, ContributionsCacheKey cacheKey = {llmqType, pQuorumBaseBlockIndex->GetBlockHash(), proTxHash}; auto it = contributionsCache.find(cacheKey); if (it == contributionsCache.end()) { - CDataStream s(SER_DISK, CLIENT_VERSION); + DataStream s{}; if (!db->ReadDataStream(std::make_tuple(DB_VVEC, llmqType, pQuorumBaseBlockIndex->GetBlockHash(), proTxHash), s)) { LogPrint(BCLog::LLMQ, "%s -- this node does not have vvec for llmq=%d block=%s protx=%s\n", __func__, std23::to_underlying(llmqType), pQuorumBaseBlockIndex->GetBlockHash().ToString(), diff --git a/src/llmq/quorums.cpp b/src/llmq/quorums.cpp index 6e13f8251dd6..ebe1a16c3e28 100644 --- a/src/llmq/quorums.cpp +++ b/src/llmq/quorums.cpp @@ -212,7 +212,7 @@ void CQuorum::WriteContributions(CDBWrapper& db) const bool CQuorum::ReadContributions(const CDBWrapper& db) { uint256 dbKey = MakeQuorumKey(*this); - CDataStream s(SER_DISK, CLIENT_VERSION); + DataStream s{}; if (!db.ReadDataStream(std::make_pair(DB_QUORUM_QUORUM_VVEC, dbKey), s)) { return false; diff --git a/src/llmq/quorumsman.cpp b/src/llmq/quorumsman.cpp index 48a8fd5bbcdf..b91e04a41b3f 100644 --- a/src/llmq/quorumsman.cpp +++ b/src/llmq/quorumsman.cpp @@ -643,7 +643,7 @@ void CQuorumManager::MigrateOldQuorumDB(CEvoDB& evoDb) const int count{0}; while (pcursor->Valid()) { decltype(start) k; - CDataStream s(SER_DISK, CLIENT_VERSION); + DataStream s{}; CBLSSecretKey sk; if (!pcursor->GetKey(k) || std::get<0>(k) != prefix) { diff --git a/src/llmq/signing.cpp b/src/llmq/signing.cpp index f374d28963cf..d4587e61e206 100644 --- a/src/llmq/signing.cpp +++ b/src/llmq/signing.cpp @@ -96,7 +96,7 @@ bool CRecoveredSigsDb::ReadRecoveredSig(Consensus::LLMQType llmqType, const uint { auto k = std::make_tuple(std::string("rs_r"), llmqType, id); - CDataStream ds(SER_DISK, CLIENT_VERSION); + DataStream ds{}; if (!db->ReadDataStream(k, ds)) { return false; } @@ -239,7 +239,7 @@ struct RecoveredSigMarker { std::optional ReadRecoveredSigMarker(const CDBWrapper& db, const std::string& prefix, const uint256& hash) { - CDataStream value{SER_DISK, CLIENT_VERSION}; + DataStream value{}; if (!db.ReadDataStream(std::make_pair(prefix, hash), value)) return std::nullopt; RecoveredSigMarker marker; try { diff --git a/src/primitives/block.h b/src/primitives/block.h index 8789a21cbe37..486f855e6204 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -245,6 +245,15 @@ class CBlock : public CBlockHeader */ struct CBlockLocator { + /** Historically CBlockLocator's version field has been written to network + * streams as the negotiated protocol version and to disk streams as the + * client version, but the value has never been used. + * + * Hard-code to the highest protocol version ever written to a network stream. + * SerParams can be used if the field requires any meaning in the future, + **/ + static constexpr int DUMMY_VERSION = 70242; + std::vector vHave; CBlockLocator() {} @@ -253,9 +262,8 @@ struct CBlockLocator SERIALIZE_METHODS(CBlockLocator, obj) { - int nVersion = s.GetVersion(); - if (!(s.GetType() & SER_GETHASH)) - READWRITE(nVersion); + int nVersion = DUMMY_VERSION; + READWRITE(nVersion); READWRITE(obj.vHave); } diff --git a/src/validation.cpp b/src/validation.cpp index 12db50076cc0..33f8f2a31321 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include From 036f2ede142a4cf0fbc693c66e71c0cb5d20c4c1 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Thu, 14 Sep 2023 09:39:37 +0100 Subject: [PATCH 08/40] Merge bitcoin/bitcoin#28458: refactor: Remove unused GetType() from CBufferedFile and CAutoFile BACKPORT NOTE: CDeterministicMN, CDiskBlockIndex and CBlockLocator no longer read GetVersion() (see bitcoin#28428 and "refactor: always serialize CDeterministicMN::nType") dbwrapper.cpp no longer uses CLIENT_VERSION, so its clientversion.h include is dropped. dbwrapper.h keeps it for CDBTransaction's GetSerializeSize() estimate and CompactRange(). Backport done in full now ---- fa19c914f7fe7be127c0fb330b41ff7c091f40aa scripted-diff: Rename CBufferedFile to BufferedFile (MarcoFalke) fa2f2413b87f5fc1e5c92bf510beebdcd0091714 Remove unused GetType() from CBufferedFile and CAutoFile (MarcoFalke) 5c2b3cd4b856f1bb536daaf7f576b1b1b42293ca dbwrapper: Use DataStream for batch operations (TheCharlatan) Pull request description: This refactor is required for https://github.com/bitcoin/bitcoin/pull/28052 and https://github.com/bitcoin/bitcoin/pull/28451 Thus, split it out. ACKs for top commit: ajtowns: utACK fa19c914f7fe7be127c0fb330b41ff7c091f40aa TheCharlatan: ACK fa19c914f7fe7be127c0fb330b41ff7c091f40aa Tree-SHA512: d9c232324702512e45fd73ec3e3170f1e8a8c8f9c49cb613a6b693a9f83358914155527ace2517a2cd626a0cedcada56ef70a2a7812edafb1888fd6765eebba2 Co-authored-by: fanquake --- src/dbwrapper.cpp | 9 ++++----- src/dbwrapper.h | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 8d07afbc56f2..e0c124815798 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -4,7 +4,6 @@ #include -#include #include #include #include @@ -157,9 +156,9 @@ struct CDBBatch::WriteBatchImpl { leveldb::WriteBatch batch; }; -CDBBatch::CDBBatch(const CDBWrapper& _parent) : parent(_parent), - m_impl_batch{std::make_unique()}, - ssValue(SER_DISK, CLIENT_VERSION){}; +CDBBatch::CDBBatch(const CDBWrapper& _parent) + : parent{_parent}, + m_impl_batch{std::make_unique()} {}; CDBBatch::~CDBBatch() = default; @@ -169,7 +168,7 @@ void CDBBatch::Clear() size_estimate = 0; } -void CDBBatch::WriteImpl(Span key, CDataStream& ssValue) +void CDBBatch::WriteImpl(Span key, DataStream& ssValue) { leveldb::Slice slKey(CharCast(key.data()), key.size()); ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 8c3364f1887c..4c09a1174339 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -91,11 +91,11 @@ class CDBBatch const std::unique_ptr m_impl_batch; DataStream ssKey{}; - CDataStream ssValue; + DataStream ssValue{}; size_t size_estimate{0}; - void WriteImpl(Span key, CDataStream& ssValue); + void WriteImpl(Span key, DataStream& ssValue); void EraseImpl(Span key); public: From 1557a3689ceae9c00602783cb0e0e02ef8bdf0b7 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Thu, 14 Sep 2023 13:15:24 -0400 Subject: [PATCH 09/40] Merge bitcoin/bitcoin#27850: test: Add unit & functional test coverage for blockstore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit de8f9123afbecc3b4f59fa80af8148bc865d0588 test: cover read-only blockstore (Matthew Zipkin) 5c2185b3b624ce87320ec16412f98ab591a5860c ci: enable chattr +i capability inside containers (Matthew Zipkin) e573f2420244c583e218f51cd0d3a3cac6731003 unit test: add coverage for BlockManager (Matthew Zipkin) Pull request description: This PR adds unit and functional tests to cover the behavior described in #2039. In particular, that bitcoind will crash on startup if a reindex is requested but the `blk` files are read-only. Eventually this behavior can be updated with https://github.com/bitcoin/bitcoin/pull/27039. This PR just commits the test coverage from #27039 as suggested in https://github.com/bitcoin/bitcoin/pull/27039#issuecomment-1584915782 ACKs for top commit: jonatack: ACK de8f9123afbecc3b4f59fa80af8148bc865d0588 modulo suggestions in https://github.com/bitcoin/bitcoin/pull/27850#discussion_r1319010039, tested on macOS, but not on Linux for the Linux-related change in the last push achow101: ACK de8f9123afbecc3b4f59fa80af8148bc865d0588 MarcoFalke: lgtm ACK de8f9123afbecc3b4f59fa80af8148bc865d0588 📶 Tree-SHA512: b9bd684035dcea11c901b649fc39f397a2155a9a8459f3348e67947e387e45312fddeccb52981aef486f8a31deebb5356a7901c1bb94b78f82c24192a369af73 Co-authored-by: Andrew Chow --- .github/workflows/test-src.yml | 3 +- contrib/containers/ci/ci-slim.Dockerfile | 1 + src/test/blockmanager_tests.cpp | 69 +++++++++++++++++++++ test/functional/feature_reindex_readonly.py | 64 +++++++++++++++++++ test/functional/test_runner.py | 1 + 5 files changed, 137 insertions(+), 1 deletion(-) create mode 100755 test/functional/feature_reindex_readonly.py diff --git a/.github/workflows/test-src.yml b/.github/workflows/test-src.yml index d6d9b7efe58f..3b25c11a3ef8 100644 --- a/.github/workflows/test-src.yml +++ b/.github/workflows/test-src.yml @@ -41,7 +41,8 @@ jobs: image: ${{ inputs.container-path }} # zizmor: ignore[unpinned-images] built earlier in this run by build-container.yml # SYS_PTRACE is required by LeakSanitizer's stop-the-world # (https://github.com/google/sanitizers/issues/764) - options: --user root --cap-add SYS_PTRACE + # LINUX_IMMUTABLE lets feature_reindex_readonly.py run chattr +i + options: --user root --cap-add LINUX_IMMUTABLE --cap-add SYS_PTRACE steps: - name: Checkout code uses: actions/checkout@v6 diff --git a/contrib/containers/ci/ci-slim.Dockerfile b/contrib/containers/ci/ci-slim.Dockerfile index a4aa61b90d12..43ccd5405a91 100644 --- a/contrib/containers/ci/ci-slim.Dockerfile +++ b/contrib/containers/ci/ci-slim.Dockerfile @@ -40,6 +40,7 @@ RUN set -ex; \ build-essential \ ca-certificates \ curl \ + e2fsprogs \ g++ \ git \ jq \ diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index 4ce0bd7bd1d5..052097dd0ceb 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include