Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions dash-spv/src/sync/filters/batch.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
use super::manager::DerivedScripts;
use dashcore::bip158::BlockFilter;
use dashcore::ScriptBuf;
use key_wallet_manager::{FilterMatchKey, WalletId};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::mem;
use std::sync::Arc;

/// A completed batch of compact block filters ready for verification.
Expand Down Expand Up @@ -34,6 +37,8 @@ pub(super) struct FiltersBatch {
/// Every script and filter element already matched against this batch's
/// filters, per wallet, as the bytes the filters were queried with.
tested: HashMap<WalletId, HashSet<Vec<u8>>>,
/// Newly derived scripts still to be matched against this batch, per wallet.
deferred_scripts: HashMap<WalletId, DerivedScripts>,
}

impl FiltersBatch {
Expand All @@ -52,6 +57,7 @@ impl FiltersBatch {
pending_blocks: 0,
scanned_wallets: BTreeMap::new(),
tested: HashMap::new(),
deferred_scripts: HashMap::new(),
}
}
/// Start height of this batch (inclusive).
Expand Down Expand Up @@ -125,6 +131,30 @@ impl FiltersBatch {
monitored.iter().filter(move |item| tested.is_none_or(|t| !t.contains(item.as_ref())))
}

/// Defer the wallet's `scripts` this batch still has to be matched against,
/// keeping the `pass` that derived them. An unscanned batch defers
/// nothing, since its scan matches every script the wallet watches.
pub(super) fn defer(&mut self, wallet_id: WalletId, pass: u64, scripts: &[ScriptBuf]) {
if !self.scanned || !self.scanned_wallets.contains_key(&wallet_id) {
return;
}
let missing: HashSet<ScriptBuf> = self.untested(&wallet_id, scripts).cloned().collect();
if !missing.is_empty() {
let deferred = self.deferred_scripts.entry(wallet_id).or_default();
deferred.entry(pass).or_default().extend(missing);
}
}

/// Whether scripts are deferred for this batch.
pub(super) fn has_deferred(&self) -> bool {
!self.deferred_scripts.is_empty()
}

/// Take the deferred scripts for matching.
pub(super) fn take_deferred(&mut self) -> HashMap<WalletId, DerivedScripts> {
mem::take(&mut self.deferred_scripts)
}

/// Record the wallets that were behind for this batch at scan time, each
/// with its `account_generation` snapshot.
pub(super) fn set_scanned_wallets(&mut self, wallets: BTreeMap<WalletId, u64>) {
Expand Down
48 changes: 39 additions & 9 deletions dash-spv/src/sync/filters/block_match_tracker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,12 +73,19 @@ pub(super) struct BlockMatchTracker {
/// Number of the last pass given to `record_processed`. Never reset, so a
/// record that outlives a `clear` can't look newer than a later pass.
passes: u64,
/// Whether the wallet checks a block against the addresses it derives
/// while applying it, so the deriving pass already saw its own scripts.
checks_derived_addresses: bool,
}

impl BlockMatchTracker {
/// Create an empty tracker.
pub(super) fn new() -> Self {
Self::default()
/// Create an empty tracker for a wallet that does or does not check a block
/// against the addresses it derives, see `WalletInterface::checks_derived_addresses`.
pub(super) fn new(checks_derived_addresses: bool) -> Self {
Self {
checks_derived_addresses,
..Self::default()
}
}

/// Track a filter match for a block against a candidate wallet set,
Expand Down Expand Up @@ -111,7 +118,8 @@ impl BlockMatchTracker {
/// Like [`Self::track`], but for matches on newly derived scripts. Each
/// candidate wallet maps to the pass that derived its matching scripts. A
/// wallet whose last pass of this block ran later already saw them and is
/// skipped, and so is a wallet the block is already queued for.
/// skipped, and so is a wallet the block is already queued for. The deriving
/// pass itself saw them only if the wallet checks derived addresses.
pub(super) fn track_for_new_scripts(
&mut self,
key: &FilterMatchKey,
Expand All @@ -122,7 +130,13 @@ impl BlockMatchTracker {
let residual: BTreeSet<WalletId> = candidate_wallets
.into_iter()
.filter(|(id, derived_at)| {
processed.and_then(|p| p.get(id)).is_none_or(|pass| pass <= derived_at)
processed.and_then(|p| p.get(id)).is_none_or(|pass| {
if self.checks_derived_addresses {
pass < derived_at
} else {
pass <= derived_at
}
})
})
.map(|(id, _)| id)
.collect();
Expand Down Expand Up @@ -250,11 +264,12 @@ mod tests {
/// NewlyTracked again for a residual wallet after the first wallet's
/// processing is recorded, and AlreadyProcessed once every candidate is
/// covered. `track_for_new_scripts` re-queues the block only for wallets
/// whose last pass ran no later than the pass that derived the scripts. A
/// whose last pass ran no later than the pass that derived the scripts, or
/// earlier than it for a wallet that checks the addresses it derives. A
/// block stays in flight until every requested wallet has reported.
#[test]
fn track_state_machine() {
let mut tracker = BlockMatchTracker::new();
let mut tracker = BlockMatchTracker::new(false);
let hash = hash_n(0);
let key = FilterMatchKey::new(100, hash);
let wallet_a: WalletId = [0xA1; 32];
Expand Down Expand Up @@ -328,14 +343,29 @@ mod tests {
tracker.track(&FilterMatchKey::new(200, other), 0, both.clone());
assert_eq!(tracker.finish_in_flight(&other, &BTreeSet::from([wallet_a])), None);
assert_eq!(tracker.finish_in_flight(&other, &both), Some((200, 0)));

// A wallet that checks a block against the addresses it derives saw
// them in the deriving pass. Scripts of a later pass still re-queue it.
let mut checking = BlockMatchTracker::new(true);
let pass = checking.record_processed(100, hash, &BTreeSet::from([wallet_a]));
assert_eq!(
checking.track_for_new_scripts(&key, 5000, BTreeMap::from([(wallet_a, pass)])),
BlockTrackResult::AlreadyProcessed
);
assert_eq!(
checking.track_for_new_scripts(&key, 5000, BTreeMap::from([(wallet_a, pass + 1)])),
BlockTrackResult::NewlyTracked {
wallets: BTreeSet::from([wallet_a])
}
);
}

/// `prune_at_or_below` drops every entry at or below the given height
/// while retaining strictly higher entries. Idempotent under repeated
/// calls with the same threshold.
#[test]
fn prune_at_or_below_drops_low_entries() {
let mut tracker = BlockMatchTracker::new();
let mut tracker = BlockMatchTracker::new(false);
let wallet: WalletId = [0xFA; 32];
let h_low = hash_n(1);
let h_mid = hash_n(2);
Expand Down Expand Up @@ -364,7 +394,7 @@ mod tests {
/// flips `is_empty`, and `clear` returns to the initial state.
#[test]
fn is_empty_and_clear_cover_both_maps() {
let mut tracker = BlockMatchTracker::new();
let mut tracker = BlockMatchTracker::new(false);
let wallet: WalletId = [0xCC; 32];
let hash = hash_n(0);
let key = FilterMatchKey::new(100, hash);
Expand Down
114 changes: 63 additions & 51 deletions dash-spv/src/sync/filters/coinjoin_gap_discovery_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@
//! 5. [`provider_wallet_rescan_prefers_an_untested_outpoint_over_an_older_script`]
//! — an untested outpoint re-applies a block even when an older script
//! group also matches it.
//! 6. [`coinjoin_scripts_derived_in_one_batch_find_blocks_in_the_next`] —
//! scripts derived by the lowest batch are deferred to the next batch and
//! still find its blocks before it commits.
//!
//! Each test drives the manager exactly the way the production event loop
//! does: `try_process_batch` → `BlocksNeeded` → (blocks-manager stand-in)
Expand All @@ -49,9 +52,10 @@ use dashcore::{
};
use key_wallet::account::ManagedAccountTrait;
use key_wallet::gap_limit::DEFAULT_COINJOIN_GAP_LIMIT;
use key_wallet::managed_account::address_pool::{AddressPool, AddressPoolType, KeySource};
use key_wallet::managed_account::address_pool::{AddressPool, AddressPoolType};
use key_wallet::wallet::initialization::WalletAccountCreationOptions;
use key_wallet::wallet::managed_wallet_info::ManagedWalletInfo;
use key_wallet_manager::test_utils::coinjoin_external_addresses;
use key_wallet_manager::WalletManager;

/// Deterministic test wallet seed (standard BIP-39 test vector mnemonic).
Expand Down Expand Up @@ -98,43 +102,6 @@ async fn setup() -> (CjFiltersManager, Arc<RwLock<WalletManager<ManagedWalletInf
(manager, wallet, wallet_id)
}

/// Derive the wallet's first `count` CoinJoin-account-0 External addresses
/// OFFLINE (via a standalone pool over the same xpub + base path), so test
/// blocks can pay indices far beyond the wallet's live watch window without
/// touching the wallet's own state.
async fn coinjoin_external_addresses(
wallet: &Arc<RwLock<WalletManager<ManagedWalletInfo>>>,
wallet_id: &WalletId,
count: u32,
) -> Vec<Address> {
let wm = wallet.read().await;
let signing_wallet = wm.get_wallet(wallet_id).expect("wallet present");
let xpub = signing_wallet
.accounts
.coinjoin_accounts
.get(&0)
.expect("CoinJoin account 0 (Default options)")
.account_xpub;
let info = wm.get_wallet_info(wallet_id).expect("wallet info present");
let live_pool = coinjoin_external_pool(info);
let mut pool = AddressPool::new(
live_pool.base_path.clone(),
AddressPoolType::External,
count,
Network::Regtest,
&KeySource::Public(xpub),
)
.expect("derive standalone CoinJoin External pool");
// `AddressPool::new` pre-generates `count` addresses; extend defensively
// in case the constructor semantics change.
if pool.highest_generated.map(|h| h + 1).unwrap_or(0) < count {
let missing = count - pool.highest_generated.map(|h| h + 1).unwrap_or(0);
pool.generate_addresses(missing, &KeySource::Public(xpub), true)
.expect("extend standalone pool");
}
(0..count).map(|i| pool.address_at_index(i).expect("address generated")).collect()
}

/// The wallet's LIVE CoinJoin-account-0 External pool.
fn coinjoin_external_pool(info: &ManagedWalletInfo) -> &AddressPool {
let account =
Expand Down Expand Up @@ -208,7 +175,8 @@ fn filter_with_elements(block: &Block, elements: &[Vec<u8>]) -> BlockFilter {
/// job), feed the resulting `BlockProcessed` events (carrying the wallet's
/// freshly derived scripts) back into the filters manager, and repeat until
/// no more blocks are requested. Blocks are processed in height order, like
/// the real `BlocksPipeline`. Returns how often each block was applied.
/// the real `BlocksPipeline`, and the manager ticks once per round. Returns how
/// often each block was applied.
async fn drive_to_quiescence(
manager: &mut CjFiltersManager,
wallet: &Arc<RwLock<WalletManager<ManagedWalletInfo>>>,
Expand All @@ -220,6 +188,7 @@ async fn drive_to_quiescence(
let mut passes: HashMap<BlockHash, usize> = HashMap::new();
let mut events = initial_events;
for _round in 0..64 {
events.extend(manager.tick(&network).await.expect("tick"));
let mut pending: BTreeMap<(u32, BlockHash), BTreeSet<WalletId>> = BTreeMap::new();
for event in events.drain(..) {
if let SyncEvent::BlocksNeeded {
Expand Down Expand Up @@ -270,7 +239,7 @@ async fn drive_to_quiescence(
#[tokio::test]
async fn coinjoin_gap_limit_dense_same_batch_recovers() {
let (mut manager, wallet, wallet_id) = setup().await;
let addresses = coinjoin_external_addresses(&wallet, &wallet_id, (G + 40) as u32).await;
let addresses = coinjoin_external_addresses(&*wallet.read().await, &wallet_id, (G + 40) as u32);

let (highest_used, highest_generated, _) = coinjoin_pool_state(&wallet, &wallet_id).await;
assert_eq!(highest_used, None, "fresh wallet must have no used CoinJoin indices");
Expand Down Expand Up @@ -313,7 +282,7 @@ async fn coinjoin_gap_limit_dense_same_batch_recovers() {
#[tokio::test]
async fn coinjoin_gap_limit_inversion_within_batch_recovers() {
let (mut manager, wallet, wallet_id) = setup().await;
let addresses = coinjoin_external_addresses(&wallet, &wallet_id, (G + 22) as u32).await;
let addresses = coinjoin_external_addresses(&*wallet.read().await, &wallet_id, (G + 22) as u32);

let (block_a, filter_a, key_a) = block_paying(10, &addresses[(G + 10)..=(G + 21)]);
let (block_b, filter_b, key_b) = block_paying(20, &addresses[0..=29]);
Expand All @@ -337,17 +306,20 @@ async fn coinjoin_gap_limit_inversion_within_batch_recovers() {
);
}

/// Block A pays index 5 and G+2 in one transaction. Its first pass only sees
/// index 5, and only then derives G+2, so A must be applied again. Block B
/// pays index 1 and G+3 and is applied after A derived G+3, so its first pass
/// already credited G+3 and the rescan must not apply it again.
/// Block A pays G+2 in its first transaction and index 5 in its second. Using
/// index 5 derives G+2, and the same application checks the block again, so A
/// is applied once. Block B pays index 1 and G+3 and is applied after A derived
/// G+3, so its first pass already credited G+3 and the rescan must not apply it again.
#[tokio::test]
async fn coinjoin_rescan_skips_blocks_applied_after_the_scripts() {
let (mut manager, wallet, wallet_id) = setup().await;
let addresses = coinjoin_external_addresses(&wallet, &wallet_id, (G + 4) as u32).await;
let addresses = coinjoin_external_addresses(&*wallet.read().await, &wallet_id, (G + 4) as u32);

let (block_a, filter_a, key_a) =
block_paying(10, &[addresses[5].clone(), addresses[G + 2].clone()]);
let (pays_g2, _, _) = block_paying(10, &[addresses[G + 2].clone()]);
let (pays_5, _, _) = block_paying(11, &[addresses[5].clone()]);
let block_a = Block::dummy(10, vec![pays_g2.txdata[0].clone(), pays_5.txdata[0].clone()]);
let (filter_a, key_a) =
(BlockFilter::dummy(&block_a), FilterMatchKey::new(10, block_a.block_hash()));
let (block_b, filter_b, key_b) =
block_paying(20, &[addresses[1].clone(), addresses[G + 3].clone()]);
let (hash_a, hash_b) = (block_a.block_hash(), block_b.block_hash());
Expand All @@ -364,7 +336,9 @@ async fn coinjoin_rescan_skips_blocks_applied_after_the_scripts() {
let (highest_used, _, used_count) = coinjoin_pool_state(&wallet, &wallet_id).await;
assert_eq!(highest_used, Some((G + 3) as u32));
assert_eq!(used_count, 4, "indices 1, 5, G+2 and G+3 must all be credited");
assert_eq!(passes.get(&hash_a), Some(&2), "A predates the scripts it derived");
let balance = wallet.read().await.get_wallet_balance(&wallet_id).expect("balance");
assert_eq!(balance.confirmed(), 4 * DENOMINATION);
assert_eq!(passes.get(&hash_a), Some(&1), "A credits the scripts it derived itself");
assert_eq!(passes.get(&hash_b), Some(&1), "B was applied after G+3 was derived");
}

Expand Down Expand Up @@ -397,7 +371,7 @@ async fn run_batch(
#[tokio::test]
async fn provider_wallet_rescans_only_untested_outpoints() {
let (mut manager, wallet, wallet_id) = setup().await;
let addresses = coinjoin_external_addresses(&wallet, &wallet_id, 7).await;
let addresses = coinjoin_external_addresses(&*wallet.read().await, &wallet_id, 7);

// Batch 0: A creates the outpoint the later batches refer to.
let (block_a, filter_a, key_a) = block_paying(10, &[addresses[5].clone()]);
Expand Down Expand Up @@ -433,7 +407,7 @@ async fn provider_wallet_rescans_only_untested_outpoints() {
#[tokio::test]
async fn provider_wallet_rescan_prefers_an_untested_outpoint_over_an_older_script() {
let (mut manager, wallet, wallet_id) = setup().await;
let addresses = coinjoin_external_addresses(&wallet, &wallet_id, (G + 4) as u32).await;
let addresses = coinjoin_external_addresses(&*wallet.read().await, &wallet_id, (G + 4) as u32);

let a = block_paying(10, &[addresses[5].clone()]);
let z = block_paying(30, &[addresses[2].clone()]);
Expand All @@ -445,3 +419,41 @@ async fn provider_wallet_rescan_prefers_an_untested_outpoint_over_an_older_scrip
let passes = run_batch(&mut manager, &wallet, 0, vec![a, (block_x, filter_x, key_x), z]).await;
assert_eq!(passes.get(&hash_x), Some(&2), "X never saw the outpoint Z created");
}

/// Block A in the first batch pays index 5 and G+4, which derives the window
/// up to 2G+4. The next batch was scanned before that, and its blocks pay G+2
/// and G+6. Both are found by the deferred rescan, and every block is applied once.
#[tokio::test]
async fn coinjoin_scripts_derived_in_one_batch_find_blocks_in_the_next() {
let (mut manager, wallet, wallet_id) = setup().await;
let addresses = coinjoin_external_addresses(&*wallet.read().await, &wallet_id, (G + 7) as u32);

let (block_a, filter_a, key_a) =
block_paying(10, &[addresses[5].clone(), addresses[G + 4].clone()]);
let (block_b, filter_b, key_b) = block_paying(120, &[addresses[G + 6].clone()]);
let (block_c, filter_c, key_c) = block_paying(150, &[addresses[G + 2].clone()]);
let blocks: HashMap<BlockHash, Block> =
[block_a, block_b, block_c].into_iter().map(|block| (block.block_hash(), block)).collect();

let mut batch = FiltersBatch::new(0, 99, HashMap::from([(key_a, filter_a)]));
batch.mark_verified();
manager.active_batches.insert(0, batch);
let mut batch =
FiltersBatch::new(100, 199, HashMap::from([(key_b, filter_b), (key_c, filter_c)]));
batch.mark_verified();
manager.active_batches.insert(100, batch);
manager.progress.update_stored_height(199);

let initial_events = manager.try_process_batch().await.unwrap();
let passes = drive_to_quiescence(&mut manager, &wallet, &blocks, initial_events).await;

let (highest_used, _, used_count) = coinjoin_pool_state(&wallet, &wallet_id).await;
assert_eq!(highest_used, Some((G + 6) as u32));
assert_eq!(used_count, 4, "indices 5, G+2, G+4 and G+6 must all be credited");
let balance = wallet.read().await.get_wallet_balance(&wallet_id).expect("balance");
assert_eq!(balance.confirmed(), 4 * DENOMINATION);
assert_eq!(passes.len(), 3);
assert!(passes.values().all(|count| *count == 1), "each block is applied once: {passes:?}");
assert!(manager.active_batches.is_empty(), "both batches commit");
assert_eq!(manager.progress.committed_height(), 199);
}
Loading
Loading