diff --git a/dash-spv/src/sync/filters/batch.rs b/dash-spv/src/sync/filters/batch.rs index 400a959d8..da7e65e2e 100644 --- a/dash-spv/src/sync/filters/batch.rs +++ b/dash-spv/src/sync/filters/batch.rs @@ -1,6 +1,9 @@ +use super::manager::DerivedScripts; use dashcore::bip158::BlockFilter; +use dashcore::ScriptBuf; use key_wallet_manager::{FilterMatchKey, WalletId}; use std::collections::{BTreeMap, HashMap, HashSet}; +use std::mem; use std::sync::Arc; /// A completed batch of compact block filters ready for verification. @@ -34,6 +37,8 @@ pub(super) struct FiltersBatch { /// Every script and filter element already matched against this batch's /// filters, per wallet, as the bytes the filters were queried with. tested: HashMap>>, + /// Newly derived scripts still to be matched against this batch, per wallet. + deferred_scripts: HashMap, } impl FiltersBatch { @@ -52,6 +57,7 @@ impl FiltersBatch { pending_blocks: 0, scanned_wallets: BTreeMap::new(), tested: HashMap::new(), + deferred_scripts: HashMap::new(), } } /// Start height of this batch (inclusive). @@ -125,6 +131,30 @@ impl FiltersBatch { monitored.iter().filter(move |item| tested.is_none_or(|t| !t.contains(item.as_ref()))) } + /// Defer the wallet's `scripts` this batch still has to be matched against, + /// keeping the `pass` that derived them. An unscanned batch defers + /// nothing, since its scan matches every script the wallet watches. + pub(super) fn defer(&mut self, wallet_id: WalletId, pass: u64, scripts: &[ScriptBuf]) { + if !self.scanned || !self.scanned_wallets.contains_key(&wallet_id) { + return; + } + let missing: HashSet = self.untested(&wallet_id, scripts).cloned().collect(); + if !missing.is_empty() { + let deferred = self.deferred_scripts.entry(wallet_id).or_default(); + deferred.entry(pass).or_default().extend(missing); + } + } + + /// Whether scripts are deferred for this batch. + pub(super) fn has_deferred(&self) -> bool { + !self.deferred_scripts.is_empty() + } + + /// Take the deferred scripts for matching. + pub(super) fn take_deferred(&mut self) -> HashMap { + mem::take(&mut self.deferred_scripts) + } + /// Record the wallets that were behind for this batch at scan time, each /// with its `account_generation` snapshot. pub(super) fn set_scanned_wallets(&mut self, wallets: BTreeMap) { diff --git a/dash-spv/src/sync/filters/block_match_tracker.rs b/dash-spv/src/sync/filters/block_match_tracker.rs index d63d39325..a08f196ac 100644 --- a/dash-spv/src/sync/filters/block_match_tracker.rs +++ b/dash-spv/src/sync/filters/block_match_tracker.rs @@ -73,12 +73,19 @@ pub(super) struct BlockMatchTracker { /// Number of the last pass given to `record_processed`. Never reset, so a /// record that outlives a `clear` can't look newer than a later pass. passes: u64, + /// Whether the wallet checks a block against the addresses it derives + /// while applying it, so the deriving pass already saw its own scripts. + checks_derived_addresses: bool, } impl BlockMatchTracker { - /// Create an empty tracker. - pub(super) fn new() -> Self { - Self::default() + /// Create an empty tracker for a wallet that does or does not check a block + /// against the addresses it derives, see `WalletInterface::checks_derived_addresses`. + pub(super) fn new(checks_derived_addresses: bool) -> Self { + Self { + checks_derived_addresses, + ..Self::default() + } } /// Track a filter match for a block against a candidate wallet set, @@ -111,7 +118,8 @@ impl BlockMatchTracker { /// Like [`Self::track`], but for matches on newly derived scripts. Each /// candidate wallet maps to the pass that derived its matching scripts. A /// wallet whose last pass of this block ran later already saw them and is - /// skipped, and so is a wallet the block is already queued for. + /// skipped, and so is a wallet the block is already queued for. The deriving + /// pass itself saw them only if the wallet checks derived addresses. pub(super) fn track_for_new_scripts( &mut self, key: &FilterMatchKey, @@ -122,7 +130,13 @@ impl BlockMatchTracker { let residual: BTreeSet = candidate_wallets .into_iter() .filter(|(id, derived_at)| { - processed.and_then(|p| p.get(id)).is_none_or(|pass| pass <= derived_at) + processed.and_then(|p| p.get(id)).is_none_or(|pass| { + if self.checks_derived_addresses { + pass < derived_at + } else { + pass <= derived_at + } + }) }) .map(|(id, _)| id) .collect(); @@ -250,11 +264,12 @@ mod tests { /// NewlyTracked again for a residual wallet after the first wallet's /// processing is recorded, and AlreadyProcessed once every candidate is /// covered. `track_for_new_scripts` re-queues the block only for wallets - /// whose last pass ran no later than the pass that derived the scripts. A + /// whose last pass ran no later than the pass that derived the scripts, or + /// earlier than it for a wallet that checks the addresses it derives. A /// block stays in flight until every requested wallet has reported. #[test] fn track_state_machine() { - let mut tracker = BlockMatchTracker::new(); + let mut tracker = BlockMatchTracker::new(false); let hash = hash_n(0); let key = FilterMatchKey::new(100, hash); let wallet_a: WalletId = [0xA1; 32]; @@ -328,6 +343,21 @@ mod tests { tracker.track(&FilterMatchKey::new(200, other), 0, both.clone()); assert_eq!(tracker.finish_in_flight(&other, &BTreeSet::from([wallet_a])), None); assert_eq!(tracker.finish_in_flight(&other, &both), Some((200, 0))); + + // A wallet that checks a block against the addresses it derives saw + // them in the deriving pass. Scripts of a later pass still re-queue it. + let mut checking = BlockMatchTracker::new(true); + let pass = checking.record_processed(100, hash, &BTreeSet::from([wallet_a])); + assert_eq!( + checking.track_for_new_scripts(&key, 5000, BTreeMap::from([(wallet_a, pass)])), + BlockTrackResult::AlreadyProcessed + ); + assert_eq!( + checking.track_for_new_scripts(&key, 5000, BTreeMap::from([(wallet_a, pass + 1)])), + BlockTrackResult::NewlyTracked { + wallets: BTreeSet::from([wallet_a]) + } + ); } /// `prune_at_or_below` drops every entry at or below the given height @@ -335,7 +365,7 @@ mod tests { /// calls with the same threshold. #[test] fn prune_at_or_below_drops_low_entries() { - let mut tracker = BlockMatchTracker::new(); + let mut tracker = BlockMatchTracker::new(false); let wallet: WalletId = [0xFA; 32]; let h_low = hash_n(1); let h_mid = hash_n(2); @@ -364,7 +394,7 @@ mod tests { /// flips `is_empty`, and `clear` returns to the initial state. #[test] fn is_empty_and_clear_cover_both_maps() { - let mut tracker = BlockMatchTracker::new(); + let mut tracker = BlockMatchTracker::new(false); let wallet: WalletId = [0xCC; 32]; let hash = hash_n(0); let key = FilterMatchKey::new(100, hash); diff --git a/dash-spv/src/sync/filters/coinjoin_gap_discovery_tests.rs b/dash-spv/src/sync/filters/coinjoin_gap_discovery_tests.rs index 8ce0cf0ec..dcd701234 100644 --- a/dash-spv/src/sync/filters/coinjoin_gap_discovery_tests.rs +++ b/dash-spv/src/sync/filters/coinjoin_gap_discovery_tests.rs @@ -28,6 +28,9 @@ //! 5. [`provider_wallet_rescan_prefers_an_untested_outpoint_over_an_older_script`] //! — an untested outpoint re-applies a block even when an older script //! group also matches it. +//! 6. [`coinjoin_scripts_derived_in_one_batch_find_blocks_in_the_next`] — +//! scripts derived by the lowest batch are deferred to the next batch and +//! still find its blocks before it commits. //! //! Each test drives the manager exactly the way the production event loop //! does: `try_process_batch` → `BlocksNeeded` → (blocks-manager stand-in) @@ -49,9 +52,10 @@ use dashcore::{ }; use key_wallet::account::ManagedAccountTrait; use key_wallet::gap_limit::DEFAULT_COINJOIN_GAP_LIMIT; -use key_wallet::managed_account::address_pool::{AddressPool, AddressPoolType, KeySource}; +use key_wallet::managed_account::address_pool::{AddressPool, AddressPoolType}; use key_wallet::wallet::initialization::WalletAccountCreationOptions; use key_wallet::wallet::managed_wallet_info::ManagedWalletInfo; +use key_wallet_manager::test_utils::coinjoin_external_addresses; use key_wallet_manager::WalletManager; /// Deterministic test wallet seed (standard BIP-39 test vector mnemonic). @@ -98,43 +102,6 @@ async fn setup() -> (CjFiltersManager, Arc>>, - wallet_id: &WalletId, - count: u32, -) -> Vec
{ - let wm = wallet.read().await; - let signing_wallet = wm.get_wallet(wallet_id).expect("wallet present"); - let xpub = signing_wallet - .accounts - .coinjoin_accounts - .get(&0) - .expect("CoinJoin account 0 (Default options)") - .account_xpub; - let info = wm.get_wallet_info(wallet_id).expect("wallet info present"); - let live_pool = coinjoin_external_pool(info); - let mut pool = AddressPool::new( - live_pool.base_path.clone(), - AddressPoolType::External, - count, - Network::Regtest, - &KeySource::Public(xpub), - ) - .expect("derive standalone CoinJoin External pool"); - // `AddressPool::new` pre-generates `count` addresses; extend defensively - // in case the constructor semantics change. - if pool.highest_generated.map(|h| h + 1).unwrap_or(0) < count { - let missing = count - pool.highest_generated.map(|h| h + 1).unwrap_or(0); - pool.generate_addresses(missing, &KeySource::Public(xpub), true) - .expect("extend standalone pool"); - } - (0..count).map(|i| pool.address_at_index(i).expect("address generated")).collect() -} - /// The wallet's LIVE CoinJoin-account-0 External pool. fn coinjoin_external_pool(info: &ManagedWalletInfo) -> &AddressPool { let account = @@ -208,7 +175,8 @@ fn filter_with_elements(block: &Block, elements: &[Vec]) -> BlockFilter { /// job), feed the resulting `BlockProcessed` events (carrying the wallet's /// freshly derived scripts) back into the filters manager, and repeat until /// no more blocks are requested. Blocks are processed in height order, like -/// the real `BlocksPipeline`. Returns how often each block was applied. +/// the real `BlocksPipeline`, and the manager ticks once per round. Returns how +/// often each block was applied. async fn drive_to_quiescence( manager: &mut CjFiltersManager, wallet: &Arc>>, @@ -220,6 +188,7 @@ async fn drive_to_quiescence( let mut passes: HashMap = HashMap::new(); let mut events = initial_events; for _round in 0..64 { + events.extend(manager.tick(&network).await.expect("tick")); let mut pending: BTreeMap<(u32, BlockHash), BTreeSet> = BTreeMap::new(); for event in events.drain(..) { if let SyncEvent::BlocksNeeded { @@ -270,7 +239,7 @@ async fn drive_to_quiescence( #[tokio::test] async fn coinjoin_gap_limit_dense_same_batch_recovers() { let (mut manager, wallet, wallet_id) = setup().await; - let addresses = coinjoin_external_addresses(&wallet, &wallet_id, (G + 40) as u32).await; + let addresses = coinjoin_external_addresses(&*wallet.read().await, &wallet_id, (G + 40) as u32); let (highest_used, highest_generated, _) = coinjoin_pool_state(&wallet, &wallet_id).await; assert_eq!(highest_used, None, "fresh wallet must have no used CoinJoin indices"); @@ -313,7 +282,7 @@ async fn coinjoin_gap_limit_dense_same_batch_recovers() { #[tokio::test] async fn coinjoin_gap_limit_inversion_within_batch_recovers() { let (mut manager, wallet, wallet_id) = setup().await; - let addresses = coinjoin_external_addresses(&wallet, &wallet_id, (G + 22) as u32).await; + let addresses = coinjoin_external_addresses(&*wallet.read().await, &wallet_id, (G + 22) as u32); let (block_a, filter_a, key_a) = block_paying(10, &addresses[(G + 10)..=(G + 21)]); let (block_b, filter_b, key_b) = block_paying(20, &addresses[0..=29]); @@ -337,17 +306,20 @@ async fn coinjoin_gap_limit_inversion_within_batch_recovers() { ); } -/// Block A pays index 5 and G+2 in one transaction. Its first pass only sees -/// index 5, and only then derives G+2, so A must be applied again. Block B -/// pays index 1 and G+3 and is applied after A derived G+3, so its first pass -/// already credited G+3 and the rescan must not apply it again. +/// Block A pays G+2 in its first transaction and index 5 in its second. Using +/// index 5 derives G+2, and the same application checks the block again, so A +/// is applied once. Block B pays index 1 and G+3 and is applied after A derived +/// G+3, so its first pass already credited G+3 and the rescan must not apply it again. #[tokio::test] async fn coinjoin_rescan_skips_blocks_applied_after_the_scripts() { let (mut manager, wallet, wallet_id) = setup().await; - let addresses = coinjoin_external_addresses(&wallet, &wallet_id, (G + 4) as u32).await; + let addresses = coinjoin_external_addresses(&*wallet.read().await, &wallet_id, (G + 4) as u32); - let (block_a, filter_a, key_a) = - block_paying(10, &[addresses[5].clone(), addresses[G + 2].clone()]); + let (pays_g2, _, _) = block_paying(10, &[addresses[G + 2].clone()]); + let (pays_5, _, _) = block_paying(11, &[addresses[5].clone()]); + let block_a = Block::dummy(10, vec![pays_g2.txdata[0].clone(), pays_5.txdata[0].clone()]); + let (filter_a, key_a) = + (BlockFilter::dummy(&block_a), FilterMatchKey::new(10, block_a.block_hash())); let (block_b, filter_b, key_b) = block_paying(20, &[addresses[1].clone(), addresses[G + 3].clone()]); let (hash_a, hash_b) = (block_a.block_hash(), block_b.block_hash()); @@ -364,7 +336,9 @@ async fn coinjoin_rescan_skips_blocks_applied_after_the_scripts() { let (highest_used, _, used_count) = coinjoin_pool_state(&wallet, &wallet_id).await; assert_eq!(highest_used, Some((G + 3) as u32)); assert_eq!(used_count, 4, "indices 1, 5, G+2 and G+3 must all be credited"); - assert_eq!(passes.get(&hash_a), Some(&2), "A predates the scripts it derived"); + let balance = wallet.read().await.get_wallet_balance(&wallet_id).expect("balance"); + assert_eq!(balance.confirmed(), 4 * DENOMINATION); + assert_eq!(passes.get(&hash_a), Some(&1), "A credits the scripts it derived itself"); assert_eq!(passes.get(&hash_b), Some(&1), "B was applied after G+3 was derived"); } @@ -397,7 +371,7 @@ async fn run_batch( #[tokio::test] async fn provider_wallet_rescans_only_untested_outpoints() { let (mut manager, wallet, wallet_id) = setup().await; - let addresses = coinjoin_external_addresses(&wallet, &wallet_id, 7).await; + let addresses = coinjoin_external_addresses(&*wallet.read().await, &wallet_id, 7); // Batch 0: A creates the outpoint the later batches refer to. let (block_a, filter_a, key_a) = block_paying(10, &[addresses[5].clone()]); @@ -433,7 +407,7 @@ async fn provider_wallet_rescans_only_untested_outpoints() { #[tokio::test] async fn provider_wallet_rescan_prefers_an_untested_outpoint_over_an_older_script() { let (mut manager, wallet, wallet_id) = setup().await; - let addresses = coinjoin_external_addresses(&wallet, &wallet_id, (G + 4) as u32).await; + let addresses = coinjoin_external_addresses(&*wallet.read().await, &wallet_id, (G + 4) as u32); let a = block_paying(10, &[addresses[5].clone()]); let z = block_paying(30, &[addresses[2].clone()]); @@ -445,3 +419,41 @@ async fn provider_wallet_rescan_prefers_an_untested_outpoint_over_an_older_scrip let passes = run_batch(&mut manager, &wallet, 0, vec![a, (block_x, filter_x, key_x), z]).await; assert_eq!(passes.get(&hash_x), Some(&2), "X never saw the outpoint Z created"); } + +/// Block A in the first batch pays index 5 and G+4, which derives the window +/// up to 2G+4. The next batch was scanned before that, and its blocks pay G+2 +/// and G+6. Both are found by the deferred rescan, and every block is applied once. +#[tokio::test] +async fn coinjoin_scripts_derived_in_one_batch_find_blocks_in_the_next() { + let (mut manager, wallet, wallet_id) = setup().await; + let addresses = coinjoin_external_addresses(&*wallet.read().await, &wallet_id, (G + 7) as u32); + + let (block_a, filter_a, key_a) = + block_paying(10, &[addresses[5].clone(), addresses[G + 4].clone()]); + let (block_b, filter_b, key_b) = block_paying(120, &[addresses[G + 6].clone()]); + let (block_c, filter_c, key_c) = block_paying(150, &[addresses[G + 2].clone()]); + let blocks: HashMap = + [block_a, block_b, block_c].into_iter().map(|block| (block.block_hash(), block)).collect(); + + let mut batch = FiltersBatch::new(0, 99, HashMap::from([(key_a, filter_a)])); + batch.mark_verified(); + manager.active_batches.insert(0, batch); + let mut batch = + FiltersBatch::new(100, 199, HashMap::from([(key_b, filter_b), (key_c, filter_c)])); + batch.mark_verified(); + manager.active_batches.insert(100, batch); + manager.progress.update_stored_height(199); + + let initial_events = manager.try_process_batch().await.unwrap(); + let passes = drive_to_quiescence(&mut manager, &wallet, &blocks, initial_events).await; + + let (highest_used, _, used_count) = coinjoin_pool_state(&wallet, &wallet_id).await; + assert_eq!(highest_used, Some((G + 6) as u32)); + assert_eq!(used_count, 4, "indices 5, G+2, G+4 and G+6 must all be credited"); + let balance = wallet.read().await.get_wallet_balance(&wallet_id).expect("balance"); + assert_eq!(balance.confirmed(), 4 * DENOMINATION); + assert_eq!(passes.len(), 3); + assert!(passes.values().all(|count| *count == 1), "each block is applied once: {passes:?}"); + assert!(manager.active_batches.is_empty(), "both batches commit"); + assert_eq!(manager.progress.committed_height(), 199); +} diff --git a/dash-spv/src/sync/filters/manager.rs b/dash-spv/src/sync/filters/manager.rs index 1a2d56560..313483d2b 100644 --- a/dash-spv/src/sync/filters/manager.rs +++ b/dash-spv/src/sync/filters/manager.rs @@ -5,7 +5,6 @@ //! Emits FiltersStored, FiltersSyncComplete and BlocksNeeded events. use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use std::mem; use std::sync::Arc; use dashcore::bip158::{BlockFilter, FilterQuery}; @@ -118,7 +117,7 @@ fn rescan_match( const BATCH_PROCESSING_SIZE: u32 = 5000; /// One wallet's scripts to rescan, keyed by the block pass that derived them. -type DerivedScripts = BTreeMap>; +pub(super) type DerivedScripts = BTreeMap>; /// Pass for scripts of unknown origin. No pass is later, so every block they /// match is re-applied. @@ -183,9 +182,6 @@ pub struct FiltersManager< /// `BlockProcessed` and the per-wallet record of which wallets already /// have a given processed block applied. pub(super) tracker: BlockMatchTracker, - /// Scripts found by processed blocks since the last rescan, rescanned - /// together. Grouped per wallet by the pass that derived them. - pub(super) pending_new_scripts: BTreeMap>>, } impl @@ -199,6 +195,7 @@ impl>, ) -> Self { let committed_height = wallet.read().await.synced_height(); + let checks_derived_addresses = wallet.read().await.checks_derived_addresses(); let stored_height = filter_storage.read().await.filter_tip_height().await.unwrap_or(0); let target_height = header_storage.read().await.get_tip().await.map(|t| t.height()).unwrap_or(0); @@ -229,8 +226,7 @@ impl 0 { @@ -868,33 +864,45 @@ impl SyncResult> { - let new_scripts = mem::take(&mut self.pending_new_scripts); - let mut events = Vec::new(); - let starts: Vec = - self.active_batches.iter().filter(|(_, b)| b.scanned()).map(|(s, _)| *s).collect(); - for batch_start in starts { - let Some(batch) = self.active_batches.get(&batch_start) else { - continue; - }; - let mut untested: HashMap = HashMap::new(); - for (id, by_pass) in &new_scripts { - if !batch.scanned_wallets().contains_key(id) { - continue; - } - for (pass, scripts) in by_pass { - let missing: HashSet = - batch.untested(id, scripts).cloned().collect(); - if !missing.is_empty() { - untested.entry(*id).or_default().insert(*pass, missing); - } - } + /// Defer `new_scripts`, derived by `pass`, to every scanned batch that has not matched them yet, + /// then rescan the lowest batch at once: the next commit waits on its blocks. + /// The other batches are rescanned by [`Self::rescan_next_deferred`]. + pub(super) async fn defer_new_scripts( + &mut self, + new_scripts: &BTreeMap>, + pass: u64, + ) -> SyncResult> { + if new_scripts.is_empty() { + return Ok(vec![]); + } + for batch in self.active_batches.values_mut() { + for (wallet_id, scripts) in new_scripts { + batch.defer(*wallet_id, pass, scripts); } - events.extend(self.rescan_batch(batch_start, &untested).await?); } - Ok(events) + let Some(&lowest) = self.active_batches.keys().next() else { + return Ok(vec![]); + }; + self.rescan_deferred(lowest).await + } + + /// Rescan the lowest batch with deferred scripts. One batch per call keeps a + /// tick short, so `BlockProcessed` events are not queued behind every batch. + pub(super) async fn rescan_next_deferred(&mut self) -> SyncResult> { + let next = self.active_batches.iter().find(|(_, batch)| batch.has_deferred()); + match next.map(|(start, _)| *start) { + Some(batch_start) => self.rescan_deferred(batch_start).await, + None => Ok(vec![]), + } + } + + /// Match the scripts deferred for the batch at `batch_start` against it. + async fn rescan_deferred(&mut self, batch_start: u32) -> SyncResult> { + let Some(batch) = self.active_batches.get_mut(&batch_start) else { + return Ok(vec![]); + }; + let deferred = batch.take_deferred(); + self.rescan_batch(batch_start, &deferred).await } /// Rescan a specific batch for newly discovered scriptPubKeys, attributed @@ -2309,6 +2317,8 @@ mod tests { /// `rescan_batch` re-queues an already processed block only when its last /// pass ran no later than the pass that derived the newest scripts matching it. + /// The mock wallet keeps the default `checks_derived_addresses`, so the + /// deriving block itself is re-queued. #[tokio::test] async fn test_rescan_batch_reprocesses_already_processed_block() { let old_address = Address::dummy(Network::Regtest, 33); @@ -2630,11 +2640,13 @@ mod tests { assert_eq!(heights, vec![30, 50, 60]); } - /// The per-tick rescan does not request an in-flight block again for a - /// wallet it is already queued for, since the queued pass sees the new - /// scripts. It still requests the block for a wallet not yet queued. + /// New scripts are matched against the lowest batch while their + /// `BlockProcessed` is handled, and against the other batches on the next + /// tick. An in-flight block is not requested again for a wallet it is + /// already queued for, since the queued pass sees the new scripts. It is + /// still requested for a wallet not yet queued. #[tokio::test] - async fn test_tick_rescan_requests_in_flight_block_only_for_new_wallets() { + async fn test_new_scripts_request_the_lowest_batch_first() { let wallet_a: WalletId = [0x1A; 32]; let wallet_b: WalletId = [0x1B; 32]; let watched = Address::dummy(Network::Regtest, 51); @@ -2651,35 +2663,41 @@ mod tests { multi.insert_wallet(wallet_b, MockWalletState::default()); let mut manager = create_multi_test_manager(Arc::new(RwLock::new(multi))).await; manager.set_state(SyncState::Syncing); - manager.progress.update_stored_height(99); - manager.progress.update_filter_header_tip_height(99); + manager.progress.update_stored_height(199); + manager.progress.update_filter_header_tip_height(199); - // Block 40 pays the watched address and the one derived below. - let block = Block::dummy( + // Block 40 pays the watched address and the one derived below. Block + // 140, in the next batch, pays only the derived one. + let block_40 = Block::dummy( 40, vec![ Transaction::dummy(&watched, 0..0, &[40]), Transaction::dummy(&derived, 0..0, &[41]), ], ); - let key_40 = FilterMatchKey::new(40, block.block_hash()); - let filter_40 = BlockFilter::dummy(&block); + let key_40 = FilterMatchKey::new(40, block_40.block_hash()); + let (key_140, filter_140) = filter_for_address(140, &derived); + let filter_40 = BlockFilter::dummy(&block_40); let mut batch = FiltersBatch::new(0, 99, HashMap::from([(key_40.clone(), filter_40)])); batch.mark_verified(); manager.active_batches.insert(0, batch); + let mut batch = FiltersBatch::new(100, 199, HashMap::from([(key_140.clone(), filter_140)])); + batch.mark_verified(); + manager.active_batches.insert(100, batch); let events = manager.scan_batch(0).await.unwrap(); assert!(matches!( events.as_slice(), [SyncEvent::BlocksNeeded { blocks }] if blocks.get(&key_40) == Some(&BTreeSet::from([wallet_a])) )); + assert!(manager.scan_batch(100).await.unwrap().is_empty()); let network: Arc = Arc::new(MockNetworkManager::new()); - let requested_40 = |events: &[SyncEvent]| { + let requested = |events: &[SyncEvent], key: &FilterMatchKey| { events.iter().find_map(|e| match e { SyncEvent::BlocksNeeded { blocks, - } => blocks.get(&key_40).cloned(), + } => blocks.get(key).cloned(), _ => None, }) }; @@ -2693,18 +2711,26 @@ mod tests { }; // Block 20 derived the script for wallet A while block 40 was queued - // for A: nothing is requested. - manager.handle_sync_event(&report(wallet_a), &network).await.unwrap(); + // for A: nothing is requested from the lowest batch, and block 140 + // waits for the tick. + let events = manager.handle_sync_event(&report(wallet_a), &network).await.unwrap(); + assert_eq!(requested(&events, &key_40), None, "block 40 is already queued for wallet A"); + assert_eq!(requested(&events, &key_140), None); let events = manager.tick(&network).await.unwrap(); - assert_eq!(requested_40(&events), None, "block 40 is already queued for wallet A"); + assert_eq!(requested(&events, &key_140), Some(BTreeSet::from([wallet_a]))); - // The same script for wallet B adds B to the queued pass. - manager.handle_sync_event(&report(wallet_b), &network).await.unwrap(); + // The same script for wallet B adds B to the queued pass of block 40 + // at once, and requests block 140 for B on the next tick. + let events = manager.handle_sync_event(&report(wallet_b), &network).await.unwrap(); + assert_eq!(requested(&events, &key_40), Some(BTreeSet::from([wallet_b]))); + assert_eq!(requested(&events, &key_140), None); let events = manager.tick(&network).await.unwrap(); - assert_eq!(requested_40(&events), Some(BTreeSet::from([wallet_b]))); + assert_eq!(requested(&events, &key_140), Some(BTreeSet::from([wallet_b]))); + assert!(manager.tick(&network).await.unwrap().is_empty()); - // Block 40 is counted once, however often the rescan matched it. + // Each block is counted once, however often a rescan matched it. assert_eq!(manager.active_batches.get(&0).unwrap().pending_blocks(), 1); + assert_eq!(manager.active_batches.get(&100).unwrap().pending_blocks(), 1); } /// `scan_batch` `AlreadyProcessed` path: when every candidate wallet has @@ -3132,6 +3158,72 @@ mod tests { ); } + /// Scripts deferred for a batch are matched against it before it commits, + /// with the pass that derived them. A block applied by a later pass is not + /// requested again, and the batch waits for the other block. + #[tokio::test] + async fn test_commit_rescans_deferred_scripts_with_their_pass() { + let wallet_id: WalletId = [2; 32]; + let watched = Address::dummy(Network::Testnet, 13); + let derived = Address::dummy(Network::Testnet, 14); + + let mut multi = MultiMockWallet::new(); + multi.insert_wallet( + wallet_id, + MockWalletState { + addresses: vec![watched.clone()], + ..MockWalletState::default() + }, + ); + let mut manager = create_multi_test_manager(Arc::new(RwLock::new(multi))).await; + manager.set_state(SyncState::Syncing); + manager.progress.update_stored_height(199); + manager.progress.update_filter_header_tip_height(199); + + let (key_140, filter_140) = filter_for_address(140, &derived); + let (key_150, filter_150) = filter_for_address(150, &derived); + let mut batch = FiltersBatch::new(0, 99, HashMap::new()); + batch.mark_verified(); + manager.active_batches.insert(0, batch); + let filters = HashMap::from([(key_140.clone(), filter_140), (key_150.clone(), filter_150)]); + let mut batch = FiltersBatch::new(100, 199, filters); + batch.mark_verified(); + manager.active_batches.insert(100, batch); + assert!(manager.scan_batch(0).await.unwrap().is_empty()); + assert!(manager.scan_batch(100).await.unwrap().is_empty()); + + // The pass of block 20 derives `derived`. + manager.wallet.write().await.set_scan_addresses(wallet_id, vec![watched, derived.clone()]); + let network: Arc = Arc::new(MockNetworkManager::new()); + let event = SyncEvent::BlockProcessed { + block_hash: Header::dummy(20).block_hash(), + height: 20, + wallets: BTreeSet::from([wallet_id]), + new_scripts: BTreeMap::from([(wallet_id, vec![derived.script_pubkey()])]), + confirmed_txids: vec![], + }; + let events = manager.handle_sync_event(&event, &network).await.unwrap(); + assert!(events.is_empty(), "the lowest batch has no block paying `derived`"); + // Block 140 was applied by a later pass. + manager.tracker.record_processed(140, *key_140.hash(), &BTreeSet::from([wallet_id])); + + // Batch 0 commits. Batch 100 matches the deferred script first. + let events = manager.try_process_batch().await.unwrap(); + let requested: Vec = events + .iter() + .filter_map(|e| match e { + SyncEvent::BlocksNeeded { + blocks, + } => Some(blocks.keys().cloned()), + _ => None, + }) + .flatten() + .collect(); + assert_eq!(requested, vec![key_150]); + assert_eq!(manager.progress.committed_height(), 99); + assert_eq!(manager.active_batches.get(&100).unwrap().pending_blocks(), 1); + } + /// Proves a batch scanned with a query missing one address — the shape of /// a lost `new_scripts` notification — does not commit until /// reconciliation matches that address and finds its block. diff --git a/dash-spv/src/sync/filters/sync_manager.rs b/dash-spv/src/sync/filters/sync_manager.rs index 2df026452..3ca83fd9c 100644 --- a/dash-spv/src/sync/filters/sync_manager.rs +++ b/dash-spv/src/sync/filters/sync_manager.rs @@ -183,21 +183,9 @@ impl< // `tracker.track` residual. let pass = self.tracker.record_processed(*height, *block_hash, wallets); - // Rescanned on the next tick or before the next commit, together - // with whatever the blocks processed meanwhile find: once per block, - // every active batch is re-matched for each block. - for (wallet_id, scripts) in new_scripts { - self.pending_new_scripts - .entry(*wallet_id) - .or_default() - .entry(pass) - .or_default() - .extend(scripts.iter().cloned()); - } - let mut events = Vec::new(); - // Check if this block is part of our tracked blocks - if let Some((_, batch_start)) = self.tracker.finish_in_flight(block_hash, wallets) { + let tracked = self.tracker.finish_in_flight(block_hash, wallets); + if let Some((_, batch_start)) = tracked { if let Some(batch) = self.active_batches.get_mut(&batch_start) { batch.decrement_pending_blocks(); tracing::debug!( @@ -208,6 +196,9 @@ impl< batch.pending_blocks() ); } + } + let mut events = self.defer_new_scripts(new_scripts, pass).await?; + if tracked.is_some() { events.extend(self.try_process_batch().await?); } return Ok(events); @@ -273,7 +264,7 @@ impl< self.filter_pipeline.send_pending(network, &*header_storage).await?; drop(header_storage); - let mut events = self.rescan_active_batches().await?; + let mut events = self.rescan_next_deferred().await?; // Store completed batches and do speculative matching events.extend(self.store_and_match_batches().await?); diff --git a/key-wallet-manager/src/process_block.rs b/key-wallet-manager/src/process_block.rs index 022980a74..ed6459010 100644 --- a/key-wallet-manager/src/process_block.rs +++ b/key-wallet-manager/src/process_block.rs @@ -5,16 +5,29 @@ use async_trait::async_trait; use dashcore::ephemerealdata::chain_lock::ChainLock; use dashcore::ephemerealdata::instant_lock::InstantLock; use dashcore::prelude::CoreBlockHeight; -use dashcore::{Address, Block, BlockHash, ScriptBuf, Transaction}; +use dashcore::{Address, Block, BlockHash, ScriptBuf, Transaction, Txid}; use hex_conservative::DisplayHex; use key_wallet::account::AccountType; use key_wallet::managed_account::transaction_record::TransactionRecord; use key_wallet::transaction_checking::{BlockInfo, DerivedAddressInfo, TransactionContext}; use key_wallet::wallet::managed_wallet_info::wallet_info_interface::WalletInfoInterface; use key_wallet::WalletCoreBalance; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use tokio::sync::broadcast; +/// Whether both records are of the same transaction in the same account. +fn same_record(a: &TransactionRecord, b: &TransactionRecord) -> bool { + a.txid == b.txid && a.account_type == b.account_type +} + +/// Adds `record`, replacing an earlier version of the same record. +fn upsert_record(records: &mut Vec, record: TransactionRecord) { + match records.iter_mut().find(|r| same_record(r, &record)) { + Some(existing) => *existing = record, + None => records.push(record), + } +} + #[async_trait] impl WalletInterface for WalletManager { async fn process_block_for_wallets( @@ -50,76 +63,105 @@ impl WalletInterface for WalletM let mut per_wallet_inserted: BTreeMap> = BTreeMap::new(); let mut per_wallet_updated: BTreeMap> = BTreeMap::new(); let mut per_wallet_derived: BTreeMap> = BTreeMap::new(); - let mut relevant_positions: BTreeMap> = BTreeMap::new(); - - for (position, tx) in block.txdata.iter().enumerate() { - // Stamp each record with its `block.vtx` index so consumers - // can replay Core's same-block apply order (e.g. multiple - // provider updates for one masternode in a single block). - let tx_info = info.with_position(position as u32); - let block_context = if all_chainlocked { - TransactionContext::InChainLockedBlock(tx_info) - } else { - TransactionContext::InBlock(tx_info) - }; - let check_result = - self.check_transaction_in_wallets(tx, block_context, wallets, true, false).await; - - if !check_result.affected_wallets.is_empty() { - if check_result.is_new_transaction { - result.new_txids.push(tx.txid()); + let mut relevant_positions: BTreeMap> = BTreeMap::new(); + let mut listed_txids: HashSet = HashSet::new(); + + // A transaction can pay an address that a later transaction of this + // block derives. So every wallet that derived addresses checks the + // whole block again, until none derives more. + let mut pass_wallets = wallets.clone(); + while !pass_wallets.is_empty() { + let mut derived_wallets = BTreeSet::new(); + for (position, tx) in block.txdata.iter().enumerate() { + // Stamp each record with its `block.vtx` index so consumers + // can replay Core's same-block apply order (e.g. multiple + // provider updates for one masternode in a single block). + let tx_info = info.with_position(position as u32); + let block_context = if all_chainlocked { + TransactionContext::InChainLockedBlock(tx_info) } else { - result.existing_txids.push(tx.txid()); + TransactionContext::InBlock(tx_info) + }; + let check_result = self + .check_transaction_in_wallets(tx, block_context, &pass_wallets, true, false) + .await; + + let txid = tx.txid(); + if !check_result.affected_wallets.is_empty() && listed_txids.insert(txid) { + if check_result.is_new_transaction { + result.new_txids.push(txid); + } else { + result.existing_txids.push(txid); + } + } + for wallet_id in &check_result.affected_wallets { + relevant_positions.entry(*wallet_id).or_default().insert(position); } - } - for wallet_id in &check_result.affected_wallets { - relevant_positions.entry(*wallet_id).or_default().push(position); - } - for (wallet_id, derived) in check_result.new_addresses { - let scripts = - derived.iter().map(|d| d.info.script_pubkey.clone()).collect::>(); - result.new_scripts.entry(wallet_id).or_default().extend(scripts); - per_wallet_derived.entry(wallet_id).or_default().extend(derived); - } - for (wallet_id, records) in check_result.per_wallet_new_records { - per_wallet_inserted.entry(wallet_id).or_default().extend(records); - } - for (wallet_id, records) in check_result.per_wallet_updated_records { - per_wallet_updated.entry(wallet_id).or_default().extend(records); - } - // Emitted per transaction rather than batched into the block - // event: a sweep names the transaction that superseded the - // removed ones, and that attribution is lost once the block's - // transactions are folded together. - let mut per_wallet_released = check_result.per_wallet_released_outpoints; - for (wallet_id, txids) in check_result.per_wallet_swept { - if txids.is_empty() { - continue; + for (wallet_id, derived) in check_result.new_addresses { + derived_wallets.insert(wallet_id); + let scripts = + derived.iter().map(|d| d.info.script_pubkey.clone()).collect::>(); + result.new_scripts.entry(wallet_id).or_default().extend(scripts); + per_wallet_derived.entry(wallet_id).or_default().extend(derived); } - let Some(info) = self.wallet_infos.get(&wallet_id) else { - continue; - }; - let released_outpoints = per_wallet_released.remove(&wallet_id).unwrap_or_default(); - let event = WalletEvent::TransactionsSwept { - wallet_id, - txids, - superseded_by: tx.txid(), - // The winner is a transaction of this very block, so its - // mined height is the block's height. - winner_mined_height: Some(height), - released_outpoints, - balance: info.balance(), - account_balances: BTreeMap::new(), - }; - self.emit_event(event); + for (wallet_id, records) in check_result.per_wallet_new_records { + let inserted = per_wallet_inserted.entry(wallet_id).or_default(); + for record in records { + upsert_record(inserted, record); + } + } + for (wallet_id, records) in check_result.per_wallet_updated_records { + for record in records { + // A record inserted by an earlier pass stays an insert. + let inserted = + per_wallet_inserted.get_mut(&wallet_id).and_then(|inserted| { + inserted.iter_mut().find(|r| same_record(r, &record)) + }); + match inserted { + Some(inserted) => *inserted = record, + None => upsert_record( + per_wallet_updated.entry(wallet_id).or_default(), + record, + ), + } + } + } + // Emitted per transaction rather than batched into the block + // event: a sweep names the transaction that superseded the + // removed ones, and that attribution is lost once the block's + // transactions are folded together. + let mut per_wallet_released = check_result.per_wallet_released_outpoints; + for (wallet_id, txids) in check_result.per_wallet_swept { + if txids.is_empty() { + continue; + } + let Some(info) = self.wallet_infos.get(&wallet_id) else { + continue; + }; + let released_outpoints = + per_wallet_released.remove(&wallet_id).unwrap_or_default(); + let event = WalletEvent::TransactionsSwept { + wallet_id, + txids, + superseded_by: txid, + // The winner is a transaction of this very block, so its + // mined height is the block's height. + winner_mined_height: Some(height), + released_outpoints, + balance: info.balance(), + account_balances: BTreeMap::new(), + }; + self.emit_event(event); + } + debug_assert!( + per_wallet_released.is_empty(), + "released outpoints for a wallet that emitted no sweep would be \ + dropped here, stranding those coins marked spent forever: {:?}", + per_wallet_released + ); } - debug_assert!( - per_wallet_released.is_empty(), - "released outpoints for a wallet that emitted no sweep would be \ - dropped here, stranding those coins marked spent forever: {:?}", - per_wallet_released - ); + pass_wallets = derived_wallets; } for (wallet_id, positions) in relevant_positions { @@ -147,6 +189,10 @@ impl WalletInterface for WalletM result } + fn checks_derived_addresses(&self) -> bool { + true + } + async fn process_mempool_transaction( &mut self, tx: &Transaction, @@ -674,6 +720,7 @@ mod tests { use super::*; use crate::matching::{check_compact_filters_for_elements, FilterMatchKey}; use crate::test_helpers::*; + use crate::test_utils::coinjoin_external_addresses; use dashcore::bip158::BlockFilter; use dashcore::block::{Header, Version}; use dashcore::pow::CompactTarget; @@ -682,6 +729,8 @@ mod tests { }; use key_wallet::account::ManagedAccountTrait as _; use key_wallet::account::StandardAccountType; + use key_wallet::gap_limit::DEFAULT_COINJOIN_GAP_LIMIT; + use key_wallet::managed_account::address_pool::AddressPoolType; use key_wallet::mnemonic::Language; use key_wallet::wallet::initialization::WalletAccountCreationOptions; use key_wallet::wallet::managed_wallet_info::transaction_building::AccountTypePreference; @@ -963,6 +1012,78 @@ mod tests { assert!(found, "should emit BlockProcessed for block processing"); } + /// A block paying addresses that only its own transactions derive is fully + /// credited by one call. A transaction checked again is listed and recorded once. + #[tokio::test] + async fn test_process_block_credits_addresses_the_block_derives() { + let (mut manager, wallet_id, _addr) = setup_manager_with_wallet(); + let gap = DEFAULT_COINJOIN_GAP_LIMIT; + let addresses = coinjoin_external_addresses(&manager, &wallet_id, gap + 9); + let derived_by_late = &addresses[(gap + 2) as usize]; + let also_derived_by_late = &addresses[(gap + 8) as usize]; + + // `late` uses index 8, which derives up to G+8. `early` pays G+2, and + // `both` pays index 6 and G+8, past what using 6 alone derives. + let early = create_tx_paying_to(derived_by_late, 0x21); + let mut both = create_tx_paying_to(&addresses[6], 0x23); + both.output.push(TxOut { + value: TX_AMOUNT, + script_pubkey: also_derived_by_late.script_pubkey(), + }); + let late = create_tx_paying_to(&addresses[8], 0x22); + let block = make_block(vec![early.clone(), both.clone(), late.clone()]); + + let mut rx = manager.subscribe_events(); + let result = manager + .process_block_for_wallets( + &block, + block.block_hash(), + 100, + &BTreeSet::from([wallet_id]), + ) + .await; + + assert_eq!( + manager.get_wallet_balance(&wallet_id).expect("balance").confirmed(), + 4 * TX_AMOUNT + ); + let pool_used = coinjoin_account(&manager, &wallet_id) + .managed_account_type() + .address_pools() + .iter() + .find(|p| p.pool_type == AddressPoolType::External) + .and_then(|p| p.highest_used); + assert_eq!(pool_used, Some(gap + 8)); + + let txids = BTreeSet::from([early.txid(), late.txid(), both.txid()]); + assert_eq!(result.new_txids.len(), 3); + assert_eq!(result.new_txids.iter().copied().collect::>(), txids); + assert!(result.existing_txids.is_empty()); + assert!(result.new_scripts[&wallet_id].contains(&derived_by_late.script_pubkey())); + + // The second pass corrects `both`, which the first recorded owning one + // output. The event carries it once, as inserted, with both outputs. + let inserted: Vec = drain_events(&mut rx) + .into_iter() + .filter_map(|event| match event { + WalletEvent::BlockProcessed { + inserted, + updated, + .. + } => { + assert!(updated.is_empty()); + Some(inserted) + } + _ => None, + }) + .flatten() + .collect(); + assert_eq!(inserted.len(), 3); + assert_eq!(inserted.iter().map(|record| record.txid).collect::>(), txids); + let both_record = inserted.iter().find(|record| record.txid == both.txid()); + assert_eq!(both_record.map(|record| record.net_amount), Some(2 * TX_AMOUNT as i64)); + } + #[tokio::test] async fn test_update_wallet_synced_height_emits_sync_height_advanced() { let (mut manager, wallet_id, _addr) = setup_manager_with_wallet(); diff --git a/key-wallet-manager/src/test_utils/coinjoin.rs b/key-wallet-manager/src/test_utils/coinjoin.rs new file mode 100644 index 000000000..474ce5eb2 --- /dev/null +++ b/key-wallet-manager/src/test_utils/coinjoin.rs @@ -0,0 +1,33 @@ +use crate::{WalletId, WalletManager}; +use dashcore::Address; +use key_wallet::account::ManagedAccountTrait; +use key_wallet::managed_account::address_pool::{AddressPool, AddressPoolType, KeySource}; +use key_wallet::wallet::managed_wallet_info::ManagedWalletInfo; + +/// Returns the first `count` External addresses of the wallet's CoinJoin account 0. +/// They come from a standalone pool over the account xpub, so they can lie beyond +/// the wallet's watch window without changing its state. +pub fn coinjoin_external_addresses( + manager: &WalletManager, + wallet_id: &WalletId, + count: u32, +) -> Vec
{ + let wallet = manager.get_wallet(wallet_id).expect("wallet present"); + let xpub = wallet.accounts.coinjoin_accounts.get(&0).expect("CoinJoin account 0").account_xpub; + let info = manager.get_wallet_info(wallet_id).expect("wallet info present"); + let account = info.accounts.coinjoin_accounts.get(&0).expect("managed CoinJoin account 0"); + let pools = account.managed_account_type().address_pools(); + let live = pools + .iter() + .find(|p| p.pool_type == AddressPoolType::External) + .expect("CoinJoin External pool"); + let pool = AddressPool::new( + live.base_path.clone(), + AddressPoolType::External, + count, + wallet.network, + &KeySource::Public(xpub), + ) + .expect("derive standalone CoinJoin External pool"); + (0..count).map(|i| pool.address_at_index(i).expect("address generated")).collect() +} diff --git a/key-wallet-manager/src/test_utils/mod.rs b/key-wallet-manager/src/test_utils/mod.rs index 252be4387..d1f4bc21c 100644 --- a/key-wallet-manager/src/test_utils/mod.rs +++ b/key-wallet-manager/src/test_utils/mod.rs @@ -1,5 +1,7 @@ +mod coinjoin; mod mock_wallet; +pub use coinjoin::coinjoin_external_addresses; pub use mock_wallet::MockWallet; pub use mock_wallet::MockWalletState; pub use mock_wallet::MultiMockWallet; diff --git a/key-wallet-manager/src/wallet_interface.rs b/key-wallet-manager/src/wallet_interface.rs index c1b6360e3..7941a2cc3 100644 --- a/key-wallet-manager/src/wallet_interface.rs +++ b/key-wallet-manager/src/wallet_interface.rs @@ -74,6 +74,13 @@ pub trait WalletInterface: Send + Sync + 'static { wallets: &BTreeSet, ) -> BlockProcessingResult; + /// Whether [`Self::process_block_for_wallets`] checks every transaction of + /// the block against the addresses that call derives. The filter rescan then + /// skips the block for them instead of applying it again. + fn checks_derived_addresses(&self) -> bool { + false + } + /// Called when a transaction is seen in the mempool. /// Returns whether the transaction was relevant and any new addresses generated. /// When `instant_lock` is `Some`, the transaction already has an IS lock.