diff --git a/dash-spv/src/sync/blocks/manager.rs b/dash-spv/src/sync/blocks/manager.rs index a04df3c5b..8a1648a61 100644 --- a/dash-spv/src/sync/blocks/manager.rs +++ b/dash-spv/src/sync/blocks/manager.rs @@ -104,7 +104,7 @@ impl BlocksManager BlocksManager>) { + /// Re-applies stored blocks to the wallets in `reapply`, returning a + /// `BlockProcessed` per pass so the filters manager learns the scripts each + /// pass derived, in the order the passes ran. + async fn reapply_blocks( + &mut self, + reapply: BTreeMap>, + ) -> Vec { + let mut events = Vec::new(); let mut queue: BTreeSet<(u32, WalletId)> = reapply .into_iter() .flat_map(|(wallet_id, heights)| heights.into_iter().map(move |h| (h, wallet_id))) @@ -180,13 +187,23 @@ impl BlocksManager; @@ -349,11 +366,15 @@ mod tests { assert_eq!(processed[0].1, 100); } + /// A re-application reports its own `BlockProcessed` with the scripts it + /// derived, but does not count as another processed block. #[tokio::test] async fn test_process_buffered_blocks_reapplies_requested_stored_block() { let storage = DiskStorageManager::with_temp_dir().await.unwrap(); let mut wallet = MockWallet::new(); wallet.set_reapply_heights(100, BTreeSet::from([200])); + let derived = Address::dummy(Network::Regtest, 7); + wallet.set_derived_addresses(200, vec![derived.clone()]); let wallet = Arc::new(RwLock::new(wallet)); let mut manager: TestBlocksManager = BlocksManager::new(wallet.clone(), storage.block_headers(), storage.blocks()).await; @@ -373,9 +394,23 @@ mod tests { ); let events = manager.process_buffered_blocks().await.unwrap(); + let reported: Vec<_> = events + .iter() + .filter_map(|event| match event { + SyncEvent::BlockProcessed { + height, + wallets, + new_scripts, + .. + } => Some((*height, wallets, new_scripts)), + _ => None, + }) + .collect(); + let wallets = BTreeSet::from([MOCK_WALLET_ID]); + let derived_scripts = BTreeMap::from([(MOCK_WALLET_ID, vec![derived.script_pubkey()])]); assert_eq!( - events.iter().filter(|e| matches!(e, SyncEvent::BlockProcessed { .. })).count(), - 1 + reported, + vec![(100, &wallets, &BTreeMap::new()), (200, &wallets, &derived_scripts)] ); let processed = wallet.read().await.processed_blocks(); diff --git a/dash-spv/src/sync/events.rs b/dash-spv/src/sync/events.rs index 5a04b67d6..9b6d91787 100644 --- a/dash-spv/src/sync/events.rs +++ b/dash-spv/src/sync/events.rs @@ -94,7 +94,8 @@ pub enum SyncEvent { blocks: BTreeMap>, }, - /// Block downloaded and processed through wallet. + /// Block downloaded and processed through wallet. Also emitted for each + /// re-application of a stored block, with the one wallet it was re-applied for. /// /// Emitted by: `BlocksManager` /// Consumed by: `FiltersManager` (for gap limit rescanning), diff --git a/dash-spv/src/sync/filters/batch.rs b/dash-spv/src/sync/filters/batch.rs index 74887ca11..400a959d8 100644 --- a/dash-spv/src/sync/filters/batch.rs +++ b/dash-spv/src/sync/filters/batch.rs @@ -1,5 +1,4 @@ use dashcore::bip158::BlockFilter; -use dashcore::ScriptBuf; use key_wallet_manager::{FilterMatchKey, WalletId}; use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; @@ -32,8 +31,9 @@ pub(super) struct FiltersBatch { /// current account set (dashpay/rust-dashcore#649). Already-synced wallets /// must not be touched. scanned_wallets: BTreeMap, - /// Every script already matched against this batch's filters, per wallet. - tested_scripts: HashMap>, + /// Every script and filter element already matched against this batch's + /// filters, per wallet, as the bytes the filters were queried with. + tested: HashMap>>, } impl FiltersBatch { @@ -51,7 +51,7 @@ impl FiltersBatch { scanned: false, pending_blocks: 0, scanned_wallets: BTreeMap::new(), - tested_scripts: HashMap::new(), + tested: HashMap::new(), } } /// Start height of this batch (inclusive). @@ -103,23 +103,26 @@ impl FiltersBatch { self.pending_blocks = self.pending_blocks.saturating_sub(1); self.pending_blocks } - /// Record that `scripts` have been matched against this batch's filters. - pub(super) fn mark_tested>( + /// Record that `items`, scripts or filter elements, have been matched + /// against this batch's filters. + pub(super) fn mark_tested>( &mut self, wallet_id: WalletId, - scripts: I, + items: impl IntoIterator, ) { - self.tested_scripts.entry(wallet_id).or_default().extend(scripts); + let tested = self.tested.entry(wallet_id).or_default(); + tested.extend(items.into_iter().map(|item| item.as_ref().to_vec())); } - /// The wallet's scripts that this batch has never been matched against. - pub(super) fn untested<'a>( + /// The wallet's scripts or filter elements that this batch has never been + /// matched against. + pub(super) fn untested<'a, T: AsRef<[u8]>>( &'a self, wallet_id: &WalletId, - monitored: &'a [ScriptBuf], - ) -> impl Iterator { - let tested = self.tested_scripts.get(wallet_id); - monitored.iter().filter(move |script| tested.is_none_or(|t| !t.contains(*script))) + monitored: &'a [T], + ) -> impl Iterator { + let tested = self.tested.get(wallet_id); + monitored.iter().filter(move |item| tested.is_none_or(|t| !t.contains(item.as_ref()))) } /// Record the wallets that were behind for this batch at scan time, each diff --git a/dash-spv/src/sync/filters/block_match_tracker.rs b/dash-spv/src/sync/filters/block_match_tracker.rs index e0d911b47..d63d39325 100644 --- a/dash-spv/src/sync/filters/block_match_tracker.rs +++ b/dash-spv/src/sync/filters/block_match_tracker.rs @@ -5,12 +5,15 @@ //! //! - `blocks_remaining`: in-flight matched blocks awaiting `BlockProcessed`, //! keyed by block hash. The associated `(height, batch_start)` lets the -//! `BlockProcessed` handler decrement the right batch's `pending_blocks`. +//! `BlockProcessed` handler decrement the right batch's `pending_blocks` +//! once every requested wallet has reported. The wallets still waiting keep +//! a rescan from requesting the block again for them. //! - `processed_blocks_per_wallet`: which wallets have already had each //! processed block applied to their state, keyed by height (so commit-time //! pruning is one `split_off` call) then by hash. Lets a runtime-added //! wallet still receive a block that was previously processed for another -//! wallet only: the gate is per-wallet, not global. +//! wallet only: the gate is per-wallet, not global. The number of each +//! wallet's last pass lets a rescan skip a block that already saw its scripts. //! //! These two maps are coupled: every call site that consults one consults the //! other, and the lifecycle (track on filter match, record on @@ -41,23 +44,35 @@ pub(super) enum BlockTrackResult { InFlight { wallets: BTreeSet, }, - /// All candidate wallets already have this block applied. Caller skips it. + /// All candidate wallets already have this block applied, or for + /// `track_for_new_scripts` are already queued for it. Caller skips it. AlreadyProcessed, } +/// A matched block awaiting `BlockProcessed`. +#[derive(Debug)] +struct InFlightBlock { + height: u32, + batch_start: u32, + /// Requested wallets whose pass has not reported yet. + awaiting: BTreeSet, +} + /// Per-block tracking state for matched blocks flowing through the filter → /// block → wallet pipeline. See module-level docs for the invariants. #[derive(Debug, Default)] pub(super) struct BlockMatchTracker { - /// In-flight matched blocks awaiting `BlockProcessed`. Maps - /// `block_hash → (height, batch_start)` so the `BlockProcessed` handler - /// can decrement the right batch's `pending_blocks` count. - blocks_remaining: BTreeMap, + /// In-flight matched blocks awaiting `BlockProcessed`, keyed by hash. + blocks_remaining: BTreeMap, /// Per-(height, hash) record of which wallets have had this block - /// applied. Bounded by `prune_at_or_below` after every commit, since - /// below `committed_height` a new wallet can only re-enter via the `tick` - /// rescan trigger which calls `clear` outright. - processed_blocks_per_wallet: BTreeMap>>, + /// applied, with the number of each wallet's last pass. Bounded by + /// `prune_at_or_below` after every commit, since below `committed_height` + /// a new wallet can only re-enter via the `tick` rescan trigger which + /// calls `clear` outright. + processed_blocks_per_wallet: BTreeMap>>, + /// Number of the last pass given to `record_processed`. Never reset, so a + /// record that outlives a `clear` can't look newer than a later pass. + passes: u64, } impl BlockMatchTracker { @@ -81,62 +96,114 @@ impl BlockMatchTracker { if residual.is_empty() { return BlockTrackResult::AlreadyProcessed; } - if self.blocks_remaining.contains_key(key.hash()) { + if let Some(in_flight) = self.blocks_remaining.get_mut(key.hash()) { + in_flight.awaiting.extend(residual.iter().copied()); return BlockTrackResult::InFlight { wallets: residual, }; } - self.blocks_remaining.insert(*key.hash(), (key.height(), batch_start)); + self.insert_in_flight(key, batch_start, &residual); BlockTrackResult::NewlyTracked { wallets: residual, } } - /// Like [`Self::track`], but for matches driven by newly derived - /// scripts: the prior processing of this block predates those scripts, - /// so processed records do not cover them and the block must be - /// re-applied to the full candidate set. Never returns - /// `AlreadyProcessed`. + /// Like [`Self::track`], but for matches on newly derived scripts. Each + /// candidate wallet maps to the pass that derived its matching scripts. A + /// wallet whose last pass of this block ran later already saw them and is + /// skipped, and so is a wallet the block is already queued for. pub(super) fn track_for_new_scripts( &mut self, key: &FilterMatchKey, batch_start: u32, - candidate_wallets: BTreeSet, + candidate_wallets: BTreeMap, ) -> BlockTrackResult { - if self.blocks_remaining.contains_key(key.hash()) { + let processed = self.processed_passes(key); + let residual: BTreeSet = candidate_wallets + .into_iter() + .filter(|(id, derived_at)| { + processed.and_then(|p| p.get(id)).is_none_or(|pass| pass <= derived_at) + }) + .map(|(id, _)| id) + .collect(); + if residual.is_empty() { + return BlockTrackResult::AlreadyProcessed; + } + if let Some(in_flight) = self.blocks_remaining.get_mut(key.hash()) { + // Passes report in the order they ran and the pass that derived the + // scripts has reported, so the queued pass runs after it and sees them. + let added: BTreeSet = + residual.difference(&in_flight.awaiting).copied().collect(); + if added.is_empty() { + return BlockTrackResult::AlreadyProcessed; + } + in_flight.awaiting.extend(added.iter().copied()); return BlockTrackResult::InFlight { - wallets: candidate_wallets, + wallets: added, }; } - self.blocks_remaining.insert(*key.hash(), (key.height(), batch_start)); + self.insert_in_flight(key, batch_start, &residual); BlockTrackResult::NewlyTracked { - wallets: candidate_wallets, + wallets: residual, } } - /// Record that `wallets` have had the block at `(height, hash)` applied - /// to their state. Idempotent: existing entries merge, never shrink. + /// Records that `wallets` have had the block at `(height, hash)` applied + /// and returns the number of this pass. Passes are numbered in the order + /// their `BlockProcessed` arrives, which is the order they ran. pub(super) fn record_processed( &mut self, height: u32, hash: BlockHash, wallets: &BTreeSet, - ) { - if wallets.is_empty() { - return; + ) -> u64 { + self.passes += 1; + let pass = self.passes; + if !wallets.is_empty() { + self.processed_blocks_per_wallet + .entry(height) + .or_default() + .entry(hash) + .or_default() + .extend(wallets.iter().map(|id| (*id, pass))); + } + pass + } + + /// Marks the pass of the in-flight block `hash` for `wallets` as reported. + /// Returns its `(height, batch_start)` and stops tracking it once no + /// requested wallet is still waiting. A pass for only some of them, such as + /// a re-application, leaves the block in flight for the others. + pub(super) fn finish_in_flight( + &mut self, + hash: &BlockHash, + wallets: &BTreeSet, + ) -> Option<(u32, u32)> { + let block = self.blocks_remaining.get_mut(hash)?; + block.awaiting.retain(|id| !wallets.contains(id)); + if !block.awaiting.is_empty() { + return None; } - self.processed_blocks_per_wallet - .entry(height) - .or_default() - .entry(hash) - .or_default() - .extend(wallets.iter().copied()); + self.blocks_remaining.remove(hash).map(|block| (block.height, block.batch_start)) } - /// Remove the in-flight entry for `hash`, returning its - /// `(height, batch_start)` if it was tracked. - pub(super) fn finish_in_flight(&mut self, hash: &BlockHash) -> Option<(u32, u32)> { - self.blocks_remaining.remove(hash) + /// The number of the last pass that applied the block at `key` to `wallet_id`. + pub(super) fn last_pass(&self, key: &FilterMatchKey, wallet_id: &WalletId) -> Option { + self.processed_passes(key).and_then(|passes| passes.get(wallet_id)).copied() + } + + fn insert_in_flight( + &mut self, + key: &FilterMatchKey, + batch_start: u32, + wallets: &BTreeSet, + ) { + let block = InFlightBlock { + height: key.height(), + batch_start, + awaiting: wallets.clone(), + }; + self.blocks_remaining.insert(*key.hash(), block); } /// Drop every per-wallet processing record at or below `height`. Called @@ -161,11 +228,12 @@ impl BlockMatchTracker { /// Wallets that have already had this block applied to their state. fn already_processed_wallets(&self, key: &FilterMatchKey) -> BTreeSet { - self.processed_blocks_per_wallet - .get(&key.height()) - .and_then(|m| m.get(key.hash())) - .cloned() - .unwrap_or_default() + self.processed_passes(key).map(|p| p.keys().copied().collect()).unwrap_or_default() + } + + /// The last pass that applied this block, per wallet. + fn processed_passes(&self, key: &FilterMatchKey) -> Option<&BTreeMap> { + self.processed_blocks_per_wallet.get(&key.height()).and_then(|m| m.get(key.hash())) } } @@ -181,7 +249,9 @@ mod tests { /// match, InFlight on re-match while the block is awaiting processing, /// NewlyTracked again for a residual wallet after the first wallet's /// processing is recorded, and AlreadyProcessed once every candidate is - /// covered. + /// covered. `track_for_new_scripts` re-queues the block only for wallets + /// whose last pass ran no later than the pass that derived the scripts. A + /// block stays in flight until every requested wallet has reported. #[test] fn track_state_machine() { let mut tracker = BlockMatchTracker::new(); @@ -197,7 +267,7 @@ mod tests { wallets: BTreeSet::from([wallet_a]) } ); - assert_eq!(tracker.finish_in_flight(&hash), Some((100, 0))); + assert_eq!(tracker.finish_in_flight(&hash, &BTreeSet::from([wallet_a])), Some((100, 0))); // Put it back in flight to continue the scenario. assert!(matches!( tracker.track(&key, 0, BTreeSet::from([wallet_a])), @@ -213,8 +283,8 @@ mod tests { ); // Block is delivered and processed for {A}. - assert!(tracker.finish_in_flight(&hash).is_some()); - tracker.record_processed(100, hash, &BTreeSet::from([wallet_a])); + assert!(tracker.finish_in_flight(&hash, &BTreeSet::from([wallet_a])).is_some()); + let pass_a = tracker.record_processed(100, hash, &BTreeSet::from([wallet_a])); // Late-added B's filter matches the same block: residual is {B} and // it gets re-queued via NewlyTracked. @@ -226,29 +296,38 @@ mod tests { ); // After B is processed, both wallets are covered: AlreadyProcessed. - assert!(tracker.finish_in_flight(&hash).is_some()); + assert!(tracker.finish_in_flight(&hash, &BTreeSet::from([wallet_b])).is_some()); tracker.record_processed(100, hash, &BTreeSet::from([wallet_b])); assert_eq!( tracker.track(&key, 5000, BTreeSet::from([wallet_a, wallet_b])), BlockTrackResult::AlreadyProcessed ); - // A match on newly derived scripts re-queues the fully-processed - // block anyway: the prior processing predates those scripts. + // Scripts derived by A's pass re-queue the block for A, since that + // pass derived them only after checking the block. B's pass ran later. + let derived_by_a = BTreeMap::from([(wallet_a, pass_a), (wallet_b, pass_a)]); assert_eq!( - tracker.track_for_new_scripts(&key, 5000, BTreeSet::from([wallet_a])), + tracker.track_for_new_scripts(&key, 5000, derived_by_a.clone()), BlockTrackResult::NewlyTracked { wallets: BTreeSet::from([wallet_a]) } ); - // While re-queued the block reports InFlight, preserving the - // one-increment-one-decrement pending accounting. + + // The re-queued pass ran after the derivation. + assert!(tracker.finish_in_flight(&hash, &BTreeSet::from([wallet_a])).is_some()); + tracker.record_processed(100, hash, &BTreeSet::from([wallet_a])); assert_eq!( - tracker.track_for_new_scripts(&key, 5000, BTreeSet::from([wallet_a])), - BlockTrackResult::InFlight { - wallets: BTreeSet::from([wallet_a]) - } + tracker.track_for_new_scripts(&key, 5000, derived_by_a), + BlockTrackResult::AlreadyProcessed ); + + // A block queued for A and B stays in flight until both have reported. + // A pass for A alone, such as a re-application, leaves it waiting for B. + let both = BTreeSet::from([wallet_a, wallet_b]); + let other = hash_n(1); + tracker.track(&FilterMatchKey::new(200, other), 0, both.clone()); + assert_eq!(tracker.finish_in_flight(&other, &BTreeSet::from([wallet_a])), None); + assert_eq!(tracker.finish_in_flight(&other, &both), Some((200, 0))); } /// `prune_at_or_below` drops every entry at or below the given height diff --git a/dash-spv/src/sync/filters/coinjoin_gap_discovery_tests.rs b/dash-spv/src/sync/filters/coinjoin_gap_discovery_tests.rs index 5e4e75598..8ce0cf0ec 100644 --- a/dash-spv/src/sync/filters/coinjoin_gap_discovery_tests.rs +++ b/dash-spv/src/sync/filters/coinjoin_gap_discovery_tests.rs @@ -20,6 +20,14 @@ //! 2. [`coinjoin_gap_limit_inversion_within_batch_recovers`] — a gap-window //! output in an EARLIER block of the SAME (still-active) batch is //! recovered by the commit-time rescan. GREEN since #820. +//! 3. [`coinjoin_rescan_skips_blocks_applied_after_the_scripts`] — the +//! rescan re-applies only blocks whose last pass predates the matching +//! scripts. +//! 4. [`provider_wallet_rescans_only_untested_outpoints`] — a rescan matches a +//! batch only against the UTXO outpoints it was never matched against. +//! 5. [`provider_wallet_rescan_prefers_an_untested_outpoint_over_an_older_script`] +//! — an untested outpoint re-applies a block even when an older script +//! group also matches it. //! //! Each test drives the manager exactly the way the production event loop //! does: `try_process_batch` → `BlocksNeeded` → (blocks-manager stand-in) @@ -34,6 +42,8 @@ use crate::storage::{ PersistentFilterStorage, StorageManager, }; use crate::test_utils::MockNetworkManager; +use dashcore::bip158::BlockFilterWriter; +use dashcore::consensus::encode::serialize; use dashcore::{ Address, Block, BlockHash, Network, OutPoint, Transaction, TxIn, TxOut, Txid, Witness, }; @@ -180,20 +190,34 @@ fn block_paying(height: u32, addresses: &[Address]) -> (Block, BlockFilter, Filt (block, filter, key) } +/// The filter of `block` with `elements` added to its output scripts, the way +/// Dash Core adds a `ProRegTx`'s collateral outpoint. +fn filter_with_elements(block: &Block, elements: &[Vec]) -> BlockFilter { + let mut content = Vec::new(); + let mut writer = BlockFilterWriter::new(&mut content, block); + writer.add_output_scripts(); + for element in elements { + writer.add_element(element); + } + writer.finish().expect("write test filter"); + BlockFilter::new(&content) +} + /// Drive the production event loop to quiescence: consume `BlocksNeeded` by /// processing the named blocks through the real wallet (the blocks-manager's /// job), feed the resulting `BlockProcessed` events (carrying the wallet's /// freshly derived scripts) back into the filters manager, and repeat until /// no more blocks are requested. Blocks are processed in height order, like -/// the real `BlocksPipeline`. +/// the real `BlocksPipeline`. Returns how often each block was applied. async fn drive_to_quiescence( manager: &mut CjFiltersManager, wallet: &Arc>>, blocks: &HashMap, initial_events: Vec, -) { +) -> HashMap { let network: Arc = Arc::new(MockNetworkManager::new()); + let mut passes: HashMap = HashMap::new(); let mut events = initial_events; for _round in 0..64 { let mut pending: BTreeMap<(u32, BlockHash), BTreeSet> = BTreeMap::new(); @@ -208,7 +232,7 @@ async fn drive_to_quiescence( } } if pending.is_empty() { - return; + return passes; } let mut next_events = Vec::new(); @@ -219,6 +243,7 @@ async fn drive_to_quiescence( .await .process_block_for_wallets(block, block_hash, height, &wallets) .await; + *passes.entry(block_hash).or_default() += 1; let confirmed_txids = result.relevant_txids().cloned().collect(); let event = SyncEvent::BlockProcessed { block_hash, @@ -311,3 +336,112 @@ async fn coinjoin_gap_limit_inversion_within_batch_recovers() { recovered by the commit-time rescan (PR #820). used_count={used_count}" ); } + +/// Block A pays index 5 and G+2 in one transaction. Its first pass only sees +/// index 5, and only then derives G+2, so A must be applied again. Block B +/// pays index 1 and G+3 and is applied after A derived G+3, so its first pass +/// already credited G+3 and the rescan must not apply it again. +#[tokio::test] +async fn coinjoin_rescan_skips_blocks_applied_after_the_scripts() { + let (mut manager, wallet, wallet_id) = setup().await; + let addresses = coinjoin_external_addresses(&wallet, &wallet_id, (G + 4) as u32).await; + + let (block_a, filter_a, key_a) = + block_paying(10, &[addresses[5].clone(), addresses[G + 2].clone()]); + let (block_b, filter_b, key_b) = + block_paying(20, &[addresses[1].clone(), addresses[G + 3].clone()]); + let (hash_a, hash_b) = (block_a.block_hash(), block_b.block_hash()); + let blocks: HashMap = HashMap::from([(hash_a, block_a), (hash_b, block_b)]); + + let mut batch = FiltersBatch::new(0, 99, HashMap::from([(key_a, filter_a), (key_b, filter_b)])); + batch.mark_verified(); + manager.active_batches.insert(0, batch); + manager.progress.update_stored_height(99); + + let initial_events = manager.try_process_batch().await.unwrap(); + let passes = drive_to_quiescence(&mut manager, &wallet, &blocks, initial_events).await; + + let (highest_used, _, used_count) = coinjoin_pool_state(&wallet, &wallet_id).await; + assert_eq!(highest_used, Some((G + 3) as u32)); + assert_eq!(used_count, 4, "indices 1, 5, G+2 and G+3 must all be credited"); + assert_eq!(passes.get(&hash_a), Some(&2), "A predates the scripts it derived"); + assert_eq!(passes.get(&hash_b), Some(&1), "B was applied after G+3 was derived"); +} + +/// Runs one batch of `blocks` at `batch_start` through scan, block passes and +/// commit, returning how often each block was applied. +async fn run_batch( + manager: &mut CjFiltersManager, + wallet: &Arc>>, + batch_start: u32, + blocks: Vec<(Block, BlockFilter, FilterMatchKey)>, +) -> HashMap { + let batch_end = batch_start + 99; + let filters = blocks.iter().map(|(_, filter, key)| (key.clone(), filter.clone())).collect(); + let mut batch = FiltersBatch::new(batch_start, batch_end, filters); + batch.mark_verified(); + manager.active_batches.insert(batch_start, batch); + manager.progress.update_stored_height(batch_end); + + let blocks = blocks.into_iter().map(|(block, _, _)| (block.block_hash(), block)).collect(); + let initial_events = manager.try_process_batch().await.unwrap(); + let passes = drive_to_quiescence(manager, wallet, &blocks, initial_events).await; + assert!(!manager.active_batches.contains_key(&batch_start), "the batch must commit"); + passes +} + +/// A provider wallet watches every UTXO outpoint, which a `ProRegTx` adds to +/// its block's filter as collateral. A rescan matches a batch only against the +/// outpoints the batch was never matched against, so a block is not applied +/// again for an outpoint its pass already knew. +#[tokio::test] +async fn provider_wallet_rescans_only_untested_outpoints() { + let (mut manager, wallet, wallet_id) = setup().await; + let addresses = coinjoin_external_addresses(&wallet, &wallet_id, 7).await; + + // Batch 0: A creates the outpoint the later batches refer to. + let (block_a, filter_a, key_a) = block_paying(10, &[addresses[5].clone()]); + let outpoint_a = serialize(&OutPoint::new(block_a.txdata[0].txid(), 0)); + run_batch(&mut manager, &wallet, 0, vec![(block_a, filter_a, key_a)]).await; + let elements = wallet.read().await.monitored_filter_elements_for(&wallet_id); + assert!(elements.contains(&outpoint_a), "a provider wallet watches its UTXO outpoints"); + + // Batch 1: B creates an outpoint without deriving an address, and E matches + // only that outpoint. Only the commit-time reconcile can find E. + let b = block_paying(110, &[addresses[2].clone()]); + let outpoint_b = serialize(&OutPoint::new(b.0.txdata[0].txid(), 0)); + let (block_e, _, key_e) = block_paying(130, &[]); + let filter_e = filter_with_elements(&block_e, &[outpoint_b]); + let hash_e = block_e.block_hash(); + let passes = run_batch(&mut manager, &wallet, 100, vec![b, (block_e, filter_e, key_e)]).await; + assert_eq!(passes.get(&hash_e), Some(&1), "E matches only the outpoint B created"); + + // Batch 2 is scanned with A's outpoint, so C's pass knows it. D derives an + // address, which rescans the batch before it commits. + let (block_c, _, key_c) = block_paying(210, &[addresses[1].clone()]); + let filter_c = filter_with_elements(&block_c, &[outpoint_a]); + let hash_c = block_c.block_hash(); + let d = block_paying(220, &[addresses[6].clone()]); + let passes = run_batch(&mut manager, &wallet, 200, vec![(block_c, filter_c, key_c), d]).await; + assert_eq!(passes.get(&hash_c), Some(&1), "C already knew A's outpoint"); +} + +/// Block X pays index G+3, which block A derived in an earlier pass, so X's +/// pass already credited it. X's filter also carries the outpoint that block Z +/// creates after X's pass. The rescan matches both, and the outpoint must win: +/// X is applied again, since no pass of X saw that outpoint. +#[tokio::test] +async fn provider_wallet_rescan_prefers_an_untested_outpoint_over_an_older_script() { + let (mut manager, wallet, wallet_id) = setup().await; + let addresses = coinjoin_external_addresses(&wallet, &wallet_id, (G + 4) as u32).await; + + let a = block_paying(10, &[addresses[5].clone()]); + let z = block_paying(30, &[addresses[2].clone()]); + let outpoint_z = serialize(&OutPoint::new(z.0.txdata[0].txid(), 0)); + let (block_x, _, key_x) = block_paying(20, &[addresses[1].clone(), addresses[G + 3].clone()]); + let filter_x = filter_with_elements(&block_x, &[outpoint_z]); + let hash_x = block_x.block_hash(); + + let passes = run_batch(&mut manager, &wallet, 0, vec![a, (block_x, filter_x, key_x), z]).await; + assert_eq!(passes.get(&hash_x), Some(&2), "X never saw the outpoint Z created"); +} diff --git a/dash-spv/src/sync/filters/manager.rs b/dash-spv/src/sync/filters/manager.rs index fce963511..1a2d56560 100644 --- a/dash-spv/src/sync/filters/manager.rs +++ b/dash-spv/src/sync/filters/manager.rs @@ -5,6 +5,7 @@ //! Emits FiltersStored, FiltersSyncComplete and BlocksNeeded events. use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::mem; use std::sync::Arc; use dashcore::bip158::{BlockFilter, FilterQuery}; @@ -22,6 +23,7 @@ use crate::validation::{FilterValidationInput, FilterValidator, Validator}; use crate::sync::progress::ProgressPercentage; use dashcore::hash_types::FilterHeader; +use hex_conservative::DisplayHex; use key_wallet_manager::WalletInterface; use key_wallet_manager::{check_compact_filters_shard, FilterMatchKey, WalletId}; use tokio::sync::RwLock; @@ -58,9 +60,70 @@ async fn match_filters( } matches } + +/// How a rescan matched a block for one wallet. +#[derive(Clone, Copy, Debug)] +enum RescanMatch { + /// Through the scripts derived by this pass, the newest that match. + Scripts(u64), + /// Through a filter element the batch had never been matched against. + Element, + /// The filter could not be decoded. + DecodeFailure, +} + +impl RescanMatch { + /// The pass the match is compared against. Without a known pass the block + /// is re-applied. + fn pass(self) -> u64 { + match self { + RescanMatch::Scripts(pass) => pass, + RescanMatch::Element | RescanMatch::DecodeFailure => UNKNOWN_PASS, + } + } +} + +/// Returns how `filter`, already matched by the rescan, matched the wallet's +/// untested `elements` or `by_pass`. An element wins over any script group, +/// because no earlier pass of the block can have seen it. +fn rescan_match( + filter: Option<&BlockFilter>, + key: &FilterMatchKey, + by_pass: &DerivedScripts, + elements: &[Vec], +) -> RescanMatch { + let Some(filter) = filter else { + return RescanMatch::DecodeFailure; + }; + if !elements.is_empty() { + let query: FilterQuery = elements.iter().map(Vec::as_slice).collect(); + match filter.match_any(key.hash(), &query) { + Ok(true) => return RescanMatch::Element, + Ok(false) => {} + Err(_) => return RescanMatch::DecodeFailure, + } + } + for (pass, scripts) in by_pass.iter().rev() { + let query: FilterQuery = scripts.iter().map(|script| script.as_bytes()).collect(); + match filter.match_any(key.hash(), &query) { + Ok(true) => return RescanMatch::Scripts(*pass), + Ok(false) => {} + Err(_) => return RescanMatch::DecodeFailure, + } + } + RescanMatch::Element +} + /// Batch size for processing filters. const BATCH_PROCESSING_SIZE: u32 = 5000; +/// One wallet's scripts to rescan, keyed by the block pass that derived them. +type DerivedScripts = BTreeMap>; + +/// Pass for scripts of unknown origin. No pass is later, so every block they +/// match is re-applied. +const UNKNOWN_PASS: u64 = u64::MAX; + /// Snapshot of a behind wallet's compact-filter query inputs for a batch scan. struct WalletScanState { /// The wallet these inputs belong to. @@ -120,8 +183,9 @@ pub struct FiltersManager< /// `BlockProcessed` and the per-wallet record of which wallets already /// have a given processed block applied. pub(super) tracker: BlockMatchTracker, - /// Scripts found by processed blocks since the last tick, rescanned together. - pub(super) pending_new_scripts: BTreeMap>, + /// Scripts found by processed blocks since the last rescan, rescanned + /// together. Grouped per wallet by the pass that derived them. + pub(super) pending_new_scripts: BTreeMap>>, } impl @@ -606,10 +670,13 @@ impl 0 { - // Reconciliation found blocks; converge before committing. + // The rescans found blocks; converge before committing. break; } } @@ -769,40 +836,42 @@ impl> = HashMap::new(); + let mut untested: HashMap = HashMap::new(); { let wallet = self.wallet.read().await; let Some(batch) = self.active_batches.get(&batch_start) else { return Ok(vec![]); }; + // Every wallet is passed on, even without untested scripts, because + // `rescan_batch` also matches the filter elements the batch never saw. for wallet_id in &wallets { let monitored = wallet.scan_script_pubkeys_for(wallet_id); let missing: HashSet = batch.untested(wallet_id, &monitored).cloned().collect(); + let by_pass = untested.entry(*wallet_id).or_default(); if !missing.is_empty() { - untested.insert(*wallet_id, missing); + by_pass.insert(UNKNOWN_PASS, missing); } } } - if untested.is_empty() { - return Ok(vec![]); - } - tracing::debug!( - "Reconcile batch {}: {} script(s) the wallet watches had never been matched here", - batch_start, - untested.values().map(HashSet::len).sum::(), - ); + let missing_scripts: usize = + untested.values().flat_map(BTreeMap::values).map(HashSet::len).sum(); + if missing_scripts > 0 { + tracing::debug!( + "Reconcile batch {}: {} script(s) the wallet watches had never been matched here", + batch_start, + missing_scripts, + ); + } // Same accounting as any other newly derived script. self.rescan_batch(batch_start, &untested).await } - /// Match freshly derived scripts against every already-scanned active batch now, so + /// Match `pending_new_scripts` against every already-scanned active batch now, so /// their blocks are requested in parallel instead of one batch at a time at commit. - pub(super) async fn rescan_active_batches( - &mut self, - new_scripts: &BTreeMap>, - ) -> SyncResult> { + pub(super) async fn rescan_active_batches(&mut self) -> SyncResult> { + let new_scripts = mem::take(&mut self.pending_new_scripts); let mut events = Vec::new(); let starts: Vec = self.active_batches.iter().filter(|(_, b)| b.scanned()).map(|(s, _)| *s).collect(); @@ -810,12 +879,19 @@ impl> = new_scripts - .iter() - .filter(|(id, _)| batch.scanned_wallets().contains_key(*id)) - .map(|(id, scripts)| (*id, batch.untested(id, scripts).cloned().collect())) - .filter(|(_, scripts): &(WalletId, HashSet)| !scripts.is_empty()) - .collect(); + let mut untested: HashMap = HashMap::new(); + for (id, by_pass) in &new_scripts { + if !batch.scanned_wallets().contains_key(id) { + continue; + } + for (pass, scripts) in by_pass { + let missing: HashSet = + batch.untested(id, scripts).cloned().collect(); + if !missing.is_empty() { + untested.entry(*id).or_default().insert(*pass, missing); + } + } + } events.extend(self.rescan_batch(batch_start, &untested).await?); } Ok(events) @@ -827,7 +903,7 @@ impl>, + new_scripts: &HashMap, ) -> SyncResult> { if new_scripts.is_empty() { return Ok(vec![]); @@ -837,6 +913,27 @@ impl = HashMap::new(); + let mut filter_elements: HashMap>> = HashMap::new(); + { + let wallet = self.wallet.read().await; + for id in new_scripts.keys() { + synced_heights.insert(*id, wallet.wallet_synced_height(id)); + let elements = wallet.monitored_filter_elements_for(id); + let untested: Vec> = batch.untested(id, &elements).cloned().collect(); + if !untested.is_empty() { + filter_elements.insert(*id, untested); + } + } + } + if filter_elements.is_empty() && new_scripts.values().all(BTreeMap::is_empty) { + return Ok(vec![]); + } + tracing::info!( "Rescan filters ({}-{}) for new scripts across {} wallets", batch.start_height(), @@ -847,8 +944,11 @@ impl = HashMap::new(); - let mut filter_elements: HashMap>> = HashMap::new(); - { - let wallet = self.wallet.read().await; - for id in new_scripts.keys() { - synced_heights.insert(*id, wallet.wallet_synced_height(id)); - filter_elements.insert(*id, wallet.monitored_filter_elements_for(id)); - } - } - - let mut block_to_wallets: BTreeMap> = BTreeMap::new(); - for (wallet_id, scripts) in new_scripts { + // A match through a newly tested filter element has no known pass, so + // its block is re-applied once per element and batch. + let mut block_to_wallets: BTreeMap> = + BTreeMap::new(); + for (wallet_id, by_pass) in new_scripts { let elements = filter_elements.remove(wallet_id).unwrap_or_default(); - if scripts.is_empty() && elements.is_empty() { + let scripts_vec: Vec = by_pass.values().flatten().cloned().collect(); + if scripts_vec.is_empty() && elements.is_empty() { continue; } - let scripts_vec: Vec = scripts.iter().cloned().collect(); let min_synced = synced_heights.get(wallet_id).copied().unwrap_or(0); - let matches = match_filters(&batch_filters, scripts_vec, elements, min_synced).await; + let matches = + match_filters(&batch_filters, scripts_vec, elements.clone(), min_synced).await; for key in matches { - block_to_wallets.entry(key).or_default().insert(*wallet_id); + let matched = rescan_match(batch_filters.get(&key), &key, by_pass, &elements); + block_to_wallets.entry(key).or_default().insert(*wallet_id, matched); } } @@ -895,8 +986,29 @@ impl { @@ -908,7 +1020,6 @@ impl { blocks_needed.insert(key, wallets); } - // Never returned by track_for_new_scripts. BlockTrackResult::AlreadyProcessed => {} } } @@ -993,7 +1104,8 @@ impl> = HashMap::new(); - new_scripts.insert(wallet_a, HashSet::from([address_a.script_pubkey()])); - new_scripts.insert(wallet_b, HashSet::from([address_b.script_pubkey()])); + let mut new_scripts: HashMap = HashMap::new(); + new_scripts.insert( + wallet_a, + DerivedScripts::from([(0, HashSet::from([address_a.script_pubkey()]))]), + ); + new_scripts.insert( + wallet_b, + DerivedScripts::from([(0, HashSet::from([address_b.script_pubkey()]))]), + ); let events = manager.rescan_batch(0, &new_scripts).await.unwrap(); @@ -2189,30 +2307,40 @@ mod tests { assert!(!attr_b.contains(&wallet_a)); } - /// A block that was already processed for a wallet is re-queued by - /// `rescan_batch` when newly derived scripts match it: the prior - /// processing predates those scripts, so the block must be re-applied - /// against the extended pools. Plain scans keep skipping processed - /// blocks. + /// `rescan_batch` re-queues an already processed block only when its last + /// pass ran no later than the pass that derived the newest scripts matching it. #[tokio::test] async fn test_rescan_batch_reprocesses_already_processed_block() { - let address = Address::dummy(Network::Regtest, 33); + let old_address = Address::dummy(Network::Regtest, 33); + let new_address = Address::dummy(Network::Regtest, 34); let mut manager = create_test_manager().await; manager.set_state(SyncState::Syncing); - let (key, filter) = filter_for_address(20, &address); - let mut filters: HashMap = HashMap::new(); - filters.insert(key.clone(), filter); + let (key_20, filter_20) = filter_for_address(20, &old_address); + let (key_30, filter_30) = filter_for_address(30, &old_address); + let (key_40, filter_40) = filter_for_address(40, &new_address); + let filters = HashMap::from([ + (key_20.clone(), filter_20), + (key_30.clone(), filter_30), + (key_40.clone(), filter_40), + ]); let mut batch = FiltersBatch::new(0, 99, filters); batch.mark_verified(); manager.active_batches.insert(0, batch); - // The block was downloaded and processed for the wallet before the - // rescan script existed. - manager.tracker.record_processed(20, *key.hash(), &BTreeSet::from([MOCK_WALLET_ID])); + let wallets = BTreeSet::from([MOCK_WALLET_ID]); + let pass_20 = manager.tracker.record_processed(20, *key_20.hash(), &wallets); + manager.tracker.record_processed(30, *key_30.hash(), &wallets); + let pass_40 = manager.tracker.record_processed(40, *key_40.hash(), &wallets); - let mut new_scripts: HashMap> = HashMap::new(); - new_scripts.insert(MOCK_WALLET_ID, HashSet::from([address.script_pubkey()])); + // Block 20's pass derived `old_address`, a pass after block 40's derived `new_address`. + let new_scripts = HashMap::from([( + MOCK_WALLET_ID, + DerivedScripts::from([ + (pass_20, HashSet::from([old_address.script_pubkey()])), + (pass_40 + 1, HashSet::from([new_address.script_pubkey()])), + ]), + )]); let events = manager.rescan_batch(0, &new_scripts).await.unwrap(); @@ -2224,12 +2352,17 @@ mod tests { } => Some(blocks), _ => None, }) - .expect("BlocksNeeded event for the re-queued block"); - assert!(blocks.get(&key).expect("re-queued block entry").contains(&MOCK_WALLET_ID)); - - // The re-queued block is accounted exactly once so the batch cannot + .expect("BlocksNeeded event for the re-queued blocks"); + // Block 20 ran the pass that derived `old_address`, block 40 ran before + // `new_address` existed. + assert!(blocks.get(&key_20).expect("block 20 re-queued").contains(&MOCK_WALLET_ID)); + assert!(blocks.get(&key_40).expect("block 40 re-queued").contains(&MOCK_WALLET_ID)); + // Block 30 ran after `old_address` was derived. + assert!(!blocks.contains_key(&key_30), "block 30 already saw `old_address`"); + + // Each re-queued block is accounted exactly once so the batch cannot // commit before its re-processing completes. - assert_eq!(manager.active_batches.get(&0).unwrap().pending_blocks(), 1); + assert_eq!(manager.active_batches.get(&0).unwrap().pending_blocks(), 2); } /// `rescan_batch` honours each wallet's own `synced_height`: a new @@ -2288,11 +2421,17 @@ mod tests { // wallet_high also "discovers" address_low's script to demonstrate // that even when a new script would match a low height, the per-wallet // synced_height filter prevents emitting it. - let mut new_scripts: HashMap> = HashMap::new(); - new_scripts.insert(wallet_low, HashSet::from([address_low.script_pubkey()])); + let mut new_scripts: HashMap = HashMap::new(); + new_scripts.insert( + wallet_low, + DerivedScripts::from([(0, HashSet::from([address_low.script_pubkey()]))]), + ); new_scripts.insert( wallet_high, - HashSet::from([address_low.script_pubkey(), address_high.script_pubkey()]), + DerivedScripts::from([( + 0, + HashSet::from([address_low.script_pubkey(), address_high.script_pubkey()]), + )]), ); let events = manager.rescan_batch(0, &new_scripts).await.unwrap(); @@ -2411,6 +2550,163 @@ mod tests { assert!(attribution.contains(&wallet_id)); } + /// A re-application that derives scripts reports them with its own pass. + /// A block applied after it already saw them, so the rescan before the + /// commit does not apply that block again. + #[tokio::test] + async fn test_scripts_derived_by_a_re_application_carry_its_pass() { + let storage = DiskStorageManager::with_temp_dir().await.unwrap(); + let watched = Address::dummy(Network::Regtest, 61); + let derived = Address::dummy(Network::Regtest, 62); + + // Applying block 30 re-applies the stored block 50, which derives + // `derived`. Block 60 pays both addresses and is applied afterwards. + let mut wallet = MockWallet::new(); + wallet.set_addresses(vec![watched.clone()]); + wallet.set_reapply_heights(30, BTreeSet::from([50])); + wallet.set_derived_addresses(50, vec![derived.clone()]); + let wallet = Arc::new(RwLock::new(wallet)); + + let block_30 = Block::dummy(30, vec![Transaction::dummy(&watched, 0..0, &[30])]); + let block_50 = Block::dummy(50, vec![Transaction::dummy(&watched, 0..0, &[50])]); + let block_60 = Block::dummy( + 60, + vec![ + Transaction::dummy(&watched, 0..0, &[60]), + Transaction::dummy(&derived, 0..0, &[61]), + ], + ); + let key_60 = FilterMatchKey::new(60, block_60.block_hash()); + let filters = HashMap::from([ + (FilterMatchKey::new(30, block_30.block_hash()), BlockFilter::dummy(&block_30)), + (key_60.clone(), BlockFilter::dummy(&block_60)), + ]); + { + let blocks = storage.blocks(); + let mut blocks = blocks.write().await; + for (height, block) in [(30, &block_30), (50, &block_50), (60, &block_60)] { + blocks.store_block(height, HashedBlock::from(block)).await.unwrap(); + } + } + + let mut filters_manager = FiltersManager::new( + Arc::clone(&wallet), + storage.block_headers(), + storage.filter_headers(), + storage.filters(), + ) + .await; + filters_manager.set_state(SyncState::Syncing); + let mut batch = FiltersBatch::new(0, 99, filters); + batch.mark_verified(); + filters_manager.active_batches.insert(0, batch); + filters_manager.progress.update_stored_height(99); + filters_manager.progress.update_filter_header_tip_height(99); + let mut blocks_manager = + BlocksManager::new(Arc::clone(&wallet), storage.block_headers(), storage.blocks()) + .await; + let network: Arc = Arc::new(MockNetworkManager::new()); + + let needed = filters_manager.try_process_batch().await.unwrap(); + let mut requested_60 = 0; + for event in &needed { + for processed in blocks_manager.handle_sync_event(event, &network).await.unwrap() { + for event in filters_manager.handle_sync_event(&processed, &network).await.unwrap() + { + if let SyncEvent::BlocksNeeded { + blocks, + } = &event + { + requested_60 += usize::from(blocks.contains_key(&key_60)); + } + } + } + } + + assert_eq!(requested_60, 0, "block 60 was applied after `derived` existed"); + assert!(filters_manager.active_batches.is_empty(), "the batch commits"); + let processed = wallet.read().await.processed_blocks(); + let heights: Vec = processed.lock().await.iter().map(|(_, height)| *height).collect(); + assert_eq!(heights, vec![30, 50, 60]); + } + + /// The per-tick rescan does not request an in-flight block again for a + /// wallet it is already queued for, since the queued pass sees the new + /// scripts. It still requests the block for a wallet not yet queued. + #[tokio::test] + async fn test_tick_rescan_requests_in_flight_block_only_for_new_wallets() { + let wallet_a: WalletId = [0x1A; 32]; + let wallet_b: WalletId = [0x1B; 32]; + let watched = Address::dummy(Network::Regtest, 51); + let derived = Address::dummy(Network::Regtest, 52); + + let mut multi = MultiMockWallet::new(); + multi.insert_wallet( + wallet_a, + MockWalletState { + addresses: vec![watched.clone()], + ..MockWalletState::default() + }, + ); + multi.insert_wallet(wallet_b, MockWalletState::default()); + let mut manager = create_multi_test_manager(Arc::new(RwLock::new(multi))).await; + manager.set_state(SyncState::Syncing); + manager.progress.update_stored_height(99); + manager.progress.update_filter_header_tip_height(99); + + // Block 40 pays the watched address and the one derived below. + let block = Block::dummy( + 40, + vec![ + Transaction::dummy(&watched, 0..0, &[40]), + Transaction::dummy(&derived, 0..0, &[41]), + ], + ); + let key_40 = FilterMatchKey::new(40, block.block_hash()); + let filter_40 = BlockFilter::dummy(&block); + let mut batch = FiltersBatch::new(0, 99, HashMap::from([(key_40.clone(), filter_40)])); + batch.mark_verified(); + manager.active_batches.insert(0, batch); + + let events = manager.scan_batch(0).await.unwrap(); + assert!(matches!( + events.as_slice(), + [SyncEvent::BlocksNeeded { blocks }] if blocks.get(&key_40) == Some(&BTreeSet::from([wallet_a])) + )); + + let network: Arc = Arc::new(MockNetworkManager::new()); + let requested_40 = |events: &[SyncEvent]| { + events.iter().find_map(|e| match e { + SyncEvent::BlocksNeeded { + blocks, + } => blocks.get(&key_40).cloned(), + _ => None, + }) + }; + let block_20 = Header::dummy(20).block_hash(); + let report = |wallet_id: WalletId| SyncEvent::BlockProcessed { + block_hash: block_20, + height: 20, + wallets: BTreeSet::from([wallet_id]), + new_scripts: BTreeMap::from([(wallet_id, vec![derived.script_pubkey()])]), + confirmed_txids: vec![], + }; + + // Block 20 derived the script for wallet A while block 40 was queued + // for A: nothing is requested. + manager.handle_sync_event(&report(wallet_a), &network).await.unwrap(); + let events = manager.tick(&network).await.unwrap(); + assert_eq!(requested_40(&events), None, "block 40 is already queued for wallet A"); + + // The same script for wallet B adds B to the queued pass. + manager.handle_sync_event(&report(wallet_b), &network).await.unwrap(); + let events = manager.tick(&network).await.unwrap(); + assert_eq!(requested_40(&events), Some(BTreeSet::from([wallet_b]))); + + // Block 40 is counted once, however often the rescan matched it. + assert_eq!(manager.active_batches.get(&0).unwrap().pending_blocks(), 1); + } + /// `scan_batch` `AlreadyProcessed` path: when every candidate wallet has /// already had this block processed, the block is skipped (no /// `BlocksNeeded`). @@ -2696,8 +2992,14 @@ mod tests { manager.tracker.track(&FilterMatchKey::new(5100, hash2), 5000, BTreeSet::from([wallet])); // Each block round-trips its (height, batch_start) on `finish_in_flight`. - assert_eq!(manager.tracker.finish_in_flight(&hash1), Some((100, 0))); - assert_eq!(manager.tracker.finish_in_flight(&hash2), Some((5100, 5000))); + assert_eq!( + manager.tracker.finish_in_flight(&hash1, &BTreeSet::from([wallet])), + Some((100, 0)) + ); + assert_eq!( + manager.tracker.finish_in_flight(&hash2, &BTreeSet::from([wallet])), + Some((5100, 5000)) + ); } #[tokio::test] @@ -2729,7 +3031,10 @@ mod tests { // Block is delivered and processed for {A}. Round-trip the (height, // batch_start) tuple while removing the in-flight entry, then record // the processing. - assert_eq!(manager.tracker.finish_in_flight(&hash), Some((100, 0))); + assert_eq!( + manager.tracker.finish_in_flight(&hash, &BTreeSet::from([wallet_a])), + Some((100, 0)) + ); manager.tracker.record_processed(100, hash, &BTreeSet::from([wallet_a])); // Late-added wallet B's filter matches the same block. A is already @@ -2742,7 +3047,10 @@ mod tests { wallets: BTreeSet::from([wallet_b]) } ); - assert_eq!(manager.tracker.finish_in_flight(&hash), Some((100, 5000))); + assert_eq!( + manager.tracker.finish_in_flight(&hash, &BTreeSet::from([wallet_b])), + Some((100, 5000)) + ); // After B is also processed, a third match including only A and B // returns AlreadyProcessed since both are covered. @@ -2751,7 +3059,7 @@ mod tests { manager.tracker.track(&key, 5000, BTreeSet::from([wallet_a, wallet_b])), BlockTrackResult::AlreadyProcessed ); - assert!(manager.tracker.finish_in_flight(&hash).is_none()); + assert!(manager.tracker.finish_in_flight(&hash, &BTreeSet::from([wallet_b])).is_none()); } #[tokio::test] @@ -2808,7 +3116,10 @@ mod tests { let wallet_id: WalletId = [5; 32]; let script = Address::dummy(Network::Testnet, 21).script_pubkey(); - let scripts = HashMap::from([(wallet_id, HashSet::from([script.clone()]))]); + let scripts = HashMap::from([( + wallet_id, + DerivedScripts::from([(0, HashSet::from([script.clone()]))]), + )]); let events = manager.rescan_batch(0, &scripts).await.unwrap(); assert!(events.is_empty(), "an empty batch cannot match anything"); diff --git a/dash-spv/src/sync/filters/sync_manager.rs b/dash-spv/src/sync/filters/sync_manager.rs index 235d6bcb0..2df026452 100644 --- a/dash-spv/src/sync/filters/sync_manager.rs +++ b/dash-spv/src/sync/filters/sync_manager.rs @@ -181,21 +181,23 @@ impl< // Record per-wallet processing so a future scan can give a // late-added wallet its own pass at this block via the // `tracker.track` residual. - self.tracker.record_processed(*height, *block_hash, wallets); + let pass = self.tracker.record_processed(*height, *block_hash, wallets); - // Rescanned on the next tick, together with whatever the blocks - // processed meanwhile find: once per block, every active batch is - // re-matched for each block. + // Rescanned on the next tick or before the next commit, together + // with whatever the blocks processed meanwhile find: once per block, + // every active batch is re-matched for each block. for (wallet_id, scripts) in new_scripts { self.pending_new_scripts .entry(*wallet_id) .or_default() + .entry(pass) + .or_default() .extend(scripts.iter().cloned()); } let mut events = Vec::new(); // Check if this block is part of our tracked blocks - if let Some((_, batch_start)) = self.tracker.finish_in_flight(block_hash) { + if let Some((_, batch_start)) = self.tracker.finish_in_flight(block_hash, wallets) { if let Some(batch) = self.active_batches.get_mut(&batch_start) { batch.decrement_pending_blocks(); tracing::debug!( @@ -271,8 +273,7 @@ impl< self.filter_pipeline.send_pending(network, &*header_storage).await?; drop(header_storage); - let new_scripts = std::mem::take(&mut self.pending_new_scripts); - let mut events = self.rescan_active_batches(&new_scripts).await?; + let mut events = self.rescan_active_batches().await?; // Store completed batches and do speculative matching events.extend(self.store_and_match_batches().await?); diff --git a/key-wallet-manager/src/test_utils/mock_wallet.rs b/key-wallet-manager/src/test_utils/mock_wallet.rs index 8d96358a6..93cafe34c 100644 --- a/key-wallet-manager/src/test_utils/mock_wallet.rs +++ b/key-wallet-manager/src/test_utils/mock_wallet.rs @@ -45,6 +45,8 @@ pub struct MockWallet { /// Monitor revision counter for staleness detection. monitor_revision: u64, reapply_heights: BTreeMap>, + /// Addresses the wallet derives the first time it processes each height. + derived_addresses: BTreeMap>, } impl Default for MockWallet { @@ -73,6 +75,7 @@ impl MockWallet { processed_instant_locks: Arc::new(Mutex::new(Vec::new())), monitor_revision: 0, reapply_heights: BTreeMap::new(), + derived_addresses: BTreeMap::new(), } } @@ -80,6 +83,12 @@ impl MockWallet { self.reapply_heights.insert(height, heights); } + /// Make processing `height` start watching `addresses` and report their + /// scripts as newly derived. + pub fn set_derived_addresses(&mut self, height: u32, addresses: Vec
) { + self.derived_addresses.insert(height, addresses); + } + /// Override the wallet id used for per-wallet API surfaces. pub fn set_wallet_id(&mut self, wallet_id: WalletId) { self.wallet_id = wallet_id; @@ -142,6 +151,21 @@ impl WalletInterface for MockWallet { if !wallets.contains(&self.wallet_id) { return BlockProcessingResult::default(); } + let derived: Vec
= self + .derived_addresses + .get(&height) + .into_iter() + .flatten() + .filter(|address| !self.addresses.contains(address)) + .cloned() + .collect(); + let mut new_scripts = BTreeMap::new(); + if !derived.is_empty() { + self.monitor_revision += 1; + new_scripts + .insert(self.wallet_id, derived.iter().map(Address::script_pubkey).collect()); + self.addresses.extend(derived); + } let mut processed = self.processed_blocks.lock().await; processed.push((block.block_hash(), height)); if height > self.last_processed_height { @@ -151,7 +175,7 @@ impl WalletInterface for MockWallet { BlockProcessingResult { new_txids: block.txdata.iter().map(|tx| tx.txid()).collect(), existing_txids: Vec::new(), - new_scripts: Default::default(), + new_scripts, reapply_heights: self .reapply_heights .get(&height)