From 4915b17cc27825f419f1beccaa59e4d8830a45a2 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Sat, 31 Oct 2026 00:43:37 +1000 Subject: [PATCH 1/9] refactor(dash-spv): inject the clock into masternode `tick` `tick` read the clock itself through `Instant::now()` and `elapsed()`, so a test could only reach one of its timeouts by rewinding the instants the manager stores. `tick` now passes `Instant::now()` to `tick_at`, which reads every clock it checks from that instant, as `InstantSendManager::tick_at` does. Behaviour is unchanged. --- dash-spv/src/sync/masternodes/sync_manager.rs | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/dash-spv/src/sync/masternodes/sync_manager.rs b/dash-spv/src/sync/masternodes/sync_manager.rs index 2ebe4e9b9..69ce6b86a 100644 --- a/dash-spv/src/sync/masternodes/sync_manager.rs +++ b/dash-spv/src/sync/masternodes/sync_manager.rs @@ -405,6 +405,22 @@ impl SyncManager for MasternodesManager { } async fn tick(&mut self, network: &Arc) -> SyncResult> { + self.tick_at(Instant::now(), network).await + } + + fn progress(&self) -> SyncManagerProgress { + SyncManagerProgress::Masternodes(self.progress.clone()) + } +} + +impl MasternodesManager { + /// `tick` with the current time injected, so tests can drive its clocks + /// with a chosen instant. The trait `tick` passes `Instant::now()`. + pub(super) async fn tick_at( + &mut self, + now: Instant, + network: &Arc, + ) -> SyncResult> { // Handle ticks for both Syncing (initial) and Synced (incremental updates) if !matches!(self.state(), SyncState::Syncing | SyncState::Synced) { return Ok(vec![]); @@ -444,7 +460,7 @@ impl SyncManager for MasternodesManager { // remaining timeout - that peer has already had its turn. if let Some(in_flight) = self.sync_state.qrinfo_in_flight { let timeout = qrinfo_timeout_for(self.sync_state.qrinfo_retry_count); - let timed_out = in_flight.wait_start.elapsed() > timeout; + let timed_out = now.saturating_duration_since(in_flight.wait_start) > timeout; if timed_out || in_flight.rejected { if self.sync_state.qrinfo_retry_count < MAX_RETRY_ATTEMPTS - 1 { if in_flight.rejected { @@ -502,23 +518,22 @@ impl SyncManager for MasternodesManager { if self.state() == SyncState::Syncing && self.sync_state.qrinfo_in_flight.is_none() && self.sync_state.mnlistdiff_pipeline.is_complete() - && self - .sync_state - .last_qrinfo_dispatch - .is_some_and(|dispatched| dispatched.elapsed() > QRINFO_STALL_WATCHDOG) + && self.sync_state.last_qrinfo_dispatch.is_some_and(|dispatched| { + now.saturating_duration_since(dispatched) > QRINFO_STALL_WATCHDOG + }) { tracing::warn!( stalled_for_secs = self .sync_state .last_qrinfo_dispatch - .map(|d| d.elapsed().as_secs()) + .map(|d| now.saturating_duration_since(d).as_secs()) .unwrap_or_default(), "Masternode sync stalled in Syncing with nothing in flight, re-dispatching QRInfo" ); // Re-stamp before dispatching so a dispatch that returns without // sending (no stored tip, tip at genesis) re-arms the watchdog for // another full interval instead of retrying on every 100ms tick. - self.sync_state.last_qrinfo_dispatch = Some(Instant::now()); + self.sync_state.last_qrinfo_dispatch = Some(now); self.sync_state.qrinfo_retry_count = 0; self.sync_state.clear_pending(); return self.send_qrinfo_for_tip(network).await; @@ -526,10 +541,6 @@ impl SyncManager for MasternodesManager { Ok(vec![]) } - - fn progress(&self) -> SyncManagerProgress { - SyncManagerProgress::Masternodes(self.progress.clone()) - } } #[cfg(test)] From 330af8bb1d12b99526a8e913dff94009af2441ca Mon Sep 17 00:00:00 2001 From: xdustinface Date: Sat, 31 Oct 2026 15:08:11 +1000 Subject: [PATCH 2/9] refactor(dash-spv): rename `request_answered` to `release_request` A sync manager also needs to stop the broker's tracking of a request it no longer wants, without any response arriving, as the masternode pipeline does when it gives up a diff. `request_answered` named only the response case. `release_request` and `Broker::release` cover both, and `MockNetworkManager` records released keys. --- dash-spv/src/network/broker.rs | 6 +++--- dash-spv/src/network/manager.rs | 6 +++--- dash-spv/src/network/mod.rs | 2 +- dash-spv/src/sync/block_headers/manager.rs | 2 +- dash-spv/src/sync/blocks/sync_manager.rs | 2 +- .../src/sync/filter_headers/sync_manager.rs | 2 +- dash-spv/src/sync/filters/sync_manager.rs | 2 +- dash-spv/src/sync/masternodes/sync_manager.rs | 2 +- dash-spv/src/test_utils/network.rs | 18 +++++++++--------- 9 files changed, 21 insertions(+), 21 deletions(-) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index c98a6afc6..c275c2fe1 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -70,7 +70,7 @@ struct OnWire { } /// The request queue plus the registry of every keyed request in play, queued or on -/// the wire. A key stays registered until answered, so re-declaring it is a no-op. +/// the wire. A key stays registered until released, so re-declaring it is a no-op. pub struct Broker { queue: MsgQueue, requests: Mutex>, @@ -110,7 +110,7 @@ impl Broker { self.queue.push(height, msg).await; } - pub async fn answered(&self, key: &RequestKey) { + pub async fn release(&self, key: &RequestKey) { if matches!(self.requests.lock().await.remove(key), Some(ReqState::OnWire(_))) { self.wake(); } @@ -625,7 +625,7 @@ mod tests { broker.enqueue(0, get_cfilters(0)).await; assert_eq!(broker.len(), 0); - broker.answered(&RequestKey::CFilters(0)).await; + broker.release(&RequestKey::CFilters(0)).await; assert_eq!(broker.state_of(&RequestKey::CFilters(0)).await, None); broker.enqueue(0, get_cfilters(0)).await; assert_eq!(broker.len(), 1); diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 3ead8f18f..a0cae9eb7 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -330,8 +330,8 @@ impl NetworkManager for PeerNetworkManager { } } - async fn request_answered(&self, key: RequestKey) { - self.state.broker.answered(&key).await; + async fn release_request(&self, key: RequestKey) { + self.state.broker.release(&key).await; } /// Broadcast a message to all connected peers @@ -1227,7 +1227,7 @@ fn spawn_pump( } let mt = MessageType::from_cmd(msg.cmd()); - // Early router wake on inbound traffic (`request_answered` notifies again for the + // Early router wake on inbound traffic (`release_request` notifies again for the // real capacity). `cfilter` is skipped: only the last of ~1000 frees anything. if mt != Some(MessageType::CFilter) { broker.wake(); diff --git a/dash-spv/src/network/mod.rs b/dash-spv/src/network/mod.rs index d9d0f22f7..cd3b7fbf0 100644 --- a/dash-spv/src/network/mod.rs +++ b/dash-spv/src/network/mod.rs @@ -116,7 +116,7 @@ pub trait NetworkManager: Send + Sync + 'static { async fn dispatch_local(&self, msg: NetworkMessage); /// Stops timeout/retry tracking of `key`. - async fn request_answered(&self, key: RequestKey); + async fn release_request(&self, key: RequestKey); async fn subscribe(&self, kinds: &[MessageType]) -> UnboundedReceiver; diff --git a/dash-spv/src/sync/block_headers/manager.rs b/dash-spv/src/sync/block_headers/manager.rs index 427a36019..d2da9ae9c 100644 --- a/dash-spv/src/sync/block_headers/manager.rs +++ b/dash-spv/src/sync/block_headers/manager.rs @@ -181,7 +181,7 @@ impl BlockHeadersManager { if matched.is_some() { if let Some(locator) = answered_locator { - network.request_answered(RequestKey::Headers(locator)).await; + network.release_request(RequestKey::Headers(locator)).await; } } else if !headers.is_empty() { if let Some(fork_height) = self.fork_height(headers).await? { diff --git a/dash-spv/src/sync/blocks/sync_manager.rs b/dash-spv/src/sync/blocks/sync_manager.rs index 701b7b0b0..29a65469a 100644 --- a/dash-spv/src/sync/blocks/sync_manager.rs +++ b/dash-spv/src/sync/blocks/sync_manager.rs @@ -86,7 +86,7 @@ impl SyncM return Ok(vec![]); } - network.request_answered(RequestKey::Block(*hashed_block.hash())).await; + network.release_request(RequestKey::Block(*hashed_block.hash())).await; // Look up height for storage let height = self diff --git a/dash-spv/src/sync/filter_headers/sync_manager.rs b/dash-spv/src/sync/filter_headers/sync_manager.rs index 8ded4b454..73d6c1b5c 100644 --- a/dash-spv/src/sync/filter_headers/sync_manager.rs +++ b/dash-spv/src/sync/filter_headers/sync_manager.rs @@ -55,7 +55,7 @@ impl SyncManager for FilterHeade return Ok(vec![]); }; - network.request_answered(RequestKey::CfHeaders(cfheaders.stop_hash)).await; + network.release_request(RequestKey::CfHeaders(cfheaders.stop_hash)).await; let mut events = Vec::new(); diff --git a/dash-spv/src/sync/filters/sync_manager.rs b/dash-spv/src/sync/filters/sync_manager.rs index f0e7e0630..235d6bcb0 100644 --- a/dash-spv/src/sync/filters/sync_manager.rs +++ b/dash-spv/src/sync/filters/sync_manager.rs @@ -142,7 +142,7 @@ impl< // One `getcfilters` is answered only once its whole batch has arrived, so // individual `cfilter`s are never answered on their own. if let Some(batch_start) = batch_completed { - network.request_answered(RequestKey::CFilters(batch_start)).await; + network.release_request(RequestKey::CFilters(batch_start)).await; } // No `send_pending` here: re-declaring per `cfilter` would re-scan every diff --git a/dash-spv/src/sync/masternodes/sync_manager.rs b/dash-spv/src/sync/masternodes/sync_manager.rs index 69ce6b86a..9125f4eb9 100644 --- a/dash-spv/src/sync/masternodes/sync_manager.rs +++ b/dash-spv/src/sync/masternodes/sync_manager.rs @@ -218,7 +218,7 @@ impl SyncManager for MasternodesManager { self.progress.add_diffs_processed(1); self.sync_state.mnlistdiff_pipeline.receive(diff); - network.request_answered(RequestKey::MnListDiff(diff.block_hash)).await; + network.release_request(RequestKey::MnListDiff(diff.block_hash)).await; if !work_block_lists.is_empty() { self.sync_state.mnlistdiff_pipeline.queue_requests(work_block_lists); self.sync_state.mnlistdiff_pipeline.send_pending(network).await?; diff --git a/dash-spv/src/test_utils/network.rs b/dash-spv/src/test_utils/network.rs index 582c60733..7aad0f878 100644 --- a/dash-spv/src/test_utils/network.rs +++ b/dash-spv/src/test_utils/network.rs @@ -28,7 +28,7 @@ pub struct MockNetworkManager { sent: Mutex>, sent_to: Mutex>, broadcasts: Mutex>, - answered: Mutex>, + released: Mutex>, /// Mirrors the broker's de-duplication, so a manager that re-declares a request /// it never correlated fails here instead of hanging against a real broker. in_play: Mutex>, @@ -51,7 +51,7 @@ impl MockNetworkManager { sent: Mutex::new(Vec::new()), sent_to: Mutex::new(Vec::new()), broadcasts: Mutex::new(Vec::new()), - answered: Mutex::new(Vec::new()), + released: Mutex::new(Vec::new()), in_play: Mutex::new(HashSet::new()), subscribers: Mutex::new(Vec::new()), events_tx: RwLock::new(events_tx), @@ -78,12 +78,12 @@ impl MockNetworkManager { self.broadcasts.lock().await.clone() } - pub async fn answered_keys(&self) -> Vec { - self.answered.lock().await.clone() + pub async fn released_keys(&self) -> Vec { + self.released.lock().await.clone() } /// Clears the recorders but not the in-play requests, which stay de-duplicated - /// until answered or [`Self::release_requests_in_play`]. + /// until released or [`Self::release_requests_in_play`]. pub async fn clear_sent(&self) { self.sent.lock().await.clear(); self.sent_to.lock().await.clear(); @@ -136,7 +136,7 @@ impl NetworkManager for MockNetworkManager { async fn stop(&self) {} async fn enqueue(&self, _height: u32, msg: NetworkMessage) { - // Same as the broker: a keyed request in play is dropped until answered; + // Same as the broker: a keyed request in play is dropped until released, and // keyless traffic is never de-duplicated. let keys = request_keys(&msg); if !keys.is_empty() { @@ -166,9 +166,9 @@ impl NetworkManager for MockNetworkManager { self.inject(test_socket_address(0), msg).await; } - async fn request_answered(&self, key: RequestKey) { + async fn release_request(&self, key: RequestKey) { self.in_play.lock().await.remove(&key); - self.answered.lock().await.push(key); + self.released.lock().await.push(key); } async fn subscribe(&self, kinds: &[MessageType]) -> UnboundedReceiver { @@ -235,7 +235,7 @@ mod tests { mock.send(get_cfilters(0)).await; assert_eq!(mock.sent_messages().await.len(), 1); - mock.request_answered(RequestKey::CFilters(0)).await; + mock.release_request(RequestKey::CFilters(0)).await; assert_eq!(mock.requests_in_play().await, 0); mock.send(get_cfilters(0)).await; From dfd1e22381c0ac3b1c670475b350b313050b6c95 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Sat, 31 Oct 2026 15:09:12 +1000 Subject: [PATCH 3/9] fix(dash-spv): drop a still-queued request when released `Broker::release` removed the registry entry but left a request still queued in `MsgQueue`, which `route` later sent untracked. That happens when a response arrives after a timeout re-queued its request, and when a caller releases a request it no longer wants. When the same key was declared again, as an `mnlistdiff` re-asked from another base is, `route` registered the stale message under the new request and its retries resent the stale one. Releasing a queued key now also drops its queued message, unless another of its keys is still in play, as a block of a multi-block `getdata` can be. Both new broker tests fail without the removal. --- dash-spv/src/network/broker.rs | 75 +++++++++++++++++++++++++++++++--- dash-spv/src/network/mod.rs | 3 +- 2 files changed, 71 insertions(+), 7 deletions(-) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index c275c2fe1..6d67b0bf2 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -110,9 +110,25 @@ impl Broker { self.queue.push(height, msg).await; } + /// Unregisters `key`. If its request was on the wire, wakes the router to send more. + /// If it was still queued, drops it unless another of its keys is still registered, + /// since sent later it would go out untracked or under a new request for the same key. pub async fn release(&self, key: &RequestKey) { - if matches!(self.requests.lock().await.remove(key), Some(ReqState::OnWire(_))) { - self.wake(); + let mut reqs = self.requests.lock().await; + match reqs.remove(key) { + Some(ReqState::OnWire(_)) => { + drop(reqs); + self.wake(); + } + Some(ReqState::Queued(_)) => { + self.queue + .remove(|msg| { + let keys = request_keys(msg); + keys.contains(key) && !keys.iter().any(|k| reqs.contains_key(k)) + }) + .await; + } + None => {} } } @@ -494,6 +510,17 @@ impl MsgQueue { out } + async fn remove(&self, matches: impl Fn(&NetworkMessage) -> bool) { + let mut removed = 0; + for class in DRAIN_PRIORITY { + let mut q = self.queue(class).lock().await; + let before = q.len(); + q.retain(|_, msg| !matches(msg)); + removed += before - q.len(); + } + self.len.fetch_sub(removed, Ordering::SeqCst); + } + async fn push(&self, height: u32, msg: NetworkMessage) { let key = (height, self.back_seq.fetch_add(1, Ordering::SeqCst)); self.queue(classify(&msg)).lock().await.insert(key, msg); @@ -549,6 +576,14 @@ mod tests { }) } + /// Requests the diff from block `base` to block 4. + fn get_mnlistdiff(base: u32) -> NetworkMessage { + NetworkMessage::GetMnListD(GetMnListDiff { + base_block_hash: BlockHash::dummy(base), + block_hash: BlockHash::dummy(4), + }) + } + fn block(n: u32) -> Inventory { Inventory::Block(BlockHash::dummy(n)) } @@ -573,10 +608,7 @@ mod tests { #[test_case(get_headers(1) => vec![RequestKey::Headers(BlockHash::dummy(1))]; "getheaders")] #[test_case(get_cfheaders(2) => vec![RequestKey::CfHeaders(BlockHash::dummy(2))]; "getcfheaders")] #[test_case(get_cfilters(100) => vec![RequestKey::CFilters(100)]; "getcfilters")] - #[test_case(NetworkMessage::GetMnListD(GetMnListDiff { - base_block_hash: BlockHash::dummy(3), - block_hash: BlockHash::dummy(4), - }) => vec![RequestKey::MnListDiff(BlockHash::dummy(4))]; "getmnlistd")] + #[test_case(get_mnlistdiff(3) => vec![RequestKey::MnListDiff(BlockHash::dummy(4))]; "getmnlistd")] #[test_case(NetworkMessage::GetData(vec![block(1), tx(7), block(2)]) => vec![RequestKey::Block(BlockHash::dummy(1)), RequestKey::Block(BlockHash::dummy(2))]; "getdata splits per block and ignores other inventory")] @@ -631,6 +663,37 @@ mod tests { assert_eq!(broker.len(), 1); } + #[test_case(get_mnlistdiff(3), vec![RequestKey::MnListDiff(BlockHash::dummy(4))], 0; "its only key")] + #[test_case(NetworkMessage::GetData(vec![block(1), block(2)]), vec![RequestKey::Block(BlockHash::dummy(1))], 1; "one of its keys")] + #[test_case(NetworkMessage::GetData(vec![block(1), block(2)]), vec![RequestKey::Block(BlockHash::dummy(1)), RequestKey::Block(BlockHash::dummy(2))], 0; "all of its keys")] + #[tokio::test] + async fn a_released_request_is_dropped_once_none_of_its_keys_is_in_play( + msg: NetworkMessage, + released: Vec, + queued: usize, + ) { + let broker = Broker::new(); + broker.enqueue(0, msg).await; + broker.enqueue(0, get_cfilters(0)).await; + for key in &released { + broker.release(key).await; + } + assert_eq!(broker.len(), queued + 1, "the other request stays queued"); + assert_eq!(broker.queue.pop_n(broker.len()).await.len(), queued + 1); + } + + #[tokio::test] + async fn a_request_released_and_declared_from_another_base_replaces_the_queued_one() { + let broker = Broker::new(); + broker.enqueue(0, get_mnlistdiff(3)).await; + broker.release(&RequestKey::MnListDiff(BlockHash::dummy(4))).await; + broker.enqueue(0, get_mnlistdiff(5)).await; + + let queued: Vec = + broker.queue.pop_n(broker.len()).await.into_iter().map(|(_, msg)| msg).collect(); + assert_eq!(queued, vec![get_mnlistdiff(5)]); + } + #[test_case(vec![get_cfilters(0), get_cfilters(0), get_cfilters(1000)], 2, 2; "a request in play is dropped")] #[test_case(vec![NetworkMessage::MemPool, NetworkMessage::MemPool], 2, 0; "untracked traffic is never deduplicated")] #[tokio::test] diff --git a/dash-spv/src/network/mod.rs b/dash-spv/src/network/mod.rs index cd3b7fbf0..7d9bb16ff 100644 --- a/dash-spv/src/network/mod.rs +++ b/dash-spv/src/network/mod.rs @@ -115,7 +115,8 @@ pub trait NetworkManager: Send + Sync + 'static { /// Inject a message into the local pump as if received from a peer. async fn dispatch_local(&self, msg: NetworkMessage); - /// Stops timeout/retry tracking of `key`. + /// Stops tracking `key` once it is answered or no longer wanted. A request not yet + /// sent is dropped. async fn release_request(&self, key: RequestKey); async fn subscribe(&self, kinds: &[MessageType]) -> UnboundedReceiver; From b2c794bf9841b589136e5570b8628c8e3fc2f9cf Mon Sep 17 00:00:00 2001 From: xdustinface Date: Sat, 31 Oct 2026 00:43:38 +1000 Subject: [PATCH 4/9] fix(dash-spv): track each wanted `mnlistdiff` until settled `clear_pending` dropped the wanted diffs without telling the broker. The broker keeps a request until it is answered, so it went on sending a dropped `GetMnListDiff` to new peers. Their answers were ignored as unexpected, so the request never settled, and each one that timed out kicked the peer that had answered it. A reconnect hits this: the diffs a disconnect kept are dropped when the sync restarts from a fresh QRInfo. The pipeline also stored only target to base. The handler worked out whether a response was a tip update by comparing its base with the engine's newest list, which holds only while a work-block list's base lies below the newest list, so a work-block list whose base equalled the newest list was applied as a tip update and moved the newest list. A response was matched by target alone, so a diff for a wanted target built from another base was applied as if it answered the request, although Core echoes the requested base. And a diff for a block the header chain could not place was left wanted, so the peer that answered it was kicked as stalled, again and again. The pipeline now keeps one `WantedDiff` per target with its base and `DiffKind`, set by the caller that queues it: `send_tip_mnlistdiff_update` queues a `TipUpdate`, the QRInfo and post-apply paths queue `WorkBlockList`s. `match_response` requires the requested base and returns the kind. `clear` releases every request it drops through `release_request`, and `queue_requests` releases a target queued again from another base, which also drops the old request if it is still queued, since the broker keys `RequestKey::MnListDiff` by target alone and would keep sending the old request, whose answers no longer match. `settle_diff` is now the one place a matched diff is applied or given up. It releases the diff through the pipeline's `release`, counts it as processed only when it was applied, and completes the pipeline once nothing is left. A tip update given up publishes nothing, a work-block list given up still lets its tip update complete. A diff the engine refuses for any reason, an unplaceable block included, is given up. `test_clear` and `test_on_disconnect_requeues_instead_of_clearing` check that dropped diffs are released, `test_queue_requests` checks the release on a base change, and `test_match_response` and `a_tip_update_completes_when_its_work_block_list_fails` check that a diff from another base answers nothing. --- dash-spv/src/sync/masternodes/manager.rs | 51 +++- dash-spv/src/sync/masternodes/pipeline.rs | 237 +++++++++++------- dash-spv/src/sync/masternodes/sync_manager.rs | 174 +++++++------ 3 files changed, 282 insertions(+), 180 deletions(-) diff --git a/dash-spv/src/sync/masternodes/manager.rs b/dash-spv/src/sync/masternodes/manager.rs index 8ac2c9372..73a6648cd 100644 --- a/dash-spv/src/sync/masternodes/manager.rs +++ b/dash-spv/src/sync/masternodes/manager.rs @@ -11,7 +11,7 @@ use crate::sml_engine::{MasternodeListEngine, QRInfoFeedResult}; use dashcore::sml::llmq_type::network::NetworkLLMQExt; use tokio::sync::RwLock; -use super::pipeline::MnListDiffPipeline; +use super::pipeline::{DiffKind, MnListDiffPipeline}; use crate::error::SyncResult; use crate::network::{NetworkManager, SendRequest}; use crate::storage::{BlockHeaderStorage, PersistentMasternodeStorage}; @@ -58,6 +58,13 @@ impl Default for PipelineMode { } } +/// How a wanted diff was settled. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum DiffOutcome { + Applied, + GivenUp, +} + /// In-flight QRInfo request: tip hash that was requested and when the request fired. /// Held inside `MasternodeSyncState::qrinfo_in_flight` while a QRInfo is outstanding. #[derive(Debug, Clone, Copy)] @@ -137,8 +144,8 @@ impl MasternodeSyncState { !self.mnlistdiff_pipeline.is_complete() || self.qrinfo_in_flight.is_some() } - pub(super) fn clear_pending(&mut self) { - self.mnlistdiff_pipeline.clear(); + pub(super) async fn clear_pending(&mut self, network: &Arc) { + self.mnlistdiff_pipeline.clear(network).await; self.qrinfo_in_flight = None; self.pipeline_mode = PipelineMode::default(); } @@ -424,14 +431,42 @@ impl MasternodesManager { } self.sync_state.pipeline_mode = PipelineMode::Incremental; - self.sync_state.mnlistdiff_pipeline.queue_requests(vec![(base_hash, new_tip_hash)]); + self.sync_state + .mnlistdiff_pipeline + .queue_requests(DiffKind::TipUpdate, vec![(base_hash, new_tip_hash)], network) + .await; self.sync_state.mnlistdiff_pipeline.send_pending(network).await?; Ok(vec![]) } - /// Dispatch pipeline completion based on the current `PipelineMode`. Called when - /// the mnlistdiff pipeline drains, from either the message handler or the tick - /// handler's timeout-cleanup path. + /// Settle a wanted diff that was applied or given up, then complete the + /// pipeline once nothing is left in it. + /// + /// A tip update given up leaves the engine unchanged, so it publishes + /// nothing. A work-block list given up leaves the tip it was fetched for + /// applied, so that pipeline still completes. + pub(super) async fn settle_diff( + &mut self, + target: BlockHash, + outcome: DiffOutcome, + network: &Arc, + ) -> SyncResult> { + let Some(kind) = self.sync_state.mnlistdiff_pipeline.release(target, network).await else { + return Ok(vec![]); + }; + match (outcome, kind) { + (DiffOutcome::Applied, _) => self.progress.add_diffs_processed(1), + (DiffOutcome::GivenUp, DiffKind::TipUpdate) => return Ok(vec![]), + (DiffOutcome::GivenUp, DiffKind::WorkBlockList) => {} + } + if !self.sync_state.mnlistdiff_pipeline.is_complete() { + return Ok(vec![]); + } + tracing::info!("All MnListDiff responses received"); + self.complete_pipeline(network).await + } + + /// Dispatch pipeline completion based on the current `PipelineMode`. /// /// After an `Incremental` pipeline finishes, re-evaluate `next_pipeline_mode` at /// the now-advanced tip and fire a catch-up QRInfo if the cycle gate now picks @@ -467,7 +502,7 @@ impl MasternodesManager { "Incremental complete, cycle gate now picks QRInfo, firing catch-up" ); self.sync_state.qrinfo_retry_count = 0; - self.sync_state.clear_pending(); + self.sync_state.clear_pending(network).await; match self.send_qrinfo_for_tip(network).await { Ok(extra) => events.extend(extra), Err(e) => tracing::warn!( diff --git a/dash-spv/src/sync/masternodes/pipeline.rs b/dash-spv/src/sync/masternodes/pipeline.rs index e11312c93..1ad254472 100644 --- a/dash-spv/src/sync/masternodes/pipeline.rs +++ b/dash-spv/src/sync/masternodes/pipeline.rs @@ -1,37 +1,70 @@ //! MnListDiff pipeline implementation. use std::collections::HashMap; +use std::mem; use std::sync::Arc; use crate::error::SyncResult; -use crate::network::{NetworkManager, SendRequest}; +use crate::network::{NetworkManager, RequestKey, SendRequest}; use dashcore::network::message::NetworkMessage; use dashcore::network::message_sml::{GetMnListDiff, MnListDiff}; use dashcore::BlockHash; +/// What a wanted diff is for, recorded when it is queued. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum DiffKind { + /// Moves the newest list to the header tip. + TipUpdate, + /// Fetches the list at a quorum's work block, needed to validate that quorum. + WorkBlockList, +} + +#[derive(Debug)] +struct WantedDiff { + base: BlockHash, + kind: DiffKind, +} + /// Pipeline for downloading MnListDiff messages for quorum validation. #[derive(Debug, Default)] pub(super) struct MnListDiffPipeline { - /// Wanted requests: target_hash -> base_hash. - base_hashes: HashMap, + /// Keyed by target, which is also the request's key at the broker. + wanted: HashMap, } impl MnListDiffPipeline { - /// Clear all state. - pub(super) fn clear(&mut self) { - self.base_hashes.clear(); + /// Drops every wanted diff and releases it from the broker. Otherwise the broker keeps + /// retrying it, and kicks peers as stalled when we ignore their answers. + pub(super) async fn clear(&mut self, network: &Arc) { + for target in mem::take(&mut self.wanted).into_keys() { + network.release_request(RequestKey::MnListDiff(target)).await; + } } - /// Queue MnListDiff requests. - /// - /// Each request is a (base_hash, target_hash) pair. - pub(super) fn queue_requests(&mut self, requests: Vec<(BlockHash, BlockHash)>) { - for (base_hash, target_hash) in requests { - self.base_hashes.insert(target_hash, base_hash); + /// Queues `(base, target)` requests of one kind. A target already wanted from a different + /// base is released from the broker. The broker keys requests by target only, so it would + /// keep sending the old request, whose answers no longer match. + pub(super) async fn queue_requests( + &mut self, + kind: DiffKind, + requests: Vec<(BlockHash, BlockHash)>, + network: &Arc, + ) { + for (base, target) in requests { + let replaced = self.wanted.insert( + target, + WantedDiff { + base, + kind, + }, + ); + if replaced.is_some_and(|replaced| replaced.base != base) { + network.release_request(RequestKey::MnListDiff(target)).await; + } } - if !self.base_hashes.is_empty() { - tracing::info!("Queued {} MnListDiff requests", self.base_hashes.len()); + if !self.wanted.is_empty() { + tracing::info!("Queued {} MnListDiff requests", self.wanted.len()); } } @@ -40,12 +73,8 @@ impl MnListDiffPipeline { &mut self, network: &Arc, ) -> SyncResult<()> { - if self.base_hashes.is_empty() { - return Ok(()); - } - let requests: Vec<(BlockHash, BlockHash)> = - self.base_hashes.iter().map(|(target, base)| (*base, *target)).collect(); + self.wanted.iter().map(|(target, wanted)| (wanted.base, *target)).collect(); for (base_block_hash, block_hash) in requests { network @@ -64,33 +93,31 @@ impl MnListDiffPipeline { Ok(()) } - /// Check if response matches a wanted request. - pub(super) fn match_response(&self, diff: &MnListDiff) -> bool { - self.base_hashes.contains_key(&diff.block_hash) + /// Returns the kind of the wanted request that `diff` answers, if any. Core echoes the + /// requested base, so a diff with a different base answers nothing. + pub(super) fn match_response(&self, diff: &MnListDiff) -> Option { + self.wanted + .get(&diff.block_hash) + .filter(|wanted| wanted.base == diff.base_block_hash) + .map(|wanted| wanted.kind) } - /// Receive a MnListDiff response. - /// - /// Returns true if the diff was expected, false if unexpected. - pub(super) fn receive(&mut self, diff: &MnListDiff) -> bool { - let target_hash = diff.block_hash; - - if self.base_hashes.remove(&target_hash).is_none() { - return false; - } - - tracing::debug!( - "Received MnListDiff for {} ({} remaining)", - target_hash, - self.base_hashes.len() - ); - - true + /// Stop wanting `target` and release it from the broker. Returns its kind, + /// or `None` when it was not wanted. + pub(super) async fn release( + &mut self, + target: BlockHash, + network: &Arc, + ) -> Option { + let wanted = self.wanted.remove(&target)?; + network.release_request(RequestKey::MnListDiff(target)).await; + tracing::debug!("Settled MnListDiff for {} ({} remaining)", target, self.wanted.len()); + Some(wanted.kind) } /// Check if pipeline has no pending work. pub(super) fn is_complete(&self) -> bool { - self.base_hashes.is_empty() + self.wanted.is_empty() } } @@ -101,6 +128,7 @@ mod tests { use dashcore_hashes::Hash; use super::*; + use crate::test_utils::MockNetworkManager; /// Create a minimal MnListDiff for testing. fn create_test_diff(base_hash: BlockHash, target_hash: BlockHash) -> MnListDiff { @@ -137,100 +165,129 @@ mod tests { } } + fn test_network() -> (Arc, Arc) { + let mock = Arc::new(MockNetworkManager::new()); + (mock.clone(), mock) + } + #[test] fn test_pipeline_new() { let pipeline = MnListDiffPipeline::default(); assert!(pipeline.is_complete()); } - #[test] - fn test_queue_requests() { + #[tokio::test] + async fn test_queue_requests() { let mut pipeline = MnListDiffPipeline::default(); + let (network, mock) = test_network(); let base1 = BlockHash::from_byte_array([0x01; 32]); let target1 = BlockHash::from_byte_array([0x02; 32]); let base2 = BlockHash::from_byte_array([0x03; 32]); let target2 = BlockHash::from_byte_array([0x04; 32]); - pipeline.queue_requests(vec![(base1, target1), (base2, target2)]); + pipeline + .queue_requests( + DiffKind::WorkBlockList, + vec![(base1, target1), (base2, target2)], + &network, + ) + .await; assert!(!pipeline.is_complete()); - assert_eq!(pipeline.base_hashes.len(), 2); - assert_eq!(pipeline.base_hashes.get(&target1), Some(&base1)); - assert_eq!(pipeline.base_hashes.get(&target2), Some(&base2)); + assert_eq!(pipeline.wanted.len(), 2); + assert_eq!(pipeline.wanted[&target1].base, base1); + assert_eq!(pipeline.wanted[&target2].base, base2); + + // Re-queued from the same base, a target stays at the broker. From a different base + // it is released, so the new request goes out. + pipeline.send_pending(&network).await.unwrap(); + mock.clear_sent().await; + pipeline.queue_requests(DiffKind::WorkBlockList, vec![(base1, target1)], &network).await; + pipeline.queue_requests(DiffKind::WorkBlockList, vec![(base1, target2)], &network).await; + assert_eq!(mock.released_keys().await, vec![RequestKey::MnListDiff(target2)]); + pipeline.send_pending(&network).await.unwrap(); + let sent: Vec<_> = mock + .sent_messages() + .await + .into_iter() + .filter_map(|msg| match msg { + NetworkMessage::GetMnListD(get) => Some((get.base_block_hash, get.block_hash)), + _ => None, + }) + .collect(); + assert_eq!(sent, vec![(base1, target2)]); } - #[test] - fn test_match_response() { + #[tokio::test] + async fn test_match_response() { let mut pipeline = MnListDiffPipeline::default(); + let (network, _mock) = test_network(); let base = BlockHash::from_byte_array([0x01; 32]); let target = BlockHash::from_byte_array([0x02; 32]); + let work_target = BlockHash::from_byte_array([0x03; 32]); - pipeline.queue_requests(vec![(base, target)]); + pipeline.queue_requests(DiffKind::TipUpdate, vec![(base, target)], &network).await; + pipeline.queue_requests(DiffKind::WorkBlockList, vec![(base, work_target)], &network).await; - let diff = create_test_diff(base, target); - assert!(pipeline.match_response(&diff)); + assert_eq!( + pipeline.match_response(&create_test_diff(base, target)), + Some(DiffKind::TipUpdate) + ); + assert_eq!( + pipeline.match_response(&create_test_diff(base, work_target)), + Some(DiffKind::WorkBlockList) + ); // Unknown hash should not match let unknown_diff = create_test_diff(base, BlockHash::from_byte_array([0xFF; 32])); - assert!(!pipeline.match_response(&unknown_diff)); - } - - #[test] - fn test_receive() { - let mut pipeline = MnListDiffPipeline::default(); + assert_eq!(pipeline.match_response(&unknown_diff), None); - let base = BlockHash::from_byte_array([0x01; 32]); - let target = BlockHash::from_byte_array([0x02; 32]); - - pipeline.queue_requests(vec![(base, target)]); - - let diff = create_test_diff(base, target); - assert!(pipeline.receive(&diff)); - assert!(pipeline.is_complete()); - assert!(pipeline.base_hashes.is_empty()); + // A wanted target with a different base does not match either, since it would build + // a different list. + let other_base = create_test_diff(BlockHash::from_byte_array([0xEE; 32]), target); + assert_eq!(pipeline.match_response(&other_base), None); } - #[test] - fn test_receive_unexpected() { - let mut pipeline = MnListDiffPipeline::default(); - - let diff = create_test_diff( - BlockHash::from_byte_array([0x01; 32]), - BlockHash::from_byte_array([0x02; 32]), - ); - - // Receiving unexpected diff should return false - assert!(!pipeline.receive(&diff)); - } - - #[test] - fn test_receive_duplicate() { + #[tokio::test] + async fn test_release() { let mut pipeline = MnListDiffPipeline::default(); + let (network, mock) = test_network(); let base = BlockHash::from_byte_array([0x01; 32]); let target = BlockHash::from_byte_array([0x02; 32]); - pipeline.queue_requests(vec![(base, target)]); - let diff = create_test_diff(base, target); + pipeline.queue_requests(DiffKind::TipUpdate, vec![(base, target)], &network).await; + pipeline.send_pending(&network).await.unwrap(); - assert!(pipeline.receive(&diff)); - assert!(!pipeline.receive(&diff)); + assert_eq!(pipeline.release(target, &network).await, Some(DiffKind::TipUpdate)); assert!(pipeline.is_complete()); + assert_eq!(mock.released_keys().await, vec![RequestKey::MnListDiff(target)]); + + // Releasing it again, or one never wanted, releases nothing. + assert_eq!(pipeline.release(target, &network).await, None); + assert_eq!(pipeline.release(base, &network).await, None); + assert_eq!(mock.released_keys().await.len(), 1); } - #[test] - fn test_clear() { + #[tokio::test] + async fn test_clear() { let mut pipeline = MnListDiffPipeline::default(); + let (network, mock) = test_network(); let base = BlockHash::from_byte_array([0x01; 32]); let target = BlockHash::from_byte_array([0x02; 32]); - pipeline.queue_requests(vec![(base, target)]); - pipeline.clear(); + pipeline.queue_requests(DiffKind::WorkBlockList, vec![(base, target)], &network).await; + pipeline.send_pending(&network).await.unwrap(); + pipeline.clear(&network).await; assert!(pipeline.is_complete()); - assert!(pipeline.base_hashes.is_empty()); + assert_eq!( + mock.released_keys().await, + vec![RequestKey::MnListDiff(target)], + "a dropped request must be released, or the broker retries it forever" + ); } } diff --git a/dash-spv/src/sync/masternodes/sync_manager.rs b/dash-spv/src/sync/masternodes/sync_manager.rs index 9125f4eb9..a384ac886 100644 --- a/dash-spv/src/sync/masternodes/sync_manager.rs +++ b/dash-spv/src/sync/masternodes/sync_manager.rs @@ -1,6 +1,7 @@ -use super::manager::PipelineMode; +use super::manager::{DiffOutcome, PipelineMode}; +use super::pipeline::DiffKind; use crate::error::SyncResult; -use crate::network::{InboundMessage, MessageType, NetworkManager, RequestKey}; +use crate::network::{InboundMessage, MessageType, NetworkManager}; use crate::storage::BlockHeaderStorage; use crate::sync::{ ManagerIdentifier, MasternodesManager, SyncEvent, SyncManager, SyncManagerProgress, SyncState, @@ -153,7 +154,10 @@ impl SyncManager for MasternodesManager { self.sync_state.last_processed_qrinfo_tip = Some(qr_info.mn_list_diff_tip.block_hash); self.sync_state.qrinfo_received(); - self.sync_state.mnlistdiff_pipeline.queue_requests(request_pairs); + self.sync_state + .mnlistdiff_pipeline + .queue_requests(DiffKind::WorkBlockList, request_pairs, network) + .await; self.sync_state.mnlistdiff_pipeline.send_pending(network).await?; self.progress.bump_last_activity(); @@ -165,42 +169,22 @@ impl SyncManager for MasternodesManager { } NetworkMessage::MnListDiff(diff) => { - // Check if this diff matches an in-flight request - if !self.sync_state.mnlistdiff_pipeline.match_response(diff) { + let Some(kind) = self.sync_state.mnlistdiff_pipeline.match_response(diff) else { tracing::debug!("Received unexpected MnListDiff for {}", diff.block_hash); return Ok(vec![]); - } + }; tracing::debug!("Processing MnListDiff message for {}", diff.block_hash); let mut engine = self.engine.write().await; - let Some(target_height) = engine.height_if_known(&diff.block_hash).await else { - drop(engine); - tracing::warn!( - "Height not found for MnListDiff block {}, leaving wanted for retry", - diff.block_hash - ); - // Not answered, so the broker's timeout/retry re-sends it. - self.sync_state.mnlistdiff_pipeline.send_pending(network).await?; - return Ok(vec![]); - }; - // Only a tip update builds on the newest list. The work-block - // lists it may ask for below build on older ones. - let extends_tip = engine - .latest_masternode_list() - .is_some_and(|list| list.block_hash == diff.base_block_hash); - let apply_ok = match engine.apply_diff((**diff).clone()).await { - Ok(_) => { - tracing::debug!("Applied MnListDiff at height {}", target_height); - true + let outcome = match engine.apply_diff((**diff).clone()).await { + Ok(()) => { + tracing::debug!("Applied MnListDiff for {}", diff.block_hash); + DiffOutcome::Applied } Err(e) => { - tracing::warn!( - "Failed to apply MnListDiff at height {}: {}", - target_height, - e - ); - false + tracing::warn!("Giving up MnListDiff for {}: {}", diff.block_hash, e); + DiffOutcome::GivenUp } }; // A tip update can bring a quorum whose work block lies below @@ -208,39 +192,24 @@ impl SyncManager for MasternodesManager { // first quorum after a start does. Fetch those lists now, as // the QRInfo path does, instead of leaving the quorum // `Skipped(MissedList)` until the next QRInfo, a rotation cycle - // later. The pipeline completes only once they are applied. - let work_block_lists = if apply_ok && extends_tip { - engine.missing_work_block_list_requests_for(diff).await - } else { - Vec::new() - }; + // later. The pipeline completes only once they are applied or + // given up. + let work_block_lists = + if outcome == DiffOutcome::Applied && kind == DiffKind::TipUpdate { + engine.missing_work_block_list_requests_for(diff).await + } else { + Vec::new() + }; drop(engine); - self.progress.add_diffs_processed(1); - self.sync_state.mnlistdiff_pipeline.receive(diff); - network.release_request(RequestKey::MnListDiff(diff.block_hash)).await; if !work_block_lists.is_empty() { - self.sync_state.mnlistdiff_pipeline.queue_requests(work_block_lists); + self.sync_state + .mnlistdiff_pipeline + .queue_requests(DiffKind::WorkBlockList, work_block_lists, network) + .await; self.sync_state.mnlistdiff_pipeline.send_pending(network).await?; } - - // Check if all responses received - if self.sync_state.mnlistdiff_pipeline.is_complete() { - // In `Incremental` mode, a failed tip update means the engine - // state is unchanged. Skip completion to avoid emitting a - // spurious `MasternodeStateUpdated` for stale state. The next - // `BlockHeadersStored` event will re-drive an incremental update. - // A failed work-block list leaves the tip it was fetched for - // applied, so that pipeline still completes. - if !apply_ok - && extends_tip - && matches!(self.sync_state.pipeline_mode, PipelineMode::Incremental) - { - return Ok(vec![]); - } - tracing::info!("All MnListDiff responses received"); - return self.complete_pipeline(network).await; - } + return self.settle_diff(diff.block_hash, outcome, network).await; } _ => {} @@ -301,7 +270,7 @@ impl SyncManager for MasternodesManager { self.progress.current_height() ); self.sync_state.qrinfo_retry_count = 0; - self.sync_state.clear_pending(); + self.sync_state.clear_pending(network).await; return self.send_qrinfo_for_tip(network).await; } PipelineMode::Incremental => { @@ -383,7 +352,7 @@ impl SyncManager for MasternodesManager { return Ok(vec![]); } self.sync_state.qrinfo_retry_count = 0; - self.sync_state.clear_pending(); + self.sync_state.clear_pending(network).await; return self.send_qrinfo_for_tip(network).await; } PipelineMode::Incremental => { @@ -396,7 +365,7 @@ impl SyncManager for MasternodesManager { self.progress.block_header_tip_height() ); self.sync_state.qrinfo_retry_count = 0; - self.sync_state.clear_pending(); + self.sync_state.clear_pending(network).await; return self.send_qrinfo_for_tip(network).await; } } @@ -440,7 +409,7 @@ impl MasternodesManager { } => { if self.sync_state.qrinfo_in_flight.is_none() { self.sync_state.qrinfo_retry_count = 0; - self.sync_state.clear_pending(); + self.sync_state.clear_pending(network).await; return self.send_qrinfo_for_tip(network).await; } } @@ -476,7 +445,7 @@ impl MasternodesManager { ); } self.sync_state.qrinfo_retry_count += 1; - self.sync_state.clear_pending(); + self.sync_state.clear_pending(network).await; return self.send_qrinfo_for_tip(network).await; } else { tracing::warn!( @@ -484,7 +453,7 @@ impl MasternodesManager { last_attempt_rejected = in_flight.rejected, "QRInfo failed on every attempt, giving up this round" ); - self.sync_state.clear_pending(); + self.sync_state.clear_pending(network).await; return self.complete_pipeline(network).await; } } @@ -535,7 +504,7 @@ impl MasternodesManager { // another full interval instead of retrying on every 100ms tick. self.sync_state.last_qrinfo_dispatch = Some(now); self.sync_state.qrinfo_retry_count = 0; - self.sync_state.clear_pending(); + self.sync_state.clear_pending(network).await; return self.send_qrinfo_for_tip(network).await; } @@ -546,10 +515,11 @@ impl MasternodesManager { #[cfg(test)] mod tests { use super::super::manager::{MasternodeSyncState, QRInfoInFlight}; + use super::super::pipeline::DiffKind; use super::{ qrinfo_timeout_for, MAX_RETRY_ATTEMPTS, QRINFO_STALL_WATCHDOG, QRINFO_TIMEOUT_SCHEDULE_SECS, }; - use crate::network::{InboundMessage, NetworkManager}; + use crate::network::{InboundMessage, NetworkManager, RequestKey}; use crate::sml_engine::test_support::quorum_entry; use crate::sml_engine::MasternodeListEngine; use crate::storage::{ @@ -941,41 +911,57 @@ mod tests { assert!(diff_requests(&mock).await.is_empty(), "nothing more to fetch"); } - /// A tip update the engine rejects leaves the state unchanged, so it + /// A tip update the engine refuses leaves the state unchanged, so it /// publishes nothing and asks for no work-block list. #[tokio::test] - async fn a_failed_tip_update_publishes_nothing() { - let (mut manager, engine, network, mock, tip_update, _hashes) = + async fn a_refused_tip_update_is_given_up() { + let (mut manager, engine, network, mock, tip_update, hashes) = manager_awaiting_a_tip_update_with_a_new_quorum().await; - let rejected = MnListDiff { + let target = hashes[TIP as usize + 1]; + let refused = MnListDiff { // A new quorum needs a ChainLock signature to be applied. quorums_chainlock_signatures: Vec::new(), ..tip_update }; - let events = deliver(&mut manager, &network, rejected).await; + let events = deliver(&mut manager, &network, refused).await; + assert!(events.is_empty(), "a given-up tip update publishes nothing, got {events:?}"); + assert_eq!(mock.released_keys().await, vec![RequestKey::MnListDiff(target)]); assert_eq!(engine.read().await.latest_masternode_list().unwrap().known_height, TIP); - assert!(events.is_empty(), "a rejected tip update must not complete, got {events:?}"); - assert!(diff_requests(&mock).await.is_empty(), "a rejected tip update asks for nothing"); + assert!(manager.sync_state.mnlistdiff_pipeline.is_complete()); + assert!(diff_requests(&mock).await.is_empty(), "a refused tip update asks for nothing"); } /// A work-block list that fails to apply leaves the tip update it was /// fetched for applied, so the update still completes. #[tokio::test] async fn a_tip_update_completes_when_its_work_block_list_fails() { - let (mut manager, _engine, network, mock, tip_update, hashes) = + let (mut manager, engine, network, mock, tip_update, hashes) = manager_awaiting_a_tip_update_with_a_new_quorum().await; deliver(&mut manager, &network, tip_update).await; - let (_, block_hash) = + let (base_block_hash, block_hash) = *diff_requests(&mock).await.first().expect("the work-block list is asked for"); - let unusable = MnListDiff { - // No list at this base, so the engine cannot apply it. + let from_another_base = MnListDiff { base_block_hash: hashes[1], block_hash, ..MnListDiff::dummy_between(0, 0) }; + let events = deliver(&mut manager, &network, from_another_base).await; + assert!(events.is_empty(), "a diff from another base answers nothing, got {events:?}"); + assert!( + !manager.sync_state.mnlistdiff_pipeline.is_complete(), + "the work-block list is still wanted" + ); + + // Remove the base list so the engine cannot apply the work-block list. + engine.write().await.masternode_lists.remove(&OLDER); + let unusable = MnListDiff { + base_block_hash, + block_hash, + ..MnListDiff::dummy_between(0, 0) + }; let events = deliver(&mut manager, &network, unusable).await; assert!( @@ -1085,7 +1071,7 @@ mod tests { let (mut manager, network, mock, _) = syncing_manager_awaiting_qrinfo(200).await; // Strand the manager the way a dispatch failure would. - manager.sync_state.clear_pending(); + manager.sync_state.clear_pending(&network).await; assert_eq!(manager.state(), SyncState::Syncing); assert!(manager.sync_state.qrinfo_in_flight.is_none()); assert!(manager.sync_state.mnlistdiff_pipeline.is_complete()); @@ -1121,11 +1107,15 @@ mod tests { /// `pipeline_mode`, forcing a full QRInfo re-run on every reconnect. #[tokio::test] async fn test_on_disconnect_requeues_instead_of_clearing() { - let (mut manager, network, _mock, tip_hash) = syncing_manager_awaiting_qrinfo(200).await; + let (mut manager, network, mock, tip_hash) = syncing_manager_awaiting_qrinfo(200).await; let base = BlockHash::from_slice(&[0x11; 32]).unwrap(); let target = BlockHash::from_slice(&[0x22; 32]).unwrap(); - manager.sync_state.mnlistdiff_pipeline.queue_requests(vec![(base, target)]); + manager + .sync_state + .mnlistdiff_pipeline + .queue_requests(DiffKind::WorkBlockList, vec![(base, target)], &network) + .await; manager.sync_state.mnlistdiff_pipeline.send_pending(&network).await.expect("send succeeds"); manager.on_disconnect(); @@ -1147,6 +1137,22 @@ mod tests { manager.sync_state.last_processed_qrinfo_tip, None, "the dedup guard must not reject the reconnect's response" ); + + // A reconnect restarts the sync from a fresh QRInfo and drops the diff. It must be + // released too, or the broker keeps retrying it and kicks every peer whose answer we + // ignore. + manager.stop_sync(); + manager + .handle_sync_event( + &SyncEvent::BlockHeaderSyncComplete { + tip_height: 200, + }, + &network, + ) + .await + .expect("the restart succeeds"); + assert!(manager.sync_state.mnlistdiff_pipeline.is_complete()); + assert_eq!(mock.requests_in_play().await, 0, "the dropped diff must be released"); } /// `tick` must reissue MnListDiff requests that a disconnect moved back to @@ -1159,7 +1165,11 @@ mod tests { let base = BlockHash::from_slice(&[0x11; 32]).unwrap(); let target = BlockHash::from_slice(&[0x22; 32]).unwrap(); - manager.sync_state.mnlistdiff_pipeline.queue_requests(vec![(base, target)]); + manager + .sync_state + .mnlistdiff_pipeline + .queue_requests(DiffKind::WorkBlockList, vec![(base, target)], &network) + .await; manager.sync_state.mnlistdiff_pipeline.send_pending(&network).await.expect("send succeeds"); mock.clear_sent().await; From c5a34805977e47c0aeae6321acfd54e432c7100d Mon Sep 17 00:00:00 2001 From: xdustinface Date: Sat, 31 Oct 2026 00:43:38 +1000 Subject: [PATCH 5/9] fix(dash-spv): give up `mnlistdiff`s after a minute The broker retries an unanswered request forever, across peers, and the masternode manager waited on it just as long. A diff no peer can answer, such as one for a block a reorg orphaned, kept the pipeline busy for good, which held back every later tip update and QRInfo. Each wanted diff now records when it was queued, and `tick_at` gives up those wanted longer than `MNLISTDIFF_DEADLINE` (60 s) through `settle_diff`, before it re-declares the rest. A given-up diff is released from the broker. The broker hides its attempts, so this deadline is the only clock. A full disconnect never runs it, because `tick` is idle until the sync restarts and the restart drops the old diffs. A tip update given up, at its deadline or refused, left the pipeline empty with the header tip still ahead, so `tick` asked for it again on every 100 ms pass. `given_up_tip` now holds it back until the header tip moves or `MNLISTDIFF_DEADLINE` has passed since it was given up, so a chain without new blocks, such as regtest or a devnet, still asks for it again at most once a minute. A disconnect clears it. `an_unanswered_diff_is_given_up_at_its_deadline` drives `tick_at` past the deadline of a work-block list and of a tip update, and `a_tip_update_given_up_is_not_asked_again_until_the_tip_moves` covers the guard. --- dash-spv/src/sync/masternodes/manager.rs | 43 ++++- dash-spv/src/sync/masternodes/pipeline.rs | 20 +++ dash-spv/src/sync/masternodes/sync_manager.rs | 155 ++++++++++++++++-- 3 files changed, 199 insertions(+), 19 deletions(-) diff --git a/dash-spv/src/sync/masternodes/manager.rs b/dash-spv/src/sync/masternodes/manager.rs index 73a6648cd..f07accf0f 100644 --- a/dash-spv/src/sync/masternodes/manager.rs +++ b/dash-spv/src/sync/masternodes/manager.rs @@ -11,7 +11,7 @@ use crate::sml_engine::{MasternodeListEngine, QRInfoFeedResult}; use dashcore::sml::llmq_type::network::NetworkLLMQExt; use tokio::sync::RwLock; -use super::pipeline::{DiffKind, MnListDiffPipeline}; +use super::pipeline::{DiffKind, MnListDiffPipeline, MNLISTDIFF_DEADLINE}; use crate::error::SyncResult; use crate::network::{NetworkManager, SendRequest}; use crate::storage::{BlockHeaderStorage, PersistentMasternodeStorage}; @@ -133,6 +133,10 @@ pub(super) struct MasternodeSyncState { /// without sending (no stored tip, tip at genesis) cannot turn it into a /// per-tick retry loop. pub(super) last_qrinfo_dispatch: Option, + /// The last tip update given up, and when. It is not asked for again until the header tip + /// moves or [`MNLISTDIFF_DEADLINE`] passes. Without the wait, `tick` would re-ask it on every + /// pass. Without the deadline, a chain with no new blocks would never retry it. + pub(super) given_up_tip: Option<(BlockHash, Instant)>, } impl MasternodeSyncState { @@ -426,7 +430,9 @@ impl MasternodesManager { return Ok(vec![]); }; - if base_hash == new_tip_hash { + if base_hash == new_tip_hash + || self.sync_state.given_up_tip.is_some_and(|(tip, _)| tip == new_tip_hash) + { return Ok(vec![]); } @@ -439,12 +445,12 @@ impl MasternodesManager { Ok(vec![]) } - /// Settle a wanted diff that was applied or given up, then complete the - /// pipeline once nothing is left in it. + /// Settles a wanted diff that was applied or given up, and completes the pipeline once it + /// is empty. /// - /// A tip update given up leaves the engine unchanged, so it publishes - /// nothing. A work-block list given up leaves the tip it was fetched for - /// applied, so that pipeline still completes. + /// A given-up tip update changed nothing, so it publishes nothing and waits before it is + /// asked for again, see [`MasternodeSyncState::given_up_tip`]. A given-up work-block list + /// still lets the tip update it was fetched for complete. pub(super) async fn settle_diff( &mut self, target: BlockHash, @@ -456,7 +462,10 @@ impl MasternodesManager { }; match (outcome, kind) { (DiffOutcome::Applied, _) => self.progress.add_diffs_processed(1), - (DiffOutcome::GivenUp, DiffKind::TipUpdate) => return Ok(vec![]), + (DiffOutcome::GivenUp, DiffKind::TipUpdate) => { + self.sync_state.given_up_tip = Some((target, Instant::now())); + return Ok(vec![]); + } (DiffOutcome::GivenUp, DiffKind::WorkBlockList) => {} } if !self.sync_state.mnlistdiff_pipeline.is_complete() { @@ -466,6 +475,24 @@ impl MasternodesManager { self.complete_pipeline(network).await } + /// Give up every diff wanted past [`MNLISTDIFF_DEADLINE`] at `now`. + pub(super) async fn give_up_expired_diffs( + &mut self, + now: Instant, + network: &Arc, + ) -> SyncResult> { + let mut events = Vec::new(); + for target in self.sync_state.mnlistdiff_pipeline.expired(now) { + tracing::warn!( + "No usable MnListDiff for {} within {}s, giving it up", + target, + MNLISTDIFF_DEADLINE.as_secs() + ); + events.extend(self.settle_diff(target, DiffOutcome::GivenUp, network).await?); + } + Ok(events) + } + /// Dispatch pipeline completion based on the current `PipelineMode`. /// /// After an `Incremental` pipeline finishes, re-evaluate `next_pipeline_mode` at diff --git a/dash-spv/src/sync/masternodes/pipeline.rs b/dash-spv/src/sync/masternodes/pipeline.rs index 1ad254472..41d3de5d3 100644 --- a/dash-spv/src/sync/masternodes/pipeline.rs +++ b/dash-spv/src/sync/masternodes/pipeline.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use std::mem; use std::sync::Arc; +use std::time::{Duration, Instant}; use crate::error::SyncResult; use crate::network::{NetworkManager, RequestKey, SendRequest}; @@ -10,6 +11,11 @@ use dashcore::network::message::NetworkMessage; use dashcore::network::message_sml::{GetMnListDiff, MnListDiff}; use dashcore::BlockHash; +/// How long a diff may stay wanted before it is given up. The broker retries a request +/// forever, so a diff no peer can answer would otherwise block every later tip update and +/// QRInfo. +pub(super) const MNLISTDIFF_DEADLINE: Duration = Duration::from_secs(60); + /// What a wanted diff is for, recorded when it is queued. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum DiffKind { @@ -23,6 +29,7 @@ pub(super) enum DiffKind { struct WantedDiff { base: BlockHash, kind: DiffKind, + queued_at: Instant, } /// Pipeline for downloading MnListDiff messages for quorum validation. @@ -50,12 +57,14 @@ impl MnListDiffPipeline { requests: Vec<(BlockHash, BlockHash)>, network: &Arc, ) { + let queued_at = Instant::now(); for (base, target) in requests { let replaced = self.wanted.insert( target, WantedDiff { base, kind, + queued_at, }, ); if replaced.is_some_and(|replaced| replaced.base != base) { @@ -115,6 +124,17 @@ impl MnListDiffPipeline { Some(wanted.kind) } + /// The targets wanted for longer than [`MNLISTDIFF_DEADLINE`] at `now`. + pub(super) fn expired(&self, now: Instant) -> Vec { + self.wanted + .iter() + .filter(|(_, wanted)| { + now.saturating_duration_since(wanted.queued_at) > MNLISTDIFF_DEADLINE + }) + .map(|(target, _)| *target) + .collect() + } + /// Check if pipeline has no pending work. pub(super) fn is_complete(&self) -> bool { self.wanted.is_empty() diff --git a/dash-spv/src/sync/masternodes/sync_manager.rs b/dash-spv/src/sync/masternodes/sync_manager.rs index a384ac886..f1de786e5 100644 --- a/dash-spv/src/sync/masternodes/sync_manager.rs +++ b/dash-spv/src/sync/masternodes/sync_manager.rs @@ -1,5 +1,5 @@ use super::manager::{DiffOutcome, PipelineMode}; -use super::pipeline::DiffKind; +use super::pipeline::{DiffKind, MNLISTDIFF_DEADLINE}; use crate::error::SyncResult; use crate::network::{InboundMessage, MessageType, NetworkManager}; use crate::storage::BlockHeaderStorage; @@ -78,6 +78,7 @@ impl SyncManager for MasternodesManager { // reject a response for a tip whose earlier attempt died with the old peer. self.sync_state.qrinfo_retry_count = 0; self.sync_state.last_processed_qrinfo_tip = None; + self.sync_state.given_up_tip = None; } async fn handle_message( @@ -383,8 +384,8 @@ impl SyncManager for MasternodesManager { } impl MasternodesManager { - /// `tick` with the current time injected, so tests can drive its clocks - /// with a chosen instant. The trait `tick` passes `Instant::now()`. + /// `tick` at a given time, so tests can pass a future instant to reach the diff deadline. + /// The trait `tick` passes `Instant::now()`. pub(super) async fn tick_at( &mut self, now: Instant, @@ -395,6 +396,14 @@ impl MasternodesManager { return Ok(vec![]); } + if self + .sync_state + .given_up_tip + .is_some_and(|(_, at)| now.saturating_duration_since(at) > MNLISTDIFF_DEADLINE) + { + self.sync_state.given_up_tip = None; + } + // If Synced with no pending requests, check whether new headers arrived // while the initial sync was in progress. BlockHeadersStored events that // landed during Syncing state updated block_header_tip_height but couldn't @@ -460,15 +469,12 @@ impl MasternodesManager { return Ok(vec![]); } - // Re-declare outstanding MnListDiff work; the broker owns pacing and retry. + // Re-declare outstanding MnListDiff work. The broker paces and retries it but never + // gives a request up, so diffs past their deadline are given up here first. if !self.sync_state.mnlistdiff_pipeline.is_complete() { + let events = self.give_up_expired_diffs(now, network).await?; self.sync_state.mnlistdiff_pipeline.send_pending(network).await?; - - // Check if complete after handling timeouts - if self.sync_state.mnlistdiff_pipeline.is_complete() { - tracing::info!("MnListDiff pipeline complete"); - return self.complete_pipeline(network).await; - } + return Ok(events); } // Stall watchdog. `Syncing` with no QRInfo in flight and an empty diff @@ -515,7 +521,7 @@ impl MasternodesManager { #[cfg(test)] mod tests { use super::super::manager::{MasternodeSyncState, QRInfoInFlight}; - use super::super::pipeline::DiffKind; + use super::super::pipeline::{DiffKind, MNLISTDIFF_DEADLINE}; use super::{ qrinfo_timeout_for, MAX_RETRY_ATTEMPTS, QRINFO_STALL_WATCHDOG, QRINFO_TIMEOUT_SCHEDULE_SECS, }; @@ -930,9 +936,51 @@ mod tests { assert_eq!(mock.released_keys().await, vec![RequestKey::MnListDiff(target)]); assert_eq!(engine.read().await.latest_masternode_list().unwrap().known_height, TIP); assert!(manager.sync_state.mnlistdiff_pipeline.is_complete()); + assert_eq!(manager.sync_state.given_up_tip.map(|(tip, _)| tip), Some(target)); assert!(diff_requests(&mock).await.is_empty(), "a refused tip update asks for nothing"); } + /// A tip update the engine cannot apply, here for a base list it no longer + /// holds, is given up. It is not asked for again on every tick, only once + /// the header tip moves. + #[tokio::test] + async fn a_tip_update_given_up_is_not_asked_again_until_the_tip_moves() { + let storage = DiskStorageManager::with_temp_dir().await.unwrap(); + let (mut manager, engine, network, mock, hashes) = + manager_after_a_tip_update_request(&storage, |hashes| { + vec![ + MasternodeList::empty(hashes[OLDER as usize], OLDER), + MasternodeList::empty(hashes[TIP as usize], TIP), + ] + }) + .await; + engine.write().await.masternode_lists.remove(&TIP); + let tip_update = MnListDiff { + base_block_hash: hashes[TIP as usize], + block_hash: hashes[TIP as usize + 1], + ..MnListDiff::dummy_between(0, 0) + }; + + let events = deliver(&mut manager, &network, tip_update).await; + + assert!(events.is_empty(), "a given-up tip update publishes nothing, got {events:?}"); + assert!(manager.sync_state.mnlistdiff_pipeline.is_complete()); + assert_eq!( + mock.released_keys().await, + vec![RequestKey::MnListDiff(hashes[TIP as usize + 1])] + ); + + manager.tick(&network).await.unwrap(); + assert!(diff_requests(&mock).await.is_empty(), "the given-up tip is not asked again"); + + let next = announce_tip(&mut manager, &network, TIP + 2).await; + assert_eq!( + diff_requests(&mock).await, + vec![(hashes[OLDER as usize], next)], + "a new tip is asked for from the newest list left" + ); + } + /// A work-block list that fails to apply leaves the tip update it was /// fetched for applied, so the update still completes. #[tokio::test] @@ -973,6 +1021,91 @@ mod tests { ); } + /// A diff no peer answers is given up at its deadline and released from the broker, which + /// would otherwise retry it forever. A given-up work-block list lets its tip update + /// complete. A given-up tip update publishes nothing. + #[tokio::test] + async fn an_unanswered_diff_is_given_up_at_its_deadline() { + let (mut manager, _engine, network, mock, tip_update, _hashes) = + manager_awaiting_a_tip_update_with_a_new_quorum().await; + let before_queued = Instant::now(); + deliver(&mut manager, &network, tip_update).await; + let after_queued = Instant::now(); + let (_, work_target) = + *diff_requests(&mock).await.first().expect("the work-block list is asked for"); + + let events = manager.tick_at(before_queued + MNLISTDIFF_DEADLINE, &network).await.unwrap(); + assert!(events.is_empty(), "not given up before its deadline, got {events:?}"); + assert!(!manager.sync_state.mnlistdiff_pipeline.is_complete()); + + let events = manager + .tick_at(after_queued + MNLISTDIFF_DEADLINE + Duration::from_secs(1), &network) + .await + .unwrap(); + assert!( + matches!( + events.as_slice(), + [SyncEvent::MasternodeStateUpdated { height, .. }] if *height == TIP + 1 + ), + "the tip update completes once its work-block list is given up, got {events:?}" + ); + assert!(mock.released_keys().await.contains(&RequestKey::MnListDiff(work_target))); + assert_eq!(manager.progress.diffs_processed(), 1, "a silent minute is not progress"); + + let storage = DiskStorageManager::with_temp_dir().await.unwrap(); + let (mut manager, _engine, network, mock, hashes) = + manager_after_a_tip_update_request(&storage, |hashes| { + vec![MasternodeList::empty(hashes[TIP as usize], TIP)] + }) + .await; + let after_queued = Instant::now(); + let target = hashes[TIP as usize + 1]; + + let events = manager + .tick_at(after_queued + MNLISTDIFF_DEADLINE + Duration::from_secs(1), &network) + .await + .unwrap(); + assert!(events.is_empty(), "a given-up tip update publishes nothing, got {events:?}"); + assert_eq!(mock.released_keys().await, vec![RequestKey::MnListDiff(target)]); + + // The header tip stays put, like on a chain with no new blocks, so the given-up tip + // update is asked for again only after its own deadline passes. + manager.tick(&network).await.unwrap(); + assert!(diff_requests(&mock).await.is_empty(), "not asked for again at once"); + manager + .tick_at(Instant::now() + MNLISTDIFF_DEADLINE + Duration::from_secs(1), &network) + .await + .unwrap(); + assert_eq!(diff_requests(&mock).await, vec![(hashes[TIP as usize], target)]); + } + + /// Stores the header at `height` and tells the manager it is the new + /// tip. Returns its hash. + async fn announce_tip( + manager: &mut MasternodesManager, + network: &Arc, + height: u32, + ) -> BlockHash { + let header = Header::dummy(height); + manager + .header_storage + .write() + .await + .store_headers(&[HashedBlockHeader::from(&header)]) + .await + .unwrap(); + manager + .handle_sync_event( + &SyncEvent::BlockHeadersStored { + tip_height: height, + }, + network, + ) + .await + .unwrap(); + header.block_hash() + } + /// A QRInfo the engine rejects must not release the request slot. /// /// `qrinfo_in_flight` is the only thing that arms `tick`'s retry ladder, so From 7b19e7f53c09dbe824b6551572ac3a40023b46c0 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Sat, 31 Oct 2026 00:43:38 +1000 Subject: [PATCH 6/9] fix(dash-spv): ban peers only for self-contradicting diffs Every diff the engine refused was given up on first sight, whoever was at fault. A peer that sent a diff contradicting itself was never held to account, and one such peer could cancel a tip update or a work-block list that the broker could have asked another peer for. `FeedError::blames_peer` now tells the faults apart in one exhaustive match. Only `InvalidCoinbaseProof` and `InvalidIndexInSignatureSet` prove the peer wrong whatever our state, since the diff then contradicts itself or the header that commits to it. Such a peer is banned, the diff stays wanted so the broker sends it to another peer, and the handler returns `MasternodeSyncFailed`, as the block and header managers do after their bans. Every other refusal can be a gap or bug of ours, or depend on our own state, so it is still given up without a ban. `CoinbaseCommitmentMismatch` checks the list built from our stored base, `IncompleteSignatureSet` fires on a devnet before V20 because its activation height is 0, `IncompleteMnListDiff` is what Core sends for a list from genesis to a block before any masternode registered, and `BaseBlockNotGenesis` and `BaseBlockHashMismatch` judge the base we requested against what we hold. Banning for those would ban every honest peer in turn, and with a single configured peer the ban would close the connection and loop through a full disconnect and restart. `a_work_block_list_that_contradicts_itself_bans_its_peer` follows the ban through to another peer's answer completing the tip update, `a_refused_tip_update_is_given_up_without_a_ban` checks that a missing ChainLock signature bans nobody, and `a_refused_diff_is_blamed_only_when_our_state_cannot_explain_it` pins the verdict for every refusal. `MockNetworkManager` now records the peers it is asked to ban. --- dash-spv/src/sml_engine.rs | 45 ++++++++ dash-spv/src/sync/masternodes/sync_manager.rs | 100 +++++++++++++++--- dash-spv/src/test_utils/network.rs | 10 +- 3 files changed, 137 insertions(+), 18 deletions(-) diff --git a/dash-spv/src/sml_engine.rs b/dash-spv/src/sml_engine.rs index f24e9a1a0..ec938f1c1 100644 --- a/dash-spv/src/sml_engine.rs +++ b/dash-spv/src/sml_engine.rs @@ -71,6 +71,34 @@ pub enum FeedError { Refused(#[from] E), } +impl FeedError { + /// Whether the refusal proves the peer sent a bad diff. Only a diff that contradicts itself + /// or its block header does. Any other refusal can come from our side, such as a missing or + /// stale base list, a bug, or network-specific activation heights. + pub(crate) fn blames_peer(&self) -> bool { + match self { + FeedError::Refused( + SmlError::InvalidCoinbaseProof(_) | SmlError::InvalidIndexInSignatureSet(_), + ) => true, + // `IncompleteMnListDiff` also fires on honest diffs: a list from genesis to a block + // before any masternode registered is empty, and Core sends it that way. + FeedError::HeaderStorage(_) + | FeedError::Refused( + SmlError::BlockHashLookupFailed(_) + | SmlError::MissingStartMasternodeList(_) + | SmlError::BaseBlockHashMismatch { + .. + } + | SmlError::BaseBlockNotGenesis(_) + | SmlError::CorruptedCodeExecution(_) + | SmlError::IncompleteMnListDiff + | SmlError::IncompleteSignatureSet + | SmlError::CoinbaseCommitmentMismatch(_), + ) => false, + } + } +} + impl From for FeedError { fn from(error: StorageError) -> Self { FeedError::HeaderStorage(error) @@ -2965,4 +2993,21 @@ mod tests { ); } } + + #[test_case(FeedError::HeaderStorage(StorageError::ReadFailed("unreadable".into())) => false; "header storage")] + #[test_case(SmlError::BlockHashLookupFailed(BlockHash::dummy(1)).into() => false; "block hash lookup failed")] + #[test_case(SmlError::MissingStartMasternodeList(BlockHash::dummy(1)).into() => false; "missing start list")] + #[test_case(SmlError::BaseBlockHashMismatch { expected: BlockHash::dummy(1), found: BlockHash::dummy(2) }.into() => false; "base block hash mismatch")] + #[test_case(SmlError::BaseBlockNotGenesis(BlockHash::dummy(1)).into() => false; "base block not genesis")] + #[test_case(SmlError::CorruptedCodeExecution("bug".into()).into() => false; "corrupted code execution")] + #[test_case(SmlError::InvalidCoinbaseProof(BlockHash::dummy(1)).into() => true; "invalid coinbase proof")] + #[test_case(SmlError::InvalidIndexInSignatureSet(3).into() => true; "invalid signature set index")] + #[test_case(SmlError::IncompleteMnListDiff.into() => false; "incomplete diff")] + #[test_case(SmlError::IncompleteSignatureSet.into() => false; "incomplete signature set")] + #[test_case(SmlError::CoinbaseCommitmentMismatch(BlockHash::dummy(1)).into() => false; "coinbase commitment mismatch")] + fn a_refused_diff_is_blamed_only_when_our_state_cannot_explain_it( + error: FeedError, + ) -> bool { + error.blames_peer() + } } diff --git a/dash-spv/src/sync/masternodes/sync_manager.rs b/dash-spv/src/sync/masternodes/sync_manager.rs index f1de786e5..4a222d72b 100644 --- a/dash-spv/src/sync/masternodes/sync_manager.rs +++ b/dash-spv/src/sync/masternodes/sync_manager.rs @@ -83,7 +83,7 @@ impl SyncManager for MasternodesManager { async fn handle_message( &mut self, - _peer: SocketAddr, + peer: SocketAddr, msg: InboundMessage, network: &Arc, ) -> SyncResult> { @@ -183,6 +183,18 @@ impl SyncManager for MasternodesManager { tracing::debug!("Applied MnListDiff for {}", diff.block_hash); DiffOutcome::Applied } + Err(e) if e.blames_peer() => { + drop(engine); + // Kept wanted. The ban closes the connection and the broker sends + // the request to another peer. + tracing::warn!( + "Banning {} for an MnListDiff the engine refused: {}", + peer, + e + ); + network.ban_peer(peer).await; + return Err(SyncError::MasternodeSyncFailed(e.to_string())); + } Err(e) => { tracing::warn!("Giving up MnListDiff for {}: {}", diff.block_hash, e); DiffOutcome::GivenUp @@ -549,6 +561,7 @@ mod tests { use dashcore::{BlockHash, Network, QuorumHash}; use dashcore_hashes::Hash; use std::collections::BTreeMap; + use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; use std::time::Instant; @@ -848,7 +861,15 @@ mod tests { network: &Arc, diff: MnListDiff, ) -> Vec { - let peer = "127.0.0.1:19999".parse().unwrap(); + deliver_from(manager, network, test_socket_address(1), diff).await + } + + async fn deliver_from( + manager: &mut MasternodesManager, + network: &Arc, + peer: SocketAddr, + diff: MnListDiff, + ) -> Vec { manager .handle_message(peer, NetworkMessage::MnListDiff(Box::new(diff)).into(), network) .await @@ -917,10 +938,11 @@ mod tests { assert!(diff_requests(&mock).await.is_empty(), "nothing more to fetch"); } - /// A tip update the engine refuses leaves the state unchanged, so it - /// publishes nothing and asks for no work-block list. + /// A refused tip update changes nothing, so it publishes nothing and asks for no work-block + /// list. A missing ChainLock signature can come from our own activation heights, so the + /// update is given up without banning the peer. #[tokio::test] - async fn a_refused_tip_update_is_given_up() { + async fn a_refused_tip_update_is_given_up_without_a_ban() { let (mut manager, engine, network, mock, tip_update, hashes) = manager_awaiting_a_tip_update_with_a_new_quorum().await; let target = hashes[TIP as usize + 1]; @@ -935,13 +957,62 @@ mod tests { assert!(events.is_empty(), "a given-up tip update publishes nothing, got {events:?}"); assert_eq!(mock.released_keys().await, vec![RequestKey::MnListDiff(target)]); assert_eq!(engine.read().await.latest_masternode_list().unwrap().known_height, TIP); - assert!(manager.sync_state.mnlistdiff_pipeline.is_complete()); - assert_eq!(manager.sync_state.given_up_tip.map(|(tip, _)| tip), Some(target)); assert!(diff_requests(&mock).await.is_empty(), "a refused tip update asks for nothing"); + assert!(mock.banned_peers().await.is_empty(), "the peer is not proven wrong"); } - /// A tip update the engine cannot apply, here for a base list it no longer - /// holds, is given up. It is not asked for again on every tick, only once + /// A work-block list with a ChainLock signature for a quorum it does not contain contradicts + /// itself, so its peer is banned. The list stays wanted, and another peer's answer completes + /// the tip update. + #[tokio::test] + async fn a_work_block_list_that_contradicts_itself_bans_its_peer() { + let (mut manager, _engine, network, mock, tip_update, _hashes) = + manager_awaiting_a_tip_update_with_a_new_quorum().await; + deliver(&mut manager, &network, tip_update).await; + let (base_block_hash, block_hash) = + *diff_requests(&mock).await.first().expect("the work-block list is asked for"); + let contradictory = MnListDiff { + base_block_hash, + block_hash, + quorums_chainlock_signatures: vec![QuorumCLSigObject { + signature: BLSSignature::from([2; 96]), + index_set: vec![0], + }], + ..MnListDiff::dummy_between(0, 0) + }; + + let result = manager + .handle_message( + test_socket_address(1), + NetworkMessage::MnListDiff(Box::new(contradictory)).into(), + &network, + ) + .await; + + assert!(matches!(result, Err(SyncError::MasternodeSyncFailed(_))), "got {result:?}"); + assert_eq!(mock.banned_peers().await, vec![test_socket_address(1)]); + assert!( + !mock.released_keys().await.contains(&RequestKey::MnListDiff(block_hash)), + "the work-block list must stay at the broker" + ); + + let honest = MnListDiff { + base_block_hash, + block_hash, + ..MnListDiff::dummy_between(0, 0) + }; + let events = deliver_from(&mut manager, &network, test_socket_address(2), honest).await; + assert!( + matches!( + events.as_slice(), + [SyncEvent::MasternodeStateUpdated { height, .. }] if *height == TIP + 1 + ), + "the honest answer completes the tip update, got {events:?}" + ); + } + + /// A tip update that fails on our side, here because the engine dropped its base list, is + /// given up without blaming the peer. It is not asked for again on every tick, only once /// the header tip moves. #[tokio::test] async fn a_tip_update_given_up_is_not_asked_again_until_the_tip_moves() { @@ -961,14 +1032,8 @@ mod tests { ..MnListDiff::dummy_between(0, 0) }; - let events = deliver(&mut manager, &network, tip_update).await; - - assert!(events.is_empty(), "a given-up tip update publishes nothing, got {events:?}"); - assert!(manager.sync_state.mnlistdiff_pipeline.is_complete()); - assert_eq!( - mock.released_keys().await, - vec![RequestKey::MnListDiff(hashes[TIP as usize + 1])] - ); + deliver(&mut manager, &network, tip_update).await; + assert!(mock.banned_peers().await.is_empty(), "a gap of ours is not the peer's fault"); manager.tick(&network).await.unwrap(); assert!(diff_requests(&mock).await.is_empty(), "the given-up tip is not asked again"); @@ -1050,6 +1115,7 @@ mod tests { "the tip update completes once its work-block list is given up, got {events:?}" ); assert!(mock.released_keys().await.contains(&RequestKey::MnListDiff(work_target))); + assert!(mock.banned_peers().await.is_empty(), "silence is not a refusal"); assert_eq!(manager.progress.diffs_processed(), 1, "a silent minute is not progress"); let storage = DiskStorageManager::with_temp_dir().await.unwrap(); diff --git a/dash-spv/src/test_utils/network.rs b/dash-spv/src/test_utils/network.rs index 7aad0f878..9281b65c0 100644 --- a/dash-spv/src/test_utils/network.rs +++ b/dash-spv/src/test_utils/network.rs @@ -29,6 +29,7 @@ pub struct MockNetworkManager { sent_to: Mutex>, broadcasts: Mutex>, released: Mutex>, + banned: Mutex>, /// Mirrors the broker's de-duplication, so a manager that re-declares a request /// it never correlated fails here instead of hanging against a real broker. in_play: Mutex>, @@ -52,6 +53,7 @@ impl MockNetworkManager { sent_to: Mutex::new(Vec::new()), broadcasts: Mutex::new(Vec::new()), released: Mutex::new(Vec::new()), + banned: Mutex::new(Vec::new()), in_play: Mutex::new(HashSet::new()), subscribers: Mutex::new(Vec::new()), events_tx: RwLock::new(events_tx), @@ -82,6 +84,10 @@ impl MockNetworkManager { self.released.lock().await.clone() } + pub async fn banned_peers(&self) -> Vec { + self.banned.lock().await.clone() + } + /// Clears the recorders but not the in-play requests, which stay de-duplicated /// until released or [`Self::release_requests_in_play`]. pub async fn clear_sent(&self) { @@ -193,7 +199,9 @@ impl NetworkManager for MockNetworkManager { self.connected.load(Ordering::SeqCst) } - async fn ban_peer(&self, _addr: SocketAddr) {} + async fn ban_peer(&self, addr: SocketAddr) { + self.banned.lock().await.push(addr); + } } #[cfg(test)] From 460bd075bc4137952b365c3c73ca67da44e3a23e Mon Sep 17 00:00:00 2001 From: xdustinface Date: Sat, 31 Oct 2026 00:43:38 +1000 Subject: [PATCH 7/9] fix(dash-spv): retry a given-up work-block list once A tip update only asks for the work-block lists of the quorums it brings, so a list given up at its deadline or refused by the engine left its quorum `Skipped(MissedList)` until the next QRInfo, a rotation cycle later. Given-up work-block lists are now remembered in one `WorkListRetry` map, and the next tip update applied also retries those a quorum of the newest list still needs, from the nearest lower list, through `readable_missing_work_block_list_requests`. Each is retried at most once, since the next QRInfo asks for every missing list anyway. The map is cleared when a QRInfo queues its lists and on disconnect. `a_given_up_work_block_list_is_retried_once` fails without the retry and when a list is retried a second time. --- dash-spv/src/sync/masternodes/manager.rs | 46 ++++++++++++- dash-spv/src/sync/masternodes/sync_manager.rs | 68 ++++++++++++++++++- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/dash-spv/src/sync/masternodes/manager.rs b/dash-spv/src/sync/masternodes/manager.rs index f07accf0f..d2f696058 100644 --- a/dash-spv/src/sync/masternodes/manager.rs +++ b/dash-spv/src/sync/masternodes/manager.rs @@ -19,7 +19,7 @@ use crate::sync::{MasternodesProgress, SyncEvent, SyncManager, SyncState}; use dashcore::network::message::NetworkMessage; use dashcore::network::message_qrinfo::{GetQRInfo, QRInfo}; use dashcore::BlockHash; -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; /// Single enum that serves two roles in the masternode-sync flow: /// @@ -65,6 +65,15 @@ pub(super) enum DiffOutcome { GivenUp, } +/// Whether the one retry of a given-up work-block list has gone out yet. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum WorkListRetry { + /// Retried with the next applied tip update, if a quorum still needs the list. + Pending, + /// Already retried. From here on only the next QRInfo fetches the list. + Sent, +} + /// In-flight QRInfo request: tip hash that was requested and when the request fired. /// Held inside `MasternodeSyncState::qrinfo_in_flight` while a QRInfo is outstanding. #[derive(Debug, Clone, Copy)] @@ -137,6 +146,8 @@ pub(super) struct MasternodeSyncState { /// moves or [`MNLISTDIFF_DEADLINE`] passes. Without the wait, `tick` would re-ask it on every /// pass. Without the deadline, a chain with no new blocks would never retry it. pub(super) given_up_tip: Option<(BlockHash, Instant)>, + /// Targets of the work-block lists given up since the last QRInfo. + pub(super) given_up_work_lists: HashMap, } impl MasternodeSyncState { @@ -154,6 +165,35 @@ impl MasternodeSyncState { self.pipeline_mode = PipelineMode::default(); } + /// Returns the `(base, target)` requests to retry for given-up work-block lists that a quorum + /// in the newest list still needs, and marks each as retried. Each base is the nearest lower + /// list. A list is retried once. + pub(super) async fn retry_given_up_work_lists( + &mut self, + engine: &MasternodeListEngine, + ) -> Vec<(BlockHash, BlockHash)> { + if !self.given_up_work_lists.values().any(|retry| *retry == WorkListRetry::Pending) { + return Vec::new(); + } + let requests: Vec<_> = engine + .readable_missing_work_block_list_requests() + .await + .into_iter() + .filter(|(_, target)| { + self.given_up_work_lists.get(target) == Some(&WorkListRetry::Pending) + }) + .collect(); + self.given_up_work_lists.retain(|_, retry| *retry == WorkListRetry::Sent); + for (_, target) in &requests { + self.given_up_work_lists.insert(*target, WorkListRetry::Sent); + } + requests + } + + pub(super) fn forget_given_up_work_lists(&mut self) { + self.given_up_work_lists.clear(); + } + /// Record that a QRInfo request was actually fired for `tip_height`. Bumps /// the per-cycle attempt counter and sets the per-tip gate so subsequent /// calls to [`MasternodesManager::next_pipeline_mode`] at the same tip @@ -466,7 +506,9 @@ impl MasternodesManager { self.sync_state.given_up_tip = Some((target, Instant::now())); return Ok(vec![]); } - (DiffOutcome::GivenUp, DiffKind::WorkBlockList) => {} + (DiffOutcome::GivenUp, DiffKind::WorkBlockList) => { + self.sync_state.given_up_work_lists.entry(target).or_insert(WorkListRetry::Pending); + } } if !self.sync_state.mnlistdiff_pipeline.is_complete() { return Ok(vec![]); diff --git a/dash-spv/src/sync/masternodes/sync_manager.rs b/dash-spv/src/sync/masternodes/sync_manager.rs index 4a222d72b..4a9320ca6 100644 --- a/dash-spv/src/sync/masternodes/sync_manager.rs +++ b/dash-spv/src/sync/masternodes/sync_manager.rs @@ -79,6 +79,7 @@ impl SyncManager for MasternodesManager { self.sync_state.qrinfo_retry_count = 0; self.sync_state.last_processed_qrinfo_tip = None; self.sync_state.given_up_tip = None; + self.sync_state.forget_given_up_work_lists(); } async fn handle_message( @@ -155,6 +156,7 @@ impl SyncManager for MasternodesManager { self.sync_state.last_processed_qrinfo_tip = Some(qr_info.mn_list_diff_tip.block_hash); self.sync_state.qrinfo_received(); + self.sync_state.forget_given_up_work_lists(); self.sync_state .mnlistdiff_pipeline .queue_requests(DiffKind::WorkBlockList, request_pairs, network) @@ -209,7 +211,9 @@ impl SyncManager for MasternodesManager { // given up. let work_block_lists = if outcome == DiffOutcome::Applied && kind == DiffKind::TipUpdate { - engine.missing_work_block_list_requests_for(diff).await + let mut lists = engine.missing_work_block_list_requests_for(diff).await; + lists.extend(self.sync_state.retry_given_up_work_lists(&engine).await); + lists } else { Vec::new() }; @@ -1172,6 +1176,68 @@ mod tests { header.block_hash() } + /// Stores the header at `height`, lets the manager request the tip update to it, and + /// delivers an empty one. Returns the diffs requested after it applies, and its events. + async fn apply_an_empty_tip_update_to( + manager: &mut MasternodesManager, + network: &Arc, + mock: &MockNetworkManager, + height: u32, + ) -> (Vec<(BlockHash, BlockHash)>, Vec) { + let tip = announce_tip(manager, network, height).await; + let [(base_block_hash, block_hash)] = diff_requests(mock).await[..] else { + panic!("the tip update to {height} must be asked for alone"); + }; + assert_eq!(block_hash, tip); + let tip_update = MnListDiff { + base_block_hash, + block_hash, + ..MnListDiff::dummy_between(0, 0) + }; + let events = deliver(manager, network, tip_update).await; + (diff_requests(mock).await, events) + } + + /// A given-up work-block list leaves its quorum unverified, and a tip update only fetches + /// lists for the quorums it brings. So the next applied tip update retries the given-up list, + /// but not lists that were never asked for. A list is retried once. Given up a second time, + /// it waits for the next QRInfo. + #[tokio::test] + async fn a_given_up_work_block_list_is_retried_once() { + let (mut manager, _engine, network, mock, tip_update, hashes) = + manager_awaiting_a_tip_update_with_a_new_quorum().await; + let work_list = + (hashes[OLDER as usize], hashes[(MINED - QUORUM_MEMBER_LIST_OFFSET) as usize]); + deliver(&mut manager, &network, tip_update).await; + assert_eq!(diff_requests(&mock).await, vec![work_list]); + let events = manager + .tick_at(Instant::now() + MNLISTDIFF_DEADLINE + Duration::from_secs(1), &network) + .await + .unwrap(); + assert_eq!(events.len(), 1, "the tip update completes once its list is given up"); + + let (requests, events) = + apply_an_empty_tip_update_to(&mut manager, &network, &mock, TIP + 2).await; + assert_eq!(requests, vec![work_list], "the given-up list is retried"); + assert!(events.is_empty(), "the tip update waits for the list, got {events:?}"); + let events = manager + .tick_at(Instant::now() + MNLISTDIFF_DEADLINE + Duration::from_secs(1), &network) + .await + .unwrap(); + assert_eq!(events.len(), 1, "the tip update completes once its list is given up"); + + let (requests, events) = + apply_an_empty_tip_update_to(&mut manager, &network, &mock, TIP + 3).await; + assert!(requests.is_empty(), "a list is retried once only, got {requests:?}"); + assert!( + matches!( + events.as_slice(), + [SyncEvent::MasternodeStateUpdated { height, .. }] if *height == TIP + 3 + ), + "got {events:?}" + ); + } + /// A QRInfo the engine rejects must not release the request slot. /// /// `qrinfo_in_flight` is the only thing that arms `tick`'s retry ladder, so From fa5bfcc62337c75900bc56566f37e3f785009123 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Mon, 2 Nov 2026 01:47:24 +1000 Subject: [PATCH 8/9] refactor(dash-spv): pass the tick time to diff deadlines `queue_requests`, `settle_diff` and `send_tip_mnlistdiff_update` take `now` instead of reading the clock themselves. A diff queued or given up from `tick_at` is stamped with the tick's time, so both deadlines follow the same clock the tests inject. Message handlers pass `Instant::now()`. --- dash-spv/src/sync/masternodes/manager.rs | 8 +-- dash-spv/src/sync/masternodes/pipeline.rs | 50 +++++++++++++++---- dash-spv/src/sync/masternodes/sync_manager.rs | 28 +++++------ 3 files changed, 57 insertions(+), 29 deletions(-) diff --git a/dash-spv/src/sync/masternodes/manager.rs b/dash-spv/src/sync/masternodes/manager.rs index d2f696058..3bc31f4d2 100644 --- a/dash-spv/src/sync/masternodes/manager.rs +++ b/dash-spv/src/sync/masternodes/manager.rs @@ -452,6 +452,7 @@ impl MasternodesManager { /// lightweight completion path when the response drains the pipeline. pub(super) async fn send_tip_mnlistdiff_update( &mut self, + now: Instant, network: &Arc, ) -> SyncResult> { let new_tip_hash = { @@ -479,7 +480,7 @@ impl MasternodesManager { self.sync_state.pipeline_mode = PipelineMode::Incremental; self.sync_state .mnlistdiff_pipeline - .queue_requests(DiffKind::TipUpdate, vec![(base_hash, new_tip_hash)], network) + .queue_requests(DiffKind::TipUpdate, vec![(base_hash, new_tip_hash)], now, network) .await; self.sync_state.mnlistdiff_pipeline.send_pending(network).await?; Ok(vec![]) @@ -495,6 +496,7 @@ impl MasternodesManager { &mut self, target: BlockHash, outcome: DiffOutcome, + now: Instant, network: &Arc, ) -> SyncResult> { let Some(kind) = self.sync_state.mnlistdiff_pipeline.release(target, network).await else { @@ -503,7 +505,7 @@ impl MasternodesManager { match (outcome, kind) { (DiffOutcome::Applied, _) => self.progress.add_diffs_processed(1), (DiffOutcome::GivenUp, DiffKind::TipUpdate) => { - self.sync_state.given_up_tip = Some((target, Instant::now())); + self.sync_state.given_up_tip = Some((target, now)); return Ok(vec![]); } (DiffOutcome::GivenUp, DiffKind::WorkBlockList) => { @@ -530,7 +532,7 @@ impl MasternodesManager { target, MNLISTDIFF_DEADLINE.as_secs() ); - events.extend(self.settle_diff(target, DiffOutcome::GivenUp, network).await?); + events.extend(self.settle_diff(target, DiffOutcome::GivenUp, now, network).await?); } Ok(events) } diff --git a/dash-spv/src/sync/masternodes/pipeline.rs b/dash-spv/src/sync/masternodes/pipeline.rs index 41d3de5d3..06e97fc35 100644 --- a/dash-spv/src/sync/masternodes/pipeline.rs +++ b/dash-spv/src/sync/masternodes/pipeline.rs @@ -48,23 +48,23 @@ impl MnListDiffPipeline { } } - /// Queues `(base, target)` requests of one kind. A target already wanted from a different - /// base is released from the broker. The broker keys requests by target only, so it would - /// keep sending the old request, whose answers no longer match. + /// Queues `(base, target)` requests of one kind, wanted from `now`. A target already wanted + /// from a different base is released from the broker. The broker keys requests by target + /// only, so it would keep sending the old request, whose answers no longer match. pub(super) async fn queue_requests( &mut self, kind: DiffKind, requests: Vec<(BlockHash, BlockHash)>, + now: Instant, network: &Arc, ) { - let queued_at = Instant::now(); for (base, target) in requests { let replaced = self.wanted.insert( target, WantedDiff { base, kind, - queued_at, + queued_at: now, }, ); if replaced.is_some_and(|replaced| replaced.base != base) { @@ -210,6 +210,7 @@ mod tests { .queue_requests( DiffKind::WorkBlockList, vec![(base1, target1), (base2, target2)], + Instant::now(), &network, ) .await; @@ -223,8 +224,22 @@ mod tests { // it is released, so the new request goes out. pipeline.send_pending(&network).await.unwrap(); mock.clear_sent().await; - pipeline.queue_requests(DiffKind::WorkBlockList, vec![(base1, target1)], &network).await; - pipeline.queue_requests(DiffKind::WorkBlockList, vec![(base1, target2)], &network).await; + pipeline + .queue_requests( + DiffKind::WorkBlockList, + vec![(base1, target1)], + Instant::now(), + &network, + ) + .await; + pipeline + .queue_requests( + DiffKind::WorkBlockList, + vec![(base1, target2)], + Instant::now(), + &network, + ) + .await; assert_eq!(mock.released_keys().await, vec![RequestKey::MnListDiff(target2)]); pipeline.send_pending(&network).await.unwrap(); let sent: Vec<_> = mock @@ -248,8 +263,17 @@ mod tests { let target = BlockHash::from_byte_array([0x02; 32]); let work_target = BlockHash::from_byte_array([0x03; 32]); - pipeline.queue_requests(DiffKind::TipUpdate, vec![(base, target)], &network).await; - pipeline.queue_requests(DiffKind::WorkBlockList, vec![(base, work_target)], &network).await; + pipeline + .queue_requests(DiffKind::TipUpdate, vec![(base, target)], Instant::now(), &network) + .await; + pipeline + .queue_requests( + DiffKind::WorkBlockList, + vec![(base, work_target)], + Instant::now(), + &network, + ) + .await; assert_eq!( pipeline.match_response(&create_test_diff(base, target)), @@ -278,7 +302,9 @@ mod tests { let base = BlockHash::from_byte_array([0x01; 32]); let target = BlockHash::from_byte_array([0x02; 32]); - pipeline.queue_requests(DiffKind::TipUpdate, vec![(base, target)], &network).await; + pipeline + .queue_requests(DiffKind::TipUpdate, vec![(base, target)], Instant::now(), &network) + .await; pipeline.send_pending(&network).await.unwrap(); assert_eq!(pipeline.release(target, &network).await, Some(DiffKind::TipUpdate)); @@ -299,7 +325,9 @@ mod tests { let base = BlockHash::from_byte_array([0x01; 32]); let target = BlockHash::from_byte_array([0x02; 32]); - pipeline.queue_requests(DiffKind::WorkBlockList, vec![(base, target)], &network).await; + pipeline + .queue_requests(DiffKind::WorkBlockList, vec![(base, target)], Instant::now(), &network) + .await; pipeline.send_pending(&network).await.unwrap(); pipeline.clear(&network).await; diff --git a/dash-spv/src/sync/masternodes/sync_manager.rs b/dash-spv/src/sync/masternodes/sync_manager.rs index 4a9320ca6..358f1673f 100644 --- a/dash-spv/src/sync/masternodes/sync_manager.rs +++ b/dash-spv/src/sync/masternodes/sync_manager.rs @@ -159,7 +159,7 @@ impl SyncManager for MasternodesManager { self.sync_state.forget_given_up_work_lists(); self.sync_state .mnlistdiff_pipeline - .queue_requests(DiffKind::WorkBlockList, request_pairs, network) + .queue_requests(DiffKind::WorkBlockList, request_pairs, Instant::now(), network) .await; self.sync_state.mnlistdiff_pipeline.send_pending(network).await?; @@ -219,14 +219,15 @@ impl SyncManager for MasternodesManager { }; drop(engine); + let now = Instant::now(); if !work_block_lists.is_empty() { self.sync_state .mnlistdiff_pipeline - .queue_requests(DiffKind::WorkBlockList, work_block_lists, network) + .queue_requests(DiffKind::WorkBlockList, work_block_lists, now, network) .await; self.sync_state.mnlistdiff_pipeline.send_pending(network).await?; } - return self.settle_diff(diff.block_hash, outcome, network).await; + return self.settle_diff(diff.block_hash, outcome, now, network).await; } _ => {} @@ -296,7 +297,7 @@ impl SyncManager for MasternodesManager { tip_height, self.progress.current_height() ); - return self.send_tip_mnlistdiff_update(network).await; + return self.send_tip_mnlistdiff_update(Instant::now(), network).await; } } } @@ -373,7 +374,7 @@ impl SyncManager for MasternodesManager { return self.send_qrinfo_for_tip(network).await; } PipelineMode::Incremental => { - return self.send_tip_mnlistdiff_update(network).await; + return self.send_tip_mnlistdiff_update(Instant::now(), network).await; } } } @@ -439,7 +440,7 @@ impl MasternodesManager { } } PipelineMode::Incremental => { - return self.send_tip_mnlistdiff_update(network).await; + return self.send_tip_mnlistdiff_update(now, network).await; } } } @@ -773,7 +774,7 @@ mod tests { manager.progress.update_block_header_tip_height(TIP + 1); let mock = Arc::new(MockNetworkManager::new()); let network: Arc = mock.clone(); - manager.send_tip_mnlistdiff_update(&network).await.unwrap(); + manager.send_tip_mnlistdiff_update(Instant::now(), &network).await.unwrap(); assert_eq!(diff_requests(&mock).await.len(), 1, "the tip update is sent"); (manager, engine, network, mock, hashes) } @@ -1128,13 +1129,10 @@ mod tests { vec![MasternodeList::empty(hashes[TIP as usize], TIP)] }) .await; - let after_queued = Instant::now(); + let given_up_at = Instant::now() + MNLISTDIFF_DEADLINE + Duration::from_secs(1); let target = hashes[TIP as usize + 1]; - let events = manager - .tick_at(after_queued + MNLISTDIFF_DEADLINE + Duration::from_secs(1), &network) - .await - .unwrap(); + let events = manager.tick_at(given_up_at, &network).await.unwrap(); assert!(events.is_empty(), "a given-up tip update publishes nothing, got {events:?}"); assert_eq!(mock.released_keys().await, vec![RequestKey::MnListDiff(target)]); @@ -1143,7 +1141,7 @@ mod tests { manager.tick(&network).await.unwrap(); assert!(diff_requests(&mock).await.is_empty(), "not asked for again at once"); manager - .tick_at(Instant::now() + MNLISTDIFF_DEADLINE + Duration::from_secs(1), &network) + .tick_at(given_up_at + MNLISTDIFF_DEADLINE + Duration::from_secs(1), &network) .await .unwrap(); assert_eq!(diff_requests(&mock).await, vec![(hashes[TIP as usize], target)]); @@ -1379,7 +1377,7 @@ mod tests { manager .sync_state .mnlistdiff_pipeline - .queue_requests(DiffKind::WorkBlockList, vec![(base, target)], &network) + .queue_requests(DiffKind::WorkBlockList, vec![(base, target)], Instant::now(), &network) .await; manager.sync_state.mnlistdiff_pipeline.send_pending(&network).await.expect("send succeeds"); @@ -1433,7 +1431,7 @@ mod tests { manager .sync_state .mnlistdiff_pipeline - .queue_requests(DiffKind::WorkBlockList, vec![(base, target)], &network) + .queue_requests(DiffKind::WorkBlockList, vec![(base, target)], Instant::now(), &network) .await; manager.sync_state.mnlistdiff_pipeline.send_pending(&network).await.expect("send succeeds"); mock.clear_sent().await; From 839aa46ba675c5ad64617b29e65359a0e0faa9a0 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Mon, 2 Nov 2026 01:48:44 +1000 Subject: [PATCH 9/9] fix(dash-spv): keep the deadline of a re-queued `mnlistdiff` A target queued again from the same base replaced its entry, which reset `queued_at` and pushed back its 60s deadline. It now keeps its entry, so only a request from a different base starts a new deadline. --- dash-spv/src/sync/masternodes/pipeline.rs | 36 +++++++++++++---------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/dash-spv/src/sync/masternodes/pipeline.rs b/dash-spv/src/sync/masternodes/pipeline.rs index 06e97fc35..20f8733fa 100644 --- a/dash-spv/src/sync/masternodes/pipeline.rs +++ b/dash-spv/src/sync/masternodes/pipeline.rs @@ -49,8 +49,10 @@ impl MnListDiffPipeline { } /// Queues `(base, target)` requests of one kind, wanted from `now`. A target already wanted - /// from a different base is released from the broker. The broker keys requests by target - /// only, so it would keep sending the old request, whose answers no longer match. + /// from the same base keeps its entry, so queuing it again does not push back its deadline. + /// A target already wanted from a different base is released from the broker. The broker + /// keys requests by target only, so it would keep sending the old request, whose answers no + /// longer match. pub(super) async fn queue_requests( &mut self, kind: DiffKind, @@ -59,6 +61,9 @@ impl MnListDiffPipeline { network: &Arc, ) { for (base, target) in requests { + if self.wanted.get(&target).is_some_and(|wanted| wanted.base == base) { + continue; + } let replaced = self.wanted.insert( target, WantedDiff { @@ -67,7 +72,7 @@ impl MnListDiffPipeline { queued_at: now, }, ); - if replaced.is_some_and(|replaced| replaced.base != base) { + if replaced.is_some() { network.release_request(RequestKey::MnListDiff(target)).await; } } @@ -206,11 +211,12 @@ mod tests { let base2 = BlockHash::from_byte_array([0x03; 32]); let target2 = BlockHash::from_byte_array([0x04; 32]); + let queued_at = Instant::now(); pipeline .queue_requests( DiffKind::WorkBlockList, vec![(base1, target1), (base2, target2)], - Instant::now(), + queued_at, &network, ) .await; @@ -220,27 +226,25 @@ mod tests { assert_eq!(pipeline.wanted[&target1].base, base1); assert_eq!(pipeline.wanted[&target2].base, base2); - // Re-queued from the same base, a target stays at the broker. From a different base - // it is released, so the new request goes out. + // A target queued again from the same base stays at the broker and keeps its deadline. + // A target queued again from a different base is released, so the new request goes out + // with a new deadline. pipeline.send_pending(&network).await.unwrap(); mock.clear_sent().await; + let queued_again_at = queued_at + MNLISTDIFF_DEADLINE / 2; pipeline .queue_requests( DiffKind::WorkBlockList, - vec![(base1, target1)], - Instant::now(), - &network, - ) - .await; - pipeline - .queue_requests( - DiffKind::WorkBlockList, - vec![(base1, target2)], - Instant::now(), + vec![(base1, target1), (base1, target2)], + queued_again_at, &network, ) .await; assert_eq!(mock.released_keys().await, vec![RequestKey::MnListDiff(target2)]); + assert_eq!( + pipeline.expired(queued_at + MNLISTDIFF_DEADLINE + Duration::from_secs(1)), + vec![target1] + ); pipeline.send_pending(&network).await.unwrap(); let sent: Vec<_> = mock .sent_messages()