From 81acc285d04c992d429dd2a9f5311679b793699d Mon Sep 17 00:00:00 2001 From: xdustinface Date: Fri, 6 Nov 2026 00:44:49 +1000 Subject: [PATCH 1/4] fix(dash-spv): keep re-queued work off kicked peers The timeout monitor re-queued a culprit's requests and woke the router before it removed the culprit from the connected set, so the router could send the freed work straight back to the peer being closed. Remove and close the culprits first, then re-queue. The router also works from a snapshot of the set that can predate the kick, so `Broker::route` now skips a peer whose connection is already closed. --- dash-spv/src/network/broker.rs | 19 ++++++++++++++++++- dash-spv/src/network/manager.rs | 10 +++++----- dash-spv/src/network/peer.rs | 4 ++++ 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index a3c1a41af..ff67d8c2b 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -282,7 +282,8 @@ impl Broker { let mut on_wire: Vec<(u32, NetworkMessage, PeerHandle)> = Vec::new(); let mut msgs = msgs.into_iter(); for (height, msg) in msgs.by_ref() { - let open = peers.iter().filter(|p| p.in_flight_bytes() < p.cap()); + // `peers` can be a snapshot taken before the timeout monitor closed one of them. + let open = peers.iter().filter(|p| !p.is_closed() && p.in_flight_bytes() < p.cap()); let Some(peer) = choose_peer(open, classify(&msg)) else { unsent.push((height, msg)); break; @@ -668,6 +669,22 @@ mod tests { } } + #[tokio::test] + async fn requeued_work_skips_a_closed_peer_still_listed() { + let broker = Broker::new(); + let gone = ConnectedPeer::for_test(conn(1, 0)).await; + let other = ConnectedPeer::for_test(conn(2, 0)).await; + broker.mark_on_wire(&get_cfilters(0), gone.id()).await; + gone.close(); + broker.requeue_from(&HashSet::from([gone.id()])).await; + + // Only the charge matters here: the loopback has no reader to answer it. + broker.route(&[gone.handle(), other.handle()], u64::MAX, &ResponseCosts::new()).await; + + assert_eq!(gone.in_flight_bytes(), 0); + assert!(other.in_flight_bytes() > 0); + } + #[tokio::test] async fn a_requeued_request_dedups_until_its_response_arrives() { let broker = Broker::new(); diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index e96e8eba1..7063585d8 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1093,15 +1093,15 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) continue; } - // Re-queue every on-wire request of a culprit (fresh ones too: the connection is - // going away), keeping keys as Queued so de-dup holds. - let retried = broker.requeue_from(&culprits).await; - - // Drop culprits still in the set (some may already be gone). + // Drop culprits still in the set (some may already be gone) before their work + // is re-queued, so the router cannot hand it back to them. let (kicked, remaining) = connected.remove(&culprits).await; for peer in &kicked { peer.close(); } + // Re-queue every on-wire request of a culprit (fresh ones too: the connection is + // going away), keeping keys as Queued so de-dup holds. + let retried = broker.requeue_from(&culprits).await; let dropped = kicked.len(); for peer in &culprits { bans.ban(peer.addr(), Ban::Kicked).await; diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index 850ac5bee..c68a35692 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -269,6 +269,10 @@ impl PeerHandle { self.cap.load(Ordering::Relaxed) } + pub(crate) fn is_closed(&self) -> bool { + self.token.is_cancelled() + } + pub async fn send(&self, msg: &NetworkMessage) -> NetworkResult<()> { // The sync layer sends a plain `getheaders`, since it does not know the peer. // The reader decompresses `Headers2` back to `Headers`. From 6c0134c0cea3576ba075c3bd78cf6886fe2123c3 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Wed, 11 Nov 2026 11:05:39 +1000 Subject: [PATCH 2/4] fix(dash-spv): skip closed peers when hedging The flow controller hands `Broker::hedge` a snapshot of the peer handles, so a peer the timeout monitor kicks right after the snapshot is still listed. `PeerHandle::send` does not check the closed token and the snapshot keeps the write half alive, so the send to such a peer succeeds. The hedge was then recorded against a dead peer, and the lowest block got no working second copy until its original request timed out. `hedge` now skips a closed peer, as `Broker::route` already does. --- dash-spv/src/network/broker.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index ff67d8c2b..ec5e09732 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -355,7 +355,7 @@ impl Broker { } let peer = peers .iter() - .filter(|p| p.id() != o.peer && p.in_flight_bytes() < p.cap()) + .filter(|p| !p.is_closed() && p.id() != o.peer && p.in_flight_bytes() < p.cap()) .max_by_key(|p| p.cap().saturating_sub(p.in_flight_bytes()))?; (key.clone(), o.msg.clone(), o.height, peer.clone()) }; @@ -704,14 +704,19 @@ mod tests { } /// A block at a slow peer gets a second copy only once it is the lowest on the wire and - /// its peer stopped delivering. + /// its peer stopped delivering. Peer 3 has the most room but was kicked after the + /// snapshot of `peers` was taken, so it never gets the copy. #[tokio::test(start_paused = true)] async fn hedge_asks_another_peer_only_for_the_lowest_stalled_block() { let broker = Broker::new(); let costs = ResponseCosts::new(); + let kicked = ConnectedPeer::for_test(conn(3, 0)).await; + kicked.set_cap(u64::MAX); + kicked.close(); let peers = [ ConnectedPeer::for_test(conn(1, 0)).await.handle(), ConnectedPeer::for_test(conn(2, 0)).await.handle(), + kicked.handle(), ]; let (slow, idle) = (&peers[..1], peers[1].id()); let get_block = |n: u32| NetworkMessage::GetData(vec![block(n)]); From 63ef6d17b4e8254c6d82f60685a7f0da0bd18c5e Mon Sep 17 00:00:00 2001 From: xdustinface Date: Wed, 11 Nov 2026 23:43:59 +1000 Subject: [PATCH 3/4] test(dash-spv): route a kicked peer's requeued work elsewhere Runs the real timeout monitor and router with a stalled peer and an idle one. The test holds the filter queue while the kick fires, which stops the monitor inside its re-queue, and checks that the stalled peer is already closed and out of the connected set at that point. Re-queuing before the removal fails it: on a multi-thread runtime the router could snapshot the set in that window and hand the work back to the stalled peer. Once the queue is released, the re-queued request goes to the remaining peer and nothing is charged to the kicked connection. `Broker::hold_queue_of` is a test-only helper that locks the queue a message waits in. --- dash-spv/src/network/broker.rs | 7 ++++++ dash-spv/src/network/manager.rs | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index ec5e09732..0114e9b42 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -393,6 +393,13 @@ impl Broker { ReqState::Arrived => "arrived", }) } + + /// Locks the queue `msg` waits in until the returned guard drops, so a test can stop a + /// task at the moment it queues such a message. + #[cfg(test)] + pub async fn hold_queue_of(&self, msg: &NetworkMessage) -> impl Sized + '_ { + self.queue.queue(classify(msg)).lock().await + } } #[derive(Clone, Copy, PartialEq, Eq, Debug)] diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 7063585d8..90e479c6f 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1327,6 +1327,8 @@ mod tests { use dashcore::network::message_filter::{CFilter, GetCFilters}; use dashcore::BlockHash; use test_case::test_case; + use tokio::task; + use tokio::time; fn get_cfilters(start: u32) -> NetworkMessage { NetworkMessage::GetCFilters(GetCFilters { @@ -1580,6 +1582,46 @@ mod tests { assert_eq!(net.state.broker.len(), stalls as usize); } + /// A timeout kick re-queues the stalled peer's work only once that peer is closed and out + /// of the set, so no router snapshot can hand the work back. Holding the filter queue + /// stops the monitor at that point. The work then goes to the peer that stays. + #[tokio::test(start_paused = true)] + async fn a_kicked_peer_gets_none_of_its_requeued_work() { + let net = broker().await; + let session = net.session.lock().await.clone(); + let mut handles = Vec::new(); + for n in 1..=2 { + let peer = ConnectedPeer::for_test(conn(n, 0)).await; + handles.push(peer.handle()); + let admitted = + net.state.connected.admit(peer, &session, net.state.max_peers, None).await; + assert!(matches!(admitted, Admission::Joined)); + } + net.state.broker.mark_on_wire(&get_cfilters(0), conn(1, 0)).await; + + let held = net.state.broker.hold_queue_of(&get_cfilters(0)).await; + time::advance(REQUEST_TIMEOUT + TIMEOUT_CHECK * 2).await; + for _ in 0..4 { + task::yield_now().await; + } + let key = RequestKey::CFilters(0); + assert_eq!( + net.state.broker.state_of(&key).await, + Some("queued"), + "the monitor reached the re-queue" + ); + assert_eq!(net.connected_count().await, 1, "re-queued before the kick"); + assert!(handles[0].is_closed(), "re-queued before the stalled peer was closed"); + drop(held); + for _ in 0..4 { + task::yield_now().await; + } + + assert_eq!(net.state.broker.state_of(&key).await, Some("on_wire")); + assert_eq!(handles[0].in_flight_bytes(), 0, "work went back to the kicked peer"); + assert!(handles[1].in_flight_bytes() > 0, "the peer that stays carries the work"); + } + /// A peer that was silent while idle is timed from the request it is given, and /// kicked only when it stays silent with that work for the full timeout. #[tokio::test(start_paused = true)] From b21026053b3f36346a39edd1f3dfe6c12c2206ec Mon Sep 17 00:00:00 2001 From: xdustinface Date: Thu, 12 Nov 2026 02:04:23 +1000 Subject: [PATCH 4/4] fix(dash-spv): leave closed peers out of the routing budget `Broker::route` summed the room and in-flight bytes of every listed peer, but sends only to peers that are not closed. A peer closed by a ban or a failed write stays in the connected set until it is removed, so its room let `route` pop work it could not send. `push_front_all` put the work back and stored a wake, and the router loop ran again at once, spinning until the peer was gone. The budget now counts open peers only. With no open peer that has room, `route` pops nothing and the router waits for a real wake. --- dash-spv/src/network/broker.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index 0114e9b42..f111931bd 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -257,9 +257,11 @@ impl Broker { if peers.is_empty() { return 0; } - let total_in_flight: u64 = peers.iter().map(PeerHandle::in_flight_bytes).sum(); - let per_peer_room: u64 = - peers.iter().map(|p| p.cap().saturating_sub(p.in_flight_bytes())).sum(); + // A closed peer still listed is sent nothing, so its room would pop work that only + // goes back to the queue and wakes the router again at once. + let live = || peers.iter().filter(|p| !p.is_closed()); + let total_in_flight: u64 = live().map(PeerHandle::in_flight_bytes).sum(); + let per_peer_room: u64 = live().map(|p| p.cap().saturating_sub(p.in_flight_bytes())).sum(); let budget = per_peer_room.min(global_cap.saturating_sub(total_in_flight)); if budget == 0 { return 0; @@ -582,6 +584,7 @@ mod tests { use dashcore::network::message_filter::{GetCFHeaders, GetCFilters}; use dashcore::network::message_sml::GetMnListDiff; use dashcore::Txid; + use futures::FutureExt; use test_case::test_case; fn get_headers(locator: u32) -> NetworkMessage { @@ -684,6 +687,13 @@ mod tests { broker.mark_on_wire(&get_cfilters(0), gone.id()).await; gone.close(); broker.requeue_from(&HashSet::from([gone.id()])).await; + broker.notified().await; + + // With only the closed peer listed nothing is popped, so the router is not woken + // again and waits for a real change instead of spinning. + assert_eq!(broker.route(&[gone.handle()], u64::MAX, &ResponseCosts::new()).await, 0); + assert_eq!(broker.state_of(&RequestKey::CFilters(0)).await, Some("queued")); + assert!(broker.notified().now_or_never().is_none(), "the router was woken again"); // Only the charge matters here: the loopback has no reader to answer it. broker.route(&[gone.handle(), other.handle()], u64::MAX, &ResponseCosts::new()).await;