diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index a3c1a41af..f111931bd 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -257,9 +257,11 @@ impl Broker { if peers.is_empty() { return 0; } - let total_in_flight: u64 = peers.iter().map(PeerHandle::in_flight_bytes).sum(); - let per_peer_room: u64 = - peers.iter().map(|p| p.cap().saturating_sub(p.in_flight_bytes())).sum(); + // A closed peer still listed is sent nothing, so its room would pop work that only + // goes back to the queue and wakes the router again at once. + let live = || peers.iter().filter(|p| !p.is_closed()); + let total_in_flight: u64 = live().map(PeerHandle::in_flight_bytes).sum(); + let per_peer_room: u64 = live().map(|p| p.cap().saturating_sub(p.in_flight_bytes())).sum(); let budget = per_peer_room.min(global_cap.saturating_sub(total_in_flight)); if budget == 0 { return 0; @@ -282,7 +284,8 @@ impl Broker { let mut on_wire: Vec<(u32, NetworkMessage, PeerHandle)> = Vec::new(); let mut msgs = msgs.into_iter(); for (height, msg) in msgs.by_ref() { - let open = peers.iter().filter(|p| p.in_flight_bytes() < p.cap()); + // `peers` can be a snapshot taken before the timeout monitor closed one of them. + let open = peers.iter().filter(|p| !p.is_closed() && p.in_flight_bytes() < p.cap()); let Some(peer) = choose_peer(open, classify(&msg)) else { unsent.push((height, msg)); break; @@ -354,7 +357,7 @@ impl Broker { } let peer = peers .iter() - .filter(|p| p.id() != o.peer && p.in_flight_bytes() < p.cap()) + .filter(|p| !p.is_closed() && p.id() != o.peer && p.in_flight_bytes() < p.cap()) .max_by_key(|p| p.cap().saturating_sub(p.in_flight_bytes()))?; (key.clone(), o.msg.clone(), o.height, peer.clone()) }; @@ -392,6 +395,13 @@ impl Broker { ReqState::Arrived => "arrived", }) } + + /// Locks the queue `msg` waits in until the returned guard drops, so a test can stop a + /// task at the moment it queues such a message. + #[cfg(test)] + pub async fn hold_queue_of(&self, msg: &NetworkMessage) -> impl Sized + '_ { + self.queue.queue(classify(msg)).lock().await + } } #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -574,6 +584,7 @@ mod tests { use dashcore::network::message_filter::{GetCFHeaders, GetCFilters}; use dashcore::network::message_sml::GetMnListDiff; use dashcore::Txid; + use futures::FutureExt; use test_case::test_case; fn get_headers(locator: u32) -> NetworkMessage { @@ -668,6 +679,29 @@ mod tests { } } + #[tokio::test] + async fn requeued_work_skips_a_closed_peer_still_listed() { + let broker = Broker::new(); + let gone = ConnectedPeer::for_test(conn(1, 0)).await; + let other = ConnectedPeer::for_test(conn(2, 0)).await; + broker.mark_on_wire(&get_cfilters(0), gone.id()).await; + gone.close(); + broker.requeue_from(&HashSet::from([gone.id()])).await; + broker.notified().await; + + // With only the closed peer listed nothing is popped, so the router is not woken + // again and waits for a real change instead of spinning. + assert_eq!(broker.route(&[gone.handle()], u64::MAX, &ResponseCosts::new()).await, 0); + assert_eq!(broker.state_of(&RequestKey::CFilters(0)).await, Some("queued")); + assert!(broker.notified().now_or_never().is_none(), "the router was woken again"); + + // Only the charge matters here: the loopback has no reader to answer it. + broker.route(&[gone.handle(), other.handle()], u64::MAX, &ResponseCosts::new()).await; + + assert_eq!(gone.in_flight_bytes(), 0); + assert!(other.in_flight_bytes() > 0); + } + #[tokio::test] async fn a_requeued_request_dedups_until_its_response_arrives() { let broker = Broker::new(); @@ -687,14 +721,19 @@ mod tests { } /// A block at a slow peer gets a second copy only once it is the lowest on the wire and - /// its peer stopped delivering. + /// its peer stopped delivering. Peer 3 has the most room but was kicked after the + /// snapshot of `peers` was taken, so it never gets the copy. #[tokio::test(start_paused = true)] async fn hedge_asks_another_peer_only_for_the_lowest_stalled_block() { let broker = Broker::new(); let costs = ResponseCosts::new(); + let kicked = ConnectedPeer::for_test(conn(3, 0)).await; + kicked.set_cap(u64::MAX); + kicked.close(); let peers = [ ConnectedPeer::for_test(conn(1, 0)).await.handle(), ConnectedPeer::for_test(conn(2, 0)).await.handle(), + kicked.handle(), ]; let (slow, idle) = (&peers[..1], peers[1].id()); let get_block = |n: u32| NetworkMessage::GetData(vec![block(n)]); diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index e96e8eba1..90e479c6f 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1093,15 +1093,15 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) continue; } - // Re-queue every on-wire request of a culprit (fresh ones too: the connection is - // going away), keeping keys as Queued so de-dup holds. - let retried = broker.requeue_from(&culprits).await; - - // Drop culprits still in the set (some may already be gone). + // Drop culprits still in the set (some may already be gone) before their work + // is re-queued, so the router cannot hand it back to them. let (kicked, remaining) = connected.remove(&culprits).await; for peer in &kicked { peer.close(); } + // Re-queue every on-wire request of a culprit (fresh ones too: the connection is + // going away), keeping keys as Queued so de-dup holds. + let retried = broker.requeue_from(&culprits).await; let dropped = kicked.len(); for peer in &culprits { bans.ban(peer.addr(), Ban::Kicked).await; @@ -1327,6 +1327,8 @@ mod tests { use dashcore::network::message_filter::{CFilter, GetCFilters}; use dashcore::BlockHash; use test_case::test_case; + use tokio::task; + use tokio::time; fn get_cfilters(start: u32) -> NetworkMessage { NetworkMessage::GetCFilters(GetCFilters { @@ -1580,6 +1582,46 @@ mod tests { assert_eq!(net.state.broker.len(), stalls as usize); } + /// A timeout kick re-queues the stalled peer's work only once that peer is closed and out + /// of the set, so no router snapshot can hand the work back. Holding the filter queue + /// stops the monitor at that point. The work then goes to the peer that stays. + #[tokio::test(start_paused = true)] + async fn a_kicked_peer_gets_none_of_its_requeued_work() { + let net = broker().await; + let session = net.session.lock().await.clone(); + let mut handles = Vec::new(); + for n in 1..=2 { + let peer = ConnectedPeer::for_test(conn(n, 0)).await; + handles.push(peer.handle()); + let admitted = + net.state.connected.admit(peer, &session, net.state.max_peers, None).await; + assert!(matches!(admitted, Admission::Joined)); + } + net.state.broker.mark_on_wire(&get_cfilters(0), conn(1, 0)).await; + + let held = net.state.broker.hold_queue_of(&get_cfilters(0)).await; + time::advance(REQUEST_TIMEOUT + TIMEOUT_CHECK * 2).await; + for _ in 0..4 { + task::yield_now().await; + } + let key = RequestKey::CFilters(0); + assert_eq!( + net.state.broker.state_of(&key).await, + Some("queued"), + "the monitor reached the re-queue" + ); + assert_eq!(net.connected_count().await, 1, "re-queued before the kick"); + assert!(handles[0].is_closed(), "re-queued before the stalled peer was closed"); + drop(held); + for _ in 0..4 { + task::yield_now().await; + } + + assert_eq!(net.state.broker.state_of(&key).await, Some("on_wire")); + assert_eq!(handles[0].in_flight_bytes(), 0, "work went back to the kicked peer"); + assert!(handles[1].in_flight_bytes() > 0, "the peer that stays carries the work"); + } + /// A peer that was silent while idle is timed from the request it is given, and /// kicked only when it stays silent with that work for the full timeout. #[tokio::test(start_paused = true)] diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index 850ac5bee..c68a35692 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -269,6 +269,10 @@ impl PeerHandle { self.cap.load(Ordering::Relaxed) } + pub(crate) fn is_closed(&self) -> bool { + self.token.is_cancelled() + } + pub async fn send(&self, msg: &NetworkMessage) -> NetworkResult<()> { // The sync layer sends a plain `getheaders`, since it does not know the peer. // The reader decompresses `Headers2` back to `Headers`.