Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 45 additions & 6 deletions dash-spv/src/network/broker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -257,9 +257,11 @@ impl Broker {
if peers.is_empty() {
return 0;
}
let total_in_flight: u64 = peers.iter().map(PeerHandle::in_flight_bytes).sum();
let per_peer_room: u64 =
peers.iter().map(|p| p.cap().saturating_sub(p.in_flight_bytes())).sum();
// A closed peer still listed is sent nothing, so its room would pop work that only
// goes back to the queue and wakes the router again at once.
let live = || peers.iter().filter(|p| !p.is_closed());
let total_in_flight: u64 = live().map(PeerHandle::in_flight_bytes).sum();
let per_peer_room: u64 = live().map(|p| p.cap().saturating_sub(p.in_flight_bytes())).sum();
let budget = per_peer_room.min(global_cap.saturating_sub(total_in_flight));
if budget == 0 {
return 0;
Expand All @@ -282,7 +284,8 @@ impl Broker {
let mut on_wire: Vec<(u32, NetworkMessage, PeerHandle)> = Vec::new();
let mut msgs = msgs.into_iter();
for (height, msg) in msgs.by_ref() {
let open = peers.iter().filter(|p| p.in_flight_bytes() < p.cap());
// `peers` can be a snapshot taken before the timeout monitor closed one of them.
let open = peers.iter().filter(|p| !p.is_closed() && p.in_flight_bytes() < p.cap());
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
let Some(peer) = choose_peer(open, classify(&msg)) else {
unsent.push((height, msg));
break;
Expand Down Expand Up @@ -354,7 +357,7 @@ impl Broker {
}
let peer = peers
.iter()
.filter(|p| p.id() != o.peer && p.in_flight_bytes() < p.cap())
.filter(|p| !p.is_closed() && p.id() != o.peer && p.in_flight_bytes() < p.cap())
.max_by_key(|p| p.cap().saturating_sub(p.in_flight_bytes()))?;
(key.clone(), o.msg.clone(), o.height, peer.clone())
};
Expand Down Expand Up @@ -392,6 +395,13 @@ impl Broker {
ReqState::Arrived => "arrived",
})
}

/// Locks the queue `msg` waits in until the returned guard drops, so a test can stop a
/// task at the moment it queues such a message.
#[cfg(test)]
pub async fn hold_queue_of(&self, msg: &NetworkMessage) -> impl Sized + '_ {
self.queue.queue(classify(msg)).lock().await
}
}

#[derive(Clone, Copy, PartialEq, Eq, Debug)]
Expand Down Expand Up @@ -574,6 +584,7 @@ mod tests {
use dashcore::network::message_filter::{GetCFHeaders, GetCFilters};
use dashcore::network::message_sml::GetMnListDiff;
use dashcore::Txid;
use futures::FutureExt;
use test_case::test_case;

fn get_headers(locator: u32) -> NetworkMessage {
Expand Down Expand Up @@ -668,6 +679,29 @@ mod tests {
}
}

#[tokio::test]
async fn requeued_work_skips_a_closed_peer_still_listed() {
let broker = Broker::new();
let gone = ConnectedPeer::for_test(conn(1, 0)).await;
let other = ConnectedPeer::for_test(conn(2, 0)).await;
broker.mark_on_wire(&get_cfilters(0), gone.id()).await;
gone.close();
broker.requeue_from(&HashSet::from([gone.id()])).await;
broker.notified().await;

// With only the closed peer listed nothing is popped, so the router is not woken
// again and waits for a real change instead of spinning.
assert_eq!(broker.route(&[gone.handle()], u64::MAX, &ResponseCosts::new()).await, 0);
assert_eq!(broker.state_of(&RequestKey::CFilters(0)).await, Some("queued"));
assert!(broker.notified().now_or_never().is_none(), "the router was woken again");

// Only the charge matters here: the loopback has no reader to answer it.
broker.route(&[gone.handle(), other.handle()], u64::MAX, &ResponseCosts::new()).await;

assert_eq!(gone.in_flight_bytes(), 0);
assert!(other.in_flight_bytes() > 0);
}

#[tokio::test]
async fn a_requeued_request_dedups_until_its_response_arrives() {
let broker = Broker::new();
Expand All @@ -687,14 +721,19 @@ mod tests {
}

/// A block at a slow peer gets a second copy only once it is the lowest on the wire and
/// its peer stopped delivering.
/// its peer stopped delivering. Peer 3 has the most room but was kicked after the
/// snapshot of `peers` was taken, so it never gets the copy.
#[tokio::test(start_paused = true)]
async fn hedge_asks_another_peer_only_for_the_lowest_stalled_block() {
let broker = Broker::new();
let costs = ResponseCosts::new();
let kicked = ConnectedPeer::for_test(conn(3, 0)).await;
kicked.set_cap(u64::MAX);
kicked.close();
let peers = [
ConnectedPeer::for_test(conn(1, 0)).await.handle(),
ConnectedPeer::for_test(conn(2, 0)).await.handle(),
kicked.handle(),
];
let (slow, idle) = (&peers[..1], peers[1].id());
let get_block = |n: u32| NetworkMessage::GetData(vec![block(n)]);
Expand Down
52 changes: 47 additions & 5 deletions dash-spv/src/network/manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1093,15 +1093,15 @@ fn spawn_timeout_monitor(state: Arc<NetworkState>, shutdown: CancellationToken)
continue;
}

// Re-queue every on-wire request of a culprit (fresh ones too: the connection is
// going away), keeping keys as Queued so de-dup holds.
let retried = broker.requeue_from(&culprits).await;

// Drop culprits still in the set (some may already be gone).
// Drop culprits still in the set (some may already be gone) before their work
// is re-queued, so the router cannot hand it back to them.
let (kicked, remaining) = connected.remove(&culprits).await;

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

✨ Suggestion, not blocking · Testing · Moderate

No test covers the reordered kick sequence in the timeout monitor

Why: The main fix moves connected.remove and peer.close() before broker.requeue_from. No test fails if the order is reverted. The new broker test only checks the is_closed filter in route and hedge.

How to fix it, why, 6 places read, and 3 other checks

How to fix it
Add a test with a paused clock. It should run a real router and monitor with a stalled peer and a healthy peer. It should then assert that the re-queued request is never sent to the kicked connection.

Why it matters
The existing Departure::Kick case in a_departed_peer_is_announced_once checks announcements, not the order. AGENTS.md says "Every fix comes with a regression test that fails without it."

Checked in the code
On the commented line: Now calls connected.remove(&culprits) and peer.close() before broker.requeue_from(&culprits), but nothing asserts that order. let (kicked, remaining) = connected.remove(&culprits).await; now runs before broker.requeue_from(&culprits) at line 1104

  • requeued_work_skips_a_closed_peer_still_listed calls close() and requeue_from by hand, so it does not exercise the monitor ordering.
  • requeued_work_skips_a_closed_peer_still_listed calls gone.close() before requeue_from, so it covers only the is_closed filter in route

#[tokio::test]
async fn requeued_work_skips_a_closed_peer_still_listed() {
let broker = Broker::new();
let gone = ConnectedPeer::for_test(conn(1, 0)).await;

The Departure::Kick case asserts only on NetworkEvents and connected_count, never on which peer the re-queued getcfilters is charged to

)
};
idx += 1;
tx.send((addr, shared)).is_ok()

// going away), keeping keys as Queued so de-dup holds.
let retried = broker.requeue_from(&culprits).await;
let dropped = kicked.len();
for peer in &culprits {
bans.ban(peer.addr(), Ban::Kicked).await;

  • The Departure::Kick case in a_departed_peer_is_announced_once (manager.rs tests) checks announcements and the final peer count, not where re-queued work go…
  • AGENTS.md: Every fix comes with a regression test that fails without it.
  • dash-spv/AGENTS.md: A change to sync, restart or disconnect handling needs a dashd test in tests/dashd_sync/ or tests/dashd_masternode/.
  • Create an issue for this

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This round raises this problem again as a warning.

⚠️ Warning, not blocking · Testing · Moderate

Regression test does not fail when the reorder in the timeout monitor is reverted

What goes wrong: a_kicked_peer_gets_none_of_its_requeued_work runs on the current-thread test runtime. With the old order, requeue_from only stores a notify permit, and the monitor goes on to connected.remove and peer.close() with no yielding await in between. The router wakes only after the kick is done, and it takes a fresh peer snapshot, so it never sees the kicked peer. The test therefore passes even without the reorder, and without the is_closed filter too.

Fix: Make the test force the router to run between the requeue and the removal, or drive the old interleaving directly. For example, run on a multi-thread runtime, or have the router hold a pre-kick snapshot and call route after requeue_from and before remove. Then confirm the test fails with the old order.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Partly fixed. Still open: Nothing fails if the monitor goes back to re-queuing work before it removes and closes the culprits, so the reorder has no working regression test.

for peer in &kicked {
peer.close();
}
// Re-queue every on-wire request of a culprit (fresh ones too: the connection is
// going away), keeping keys as Queued so de-dup holds.
let retried = broker.requeue_from(&culprits).await;
Comment thread
xdustinface marked this conversation as resolved.
Resolved
let dropped = kicked.len();
for peer in &culprits {
bans.ban(peer.addr(), Ban::Kicked).await;
Expand Down Expand Up @@ -1327,6 +1327,8 @@ mod tests {
use dashcore::network::message_filter::{CFilter, GetCFilters};
use dashcore::BlockHash;
use test_case::test_case;
use tokio::task;
use tokio::time;

fn get_cfilters(start: u32) -> NetworkMessage {
NetworkMessage::GetCFilters(GetCFilters {
Expand Down Expand Up @@ -1580,6 +1582,46 @@ mod tests {
assert_eq!(net.state.broker.len(), stalls as usize);
}

/// A timeout kick re-queues the stalled peer's work only once that peer is closed and out
/// of the set, so no router snapshot can hand the work back. Holding the filter queue
/// stops the monitor at that point. The work then goes to the peer that stays.
#[tokio::test(start_paused = true)]
async fn a_kicked_peer_gets_none_of_its_requeued_work() {
let net = broker().await;
let session = net.session.lock().await.clone();
let mut handles = Vec::new();
for n in 1..=2 {
let peer = ConnectedPeer::for_test(conn(n, 0)).await;
handles.push(peer.handle());
let admitted =
net.state.connected.admit(peer, &session, net.state.max_peers, None).await;
assert!(matches!(admitted, Admission::Joined));
}
net.state.broker.mark_on_wire(&get_cfilters(0), conn(1, 0)).await;

let held = net.state.broker.hold_queue_of(&get_cfilters(0)).await;
time::advance(REQUEST_TIMEOUT + TIMEOUT_CHECK * 2).await;
for _ in 0..4 {
task::yield_now().await;
}
let key = RequestKey::CFilters(0);
assert_eq!(
net.state.broker.state_of(&key).await,
Some("queued"),
"the monitor reached the re-queue"
);
assert_eq!(net.connected_count().await, 1, "re-queued before the kick");
assert!(handles[0].is_closed(), "re-queued before the stalled peer was closed");
drop(held);
for _ in 0..4 {
task::yield_now().await;
}

assert_eq!(net.state.broker.state_of(&key).await, Some("on_wire"));
assert_eq!(handles[0].in_flight_bytes(), 0, "work went back to the kicked peer");
assert!(handles[1].in_flight_bytes() > 0, "the peer that stays carries the work");
}

/// A peer that was silent while idle is timed from the request it is given, and
/// kicked only when it stays silent with that work for the full timeout.
#[tokio::test(start_paused = true)]
Expand Down
4 changes: 4 additions & 0 deletions dash-spv/src/network/peer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,10 @@ impl PeerHandle {
self.cap.load(Ordering::Relaxed)
}

pub(crate) fn is_closed(&self) -> bool {
self.token.is_cancelled()
}

pub async fn send(&self, msg: &NetworkMessage) -> NetworkResult<()> {
// The sync layer sends a plain `getheaders`, since it does not know the peer.
// The reader decompresses `Headers2` back to `Headers`.
Expand Down
Loading