From 81acc285d04c992d429dd2a9f5311679b793699d Mon Sep 17 00:00:00 2001 From: xdustinface Date: Fri, 6 Nov 2026 00:44:49 +1000 Subject: [PATCH 1/9] fix(dash-spv): keep re-queued work off kicked peers The timeout monitor re-queued a culprit's requests and woke the router before it removed the culprit from the connected set, so the router could send the freed work straight back to the peer being closed. Remove and close the culprits first, then re-queue. The router also works from a snapshot of the set that can predate the kick, so `Broker::route` now skips a peer whose connection is already closed. --- dash-spv/src/network/broker.rs | 19 ++++++++++++++++++- dash-spv/src/network/manager.rs | 10 +++++----- dash-spv/src/network/peer.rs | 4 ++++ 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index a3c1a41af..ff67d8c2b 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -282,7 +282,8 @@ impl Broker { let mut on_wire: Vec<(u32, NetworkMessage, PeerHandle)> = Vec::new(); let mut msgs = msgs.into_iter(); for (height, msg) in msgs.by_ref() { - let open = peers.iter().filter(|p| p.in_flight_bytes() < p.cap()); + // `peers` can be a snapshot taken before the timeout monitor closed one of them. + let open = peers.iter().filter(|p| !p.is_closed() && p.in_flight_bytes() < p.cap()); let Some(peer) = choose_peer(open, classify(&msg)) else { unsent.push((height, msg)); break; @@ -668,6 +669,22 @@ mod tests { } } + #[tokio::test] + async fn requeued_work_skips_a_closed_peer_still_listed() { + let broker = Broker::new(); + let gone = ConnectedPeer::for_test(conn(1, 0)).await; + let other = ConnectedPeer::for_test(conn(2, 0)).await; + broker.mark_on_wire(&get_cfilters(0), gone.id()).await; + gone.close(); + broker.requeue_from(&HashSet::from([gone.id()])).await; + + // Only the charge matters here: the loopback has no reader to answer it. + broker.route(&[gone.handle(), other.handle()], u64::MAX, &ResponseCosts::new()).await; + + assert_eq!(gone.in_flight_bytes(), 0); + assert!(other.in_flight_bytes() > 0); + } + #[tokio::test] async fn a_requeued_request_dedups_until_its_response_arrives() { let broker = Broker::new(); diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index e96e8eba1..7063585d8 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1093,15 +1093,15 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) continue; } - // Re-queue every on-wire request of a culprit (fresh ones too: the connection is - // going away), keeping keys as Queued so de-dup holds. - let retried = broker.requeue_from(&culprits).await; - - // Drop culprits still in the set (some may already be gone). + // Drop culprits still in the set (some may already be gone) before their work + // is re-queued, so the router cannot hand it back to them. let (kicked, remaining) = connected.remove(&culprits).await; for peer in &kicked { peer.close(); } + // Re-queue every on-wire request of a culprit (fresh ones too: the connection is + // going away), keeping keys as Queued so de-dup holds. + let retried = broker.requeue_from(&culprits).await; let dropped = kicked.len(); for peer in &culprits { bans.ban(peer.addr(), Ban::Kicked).await; diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index 850ac5bee..c68a35692 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -269,6 +269,10 @@ impl PeerHandle { self.cap.load(Ordering::Relaxed) } + pub(crate) fn is_closed(&self) -> bool { + self.token.is_cancelled() + } + pub async fn send(&self, msg: &NetworkMessage) -> NetworkResult<()> { // The sync layer sends a plain `getheaders`, since it does not know the peer. // The reader decompresses `Headers2` back to `Headers`. From 6c0134c0cea3576ba075c3bd78cf6886fe2123c3 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Wed, 11 Nov 2026 11:05:39 +1000 Subject: [PATCH 2/9] fix(dash-spv): skip closed peers when hedging The flow controller hands `Broker::hedge` a snapshot of the peer handles, so a peer the timeout monitor kicks right after the snapshot is still listed. `PeerHandle::send` does not check the closed token and the snapshot keeps the write half alive, so the send to such a peer succeeds. The hedge was then recorded against a dead peer, and the lowest block got no working second copy until its original request timed out. `hedge` now skips a closed peer, as `Broker::route` already does. --- dash-spv/src/network/broker.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index ff67d8c2b..ec5e09732 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -355,7 +355,7 @@ impl Broker { } let peer = peers .iter() - .filter(|p| p.id() != o.peer && p.in_flight_bytes() < p.cap()) + .filter(|p| !p.is_closed() && p.id() != o.peer && p.in_flight_bytes() < p.cap()) .max_by_key(|p| p.cap().saturating_sub(p.in_flight_bytes()))?; (key.clone(), o.msg.clone(), o.height, peer.clone()) }; @@ -704,14 +704,19 @@ mod tests { } /// A block at a slow peer gets a second copy only once it is the lowest on the wire and - /// its peer stopped delivering. + /// its peer stopped delivering. Peer 3 has the most room but was kicked after the + /// snapshot of `peers` was taken, so it never gets the copy. #[tokio::test(start_paused = true)] async fn hedge_asks_another_peer_only_for_the_lowest_stalled_block() { let broker = Broker::new(); let costs = ResponseCosts::new(); + let kicked = ConnectedPeer::for_test(conn(3, 0)).await; + kicked.set_cap(u64::MAX); + kicked.close(); let peers = [ ConnectedPeer::for_test(conn(1, 0)).await.handle(), ConnectedPeer::for_test(conn(2, 0)).await.handle(), + kicked.handle(), ]; let (slow, idle) = (&peers[..1], peers[1].id()); let get_block = |n: u32| NetworkMessage::GetData(vec![block(n)]); From 0467c8c5afe8c36200109ab2bd57cf8e2e592a81 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Wed, 11 Nov 2026 00:01:19 +1000 Subject: [PATCH 3/9] fix(dash-spv): count partial messages as progress A slow peer streaming one large reply, such as a `headers2` of up to 8000 headers (about 344 KB), was kicked by `Broker::expire` because only a completed message counted as progress on its request, although its bytes kept arriving. The reader now records whether bytes of an unfinished message sit in its buffer, which works for v1 and v2 framing alike without knowing the command. Each tick the timeout monitor collects the peers whose byte counter grew while such a message is arriving and refreshes their requests with one `Broker::receiving` call. Small unrelated messages such as pings, `inv`s and `addr`s arrive whole, so a peer that ignores a request while relaying them still stalls after one timeout. A request also stalls once its peer completed no answer for `ANSWER_TIMEOUTS` (6) timeouts, so a peer trickling bytes into a message that never finishes cannot keep its request alive forever. The bound counts from the last answer, not from send time: peers answer in order, and a slow peer's queue can hold a request well past a minute while every answer ahead of it arrives. `Broker::hedge` waits on answers only, as before, so a block still arriving slowly can be hedged. --- dash-spv/src/network/broker.rs | 53 +++++++++++++--- dash-spv/src/network/manager.rs | 105 +++++++++++++++++++++++--------- dash-spv/src/network/peer.rs | 66 +++++++++++++++++++- 3 files changed, 183 insertions(+), 41 deletions(-) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index ec5e09732..31c227f85 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -12,6 +12,10 @@ use crate::network::costs::ResponseCosts; use crate::network::peer::{PeerHandle, PeerId}; const KICKING_ATTEMPTS: u32 = 2; +/// How many timeouts a request lives on bytes of a message still arriving, without an +/// answer. A big answer on a slow link must fit, but a message that never finishes must +/// not keep its request alive forever. +const ANSWER_TIMEOUTS: u32 = 6; const HEDGE_AFTER: Duration = Duration::from_secs(2); const MIN_IN_FLIGHT_PER_CLASS: usize = 4; const MSG_CLASSES: usize = 5; @@ -62,8 +66,12 @@ struct OnWire { /// The connection, not the address: a late close of an old connection must not /// pull back the work of a new one to the same address. peer: PeerId, - /// Refreshed by every piece of a streaming response. + /// Refreshed by every answer and by bytes of a message still arriving, since a big + /// answer can take longer than a timeout to arrive. last_progress: Instant, + /// Refreshed only by answers, so a block still arriving slowly can be hedged and a + /// message that never finishes cannot keep the request alive forever. + last_answer: Instant, msg: NetworkMessage, height: u32, hedge: Option, @@ -205,18 +213,33 @@ impl Broker { /// Peers answer in order, so any answer is progress on all of its requests. pub async fn progress(&self, peer: PeerId) { + self.refresh(&[peer], true).await; + } + + /// Bytes of a message still arriving from any of `peers` are progress on their requests. + pub async fn receiving(&self, peers: &[PeerId]) { + if !peers.is_empty() { + self.refresh(peers, false).await; + } + } + + async fn refresh(&self, peers: &[PeerId], answered: bool) { let now = Instant::now(); for state in self.requests.lock().await.values_mut() { if let ReqState::OnWire(o) = state { - if o.peer == peer { + if peers.contains(&o.peer) { o.last_progress = now; + if answered { + o.last_answer = now; + } } } } } - /// Returns the peers to kick for a stalled request. A request that already - /// stalled [`KICKING_ATTEMPTS`] times is retried without blaming its peer. + /// Returns the peers to kick for a stalled request. A request stalls after a timeout + /// without progress, or after [`ANSWER_TIMEOUTS`] timeouts without an answer. A request + /// that already stalled [`KICKING_ATTEMPTS`] times is retried without blaming its peer. pub async fn expire(&self, timeout: impl Fn(&PeerId) -> Duration) -> HashSet { let now = Instant::now(); let mut culprits = HashSet::new(); @@ -225,7 +248,10 @@ impl Broker { let ReqState::OnWire(o) = state else { continue; }; - if now.duration_since(o.last_progress) <= timeout(&o.peer) { + let timeout = timeout(&o.peer); + if now.duration_since(o.last_progress) <= timeout + && now.duration_since(o.last_answer) <= timeout * ANSWER_TIMEOUTS + { continue; } if o.attempts < KICKING_ATTEMPTS { @@ -315,6 +341,7 @@ impl Broker { msg: msg.clone(), height, last_progress: Instant::now(), + last_answer: Instant::now(), hedge: None, attempts: attempts + 1, })); @@ -335,7 +362,7 @@ impl Broker { sent } - /// Endgame: asks a second peer for the lowest block on the wire once its peer delivered + /// Endgame: asks a second peer for the lowest block on the wire once its peer answered /// nothing for [`HEDGE_AFTER`], and returns its height. Blocks are processed in height /// order, so only the lowest one holds up progress and a second copy of any other is wasted. pub async fn hedge(&self, peers: &[PeerHandle]) -> Option { @@ -350,7 +377,7 @@ impl Broker { _ => None, }) .min_by_key(|(_, o)| o.height)?; - if o.hedge.is_some() || o.last_progress.elapsed() < HEDGE_AFTER { + if o.hedge.is_some() || o.last_answer.elapsed() < HEDGE_AFTER { return None; } let peer = peers @@ -376,6 +403,7 @@ impl Broker { let on_wire = OnWire { peer, last_progress: Instant::now(), + last_answer: Instant::now(), msg: msg.clone(), height: 0, hedge: None, @@ -385,6 +413,14 @@ impl Broker { } } + #[cfg(test)] + pub async fn carrier_of(&self, key: &RequestKey) -> Option { + match self.requests.lock().await.get(key) { + Some(ReqState::OnWire(o)) => Some(o.peer), + _ => None, + } + } + #[cfg(test)] pub async fn state_of(&self, key: &RequestKey) -> Option<&'static str> { self.requests.lock().await.get(key).map(|s| match s { @@ -749,8 +785,9 @@ mod tests { tokio::time::advance(HEDGE_AFTER / 2).await; assert_eq!(broker.hedge(&peers).await, None, "the slow peer is still delivering"); + broker.receiving(&[slow[0].id()]).await; tokio::time::advance(HEDGE_AFTER / 2).await; - assert_eq!(broker.hedge(&peers).await, Some(10)); + assert_eq!(broker.hedge(&peers).await, Some(10), "a block still arriving is hedged"); assert_eq!(broker.hedge(&peers).await, None, "later blocks wait for the lowest"); assert_eq!(hedged_blocks().await, [10]); diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 7063585d8..af9ad5da9 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1055,19 +1055,26 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) // bytes are freed on the peer separately, so a wedged peer can sit at its cap // with no registry entries. Pending work plus a byte counter // frozen for REQUEST_TIMEOUT means stuck, whatever the registry thinks. - let mut culprits: HashSet = { + let (mut culprits, streaming) = { let peers = connected.read().await; let live: HashSet = peers.iter().map(|p| p.id()).collect(); progress.retain(|addr, _| live.contains(addr)); let mut culprits = HashSet::new(); + let mut streaming = Vec::new(); for peer in peers.iter() { let addr = peer.id(); let bytes = peer.bytes_read(); let in_flight = peer.in_flight_bytes(); let entry = progress.entry(addr).or_insert((bytes, now)); + let grew = bytes > entry.0; + // Small unrelated messages arrive whole, so only bytes of a message + // still arriving are progress on a request. + if grew && peer.partial_message() { + streaming.push(addr); + } // An idle peer owes no bytes, so its silence before it gets work // must not count against its first request. - if bytes > entry.0 || in_flight == 0 { + if grew || in_flight == 0 { *entry = (bytes, now); } if in_flight > 0 && now.duration_since(entry.1) > request_timeout(peer.lag_ms()) @@ -1075,8 +1082,9 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) culprits.insert(addr); } } - culprits + (culprits, streaming) }; + broker.receiving(&streaming).await; // A request that stopped making progress also condemns its peer: the // per-peer byte check above sees unrelated traffic and judges it healthy. let timeouts: HashMap = connected @@ -1350,6 +1358,13 @@ mod tests { PeerNetworkManager::new(&config).await } + /// Lets the spawned tasks run what the last time advance made due. + async fn settle() { + for _ in 0..4 { + tokio::task::yield_now().await; + } + } + /// The events received until none arrives for a while. async fn drain(events: &mut broadcast::Receiver) -> Vec { let mut received = Vec::new(); @@ -1544,40 +1559,70 @@ mod tests { assert_eq!(net.connected_count().await, 0); } - #[test_case(None, true; "a silent request")] - #[test_case(Some(1), false; "the carrier streaming keeps it alive")] - #[test_case(Some(2), true; "another peer streaming is not progress")] + /// Counts `n` bytes as read from connected peer `peer`, leaving a message unfinished + /// if `partial`. + async fn receive(net: &PeerNetworkManager, peer: u8, n: u64, partial: bool) { + for p in net.state.connected.read().await.iter() { + if p.id() == conn(peer, 0) { + p.receive_for_test(n, partial); + } + } + } + + enum Feed { + Nothing, + /// A complete answer from this peer every second. + Answers(u8), + /// More bytes of one long message from this peer every second. + Streams(u8), + /// Small complete messages, such as pings, from this peer every second. + Chatters(u8), + } + + /// Peer 1 carries a request for `secs` seconds beside idle peer 2. + #[test_case(Feed::Nothing, 11, true; "a silent request")] + #[test_case(Feed::Answers(1), 20, false; "the carrier answering keeps it alive")] + #[test_case(Feed::Answers(2), 11, true; "another peer answering is not progress")] + #[test_case(Feed::Streams(1), 55, false; "the carrier streaming one long answer keeps it alive")] + #[test_case(Feed::Streams(1), 61, true; "the carrier streaming one answer past the bound stalls")] + #[test_case(Feed::Streams(2), 11, true; "another peer streaming is not progress")] + #[test_case(Feed::Chatters(1), 11, true; "the carrier sending only small messages stalls")] #[tokio::test(start_paused = true)] - async fn the_timeout_monitor_requeues_stalled_requests(streamer: Option, stalls: bool) { + async fn the_timeout_monitor_requeues_stalled_requests(feed: Feed, secs: u64, stalls: bool) { let net = broker().await; + let session = net.session.lock().await.clone(); + for n in 1..=2 { + let peer = ConnectedPeer::for_test(conn(n, 0)).await; + let admitted = + net.state.connected.admit(peer, &session, net.state.max_peers, None).await; + assert!(matches!(admitted, Admission::Joined)); + } net.state.broker.mark_on_wire(&get_cfilters(0), conn(1, 0)).await; - // Two gaps that each stay under the deadline but together exceed it. - for _ in 0..2 { - tokio::time::advance(REQUEST_TIMEOUT - TIMEOUT_CHECK).await; - tokio::task::yield_now().await; - if let Some(peer) = streamer { - let piece = NetworkMessage::CFilter(CFilter { - filter_type: 0, - block_hash: BlockHash::dummy(1), - filter: vec![0u8; 4], - }); - net.state - .inbound - .send(PeerEvent::Message(conn(peer, 0), piece.into())) - .expect("the pump is running"); + for _ in 0..secs { + tokio::time::advance(TIMEOUT_CHECK).await; + match feed { + Feed::Nothing => {} + Feed::Answers(peer) => { + let piece = NetworkMessage::CFilter(CFilter { + filter_type: 0, + block_hash: BlockHash::dummy(1), + filter: vec![0u8; 4], + }); + net.state + .inbound + .send(PeerEvent::Message(conn(peer, 0), piece.into())) + .expect("the pump is running"); + } + Feed::Streams(peer) => receive(&net, peer, 1000, true).await, + Feed::Chatters(peer) => receive(&net, peer, 100, false).await, } - tokio::task::yield_now().await; - tokio::task::yield_now().await; + settle().await; } - let want = if stalls { - "queued" - } else { - "on_wire" - }; - assert_eq!(net.state.broker.state_of(&RequestKey::CFilters(0)).await, Some(want)); - assert_eq!(net.state.broker.len(), stalls as usize); + // A stalled request is re-queued and may already be routed to peer 2. + let carrier = net.state.broker.carrier_of(&RequestKey::CFilters(0)).await; + assert_eq!(carrier == Some(conn(1, 0)), !stalls, "carrier {carrier:?}"); } /// A peer that was silent while idle is timed from the request it is given, and diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index c68a35692..5d63fa243 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -17,7 +17,7 @@ use dashcore::{ use futures::lock::Mutex; use rand::Rng; use std::net::SocketAddr; -use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; #[cfg(test)] @@ -137,6 +137,23 @@ impl Decoder for SizedCodec { } } +/// Frames messages and records whether the bytes of an unfinished one are buffered. +struct TrackedCodec { + codec: SizedCodec, + partial: Arc, +} + +impl Decoder for TrackedCodec { + type Item = (RawNetworkMessage, u64); + type Error = encode::Error; + + fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { + let frame = self.codec.decode(src); + self.partial.store(!src.is_empty(), Ordering::Relaxed); + frame + } +} + /// The socket's write half, framing each message for the connection's transport. struct PeerWriter { half: OwnedWriteHalf, @@ -172,7 +189,7 @@ impl PeerWriter { /// the peer's version packet. type LinkReader = Either, OwnedReadHalf>; -type PeerReader = FramedRead, SizedCodec>; +type PeerReader = FramedRead, TrackedCodec>; /// A TCP connection with its transport negotiated, ready for the version handshake. struct Link { @@ -304,6 +321,7 @@ pub struct ConnectedPeer { version: VersionMessage, lag_ms: AtomicU32, bytes: Arc, + partial: Arc, } pub struct DisconnectedPeer { @@ -346,6 +364,11 @@ impl ConnectedPeer { self.bytes.load(Ordering::Relaxed) } + /// Whether part of a message has arrived and the rest has not. + pub fn partial_message(&self) -> bool { + self.partial.load(Ordering::Relaxed) + } + pub fn in_flight_bytes(&self) -> u64 { self.handle.in_flight_bytes() } @@ -389,8 +412,17 @@ impl ConnectedPeer { version: build_version(id.addr(), "test"), lag_ms: AtomicU32::new(1), bytes: Arc::new(AtomicU64::new(0)), + partial: Arc::new(AtomicBool::new(false)), } } + + /// Counts `n` bytes as read from the socket, as the reader does, leaving a message + /// unfinished if `partial`. + #[cfg(test)] + pub(crate) fn receive_for_test(&self, n: u64, partial: bool) { + self.bytes.fetch_add(n, Ordering::Relaxed); + self.partial.store(partial, Ordering::Relaxed); + } } impl DisconnectedPeer { @@ -485,13 +517,17 @@ impl DisconnectedPeer { 1 }; let peer_bytes = Arc::new(AtomicU64::new(0)); + let partial = Arc::new(AtomicBool::new(false)); let mut reader = FramedRead::new( CountingReader { inner: reader, state: state.clone(), peer_bytes: peer_bytes.clone(), }, - codec, + TrackedCodec { + codec, + partial: partial.clone(), + }, ); handshake_send( @@ -614,6 +650,7 @@ impl DisconnectedPeer { version, lag_ms: AtomicU32::new(lag_ms), bytes: peer_bytes, + partial, }) } } @@ -756,6 +793,8 @@ async fn handshake_send(writer: &mut PeerWriter, payload: NetworkMessage) -> Net mod tests { use std::io::{Error as IoError, ErrorKind}; + use test_case::test_case; + use super::*; fn io_error(kind: ErrorKind) -> ProtocolError { @@ -793,4 +832,25 @@ mod tests { wire_agent(&format!("{}é{}", "a".repeat(MAX_USER_AGENT_LEN - 3), "b".repeat(10))); assert_eq!(straddling, format!("/{}/", "a".repeat(MAX_USER_AGENT_LEN - 3))); } + + /// Whether the codec reports an unfinished message after the first `n` bytes of two + /// 32-byte pings arrive. + #[test_case(1 => true; "part of a header")] + #[test_case(24 => true; "a header without its payload")] + #[test_case(32 => false; "a whole ping")] + #[test_case(40 => true; "a whole ping and part of the next")] + fn the_codec_tracks_an_unfinished_message(n: usize) -> bool { + let magic = Network::Regtest.magic(); + let ping = encode::serialize(&RawNetworkMessage { + magic, + payload: NetworkMessage::Ping(1), + }); + let mut codec = TrackedCodec { + codec: SizedCodec::V1, + partial: Arc::new(AtomicBool::new(false)), + }; + let mut src = BytesMut::from(&[ping.clone(), ping].concat()[..n]); + while codec.decode(&mut src).unwrap().is_some() {} + codec.partial.load(Ordering::Relaxed) + } } From ee580718e782378285486fe48235936b18bea9bd Mon Sep 17 00:00:00 2001 From: xdustinface Date: Wed, 11 Nov 2026 00:02:20 +1000 Subject: [PATCH 4/9] fix(dash-spv): kick one peer per shared stall When our own link stalls, every peer goes quiet at once. The timeout monitor judged each peer alone, so peers whose timeouts started a second apart crossed them one at a time and each looked like a lone stall. A testnet run lost four healthy peers in 14 s during a 40 s outage, and with one configured peer a 30 s outage kicked it and banned it for 10 minutes. The monitor now hands its decision to a `StallJudge`. A silent peer is blamed only once our link provably works: some peer, this one included, received bytes more than a timeout into its silence. Silence that no peer witnessed is held as a possible stall of our link, for at most `LINK_STALL_TIMEOUTS` (6) request timeouts, 60 s with the 10 s base timeout, enough for the 40 s outages seen on testnet. Past the bound the set is more likely dead than our link, so the peer is kicked anyway. A new peer's first byte counter proves nothing, since its handshake may predate the stall. Silent peers are kicked one per timeout, the one silent the longest first, so a stall of our link costs at most a single peer whatever share of the set it is. A replacement joining right after a kick does not unpace the next one. When the first peer receives again after a silence nobody witnessed, our link counts as back from that moment. Both the witness and the hold bound then count from the later of a peer's silence and that return, so a peer still recovering from TCP backoff gets a full timeout instead of being kicked as soon as another peer speaks. Bytes from an idle peer later on cannot restart a dead peer's timeout. `Broker::expire` reports when each culprit's stalled request went silent, so its silence is measured the same way as the owed-bytes check. A request past `ANSWER_TIMEOUTS` is blamed from its last answer, so the peer's own recent bytes witness our link and it is kicked rather than held. While the judge holds a peer, `NetworkState` flags our link as stalled. The flow controller then skips the tick in a peer's serving rate, which otherwise stayed dragged down for 30 s after the stall, and neither the flow controller nor the supervisor names an underserving peer from rates the stall skewed. --- dash-spv/src/network/broker.rs | 26 +- dash-spv/src/network/manager.rs | 451 ++++++++++++++++++++++++++------ 2 files changed, 382 insertions(+), 95 deletions(-) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index 31c227f85..6ac1dfb41 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -237,25 +237,31 @@ impl Broker { } } - /// Returns the peers to kick for a stalled request. A request stalls after a timeout - /// without progress, or after [`ANSWER_TIMEOUTS`] timeouts without an answer. A request - /// that already stalled [`KICKING_ATTEMPTS`] times is retried without blaming its peer. - pub async fn expire(&self, timeout: impl Fn(&PeerId) -> Duration) -> HashSet { + /// Returns the peers to kick for a stalled request, each with the earliest instant + /// its stalled requests are blamed from. A request stalls after a timeout without + /// progress, or after [`ANSWER_TIMEOUTS`] timeouts without an answer. A request that + /// already stalled [`KICKING_ATTEMPTS`] times is retried without blaming its peer. + pub async fn expire(&self, timeout: impl Fn(&PeerId) -> Duration) -> HashMap { let now = Instant::now(); - let mut culprits = HashSet::new(); + let mut culprits: HashMap = HashMap::new(); let mut unanswerable = HashSet::new(); for (key, state) in self.requests.lock().await.iter() { let ReqState::OnWire(o) = state else { continue; }; let timeout = timeout(&o.peer); - if now.duration_since(o.last_progress) <= timeout - && now.duration_since(o.last_answer) <= timeout * ANSWER_TIMEOUTS - { + let silent_since = if now.duration_since(o.last_progress) > timeout { + o.last_progress + } else if now.duration_since(o.last_answer) > timeout * ANSWER_TIMEOUTS { + // Blamed from the answer, not the recent bytes, so those bytes show our + // link works and the peer is not held as a link stall. + o.last_answer + } else { continue; - } + }; if o.attempts < KICKING_ATTEMPTS { - culprits.insert(o.peer); + let since = culprits.entry(o.peer).or_insert(silent_since); + *since = (*since).min(silent_since); } else { tracing::debug!( target: "dash_spv::network", diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index af9ad5da9..cebe31e04 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -2,7 +2,7 @@ use std::{ collections::{HashMap, HashSet, VecDeque}, net::SocketAddr, sync::{ - atomic::{AtomicU32, AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, Arc, }, time::Duration, @@ -64,6 +64,11 @@ const MAX_ADDR_PER_MESSAGE: usize = 1000; /// How often the timeout monitor scans the outstanding-request registry. const TIMEOUT_CHECK: Duration = Duration::from_secs(1); +/// How many request timeouts a silent peer is held while no peer proves our link +/// works, counted from its silence or from our link's return if later. With the 10 s +/// base timeout that is 60 s, enough for the 40 s outages seen on testnet. +const LINK_STALL_TIMEOUTS: u32 = 6; + use crate::error::{NetworkError, NetworkResult}; use crate::network::bans::{Ban, Bans}; use crate::network::broker::{Broker, RequestKey}; @@ -208,6 +213,8 @@ pub struct NetworkState { pub required_services: ServiceFlags, pub user_agent: String, pub transport_preference: TransportPreference, + /// Whether the timeout monitor sees our own link stalled. + link_stalled: AtomicBool, } pub struct PeerNetworkManager { @@ -257,6 +264,7 @@ impl PeerNetworkManager { required_services, user_agent: config.user_agent.clone(), transport_preference: config.transport_preference, + link_stalled: AtomicBool::new(false), }); let shutdown = CancellationToken::new(); @@ -610,19 +618,27 @@ impl PeerFlow { } } - /// One tick with the peer's cumulative bytes received, its bytes in flight now and - /// whether the client's own cap holds back work. Returns its new cap. - fn tick(&mut self, bytes: u64, in_flight: u64, link_full: bool) -> u64 { + /// One tick with the peer's cumulative bytes received, its bytes in flight now, + /// whether the client's own cap holds back work and whether our link is stalled. + /// Returns its new cap. + fn tick(&mut self, bytes: u64, in_flight: u64, link_full: bool, link_stalled: bool) -> u64 { // The router sends a peer work until it reaches its cap or the link cap binds. // Short of both, its rate shows how much it was asked for, not its bandwidth. let bound = in_flight > 0 && (in_flight >= self.limiter.cap as u64 || link_full); let cap = self.limiter.tick(bytes, in_flight, CAP_STEP_BYTES_PER_SEC); - if bound { + self.sample(self.limiter.rate, bound, link_stalled); + cap + } + + /// Folds `rate` into the serving rate if the peer had work. Ticks are skipped once the + /// timeout monitor sees our own link stalled, since such a stall says nothing about + /// the peer and would drag its average for 30 s. + fn sample(&mut self, rate: f64, has_work: bool, link_stalled: bool) { + if has_work && !link_stalled { self.serving_ticks += 1; let alpha = 1.0 / self.serving_ticks.min(SERVING_SAMPLE_TICKS) as f64; - self.serving_rate += alpha * (self.limiter.rate - self.serving_rate); + self.serving_rate += alpha * (rate - self.serving_rate); } - cap } /// Returns the peer's serving rate in bytes/s, 0 until measured. @@ -648,6 +664,7 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) peer_wake, broker, costs, + link_stalled, .. } = &*state; let mut global = ByteLimiter::new(MIN_CAP_BYTES, bytes.load(Ordering::Relaxed)); @@ -676,9 +693,10 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) let in_flights: Vec = g.iter().map(ConnectedPeer::in_flight_bytes).collect(); let total_in_flight: u64 = in_flights.iter().sum(); let link_full = total_in_flight >= global_cap.load(Ordering::Relaxed); + let stalled = link_stalled.load(Ordering::Relaxed); for (p, &in_flight) in g.iter().zip(&in_flights) { let flow = peers.entry(p.id()).or_insert_with(|| PeerFlow::new(p.bytes_read())); - p.set_cap(flow.tick(p.bytes_read(), in_flight, link_full)); + p.set_cap(flow.tick(p.bytes_read(), in_flight, link_full, stalled)); p.set_serving_rate(flow.serving_rate()); } @@ -742,8 +760,9 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) } // Only the transition wakes the supervisor, so a peer with no replacement - // is not re-probed every tick. - let underserving = underserving_peer(&g); + // is not re-probed every tick. A stall of our own link drags every rate down + // unevenly, so nobody is underserving during one. + let underserving = (!stalled).then(|| underserving_peer(&g)).flatten(); let handles: Vec = g.iter().map(|p| p.handle()).collect(); drop(g); if broker.len() == 0 { @@ -840,7 +859,8 @@ impl Supervisor { // Only look for a laggard once full: below cap every arrival is wanted anyway. // Each swap re-sends the work the old peer held, so at most one per cooldown. let cooled = self.last_swap.lock().await.is_none_or(|t| t.elapsed() >= SWAP_COOLDOWN); - let slow = (deficit == 0 && cooled) + let stalled = self.state.link_stalled.load(Ordering::Relaxed); + let slow = (deficit == 0 && cooled && !stalled) .then(|| underserving_peer(&peers)) .flatten() .map(|slow| slow.addr); @@ -1019,17 +1039,147 @@ fn spawn_peer_supervisor(state: Arc, shutdown: CancellationToken) ) } -/// Time requests out and evict the peers that stalled on them. -/// -/// A peer is a culprit when it has in-flight work and no bytes have arrived from it -/// for a full [`REQUEST_TIMEOUT`] since it was given that work, or when one of its -/// requests made no progress for that long. Liveness is per peer, not per request -/// age: a healthy peer draining a deep backlog keeps sending bytes even while its -/// oldest request waits. -/// -/// A culprit is kicked immediately and ALL its on-wire requests are re-queued for -/// other peers; leaving it connected would just route the freed work back to it. -/// The supervisor refills the set. +/// What the timeout monitor reads of a connected peer each tick. +struct Observed { + id: PeerId, + bytes: u64, + /// Whether part of a message has arrived and the rest has not. + partial: bool, + in_flight: u64, + timeout: Duration, +} + +/// What the timeout monitor knows of a connected peer. +struct Liveness { + bytes: u64, + /// When the peer last sent bytes or was idle. + owing_since: Instant, +} + +/// Decides which peers [`spawn_timeout_monitor`] kicks. A peer silent with work is held +/// while no peer proves our link works, but for at most [`LINK_STALL_TIMEOUTS`] +/// timeouts, and silent peers are kicked one per timeout, the one silent the longest first. +#[derive(Default)] +struct StallJudge { + progress: HashMap, + /// When some peer's bytes last grew. A new peer's first counter is not growth: its + /// handshake may predate a stall of our link, so it proves nothing about the link now. + last_heard: Option, + /// The last kick of a silent peer, which paces the next. + last_kick: Option, + /// The silent peers held at the last judgement because no peer proved our link works. + held: HashSet, + /// Whether some held peer has had no witness since its silence began, so the next + /// bytes from any peer show our link is back. + link_down: bool, + /// When some peer first received after our link went down. + link_back: Option, +} + +impl StallJudge { + /// Records the peers' byte counters and returns the peers that received bytes of a + /// message still arriving. Small unrelated messages arrive whole, so only these bytes + /// are progress on a request. + fn observe(&mut self, now: Instant, peers: &[Observed]) -> Vec { + self.progress.retain(|id, _| peers.iter().any(|p| p.id == *id)); + let mut streaming = Vec::new(); + for p in peers { + let entry = self.progress.entry(p.id).or_insert(Liveness { + bytes: p.bytes, + owing_since: now, + }); + let grew = p.bytes > entry.bytes; + if grew { + if self.link_down { + self.link_back = Some(now); + } + entry.bytes = p.bytes; + self.last_heard = Some(now); + if p.partial { + streaming.push(p.id); + } + } + // An idle peer owes no bytes, so its silence before it gets work + // must not count against its first request. + if grew || p.in_flight == 0 { + entry.owing_since = now; + } + } + streaming + } + + /// Whether our own link may be stalled: a silent peer is held for lack of a witness. + fn link_stalled(&self) -> bool { + !self.held.is_empty() + } + + /// Returns the peers to kick and how many culprits are spared. `culprits` maps the + /// peers of stalled requests to when their silence began. + fn judge( + &mut self, + now: Instant, + peers: &[Observed], + mut culprits: HashMap, + ) -> (HashSet, usize) { + self.held.clear(); + self.link_down = false; + // Judged on the peer's own counters, not just the registry: streaming in-flight + // bytes are freed on the peer separately, so a wedged peer can sit at its cap + // with no registry entries. Pending work plus a byte counter + // frozen for REQUEST_TIMEOUT means stuck, whatever the registry thinks. + for p in peers.iter().filter(|p| p.in_flight > 0) { + let Some(owing_since) = self.progress.get(&p.id).map(|l| l.owing_since) else { + continue; + }; + if now.duration_since(owing_since) > p.timeout { + let since = culprits.entry(p.id).or_insert(owing_since); + *since = (*since).min(owing_since); + } + } + if culprits.is_empty() { + return (HashSet::new(), 0); + } + + let mut silent: Vec<(Instant, &Observed)> = Vec::new(); + for p in peers { + let Some(&since) = culprits.get(&p.id) else { + continue; + }; + // Silence is blamed only once any peer, this one included, received a full + // timeout into it, which proves our link works. Silence before the link came + // back does not count, so the peer's own connection gets a timeout to resume. + let witnessed_from = + |from: Instant| self.last_heard.is_some_and(|heard| heard > from + p.timeout); + let blamed_from = self.link_back.map_or(since, |back| back.max(since)); + if witnessed_from(blamed_from) + || now.duration_since(blamed_from) > p.timeout * LINK_STALL_TIMEOUTS + { + silent.push((since, p)); + } else { + self.link_down |= !witnessed_from(since); + self.held.insert(p.id); + } + } + // A silent peer may still be our link failing it, so the one silent the longest + // goes, and one kick per timeout lets such a stall cost a single peer, whatever + // share of the set it is. + silent.sort_by_key(|&(since, p)| (since, p.id.addr())); + let due = silent.first().is_some_and(|(_, worst)| { + self.last_kick.is_none_or(|at| now.duration_since(at) > worst.timeout) + }); + if due { + self.last_kick = Some(now); + } + let mut spared = self.held.clone(); + spared.extend(silent.iter().skip(usize::from(due)).map(|(_, p)| p.id)); + culprits.retain(|id, _| !spared.contains(id)); + (culprits.into_keys().collect(), spared.len()) + } +} + +/// Every [`TIMEOUT_CHECK`], kicks the peers [`StallJudge`] finds stalled on requests +/// with no progress for a [`REQUEST_TIMEOUT`], re-queues all their on-wire requests for +/// other peers and wakes the supervisor to refill the set. fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) -> JoinHandle<()> { tokio::spawn(async move { let NetworkState { @@ -1038,11 +1188,11 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) peer_wake, max_peers, bans, + link_stalled, .. } = &*state; let mut ticker = tokio::time::interval(TIMEOUT_CHECK); - // Per-peer liveness: (bytes read, when the peer last sent bytes or was idle). - let mut progress: HashMap = HashMap::new(); + let mut judge = StallJudge::default(); loop { tokio::select! { _ = shutdown.cancelled() => break, @@ -1050,54 +1200,34 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) } let now = Instant::now(); - - // Judged on the peer's own counters, not just the registry: streaming in-flight - // bytes are freed on the peer separately, so a wedged peer can sit at its cap - // with no registry entries. Pending work plus a byte counter - // frozen for REQUEST_TIMEOUT means stuck, whatever the registry thinks. - let (mut culprits, streaming) = { - let peers = connected.read().await; - let live: HashSet = peers.iter().map(|p| p.id()).collect(); - progress.retain(|addr, _| live.contains(addr)); - let mut culprits = HashSet::new(); - let mut streaming = Vec::new(); - for peer in peers.iter() { - let addr = peer.id(); - let bytes = peer.bytes_read(); - let in_flight = peer.in_flight_bytes(); - let entry = progress.entry(addr).or_insert((bytes, now)); - let grew = bytes > entry.0; - // Small unrelated messages arrive whole, so only bytes of a message - // still arriving are progress on a request. - if grew && peer.partial_message() { - streaming.push(addr); - } - // An idle peer owes no bytes, so its silence before it gets work - // must not count against its first request. - if grew || in_flight == 0 { - *entry = (bytes, now); - } - if in_flight > 0 && now.duration_since(entry.1) > request_timeout(peer.lag_ms()) - { - culprits.insert(addr); - } - } - (culprits, streaming) - }; - broker.receiving(&streaming).await; - // A request that stopped making progress also condemns its peer: the - // per-peer byte check above sees unrelated traffic and judges it healthy. - let timeouts: HashMap = connected + let peers: Vec = connected .read() .await .iter() - .map(|p| (p.id(), request_timeout(p.lag_ms()))) + .map(|p| Observed { + id: p.id(), + bytes: p.bytes_read(), + partial: p.partial_message(), + in_flight: p.in_flight_bytes(), + timeout: request_timeout(p.lag_ms()), + }) .collect(); - culprits.extend( - broker.expire(|peer| timeouts.get(peer).copied().unwrap_or(REQUEST_TIMEOUT)).await, - ); - + broker.receiving(&judge.observe(now, &peers)).await; + // A request with no progress for a timeout also condemns its peer: the + // per-peer byte check sees unrelated traffic and judges it healthy. + let timeout = |id: &PeerId| { + peers.iter().find(|p| p.id == *id).map_or(REQUEST_TIMEOUT, |p| p.timeout) + }; + let (culprits, spared) = judge.judge(now, &peers, broker.expire(timeout).await); + link_stalled.store(judge.link_stalled(), Ordering::Relaxed); if culprits.is_empty() { + if spared > 0 { + tracing::debug!( + target: "dash_spv::network", + "request timeout: kicked none, spared {}", + spared, + ); + } continue; } @@ -1121,9 +1251,10 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) tracing::warn!( target: "dash_spv::network", - "request timeout: kicked {} peer(s) {:?}, retried {} request(s)", + "request timeout: kicked {} peer(s) {:?}, spared {}, retried {} request(s)", dropped, culprits, + spared, retried, ); broker.wake(); @@ -1393,7 +1524,7 @@ mod tests { fn tick(&mut self, delivered: u64, in_flight: u64, link_full: bool) { self.bytes += delivered; - self.cap = self.flow.tick(self.bytes, in_flight, link_full); + self.cap = self.flow.tick(self.bytes, in_flight, link_full, false); } /// Bulk download at `rate` bytes/s. The router keeps the peer at its own cap, or @@ -1503,7 +1634,14 @@ mod tests { let mut late = None; match departure { Departure::Drop => disconnect_all(), - Departure::Kick => tokio::time::advance(REQUEST_TIMEOUT + TIMEOUT_CHECK * 2).await, + Departure::Kick => { + // Silent past the bound, the peers go one per timeout. + for _ in 0..2 { + tokio::time::advance(REQUEST_TIMEOUT * LINK_STALL_TIMEOUTS + TIMEOUT_CHECK) + .await; + settle().await; + } + } Departure::Stop => { net.stop().await; // A handshake that completed while `stop` ran. @@ -1579,7 +1717,7 @@ mod tests { Chatters(u8), } - /// Peer 1 carries a request for `secs` seconds beside idle peer 2. + /// Peer 1 carries a request for `secs` seconds while idle peer 2 keeps receiving. #[test_case(Feed::Nothing, 11, true; "a silent request")] #[test_case(Feed::Answers(1), 20, false; "the carrier answering keeps it alive")] #[test_case(Feed::Answers(2), 11, true; "another peer answering is not progress")] @@ -1601,6 +1739,7 @@ mod tests { for _ in 0..secs { tokio::time::advance(TIMEOUT_CHECK).await; + receive(&net, 2, 100, false).await; match feed { Feed::Nothing => {} Feed::Answers(peer) => { @@ -1631,27 +1770,169 @@ mod tests { async fn the_timeout_monitor_times_an_idle_peer_from_its_first_request() { let net = broker().await; let session = net.session.lock().await.clone(); - let id = conn(1, 0); - let peer = ConnectedPeer::for_test(id).await; + let peer = ConnectedPeer::for_test(conn(1, 0)).await; let handle = peer.handle(); - let admitted = net.state.connected.admit(peer, &session, net.state.max_peers, None).await; - assert!(matches!(admitted, Admission::Joined)); - - for _ in 0..3 { - tokio::time::advance(REQUEST_TIMEOUT).await; - tokio::task::yield_now().await; + for peer in [peer, ConnectedPeer::for_test(conn(2, 0)).await] { + let admitted = + net.state.connected.admit(peer, &session, net.state.max_peers, None).await; + assert!(matches!(admitted, Admission::Joined)); } + + // The second peer keeps receiving, so a stall is not blamed on our link. + let run = |secs: u64| { + let net = &net; + async move { + for _ in 0..secs { + tokio::time::advance(TIMEOUT_CHECK).await; + receive(net, 2, 100, false).await; + settle().await; + } + } + }; + run(REQUEST_TIMEOUT.as_secs() * 3).await; // Only the charge matters here: the loopback has no reader to answer it. let _ = handle.send(&get_cfilters(0)).await; assert!(handle.in_flight_bytes() > 0); - tokio::time::advance(TIMEOUT_CHECK * 2).await; - tokio::task::yield_now().await; - assert_eq!(net.connected_count().await, 1, "kicked right after its first request"); + run(2).await; + assert_eq!(net.connected_count().await, 2, "kicked right after its first request"); + + run(REQUEST_TIMEOUT.as_secs()).await; + assert_eq!(net.connected_count().await, 1, "a peer silent with work stays connected"); + } - tokio::time::advance(REQUEST_TIMEOUT).await; - tokio::task::yield_now().await; - assert_eq!(net.connected_count().await, 0, "a peer silent with work stays connected"); + /// What a peer of the set does, second by second. + #[derive(Clone, Copy)] + enum Role { + /// Holds a request the registry tracks and receives bytes until this second. + Registered(u64), + /// Owes bytes the registry does not track and receives bytes until this second. + Charged(u64), + /// Idle, and receives bytes from this second on. + Delivers(u64), + /// Connects at this second, idle, and receives nothing. + Joins(u64), + /// Owes bytes, receives until the first second and again from the second. + Resumes(u64, u64), + /// Idle, and receives bytes every this many seconds. + Whispers(u64), + /// Holds a request the registry tracks and receives small whole messages from + /// this second on. + Chatters(u64), + } + + /// Peer `n + 1` plays `roles[n]` for `secs` seconds. `link_stalled` is whether the + /// judge then suspects our own link. + #[test_case(&[Role::Registered(0); 3], 11, &[1, 2, 3], true; "a stall of every peer at once is our link")] + #[test_case(&[Role::Registered(0); 3], 41, &[1, 2, 3], true; "a 40 s silence of every peer kicks nobody")] + #[test_case(&[Role::Registered(0); 3], 71, &[2, 3], false; "every peer silent past the bound loses the one silent the longest")] + #[test_case(&[Role::Registered(0); 3], 72, &[3], false; "every peer silent past the bound loses one per timeout")] + #[test_case(&[Role::Charged(0); 3], 25, &[1, 2, 3], true; "a long stall of every peer is still our link")] + #[test_case(&[Role::Registered(1), Role::Registered(2), Role::Registered(3)], 25, &[1, 2, 3], true; "peers going silent a second apart in an outage stay")] + #[test_case(&[Role::Charged(1), Role::Charged(2), Role::Charged(3)], 25, &[1, 2, 3], true; "peers owing bytes and going silent a second apart stay")] + #[test_case(&[Role::Delivers(0), Role::Registered(0), Role::Delivers(0)], 11, &[1, 3], false; "a lone silent request among delivering peers is kicked")] + #[test_case(&[Role::Delivers(0), Role::Charged(0), Role::Delivers(0)], 12, &[1, 3], false; "a lone peer owing bytes among delivering peers is kicked")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(0)], 11, &[2, 3], false; "two stalls beside a delivering peer kick one")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(0)], 22, &[2], false; "the second stall goes a timeout later")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(1)], 12, &[2, 3], false; "stalls a second apart beside a delivering peer go one per timeout")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(2)], 14, &[2, 3], false; "stalls two seconds apart beside a delivering peer go one per timeout")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(1), Role::Joins(11)], 12, &[2, 3, 4], false; "a replacement joining does not unpace the next stall")] + #[test_case(&[Role::Registered(0), Role::Delivers(13), Role::Registered(1)], 24, &[2, 3], false; "a late witness kicks the peer silent the longest a timeout later")] + #[test_case(&[Role::Resumes(5, 20), Role::Resumes(5, 20), Role::Charged(5)], 25, &[1, 2, 3], true; "a peer still silent when the link is back gets a timeout")] + #[test_case(&[Role::Resumes(5, 20), Role::Resumes(5, 20), Role::Charged(5)], 31, &[1, 2], false; "a peer silent a timeout after the link is back is kicked")] + #[test_case(&[Role::Charged(0), Role::Whispers(3)], 13, &[2], false; "a lone peer owing bytes beside an idle one is no link stall")] + #[test_case(&[Role::Registered(0)], 41, &[1], true; "a sole silent peer stays through a 40 s outage")] + #[test_case(&[Role::Registered(0)], 61, &[], false; "a sole silent peer is kicked past the bound")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Joins(12)], 13, &[1, 2, 3], true; "a peer joining during an outage proves nothing")] + #[test_case(&[Role::Charged(0), Role::Delivers(30)], 15, &[1, 2], true; "a lone peer owing bytes beside a quiet idle one is held")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 35, &[1, 2, 3], true; "an idle peer receiving first after an outage gives the held peers a timeout")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 36, &[2, 3], false; "held peers silent a timeout after an idle peer receives go")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 47, &[3], false; "held peers silent after an idle peer receives go one per timeout")] + #[test_case(&[Role::Charged(0), Role::Whispers(15)], 30, &[2], false; "an idle peer's later bytes do not restart a held peer's timeout")] + #[test_case(&[Role::Resumes(0, 55), Role::Charged(0)], 65, &[1, 2], true; "a peer silent through a 55 s outage gets a timeout after it")] + #[test_case(&[Role::Resumes(0, 55), Role::Charged(0)], 66, &[1], false; "a peer silent a timeout after a 55 s outage is kicked")] + #[test_case(&[Role::Chatters(0)], 11, &[], false; "a sole peer receiving only small messages is kicked")] + #[test_case(&[Role::Chatters(25); 3], 36, &[2, 3], false; "peers receiving only small messages after an outage go one per timeout")] + fn the_stall_judge_kicks_one_peer_per_shared_stall( + roles: &[Role], + secs: u64, + kept: &[u8], + link_stalled: bool, + ) { + // Every received byte but a chatter's is part of an answer still arriving. + let peer = |n: u8, role: Role| Observed { + id: conn(n, 0), + bytes: 0, + partial: !matches!(role, Role::Chatters(_)), + in_flight: matches!(role, Role::Charged(_) | Role::Resumes(..)).into(), + timeout: REQUEST_TIMEOUT, + }; + let start = Instant::now(); + let mut peers: Vec = (1..) + .zip(roles) + .filter(|(_, role)| !matches!(role, Role::Joins(_))) + .map(|(n, &role)| peer(n, role)) + .collect(); + // The last progress of each registered request. + let mut requests: HashMap = (1..) + .zip(roles) + .filter(|(_, role)| matches!(role, Role::Registered(_) | Role::Chatters(_))) + .map(|(n, _)| (conn(n, 0), start)) + .collect(); + let mut judge = StallJudge::default(); + + for second in 1..=secs { + for (n, &role) in (1..).zip(roles) { + let receives = match role { + Role::Registered(until) | Role::Charged(until) => second <= until, + Role::Delivers(from) | Role::Chatters(from) => second >= from, + Role::Resumes(until, from) => second <= until || second >= from, + Role::Whispers(period) => second.is_multiple_of(period), + Role::Joins(at) => { + if second == at { + peers.push(peer(n, role)); + } + false + } + }; + if let Some(p) = peers.iter_mut().find(|p| receives && p.id == conn(n, 0)) { + p.bytes += 100; + } + } + let now = start + Duration::from_secs(second); + for id in judge.observe(now, &peers) { + if let Some(progress) = requests.get_mut(&id) { + *progress = now; + } + } + let stalled = requests + .iter() + .filter(|(_, &progress)| now.duration_since(progress) > REQUEST_TIMEOUT) + .map(|(&id, &progress)| (id, progress)) + .collect(); + let (kicked, _) = judge.judge(now, &peers, stalled); + assert!(kicked.len() <= 1, "second {second} kicked {kicked:?}"); + peers.retain(|p| !kicked.contains(&p.id)); + requests.retain(|id, _| !kicked.contains(id)); + } + + let left: Vec = peers.iter().map(|p| p.id).collect(); + let kept: Vec = kept.iter().map(|&n| conn(n, 0)).collect(); + assert_eq!(left, kept); + assert_eq!(judge.link_stalled(), link_stalled); + } + + /// A peer serves 1 MB/s with work for three ticks, then nothing for one. + #[test_case(false, true => 750_000; "a stall of its own drags its serving rate")] + #[test_case(true, true => 1_000_000; "a stall of our link does not")] + #[test_case(false, false => 1_000_000; "idle ticks do not")] + fn the_serving_rate_averages_ticks_with_work(link_stalled: bool, has_work: bool) -> u64 { + let mut flow = PeerFlow::new(0); + for _ in 0..3 { + flow.sample(1e6, true, false); + } + flow.sample(0.0, has_work, link_stalled); + flow.serving_rate as u64 } #[test_case(&[&[MessageType::Inv], &[MessageType::Inv]], 0, &[1, 1]; "every subscriber of the type")] From c07fa6651fb15e6f20d0e324d1b42ab1ceb06833 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Wed, 11 Nov 2026 00:02:20 +1000 Subject: [PATCH 5/9] test(dash-spv): keep a peer frozen mid-sync connected A dashd test freezes dashd with `SIGSTOP` for 30 s in the middle of a sync, three request timeouts but under the bound for holding a silent peer, then resumes it with `SIGCONT`. Its socket stays open, so the client sees a silent connection, as when our own link stalls. The test asserts the client never drops its only peer and that sync completes. Against the old timeout monitor the peer is kicked about 10 s into the freeze. `DashCoreNode::set_paused` sends the signals. The test is unix only, since Windows has no equivalent. --- dash-spv/src/test_utils/node.rs | 21 +++++++++ dash-spv/tests/dashd_sync/main.rs | 3 ++ dash-spv/tests/dashd_sync/tests_stall.rs | 55 ++++++++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 dash-spv/tests/dashd_sync/tests_stall.rs diff --git a/dash-spv/src/test_utils/node.rs b/dash-spv/src/test_utils/node.rs index e49c1d82c..cdf81c0e0 100644 --- a/dash-spv/src/test_utils/node.rs +++ b/dash-spv/src/test_utils/node.rs @@ -16,6 +16,8 @@ use std::process::ExitStatus; use std::sync::atomic::{AtomicU16, Ordering}; use std::time::Duration; use tokio::process::Child; +#[cfg(unix)] +use tokio::process::Command; use tokio::time::{sleep, timeout}; use super::fs_helpers::{copy_dir, copy_to_retain_dir}; @@ -683,6 +685,25 @@ impl DashCoreNode { tracing::info!("Disconnected peer {}", addr); } + /// Freezes dashd with `SIGSTOP`, or resumes it with `SIGCONT`. Its sockets stay open, so + /// its peers see a silent connection rather than a disconnect. + #[cfg(unix)] + pub async fn set_paused(&self, paused: bool) { + let pid = self.process.as_ref().and_then(Child::id).expect("dashd is not running"); + let signal = if paused { + "-STOP" + } else { + "-CONT" + }; + let status = Command::new("kill") + .args([signal, &pid.to_string()]) + .status() + .await + .expect("failed to run kill"); + assert!(status.success(), "kill {signal} {pid} failed: {status}"); + tracing::info!("Set paused={} on dashd", paused); + } + /// Enable or disable all P2P network activity on this node. pub fn set_network_active(&self, active: bool) { let client = self.rpc_client(); diff --git a/dash-spv/tests/dashd_sync/main.rs b/dash-spv/tests/dashd_sync/main.rs index a22afc967..52ba1e26f 100644 --- a/dash-spv/tests/dashd_sync/main.rs +++ b/dash-spv/tests/dashd_sync/main.rs @@ -11,4 +11,7 @@ mod tests_mempool; mod tests_multi_wallet; mod tests_node_start; mod tests_restart; +// SIGSTOP freezes dashd, and Windows has no equivalent. +#[cfg(unix)] +mod tests_stall; mod tests_transaction; diff --git a/dash-spv/tests/dashd_sync/tests_stall.rs b/dash-spv/tests/dashd_sync/tests_stall.rs new file mode 100644 index 000000000..9e75ad6b4 --- /dev/null +++ b/dash-spv/tests/dashd_sync/tests_stall.rs @@ -0,0 +1,55 @@ +use std::time::Duration; + +use dash_spv::network::NetworkEvent; +use dash_spv::test_utils::{TestChain, SYNC_TIMEOUT}; +use dash_spv::SyncEvent; +use tokio::sync::broadcast::error::TryRecvError; + +use super::helpers::{is_progress_event, wait_for_sync}; +use super::setup::TestContext; + +/// Verify a peer that goes silent mid-sync, as when our own link stalls, stays connected. +/// +/// `SIGSTOP` freezes dashd with its socket open for well over a request timeout, but +/// under the bound for holding a silent peer. The client must keep its only peer through +/// the freeze, and sync must complete once dashd resumes. +#[tokio::test] +async fn test_sync_keeps_a_peer_frozen_mid_sync() { + let Some(ctx) = TestContext::new(TestChain::Full).await else { + return; + }; + let mut client_handle = ctx.spawn_new_client().await; + + let mut progress_events = 0; + tokio::time::timeout(SYNC_TIMEOUT, async { + while progress_events < 5 { + match client_handle.sync_event_receiver.recv().await { + Ok(ref event) if is_progress_event(event) => progress_events += 1, + Ok(SyncEvent::SyncComplete { + .. + }) => panic!("sync completed before dashd could be frozen"), + Ok(_) => {} + Err(e) => panic!("sync events lost: {e}"), + } + } + }) + .await + .expect("Timeout waiting for sync progress"); + + ctx.dashd.node.set_paused(true).await; + tokio::time::sleep(Duration::from_secs(30)).await; + ctx.dashd.node.set_paused(false).await; + + wait_for_sync(&mut client_handle.progress_receiver, ctx.dashd.initial_height).await; + loop { + match client_handle.network_event_receiver.try_recv() { + Ok(NetworkEvent::PeerDisconnected(addr)) => panic!("peer {addr} was dropped"), + Ok(_) => {} + Err(TryRecvError::Empty) => break, + Err(e) => panic!("network events lost: {e}"), + } + } + + client_handle.stop().await; + ctx.assert_synced(&client_handle.client.progress().await).await; +} From 9607931730c4e531bbd9fc07492ffb64192b9f8d Mon Sep 17 00:00:00 2001 From: xdustinface Date: Wed, 11 Nov 2026 00:04:05 +1000 Subject: [PATCH 6/9] refactor(dash-spv): keep only a flag of held peers `StallJudge::held` collected the peers held for a possible stall of our link, but it was only read to check whether it was empty and to count the spared culprits. The judge now builds the kick set directly, from the culprits that already left plus the peer silent the longest when a kick is due, counts the held peers while judging, and keeps a `link_stalled` flag instead of the set. The kicks and the flag are unchanged. --- dash-spv/src/network/manager.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index cebe31e04..2f6c51cd2 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1067,8 +1067,9 @@ struct StallJudge { last_heard: Option, /// The last kick of a silent peer, which paces the next. last_kick: Option, - /// The silent peers held at the last judgement because no peer proved our link works. - held: HashSet, + /// Whether our own link may be stalled: the last judgement held a silent peer because + /// no peer proved our link works. + link_stalled: bool, /// Whether some held peer has had no witness since its silence began, so the next /// bytes from any peer show our link is back. link_down: bool, @@ -1108,11 +1109,6 @@ impl StallJudge { streaming } - /// Whether our own link may be stalled: a silent peer is held for lack of a witness. - fn link_stalled(&self) -> bool { - !self.held.is_empty() - } - /// Returns the peers to kick and how many culprits are spared. `culprits` maps the /// peers of stalled requests to when their silence began. fn judge( @@ -1121,7 +1117,7 @@ impl StallJudge { peers: &[Observed], mut culprits: HashMap, ) -> (HashSet, usize) { - self.held.clear(); + self.link_stalled = false; self.link_down = false; // Judged on the peer's own counters, not just the registry: streaming in-flight // bytes are freed on the peer separately, so a wedged peer can sit at its cap @@ -1140,6 +1136,7 @@ impl StallJudge { return (HashSet::new(), 0); } + let mut held = 0; let mut silent: Vec<(Instant, &Observed)> = Vec::new(); for p in peers { let Some(&since) = culprits.get(&p.id) else { @@ -1157,9 +1154,10 @@ impl StallJudge { silent.push((since, p)); } else { self.link_down |= !witnessed_from(since); - self.held.insert(p.id); + held += 1; } } + self.link_stalled = held > 0; // A silent peer may still be our link failing it, so the one silent the longest // goes, and one kick per timeout lets such a stall cost a single peer, whatever // share of the set it is. @@ -1170,10 +1168,11 @@ impl StallJudge { if due { self.last_kick = Some(now); } - let mut spared = self.held.clone(); - spared.extend(silent.iter().skip(usize::from(due)).map(|(_, p)| p.id)); - culprits.retain(|id, _| !spared.contains(id)); - (culprits.into_keys().collect(), spared.len()) + // A culprit that already left is not spared, so its requests are still re-queued. + let mut kicked: HashSet = + culprits.into_keys().filter(|id| peers.iter().all(|p| p.id != *id)).collect(); + kicked.extend(silent.first().filter(|_| due).map(|(_, p)| p.id)); + (kicked, held + silent.len() - usize::from(due)) } } @@ -1219,7 +1218,7 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) peers.iter().find(|p| p.id == *id).map_or(REQUEST_TIMEOUT, |p| p.timeout) }; let (culprits, spared) = judge.judge(now, &peers, broker.expire(timeout).await); - link_stalled.store(judge.link_stalled(), Ordering::Relaxed); + link_stalled.store(judge.link_stalled, Ordering::Relaxed); if culprits.is_empty() { if spared > 0 { tracing::debug!( @@ -1919,7 +1918,7 @@ mod tests { let left: Vec = peers.iter().map(|p| p.id).collect(); let kept: Vec = kept.iter().map(|&n| conn(n, 0)).collect(); assert_eq!(left, kept); - assert_eq!(judge.link_stalled(), link_stalled); + assert_eq!(judge.link_stalled, link_stalled); } /// A peer serves 1 MB/s with work for three ticks, then nothing for one. From 74bfd80fdd6e9066ca908d63ee15a460582084b6 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Wed, 11 Nov 2026 00:04:14 +1000 Subject: [PATCH 7/9] refactor(dash-spv): name the judge's `Liveness` map `StallJudge::progress` holds each peer's `Liveness`, and "progress" already means request progress in the broker and the monitor. The field is now `liveness`. --- dash-spv/src/network/manager.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 2f6c51cd2..774f93e72 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1061,7 +1061,7 @@ struct Liveness { /// timeouts, and silent peers are kicked one per timeout, the one silent the longest first. #[derive(Default)] struct StallJudge { - progress: HashMap, + liveness: HashMap, /// When some peer's bytes last grew. A new peer's first counter is not growth: its /// handshake may predate a stall of our link, so it proves nothing about the link now. last_heard: Option, @@ -1082,10 +1082,10 @@ impl StallJudge { /// message still arriving. Small unrelated messages arrive whole, so only these bytes /// are progress on a request. fn observe(&mut self, now: Instant, peers: &[Observed]) -> Vec { - self.progress.retain(|id, _| peers.iter().any(|p| p.id == *id)); + self.liveness.retain(|id, _| peers.iter().any(|p| p.id == *id)); let mut streaming = Vec::new(); for p in peers { - let entry = self.progress.entry(p.id).or_insert(Liveness { + let entry = self.liveness.entry(p.id).or_insert(Liveness { bytes: p.bytes, owing_since: now, }); @@ -1124,7 +1124,7 @@ impl StallJudge { // with no registry entries. Pending work plus a byte counter // frozen for REQUEST_TIMEOUT means stuck, whatever the registry thinks. for p in peers.iter().filter(|p| p.in_flight > 0) { - let Some(owing_since) = self.progress.get(&p.id).map(|l| l.owing_since) else { + let Some(owing_since) = self.liveness.get(&p.id).map(|l| l.owing_since) else { continue; }; if now.duration_since(owing_since) > p.timeout { From e89874deb40af7c20f274297e6d2b747f0dd50a3 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Wed, 11 Nov 2026 00:04:20 +1000 Subject: [PATCH 8/9] test(dash-spv): clarify the `Role::Resumes` doc The doc read "until the first second and again from the second", which mixes the ordinal with the time unit. It now names both fields as times in seconds. --- dash-spv/src/network/manager.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 774f93e72..bcca69cf1 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1811,7 +1811,8 @@ mod tests { Delivers(u64), /// Connects at this second, idle, and receives nothing. Joins(u64), - /// Owes bytes, receives until the first second and again from the second. + /// Owes bytes. Receives until the time in the first field and again from the time + /// in the second, both in seconds. Resumes(u64, u64), /// Idle, and receives bytes every this many seconds. Whispers(u64), From bb5a0d014317e39383409e771258f4eaed4f1429 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Sun, 8 Nov 2026 17:48:25 +1000 Subject: [PATCH 9/9] fix(dash-spv): settle filter requests on network arrival A `CFilters` or `CfHeaders` key stayed on the wire until its serial manager processed the reply, up to a million messages later. A kicked peer then had its delivered batches sent again, and a busy manager read as a stalled peer. - `CFilters` requests are keyed by stop hash, so the last `cfilter` of a reply identifies its request without a scan. - The pump settles the key when the reply arrives. Every way a manager drops a delivered batch releases it: a reply that leaves its batch incomplete, a filter that resolves to no stored header, a batch thrown away by a rescan and a `cfheaders` reply that matches nothing. - A batch rejected by verification or lost to a storage error is requested again instead of waited on forever. - Filters are stored in height order, so the lowest batch not yet delivered gets a second copy from the peer with the shortest queue once its peer answered nothing for `HEDGE_AFTER`, at most once per request and past the peers' caps. - Dropping a peer clears the hedges it held, so a request whose second copy was lost with its peer can be hedged again. --- dash-spv/src/network/broker.rs | 227 ++++++++++++++---- dash-spv/src/network/manager.rs | 72 +++++- dash-spv/src/sync/filter_headers/manager.rs | 25 +- .../src/sync/filter_headers/sync_manager.rs | 6 +- dash-spv/src/sync/filters/manager.rs | 183 ++++++++++---- dash-spv/src/sync/filters/pipeline.rs | 46 +++- dash-spv/src/sync/filters/sync_manager.rs | 16 +- dash-spv/src/test_utils/network.rs | 4 +- 8 files changed, 456 insertions(+), 123 deletions(-) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index 6ac1dfb41..695d31d5b 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -29,8 +29,8 @@ pub enum RequestKey { Headers(BlockHash), /// Stop hash. CfHeaders(BlockHash), - /// Start height. - CFilters(u32), + /// Stop hash. + CFilters(BlockHash), MnListDiff(BlockHash), Block(BlockHash), } @@ -41,7 +41,7 @@ pub fn request_keys(msg: &NetworkMessage) -> Vec { m.locator_hashes.first().map(|h| RequestKey::Headers(*h)).into_iter().collect() } NetworkMessage::GetCFHeaders(m) => vec![RequestKey::CfHeaders(m.stop_hash)], - NetworkMessage::GetCFilters(m) => vec![RequestKey::CFilters(m.start_height)], + NetworkMessage::GetCFilters(m) => vec![RequestKey::CFilters(m.stop_hash)], NetworkMessage::GetMnListD(m) => vec![RequestKey::MnListDiff(m.block_hash)], NetworkMessage::GetData(inv) => inv .iter() @@ -58,7 +58,7 @@ enum ReqState { /// Attempts so far. Queued(u32), OnWire(Box), - /// The block was received and waits for its manager to release the key. + /// The whole reply was received and waits for its manager to release the key. Arrived, } @@ -174,7 +174,16 @@ impl Broker { n } + /// Re-queues the requests of the `gone` peers. A request they only hedged stays with its + /// peer and may be hedged again, since the copy is lost. pub async fn requeue_from(&self, gone: &HashSet) -> usize { + for state in self.requests.lock().await.values_mut() { + if let ReqState::OnWire(o) = state { + if o.hedge.is_some_and(|hedge| gone.contains(&hedge)) { + o.hedge = None; + } + } + } self.requeue(|_, o| gone.contains(&o.peer)).await } @@ -191,12 +200,12 @@ impl Broker { } } - /// Settled on arrival, not when processed: blocks wait in height order, and that - /// wait is not the peer's. The first copy of a hedged block settles it. A request - /// put back in the queue is settled by a copy from any peer, and its message dropped. - /// The key stays registered until released, so a re-declare before then sends nothing. - pub async fn block_arrived(&self, peer: PeerId, hash: BlockHash) { - let key = RequestKey::Block(hash); + /// Settles `key` on arrival of its whole reply from `peer`, not when processed, since + /// the wait for a manager is not the peer's. The first copy of a hedged request settles + /// it. A request put back in the queue is settled by a copy from any peer, and its message + /// dropped. The key stays registered until released, so a re-declare before then sends + /// nothing, and a manager that cannot use the reply must release the key. + pub async fn arrived(&self, peer: PeerId, key: RequestKey) { let mut reqs = self.requests.lock().await; match reqs.get(&key) { Some(ReqState::OnWire(o)) if o.peer == peer || o.hedge == Some(peer) => {} @@ -339,7 +348,7 @@ impl Broker { let attempts = match slot { ReqState::Queued(attempts) => *attempts, ReqState::OnWire(o) => o.attempts, - // The block of a requeued request can arrive while it is resent. + // The reply to a requeued request can arrive while it is resent. ReqState::Arrived => continue, }; *slot = ReqState::OnWire(Box::new(OnWire { @@ -374,15 +383,7 @@ impl Broker { pub async fn hedge(&self, peers: &[PeerHandle]) -> Option { let (key, msg, height, peer) = { let reqs = self.requests.lock().await; - let (key, o) = reqs - .iter() - .filter_map(|(key, state)| match state { - ReqState::OnWire(o) if matches!(key, RequestKey::Block(_)) => { - Some((key, o.as_ref())) - } - _ => None, - }) - .min_by_key(|(_, o)| o.height)?; + let (key, o) = lowest_on_wire(&reqs, |key| matches!(key, RequestKey::Block(_)))?; if o.hedge.is_some() || o.last_answer.elapsed() < HEDGE_AFTER { return None; } @@ -392,6 +393,39 @@ impl Broker { .max_by_key(|p| p.cap().saturating_sub(p.in_flight_bytes()))?; (key.clone(), o.msg.clone(), o.height, peer.clone()) }; + self.send_hedge(key, msg, height, peer).await + } + + /// Asks a second peer for the lowest filter batch not yet delivered once its peer has + /// answered nothing for [`HEDGE_AFTER`], and returns its height. Filters are stored in + /// height order, so that batch holds back everything above it however much is downloaded. + /// One queued lower goes out next, so it is no reason to hedge the batch above it. + pub async fn hedge_filters(&self, peers: &[PeerHandle]) -> Option { + let (key, msg, height, peer) = { + let reqs = self.requests.lock().await; + let (key, o) = lowest_on_wire(&reqs, |key| matches!(key, RequestKey::CFilters(_)))?; + if o.hedge.is_some() + || o.last_answer.elapsed() < HEDGE_AFTER + || self.queue.lowest_height(MsgClass::CFilters).await.is_some_and(|h| h < o.height) + { + return None; + } + // Bulk download keeps every peer at its cap, so a cap would rule out all of them. + // Like a header request, the second copy goes where it is answered soonest. + let others = peers.iter().filter(|p| p.id() != o.peer && !p.is_closed()); + let peer = choose_peer(others, MsgClass::Headers)?; + (key.clone(), o.msg.clone(), o.height, peer.clone()) + }; + self.send_hedge(key, msg, height, peer).await + } + + async fn send_hedge( + &self, + key: RequestKey, + msg: NetworkMessage, + height: u32, + peer: PeerHandle, + ) -> Option { peer.send(&msg).await.ok()?; match self.requests.lock().await.get_mut(&key) { Some(ReqState::OnWire(o)) if o.hedge.is_none() => { @@ -437,6 +471,19 @@ impl Broker { } } +/// Returns the on-wire request of the lowest height among the keys `wanted` selects. +fn lowest_on_wire( + reqs: &HashMap, + wanted: impl Fn(&RequestKey) -> bool, +) -> Option<(&RequestKey, &OnWire)> { + reqs.iter() + .filter_map(|(key, state)| match state { + ReqState::OnWire(o) if wanted(key) => Some((key, o.as_ref())), + _ => None, + }) + .min_by_key(|(_, o)| o.height) +} + #[derive(Clone, Copy, PartialEq, Eq, Debug)] enum MsgClass { Other, @@ -536,6 +583,10 @@ impl MsgQueue { } } + async fn lowest_height(&self, class: MsgClass) -> Option { + self.queue(class).lock().await.first_key_value().map(|(&(height, _), _)| height) + } + async fn pop_n(&self, n: usize) -> Vec<(u32, NetworkMessage)> { self.pop_guaranteed(n as u64, &[0; MSG_CLASSES], |_| 1).await } @@ -630,7 +681,7 @@ mod tests { NetworkMessage::GetCFilters(GetCFilters { filter_type: 0, start_height: start, - stop_hash: BlockHash::dummy(9), + stop_hash: BlockHash::dummy(start), }) } @@ -673,7 +724,7 @@ mod tests { #[test_case(get_headers(1) => vec![RequestKey::Headers(BlockHash::dummy(1))]; "getheaders")] #[test_case(get_cfheaders(2) => vec![RequestKey::CfHeaders(BlockHash::dummy(2))]; "getcfheaders")] - #[test_case(get_cfilters(100) => vec![RequestKey::CFilters(100)]; "getcfilters")] + #[test_case(get_cfilters(100) => vec![RequestKey::CFilters(BlockHash::dummy(100))]; "getcfilters")] #[test_case(get_mnlistdiff(3) => vec![RequestKey::MnListDiff(BlockHash::dummy(4))]; "getmnlistd")] #[test_case(NetworkMessage::GetData(vec![block(1), tx(7), block(2)]) => vec![RequestKey::Block(BlockHash::dummy(1)), RequestKey::Block(BlockHash::dummy(2))]; @@ -707,7 +758,10 @@ mod tests { } else { "on_wire" }; - assert_eq!(broker.state_of(&RequestKey::CFilters(height)).await, Some(want)); + assert_eq!( + broker.state_of(&RequestKey::CFilters(BlockHash::dummy(height))).await, + Some(want) + ); } } @@ -734,13 +788,16 @@ mod tests { broker.mark_on_wire(&get_cfilters(0), peer).await; broker.requeue_from(&HashSet::from([peer])).await; broker.queue.pop_n(1).await; - assert_eq!(broker.state_of(&RequestKey::CFilters(0)).await, Some("queued")); + assert_eq!( + broker.state_of(&RequestKey::CFilters(BlockHash::dummy(0))).await, + Some("queued") + ); broker.enqueue(0, get_cfilters(0)).await; assert_eq!(broker.len(), 0); - broker.release(&RequestKey::CFilters(0)).await; - assert_eq!(broker.state_of(&RequestKey::CFilters(0)).await, None); + broker.release(&RequestKey::CFilters(BlockHash::dummy(0))).await; + assert_eq!(broker.state_of(&RequestKey::CFilters(BlockHash::dummy(0))).await, None); broker.enqueue(0, get_cfilters(0)).await; assert_eq!(broker.len(), 1); } @@ -797,28 +854,29 @@ mod tests { assert_eq!(broker.hedge(&peers).await, None, "later blocks wait for the lowest"); assert_eq!(hedged_blocks().await, [10]); - broker.block_arrived(idle, BlockHash::dummy(10)).await; + broker.arrived(idle, RequestKey::Block(BlockHash::dummy(10))).await; assert_eq!(broker.hedge(&peers).await, Some(11)); assert_eq!(hedged_blocks().await, [11]); assert_eq!(broker.hedge(slow).await, None, "no other peer to ask"); } + #[test_case(NetworkMessage::GetData(vec![block(1)]), RequestKey::Block(BlockHash::dummy(1)); "a block")] + #[test_case(get_cfilters(1), RequestKey::CFilters(BlockHash::dummy(1)); "a filter batch")] + #[test_case(get_cfheaders(1), RequestKey::CfHeaders(BlockHash::dummy(1)); "a filter header batch")] #[tokio::test] - async fn an_arrived_block_dedups_until_released() { + async fn an_arrived_reply_dedups_until_released(request: NetworkMessage, key: RequestKey) { let broker = Broker::new(); let id = conn(1, 0); - let getdata = NetworkMessage::GetData(vec![block(1)]); - let key = RequestKey::Block(BlockHash::dummy(1)); + let peer = ConnectedPeer::for_test(id).await; - broker.enqueue(0, getdata.clone()).await; - assert_eq!(broker.queue.pop_n(1).await.len(), 1); - broker.mark_on_wire(&getdata, id).await; - broker.block_arrived(conn(2, 0), BlockHash::dummy(1)).await; + broker.enqueue(0, request.clone()).await; + assert_eq!(broker.route(&[peer.handle()], u64::MAX, &ResponseCosts::new()).await, 1); + broker.arrived(conn(2, 0), key.clone()).await; assert_eq!(broker.state_of(&key).await, Some("on_wire"), "only its carrier settles it"); - broker.block_arrived(id, BlockHash::dummy(1)).await; + broker.arrived(id, key.clone()).await; assert_eq!(broker.state_of(&key).await, Some("arrived")); - broker.enqueue(0, getdata.clone()).await; + broker.enqueue(0, request.clone()).await; assert_eq!(broker.len(), 0, "a re-declare before the release must not fetch it again"); assert_eq!(broker.requeue_from(&HashSet::from([id])).await, 0); assert!(broker.expire(|_| Duration::ZERO).await.is_empty()); @@ -826,29 +884,110 @@ mod tests { broker.release(&key).await; assert_eq!(broker.state_of(&key).await, None); - broker.enqueue(0, getdata).await; + broker.enqueue(0, request).await; assert_eq!(broker.len(), 1); } + #[test_case(NetworkMessage::GetData(vec![block(1)]), RequestKey::Block(BlockHash::dummy(1)); "a block")] + #[test_case(get_cfilters(1), RequestKey::CFilters(BlockHash::dummy(1)); "a filter batch")] #[tokio::test] - async fn a_requeued_block_is_settled_by_a_copy_from_any_peer() { + async fn a_requeued_request_is_settled_by_a_copy_from_any_peer( + request: NetworkMessage, + key: RequestKey, + ) { let broker = Broker::new(); let carrier = conn(1, 0); - let getdata = NetworkMessage::GetData(vec![block(1)]); - let key = RequestKey::Block(BlockHash::dummy(1)); - broker.mark_on_wire(&getdata, carrier).await; + broker.mark_on_wire(&request, carrier).await; assert_eq!(broker.requeue_from(&HashSet::from([carrier])).await, 1); assert_eq!(broker.state_of(&key).await, Some("queued")); - broker.block_arrived(conn(2, 0), BlockHash::dummy(1)).await; + broker.arrived(conn(2, 0), key.clone()).await; assert_eq!(broker.state_of(&key).await, Some("arrived")); - assert_eq!(broker.len(), 0, "the queued getdata must not go out after the block arrived"); + assert_eq!(broker.len(), 0, "the queued request must not go out after its reply arrived"); - broker.enqueue(0, getdata).await; + broker.enqueue(0, request).await; assert_eq!(broker.len(), 0); } + /// Peers at their caps must not stop it: the batch blocks the store, and every bulk + /// download keeps them there. Peer 2 is at its cap with the batch that completes first. + #[tokio::test(start_paused = true)] + async fn hedge_filters_asks_another_peer_once_for_the_lowest_missing_batch() { + let broker = Broker::new(); + let costs = ResponseCosts::new(); + let slow = ConnectedPeer::for_test(conn(1, 0)).await; + let idle = ConnectedPeer::for_test(conn(2, 0)).await; + let peers = [slow.handle(), idle.handle()]; + let lowest = RequestKey::CFilters(BlockHash::dummy(1000)); + + slow.set_cap(1 << 30); + for start in [1000, 2000] { + broker.enqueue(start, get_cfilters(start)).await; + } + assert_eq!(broker.route(&peers[..1], u64::MAX, &costs).await, 2); + broker.enqueue(3000, get_cfilters(3000)).await; + assert_eq!(broker.route(&peers[1..], u64::MAX, &costs).await, 1); + broker.arrived(idle.id(), RequestKey::CFilters(BlockHash::dummy(3000))).await; + idle.set_cap(idle.in_flight_bytes().max(1)); + + assert_eq!(broker.hedge_filters(&peers).await, None, "it has not waited long enough"); + tokio::time::advance(HEDGE_AFTER).await; + + broker.enqueue(500, get_cfilters(500)).await; + assert_eq!(broker.hedge_filters(&peers).await, None, "the lowest one is queued"); + broker.queue.pop_n(1).await; + broker.release(&RequestKey::CFilters(BlockHash::dummy(500))).await; + + assert_eq!(broker.hedge_filters(&peers).await, Some(1000)); + assert_eq!(broker.hedge_filters(&peers).await, None, "a batch is hedged once"); + assert_eq!(broker.hedge_filters(&peers[..1]).await, None); + + broker.arrived(idle.id(), lowest.clone()).await; + assert_eq!(broker.state_of(&lowest).await, Some("arrived"), "the first copy settles it"); + assert_eq!(broker.hedge_filters(&peers).await, Some(2000)); + assert_eq!(broker.hedge_filters(&peers[..1]).await, None, "no other peer to ask"); + } + + #[tokio::test(start_paused = true)] + async fn a_hedge_lost_with_its_peer_is_sent_again() { + let broker = Broker::new(); + let holder = ConnectedPeer::for_test(conn(1, 0)).await.handle(); + let first = ConnectedPeer::for_test(conn(2, 0)).await.handle(); + let second = ConnectedPeer::for_test(conn(3, 0)).await.handle(); + broker.mark_on_wire(&get_cfilters(0), holder.id()).await; + tokio::time::advance(HEDGE_AFTER).await; + + assert_eq!(broker.hedge_filters(&[holder.clone(), first.clone()]).await, Some(0)); + assert_eq!(broker.hedge_filters(&[holder.clone(), second.clone()]).await, None); + + assert_eq!(broker.requeue_from(&HashSet::from([first.id()])).await, 0); + let key = RequestKey::CFilters(BlockHash::dummy(0)); + assert_eq!(broker.carrier_of(&key).await, Some(holder.id()), "the request stays put"); + assert_eq!(broker.hedge_filters(&[holder, second]).await, Some(0)); + } + + #[test_case(false, Some(0); "a silent peer")] + #[test_case(true, None; "a peer still delivering")] + #[tokio::test(start_paused = true)] + async fn hedge_filters_asks_another_peer_only_for_a_batch_stuck_at_a_silent_one( + delivering: bool, + hedged: Option, + ) { + let broker = Broker::new(); + let slow = ConnectedPeer::for_test(conn(1, 0)).await; + let idle = ConnectedPeer::for_test(conn(2, 0)).await; + broker.mark_on_wire(&get_cfilters(0), conn(1, 0)).await; + + tokio::time::advance(HEDGE_AFTER / 2).await; + if delivering { + broker.progress(conn(1, 0)).await; + } + tokio::time::advance(HEDGE_AFTER / 2).await; + + assert_eq!(broker.hedge_filters(&[slow.handle(), idle.handle()]).await, hedged); + } + #[test_case(get_mnlistdiff(3), vec![RequestKey::MnListDiff(BlockHash::dummy(4))], 0; "its only key")] #[test_case(NetworkMessage::GetData(vec![block(1), block(2)]), vec![RequestKey::Block(BlockHash::dummy(1))], 1; "one of its keys")] #[test_case(NetworkMessage::GetData(vec![block(1), block(2)]), vec![RequestKey::Block(BlockHash::dummy(1)), RequestKey::Block(BlockHash::dummy(2))], 0; "all of its keys")] diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index bcca69cf1..baec0ffef 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -770,6 +770,9 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) tracing::debug!(target: "dash_spv::network", "endgame: hedged block {}", height); } } + if let Some(height) = broker.hedge_filters(&handles).await { + tracing::debug!(target: "dash_spv::network", "hedged filter batch {}", height); + } if let Some(slow) = underserving.filter(|slow| Some(slow.addr) != last_underserving) { tracing::debug!( target: "dash_spv::network", @@ -1298,12 +1301,22 @@ fn spawn_pump( broker.not_found(peer, inv).await; } - // A block is settled on arrival, not when the wallet gets to it: blocks - // are processed in height order, so one can wait long behind a gap, and - // charging that wait to the peer reads as a queue and shrinks its cap. - // The first copy of a hedged block settles it: the other is a cancel. - if let NetworkMessage::Block(block) = &*msg { - broker.block_arrived(peer, block.block_hash()).await; + // A request is settled on arrival, not when its manager gets to it: blocks + // wait in height order behind a gap and filters in a long backlog, and + // charging that wait to the peer reads as a stall. The last `cfilter` of a + // reply is the one for the stop hash that keys the request. + let arrived = match &*msg { + NetworkMessage::Block(block) => Some(RequestKey::Block(block.block_hash())), + NetworkMessage::CFilter(filter) => { + Some(RequestKey::CFilters(filter.block_hash)) + } + NetworkMessage::CFHeaders(headers) => { + Some(RequestKey::CfHeaders(headers.stop_hash)) + } + _ => None, + }; + if let Some(key) = arrived { + broker.arrived(peer, key).await; } // Peers answer in order, so any answer means the peer is working @@ -1385,8 +1398,8 @@ fn spawn_pump( } let mt = MessageType::from_cmd(msg.cmd()); - // Early router wake on inbound traffic (`release_request` notifies again for the - // real capacity). `cfilter` is skipped: only the last of ~1000 frees anything. + // Early router wake on inbound traffic. `cfilter` is skipped: only the last + // of ~1000 frees anything, and `arrived` wakes the router for it. if mt != Some(MessageType::CFilter) { broker.wake(); } @@ -1470,7 +1483,7 @@ mod tests { NetworkMessage::GetCFilters(GetCFilters { filter_type: 0, start_height: start, - stop_hash: BlockHash::dummy(9), + stop_hash: BlockHash::dummy(start), }) } @@ -1759,10 +1772,49 @@ mod tests { } // A stalled request is re-queued and may already be routed to peer 2. - let carrier = net.state.broker.carrier_of(&RequestKey::CFilters(0)).await; + let carrier = net.state.broker.carrier_of(&RequestKey::CFilters(BlockHash::dummy(0))).await; assert_eq!(carrier == Some(conn(1, 0)), !stalls, "carrier {carrier:?}"); } + /// A kick re-sends what the peer still owes, not a batch that it delivered and that + /// waits for the filters manager. + #[tokio::test(start_paused = true)] + async fn a_kick_requeues_no_delivered_filter_batch() { + let net = broker().await; + let session = net.session.lock().await.clone(); + for n in 1..=2 { + let peer = ConnectedPeer::for_test(conn(n, 0)).await; + let admitted = + net.state.connected.admit(peer, &session, net.state.max_peers, None).await; + assert!(matches!(admitted, Admission::Joined)); + } + let (delivered, owed) = (get_cfilters(0), get_cfilters(1000)); + net.state.broker.mark_on_wire(&delivered, conn(1, 0)).await; + net.state.broker.mark_on_wire(&owed, conn(1, 0)).await; + let last = NetworkMessage::CFilter(CFilter { + filter_type: 0, + block_hash: BlockHash::dummy(0), + filter: vec![0u8; 4], + }); + net.state + .inbound + .send(PeerEvent::Message(conn(1, 0), last.into())) + .expect("the pump is running"); + settle().await; + + for _ in 0..12 { + tokio::time::advance(TIMEOUT_CHECK).await; + receive(&net, 2, 100, false).await; + settle().await; + } + + let broker = &net.state.broker; + let owed = RequestKey::CFilters(BlockHash::dummy(1000)); + assert_ne!(broker.carrier_of(&owed).await, Some(conn(1, 0)), "the peer must be kicked"); + let delivered = RequestKey::CFilters(BlockHash::dummy(0)); + assert_eq!(broker.state_of(&delivered).await, Some("arrived")); + } + /// A peer that was silent while idle is timed from the request it is given, and /// kicked only when it stays silent with that work for the full timeout. #[tokio::test(start_paused = true)] diff --git a/dash-spv/src/sync/filter_headers/manager.rs b/dash-spv/src/sync/filter_headers/manager.rs index 229f7da03..244db94a5 100644 --- a/dash-spv/src/sync/filter_headers/manager.rs +++ b/dash-spv/src/sync/filter_headers/manager.rs @@ -388,7 +388,7 @@ impl std::fmt::Debug #[cfg(test)] mod tests { use super::*; - use crate::network::MessageType; + use crate::network::{MessageType, RequestKey}; use crate::storage::{ DiskStorageManager, PersistentBlockHeaderStorage, PersistentFilterHeaderStorage, StorageManager, @@ -801,6 +801,29 @@ mod tests { assert_eq!(filter_tip(&storage).await, Some(3999)); } + /// The broker settles a request when its reply arrives, so a reply the pipeline does + /// not take must still release the key or a later declaration of the batch sends nothing. + #[test_case(0, false; "an empty reply")] + #[test_case(2000, true; "a reply reaching a pipeline dropped by a disconnect")] + #[tokio::test] + async fn test_cfheaders_reply_matching_nothing_releases_its_key( + count: u32, + disconnected: bool, + ) { + let storage = DiskStorageManager::with_temp_dir().await.unwrap(); + let hashes = store_block_headers(&storage, 0, 2000).await; + let (mut manager, network) = start_sync(&storage, 1999).await; + if disconnected { + manager.on_disconnect(); + } + + let reply = cfheaders(FilterHeader::all_zeros(), hashes[1999], count); + let (stored, _) = deliver(&mut manager, &network, reply).await; + + assert!(stored.is_empty()); + assert_eq!(network.released_keys().await, vec![RequestKey::CfHeaders(hashes[1999])]); + } + #[tokio::test] async fn test_buffered_cfheaders_batch_not_connecting_is_requeued_when_ready() { let storage = DiskStorageManager::with_temp_dir().await.unwrap(); diff --git a/dash-spv/src/sync/filter_headers/sync_manager.rs b/dash-spv/src/sync/filter_headers/sync_manager.rs index aaa12346c..a9216b605 100644 --- a/dash-spv/src/sync/filter_headers/sync_manager.rs +++ b/dash-spv/src/sync/filter_headers/sync_manager.rs @@ -46,9 +46,11 @@ impl SyncManager for FilterHeade msg: InboundMessage, network: &Arc, ) -> SyncResult> { + // Released whatever comes of the reply: a reply that matches nothing must not leave + // its key settled, and a batch still wanted is declared again. if let NetworkMessage::CFHeaders(cfheaders) = &*msg { + network.release_request(RequestKey::CfHeaders(cfheaders.stop_hash)).await; if self.pipeline.receive_below(cfheaders) { - network.release_request(RequestKey::CfHeaders(cfheaders.stop_hash)).await; return self.store_lower_batches().await; } } @@ -63,8 +65,6 @@ impl SyncManager for FilterHeade return Ok(vec![]); }; - network.release_request(RequestKey::CfHeaders(cfheaders.stop_hash)).await; - let mut events = Vec::new(); // Try to receive (may buffer if out of order) diff --git a/dash-spv/src/sync/filters/manager.rs b/dash-spv/src/sync/filters/manager.rs index 12d08f954..9be3795b3 100644 --- a/dash-spv/src/sync/filters/manager.rs +++ b/dash-spv/src/sync/filters/manager.rs @@ -13,8 +13,8 @@ use dashcore::ScriptBuf; use super::batch::FiltersBatch; use super::block_match_tracker::{BlockMatchTracker, BlockTrackResult}; use super::pipeline::FiltersPipeline; -use crate::error::SyncResult; -use crate::network::NetworkManager; +use crate::error::{SyncError, SyncResult}; +use crate::network::{NetworkManager, RequestKey}; use crate::storage::{BlockHeaderStorage, FilterHeaderStorage, FilterStorage}; use crate::sync::filters::util::get_prev_filter_header; use crate::sync::{FiltersProgress, SyncEvent, SyncManager, SyncState}; @@ -251,11 +251,15 @@ impl) { self.active_batches.clear(); self.tracker.clear(); self.pending_batches.clear(); + for stop_hash in self.filter_pipeline.wanted_stops() { + network.release_request(RequestKey::CFilters(*stop_hash)).await; + } self.filter_pipeline = FiltersPipeline::new(); } @@ -496,6 +500,43 @@ impl SyncResult<()> { + // Load filter headers for verification + let filter_headers = self + .filter_header_storage + .read() + .await + .load_filter_headers(batch.start_height()..batch.end_height() + 1) + .await?; + + let filter_headers_map: HashMap = filter_headers + .into_iter() + .enumerate() + .map(|(idx, header)| (batch.start_height() + idx as u32, header)) + .collect(); + + let filter_header_storage = self.filter_header_storage.read().await; + let prev_filter_header = + get_prev_filter_header(&*filter_header_storage, batch.start_height()).await?; + drop(filter_header_storage); + + let validator = FilterValidator::new(); + let validation_input = FilterValidationInput { + filters: batch.filters(), + expected_headers: &filter_headers_map, + prev_filter_header, + }; + validator.validate(validation_input)?; + + // Store verified filters to disk + let mut filter_storage = self.filter_storage.write().await; + for (key, filter) in batch.filters() { + filter_storage.store_filter(key.height(), &filter.content).await?; + } + Ok(()) + } + /// Store completed filter batches to disk and do speculative matching. /// This is decoupled from block processing - we store and match as fast as possible. pub(super) async fn store_and_match_batches(&mut self) -> SyncResult> { @@ -540,44 +581,27 @@ impl = filter_headers - .into_iter() - .enumerate() - .map(|(idx, header)| (batch.start_height() + idx as u32, header)) - .collect(); - - let filter_header_storage = self.filter_header_storage.read().await; - let prev_filter_header = - get_prev_filter_header(&*filter_header_storage, batch.start_height()).await?; - drop(filter_header_storage); - - let validator = FilterValidator::new(); - let validation_input = FilterValidationInput { - filters: batch.filters(), - expected_headers: &filter_headers_map, - prev_filter_header, - }; - validator.validate(validation_input)?; - - // Store verified filters to disk - let mut filter_storage = self.filter_storage.write().await; - for (key, filter) in batch.filters() { - filter_storage.store_filter(key.height(), &filter.content).await?; + let (start, end) = (batch.start_height(), batch.end_height()); + match self.verify_and_store(&batch).await { + Ok(()) => events.push(SyncEvent::FiltersStored { + start_height: start, + end_height: end, + }), + Err(SyncError::Validation(reason)) => { + tracing::warn!( + "Rejected filter batch {}-{}, requesting it again: {}", + start, + end, + reason + ); + self.filter_pipeline.requeue_rejected(start, end); + break; + } + Err(e) => { + self.filter_pipeline.requeue_rejected(start, end); + return Err(e); + } } - drop(filter_storage); - - events.push(SyncEvent::FiltersStored { - start_height: batch.start_height(), - end_height: batch.end_height(), - }); } // === Load filters into all active batches that overlap === @@ -1362,6 +1386,7 @@ mod tests { MockWallet, MockWalletState, MultiMockWallet, MOCK_WALLET_ID, }; use std::net::SocketAddr; + use test_case::test_case; type TestFiltersManager = FiltersManager< PersistentBlockHeaderStorage, @@ -3155,7 +3180,8 @@ mod tests { manager.filter_pipeline.init(2000, 2999); assert!(!manager.is_idle()); - manager.reset_for_rescan(); + let requests: Arc = Arc::new(MockNetworkManager::new()); + manager.reset_for_rescan(&requests).await; assert!(manager.is_idle()); } @@ -3832,8 +3858,10 @@ mod tests { /// block makes `extend_target` re-request every filter from height 1 and /// wedges the in-order store loop on a `next_batch_to_store` of 0, freezing /// `committed_height` one below tip. + #[test_case(false; "the first copy is valid")] + #[test_case(true; "a rejected copy is requested again")] #[tokio::test] - async fn test_synced_boot_then_boundary_block_commits() { + async fn test_synced_boot_then_boundary_block_commits(rejected_first: bool) { let (mut manager, headers, boundary_filter) = setup_synced_manager_at_tip().await; // Fully-synced boot leaves the manager in its default state. @@ -3890,6 +3918,19 @@ mod tests { block_hash: headers[101].block_hash(), filter: boundary_filter.content.clone(), }; + if rejected_first { + let corrupt = CFilter { + filter: vec![0xFF; 4], + ..cfilter.clone() + }; + manager + .handle_message(peer, NetworkMessage::CFilter(corrupt).into(), &requests) + .await + .unwrap(); + mock.clear_sent().await; + manager.tick(&requests).await.unwrap(); + assert_eq!(drain_getcfilters(&mock).await, 1, "the rejected batch must return"); + } manager .handle_message(peer, NetworkMessage::CFilter(cfilter).into(), &requests) .await @@ -4041,7 +4082,14 @@ mod tests { // MockWallet defaults to synced_height=0, so wallets_behind(100) = {MOCK_WALLET_ID}. assert_eq!(manager.wallet.read().await.synced_height(), 0); - let requests: Arc = Arc::new(MockNetworkManager::new()); + let mock = Arc::new(MockNetworkManager::new()); + let requests: Arc = mock.clone(); + let header_storage = manager.header_storage.clone(); + manager + .filter_pipeline + .send_pending(&requests, &*header_storage.read().await) + .await + .unwrap(); // Sanity: the pre-populated stale processed record is present, so // `track` for the same wallet would short-circuit to AlreadyProcessed. @@ -4059,6 +4107,11 @@ mod tests { // Old in-flight state was cleared and a fresh batch was created at scan_start=0. assert!(!manager.active_batches.contains_key(&101)); assert!(manager.active_batches.contains_key(&0)); + assert_eq!( + mock.released_keys().await, + vec![RequestKey::CFilters(headers[200].block_hash())], + "a dropped batch must not leave its request registered" + ); // The stale pre-populated record was wiped by `reset_for_rescan`: // a fresh `track` for the same wallet now returns `NewlyTracked`. assert!(matches!( @@ -4215,11 +4268,48 @@ mod tests { *headers[10].hash(), &filter.content ), - Some(200_000), + Some(*headers[10].hash()), "a tick re-requested filters that were still in flight" ); } + /// The broker settles a request when the last filter of its reply arrives, so + /// only the manager can tell that the reply was not enough. + #[tokio::test] + async fn a_reply_that_leaves_its_batch_incomplete_is_requested_again() { + let mut manager = create_anchored_test_manager(200_000, 10).await; + manager.filter_storage.write().await.clear_filters().await.unwrap(); + manager.progress.update_stored_height(0); + + let mock = Arc::new(MockNetworkManager::new()); + let requests: Arc = mock.clone(); + manager.handle_new_filter_headers(200_010, &requests).await.unwrap(); + assert_eq!(drain_getcfilters(&mock).await, 1); + + let stop_hash = + *manager.header_storage.read().await.get_header(200_010).await.unwrap().unwrap().hash(); + let last_filter = |block_hash| { + NetworkMessage::CFilter(CFilter { + filter_type: 0, + block_hash, + filter: vec![0; 4], + }) + .into() + }; + let peer: SocketAddr = "127.0.0.1:19999".parse().unwrap(); + + manager.handle_message(peer, last_filter(stop_hash), &requests).await.unwrap(); + assert_eq!(mock.released_keys().await, vec![RequestKey::CFilters(stop_hash)]); + + manager.tick(&requests).await.unwrap(); + assert_eq!(drain_getcfilters(&mock).await, 1, "the incomplete batch must return"); + + // A stop hash that no stored header resolves cannot reach the pipeline. + let unknown = dashcore::BlockHash::dummy(7); + assert!(manager.handle_message(peer, last_filter(unknown), &requests).await.is_err()); + assert_eq!(mock.released_keys().await.last(), Some(&RequestKey::CFilters(unknown))); + } + /// A wallet added at runtime reports `synced_height = 0` regardless of what /// the client anchored at, so raising the birth height at construction does /// not cover this path. It must converge, not rescan forever. @@ -4350,7 +4440,8 @@ mod tests { // Sanity check: the full-reset path (`reset_for_rescan`) does wipe // everything — the two paths must remain distinct. - manager.reset_for_rescan(); + let requests: Arc = Arc::new(MockNetworkManager::new()); + manager.reset_for_rescan(&requests).await; assert!(manager.active_batches.is_empty()); assert!(manager.pending_batches.is_empty()); assert!(manager.tracker.is_empty()); diff --git a/dash-spv/src/sync/filters/pipeline.rs b/dash-spv/src/sync/filters/pipeline.rs index 1b36f3c27..9731189fc 100644 --- a/dash-spv/src/sync/filters/pipeline.rs +++ b/dash-spv/src/sync/filters/pipeline.rs @@ -197,14 +197,15 @@ impl FiltersPipeline { /// Handle a received CFilter message with filter data. /// /// Buffers the filter data for batch verification and wallet matching. - /// Returns `Some(batch_start)` (the `GetCFilters` request key) when a batch - /// completes, `None` otherwise. + /// Returns the stop hash of the batch's `GetCFilters` request when that request is over: + /// the batch completed, or its last filter arrived and earlier ones are missing. Peers + /// answer in order, so nothing more will come and the request must be declared again. pub(super) fn receive_with_data( &mut self, height: u32, block_hash: BlockHash, filter_data: &[u8], - ) -> Option { + ) -> Option { // Find which batch this filter belongs to let batch_start = self.find_batch_for_height(height)?; @@ -226,7 +227,17 @@ impl FiltersPipeline { expected ); } - return None; + return if height == tracker.end_height() { + tracing::warn!( + "Filter batch {} ended incomplete with {}/{} filters, requesting it again", + batch_start, + received, + expected + ); + self.batch_stops.get(&batch_start).copied() + } else { + None + }; } let end_height = tracker.end_height(); @@ -235,7 +246,7 @@ impl FiltersPipeline { self.batch_trackers.get_mut(&batch_start).map(|t| t.take_filters()).unwrap_or_default(); self.batch_trackers.remove(&batch_start); - self.batch_stops.remove(&batch_start); + let stop_hash = self.batch_stops.remove(&batch_start); for key in filters.keys() { self.block_heights.remove(key.hash()); } @@ -249,7 +260,18 @@ impl FiltersPipeline { let batch = FiltersBatch::new(batch_start, end_height, filters); self.completed_batches.insert(batch); - Some(batch_start) + stop_hash + } + + /// Returns the stop hash of every batch still wanted, the keys of its `GetCFilters` requests. + pub(super) fn wanted_stops(&self) -> impl Iterator { + self.batch_stops.values() + } + + /// Wants the batch `start_height..=end_height` again. A rejected batch left the pipeline + /// when it completed, and without it the store order waits for it forever. + pub(super) fn requeue_rejected(&mut self, start_height: u32, end_height: u32) { + self.batch_trackers.insert(start_height, BatchTracker::new(end_height)); } /// Returns the start height of the wanted batch that contains `height`. @@ -397,20 +419,22 @@ mod tests { } #[test] - fn test_receive_batch_completion_returns_batch_start() { + fn test_receive_batch_completion_returns_the_stop_hash() { let mut pipeline = FiltersPipeline::new(); pipeline.target_height = 2; pipeline.batch_trackers.insert(0, BatchTracker::new(2)); + let stop_hash = Header::dummy(2).block_hash(); + pipeline.batch_stops.insert(0, stop_hash); - let mut completed_at = None; + let mut completed_with = None; for h in 0..=2 { let hash = Header::dummy(h).block_hash(); - if let Some(start) = pipeline.receive_with_data(h, hash, &dummy_filter_data(h)) { - completed_at = Some(start); + if let Some(stop) = pipeline.receive_with_data(h, hash, &dummy_filter_data(h)) { + completed_with = Some(stop); } } - assert_eq!(completed_at, Some(0)); + assert_eq!(completed_with, Some(stop_hash)); assert!(pipeline.batch_trackers.is_empty()); assert!(!pipeline.batch_stops.contains_key(&0)); assert_eq!(pipeline.completed_batches.len(), 1); diff --git a/dash-spv/src/sync/filters/sync_manager.rs b/dash-spv/src/sync/filters/sync_manager.rs index 7f6fef22d..e14012ba0 100644 --- a/dash-spv/src/sync/filters/sync_manager.rs +++ b/dash-spv/src/sync/filters/sync_manager.rs @@ -120,9 +120,13 @@ impl< // Find height for this filter let height = - self.filter_pipeline.resolve_height(&cfilter.block_hash, &self.header_storage).await?; + self.filter_pipeline.resolve_height(&cfilter.block_hash, &self.header_storage).await; + if !matches!(height, Ok(Some(_))) { + // This may be the last filter of its batch, which the pipeline will never see. + network.release_request(RequestKey::CFilters(cfilter.block_hash)).await; + } - let Some(h) = height else { + let Some(h) = height? else { tracing::warn!( block_hash = %cfilter.block_hash, peer = %peer, @@ -136,13 +140,13 @@ impl< }; // Buffer filter in pipeline - let batch_completed = + let request_over = self.filter_pipeline.receive_with_data(h, cfilter.block_hash, &cfilter.filter); // One `getcfilters` is answered only once its whole batch has arrived, so // individual `cfilter`s are never answered on their own. - if let Some(batch_start) = batch_completed { - network.release_request(RequestKey::CFilters(batch_start)).await; + if let Some(stop_hash) = request_over { + network.release_request(RequestKey::CFilters(stop_hash)).await; } // No `send_pending` here: re-declaring per `cfilter` would re-scan every @@ -241,7 +245,7 @@ impl< committed, restart_at ); - self.reset_for_rescan(); + self.reset_for_rescan(network).await; self.progress.update_committed_height(stale_min_synced); return self.start_download(network).await; } diff --git a/dash-spv/src/test_utils/network.rs b/dash-spv/src/test_utils/network.rs index 9281b65c0..2768bfb50 100644 --- a/dash-spv/src/test_utils/network.rs +++ b/dash-spv/src/test_utils/network.rs @@ -215,7 +215,7 @@ mod tests { NetworkMessage::GetCFilters(GetCFilters { filter_type: 0, start_height: start, - stop_hash: BlockHash::dummy(9), + stop_hash: BlockHash::dummy(start), }) } @@ -243,7 +243,7 @@ mod tests { mock.send(get_cfilters(0)).await; assert_eq!(mock.sent_messages().await.len(), 1); - mock.release_request(RequestKey::CFilters(0)).await; + mock.release_request(RequestKey::CFilters(BlockHash::dummy(0))).await; assert_eq!(mock.requests_in_play().await, 0); mock.send(get_cfilters(0)).await;