From 99f4ecf9400d7e18db204d2a0ba10007b18b673d Mon Sep 17 00:00:00 2001 From: xdustinface Date: Wed, 11 Nov 2026 00:02:20 +1000 Subject: [PATCH 01/11] fix(dash-spv): kick one peer per shared stall When our own link stalls, every peer goes quiet at once. The timeout monitor judged each peer alone, so peers whose timeouts started a second apart crossed them one at a time and each looked like a lone stall. A testnet run lost four healthy peers in 14 s during a 40 s outage, and with one configured peer a 30 s outage kicked it and banned it for 10 minutes. The monitor now hands its decision to a `StallJudge`. A silent peer is blamed only once our link provably works: some peer, this one included, received bytes more than a timeout into its silence. Silence that no peer witnessed is held as a possible stall of our link, for at most `LINK_STALL_TIMEOUTS` (6) request timeouts, 60 s with the 10 s base timeout, enough for the 40 s outages seen on testnet. Past the bound the set is more likely dead than our link, so the peer is kicked anyway. A new peer's first byte counter proves nothing, since its handshake may predate the stall. Silent peers are kicked one per timeout, the one silent the longest first, so a stall of our link never takes the whole set at once. A replacement joining right after a kick does not unpace the next one. When the first peer receives again after a silence nobody witnessed, our link counts as back from that moment. Both the witness and the hold bound then count from the later of a peer's silence and that return, so a peer still recovering from TCP backoff gets a full timeout instead of being kicked as soon as another peer speaks. Bytes from an idle peer later on cannot restart a dead peer's timeout. A link that keeps coming back for less than a timeout would push that start forward forever, so a hold also ends `MAX_HOLD_TIMEOUTS` (12) timeouts after the peer's own silence began. No single return reaches that, since the last one within a hold comes at most `LINK_STALL_TIMEOUTS` in. `Broker::expire` reports when each culprit's stalled request went silent, so its silence is measured the same way as the owed-bytes check. A request past `ANSWER_TIMEOUTS` is blamed from its last answer, so the peer's own recent bytes witness our link and it is kicked rather than held. While the judge holds a peer, `NetworkState` flags our link as stalled. The flow controller then skips the tick in a peer's serving rate, which otherwise stayed dragged down for 30 s after the stall, and neither the flow controller nor the supervisor names an underserving peer from rates the stall skewed. --- dash-spv/src/network/broker.rs | 26 +- dash-spv/src/network/manager.rs | 529 ++++++++++++++++++++++++++------ dash-spv/src/network/peer.rs | 2 + 3 files changed, 447 insertions(+), 110 deletions(-) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index 7cbbb3ca9..7f34ebce3 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -237,25 +237,31 @@ impl Broker { } } - /// Returns the peers to kick for a stalled request. A request stalls after a timeout - /// without progress, or after [`ANSWER_TIMEOUTS`] timeouts without an answer. A request - /// that already stalled [`KICKING_ATTEMPTS`] times is retried without blaming its peer. - pub async fn expire(&self, timeout: impl Fn(&PeerId) -> Duration) -> HashSet { + /// Returns the peers of stalled requests, each with the earliest instant its stalled + /// requests are blamed from. A request stalls after a timeout without + /// progress, or after [`ANSWER_TIMEOUTS`] timeouts without an answer. A request that + /// already stalled [`KICKING_ATTEMPTS`] times is retried without blaming its peer. + pub async fn expire(&self, timeout: impl Fn(&PeerId) -> Duration) -> HashMap { let now = Instant::now(); - let mut culprits = HashSet::new(); + let mut culprits: HashMap = HashMap::new(); let mut unanswerable = HashSet::new(); for (key, state) in self.requests.lock().await.iter() { let ReqState::OnWire(o) = state else { continue; }; let timeout = timeout(&o.peer); - if now.duration_since(o.last_progress) <= timeout - && now.duration_since(o.last_answer) <= timeout * ANSWER_TIMEOUTS - { + let silent_since = if now.duration_since(o.last_progress) > timeout { + o.last_progress + } else if now.duration_since(o.last_answer) > timeout * ANSWER_TIMEOUTS { + // Blamed from the answer, not the recent bytes, so those bytes show our + // link works and the peer is not held as a link stall. + o.last_answer + } else { continue; - } + }; if o.attempts < KICKING_ATTEMPTS { - culprits.insert(o.peer); + let since = culprits.entry(o.peer).or_insert(silent_since); + *since = (*since).min(silent_since); } else { tracing::debug!( target: "dash_spv::network", diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 107075df5..51e6cf029 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -2,7 +2,7 @@ use std::{ collections::{HashMap, HashSet, VecDeque}, net::SocketAddr, sync::{ - atomic::{AtomicU32, AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, Arc, }, time::Duration, @@ -44,10 +44,10 @@ const CONNECT_DEADLINE: Duration = Duration::from_secs(20); /// peer; this is just a backstop so it never sleeps on a notify that never comes. const STALL_CHECK: Duration = Duration::from_secs(5); -/// A request unanswered this long is treated as dead: the timeout monitor re-queues -/// it and kicks the peer holding it. Deliberately aggressive: replacing a peer that -/// stopped answering is faster than waiting for it; merely-slow peers are already -/// throttled by their per-peer cap. +/// A request unanswered this long is stalled. Its peer becomes a culprit, and [`StallJudge`] +/// kicks it and re-queues its work unless our own link may be the one stalled. Deliberately +/// aggressive: replacing a peer that stopped answering is faster than waiting for it, and +/// merely slow peers are already throttled by their per-peer cap. const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); /// A far peer gets this many handshake pings instead, when that is longer: its round @@ -64,6 +64,16 @@ const MAX_ADDR_PER_MESSAGE: usize = 1000; /// How often the timeout monitor scans the outstanding-request registry. const TIMEOUT_CHECK: Duration = Duration::from_secs(1); +/// How many request timeouts a silent peer is held while no peer proves our link +/// works, counted from its silence or from our link's return if later. With the 10 s +/// base timeout that is 60 s, enough for the 40 s outages seen on testnet. +const LINK_STALL_TIMEOUTS: u32 = 6; + +/// How many request timeouts a silent peer is held at most, counted from its silence, +/// however often our link returns. The link's last return within a hold is at most +/// [`LINK_STALL_TIMEOUTS`] in, so this never shortens a hold that a single return allows. +const MAX_HOLD_TIMEOUTS: u32 = 2 * LINK_STALL_TIMEOUTS; + use crate::error::{NetworkError, NetworkResult}; use crate::network::bans::{Ban, Bans}; use crate::network::broker::{Broker, RequestKey}; @@ -208,6 +218,8 @@ pub struct NetworkState { pub required_services: ServiceFlags, pub user_agent: String, pub transport_preference: TransportPreference, + /// Whether the timeout monitor sees our own link stalled. + link_stalled: AtomicBool, } pub struct PeerNetworkManager { @@ -257,6 +269,7 @@ impl PeerNetworkManager { required_services, user_agent: config.user_agent.clone(), transport_preference: config.transport_preference, + link_stalled: AtomicBool::new(false), }); let shutdown = CancellationToken::new(); @@ -610,19 +623,27 @@ impl PeerFlow { } } - /// One tick with the peer's cumulative bytes received, its bytes in flight now and - /// whether the client's own cap holds back work. Returns its new cap. - fn tick(&mut self, bytes: u64, in_flight: u64, link_full: bool) -> u64 { + /// One tick with the peer's cumulative bytes received, its bytes in flight now, + /// whether the client's own cap holds back work and whether our link is stalled. + /// Returns its new cap. + fn tick(&mut self, bytes: u64, in_flight: u64, link_full: bool, link_stalled: bool) -> u64 { // The router sends a peer work until it reaches its cap or the link cap binds. // Short of both, its rate shows how much it was asked for, not its bandwidth. let bound = in_flight > 0 && (in_flight >= self.limiter.cap as u64 || link_full); let cap = self.limiter.tick(bytes, in_flight, CAP_STEP_BYTES_PER_SEC); - if bound { + self.sample(self.limiter.rate, bound, link_stalled); + cap + } + + /// Folds `rate` into the serving rate if the peer had work. Ticks are skipped once the + /// timeout monitor sees our own link stalled, since such a stall says nothing about + /// the peer and would drag its average for 30 s. + fn sample(&mut self, rate: f64, has_work: bool, link_stalled: bool) { + if has_work && !link_stalled { self.serving_ticks += 1; let alpha = 1.0 / self.serving_ticks.min(SERVING_SAMPLE_TICKS) as f64; - self.serving_rate += alpha * (self.limiter.rate - self.serving_rate); + self.serving_rate += alpha * (rate - self.serving_rate); } - cap } /// Returns the peer's serving rate in bytes/s, 0 until measured. @@ -648,6 +669,7 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) peer_wake, broker, costs, + link_stalled, .. } = &*state; let mut global = ByteLimiter::new(MIN_CAP_BYTES, bytes.load(Ordering::Relaxed)); @@ -676,9 +698,10 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) let in_flights: Vec = g.iter().map(ConnectedPeer::in_flight_bytes).collect(); let total_in_flight: u64 = in_flights.iter().sum(); let link_full = total_in_flight >= global_cap.load(Ordering::Relaxed); + let stalled = link_stalled.load(Ordering::Relaxed); for (p, &in_flight) in g.iter().zip(&in_flights) { let flow = peers.entry(p.id()).or_insert_with(|| PeerFlow::new(p.bytes_read())); - p.set_cap(flow.tick(p.bytes_read(), in_flight, link_full)); + p.set_cap(flow.tick(p.bytes_read(), in_flight, link_full, stalled)); p.set_serving_rate(flow.serving_rate()); } @@ -742,8 +765,9 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) } // Only the transition wakes the supervisor, so a peer with no replacement - // is not re-probed every tick. - let underserving = underserving_peer(&g); + // is not re-probed every tick. A stall of our own link drags every rate down + // unevenly, so nobody is underserving during one. + let underserving = (!stalled).then(|| underserving_peer(&g)).flatten(); let handles: Vec = g.iter().map(|p| p.handle()).collect(); drop(g); if broker.len() == 0 { @@ -840,7 +864,8 @@ impl Supervisor { // Only look for a laggard once full: below cap every arrival is wanted anyway. // Each swap re-sends the work the old peer held, so at most one per cooldown. let cooled = self.last_swap.lock().await.is_none_or(|t| t.elapsed() >= SWAP_COOLDOWN); - let slow = (deficit == 0 && cooled) + let stalled = self.state.link_stalled.load(Ordering::Relaxed); + let slow = (deficit == 0 && cooled && !stalled) .then(|| underserving_peer(&peers)) .flatten() .map(|slow| slow.addr); @@ -1019,17 +1044,148 @@ fn spawn_peer_supervisor(state: Arc, shutdown: CancellationToken) ) } -/// Time requests out and evict the peers that stalled on them. -/// -/// A peer is a culprit when it has in-flight work and no bytes have arrived from it -/// for a full [`REQUEST_TIMEOUT`] since it was given that work, or when one of its -/// requests made no progress for that long. Liveness is per peer, not per request -/// age: a healthy peer draining a deep backlog keeps sending bytes even while its -/// oldest request waits. -/// -/// A culprit is kicked immediately and ALL its on-wire requests are re-queued for -/// other peers; leaving it connected would just route the freed work back to it. -/// The supervisor refills the set. +/// What the timeout monitor reads of a connected peer each tick. +struct Observed { + id: PeerId, + bytes: u64, + /// Whether part of a message has arrived and the rest has not. + partial: bool, + in_flight: u64, + timeout: Duration, +} + +/// What the timeout monitor knows of a connected peer. +struct Liveness { + bytes: u64, + /// When the peer last sent bytes or was idle. + owing_since: Instant, +} + +/// Decides which peers [`spawn_timeout_monitor`] kicks. A peer silent with work is held while +/// no peer proves our link works, as long as [`LINK_STALL_TIMEOUTS`] and [`MAX_HOLD_TIMEOUTS`] +/// allow. Silent peers are kicked one per timeout, the one silent the longest first. +#[derive(Default)] +struct StallJudge { + progress: HashMap, + /// When some peer's bytes last grew. A new peer's first counter is not growth: its + /// handshake may predate a stall of our link, so it proves nothing about the link now. + last_heard: Option, + /// The last kick of a silent peer, which paces the next. + last_kick: Option, + /// The silent peers held at the last judgement because no peer proved our link works. + held: HashSet, + /// Whether some held peer has had no witness since its silence began, so the next + /// bytes from any peer show our link is back. + link_down: bool, + /// When some peer first received after our link went down. + link_back: Option, +} + +impl StallJudge { + /// Records the peers' byte counters and returns the peers that received bytes of a + /// message still arriving. Small unrelated messages arrive whole, so only these bytes + /// are progress on a request. + fn observe(&mut self, now: Instant, peers: &[Observed]) -> Vec { + self.progress.retain(|id, _| peers.iter().any(|p| p.id == *id)); + let mut streaming = Vec::new(); + for p in peers { + let entry = self.progress.entry(p.id).or_insert(Liveness { + bytes: p.bytes, + owing_since: now, + }); + let grew = p.bytes > entry.bytes; + if grew { + if self.link_down { + self.link_back = Some(now); + } + entry.bytes = p.bytes; + self.last_heard = Some(now); + if p.partial { + streaming.push(p.id); + } + } + // An idle peer owes no bytes, so its silence before it gets work + // must not count against its first request. + if grew || p.in_flight == 0 { + entry.owing_since = now; + } + } + streaming + } + + /// Whether our own link may be stalled: a silent peer is held for lack of a witness. + fn link_stalled(&self) -> bool { + !self.held.is_empty() + } + + /// Returns the peers to kick and how many culprits are spared. `culprits` maps the + /// peers of stalled requests to when their silence began. + fn judge( + &mut self, + now: Instant, + peers: &[Observed], + mut culprits: HashMap, + ) -> (HashSet, usize) { + self.held.clear(); + self.link_down = false; + // Judged on the peer's own counters, not just the registry: streaming in-flight + // bytes are freed on the peer separately, so a wedged peer can sit at its cap + // with no registry entries. Pending work plus a byte counter + // frozen for REQUEST_TIMEOUT means stuck, whatever the registry thinks. + for p in peers.iter().filter(|p| p.in_flight > 0) { + let Some(owing_since) = self.progress.get(&p.id).map(|l| l.owing_since) else { + continue; + }; + if now.duration_since(owing_since) > p.timeout { + let since = culprits.entry(p.id).or_insert(owing_since); + *since = (*since).min(owing_since); + } + } + if culprits.is_empty() { + return (HashSet::new(), 0); + } + + let mut silent: Vec<(Instant, &Observed)> = Vec::new(); + for p in peers { + let Some(&since) = culprits.get(&p.id) else { + continue; + }; + // Silence is blamed only once any peer, this one included, received a full + // timeout into it, which proves our link works. Silence before the link came + // back does not count, so the peer's own connection gets a timeout to resume. + let witnessed_from = + |from: Instant| self.last_heard.is_some_and(|heard| heard > from + p.timeout); + let blamed_from = self.link_back.map_or(since, |back| back.max(since)); + if witnessed_from(blamed_from) + || now.duration_since(blamed_from) > p.timeout * LINK_STALL_TIMEOUTS + || now.duration_since(since) > p.timeout * MAX_HOLD_TIMEOUTS + { + silent.push((since, p)); + } else { + self.link_down |= !witnessed_from(since); + self.held.insert(p.id); + } + } + // A silent peer may still be our link failing it, so the one silent the longest + // goes, and one kick per timeout keeps a stall of our link from taking the whole set + // at once. + silent.sort_by_key(|&(since, p)| (since, p.id.addr())); + let due = silent.first().is_some_and(|(_, worst)| { + self.last_kick.is_none_or(|at| now.duration_since(at) > worst.timeout) + }); + if due { + self.last_kick = Some(now); + } + let mut spared = self.held.clone(); + spared.extend(silent.iter().skip(usize::from(due)).map(|(_, p)| p.id)); + culprits.retain(|id, _| !spared.contains(id)); + (culprits.into_keys().collect(), spared.len()) + } +} + +/// Every [`TIMEOUT_CHECK`], kicks the peers [`StallJudge`] finds stalled on requests +/// with no progress for a [`REQUEST_TIMEOUT`], re-queues all their on-wire requests for +/// other peers and wakes the supervisor to refill the set. fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) -> JoinHandle<()> { tokio::spawn(async move { let NetworkState { @@ -1038,11 +1194,11 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) peer_wake, max_peers, bans, + link_stalled, .. } = &*state; let mut ticker = tokio::time::interval(TIMEOUT_CHECK); - // Per-peer liveness: (bytes read, when the peer last sent bytes or was idle). - let mut progress: HashMap = HashMap::new(); + let mut judge = StallJudge::default(); loop { tokio::select! { _ = shutdown.cancelled() => break, @@ -1050,54 +1206,34 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) } let now = Instant::now(); - - // Judged on the peer's own counters, not just the registry: streaming in-flight - // bytes are freed on the peer separately, so a wedged peer can sit at its cap - // with no registry entries. Pending work plus a byte counter - // frozen for REQUEST_TIMEOUT means stuck, whatever the registry thinks. - let (mut culprits, streaming) = { - let peers = connected.read().await; - let live: HashSet = peers.iter().map(|p| p.id()).collect(); - progress.retain(|addr, _| live.contains(addr)); - let mut culprits = HashSet::new(); - let mut streaming = Vec::new(); - for peer in peers.iter() { - let addr = peer.id(); - let bytes = peer.bytes_read(); - let in_flight = peer.in_flight_bytes(); - let entry = progress.entry(addr).or_insert((bytes, now)); - let grew = bytes > entry.0; - // Small unrelated messages arrive whole, so only bytes of a message - // still arriving are progress on a request. - if grew && peer.partial_message() { - streaming.push(addr); - } - // An idle peer owes no bytes, so its silence before it gets work - // must not count against its first request. - if grew || in_flight == 0 { - *entry = (bytes, now); - } - if in_flight > 0 && now.duration_since(entry.1) > request_timeout(peer.lag_ms()) - { - culprits.insert(addr); - } - } - (culprits, streaming) - }; - broker.receiving(&streaming).await; - // A request that stopped making progress also condemns its peer: the - // per-peer byte check above sees unrelated traffic and judges it healthy. - let timeouts: HashMap = connected + let peers: Vec = connected .read() .await .iter() - .map(|p| (p.id(), request_timeout(p.lag_ms()))) + .map(|p| Observed { + id: p.id(), + bytes: p.bytes_read(), + partial: p.partial_message(), + in_flight: p.in_flight_bytes(), + timeout: request_timeout(p.lag_ms()), + }) .collect(); - culprits.extend( - broker.expire(|peer| timeouts.get(peer).copied().unwrap_or(REQUEST_TIMEOUT)).await, - ); - + broker.receiving(&judge.observe(now, &peers)).await; + // A request with no progress for a timeout also condemns its peer: the + // per-peer byte check sees unrelated traffic and judges it healthy. + let timeout = |id: &PeerId| { + peers.iter().find(|p| p.id == *id).map_or(REQUEST_TIMEOUT, |p| p.timeout) + }; + let (culprits, spared) = judge.judge(now, &peers, broker.expire(timeout).await); + link_stalled.store(judge.link_stalled(), Ordering::Relaxed); if culprits.is_empty() { + if spared > 0 { + tracing::debug!( + target: "dash_spv::network", + "request timeout: kicked none, spared {}", + spared, + ); + } continue; } @@ -1121,9 +1257,10 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) tracing::warn!( target: "dash_spv::network", - "request timeout: kicked {} peer(s) {:?}, retried {} request(s)", + "request timeout: kicked {} peer(s) {:?}, spared {}, retried {} request(s)", dropped, culprits, + spared, retried, ); broker.wake(); @@ -1338,6 +1475,7 @@ mod tests { use dashcore::{BlockHash, Network}; use test_case::test_case; use tokio::io::AsyncWriteExt; + use tokio::net::tcp::OwnedWriteHalf; use tokio::task; use tokio::time; @@ -1363,6 +1501,48 @@ mod tests { PeerNetworkManager::new(&config).await } + /// Advances the clock `secs` seconds while peer `id` receives a whole `ping` each second + /// through its far end `far`. + async fn advance_receiving( + net: &PeerNetworkManager, + id: PeerId, + far: &mut OwnedWriteHalf, + secs: u64, + ) { + let ping = serialize(&RawNetworkMessage { + magic: Network::Regtest.magic(), + payload: NetworkMessage::Ping(1), + }); + for _ in 0..secs { + time::advance(TIMEOUT_CHECK).await; + deliver(net, id, far, &ping).await; + settle().await; + } + } + + /// Writes `bytes` to the far end `far` of peer `id` and waits until the peer's reader + /// counted them. Socket readiness does not follow the paused clock, so a tick could + /// otherwise miss bytes already written. A kicked peer's closed socket is skipped. + async fn deliver(net: &PeerNetworkManager, id: PeerId, far: &mut OwnedWriteHalf, bytes: &[u8]) { + let read = || async { + net.state.connected.read().await.iter().find(|p| p.id() == id).map(|p| p.bytes_read()) + }; + let Some(before) = read().await else { + return; + }; + if far.write_all(bytes).await.is_err() { + return; + } + let target = before + bytes.len() as u64; + for _ in 0..100_000 { + match read().await { + Some(read) if read < target => task::yield_now().await, + _ => return, + } + } + panic!("the reader of {id:?} never counted {} bytes", bytes.len()); + } + /// Lets the spawned tasks run what the last time advance made due. async fn settle() { for _ in 0..4 { @@ -1398,7 +1578,7 @@ mod tests { fn tick(&mut self, delivered: u64, in_flight: u64, link_full: bool) { self.bytes += delivered; - self.cap = self.flow.tick(self.bytes, in_flight, link_full); + self.cap = self.flow.tick(self.bytes, in_flight, link_full, false); } /// Bulk download at `rate` bytes/s. The router keeps the peer at its own cap, or @@ -1508,7 +1688,13 @@ mod tests { let mut late = None; match departure { Departure::Drop => disconnect_all(), - Departure::Kick => tokio::time::advance(REQUEST_TIMEOUT + TIMEOUT_CHECK * 2).await, + Departure::Kick => { + // Silent past the bound, the peers go one per timeout. + for _ in 0..2 { + time::advance(REQUEST_TIMEOUT * LINK_STALL_TIMEOUTS + TIMEOUT_CHECK).await; + settle().await; + } + } Departure::Stop => { net.stop().await; // A handshake that completed while `stop` ran. @@ -1574,7 +1760,7 @@ mod tests { Chatters(u8), } - /// Peer 1 carries a request for `secs` seconds beside idle peer 2. + /// Peer 1 carries a request for `secs` seconds while idle peer 2 keeps receiving. #[test_case(Feed::Nothing, 11, true; "a silent request")] #[test_case(Feed::Answers(1), 20, false; "the carrier answering keeps it alive")] #[test_case(Feed::Answers(2), 11, true; "another peer answering is not progress")] @@ -1610,6 +1796,10 @@ mod tests { for second in 0..secs as usize { tokio::time::advance(TIMEOUT_CHECK).await; + // Peer 2 proves our link works, unless the feed already sends it bytes. + if !matches!(feed, Feed::Streams(2) | Feed::Chatters(2)) { + deliver(&net, conn(2, 0), &mut far_ends[1], &ping).await; + } match feed { Feed::Nothing => {} Feed::Answers(peer) => { @@ -1623,13 +1813,12 @@ mod tests { .send(PeerEvent::Message(conn(peer, 0), piece.into())) .expect("the pump is running"); } - // A kicked peer's socket is closed, so a write after the kick fails. Feed::Streams(peer) => { let piece = &long_answer[second * 1000..(second + 1) * 1000]; - let _ = far_ends[peer as usize - 1].write_all(piece).await; + deliver(&net, conn(peer, 0), &mut far_ends[peer as usize - 1], piece).await; } Feed::Chatters(peer) => { - let _ = far_ends[peer as usize - 1].write_all(&ping).await; + deliver(&net, conn(peer, 0), &mut far_ends[peer as usize - 1], &ping).await; } } settle().await; @@ -1647,10 +1836,11 @@ mod tests { async fn a_kicked_peer_gets_none_of_its_requeued_work() { let net = broker().await; let session = net.session.lock().await.clone(); - let mut handles = Vec::new(); - for n in 1..=2 { - let peer = ConnectedPeer::for_test(conn(n, 0)).await; - handles.push(peer.handle()); + let stalled = ConnectedPeer::for_test(conn(1, 0)).await; + let (witness, mut witness_far) = + ConnectedPeer::for_test_reading(conn(2, 0), net.state.clone()).await; + let handles = [stalled.handle(), witness.handle()]; + for peer in [stalled, witness] { let admitted = net.state.connected.admit(peer, &session, net.state.max_peers, None).await; assert!(matches!(admitted, Admission::Joined)); @@ -1658,10 +1848,8 @@ mod tests { net.state.broker.mark_on_wire(&get_cfilters(0), conn(1, 0)).await; let held = net.state.broker.hold_queue_of(&get_cfilters(0)).await; - time::advance(REQUEST_TIMEOUT + TIMEOUT_CHECK * 2).await; - for _ in 0..4 { - task::yield_now().await; - } + // The second peer keeps receiving, so the stall is not blamed on our link. + advance_receiving(&net, conn(2, 0), &mut witness_far, REQUEST_TIMEOUT.as_secs() + 2).await; let key = RequestKey::CFilters(0); assert_eq!( net.state.broker.state_of(&key).await, @@ -1686,27 +1874,168 @@ mod tests { async fn the_timeout_monitor_times_an_idle_peer_from_its_first_request() { let net = broker().await; let session = net.session.lock().await.clone(); - let id = conn(1, 0); - let peer = ConnectedPeer::for_test(id).await; + let peer = ConnectedPeer::for_test(conn(1, 0)).await; let handle = peer.handle(); - let admitted = net.state.connected.admit(peer, &session, net.state.max_peers, None).await; - assert!(matches!(admitted, Admission::Joined)); - - for _ in 0..3 { - tokio::time::advance(REQUEST_TIMEOUT).await; - tokio::task::yield_now().await; + let (witness, mut witness_far) = + ConnectedPeer::for_test_reading(conn(2, 0), net.state.clone()).await; + for peer in [peer, witness] { + let admitted = + net.state.connected.admit(peer, &session, net.state.max_peers, None).await; + assert!(matches!(admitted, Admission::Joined)); } + + // The second peer keeps receiving, so a stall is not blamed on our link. + advance_receiving(&net, conn(2, 0), &mut witness_far, REQUEST_TIMEOUT.as_secs() * 3).await; // Only the charge matters here: the loopback has no reader to answer it. let _ = handle.send(&get_cfilters(0)).await; assert!(handle.in_flight_bytes() > 0); - tokio::time::advance(TIMEOUT_CHECK * 2).await; - tokio::task::yield_now().await; - assert_eq!(net.connected_count().await, 1, "kicked right after its first request"); + advance_receiving(&net, conn(2, 0), &mut witness_far, 2).await; + assert_eq!(net.connected_count().await, 2, "kicked right after its first request"); + + advance_receiving(&net, conn(2, 0), &mut witness_far, REQUEST_TIMEOUT.as_secs()).await; + assert_eq!(net.connected_count().await, 1, "a peer silent with work stays connected"); + } + + /// What a peer of the set does, second by second. + #[derive(Clone, Copy)] + enum Role { + /// Holds a request the registry tracks and receives bytes until this second. + Registered(u64), + /// Owes bytes the registry does not track and receives bytes until this second. + Charged(u64), + /// Idle, and receives bytes from this second on. + Delivers(u64), + /// Connects at this second, idle, and receives nothing. + Joins(u64), + /// Owes bytes, receives until the first second and again from the second. + Resumes(u64, u64), + /// Idle, and receives bytes every this many seconds. + Whispers(u64), + /// Holds a request the registry tracks and receives small whole messages from + /// this second on. + Chatters(u64), + /// Owes bytes. Repeats being silent for the first field's seconds and receiving for + /// the second's. + Flaps(u64, u64), + } + + /// Peer `n + 1` plays `roles[n]` for `secs` seconds. `link_stalled` is whether the + /// judge then suspects our own link. + #[test_case(&[Role::Registered(0); 3], 11, &[1, 2, 3], true; "a stall of every peer at once is our link")] + #[test_case(&[Role::Registered(0); 3], 41, &[1, 2, 3], true; "a 40 s silence of every peer kicks nobody")] + #[test_case(&[Role::Registered(0); 3], 71, &[2, 3], false; "every peer silent past the bound loses the one silent the longest")] + #[test_case(&[Role::Registered(0); 3], 72, &[3], false; "every peer silent past the bound loses one per timeout")] + #[test_case(&[Role::Charged(0); 3], 25, &[1, 2, 3], true; "a long stall of every peer is still our link")] + #[test_case(&[Role::Registered(1), Role::Registered(2), Role::Registered(3)], 25, &[1, 2, 3], true; "peers going silent a second apart in an outage stay")] + #[test_case(&[Role::Charged(1), Role::Charged(2), Role::Charged(3)], 25, &[1, 2, 3], true; "peers owing bytes and going silent a second apart stay")] + #[test_case(&[Role::Delivers(0), Role::Registered(0), Role::Delivers(0)], 11, &[1, 3], false; "a lone silent request among delivering peers is kicked")] + #[test_case(&[Role::Delivers(0), Role::Charged(0), Role::Delivers(0)], 12, &[1, 3], false; "a lone peer owing bytes among delivering peers is kicked")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(0)], 11, &[2, 3], false; "two stalls beside a delivering peer kick one")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(0)], 22, &[2], false; "the second stall goes a timeout later")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(1)], 12, &[2, 3], false; "stalls a second apart beside a delivering peer go one per timeout")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(2)], 14, &[2, 3], false; "stalls two seconds apart beside a delivering peer go one per timeout")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(1), Role::Joins(11)], 12, &[2, 3, 4], false; "a replacement joining does not unpace the next stall")] + #[test_case(&[Role::Registered(0), Role::Delivers(13), Role::Registered(1)], 24, &[2, 3], false; "a late witness kicks the peer silent the longest a timeout later")] + #[test_case(&[Role::Resumes(5, 20), Role::Resumes(5, 20), Role::Charged(5)], 25, &[1, 2, 3], true; "a peer still silent when the link is back gets a timeout")] + #[test_case(&[Role::Resumes(5, 20), Role::Resumes(5, 20), Role::Charged(5)], 31, &[1, 2], false; "a peer silent a timeout after the link is back is kicked")] + #[test_case(&[Role::Charged(0), Role::Whispers(3)], 13, &[2], false; "a lone peer owing bytes beside an idle one is no link stall")] + #[test_case(&[Role::Registered(0)], 41, &[1], true; "a sole silent peer stays through a 40 s outage")] + #[test_case(&[Role::Registered(0)], 61, &[], false; "a sole silent peer is kicked past the bound")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Joins(12)], 13, &[1, 2, 3], true; "a peer joining during an outage proves nothing")] + #[test_case(&[Role::Charged(0), Role::Delivers(30)], 15, &[1, 2], true; "a lone peer owing bytes beside a quiet idle one is held")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 35, &[1, 2, 3], true; "an idle peer receiving first after an outage gives the held peers a timeout")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 36, &[2, 3], false; "held peers silent a timeout after an idle peer receives go")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 47, &[3], false; "held peers silent after an idle peer receives go one per timeout")] + #[test_case(&[Role::Charged(0), Role::Whispers(15)], 30, &[2], false; "an idle peer's later bytes do not restart a held peer's timeout")] + #[test_case(&[Role::Resumes(0, 55), Role::Charged(0)], 65, &[1, 2], true; "a peer silent through a 55 s outage gets a timeout after it")] + #[test_case(&[Role::Resumes(0, 55), Role::Charged(0)], 66, &[1], false; "a peer silent a timeout after a 55 s outage is kicked")] + #[test_case(&[Role::Chatters(0)], 11, &[], false; "a sole peer receiving only small messages is kicked")] + #[test_case(&[Role::Chatters(25); 3], 36, &[2, 3], false; "peers receiving only small messages after an outage go one per timeout")] + #[test_case(&[Role::Charged(0), Role::Flaps(12, 5)], 121, &[1, 2], true; "a dead peer beside a flapping link is held up to the absolute bound")] + #[test_case(&[Role::Charged(0), Role::Flaps(12, 5)], 122, &[2], false; "a dead peer beside a flapping link is kicked at the absolute bound")] + fn the_stall_judge_kicks_one_peer_per_shared_stall( + roles: &[Role], + secs: u64, + kept: &[u8], + link_stalled: bool, + ) { + // Every received byte but a chatter's is part of an answer still arriving. + let peer = |n: u8, role: Role| Observed { + id: conn(n, 0), + bytes: 0, + partial: !matches!(role, Role::Chatters(_)), + in_flight: matches!(role, Role::Charged(_) | Role::Resumes(..) | Role::Flaps(..)) + .into(), + timeout: REQUEST_TIMEOUT, + }; + let start = Instant::now(); + let mut peers: Vec = (1..) + .zip(roles) + .filter(|(_, role)| !matches!(role, Role::Joins(_))) + .map(|(n, &role)| peer(n, role)) + .collect(); + // The last progress of each registered request. + let mut requests: HashMap = (1..) + .zip(roles) + .filter(|(_, role)| matches!(role, Role::Registered(_) | Role::Chatters(_))) + .map(|(n, _)| (conn(n, 0), start)) + .collect(); + let mut judge = StallJudge::default(); + + for second in 1..=secs { + for (n, &role) in (1..).zip(roles) { + let receives = match role { + Role::Registered(until) | Role::Charged(until) => second <= until, + Role::Delivers(from) | Role::Chatters(from) => second >= from, + Role::Resumes(until, from) => second <= until || second >= from, + Role::Whispers(period) => second.is_multiple_of(period), + Role::Flaps(silent, receiving) => (second - 1) % (silent + receiving) >= silent, + Role::Joins(at) => { + if second == at { + peers.push(peer(n, role)); + } + false + } + }; + if let Some(p) = peers.iter_mut().find(|p| receives && p.id == conn(n, 0)) { + p.bytes += 100; + } + } + let now = start + Duration::from_secs(second); + for id in judge.observe(now, &peers) { + if let Some(progress) = requests.get_mut(&id) { + *progress = now; + } + } + let stalled = requests + .iter() + .filter(|(_, &progress)| now.duration_since(progress) > REQUEST_TIMEOUT) + .map(|(&id, &progress)| (id, progress)) + .collect(); + let (kicked, _) = judge.judge(now, &peers, stalled); + assert!(kicked.len() <= 1, "second {second} kicked {kicked:?}"); + peers.retain(|p| !kicked.contains(&p.id)); + requests.retain(|id, _| !kicked.contains(id)); + } + + let left: Vec = peers.iter().map(|p| p.id).collect(); + let kept: Vec = kept.iter().map(|&n| conn(n, 0)).collect(); + assert_eq!(left, kept); + assert_eq!(judge.link_stalled(), link_stalled); + } - tokio::time::advance(REQUEST_TIMEOUT).await; - tokio::task::yield_now().await; - assert_eq!(net.connected_count().await, 0, "a peer silent with work stays connected"); + /// A peer serves 1 MB/s with work for three ticks, then nothing for one. + #[test_case(false, true => 750_000; "a stall of its own drags its serving rate")] + #[test_case(true, true => 1_000_000; "a stall of our link does not")] + #[test_case(false, false => 1_000_000; "idle ticks do not")] + fn the_serving_rate_averages_ticks_with_work(link_stalled: bool, has_work: bool) -> u64 { + let mut flow = PeerFlow::new(0); + for _ in 0..3 { + flow.sample(1e6, true, false); + } + flow.sample(0.0, has_work, link_stalled); + flow.serving_rate as u64 } #[test_case(&[&[MessageType::Inv], &[MessageType::Inv]], 0, &[1, 1]; "every subscriber of the type")] diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index 7da8a886b..b8389592b 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -436,6 +436,8 @@ impl ConnectedPeer { state: Arc, ) -> (Self, OwnedWriteHalf) { let (link, far) = test_link().await; + // A test writes in small pieces, which Nagle would hold back for a delayed ACK. + far.set_nodelay(true).expect("loopback nodelay"); let (mut far_read, far_write) = far.into_split(); tokio::spawn(async move { io::copy(&mut far_read, &mut io::sink()).await }); let (reader, bytes, partial) = peer_reader(link.reader, link.codec, state.clone()); From 506eddee35e77b239eb2e375aae7f619bb823b0f Mon Sep 17 00:00:00 2001 From: xdustinface Date: Wed, 11 Nov 2026 00:02:20 +1000 Subject: [PATCH 02/11] test(dash-spv): keep a peer frozen mid-sync connected A dashd test freezes dashd with `SIGSTOP` for 30 s in the middle of a sync, three request timeouts but under the bound for holding a silent peer, then resumes it with `SIGCONT`. Its socket stays open, so the client sees a silent connection, as when our own link stalls. The test asserts the client never drops its only peer and that sync completes. Against the old timeout monitor the peer is kicked about 10 s into the freeze. `DashCoreNode::set_paused` sends the signals. The test is unix only, since Windows has no equivalent. --- dash-spv/src/test_utils/node.rs | 21 ++++++++ dash-spv/tests/dashd_sync/main.rs | 3 ++ dash-spv/tests/dashd_sync/tests_stall.rs | 68 ++++++++++++++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 dash-spv/tests/dashd_sync/tests_stall.rs diff --git a/dash-spv/src/test_utils/node.rs b/dash-spv/src/test_utils/node.rs index e49c1d82c..cdf81c0e0 100644 --- a/dash-spv/src/test_utils/node.rs +++ b/dash-spv/src/test_utils/node.rs @@ -16,6 +16,8 @@ use std::process::ExitStatus; use std::sync::atomic::{AtomicU16, Ordering}; use std::time::Duration; use tokio::process::Child; +#[cfg(unix)] +use tokio::process::Command; use tokio::time::{sleep, timeout}; use super::fs_helpers::{copy_dir, copy_to_retain_dir}; @@ -683,6 +685,25 @@ impl DashCoreNode { tracing::info!("Disconnected peer {}", addr); } + /// Freezes dashd with `SIGSTOP`, or resumes it with `SIGCONT`. Its sockets stay open, so + /// its peers see a silent connection rather than a disconnect. + #[cfg(unix)] + pub async fn set_paused(&self, paused: bool) { + let pid = self.process.as_ref().and_then(Child::id).expect("dashd is not running"); + let signal = if paused { + "-STOP" + } else { + "-CONT" + }; + let status = Command::new("kill") + .args([signal, &pid.to_string()]) + .status() + .await + .expect("failed to run kill"); + assert!(status.success(), "kill {signal} {pid} failed: {status}"); + tracing::info!("Set paused={} on dashd", paused); + } + /// Enable or disable all P2P network activity on this node. pub fn set_network_active(&self, active: bool) { let client = self.rpc_client(); diff --git a/dash-spv/tests/dashd_sync/main.rs b/dash-spv/tests/dashd_sync/main.rs index a22afc967..52ba1e26f 100644 --- a/dash-spv/tests/dashd_sync/main.rs +++ b/dash-spv/tests/dashd_sync/main.rs @@ -11,4 +11,7 @@ mod tests_mempool; mod tests_multi_wallet; mod tests_node_start; mod tests_restart; +// SIGSTOP freezes dashd, and Windows has no equivalent. +#[cfg(unix)] +mod tests_stall; mod tests_transaction; diff --git a/dash-spv/tests/dashd_sync/tests_stall.rs b/dash-spv/tests/dashd_sync/tests_stall.rs new file mode 100644 index 000000000..5cf3caf8f --- /dev/null +++ b/dash-spv/tests/dashd_sync/tests_stall.rs @@ -0,0 +1,68 @@ +use std::time::Duration; + +use dash_spv::network::NetworkEvent; +use dash_spv::test_utils::{TestChain, SYNC_TIMEOUT}; +use dash_spv::SyncEvent; +use tokio::sync::broadcast::error::TryRecvError; +use tokio::time::{sleep, timeout}; + +use super::helpers::{is_progress_event, wait_for_sync}; +use super::setup::TestContext; + +/// Verify a peer that goes silent mid-sync, as when our own link stalls, stays connected. +/// +/// `SIGSTOP` freezes dashd with its socket open for well over a request timeout, but +/// under the bound for holding a silent peer. The client must keep its only peer through +/// the freeze, and sync must complete once dashd resumes. +#[tokio::test] +async fn test_sync_keeps_a_peer_frozen_mid_sync() { + let Some(ctx) = TestContext::new(TestChain::Full).await else { + return; + }; + let mut client_handle = ctx.spawn_new_client().await; + + let mut progress_events = 0; + timeout(SYNC_TIMEOUT, async { + while progress_events < 5 { + match client_handle.sync_event_receiver.recv().await { + Ok(ref event) if is_progress_event(event) => progress_events += 1, + Ok(SyncEvent::SyncComplete { + .. + }) => panic!("sync completed before dashd could be frozen"), + Ok(_) => {} + Err(e) => panic!("sync events lost: {e}"), + } + } + }) + .await + .expect("Timeout waiting for sync progress"); + + ctx.dashd.node.set_paused(true).await; + // Sync may have finished while the loop above stopped reading, so the freeze must + // be checked to fall mid-sync. + while let Ok(event) = client_handle.sync_event_receiver.try_recv() { + assert!( + !matches!(event, SyncEvent::SyncComplete { .. }), + "sync completed before dashd was frozen" + ); + } + assert!( + !client_handle.client.progress().await.is_synced(), + "sync completed before dashd was frozen" + ); + sleep(Duration::from_secs(30)).await; + ctx.dashd.node.set_paused(false).await; + + wait_for_sync(&mut client_handle.progress_receiver, ctx.dashd.initial_height).await; + loop { + match client_handle.network_event_receiver.try_recv() { + Ok(NetworkEvent::PeerDisconnected(addr)) => panic!("peer {addr} was dropped"), + Ok(_) => {} + Err(TryRecvError::Empty) => break, + Err(e) => panic!("network events lost: {e}"), + } + } + + client_handle.stop().await; + ctx.assert_synced(&client_handle.client.progress().await).await; +} From a3dd1a9b8d96aaf27a2b17d47884677036d9dc03 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Wed, 11 Nov 2026 00:04:05 +1000 Subject: [PATCH 03/11] refactor(dash-spv): keep only a flag of held peers `StallJudge::held` collected the peers held for a possible stall of our link, but it was only read to check whether it was empty and to count the spared culprits. The judge now builds the kick set directly, from the culprits that already left plus the peer silent the longest when a kick is due, counts the held peers while judging, and keeps a `link_stalled` flag instead of the set. The kicks and the flag are unchanged. --- dash-spv/src/network/manager.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 51e6cf029..e16fba350 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1072,8 +1072,9 @@ struct StallJudge { last_heard: Option, /// The last kick of a silent peer, which paces the next. last_kick: Option, - /// The silent peers held at the last judgement because no peer proved our link works. - held: HashSet, + /// Whether our own link may be stalled: the last judgement held a silent peer because + /// no peer proved our link works. + link_stalled: bool, /// Whether some held peer has had no witness since its silence began, so the next /// bytes from any peer show our link is back. link_down: bool, @@ -1113,11 +1114,6 @@ impl StallJudge { streaming } - /// Whether our own link may be stalled: a silent peer is held for lack of a witness. - fn link_stalled(&self) -> bool { - !self.held.is_empty() - } - /// Returns the peers to kick and how many culprits are spared. `culprits` maps the /// peers of stalled requests to when their silence began. fn judge( @@ -1126,7 +1122,7 @@ impl StallJudge { peers: &[Observed], mut culprits: HashMap, ) -> (HashSet, usize) { - self.held.clear(); + self.link_stalled = false; self.link_down = false; // Judged on the peer's own counters, not just the registry: streaming in-flight // bytes are freed on the peer separately, so a wedged peer can sit at its cap @@ -1145,6 +1141,7 @@ impl StallJudge { return (HashSet::new(), 0); } + let mut held = 0; let mut silent: Vec<(Instant, &Observed)> = Vec::new(); for p in peers { let Some(&since) = culprits.get(&p.id) else { @@ -1163,9 +1160,10 @@ impl StallJudge { silent.push((since, p)); } else { self.link_down |= !witnessed_from(since); - self.held.insert(p.id); + held += 1; } } + self.link_stalled = held > 0; // A silent peer may still be our link failing it, so the one silent the longest // goes, and one kick per timeout keeps a stall of our link from taking the whole set // at once. @@ -1176,10 +1174,11 @@ impl StallJudge { if due { self.last_kick = Some(now); } - let mut spared = self.held.clone(); - spared.extend(silent.iter().skip(usize::from(due)).map(|(_, p)| p.id)); - culprits.retain(|id, _| !spared.contains(id)); - (culprits.into_keys().collect(), spared.len()) + // A culprit that already left is not spared, so its requests are still re-queued. + let mut kicked: HashSet = + culprits.into_keys().filter(|id| peers.iter().all(|p| p.id != *id)).collect(); + kicked.extend(silent.first().filter(|_| due).map(|(_, p)| p.id)); + (kicked, held + silent.len() - usize::from(due)) } } @@ -1225,7 +1224,7 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) peers.iter().find(|p| p.id == *id).map_or(REQUEST_TIMEOUT, |p| p.timeout) }; let (culprits, spared) = judge.judge(now, &peers, broker.expire(timeout).await); - link_stalled.store(judge.link_stalled(), Ordering::Relaxed); + link_stalled.store(judge.link_stalled, Ordering::Relaxed); if culprits.is_empty() { if spared > 0 { tracing::debug!( @@ -2022,7 +2021,7 @@ mod tests { let left: Vec = peers.iter().map(|p| p.id).collect(); let kept: Vec = kept.iter().map(|&n| conn(n, 0)).collect(); assert_eq!(left, kept); - assert_eq!(judge.link_stalled(), link_stalled); + assert_eq!(judge.link_stalled, link_stalled); } /// A peer serves 1 MB/s with work for three ticks, then nothing for one. From 6b9f47da06c5fa7dbe02385f627a4b7102699160 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Wed, 11 Nov 2026 00:04:14 +1000 Subject: [PATCH 04/11] refactor(dash-spv): name the judge's `Liveness` map `StallJudge::progress` holds each peer's `Liveness`, and "progress" already means request progress in the broker and the monitor. The field is now `liveness`. --- dash-spv/src/network/manager.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index e16fba350..f16649f78 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1066,7 +1066,7 @@ struct Liveness { /// allow. Silent peers are kicked one per timeout, the one silent the longest first. #[derive(Default)] struct StallJudge { - progress: HashMap, + liveness: HashMap, /// When some peer's bytes last grew. A new peer's first counter is not growth: its /// handshake may predate a stall of our link, so it proves nothing about the link now. last_heard: Option, @@ -1087,10 +1087,10 @@ impl StallJudge { /// message still arriving. Small unrelated messages arrive whole, so only these bytes /// are progress on a request. fn observe(&mut self, now: Instant, peers: &[Observed]) -> Vec { - self.progress.retain(|id, _| peers.iter().any(|p| p.id == *id)); + self.liveness.retain(|id, _| peers.iter().any(|p| p.id == *id)); let mut streaming = Vec::new(); for p in peers { - let entry = self.progress.entry(p.id).or_insert(Liveness { + let entry = self.liveness.entry(p.id).or_insert(Liveness { bytes: p.bytes, owing_since: now, }); @@ -1129,7 +1129,7 @@ impl StallJudge { // with no registry entries. Pending work plus a byte counter // frozen for REQUEST_TIMEOUT means stuck, whatever the registry thinks. for p in peers.iter().filter(|p| p.in_flight > 0) { - let Some(owing_since) = self.progress.get(&p.id).map(|l| l.owing_since) else { + let Some(owing_since) = self.liveness.get(&p.id).map(|l| l.owing_since) else { continue; }; if now.duration_since(owing_since) > p.timeout { From 3fb8f837fe2450181167b84aead6990b979e0f0e Mon Sep 17 00:00:00 2001 From: xdustinface Date: Wed, 11 Nov 2026 00:04:20 +1000 Subject: [PATCH 05/11] test(dash-spv): clarify the `Role::Resumes` doc The doc read "until the first second and again from the second", which mixes the ordinal with the time unit. It now names both fields as times in seconds. --- dash-spv/src/network/manager.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index f16649f78..e34fbaa23 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1907,7 +1907,8 @@ mod tests { Delivers(u64), /// Connects at this second, idle, and receives nothing. Joins(u64), - /// Owes bytes, receives until the first second and again from the second. + /// Owes bytes. Receives until the time in the first field and again from the time + /// in the second, both in seconds. Resumes(u64, u64), /// Idle, and receives bytes every this many seconds. Whispers(u64), From 8abf7bac86273c8d1f526549e9de0e0d9e63d729 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Fri, 13 Nov 2026 14:26:16 +1000 Subject: [PATCH 06/11] test(dash-spv): pin how a link stall gates rates and swaps Nothing failed if the flow controller sampled serving rates or named a slow peer during a stall of our own link, or if the supervisor picked a slow peer to swap during one. `the_serving_rate_averages_ticks_with_work` called the private `PeerFlow::sample` and read its private rate. `a_stall_of_our_link_leaves_the_serving_rates_alone` measures two peers through the real flow controller, with work up to their caps and bytes fed through their sockets. Once both go silent, their rates fall until the judge holds them as a possible stall of our link, and then stay put. `no_slow_peer_is_swapped_out_while_our_link_stalls` starts the supervisor during such a stall with a slow peer and a replacement that completes a real handshake, and checks that nothing is dialed. Once the peers deliver again, the swap the stall held back must go ahead. It does only because the flow controller names no slow peer during a stall: naming it again afterwards is what wakes a full supervisor, which has no timer to fall back on. The private-state test is removed, as the new test covers its stall and silence cases and `only_bandwidth_bound_service_condemns_a_peer` its idle one. --- dash-spv/src/network/manager.rs | 255 ++++++++++++++++++++++++++++++-- 1 file changed, 244 insertions(+), 11 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index e34fbaa23..99564df20 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1468,15 +1468,20 @@ mod tests { use crate::client::ClientConfig; use crate::test_utils::test_socket_address; use dashcore::consensus::encode::serialize; - use dashcore::network::message::RawNetworkMessage; + use dashcore::network::address::Address; + use dashcore::network::message::{RawNetworkMessage, RawNetworkMessageCodec}; use dashcore::network::message_blockdata::Inventory; use dashcore::network::message_filter::{CFilter, GetCFilters}; + use dashcore::network::message_network::VersionMessage; use dashcore::{BlockHash, Network}; + use std::sync::atomic::AtomicUsize; use test_case::test_case; use tokio::io::AsyncWriteExt; use tokio::net::tcp::OwnedWriteHalf; + use tokio::net::{TcpListener, TcpStream}; use tokio::task; use tokio::time; + use tokio_util::codec::FramedRead; fn get_cfilters(start: u32) -> NetworkMessage { NetworkMessage::GetCFilters(GetCFilters { @@ -1542,6 +1547,183 @@ mod tests { panic!("the reader of {id:?} never counted {} bytes", bytes.len()); } + /// Advances the clock `secs` seconds in flow controller ticks, nobody receiving. + async fn advance_silent(secs: u64) { + for _ in 0..secs * 4 { + time::advance(FLOW_TICK).await; + settle().await; + } + } + + /// A framed `cfilter` for a peer to receive. Its block hash is not the stop hash of + /// [`get_cfilters`], so it settles no charge and the peer's work stays in flight. + fn unsettling_filter() -> Vec { + serialize(&RawNetworkMessage { + magic: Network::Regtest.magic(), + payload: NetworkMessage::CFilter(CFilter { + filter_type: 0, + block_hash: BlockHash::dummy(1), + filter: vec![0; 32 * 1024], + }), + }) + } + + /// A connected peer that receives a run of [`unsettling_filter`]s at its own rate. + struct Feeder { + id: PeerId, + handle: PeerHandle, + far: OwnedWriteHalf, + rate: u64, + sent: usize, + } + + impl Feeder { + /// Delivers what the peer receives in one [`FLOW_TICK`]. + async fn tick(&mut self, net: &PeerNetworkManager, frame: &[u8]) { + let mut due = (self.rate as f64 * FLOW_TICK.as_secs_f64()) as usize; + while due > 0 { + let at = self.sent % frame.len(); + let piece = &frame[at..frame.len().min(at + due)]; + deliver(net, self.id, &mut self.far, piece).await; + self.sent += piece.len(); + due -= piece.len(); + } + } + } + + /// Connects peer `n + 1` for each of `rates` and feeds it `rates[n]` bytes/s until the + /// flow controller has measured how fast every peer serves. + async fn measured_peers(net: &PeerNetworkManager, rates: &[u64]) -> Vec { + let session = net.session.lock().await.clone(); + let mut feeders = Vec::new(); + for (n, &rate) in rates.iter().enumerate() { + let id = conn(n as u8 + 1, 0); + let (peer, far) = ConnectedPeer::for_test_reading(id, net.state.clone()).await; + let handle = peer.handle(); + let admitted = + net.state.connected.admit(peer, &session, net.state.max_peers, None).await; + assert!(matches!(admitted, Admission::Joined)); + feeders.push(Feeder { + id, + handle, + far, + rate, + sent: 0, + }); + } + feed(net, &mut feeders, SERVING_SAMPLE_TICKS * 2).await; + let measured = net.state.connected.read().await.iter().all(|p| p.serving_rate() > 0); + assert!(measured, "every peer has a serving rate"); + feeders + } + + /// Feeds every peer at its rate for `ticks` flow controller ticks, with work up to its + /// cap, as bulk download keeps it. + async fn feed(net: &PeerNetworkManager, feeders: &mut [Feeder], ticks: u32) { + let frame = unsettling_filter(); + for _ in 0..ticks { + for feeder in feeders.iter() { + while feeder.handle.in_flight_bytes() < feeder.handle.cap() { + let _ = feeder.handle.send(&get_cfilters(0)).await; + } + } + time::advance(FLOW_TICK).await; + for feeder in feeders.iter_mut() { + feeder.tick(net, &frame).await; + } + settle().await; + } + } + + /// The peers' serving rates, by connection. + async fn serving_rates(net: &PeerNetworkManager) -> Vec { + net.state.connected.read().await.iter().map(ConnectedPeer::serving_rate).collect() + } + + /// A peer on loopback that answers the version handshake once `go` is notified, then + /// answers pings. + struct HandshakingPeer { + addr: SocketAddr, + go: Arc, + accepted: Arc, + /// Whether a ping came after the handshake, which the client sends last. + pinged: Arc, + /// Whether the client closed the connection. + closed: Arc, + } + + impl HandshakingPeer { + async fn listen() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind loopback"); + let peer = HandshakingPeer { + addr: listener.local_addr().expect("bound address"), + go: Arc::new(Notify::new()), + accepted: Arc::new(AtomicUsize::new(0)), + pinged: Arc::new(AtomicBool::new(false)), + closed: Arc::new(AtomicBool::new(false)), + }; + let (addr, go) = (peer.addr, peer.go.clone()); + let (accepted, pinged, closed) = + (peer.accepted.clone(), peer.pinged.clone(), peer.closed.clone()); + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + accepted.fetch_add(1, Ordering::SeqCst); + let (go, pinged, closed) = (go.clone(), pinged.clone(), closed.clone()); + tokio::spawn(answer_handshake(stream, addr, go, pinged, closed)); + } + }); + peer + } + } + + /// Answers the client's version handshake on `stream` once `go` is notified, then its + /// pings, for [`HandshakingPeer`]. + async fn answer_handshake( + stream: TcpStream, + addr: SocketAddr, + go: Arc, + pinged: Arc, + closed: Arc, + ) { + stream.set_nodelay(true).expect("loopback nodelay"); + let (read, mut write) = stream.into_split(); + let mut frames = FramedRead::new(read, RawNetworkMessageCodec); + let framed = |payload| { + serialize(&RawNetworkMessage { + magic: Network::Regtest.magic(), + payload, + }) + }; + while let Some(Ok(raw)) = frames.next().await { + if matches!(raw.payload, NetworkMessage::Version(_)) { + break; + } + } + go.notified().await; + let services = ServiceFlags::NETWORK | ServiceFlags::COMPACT_FILTERS | ServiceFlags::BLOOM; + let version = VersionMessage::new( + services, + 0, + Address::new(&addr, ServiceFlags::NONE), + Address::new(&addr, services), + 1, + "/handshaking/".into(), + 0, + false, + [0; 32], + ); + let mut hello = framed(NetworkMessage::Version(version)); + hello.extend(framed(NetworkMessage::Verack)); + let _ = write.write_all(&hello).await; + while let Some(Ok(raw)) = frames.next().await { + if let NetworkMessage::Ping(nonce) = raw.payload { + pinged.store(true, Ordering::SeqCst); + let _ = write.write_all(&framed(NetworkMessage::Pong(nonce))).await; + } + } + closed.store(true, Ordering::SeqCst); + } + /// Lets the spawned tasks run what the last time advance made due. async fn settle() { for _ in 0..4 { @@ -2025,17 +2207,68 @@ mod tests { assert_eq!(judge.link_stalled, link_stalled); } - /// A peer serves 1 MB/s with work for three ticks, then nothing for one. - #[test_case(false, true => 750_000; "a stall of its own drags its serving rate")] - #[test_case(true, true => 1_000_000; "a stall of our link does not")] - #[test_case(false, false => 1_000_000; "idle ticks do not")] - fn the_serving_rate_averages_ticks_with_work(link_stalled: bool, has_work: bool) -> u64 { - let mut flow = PeerFlow::new(0); - for _ in 0..3 { - flow.sample(1e6, true, false); + /// A peer's silence with work drags its serving rate. Once every peer is silent long + /// enough for the judge to hold them as a possible stall of our own link, the flow + /// controller leaves the rates alone, since such a stall says nothing about the peers. + #[tokio::test(start_paused = true)] + async fn a_stall_of_our_link_leaves_the_serving_rates_alone() { + let net = broker().await; + let _feeders = measured_peers(&net, &[1 << 20, 200 << 10]).await; + let measured = serving_rates(&net).await; + + advance_silent(REQUEST_TIMEOUT.as_secs() + 2).await; + let held = serving_rates(&net).await; + assert!(held.iter().zip(&measured).all(|(h, m)| h < m), "{held:?} vs {measured:?}"); + + advance_silent(REQUEST_TIMEOUT.as_secs() * 3).await; + assert_eq!(serving_rates(&net).await, held); + assert_eq!(net.connected_count().await, 2, "a held peer was kicked"); + } + + /// Peer 2 serves a fifth of peer 1's rate, then both go silent and the supervisor starts + /// `start_at` seconds in. Past the judge's hold it must not dial. When the peers deliver + /// again, the held back swap goes ahead. + #[test_case(REQUEST_TIMEOUT.as_secs() + 2, 0; "a round during the stall dials nobody")] + #[tokio::test(start_paused = true)] + async fn no_slow_peer_is_swapped_out_while_our_link_stalls(start_at: u64, dials: usize) { + let replacement = HandshakingPeer::listen().await; + let mut config = ClientConfig::regtest().with_restrict_to_configured_peers(true); + config.add_peer(replacement.addr); + config.max_peers = 2; + config.transport_preference = TransportPreference::V1Only; + let net = PeerNetworkManager::new(&config).await; + let mut feeders = measured_peers(&net, &[1 << 20, 200 << 10]).await; + let slow = feeders[1].handle.clone(); + + advance_silent(start_at).await; + net.start().await; + settle().await; + advance_silent((REQUEST_TIMEOUT.as_secs() + 2).saturating_sub(start_at)).await; + replacement.go.notify_one(); + for _ in 0..100_000 { + if replacement.closed.load(Ordering::SeqCst) || slow.is_closed() { + break; + } + task::yield_now().await; + } + + assert_eq!(replacement.accepted.load(Ordering::SeqCst), dials); + assert_eq!(replacement.pinged.load(Ordering::SeqCst), dials > 0, "handshake finished"); + assert!(!slow.is_closed(), "the slow peer was swapped out"); + assert_eq!(net.connected_count().await, 2); + + // The peers deliver again, which proves our link works, and the held back swap goes + // ahead. Nothing but the pick of a slow peer wakes a full supervisor. + replacement.go.notify_one(); + for _ in 0..8 * 4 { + feed(&net, &mut feeders, 1).await; + if slow.is_closed() { + break; + } } - flow.sample(0.0, has_work, link_stalled); - flow.serving_rate as u64 + assert!(slow.is_closed(), "the slow peer was kept once our link recovered"); + assert_eq!(replacement.accepted.load(Ordering::SeqCst), dials + 1); + assert_eq!(net.connected_count().await, 2); } #[test_case(&[&[MessageType::Inv], &[MessageType::Inv]], 0, &[1, 1]; "every subscriber of the type")] From 10ae4d2fff168951ae53ce4f88d1462fd688e2c1 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Fri, 13 Nov 2026 14:26:17 +1000 Subject: [PATCH 07/11] fix(dash-spv): swap no slow peer once our link stalls mid-dial `repair_round` picks the slow peer to replace before it dials the candidates, which can take up to `CONNECT_DEADLINE`. If our own link stalled during the dial, `admit` still swapped out that peer, closing it and re-queuing its work in the middle of the stall. The supervisor now checks `link_stalled` again when it admits a candidate. A new case of `no_slow_peer_is_swapped_out_while_our_link_stalls` lets the stall begin while the replacement is being dialed. --- dash-spv/src/network/manager.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 99564df20..2c495c0af 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -930,7 +930,9 @@ impl Supervisor { } let new_addr = peer.addr(); - let slow = slow.filter(|_| replaced.is_none()); + // Our link can stall while the candidates are dialed, and then `slow` is stale. + let stalled = self.state.link_stalled.load(Ordering::Relaxed); + let slow = slow.filter(|_| replaced.is_none() && !stalled); match self.state.connected.admit(peer, &self.shutdown, self.state.max_peers, slow).await { Admission::Joined => accepted += 1, @@ -2226,9 +2228,10 @@ mod tests { } /// Peer 2 serves a fifth of peer 1's rate, then both go silent and the supervisor starts - /// `start_at` seconds in. Past the judge's hold it must not dial. When the peers deliver - /// again, the held back swap goes ahead. + /// `start_at` seconds in. While the judge holds them it must not swap peer 2 out, nor dial + /// once the hold began. When the peers deliver again, the held back swap goes ahead. #[test_case(REQUEST_TIMEOUT.as_secs() + 2, 0; "a round during the stall dials nobody")] + #[test_case(REQUEST_TIMEOUT.as_secs() - 2, 1; "a stall during the dial keeps the slow peer")] #[tokio::test(start_paused = true)] async fn no_slow_peer_is_swapped_out_while_our_link_stalls(start_at: u64, dials: usize) { let replacement = HandshakingPeer::listen().await; From 5bcbe83a3e330bc6a2d1dce0d581097b879f6b95 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Fri, 13 Nov 2026 15:52:01 +1000 Subject: [PATCH 08/11] fix(dash-spv): ban no peer kicked only by the hold bound A stall of our own link longer than the hold still kicks the held peers one per timeout, and every kicked peer was banned for 10 minutes with `Ban::Kicked`. Once the link came back, the supervisor could not redial the very peers our link had failed. `StallJudge::judge` now also returns the peer it kicked because a hold bound ran out with no peer witnessing its silence. The monitor still kicks that peer and re-queues its work, but does not ban it: it is most likely a victim of our link, and if it is dead after all, redialing it fails and `Ban::DialFailed` backs it off. A peer kicked while another peer proves the link works is banned as before, and so is a culprit that already left the set. Every case of `the_stall_judge_kicks_one_peer_per_shared_stall` now also states which kicks go unbanned. Through the real monitor, the peers of `a_departed_peer_is_announced_once` that are kicked past the bound stay unbanned, while the peer `the_timeout_monitor_requeues_stalled_requests` kicks beside a witness is banned. --- dash-spv/src/network/manager.rs | 116 ++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 50 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 2c495c0af..ed8f4eb5c 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1116,14 +1116,15 @@ impl StallJudge { streaming } - /// Returns the peers to kick and how many culprits are spared. `culprits` maps the - /// peers of stalled requests to when their silence began. + /// Returns the peers to kick, how many culprits are spared, and the kicked peer no peer + /// witnessed, if any. `culprits` maps the peers of stalled requests to when their + /// silence began. fn judge( &mut self, now: Instant, peers: &[Observed], mut culprits: HashMap, - ) -> (HashSet, usize) { + ) -> (HashSet, usize, Option) { self.link_stalled = false; self.link_down = false; // Judged on the peer's own counters, not just the registry: streaming in-flight @@ -1140,11 +1141,11 @@ impl StallJudge { } } if culprits.is_empty() { - return (HashSet::new(), 0); + return (HashSet::new(), 0, None); } let mut held = 0; - let mut silent: Vec<(Instant, &Observed)> = Vec::new(); + let mut silent: Vec<(Instant, &Observed, bool)> = Vec::new(); for p in peers { let Some(&since) = culprits.get(&p.id) else { continue; @@ -1155,11 +1156,12 @@ impl StallJudge { let witnessed_from = |from: Instant| self.last_heard.is_some_and(|heard| heard > from + p.timeout); let blamed_from = self.link_back.map_or(since, |back| back.max(since)); - if witnessed_from(blamed_from) + let witnessed = witnessed_from(blamed_from); + if witnessed || now.duration_since(blamed_from) > p.timeout * LINK_STALL_TIMEOUTS || now.duration_since(since) > p.timeout * MAX_HOLD_TIMEOUTS { - silent.push((since, p)); + silent.push((since, p, witnessed)); } else { self.link_down |= !witnessed_from(since); held += 1; @@ -1169,8 +1171,8 @@ impl StallJudge { // A silent peer may still be our link failing it, so the one silent the longest // goes, and one kick per timeout keeps a stall of our link from taking the whole set // at once. - silent.sort_by_key(|&(since, p)| (since, p.id.addr())); - let due = silent.first().is_some_and(|(_, worst)| { + silent.sort_by_key(|&(since, p, _)| (since, p.id.addr())); + let due = silent.first().is_some_and(|(_, worst, _)| { self.last_kick.is_none_or(|at| now.duration_since(at) > worst.timeout) }); if due { @@ -1179,14 +1181,16 @@ impl StallJudge { // A culprit that already left is not spared, so its requests are still re-queued. let mut kicked: HashSet = culprits.into_keys().filter(|id| peers.iter().all(|p| p.id != *id)).collect(); - kicked.extend(silent.first().filter(|_| due).map(|(_, p)| p.id)); - (kicked, held + silent.len() - usize::from(due)) + let kick = silent.first().filter(|_| due); + kicked.extend(kick.map(|(_, p, _)| p.id)); + let unwitnessed = kick.filter(|(_, _, witnessed)| !witnessed).map(|(_, p, _)| p.id); + (kicked, held + silent.len() - usize::from(due), unwitnessed) } } -/// Every [`TIMEOUT_CHECK`], kicks the peers [`StallJudge`] finds stalled on requests -/// with no progress for a [`REQUEST_TIMEOUT`], re-queues all their on-wire requests for -/// other peers and wakes the supervisor to refill the set. +/// Every [`TIMEOUT_CHECK`], kicks the peers [`StallJudge`] finds stalled on requests with no +/// progress for a [`REQUEST_TIMEOUT`], bans every kicked peer except one kicked with no witness, +/// re-queues their on-wire requests for other peers and wakes the supervisor to refill the set. fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) -> JoinHandle<()> { tokio::spawn(async move { let NetworkState { @@ -1225,7 +1229,8 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) let timeout = |id: &PeerId| { peers.iter().find(|p| p.id == *id).map_or(REQUEST_TIMEOUT, |p| p.timeout) }; - let (culprits, spared) = judge.judge(now, &peers, broker.expire(timeout).await); + let (culprits, spared, unwitnessed) = + judge.judge(now, &peers, broker.expire(timeout).await); link_stalled.store(judge.link_stalled, Ordering::Relaxed); if culprits.is_empty() { if spared > 0 { @@ -1248,7 +1253,9 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) // going away), keeping keys as Queued so de-dup holds. let retried = broker.requeue_from(&culprits).await; let dropped = kicked.len(); - for peer in &culprits { + // A peer kicked only because its hold ran out is likely a victim of our own link. + // If it is unreachable after all, redialing it fails and backs off. + for peer in culprits.iter().filter(|&&peer| Some(peer) != unwitnessed) { bans.ban(peer.addr(), Ban::Kicked).await; } // The set is now short: wake the supervisor, which otherwise idles. @@ -1877,6 +1884,8 @@ mod tests { time::advance(REQUEST_TIMEOUT * LINK_STALL_TIMEOUTS + TIMEOUT_CHECK).await; settle().await; } + // No peer witnessed the silence, so it may have been our link's. + assert!(net.state.bans.banned().await.is_empty(), "a held peer was banned"); } Departure::Stop => { net.stop().await; @@ -2010,6 +2019,8 @@ mod tests { // A stalled request is re-queued and may already be routed to peer 2. let carrier = net.state.broker.carrier_of(&RequestKey::CFilters(0)).await; assert_eq!(carrier == Some(conn(1, 0)), !stalls, "carrier {carrier:?}"); + let banned = net.state.bans.banned().await.contains(&conn(1, 0).addr()); + assert_eq!(banned, stalls, "a peer kicked beside a witness is banned"); } /// A timeout kick re-queues the stalled peer's work only once that peer is closed and out @@ -2105,44 +2116,45 @@ mod tests { } /// Peer `n + 1` plays `roles[n]` for `secs` seconds. `link_stalled` is whether the - /// judge then suspects our own link. - #[test_case(&[Role::Registered(0); 3], 11, &[1, 2, 3], true; "a stall of every peer at once is our link")] - #[test_case(&[Role::Registered(0); 3], 41, &[1, 2, 3], true; "a 40 s silence of every peer kicks nobody")] - #[test_case(&[Role::Registered(0); 3], 71, &[2, 3], false; "every peer silent past the bound loses the one silent the longest")] - #[test_case(&[Role::Registered(0); 3], 72, &[3], false; "every peer silent past the bound loses one per timeout")] - #[test_case(&[Role::Charged(0); 3], 25, &[1, 2, 3], true; "a long stall of every peer is still our link")] - #[test_case(&[Role::Registered(1), Role::Registered(2), Role::Registered(3)], 25, &[1, 2, 3], true; "peers going silent a second apart in an outage stay")] - #[test_case(&[Role::Charged(1), Role::Charged(2), Role::Charged(3)], 25, &[1, 2, 3], true; "peers owing bytes and going silent a second apart stay")] - #[test_case(&[Role::Delivers(0), Role::Registered(0), Role::Delivers(0)], 11, &[1, 3], false; "a lone silent request among delivering peers is kicked")] - #[test_case(&[Role::Delivers(0), Role::Charged(0), Role::Delivers(0)], 12, &[1, 3], false; "a lone peer owing bytes among delivering peers is kicked")] - #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(0)], 11, &[2, 3], false; "two stalls beside a delivering peer kick one")] - #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(0)], 22, &[2], false; "the second stall goes a timeout later")] - #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(1)], 12, &[2, 3], false; "stalls a second apart beside a delivering peer go one per timeout")] - #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(2)], 14, &[2, 3], false; "stalls two seconds apart beside a delivering peer go one per timeout")] - #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(1), Role::Joins(11)], 12, &[2, 3, 4], false; "a replacement joining does not unpace the next stall")] - #[test_case(&[Role::Registered(0), Role::Delivers(13), Role::Registered(1)], 24, &[2, 3], false; "a late witness kicks the peer silent the longest a timeout later")] - #[test_case(&[Role::Resumes(5, 20), Role::Resumes(5, 20), Role::Charged(5)], 25, &[1, 2, 3], true; "a peer still silent when the link is back gets a timeout")] - #[test_case(&[Role::Resumes(5, 20), Role::Resumes(5, 20), Role::Charged(5)], 31, &[1, 2], false; "a peer silent a timeout after the link is back is kicked")] - #[test_case(&[Role::Charged(0), Role::Whispers(3)], 13, &[2], false; "a lone peer owing bytes beside an idle one is no link stall")] - #[test_case(&[Role::Registered(0)], 41, &[1], true; "a sole silent peer stays through a 40 s outage")] - #[test_case(&[Role::Registered(0)], 61, &[], false; "a sole silent peer is kicked past the bound")] - #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Joins(12)], 13, &[1, 2, 3], true; "a peer joining during an outage proves nothing")] - #[test_case(&[Role::Charged(0), Role::Delivers(30)], 15, &[1, 2], true; "a lone peer owing bytes beside a quiet idle one is held")] - #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 35, &[1, 2, 3], true; "an idle peer receiving first after an outage gives the held peers a timeout")] - #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 36, &[2, 3], false; "held peers silent a timeout after an idle peer receives go")] - #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 47, &[3], false; "held peers silent after an idle peer receives go one per timeout")] - #[test_case(&[Role::Charged(0), Role::Whispers(15)], 30, &[2], false; "an idle peer's later bytes do not restart a held peer's timeout")] - #[test_case(&[Role::Resumes(0, 55), Role::Charged(0)], 65, &[1, 2], true; "a peer silent through a 55 s outage gets a timeout after it")] - #[test_case(&[Role::Resumes(0, 55), Role::Charged(0)], 66, &[1], false; "a peer silent a timeout after a 55 s outage is kicked")] - #[test_case(&[Role::Chatters(0)], 11, &[], false; "a sole peer receiving only small messages is kicked")] - #[test_case(&[Role::Chatters(25); 3], 36, &[2, 3], false; "peers receiving only small messages after an outage go one per timeout")] - #[test_case(&[Role::Charged(0), Role::Flaps(12, 5)], 121, &[1, 2], true; "a dead peer beside a flapping link is held up to the absolute bound")] - #[test_case(&[Role::Charged(0), Role::Flaps(12, 5)], 122, &[2], false; "a dead peer beside a flapping link is kicked at the absolute bound")] + /// judge then suspects our own link, `unbanned` the peers kicked with no witness. + #[test_case(&[Role::Registered(0); 3], 11, &[1, 2, 3], true, &[]; "a stall of every peer at once is our link")] + #[test_case(&[Role::Registered(0); 3], 41, &[1, 2, 3], true, &[]; "a 40 s silence of every peer kicks nobody")] + #[test_case(&[Role::Registered(0); 3], 71, &[2, 3], false, &[1]; "every peer silent past the bound loses the one silent the longest")] + #[test_case(&[Role::Registered(0); 3], 72, &[3], false, &[1, 2]; "every peer silent past the bound loses one per timeout")] + #[test_case(&[Role::Charged(0); 3], 25, &[1, 2, 3], true, &[]; "a long stall of every peer is still our link")] + #[test_case(&[Role::Registered(1), Role::Registered(2), Role::Registered(3)], 25, &[1, 2, 3], true, &[]; "peers going silent a second apart in an outage stay")] + #[test_case(&[Role::Charged(1), Role::Charged(2), Role::Charged(3)], 25, &[1, 2, 3], true, &[]; "peers owing bytes and going silent a second apart stay")] + #[test_case(&[Role::Delivers(0), Role::Registered(0), Role::Delivers(0)], 11, &[1, 3], false, &[]; "a lone silent request among delivering peers is kicked")] + #[test_case(&[Role::Delivers(0), Role::Charged(0), Role::Delivers(0)], 12, &[1, 3], false, &[]; "a lone peer owing bytes among delivering peers is kicked")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(0)], 11, &[2, 3], false, &[]; "two stalls beside a delivering peer kick one")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(0)], 22, &[2], false, &[]; "the second stall goes a timeout later")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(1)], 12, &[2, 3], false, &[]; "stalls a second apart beside a delivering peer go one per timeout")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(2)], 14, &[2, 3], false, &[]; "stalls two seconds apart beside a delivering peer go one per timeout")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(1), Role::Joins(11)], 12, &[2, 3, 4], false, &[]; "a replacement joining does not unpace the next stall")] + #[test_case(&[Role::Registered(0), Role::Delivers(13), Role::Registered(1)], 24, &[2, 3], false, &[]; "a late witness kicks the peer silent the longest a timeout later")] + #[test_case(&[Role::Resumes(5, 20), Role::Resumes(5, 20), Role::Charged(5)], 25, &[1, 2, 3], true, &[]; "a peer still silent when the link is back gets a timeout")] + #[test_case(&[Role::Resumes(5, 20), Role::Resumes(5, 20), Role::Charged(5)], 31, &[1, 2], false, &[]; "a peer silent a timeout after the link is back is kicked")] + #[test_case(&[Role::Charged(0), Role::Whispers(3)], 13, &[2], false, &[]; "a lone peer owing bytes beside an idle one is no link stall")] + #[test_case(&[Role::Registered(0)], 41, &[1], true, &[]; "a sole silent peer stays through a 40 s outage")] + #[test_case(&[Role::Registered(0)], 61, &[], false, &[1]; "a sole silent peer is kicked past the bound")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Joins(12)], 13, &[1, 2, 3], true, &[]; "a peer joining during an outage proves nothing")] + #[test_case(&[Role::Charged(0), Role::Delivers(30)], 15, &[1, 2], true, &[]; "a lone peer owing bytes beside a quiet idle one is held")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 35, &[1, 2, 3], true, &[]; "an idle peer receiving first after an outage gives the held peers a timeout")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 36, &[2, 3], false, &[]; "held peers silent a timeout after an idle peer receives go")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 47, &[3], false, &[]; "held peers silent after an idle peer receives go one per timeout")] + #[test_case(&[Role::Charged(0), Role::Whispers(15)], 30, &[2], false, &[]; "an idle peer's later bytes do not restart a held peer's timeout")] + #[test_case(&[Role::Resumes(0, 55), Role::Charged(0)], 65, &[1, 2], true, &[]; "a peer silent through a 55 s outage gets a timeout after it")] + #[test_case(&[Role::Resumes(0, 55), Role::Charged(0)], 66, &[1], false, &[]; "a peer silent a timeout after a 55 s outage is kicked")] + #[test_case(&[Role::Chatters(0)], 11, &[], false, &[]; "a sole peer receiving only small messages is kicked")] + #[test_case(&[Role::Chatters(25); 3], 36, &[2, 3], false, &[]; "peers receiving only small messages after an outage go one per timeout")] + #[test_case(&[Role::Charged(0), Role::Flaps(12, 5)], 121, &[1, 2], true, &[]; "a dead peer beside a flapping link is held up to the absolute bound")] + #[test_case(&[Role::Charged(0), Role::Flaps(12, 5)], 122, &[2], false, &[1]; "a dead peer beside a flapping link is kicked at the absolute bound")] fn the_stall_judge_kicks_one_peer_per_shared_stall( roles: &[Role], secs: u64, kept: &[u8], link_stalled: bool, + unbanned: &[u8], ) { // Every received byte but a chatter's is part of an answer still arriving. let peer = |n: u8, role: Role| Observed { @@ -2166,6 +2178,7 @@ mod tests { .map(|(n, _)| (conn(n, 0), start)) .collect(); let mut judge = StallJudge::default(); + let mut kicked_unbanned = Vec::new(); for second in 1..=secs { for (n, &role) in (1..).zip(roles) { @@ -2197,8 +2210,9 @@ mod tests { .filter(|(_, &progress)| now.duration_since(progress) > REQUEST_TIMEOUT) .map(|(&id, &progress)| (id, progress)) .collect(); - let (kicked, _) = judge.judge(now, &peers, stalled); + let (kicked, _, unwitnessed) = judge.judge(now, &peers, stalled); assert!(kicked.len() <= 1, "second {second} kicked {kicked:?}"); + kicked_unbanned.extend(unwitnessed); peers.retain(|p| !kicked.contains(&p.id)); requests.retain(|id, _| !kicked.contains(id)); } @@ -2207,6 +2221,8 @@ mod tests { let kept: Vec = kept.iter().map(|&n| conn(n, 0)).collect(); assert_eq!(left, kept); assert_eq!(judge.link_stalled, link_stalled); + let unbanned: Vec = unbanned.iter().map(|&n| conn(n, 0)).collect(); + assert_eq!(kicked_unbanned, unbanned, "kicked without a ban"); } /// A peer's silence with work drags its serving rate. Once every peer is silent long From ebb7dc38d93654dd931081548a25cf9bedff46bb Mon Sep 17 00:00:00 2001 From: xdustinface Date: Fri, 13 Nov 2026 21:05:16 +1000 Subject: [PATCH 09/11] fix(dash-spv): count our link's uptime during a peer's silence The judge kept a single instant for our link's return and restarted a culprit's witness clock from it. Every rule for setting it could be re-armed by ordinary traffic: two healthy peers pausing together, owing peers with reply gaps beside an idle stream, or a flapping link kept a dead culprit held up to two minutes and then kicked it unbanned, while a peer resuming on the first tick past the timeout dropped the evidence altogether. The judge now counts how long our link demonstrably worked. Each tick on which any peer's bytes grew adds the time since the tick before, at most `TIMEOUT_CHECK`, so a late monitor tick cannot count an outage as uptime. A culprit is witnessed once our link worked for more than a timeout of its silence, however outages split that time, and is kicked unbanned only after more than `LINK_STALL_TIMEOUTS` timeouts of downtime. Since uptime and downtime together are the whole silence, a hold ends within `LINK_STALL_TIMEOUTS` plus one timeouts, and `MAX_HOLD_TIMEOUTS`, `link_down` and `link_back` are gone. Uptime is kept as one cumulative sample per tick, back to the last tick before the oldest silence judged now or later: a peer's, a culprit's still waiting behind kick pacing, or a request's, which `Broker::expire` blames from at most `ANSWER_TIMEOUTS` timeouts back. A silence is measured from the last tick at or before it began, so a request sent between ticks is blamed on the first tick past its timeout. `link_stalled` stays set while any culprit is unwitnessed, held or waiting behind kick pacing. Rows of `the_stall_judge_kicks_one_peer_per_shared_stall` and `the_timeout_monitor_requeues_stalled_requests` whose request predates the judge's first tick are kicked a tick later, as uptime before that tick is unknown. Rows after an outage are kicked a tick earlier, as the tick that ends an outage counts. Sparse or flapping traffic now proves only as much uptime as it shows: a peer beside pings every 3 s is held until they add up to a timeout, and one beside a receipt every 10 s is kicked unbanned at the bound. The rows where every peer goes silent past the bound now report `link_stalled` while the rest wait to be kicked. New rows cover peers pausing together, an idle stream beside owing peers with gaps, a peer resuming on the timeout tick, 5 s and 40 s outages, culprits paced past the link's last uptime, a request sent between ticks and a sole peer streaming an answer it never finishes. --- dash-spv/src/network/broker.rs | 2 +- dash-spv/src/network/manager.rs | 178 ++++++++++++++++++++------------ 2 files changed, 114 insertions(+), 66 deletions(-) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index 7f34ebce3..f7cf4f882 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -15,7 +15,7 @@ const KICKING_ATTEMPTS: u32 = 2; /// How many timeouts a request lives on bytes of a message still arriving, without an /// answer. A big answer on a slow link must fit, but a message that never finishes must /// not keep its request alive forever. -const ANSWER_TIMEOUTS: u32 = 6; +pub(super) const ANSWER_TIMEOUTS: u32 = 6; const HEDGE_AFTER: Duration = Duration::from_secs(2); const MIN_IN_FLIGHT_PER_CLASS: usize = 4; const MSG_CLASSES: usize = 5; diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index ed8f4eb5c..25ee1dc42 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -64,19 +64,14 @@ const MAX_ADDR_PER_MESSAGE: usize = 1000; /// How often the timeout monitor scans the outstanding-request registry. const TIMEOUT_CHECK: Duration = Duration::from_secs(1); -/// How many request timeouts a silent peer is held while no peer proves our link -/// works, counted from its silence or from our link's return if later. With the 10 s -/// base timeout that is 60 s, enough for the 40 s outages seen on testnet. +/// How many request timeouts of our link's downtime a silent peer is held for: the part of +/// its silence in which no peer received anything. With the 10 s base timeout that is 60 s, +/// enough for the 40 s outages seen on testnet. const LINK_STALL_TIMEOUTS: u32 = 6; -/// How many request timeouts a silent peer is held at most, counted from its silence, -/// however often our link returns. The link's last return within a hold is at most -/// [`LINK_STALL_TIMEOUTS`] in, so this never shortens a hold that a single return allows. -const MAX_HOLD_TIMEOUTS: u32 = 2 * LINK_STALL_TIMEOUTS; - use crate::error::{NetworkError, NetworkResult}; use crate::network::bans::{Ban, Bans}; -use crate::network::broker::{Broker, RequestKey}; +use crate::network::broker::{Broker, RequestKey, ANSWER_TIMEOUTS}; use crate::network::peer_set::{Admission, PeerSet}; use crate::network::NetworkManager; use crate::network::TransportPreference; @@ -1063,25 +1058,26 @@ struct Liveness { owing_since: Instant, } -/// Decides which peers [`spawn_timeout_monitor`] kicks. A peer silent with work is held while -/// no peer proves our link works, as long as [`LINK_STALL_TIMEOUTS`] and [`MAX_HOLD_TIMEOUTS`] -/// allow. Silent peers are kicked one per timeout, the one silent the longest first. +/// Decides which peers [`spawn_timeout_monitor`] kicks. A peer silent with work is blamed once +/// our link worked for more than a timeout of its silence, or after more than +/// [`LINK_STALL_TIMEOUTS`] timeouts of downtime. At most one is kicked per timeout. #[derive(Default)] struct StallJudge { liveness: HashMap, - /// When some peer's bytes last grew. A new peer's first counter is not growth: its - /// handshake may predate a stall of our link, so it proves nothing about the link now. - last_heard: Option, /// The last kick of a silent peer, which paces the next. last_kick: Option, - /// Whether our own link may be stalled: the last judgement held a silent peer because - /// no peer proved our link works. + /// Whether our own link may be stalled: the last judgement left a culprit no peer + /// witnessed, held or waiting to be kicked. link_stalled: bool, - /// Whether some held peer has had no witness since its silence began, so the next - /// bytes from any peer show our link is back. - link_down: bool, - /// When some peer first received after our link went down. - link_back: Option, + /// When [`Self::observe`] last ran. + last_tick: Option, + /// How long our link has provably worked: each tick on which some peer's bytes grew adds + /// the time since the tick before, at most [`TIMEOUT_CHECK`]. A new peer's first counter + /// is not growth. + link_up: Duration, + /// `link_up` at each tick, back to the last tick at or before the oldest silence judged + /// now or later. + uptime: VecDeque<(Instant, Duration)>, } impl StallJudge { @@ -1090,6 +1086,7 @@ impl StallJudge { /// are progress on a request. fn observe(&mut self, now: Instant, peers: &[Observed]) -> Vec { self.liveness.retain(|id, _| peers.iter().any(|p| p.id == *id)); + let mut any_grew = false; let mut streaming = Vec::new(); for p in peers { let entry = self.liveness.entry(p.id).or_insert(Liveness { @@ -1098,11 +1095,8 @@ impl StallJudge { }); let grew = p.bytes > entry.bytes; if grew { - if self.link_down { - self.link_back = Some(now); - } + any_grew = true; entry.bytes = p.bytes; - self.last_heard = Some(now); if p.partial { streaming.push(p.id); } @@ -1113,9 +1107,23 @@ impl StallJudge { entry.owing_since = now; } } + // The cap keeps ticks the monitor ran late from counting an outage as uptime. + if let Some(last) = self.last_tick.filter(|_| any_grew) { + self.link_up += now.duration_since(last).min(TIMEOUT_CHECK); + } + self.last_tick = Some(now); + self.uptime.push_back((now, self.link_up)); streaming } + /// How long our link worked since `from`. It overcounts by up to one tick. + fn uptime_since(&self, from: Instant) -> Duration { + // The last sample at or before `from`, or the oldest kept if all are later. + let last = self.uptime.partition_point(|&(at, _)| at <= from).saturating_sub(1); + let before = self.uptime.get(last).map_or(Duration::ZERO, |&(_, up)| up); + self.link_up.saturating_sub(before) + } + /// Returns the peers to kick, how many culprits are spared, and the kicked peer no peer /// witnessed, if any. `culprits` maps the peers of stalled requests to when their /// silence began. @@ -1126,7 +1134,6 @@ impl StallJudge { mut culprits: HashMap, ) -> (HashSet, usize, Option) { self.link_stalled = false; - self.link_down = false; // Judged on the peer's own counters, not just the registry: streaming in-flight // bytes are freed on the peer separately, so a wedged peer can sit at its cap // with no registry entries. Pending work plus a byte counter @@ -1140,6 +1147,17 @@ impl StallJudge { *since = (*since).min(owing_since); } } + // Uptime is kept back to the oldest silence judged now or later: a peer's, a culprit's, + // or a request's, which `Broker::expire` blames from at most `ANSWER_TIMEOUTS` back. + let oldest = peers + .iter() + .filter_map(|p| now.checked_sub(p.timeout * ANSWER_TIMEOUTS)) + .chain(self.liveness.values().map(|l| l.owing_since)) + .chain(culprits.values().copied()) + .min(); + while self.uptime.get(1).is_some_and(|&(at, _)| oldest.is_none_or(|oldest| at <= oldest)) { + self.uptime.pop_front(); + } if culprits.is_empty() { return (HashSet::new(), 0, None); } @@ -1150,24 +1168,17 @@ impl StallJudge { let Some(&since) = culprits.get(&p.id) else { continue; }; - // Silence is blamed only once any peer, this one included, received a full - // timeout into it, which proves our link works. Silence before the link came - // back does not count, so the peer's own connection gets a timeout to resume. - let witnessed_from = - |from: Instant| self.last_heard.is_some_and(|heard| heard > from + p.timeout); - let blamed_from = self.link_back.map_or(since, |back| back.max(since)); - let witnessed = witnessed_from(blamed_from); - if witnessed - || now.duration_since(blamed_from) > p.timeout * LINK_STALL_TIMEOUTS - || now.duration_since(since) > p.timeout * MAX_HOLD_TIMEOUTS - { + // Silence is blamed once our link worked for more than a timeout of it, so the peer + // had that long to answer, however outages split the uptime. + let up = self.uptime_since(since); + let down = now.duration_since(since).saturating_sub(up); + let witnessed = up > p.timeout; + if witnessed || down > p.timeout * LINK_STALL_TIMEOUTS { silent.push((since, p, witnessed)); } else { - self.link_down |= !witnessed_from(since); held += 1; } } - self.link_stalled = held > 0; // A silent peer may still be our link failing it, so the one silent the longest // goes, and one kick per timeout keeps a stall of our link from taking the whole set // at once. @@ -1178,6 +1189,9 @@ impl StallJudge { if due { self.last_kick = Some(now); } + // The culprits waiting behind kick pacing may be our link's victims too. + let mut waiting = silent.iter().skip(usize::from(due)); + self.link_stalled = held > 0 || waiting.any(|&(_, _, witnessed)| !witnessed); // A culprit that already left is not spared, so its requests are still re-queued. let mut kicked: HashSet = culprits.into_keys().filter(|id| peers.iter().all(|p| p.id != *id)).collect(); @@ -1953,13 +1967,13 @@ mod tests { } /// Peer 1 carries a request for `secs` seconds while idle peer 2 keeps receiving. - #[test_case(Feed::Nothing, 11, true; "a silent request")] + #[test_case(Feed::Nothing, 12, true; "a silent request")] #[test_case(Feed::Answers(1), 20, false; "the carrier answering keeps it alive")] - #[test_case(Feed::Answers(2), 11, true; "another peer answering is not progress")] + #[test_case(Feed::Answers(2), 12, true; "another peer answering is not progress")] #[test_case(Feed::Streams(1), 55, false; "the carrier streaming one long answer keeps it alive")] #[test_case(Feed::Streams(1), 61, true; "the carrier streaming one answer past the bound stalls")] - #[test_case(Feed::Streams(2), 11, true; "another peer streaming is not progress")] - #[test_case(Feed::Chatters(1), 11, true; "the carrier sending only small messages stalls")] + #[test_case(Feed::Streams(2), 12, true; "another peer streaming is not progress")] + #[test_case(Feed::Chatters(1), 12, true; "the carrier sending only small messages stalls")] #[tokio::test(start_paused = true)] async fn the_timeout_monitor_requeues_stalled_requests(feed: Feed, secs: u64, stalls: bool) { let net = broker().await; @@ -2113,42 +2127,62 @@ mod tests { /// Owes bytes. Repeats being silent for the first field's seconds and receiving for /// the second's. Flaps(u64, u64), + /// Holds a request the registry tracks, sent this many milliseconds in, and receives + /// nothing. + Asks(u64), + /// Holds a request the registry tracks and receives part of its answer every this many + /// seconds, never all of it. + Streams(u64), } /// Peer `n + 1` plays `roles[n]` for `secs` seconds. `link_stalled` is whether the /// judge then suspects our own link, `unbanned` the peers kicked with no witness. #[test_case(&[Role::Registered(0); 3], 11, &[1, 2, 3], true, &[]; "a stall of every peer at once is our link")] #[test_case(&[Role::Registered(0); 3], 41, &[1, 2, 3], true, &[]; "a 40 s silence of every peer kicks nobody")] - #[test_case(&[Role::Registered(0); 3], 71, &[2, 3], false, &[1]; "every peer silent past the bound loses the one silent the longest")] - #[test_case(&[Role::Registered(0); 3], 72, &[3], false, &[1, 2]; "every peer silent past the bound loses one per timeout")] + #[test_case(&[Role::Registered(0); 3], 71, &[2, 3], true, &[1]; "every peer silent past the bound loses the one silent the longest")] + #[test_case(&[Role::Registered(0); 3], 72, &[3], true, &[1, 2]; "every peer silent past the bound loses one per timeout")] #[test_case(&[Role::Charged(0); 3], 25, &[1, 2, 3], true, &[]; "a long stall of every peer is still our link")] #[test_case(&[Role::Registered(1), Role::Registered(2), Role::Registered(3)], 25, &[1, 2, 3], true, &[]; "peers going silent a second apart in an outage stay")] #[test_case(&[Role::Charged(1), Role::Charged(2), Role::Charged(3)], 25, &[1, 2, 3], true, &[]; "peers owing bytes and going silent a second apart stay")] - #[test_case(&[Role::Delivers(0), Role::Registered(0), Role::Delivers(0)], 11, &[1, 3], false, &[]; "a lone silent request among delivering peers is kicked")] + #[test_case(&[Role::Delivers(0), Role::Registered(0), Role::Delivers(0)], 12, &[1, 3], false, &[]; "a lone silent request among delivering peers is kicked")] #[test_case(&[Role::Delivers(0), Role::Charged(0), Role::Delivers(0)], 12, &[1, 3], false, &[]; "a lone peer owing bytes among delivering peers is kicked")] - #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(0)], 11, &[2, 3], false, &[]; "two stalls beside a delivering peer kick one")] - #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(0)], 22, &[2], false, &[]; "the second stall goes a timeout later")] + #[test_case(&[Role::Delivers(0), Role::Asks(1500), Role::Delivers(0)], 12, &[1, 3], false, &[]; "a request sent between ticks is kicked on the first tick past its timeout")] + #[test_case(&[Role::Streams(3)], 61, &[], false, &[]; "a sole peer streaming a never finished answer is kicked at the answer bound")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(0)], 12, &[2, 3], false, &[]; "two stalls beside a delivering peer kick one")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(0)], 23, &[2], false, &[]; "the second stall goes a timeout later")] #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(1)], 12, &[2, 3], false, &[]; "stalls a second apart beside a delivering peer go one per timeout")] #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(2)], 14, &[2, 3], false, &[]; "stalls two seconds apart beside a delivering peer go one per timeout")] #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(1), Role::Joins(11)], 12, &[2, 3, 4], false, &[]; "a replacement joining does not unpace the next stall")] #[test_case(&[Role::Registered(0), Role::Delivers(13), Role::Registered(1)], 24, &[2, 3], false, &[]; "a late witness kicks the peer silent the longest a timeout later")] #[test_case(&[Role::Resumes(5, 20), Role::Resumes(5, 20), Role::Charged(5)], 25, &[1, 2, 3], true, &[]; "a peer still silent when the link is back gets a timeout")] #[test_case(&[Role::Resumes(5, 20), Role::Resumes(5, 20), Role::Charged(5)], 31, &[1, 2], false, &[]; "a peer silent a timeout after the link is back is kicked")] - #[test_case(&[Role::Charged(0), Role::Whispers(3)], 13, &[2], false, &[]; "a lone peer owing bytes beside an idle one is no link stall")] + #[test_case(&[Role::Charged(0), Role::Whispers(3)], 32, &[1, 2], true, &[]; "a lone peer owing bytes beside a pinging idle one is held while the pings add up")] + #[test_case(&[Role::Charged(0), Role::Whispers(3)], 33, &[2], false, &[]; "a lone peer owing bytes beside a pinging idle one is kicked once they add up to a timeout")] #[test_case(&[Role::Registered(0)], 41, &[1], true, &[]; "a sole silent peer stays through a 40 s outage")] #[test_case(&[Role::Registered(0)], 61, &[], false, &[1]; "a sole silent peer is kicked past the bound")] #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Joins(12)], 13, &[1, 2, 3], true, &[]; "a peer joining during an outage proves nothing")] #[test_case(&[Role::Charged(0), Role::Delivers(30)], 15, &[1, 2], true, &[]; "a lone peer owing bytes beside a quiet idle one is held")] - #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 35, &[1, 2, 3], true, &[]; "an idle peer receiving first after an outage gives the held peers a timeout")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 34, &[1, 2, 3], true, &[]; "an idle peer receiving first after an outage gives the held peers a timeout")] #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 36, &[2, 3], false, &[]; "held peers silent a timeout after an idle peer receives go")] #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 47, &[3], false, &[]; "held peers silent after an idle peer receives go one per timeout")] - #[test_case(&[Role::Charged(0), Role::Whispers(15)], 30, &[2], false, &[]; "an idle peer's later bytes do not restart a held peer's timeout")] - #[test_case(&[Role::Resumes(0, 55), Role::Charged(0)], 65, &[1, 2], true, &[]; "a peer silent through a 55 s outage gets a timeout after it")] - #[test_case(&[Role::Resumes(0, 55), Role::Charged(0)], 66, &[1], false, &[]; "a peer silent a timeout after a 55 s outage is kicked")] - #[test_case(&[Role::Chatters(0)], 11, &[], false, &[]; "a sole peer receiving only small messages is kicked")] + #[test_case(&[Role::Charged(0), Role::Whispers(10)], 67, &[1, 2], true, &[]; "a lone peer owing bytes beside sparse traffic is held up to the bound")] + #[test_case(&[Role::Charged(0), Role::Whispers(10)], 68, &[2], false, &[1]; "a lone peer owing bytes beside sparse traffic is kicked at the bound")] + #[test_case(&[Role::Resumes(0, 55), Role::Charged(0)], 64, &[1, 2], true, &[]; "a peer silent through a 55 s outage gets a timeout after it")] + #[test_case(&[Role::Resumes(0, 55), Role::Charged(0)], 65, &[1], false, &[]; "a peer silent a timeout after a 55 s outage is kicked")] + #[test_case(&[Role::Chatters(0)], 12, &[], false, &[]; "a sole peer receiving only small messages is kicked")] #[test_case(&[Role::Chatters(25); 3], 36, &[2, 3], false, &[]; "peers receiving only small messages after an outage go one per timeout")] - #[test_case(&[Role::Charged(0), Role::Flaps(12, 5)], 121, &[1, 2], true, &[]; "a dead peer beside a flapping link is held up to the absolute bound")] - #[test_case(&[Role::Charged(0), Role::Flaps(12, 5)], 122, &[2], false, &[1]; "a dead peer beside a flapping link is kicked at the absolute bound")] + #[test_case(&[Role::Charged(0), Role::Flaps(12, 5)], 46, &[1, 2], true, &[]; "a dead peer beside a flapping link is held while its uptime adds up")] + #[test_case(&[Role::Charged(0), Role::Flaps(12, 5)], 47, &[2], false, &[]; "a dead peer beside a flapping link is kicked once its uptime adds up to a timeout")] + #[test_case(&[Role::Charged(0), Role::Flaps(3, 7), Role::Flaps(3, 7)], 16, &[1, 2, 3], true, &[]; "a dead peer beside peers pausing together is held while their uptime adds up")] + #[test_case(&[Role::Charged(0), Role::Flaps(3, 7), Role::Flaps(3, 7)], 17, &[2, 3], false, &[]; "a dead peer beside peers pausing together is kicked once their uptime adds up")] + #[test_case(&[Role::Charged(0), Role::Delivers(0), Role::Flaps(3, 1), Role::Flaps(3, 1)], 12, &[2, 3, 4], false, &[]; "a dead peer beside an idle stream and owing peers with gaps is kicked")] + #[test_case(&[Role::Charged(0), Role::Resumes(0, 12)], 21, &[1, 2], true, &[]; "a peer silent with one resuming on the timeout tick gets a timeout")] + #[test_case(&[Role::Charged(0), Role::Resumes(0, 12)], 22, &[2], false, &[]; "a peer silent with one resuming on the timeout tick is kicked a timeout later")] + #[test_case(&[Role::Charged(0), Role::Resumes(3, 9)], 16, &[1, 2], true, &[]; "a peer silent across a 5 s outage gets a timeout of uptime")] + #[test_case(&[Role::Charged(0), Role::Resumes(3, 9)], 17, &[2], false, &[]; "a peer silent across a 5 s outage is kicked after a timeout of uptime")] + #[test_case(&[Role::Charged(0), Role::Resumes(0, 40)], 49, &[1, 2], true, &[]; "a peer silent through a 40 s outage gets a timeout after it")] + #[test_case(&[Role::Charged(0), Role::Resumes(0, 40)], 50, &[2], false, &[]; "a peer silent a timeout after a 40 s outage is kicked")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Registered(0), Role::Charged(20)], 34, &[4], true, &[]; "stalls paced past the link's last uptime keep their witness")] fn the_stall_judge_kicks_one_peer_per_shared_stall( roles: &[Role], secs: u64, @@ -2171,11 +2205,17 @@ mod tests { .filter(|(_, role)| !matches!(role, Role::Joins(_))) .map(|(n, &role)| peer(n, role)) .collect(); - // The last progress of each registered request. - let mut requests: HashMap = (1..) + // When each registered request was sent, and its last progress. None is answered. + let mut requests: HashMap = (1..) .zip(roles) - .filter(|(_, role)| matches!(role, Role::Registered(_) | Role::Chatters(_))) - .map(|(n, _)| (conn(n, 0), start)) + .filter_map(|(n, &role)| { + let sent = match role { + Role::Registered(_) | Role::Chatters(_) | Role::Streams(_) => start, + Role::Asks(ms) => start + Duration::from_millis(ms), + _ => return None, + }; + Some((conn(n, 0), (sent, sent))) + }) .collect(); let mut judge = StallJudge::default(); let mut kicked_unbanned = Vec::new(); @@ -2186,7 +2226,8 @@ mod tests { Role::Registered(until) | Role::Charged(until) => second <= until, Role::Delivers(from) | Role::Chatters(from) => second >= from, Role::Resumes(until, from) => second <= until || second >= from, - Role::Whispers(period) => second.is_multiple_of(period), + Role::Whispers(period) | Role::Streams(period) => second.is_multiple_of(period), + Role::Asks(_) => false, Role::Flaps(silent, receiving) => (second - 1) % (silent + receiving) >= silent, Role::Joins(at) => { if second == at { @@ -2201,14 +2242,21 @@ mod tests { } let now = start + Duration::from_secs(second); for id in judge.observe(now, &peers) { - if let Some(progress) = requests.get_mut(&id) { + if let Some((_, progress)) = requests.get_mut(&id) { *progress = now; } } + // As `Broker::expire` blames a request: from its progress, or from its answer. let stalled = requests .iter() - .filter(|(_, &progress)| now.duration_since(progress) > REQUEST_TIMEOUT) - .map(|(&id, &progress)| (id, progress)) + .filter_map(|(&id, &(sent, progress))| { + if now.duration_since(progress) > REQUEST_TIMEOUT { + Some((id, progress)) + } else { + (now.duration_since(sent) > REQUEST_TIMEOUT * ANSWER_TIMEOUTS) + .then_some((id, sent)) + } + }) .collect(); let (kicked, _, unwitnessed) = judge.judge(now, &peers, stalled); assert!(kicked.len() <= 1, "second {second} kicked {kicked:?}"); From 4e38fff8cb5c8acf8c41c4f3c9d6bd60730e891b Mon Sep 17 00:00:00 2001 From: xdustinface Date: Fri, 13 Nov 2026 21:06:46 +1000 Subject: [PATCH 10/11] refactor(dash-spv): fold `PeerFlow::sample` back into `tick` `sample` was split out of `tick` so a test could drive it, and that test is gone. `tick` is its only caller again, so the serving rate update goes back inline with the stall check beside it. --- dash-spv/src/network/manager.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 25ee1dc42..90d51f5f5 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -626,19 +626,14 @@ impl PeerFlow { // Short of both, its rate shows how much it was asked for, not its bandwidth. let bound = in_flight > 0 && (in_flight >= self.limiter.cap as u64 || link_full); let cap = self.limiter.tick(bytes, in_flight, CAP_STEP_BYTES_PER_SEC); - self.sample(self.limiter.rate, bound, link_stalled); - cap - } - - /// Folds `rate` into the serving rate if the peer had work. Ticks are skipped once the - /// timeout monitor sees our own link stalled, since such a stall says nothing about - /// the peer and would drag its average for 30 s. - fn sample(&mut self, rate: f64, has_work: bool, link_stalled: bool) { - if has_work && !link_stalled { + // A stall of our own link says nothing about the peer and would drag its average + // for 30 s, so ticks are skipped once the timeout monitor sees one. + if bound && !link_stalled { self.serving_ticks += 1; let alpha = 1.0 / self.serving_ticks.min(SERVING_SAMPLE_TICKS) as f64; - self.serving_rate += alpha * (rate - self.serving_rate); + self.serving_rate += alpha * (self.limiter.rate - self.serving_rate); } + cap } /// Returns the peer's serving rate in bytes/s, 0 until measured. From 46beb071a1311066f171ec3f5e7cbe6d30983eba Mon Sep 17 00:00:00 2001 From: xdustinface Date: Fri, 13 Nov 2026 21:07:10 +1000 Subject: [PATCH 11/11] test(dash-spv): build the fed `cfilter` frame inside `feed` `unsettling_filter` had `feed` as its only caller, so the frame is built where it is sent. --- dash-spv/src/network/manager.rs | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 90d51f5f5..11922f4ac 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1573,20 +1573,7 @@ mod tests { } } - /// A framed `cfilter` for a peer to receive. Its block hash is not the stop hash of - /// [`get_cfilters`], so it settles no charge and the peer's work stays in flight. - fn unsettling_filter() -> Vec { - serialize(&RawNetworkMessage { - magic: Network::Regtest.magic(), - payload: NetworkMessage::CFilter(CFilter { - filter_type: 0, - block_hash: BlockHash::dummy(1), - filter: vec![0; 32 * 1024], - }), - }) - } - - /// A connected peer that receives a run of [`unsettling_filter`]s at its own rate. + /// A connected peer that receives a run of framed `cfilter`s at its own rate. struct Feeder { id: PeerId, handle: PeerHandle, @@ -1638,7 +1625,16 @@ mod tests { /// Feeds every peer at its rate for `ticks` flow controller ticks, with work up to its /// cap, as bulk download keeps it. async fn feed(net: &PeerNetworkManager, feeders: &mut [Feeder], ticks: u32) { - let frame = unsettling_filter(); + // Its block hash is not the stop hash of `get_cfilters`, so it settles no charge and + // the peers' work stays in flight. + let frame = serialize(&RawNetworkMessage { + magic: Network::Regtest.magic(), + payload: NetworkMessage::CFilter(CFilter { + filter_type: 0, + block_hash: BlockHash::dummy(1), + filter: vec![0; 32 * 1024], + }), + }); for _ in 0..ticks { for feeder in feeders.iter() { while feeder.handle.in_flight_bytes() < feeder.handle.cap() {