From c54619d3902d7985c24c1e26dc082bc3d2f90cf9 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Sun, 8 Nov 2026 17:48:25 +1000 Subject: [PATCH] fix(dash-spv): rank a stalled peer last when routing A peer that stopped delivering kept the rates it had earned, so shortest-queue routing kept picking it. In a mainnet run one peer serving 2 to 3 MB/s stalled with blocks owed, received 51 more block requests within 4 s, and each was hedged after 10 s. Its 30 s serving rate, which routing reads first, was sampled only while the peer was bound, so it stayed near 2.5 MB/s for over a minute. A peer that owes replies and delivers under 8 KB/s for `HEDGE_AFTER` is now stalled. Routing and hedges pick a stalled peer only when every candidate is stalled. The flag clears with the first tick of real delivery, so a fast peer computing one reply for less than `HEDGE_AFTER` keeps its rank, and one computing longer is passed over only until the reply streams in. The serving rate is now sampled whenever the peer serves all it can, as its served rate is, not only while bound. --- dash-spv/src/network/broker.rs | 21 ++++- dash-spv/src/network/manager.rs | 131 +++++++++++++++++++++++++++++--- dash-spv/src/network/peer.rs | 14 +++- 3 files changed, 150 insertions(+), 16 deletions(-) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index 0fbbff9c2..8493ae105 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -16,7 +16,7 @@ const KICKING_ATTEMPTS: u32 = 2; /// answer. A big answer on a slow link must fit, but a message that never finishes must /// not keep its request alive forever. const ANSWER_TIMEOUTS: u32 = 6; -const HEDGE_AFTER: Duration = Duration::from_secs(2); +pub(super) const HEDGE_AFTER: Duration = Duration::from_secs(2); /// A request on the wire this long gets a second copy even while its peer delivers other /// replies, unless [`HEDGE_QUEUE_FACTOR`] gives it longer. const HEDGE_OVERDUE: Duration = Duration::from_secs(10); @@ -533,11 +533,24 @@ fn classify(msg: &NetworkMessage) -> MsgClass { } } +/// Picks the peer among `open` for a message of `class`. A stalled peer is picked only when +/// every peer in `open` is stalled, since its rate and cap still describe it before the stall. +fn choose_peer<'a>( + open: impl Iterator + Clone, + class: MsgClass, +) -> Option<&'a PeerHandle> { + let delivering = open.clone().filter(|p| !p.stalled()); + if delivering.clone().next().is_some() { + return choose_ranked(delivering, class); + } + choose_ranked(open, class) +} + /// Picks the peer among `open` for a message of `class`. A header or block request goes to /// the peer that will answer it soonest, since sync waits on each of them in height order. /// Bulk filter traffic goes to the peer least loaded relative to its own cap, and so does /// any request while no peer has a rate yet. -fn choose_peer<'a>( +fn choose_ranked<'a>( open: impl Iterator + Clone, class: MsgClass, ) -> Option<&'a PeerHandle> { @@ -554,9 +567,9 @@ fn choose_peer<'a>( /// Picks the peer among `others` for the second copy of a request: the one that answers it /// soonest. A peer without a serving rate looks idle before it gets work and may be slow, so -/// it is asked only when no peer with a serving rate is among `others`. +/// it is asked only when no delivering peer with a serving rate is among `others`. fn hedge_peer<'a>(others: impl Iterator + Clone) -> Option<&'a PeerHandle> { - let measured = others.clone().filter(|p| p.serving_rate() > 0); + let measured = others.clone().filter(|p| p.serving_rate() > 0 && !p.stalled()); if measured.clone().next().is_some() { return choose_peer(measured, MsgClass::Headers); } diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 9cec3eec7..c970af74a 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -106,7 +106,7 @@ const LINK_STALL_TIMEOUTS: u32 = 6; use crate::error::{NetworkError, NetworkResult}; use crate::network::bans::{Ban, Bans}; -use crate::network::broker::{Broker, RequestKey}; +use crate::network::broker::{Broker, RequestKey, HEDGE_AFTER}; use crate::network::peer_set::{Admission, PeerSet}; use crate::network::NetworkManager; use crate::network::TransportPreference; @@ -519,6 +519,10 @@ const MIN_CAP_BYTES: u64 = INITIAL_CAP_BYTES; /// the round trip, or a peer limited by its cap cannot show it could serve more. const TARGET_QUEUE_SECS: f64 = 2.0; +/// A peer owing replies that delivers less than this for [`HEDGE_AFTER`] is stalled. Pings +/// and announcements stay well below it, any reply streaming in rises above it. +const STALLED_BELOW_BYTES_PER_SEC: f64 = 8.0 * 1024.0; + /// Horizon of the served rate. A burst of `n` bytes raises the cap by about `n`. const SERVED_RATE_SECS: f64 = 2.0; @@ -528,7 +532,7 @@ const CAP_PROBE_BYTES_PER_SEC: f64 = 32.0 * 1024.0; /// Share per second of its excess over the target that a bound peer's cap gives back. const CAP_DECAY_PER_SEC: f64 = 0.5; -/// Ticks a peer must have been bandwidth bound before its rate is compared (30 s), and +/// Ticks a peer must have served all it can before its rate is compared (30 s), and /// the span its serving rate is averaged over, so a momentary dip never condemns it. A /// fast mainnet peer serves only 40 to 60% of its later rate in its first 15 s, and 70 /// to 100% by 30 s, so a newcomer judged sooner looks slow while it ramps. @@ -613,11 +617,13 @@ impl ByteLimiter { } } -/// A peer's limiter plus how fast it serves while bandwidth bound, for eviction. +/// A peer's limiter plus how fast it serves while serving all it can, for eviction. struct PeerFlow { limiter: ByteLimiter, serving_rate: f64, serving_ticks: u32, + /// Consecutive ticks in which the peer owed replies and delivered nearly nothing. + stalled_ticks: u32, } impl PeerFlow { @@ -626,18 +632,59 @@ impl PeerFlow { limiter: ByteLimiter::new(bytes), serving_rate: 0.0, serving_ticks: 0, + stalled_ticks: 0, } } /// One tick with the peer's cumulative bytes received, its bytes in flight now and /// whether our link is stalled. Returns its new cap. fn tick(&mut self, bytes: u64, in_flight: u64, link_stalled: bool) -> u64 { - let bound = self.limiter.bound(in_flight); + let serving_all = self.limiter.serving_all(in_flight); + let got = bytes.saturating_sub(self.limiter.last_bytes); let cap = self.limiter.tick(bytes, in_flight, link_stalled); - self.sample(self.limiter.rate, bound, link_stalled); + self.sample(self.limiter.rate, serving_all, link_stalled); + let stalling = (got as f64) < STALLED_BELOW_BYTES_PER_SEC * FLOW_TICK.as_secs_f64(); + if in_flight == 0 || !stalling { + self.stalled_ticks = 0; + } else if !link_stalled { + self.stalled_ticks += 1; + } cap } + /// Whether the peer owed replies and delivered nearly nothing for [`HEDGE_AFTER`]. + fn stalled(&self) -> bool { + FLOW_TICK * self.stalled_ticks >= HEDGE_AFTER + } + + /// Ticks the flow of `peer` with `in_flight` bytes owed and publishes its cap and rates. + fn update(&mut self, peer: &ConnectedPeer, in_flight: u64, link_stalled: bool) { + let (was_stalled, stalled_for) = (self.stalled(), FLOW_TICK * self.stalled_ticks); + peer.set_cap(self.tick(peer.bytes_read(), in_flight, link_stalled)); + peer.set_serving_rate(self.serving_rate()); + peer.set_served(self.limiter.served as u64); + let stalled = self.stalled(); + peer.set_stalled(stalled); + if stalled && !was_stalled { + tracing::debug!( + target: "dash_spv::network", + "{} stalled: owes {} KB and delivered under {} KB/s for {:.1}s", + peer.addr(), + in_flight >> 10, + STALLED_BELOW_BYTES_PER_SEC as u64 >> 10, + (FLOW_TICK * self.stalled_ticks).as_secs_f64(), + ); + } else if was_stalled && !stalled { + tracing::debug!( + target: "dash_spv::network", + "{} no longer stalled after {:.1}s, owes {} KB", + peer.addr(), + stalled_for.as_secs_f64(), + in_flight >> 10, + ); + } + } + /// Folds `rate` into the serving rate if the peer had work. Ticks are skipped once the /// timeout monitor sees our own link stalled, since such a stall says nothing about /// the peer and would drag its average for 30 s. @@ -662,7 +709,7 @@ impl PeerFlow { /// Sizes the byte cap of every peer with a [`ByteLimiter`] on its own throughput. Each cap /// follows its peer's share of the client's link, so a full link needs no cap of its own. -/// Also reports how fast each peer serves while bandwidth bound, for [`underserving_peer`]. +/// Also reports how fast each peer serves while serving all it can, for [`underserving_peer`]. fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) -> JoinHandle<()> { tokio::spawn(async move { let NetworkState { @@ -701,9 +748,7 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) let stalled = link_stalled.load(Ordering::Relaxed); for (p, &in_flight) in g.iter().zip(&in_flights) { let flow = peers.entry(p.id()).or_insert_with(|| PeerFlow::new(p.bytes_read())); - p.set_cap(flow.tick(p.bytes_read(), in_flight, stalled)); - p.set_serving_rate(flow.serving_rate()); - p.set_served(flow.limiter.served as u64); + flow.update(p, in_flight, stalled); } if ticks.is_multiple_of(2) { @@ -1262,7 +1307,7 @@ struct SlowPeer { } /// The connected peer clearly underserving the rest, if any: serving bytes at a small -/// fraction of the best connected peer's rate while bandwidth bound. Relative, so a slow +/// fraction of the best connected peer's rate while serving all it can. Relative, so a slow /// client link (every peer slow alike) never condemns anyone. fn underserving_peer(peers: &[ConnectedPeer]) -> Option { let measured: Vec<(SocketAddr, u64)> = peers @@ -1755,7 +1800,7 @@ mod tests { use dashcore::consensus::encode::serialize; use dashcore::network::address::Address; use dashcore::network::message::{RawNetworkMessage, RawNetworkMessageCodec}; - use dashcore::network::message_blockdata::Inventory; + use dashcore::network::message_blockdata::{GetHeadersMessage, Inventory}; use dashcore::network::message_filter::{CFilter, GetCFilters}; use dashcore::network::message_network::VersionMessage; use dashcore::{BlockHash, Network}; @@ -2727,6 +2772,70 @@ mod tests { assert_eq!(judge.link_stalled, link_stalled); } + /// Peer 1 serves 1 MB/s and owes a `headers2` reply, peer 2 serves 50 KB/s and owes a + /// block, so a new block goes to peer 1 unless it stalled. Then peer 1 delivers + /// `per_tick` bytes for `ticks` ticks. Returns which peer the next block goes to. + #[test_case(8, 64 => 2; "a peer owing replies that delivers only pings ranks last")] + #[test_case(7, 64 => 1; "a reply in the making for less than the hedge delay keeps its rank")] + #[test_case(40, 4096 => 1; "a reply streaming in slowly keeps its rank")] + #[tokio::test] + async fn a_stalled_peer_gets_no_new_blocks(ticks: u32, per_tick: u64) -> u8 { + let peers = + [ConnectedPeer::for_test(conn(1, 0)).await, ConnectedPeer::for_test(conn(2, 0)).await]; + let locator = GetHeadersMessage::new(vec![BlockHash::dummy(1)], BlockHash::dummy(0)); + peers[0].handle().send(&NetworkMessage::GetHeaders2(locator)).await.expect("loopback send"); + peers[1] + .handle() + .send(&NetworkMessage::GetData(vec![block(1)])) + .await + .expect("loopback send"); + let mut flows = peers.each_ref().map(|p| PeerFlow::new(p.bytes_read())); + let tick_s = FLOW_TICK.as_secs_f64(); + let mut tick = |deliveries: [u64; 2]| { + for ((peer, flow), bytes) in peers.iter().zip(&mut flows).zip(deliveries) { + peer.receive_for_test(bytes, false); + flow.update(peer, peer.in_flight_bytes(), false); + } + }; + for _ in 0..40 { + tick([(1e6 * tick_s) as u64, (50e3 * tick_s) as u64]); + } + for _ in 0..ticks { + tick([per_tick, (50e3 * tick_s) as u64]); + } + + let broker = Broker::new(); + broker.enqueue(0, NetworkMessage::GetData(vec![block(2)])).await; + let before = peers.each_ref().map(|p| p.in_flight_bytes()); + let handles = peers.each_ref().map(|p| p.handle()); + assert_eq!(broker.route(&handles, &ResponseCosts::new()).await, 1); + let routed = peers.iter().zip(before).position(|(p, b)| p.in_flight_bytes() > b); + routed.map_or(0, |i| i as u8 + 1) + } + + /// A burst at 5 MB/s lifts the peer's cap, then it serves 100 KB/s of a 1 MB backlog + /// below that cap. Its serving rate is measured while it is backlogged, not only once the + /// cap has fallen enough to bind it. + #[test] + fn a_backlogged_peer_is_measured_below_its_cap() { + let tick_s = FLOW_TICK.as_secs_f64(); + let mut flow = PeerFlow::new(0); + let mut bytes = 0; + for _ in 0..8 { + bytes += (5e6 * tick_s) as u64; + flow.tick(bytes, 64 * 1024, false); + } + // The cap falls below the backlog after about 34 ticks, so a peer sampled only while + // bound misses those. The 1 s rate average needs about 9 to read the backlog. + for _ in 0..SERVING_SAMPLE_TICKS + 20 { + bytes += (100e3 * tick_s) as u64; + flow.tick(bytes, 1 << 20, false); + } + + let rate = flow.serving_rate(); + assert!(rate > 0 && rate < 300_000, "{rate} after {} samples", flow.serving_ticks); + } + /// A peer serves 1 MB/s with work for three ticks, then nothing for one. #[test_case(false, true => 750_000; "a stall of its own drags its serving rate")] #[test_case(true, true => 1_000_000; "a stall of our link does not")] diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index e5022bafc..c383b54d2 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -258,11 +258,13 @@ pub struct PeerHandle { /// bandwidth-delay product), sized by the controller from its byte rate and /// service time. cap: Arc, - /// Bytes/s this peer serves while bandwidth bound, 0 until measured. + /// Bytes/s this peer serves while serving all it can, 0 until measured. serving_rate: Arc, /// Bytes/s this peer served over the last two seconds, 0 until it delivered anything. /// A rough stand-in for `serving_rate` while that is still being measured. served: Arc, + /// Whether this peer delivered nearly nothing for a while although it owes replies. + stalled: Arc, headers2: bool, /// Child of the global shutdown; cancelling it stops the reader and closes the socket. token: CancellationToken, @@ -285,6 +287,10 @@ impl PeerHandle { self.served.load(Ordering::Relaxed) } + pub(super) fn stalled(&self) -> bool { + self.stalled.load(Ordering::Relaxed) + } + pub fn in_flight_bytes(&self) -> u64 { self.ledger.in_flight_bytes() } @@ -405,6 +411,10 @@ impl ConnectedPeer { self.handle.served.store(bytes_per_sec, Ordering::Relaxed); } + pub(crate) fn set_stalled(&self, stalled: bool) { + self.handle.stalled.store(stalled, Ordering::Relaxed); + } + /// A handshaken connection `id` over a loopback socket, with no reader running. /// The far end discards what it reads, so sends succeed. #[cfg(test)] @@ -423,6 +433,7 @@ impl ConnectedPeer { cap: Arc::new(AtomicU64::new(INITIAL_CAP_BYTES)), serving_rate: Arc::new(AtomicU64::new(0)), served: Arc::new(AtomicU64::new(0)), + stalled: Arc::new(AtomicBool::new(false)), headers2: false, token: CancellationToken::new(), }, @@ -663,6 +674,7 @@ impl DisconnectedPeer { cap, serving_rate: Arc::new(AtomicU64::new(0)), served: Arc::new(AtomicU64::new(0)), + stalled: Arc::new(AtomicBool::new(false)), headers2, token, },