From a92d0948191126b7a24355bed611a689a08416a3 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Sun, 8 Nov 2026 17:48:25 +1000 Subject: [PATCH] fix(dash-spv): size byte caps from the served rate A peer's in-flight cap now follows its own served rate, `TARGET_QUEUE_SECS` of it. The cap rises at once when the rate supports it, so a fast peer reaches megabytes within a second or two, and a slow peer stays near the floor. A small probe lets a peer limited by its cap find more, and ticks of a stalled link are skipped. This replaces the round-trip based limiter, whose step grew with the queue and reached tens of megabytes on slow peers. The served rate and the cap fall only while the peer serves all it can: while it is bound at its cap, or while it holds more than `TARGET_QUEUE_SECS` of what it delivers below its cap. Below that a peer shows how much it was asked for. Less than `MIN_CAP_BYTES` in flight is a reply or two in the making and leaves both alone. Only a bound peer probes for a higher cap. On mainnet a burst raised one peer's cap to 15.7 MB while it delivered 0.4 to 0.8 MB/s, and the router queued up to 8.8 MB below that cap. The floor is two `getheaders2` replies, so a new peer has one queued behind the one it serves. A 336 KB reply did not fit the old 256 KB floor twice. The global limiter is gone. Each cap follows its peer's share of the client's link, so a full link already shows up in the peer rates, and `PeerFlow` samples the serving rate while the peer is bound to its own cap. The `peer_speed` lines drop `base` and `global_cap` and gain `served`. Replaying the six logs of the slow and fast testnet runs, the peak peer cap on the slow run falls from 4.7 MB to 2.1 MB and its implied queue from 17 s to 6 s, while over the first 60 s of the four fast logs the bytes delivered range from 6% below to 6% above the unbounded limiter and from 2% to 8% above the step-bound one. The hedges and the shortest-queue pick build on the served rate: - The frontier filter batch and the lowest endgame block get a second copy once their peer answered nothing for `HEDGE_AFTER`, or once they have been on the wire for `HEDGE_QUEUE_FACTOR` (2) times the queue they were sent behind, `in_flight / rate` of the peer at send, and never before `HEDGE_OVERDUE` (10 s). A peer that keeps trickling other replies held a frontier batch for 118 s in one testnet run. A fixed 10 s bound instead hedged the mainnet frontier batch 20 to 76 times per sync, mostly while the batch merely waited in a long queue. The queue is taken at send, so a peer that slows down does not earn more time. - The hedge log names the holder, the time on the wire, the overdue threshold, the holder's silence and the peer asked for the copy. - A peer without a serving rate is timed by its served rate of the last two seconds, published as `PeerHandle::served`, so the shortest-queue pick no longer falls back to the cap ratio whenever one busy peer is still being measured. A busy peer that delivered nothing yet goes last. - Both hedges go to the peer with a serving rate that answers soonest, and to an unmeasured peer only when no measured one qualifies. An idle newcomer used to win the second copy. --- dash-spv/src/network/broker.rs | 298 ++++++++++++++++++------ dash-spv/src/network/costs.rs | 2 +- dash-spv/src/network/manager.rs | 396 +++++++++++++++++++------------- dash-spv/src/network/peer.rs | 17 +- 4 files changed, 480 insertions(+), 233 deletions(-) diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index 695d31d5b..0fbbff9c2 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -17,6 +17,14 @@ const KICKING_ATTEMPTS: u32 = 2; /// not keep its request alive forever. const ANSWER_TIMEOUTS: u32 = 6; const HEDGE_AFTER: Duration = Duration::from_secs(2); +/// A request on the wire this long gets a second copy even while its peer delivers other +/// replies, unless [`HEDGE_QUEUE_FACTOR`] gives it longer. +const HEDGE_OVERDUE: Duration = Duration::from_secs(10); +/// A request may wait this many times its peer's queue at send before a second copy is sent +/// to a peer still delivering, so it is hedged only once its peer serves at under half the +/// rate the request was routed by. A peer's queue can hold many seconds of work, and a copy +/// sent while the batch merely waits in it is downloaded twice. +const HEDGE_QUEUE_FACTOR: f64 = 2.0; const MIN_IN_FLIGHT_PER_CLASS: usize = 4; const MSG_CLASSES: usize = 5; const DRAIN_PRIORITY: [MsgClass; MSG_CLASSES] = @@ -72,6 +80,9 @@ struct OnWire { /// Refreshed only by answers, so a block still arriving slowly can be hedged and a /// message that never finishes cannot keep the request alive forever. last_answer: Instant, + sent: Instant, + /// How long after `sent` the request is overdue although its peer keeps delivering. + overdue_after: Duration, msg: NetworkMessage, height: u32, hedge: Option, @@ -287,21 +298,13 @@ impl Broker { culprits } - /// Sends what fits in both the peers' and the global byte caps, each message to - /// the peer [`choose_peer`] picks. - pub async fn route( - &self, - peers: &[PeerHandle], - global_cap: u64, - costs: &ResponseCosts, - ) -> usize { + /// Sends what fits in the peers' byte caps, each message to the peer [`choose_peer`] + /// picks. + pub async fn route(&self, peers: &[PeerHandle], costs: &ResponseCosts) -> usize { if peers.is_empty() { return 0; } - let total_in_flight: u64 = peers.iter().map(PeerHandle::in_flight_bytes).sum(); - let per_peer_room: u64 = - peers.iter().map(|p| p.cap().saturating_sub(p.in_flight_bytes())).sum(); - let budget = per_peer_room.min(global_cap.saturating_sub(total_in_flight)); + let budget: u64 = peers.iter().map(|p| p.cap().saturating_sub(p.in_flight_bytes())).sum(); if budget == 0 { return 0; } @@ -320,7 +323,7 @@ impl Broker { // Popped but unsent goes back: its key is registered, so a dropped message // would stall its pipeline. let mut unsent: Vec<(u32, NetworkMessage)> = Vec::new(); - let mut on_wire: Vec<(u32, NetworkMessage, PeerHandle)> = Vec::new(); + let mut on_wire: Vec<(u32, NetworkMessage, PeerHandle, Duration)> = Vec::new(); let mut msgs = msgs.into_iter(); for (height, msg) in msgs.by_ref() { // `peers` can be a snapshot taken before the timeout monitor closed one of them. @@ -331,7 +334,9 @@ impl Broker { }; if peer.send(&msg).await.is_ok() { sent += 1; - on_wire.push((height, msg, peer.clone())); + // Before the next send, which queues behind this request. + let overdue_after = overdue_after(peer); + on_wire.push((height, msg, peer.clone(), overdue_after)); } else { tracing::warn!(target: "dash_spv::network", "router: send to {} failed", peer.addr()); unsent.push((height, msg)); @@ -342,7 +347,7 @@ impl Broker { if !on_wire.is_empty() { let mut reqs = self.requests.lock().await; - for (height, msg, peer) in on_wire { + for (height, msg, peer, overdue_after) in on_wire { for key in request_keys(&msg) { if let Some(slot) = reqs.get_mut(&key) { let attempts = match slot { @@ -357,6 +362,8 @@ impl Broker { height, last_progress: Instant::now(), last_answer: Instant::now(), + sent: Instant::now(), + overdue_after, hedge: None, attempts: attempts + 1, })); @@ -377,43 +384,42 @@ impl Broker { sent } - /// Endgame: asks a second peer for the lowest block on the wire once its peer answered - /// nothing for [`HEDGE_AFTER`], and returns its height. Blocks are processed in height + /// Endgame: asks a second peer for the lowest block on the wire once it is + /// [`OnWire::overdue`], and returns its height. Blocks are processed in height /// order, so only the lowest one holds up progress and a second copy of any other is wasted. pub async fn hedge(&self, peers: &[PeerHandle]) -> Option { let (key, msg, height, peer) = { let reqs = self.requests.lock().await; let (key, o) = lowest_on_wire(&reqs, |key| matches!(key, RequestKey::Block(_)))?; - if o.hedge.is_some() || o.last_answer.elapsed() < HEDGE_AFTER { + if o.hedge.is_some() || !o.overdue() { return None; } - let peer = peers + let others = peers .iter() - .filter(|p| !p.is_closed() && p.id() != o.peer && p.in_flight_bytes() < p.cap()) - .max_by_key(|p| p.cap().saturating_sub(p.in_flight_bytes()))?; + .filter(|p| !p.is_closed() && p.id() != o.peer && p.in_flight_bytes() < p.cap()); + let peer = hedge_peer(others)?; (key.clone(), o.msg.clone(), o.height, peer.clone()) }; self.send_hedge(key, msg, height, peer).await } - /// Asks a second peer for the lowest filter batch not yet delivered once its peer has - /// answered nothing for [`HEDGE_AFTER`], and returns its height. Filters are stored in - /// height order, so that batch holds back everything above it however much is downloaded. - /// One queued lower goes out next, so it is no reason to hedge the batch above it. + /// Asks a second peer for the lowest filter batch not yet delivered once it is + /// [`OnWire::overdue`], and returns its height. Filters are stored in height order, so + /// that batch holds back everything above it however much is downloaded. One queued + /// lower goes out next, so it is no reason to hedge the batch above it. pub async fn hedge_filters(&self, peers: &[PeerHandle]) -> Option { let (key, msg, height, peer) = { let reqs = self.requests.lock().await; let (key, o) = lowest_on_wire(&reqs, |key| matches!(key, RequestKey::CFilters(_)))?; if o.hedge.is_some() - || o.last_answer.elapsed() < HEDGE_AFTER + || !o.overdue() || self.queue.lowest_height(MsgClass::CFilters).await.is_some_and(|h| h < o.height) { return None; } // Bulk download keeps every peer at its cap, so a cap would rule out all of them. - // Like a header request, the second copy goes where it is answered soonest. let others = peers.iter().filter(|p| p.id() != o.peer && !p.is_closed()); - let peer = choose_peer(others, MsgClass::Headers)?; + let peer = hedge_peer(others)?; (key.clone(), o.msg.clone(), o.height, peer.clone()) }; self.send_hedge(key, msg, height, peer).await @@ -430,6 +436,16 @@ impl Broker { match self.requests.lock().await.get_mut(&key) { Some(ReqState::OnWire(o)) if o.hedge.is_none() => { o.hedge = Some(peer.id()); + tracing::debug!( + target: "dash_spv::network", + "hedge at {}: on the wire {:.1}s at {} (overdue after {:.1}s, silent {:.1}s), copy to {}", + height, + o.sent.elapsed().as_secs_f64(), + o.peer.addr(), + o.overdue_after.as_secs_f64(), + o.last_answer.elapsed().as_secs_f64(), + peer.addr(), + ); Some(height) } _ => None, @@ -444,6 +460,8 @@ impl Broker { peer, last_progress: Instant::now(), last_answer: Instant::now(), + sent: Instant::now(), + overdue_after: HEDGE_OVERDUE, msg: msg.clone(), height: 0, hedge: None, @@ -471,6 +489,14 @@ impl Broker { } } +impl OnWire { + /// Whether the request waited long enough for a second copy: its peer answered nothing + /// for [`HEDGE_AFTER`], or the request has been on the wire for `overdue_after`. + fn overdue(&self) -> bool { + self.last_answer.elapsed() >= HEDGE_AFTER || self.sent.elapsed() >= self.overdue_after + } +} + /// Returns the on-wire request of the lowest height among the keys `wanted` selects. fn lowest_on_wire( reqs: &HashMap, @@ -510,17 +536,15 @@ fn classify(msg: &NetworkMessage) -> MsgClass { /// Picks the peer among `open` for a message of `class`. A header or block request goes to /// the peer that will answer it soonest, since sync waits on each of them in height order. /// Bulk filter traffic goes to the peer least loaded relative to its own cap, and so does -/// any request while a busy peer has no serving rate yet. +/// any request while no peer has a rate yet. fn choose_peer<'a>( open: impl Iterator + Clone, class: MsgClass, ) -> Option<&'a PeerHandle> { - if matches!(class, MsgClass::Headers | MsgClass::Blocks) { - let waits: Option> = - open.clone().map(|p| Some((queued_secs(p)?, p))).collect(); - if let Some(waits) = waits { - return waits.into_iter().min_by(|a, b| a.0.total_cmp(&b.0)).map(|(_, p)| p); - } + if matches!(class, MsgClass::Headers | MsgClass::Blocks) && open.clone().any(|p| rate(p) > 0) { + // A busy peer that has not delivered anything yet has no known queue, so it goes last. + let wait = |p: &PeerHandle| queued_secs(p).unwrap_or(f64::INFINITY); + return open.min_by(|a, b| wait(a).total_cmp(&wait(b))); } open.min_by(|a, b| { (a.in_flight_bytes() as u128 * b.cap() as u128) @@ -528,10 +552,39 @@ fn choose_peer<'a>( }) } +/// Picks the peer among `others` for the second copy of a request: the one that answers it +/// soonest. A peer without a serving rate looks idle before it gets work and may be slow, so +/// it is asked only when no peer with a serving rate is among `others`. +fn hedge_peer<'a>(others: impl Iterator + Clone) -> Option<&'a PeerHandle> { + let measured = others.clone().filter(|p| p.serving_rate() > 0); + if measured.clone().next().is_some() { + return choose_peer(measured, MsgClass::Headers); + } + choose_peer(others, MsgClass::Headers) +} + +/// Bytes/s the peer serves: its serving rate once measured, its last two seconds before. +fn rate(peer: &PeerHandle) -> u64 { + match peer.serving_rate() { + 0 => peer.served(), + rate => rate, + } +} + +/// Returns how long a request just sent to `peer` may stay on the wire while `peer` keeps +/// delivering: [`HEDGE_QUEUE_FACTOR`] times the queue it waits behind, at least +/// [`HEDGE_OVERDUE`]. The queue is taken at send. Requests sent later wait behind this one, +/// and a rate taken later would give a peer that slowed down the more time the slower it got. +fn overdue_after(peer: &PeerHandle) -> Duration { + queued_secs(peer) + .and_then(|secs| Duration::try_from_secs_f64(secs * HEDGE_QUEUE_FACTOR).ok()) + .map_or(HEDGE_OVERDUE, |queued| queued.max(HEDGE_OVERDUE)) +} + /// Seconds the peer needs to serve what it already holds, `None` while it holds work -/// but has no measured serving rate. Peers answer in order, so a new request waits that long. +/// but has no rate. Peers answer in order, so a new request waits that long. fn queued_secs(peer: &PeerHandle) -> Option { - match (peer.in_flight_bytes(), peer.serving_rate()) { + match (peer.in_flight_bytes(), rate(peer)) { (0, _) => Some(0.0), (_, 0) => None, (bytes, rate) => Some(bytes as f64 / rate as f64), @@ -705,6 +758,10 @@ mod tests { Inventory::Block(BlockHash::dummy(n)) } + fn get_block(n: u32) -> NetworkMessage { + NetworkMessage::GetData(vec![block(n)]) + } + fn tx(n: u8) -> Inventory { Inventory::Transaction(Txid::from_byte_array([n; 32])) } @@ -775,7 +832,7 @@ mod tests { broker.requeue_from(&HashSet::from([gone.id()])).await; // Only the charge matters here: the loopback has no reader to answer it. - broker.route(&[gone.handle(), other.handle()], u64::MAX, &ResponseCosts::new()).await; + broker.route(&[gone.handle(), other.handle()], &ResponseCosts::new()).await; assert_eq!(gone.in_flight_bytes(), 0); assert!(other.in_flight_bytes() > 0); @@ -818,7 +875,6 @@ mod tests { kicked.handle(), ]; let (slow, idle) = (&peers[..1], peers[1].id()); - let get_block = |n: u32| NetworkMessage::GetData(vec![block(n)]); let hedged_blocks = || async { let mut hedged: Vec = broker .requests @@ -837,10 +893,10 @@ mod tests { for n in [11, 12] { broker.enqueue(n, get_block(n)).await; } - assert_eq!(broker.route(slow, u64::MAX, &costs).await, 2); + assert_eq!(broker.route(slow, &costs).await, 2); tokio::time::advance(HEDGE_AFTER).await; broker.enqueue(10, get_block(10)).await; - assert_eq!(broker.route(slow, u64::MAX, &costs).await, 1); + assert_eq!(broker.route(slow, &costs).await, 1); assert_eq!(broker.hedge(&peers).await, None, "the lowest block has not waited long enough"); tokio::time::advance(HEDGE_AFTER / 2).await; @@ -860,6 +916,18 @@ mod tests { assert_eq!(broker.hedge(slow).await, None, "no other peer to ask"); } + #[tokio::test] + async fn a_new_peer_holds_two_header_requests() { + let broker = Broker::new(); + let peer = ConnectedPeer::for_test(conn(1, 0)).await; + for n in 0..3 { + broker.enqueue(n, get_headers(n)).await; + } + + assert_eq!(broker.route(&[peer.handle()], &ResponseCosts::new()).await, 2); + assert_eq!(broker.len(), 1); + } + #[test_case(NetworkMessage::GetData(vec![block(1)]), RequestKey::Block(BlockHash::dummy(1)); "a block")] #[test_case(get_cfilters(1), RequestKey::CFilters(BlockHash::dummy(1)); "a filter batch")] #[test_case(get_cfheaders(1), RequestKey::CfHeaders(BlockHash::dummy(1)); "a filter header batch")] @@ -870,7 +938,7 @@ mod tests { let peer = ConnectedPeer::for_test(id).await; broker.enqueue(0, request.clone()).await; - assert_eq!(broker.route(&[peer.handle()], u64::MAX, &ResponseCosts::new()).await, 1); + assert_eq!(broker.route(&[peer.handle()], &ResponseCosts::new()).await, 1); broker.arrived(conn(2, 0), key.clone()).await; assert_eq!(broker.state_of(&key).await, Some("on_wire"), "only its carrier settles it"); broker.arrived(id, key.clone()).await; @@ -925,9 +993,9 @@ mod tests { for start in [1000, 2000] { broker.enqueue(start, get_cfilters(start)).await; } - assert_eq!(broker.route(&peers[..1], u64::MAX, &costs).await, 2); + assert_eq!(broker.route(&peers[..1], &costs).await, 2); broker.enqueue(3000, get_cfilters(3000)).await; - assert_eq!(broker.route(&peers[1..], u64::MAX, &costs).await, 1); + assert_eq!(broker.route(&peers[1..], &costs).await, 1); broker.arrived(idle.id(), RequestKey::CFilters(BlockHash::dummy(3000))).await; idle.set_cap(idle.in_flight_bytes().max(1)); @@ -949,6 +1017,85 @@ mod tests { assert_eq!(broker.hedge_filters(&peers[..1]).await, None, "no other peer to ask"); } + /// Peer 2 is idle and unmeasured, peer 3 measured and busy. Returns the peer asked for + /// the second copy. + #[test_case(get_cfilters(0), false => Some(3); "a filter batch goes to the measured peer")] + #[test_case(get_cfilters(0), true => Some(3); "a filter batch goes to the measured peer at its cap")] + #[test_case(get_block(0), false => Some(3); "a block goes to the measured peer")] + #[test_case(get_block(0), true => Some(2); "a block goes to an unmeasured peer while the measured one is full")] + #[tokio::test(start_paused = true)] + async fn a_hedge_prefers_a_measured_peer(request: NetworkMessage, full: bool) -> Option { + let broker = Broker::new(); + let holder = ConnectedPeer::for_test(conn(1, 0)).await; + let unmeasured = ConnectedPeer::for_test(conn(2, 0)).await; + let measured = ConnectedPeer::for_test(conn(3, 0)).await; + measured.handle().send(&get_cfilters(5000)).await.expect("loopback send"); + measured.set_serving_rate(100 << 10); + if full { + measured.set_cap(measured.in_flight_bytes()); + } + broker.mark_on_wire(&request, holder.id()).await; + tokio::time::advance(HEDGE_AFTER).await; + + let peers = [holder.handle(), unmeasured.handle(), measured.handle()]; + let hedged = match request { + NetworkMessage::GetCFilters(_) => broker.hedge_filters(&peers).await, + _ => broker.hedge(&peers).await, + }; + assert_eq!(hedged, Some(0)); + let hedge = broker.requests.lock().await.values().find_map(|state| match state { + ReqState::OnWire(o) => o.hedge, + _ => None, + }); + [2, 3].into_iter().find(|n| hedge == Some(conn(*n, 0))) + } + + /// The holder keeps delivering a 20 s queue it took the request behind, so a second copy + /// waits for twice that rather than for `HEDGE_OVERDUE`. + #[test_case(get_cfilters(1000); "a filter batch")] + #[test_case(get_block(1); "a block")] + #[tokio::test(start_paused = true)] + async fn a_request_queued_at_a_steady_peer_is_hedged_only_past_twice_its_queue( + request: NetworkMessage, + ) { + let broker = Broker::new(); + let costs = ResponseCosts::new(); + let holder = ConnectedPeer::for_test(conn(1, 0)).await; + let other = ConnectedPeer::for_test(conn(2, 0)).await; + for start in [0, 1, 2, 3] { + holder.handle().send(&get_cfilters(start)).await.expect("loopback send"); + } + holder.set_cap(1 << 30); + let queued = holder.in_flight_bytes() + costs.cost(&request); + holder.set_serving_rate(queued / 20); + broker.enqueue(1000, request.clone()).await; + assert_eq!(broker.route(&[holder.handle()], &costs).await, 1); + + let peers = [holder.handle(), other.handle()]; + let hedge = || async { + match request { + NetworkMessage::GetCFilters(_) => broker.hedge_filters(&peers).await, + _ => broker.hedge(&peers).await, + } + }; + let step = HEDGE_AFTER / 2; + let mut waited = Duration::ZERO; + // The integer rate rounds the queue up a little, so the copy goes just after 40 s. + for deadline in [HEDGE_OVERDUE, Duration::from_secs(39), Duration::from_secs(41)] { + while waited < deadline { + tokio::time::advance(step).await; + broker.progress(holder.id()).await; + waited += step; + } + let hedged = if deadline < Duration::from_secs(40) { + None + } else { + Some(1000) + }; + assert_eq!(hedge().await, hedged, "after {deadline:?}"); + } + } + #[tokio::test(start_paused = true)] async fn a_hedge_lost_with_its_peer_is_sent_again() { let broker = Broker::new(); @@ -967,25 +1114,37 @@ mod tests { assert_eq!(broker.hedge_filters(&[holder, second]).await, Some(0)); } - #[test_case(false, Some(0); "a silent peer")] - #[test_case(true, None; "a peer still delivering")] + #[test_case(get_cfilters(0), false, HEDGE_AFTER => Some(0); "a batch at a silent peer")] + #[test_case(get_cfilters(0), true, HEDGE_AFTER => None; "a batch at a peer still delivering")] + #[test_case(get_cfilters(0), true, HEDGE_OVERDUE - HEDGE_AFTER => None; "a batch at a peer still delivering, short of the deadline")] + #[test_case(get_cfilters(0), true, HEDGE_OVERDUE => Some(0); "a batch at a peer still delivering, past the deadline")] + #[test_case(get_block(0), false, HEDGE_AFTER => Some(0); "a block at a silent peer")] + #[test_case(get_block(0), true, HEDGE_OVERDUE - HEDGE_AFTER => None; "a block at a peer still delivering, short of the deadline")] + #[test_case(get_block(0), true, HEDGE_OVERDUE => Some(0); "a block at a peer still delivering, past the deadline")] #[tokio::test(start_paused = true)] - async fn hedge_filters_asks_another_peer_only_for_a_batch_stuck_at_a_silent_one( + async fn hedge_asks_another_peer_only_for_a_request_stuck_at_a_silent_or_slow_one( + request: NetworkMessage, delivering: bool, - hedged: Option, - ) { + waited: Duration, + ) -> Option { let broker = Broker::new(); let slow = ConnectedPeer::for_test(conn(1, 0)).await; let idle = ConnectedPeer::for_test(conn(2, 0)).await; - broker.mark_on_wire(&get_cfilters(0), conn(1, 0)).await; + broker.mark_on_wire(&request, conn(1, 0)).await; - tokio::time::advance(HEDGE_AFTER / 2).await; - if delivering { - broker.progress(conn(1, 0)).await; + let step = HEDGE_AFTER / 2; + for _ in 0..waited.as_millis() / step.as_millis() { + tokio::time::advance(step).await; + if delivering { + broker.progress(conn(1, 0)).await; + } } - tokio::time::advance(HEDGE_AFTER / 2).await; - assert_eq!(broker.hedge_filters(&[slow.handle(), idle.handle()]).await, hedged); + let peers = [slow.handle(), idle.handle()]; + match request { + NetworkMessage::GetCFilters(_) => broker.hedge_filters(&peers).await, + _ => broker.hedge(&peers).await, + } } #[test_case(get_mnlistdiff(3), vec![RequestKey::MnListDiff(BlockHash::dummy(4))], 0; "its only key")] @@ -1035,15 +1194,23 @@ mod tests { assert_eq!(broker.requests.lock().await.len(), tracked); } - /// `slow` holds 1 MB at 100 KB/s, a 10 s queue at a tenth of its cap. `fast` holds - /// 256 KB at `fast_rate`, half its cap. Returns which of them `msg` is routed to. - #[test_case(get_headers(1), 1 << 20 => "fast"; "a header request goes to the shortest queue")] - #[test_case(get_headers(1), 0 => "slow"; "a header request falls back to the cap ratio while a busy peer is unmeasured")] - #[test_case(NetworkMessage::GetData(vec![block(1)]), 1 << 20 => "fast"; "a block request goes to the shortest queue")] - #[test_case(NetworkMessage::GetData(vec![block(1)]), 0 => "slow"; "a block request falls back to the cap ratio while a busy peer is unmeasured")] - #[test_case(get_cfilters(0), 1 << 20 => "slow"; "bulk requests keep the cap ratio")] + /// `slow` holds 1 MB, a tenth of its cap, served at `slow_rate`. `fast` holds 256 KB, half + /// its cap, served at `fast_rate`, or at `fast_served` over the last two seconds while its + /// serving rate is unmeasured. Returns which of them `msg` is routed to. + #[test_case(get_headers(1), 100 << 10, 1 << 20, 0 => "fast"; "a header request goes to the shortest queue")] + #[test_case(get_headers(1), 100 << 10, 0, 1 << 20 => "fast"; "a peer still being measured is timed by its last two seconds")] + #[test_case(get_headers(1), 0, 1 << 20, 0 => "fast"; "a busy peer that delivered nothing yet goes last")] + #[test_case(get_headers(1), 0, 0, 0 => "slow"; "the cap ratio while no peer has a rate")] + #[test_case(get_block(1), 100 << 10, 1 << 20, 0 => "fast"; "a block request goes to the shortest queue")] + #[test_case(get_block(1), 100 << 10, 0, 1 << 20 => "fast"; "a block request times a peer by its last two seconds")] + #[test_case(get_cfilters(0), 100 << 10, 1 << 20, 0 => "slow"; "bulk requests keep the cap ratio")] #[tokio::test] - async fn route_picks_the_peer_by_class(msg: NetworkMessage, fast_rate: u64) -> &'static str { + async fn route_picks_the_peer_by_class( + msg: NetworkMessage, + slow_rate: u64, + fast_rate: u64, + fast_served: u64, + ) -> &'static str { let costs = ResponseCosts::new(); let slow = ConnectedPeer::for_test(conn(1, 0)).await; let fast = ConnectedPeer::for_test(conn(2, 0)).await; @@ -1053,13 +1220,14 @@ mod tests { fast.handle().send(&get_cfilters(4000)).await.expect("loopback send"); slow.set_cap(slow.in_flight_bytes() * 10); fast.set_cap(fast.in_flight_bytes() * 2); - slow.set_serving_rate(100 << 10); + slow.set_serving_rate(slow_rate); fast.set_serving_rate(fast_rate); + fast.set_served(fast_served); let before = (slow.in_flight_bytes(), fast.in_flight_bytes()); let broker = Broker::new(); broker.enqueue(0, msg).await; - assert_eq!(broker.route(&[slow.handle(), fast.handle()], u64::MAX, &costs).await, 1); + assert_eq!(broker.route(&[slow.handle(), fast.handle()], &costs).await, 1); match (slow.in_flight_bytes() > before.0, fast.in_flight_bytes() > before.1) { (true, false) => "slow", diff --git a/dash-spv/src/network/costs.rs b/dash-spv/src/network/costs.rs index 19b26ecae..aae47657f 100644 --- a/dash-spv/src/network/costs.rs +++ b/dash-spv/src/network/costs.rs @@ -13,7 +13,7 @@ use tokio::sync::Mutex; const HEADERS_RESP_BYTES: u64 = 2000 * 81; /// Dash Core sends up to 8000 compressed headers of about 43 bytes each. -const HEADERS2_RESP_BYTES: u64 = 8000 * 43; +pub(super) const HEADERS2_RESP_BYTES: u64 = 8000 * 43; const CFHEADERS_RESP_BYTES: u64 = 2000 * 32 + 65; const RESP_BYTES_ALPHA: f64 = 0.3; const REPLY_KINDS: usize = 10; diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index baec0ffef..e04281b14 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1,5 +1,5 @@ use std::{ - collections::{HashMap, HashSet, VecDeque}, + collections::{HashMap, HashSet}, net::SocketAddr, sync::{ atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, @@ -206,7 +206,6 @@ pub struct NetworkState { pub subscribers: Subscribers, pub inbound: UnboundedSender, pub bytes: AtomicU64, - pub global_cap: AtomicU64, pub peer_wake: Notify, pub costs: Arc, pub max_peers: usize, @@ -257,7 +256,6 @@ impl PeerNetworkManager { subscribers: Mutex::new(HashMap::new()), inbound, bytes: AtomicU64::new(0), - global_cap: AtomicU64::new(MIN_CAP_BYTES * max_peers as u64), peer_wake: Notify::new(), costs: Arc::new(ResponseCosts::new()), max_peers, @@ -413,7 +411,6 @@ fn spawn_router(state: Arc, shutdown: CancellationToken) -> JoinHa broker, connected, costs, - global_cap, .. } = &*state; loop { @@ -431,7 +428,7 @@ fn spawn_router(state: Arc, shutdown: CancellationToken) -> JoinHa let peers: Vec = connected.read().await.iter().map(|p| p.handle()).collect(); - let sent = broker.route(&peers, global_cap.load(Ordering::Relaxed), costs).await; + let sent = broker.route(&peers, costs).await; drop(peers); if sent == 0 { @@ -453,33 +450,23 @@ const FLOW_TICK: Duration = Duration::from_millis(250); /// Weight of a new tick in the rate and in-flight averages (~1 s horizon). const FLOW_EMA_ALPHA: f64 = 0.25; -/// Ticks of round-trip samples the base round trip is the minimum of (30 s). -const BASE_RTT_TICKS: u64 = 120; - -/// Shortest period between adjustments, in ticks; otherwise two round trips. -const MIN_PERIOD_TICKS: u32 = 4; - -/// Smallest cap of a peer (and of the link, per peer): one response always fits. +/// Smallest cap of a peer: two header replies, so a new peer has one queued behind the +/// one it is serving. const MIN_CAP_BYTES: u64 = INITIAL_CAP_BYTES; -/// Linear probe per second, per peer: linear in time, not per adjustment, so a far -/// peer (adjusted once per two long round trips) ramps as fast as a near one. -const CAP_STEP_BYTES_PER_SEC: f64 = 512.0 * 1024.0; - -/// Added in-flight must buy at least this share of a proportional rate gain, or the -/// path is full and the addition only queued. -const MIN_ELASTICITY: f64 = 0.3; +/// Seconds of a peer's served rate its cap allows in flight. A request sent to the peer +/// waits this long behind the ones before it, so this stays small. It must still exceed +/// the round trip, or a peer limited by its cap cannot show it could serve more. +const TARGET_QUEUE_SECS: f64 = 2.0; -/// In-flight growth below this is noise, not an experiment to judge. -const MIN_GROWTH: f64 = 0.05; +/// Horizon of the served rate. A burst of `n` bytes raises the cap by about `n`. +const SERVED_RATE_SECS: f64 = 2.0; -/// Queue at the bottleneck, as a fraction of the bandwidth-delay product, above which -/// the cap is cut by the excess (TCP Vegas' beta) whatever the rate did. -const QUEUE_HIGH: f64 = 1.0; +/// Growth per second of a bound peer's cap, to find more than the served rate shows. +const CAP_PROBE_BYTES_PER_SEC: f64 = 32.0 * 1024.0; -/// Queue floor in bytes: responses are lumpy (a filter batch can be 1 MB), so a small -/// bandwidth-delay product must still absorb a couple. -const QUEUE_FLOOR_BYTES: f64 = 1024.0 * 1024.0; +/// Share per second of its excess over the target that a bound peer's cap gives back. +const CAP_DECAY_PER_SEC: f64 = 0.5; /// Ticks a peer must have been bandwidth bound before its rate is compared, and the /// span its serving rate is averaged over, so a momentary dip never condemns it. @@ -488,48 +475,30 @@ const SERVING_SAMPLE_TICKS: u32 = 120; /// A peer serving below this fraction of the best connected peer's rate is slow. const SLOW_RATE_FRACTION: f64 = 0.25; -/// Sizes a byte cap from throughput alone. Grows it linearly while each addition of -/// in-flight bytes still buys proportionally more bytes/s; once the rate stops -/// following, scales it back by how much of the addition did not pay. As a guard, -/// by Little's law the round trip is `in_flight / rate` and its 30 s minimum the -/// path's base, so `in_flight - rate ยท base_rtt` is what sits queued (TCP Vegas): -/// more than a bandwidth-delay product of it cuts the cap by the excess. +/// Sizes a peer's byte cap in seconds of its own served rate: a fast peer reaches +/// megabytes within a second or two and a slow one stays near [`MIN_CAP_BYTES`]. The cap +/// rises at once when the served rate supports it. It only falls while the peer is +/// [`ByteLimiter::serving_all`]. Otherwise a peer shows how much it was asked for, which +/// says nothing about what it can serve. struct ByteLimiter { cap: f64, - min_cap: f64, /// Bytes/s and bytes in flight, averaged over ticks. rate: f64, in_flight: f64, + /// Bytes/s the peer serves, averaged over [`SERVED_RATE_SECS`] of ticks in which it + /// served all it could. + served: f64, last_bytes: u64, - /// Monotonic window of (tick, round trip) giving the minimum over the last - /// [`BASE_RTT_TICKS`]. - rtts: VecDeque<(u64, f64)>, - ticks: u64, - base_rtt: f64, - /// Rate and in-flight at the last adjustment, to judge what it bought. - prev_rate: f64, - prev_in_flight: f64, - since_adjust: u32, - /// Whether the cap was ever cut: until then it is still ramping, and the rate it - /// allows says nothing about the peer. - found_ceiling: bool, } impl ByteLimiter { - fn new(cap: u64, last_bytes: u64) -> Self { + fn new(last_bytes: u64) -> Self { Self { - cap: cap as f64, - min_cap: cap as f64, + cap: MIN_CAP_BYTES as f64, rate: 0.0, in_flight: 0.0, + served: 0.0, last_bytes, - rtts: VecDeque::new(), - ticks: 0, - base_rtt: 0.0, - prev_rate: 0.0, - prev_in_flight: 0.0, - since_adjust: 0, - found_ceiling: false, } } @@ -537,67 +506,47 @@ impl ByteLimiter { self.in_flight / self.rate.max(1.0) } - /// One tick with the cumulative bytes received and the bytes in flight now; - /// `step` is the linear growth per second. - fn tick(&mut self, bytes: u64, in_flight: u64, step: f64) -> u64 { + /// Whether the router has given the peer all its cap allows. + fn bound(&self, in_flight: u64) -> bool { + in_flight > 0 && in_flight >= self.cap as u64 + } + + /// Whether what the peer delivers is all it can serve: it is bound, or it holds more than + /// [`TARGET_QUEUE_SECS`] of what it delivers. A cap raised by a burst leaves the peer below + /// it, so without the second case the cap never comes down while the queue grows. + /// Less than [`MIN_CAP_BYTES`] in flight is a reply or two in the making, not a backlog. + fn serving_all(&self, in_flight: u64) -> bool { + self.bound(in_flight) + || (in_flight >= MIN_CAP_BYTES && in_flight as f64 > self.rate * TARGET_QUEUE_SECS) + } + + /// One tick with the cumulative bytes received and the bytes in flight now. A stall + /// of our own link says nothing about the peer, so it leaves the cap alone. + fn tick(&mut self, bytes: u64, in_flight: u64, link_stalled: bool) -> u64 { let tick_s = FLOW_TICK.as_secs_f64(); - let got = bytes.saturating_sub(self.last_bytes) as f64; + let got = bytes.saturating_sub(self.last_bytes) as f64 / tick_s; self.last_bytes = bytes; - self.rate += FLOW_EMA_ALPHA * (got / tick_s - self.rate); + self.rate += FLOW_EMA_ALPHA * (got - self.rate); self.in_flight += FLOW_EMA_ALPHA * (in_flight as f64 - self.in_flight); - self.ticks += 1; - - // Nothing asked or nothing arriving says nothing about the path. - if in_flight == 0 || self.rate < 1.0 || self.in_flight < 1.0 { + if link_stalled { return self.cap as u64; } - let rtt = self.rtt(); - while self.rtts.back().is_some_and(|&(_, r)| r >= rtt) { - self.rtts.pop_back(); - } - self.rtts.push_back((self.ticks, rtt)); - while self.rtts.front().is_some_and(|&(t, _)| t + BASE_RTT_TICKS < self.ticks) { - self.rtts.pop_front(); - } - self.base_rtt = self.rtts.front().map_or(rtt, |&(_, r)| r); - // One adjustment per two round trips, so each judges the last one's effect. - self.since_adjust += 1; - let period = ((2.0 * rtt / tick_s).ceil() as u32).max(MIN_PERIOD_TICKS); - if self.since_adjust < period { - return self.cap as u64; + let serving_all = self.serving_all(in_flight); + if serving_all || got > self.served { + self.served += tick_s / SERVED_RATE_SECS * (got - self.served); } - let step = step * self.since_adjust as f64 * tick_s; - self.since_adjust = 0; - let (prev_rate, prev_in_flight) = (self.prev_rate, self.prev_in_flight); - self.prev_rate = self.rate; - self.prev_in_flight = self.in_flight; - - let bdp = self.rate * self.base_rtt; - let queued = self.in_flight - bdp; - let high = (bdp * QUEUE_HIGH).max(QUEUE_FLOOR_BYTES); - let growth = if prev_in_flight > 0.0 { - self.in_flight / prev_in_flight - 1.0 - } else { - 0.0 - }; - let gain = if prev_rate > 0.0 { - self.rate / prev_rate - 1.0 - } else { - 0.0 - }; - // Growth needs the cap to be what limits us; an under-used cap proves nothing. - let limited = in_flight as f64 + step >= self.cap; - if queued > high { - self.cap -= queued - high; - self.found_ceiling = true; - } else if growth > MIN_GROWTH && gain < growth * MIN_ELASTICITY { - self.cap *= (1.0 + gain.max(0.0)) / (1.0 + growth); - self.found_ceiling = true; - } else if limited { - self.cap += step; - } - self.cap = self.cap.max(self.min_cap); + let target = self.served * TARGET_QUEUE_SECS; + if target > self.cap { + self.cap = target; + } else if serving_all { + // Only a bound peer can show it would serve more than its cap allows. + if self.bound(in_flight) { + self.cap += CAP_PROBE_BYTES_PER_SEC * tick_s; + } + self.cap -= (self.cap - target).max(0.0) * CAP_DECAY_PER_SEC * tick_s; + } + self.cap = self.cap.max(MIN_CAP_BYTES as f64); self.cap as u64 } } @@ -612,20 +561,17 @@ struct PeerFlow { impl PeerFlow { fn new(bytes: u64) -> Self { Self { - limiter: ByteLimiter::new(MIN_CAP_BYTES, bytes), + limiter: ByteLimiter::new(bytes), serving_rate: 0.0, serving_ticks: 0, } } - /// One tick with the peer's cumulative bytes received, its bytes in flight now, - /// whether the client's own cap holds back work and whether our link is stalled. - /// Returns its new cap. - fn tick(&mut self, bytes: u64, in_flight: u64, link_full: bool, link_stalled: bool) -> u64 { - // The router sends a peer work until it reaches its cap or the link cap binds. - // Short of both, its rate shows how much it was asked for, not its bandwidth. - let bound = in_flight > 0 && (in_flight >= self.limiter.cap as u64 || link_full); - let cap = self.limiter.tick(bytes, in_flight, CAP_STEP_BYTES_PER_SEC); + /// One tick with the peer's cumulative bytes received, its bytes in flight now and + /// whether our link is stalled. Returns its new cap. + fn tick(&mut self, bytes: u64, in_flight: u64, link_stalled: bool) -> u64 { + let bound = self.limiter.bound(in_flight); + let cap = self.limiter.tick(bytes, in_flight, link_stalled); self.sample(self.limiter.rate, bound, link_stalled); cap } @@ -643,7 +589,7 @@ impl PeerFlow { /// Returns the peer's serving rate in bytes/s, 0 until measured. fn serving_rate(&self) -> u64 { - let measured = self.limiter.found_ceiling && self.serving_ticks >= SERVING_SAMPLE_TICKS; + let measured = self.serving_ticks >= SERVING_SAMPLE_TICKS; if measured { self.serving_rate as u64 } else { @@ -652,14 +598,13 @@ impl PeerFlow { } } -/// Sizes the byte caps of every peer and of the client's own link, each with a -/// [`ByteLimiter`] on its own throughput: the router sends while both have room. +/// Sizes the byte cap of every peer with a [`ByteLimiter`] on its own throughput. Each cap +/// follows its peer's share of the client's link, so a full link needs no cap of its own. /// Also reports how fast each peer serves while bandwidth bound, for [`underserving_peer`]. fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) -> JoinHandle<()> { tokio::spawn(async move { let NetworkState { bytes, - global_cap, connected, peer_wake, broker, @@ -667,7 +612,6 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) link_stalled, .. } = &*state; - let mut global = ByteLimiter::new(MIN_CAP_BYTES, bytes.load(Ordering::Relaxed)); let mut peers: HashMap = HashMap::new(); let mut last_underserving: Option = None; // Bytes read, in flight and cap at each peer's last `peer_speed` line. A peer is @@ -692,20 +636,14 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) let in_flights: Vec = g.iter().map(ConnectedPeer::in_flight_bytes).collect(); let total_in_flight: u64 = in_flights.iter().sum(); - let link_full = total_in_flight >= global_cap.load(Ordering::Relaxed); let stalled = link_stalled.load(Ordering::Relaxed); for (p, &in_flight) in g.iter().zip(&in_flights) { let flow = peers.entry(p.id()).or_insert_with(|| PeerFlow::new(p.bytes_read())); - p.set_cap(flow.tick(p.bytes_read(), in_flight, link_full, stalled)); + p.set_cap(flow.tick(p.bytes_read(), in_flight, stalled)); p.set_serving_rate(flow.serving_rate()); + p.set_served(flow.limiter.served as u64); } - global.min_cap = (MIN_CAP_BYTES * g.len() as u64) as f64; - global.cap = global.cap.max(global.min_cap); - let step = CAP_STEP_BYTES_PER_SEC * g.len() as f64; - let cap = global.tick(bytes.load(Ordering::Relaxed), total_in_flight, step); - global_cap.store(cap, Ordering::Relaxed); - if ticks.is_multiple_of(2) { let caps: u64 = g.iter().map(|p| p.cap()).sum(); reported.retain(|id, _| live.contains(id)); @@ -719,40 +657,35 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) any_reported = true; tracing::debug!( target: "peer_speed", - "peer {}: cap={} KB in_flight={} KB rate={:.2} MB/s rtt={:.0}ms base={:.0}ms lag={}ms total={:.1} MB", + "peer {}: cap={} KB in_flight={} KB rate={:.2} MB/s served={:.2} MB/s rtt={:.0}ms lag={}ms total={:.1} MB", p.addr(), p.cap() >> 10, p.in_flight_bytes() >> 10, flow.limiter.rate / 1e6, + flow.limiter.served / 1e6, flow.limiter.rtt() * 1e3, - flow.limiter.base_rtt * 1e3, p.lag_ms(), p.bytes_read() as f64 / 1e6, ); } - let bind = if total_in_flight >= cap.min(caps) { - if cap <= caps { - "global" - } else { - "peers" - } + let bind = if total_in_flight >= caps { + "peers" } else { "work" }; let snapshot = - Some((bytes.load(Ordering::Relaxed), total_in_flight, cap, caps, g.len())); + Some((bytes.load(Ordering::Relaxed), total_in_flight, caps, g.len())); if any_reported || reported_bandwidth != snapshot { reported_bandwidth = snapshot; + let rate: f64 = peers.values().map(|flow| flow.limiter.rate).sum(); tracing::debug!( target: "peer_speed", - "bandwidth: {:.1} MB/s bind={} | global_cap={} sum_peer_cap={} in_flight={} KB | rtt={:.0}ms base={:.0}ms | peers={} | resp KB {:?}", - global.rate / 1e6, + "bandwidth: {:.1} MB/s bind={} | sum_peer_cap={} in_flight={} KB | rtt={:.0}ms | peers={} | resp KB {:?}", + rate / 1e6, bind, - cap >> 10, caps >> 10, total_in_flight >> 10, - global.rtt() * 1e3, - global.base_rtt * 1e3, + total_in_flight as f64 / rate.max(1.0) * 1e3, g.len(), costs.expected_kb(), ); @@ -1534,22 +1467,16 @@ mod tests { } } - fn tick(&mut self, delivered: u64, in_flight: u64, link_full: bool) { + fn tick(&mut self, delivered: u64, in_flight: u64) { self.bytes += delivered; - self.cap = self.flow.tick(self.bytes, in_flight, link_full, false); + self.cap = self.flow.tick(self.bytes, in_flight, false); } - /// Bulk download at `rate` bytes/s. The router keeps the peer at its own cap, or - /// at half of it when the client's link cap binds first. - fn bulk(&mut self, rate: u64, link_full: bool, ticks: u32) { + /// Bulk download at `rate` bytes/s, with the router keeping the peer at its cap. + fn bulk(&mut self, rate: u64, ticks: u32) { let delivered = (rate as f64 * FLOW_TICK.as_secs_f64()) as u64; for _ in 0..ticks { - let in_flight = if link_full { - self.cap / 2 - } else { - self.cap - }; - self.tick(delivered, in_flight, link_full); + self.tick(delivered, self.cap); } } @@ -1559,11 +1486,11 @@ mod tests { const REPLY_BYTES: u64 = 4 * 1024; for _ in 0..count { for _ in 1..latency { - self.tick(0, REPLY_BYTES, false); + self.tick(0, REPLY_BYTES); } - self.tick(REPLY_BYTES, 0, false); + self.tick(REPLY_BYTES, 0); for _ in 0..40 { - self.tick(0, 0, false); + self.tick(0, 0); } } } @@ -1571,18 +1498,16 @@ mod tests { /// Only a peer serving all the work its caps allow is compared. Small replies at the /// tip measure the round trip, so a far peer must not look slow for them. - #[test_case(1_000_000, false, true => None; "latency bound replies at the tip")] - #[test_case(100_000, false, false => Some(1); "a slow peer at its own cap")] - #[test_case(100_000, true, false => Some(1); "a slow peer held back by the link cap")] + #[test_case(1_000_000, true => None; "latency bound replies at the tip")] + #[test_case(100_000, false => Some(1); "a slow peer at its own cap")] #[tokio::test] async fn only_bandwidth_bound_service_condemns_a_peer( second_rate: u64, - link_full: bool, then_idle: bool, ) -> Option { let mut served = [Served::new(), Served::new()]; for (s, rate) in served.iter_mut().zip([1_000_000, second_rate]) { - s.bulk(rate, link_full, SERVING_SAMPLE_TICKS * 10); + s.bulk(rate, SERVING_SAMPLE_TICKS * 10); assert!(s.flow.serving_rate() > 0, "a bandwidth bound peer is measured"); } if then_idle { @@ -2034,4 +1959,145 @@ mod tests { fn message_type_from_cmd(cmd: &str) -> Option { MessageType::from_cmd(cmd) } + + /// Returns the initial cap and the cap after every tick of `secs`, while the sender + /// keeps the cap full and the path delivers `serve(second, in_flight)` bytes/s. + fn caps_while_full(secs: u32, serve: impl Fn(f64, f64) -> f64) -> Vec { + caps_with_work(secs, |_, cap| cap, serve) + } + + /// Like [`caps_while_full`], with `work(second, cap)` bytes in flight. + fn caps_with_work( + secs: u32, + work: impl Fn(f64, u64) -> u64, + serve: impl Fn(f64, f64) -> f64, + ) -> Vec { + let tick_s = FLOW_TICK.as_secs_f64(); + let mut limiter = ByteLimiter::new(0); + let mut bytes = 0.0; + let mut caps = vec![MIN_CAP_BYTES]; + for tick in 0..(secs as f64 / tick_s) as usize { + let second = tick as f64 * tick_s; + let in_flight = work(second, caps[caps.len() - 1]); + bytes += serve(second, in_flight as f64) * tick_s; + caps.push(limiter.tick(bytes as u64, in_flight, false)); + } + caps + } + + fn cap_after(caps: &[u64], secs: u32) -> u64 { + caps[(secs as f64 / FLOW_TICK.as_secs_f64()) as usize] + } + + #[test_case(0.1, 2; "a near link")] + #[test_case(0.8, 5; "a far link")] + #[test_case(1.5, 30; "a link whose round trip nears the target queue")] + fn a_byte_cap_ramps_to_fill_a_fast_link(round_trip: f64, within_secs: u32) { + let link_rate = 10e6; + let caps = caps_while_full(60, |_, in_flight| (in_flight / round_trip).min(link_rate)); + + let bdp = (link_rate * round_trip) as u64; + assert!(cap_after(&caps, within_secs) >= bdp, "{}", cap_after(&caps, within_secs)); + } + + #[test] + fn a_slow_peer_keeps_a_small_cap() { + let caps = caps_while_full(300, |_, _| 150_000.0); + + let peak = caps.iter().max().copied().unwrap_or_default(); + assert!(peak < MIN_CAP_BYTES + 128 * 1024, "peak cap {peak}"); + } + + /// After 10 s at 5 MB/s the peer serves 100 KB/s. With 4 MB to serve it stays below the + /// cap that burst earned, yet holds a 40 s queue, so the cap must fall all the same. + #[test_case(false; "kept full")] + #[test_case(true; "below the cap with a backlog")] + fn a_cap_falls_with_the_peer_that_slowed_down(backlog: bool) { + let work = |second: f64, cap: u64| { + if backlog && second >= 10.0 { + 4 << 20 + } else { + cap + } + }; + let caps = caps_with_work(60, work, |second, _| { + if second < 10.0 { + 5e6 + } else { + 100_000.0 + } + }); + + assert!(cap_after(&caps, 10) > 8_000_000, "{}", cap_after(&caps, 10)); + assert!(cap_after(&caps, 30) < MIN_CAP_BYTES + 128 * 1024, "{}", cap_after(&caps, 30)); + } + + /// The peer alternates a second at 4 MB/s with a silent second. Its cap follows that + /// 2 MB/s average, not each burst, which alone would ask for 8 MB. + #[test] + fn a_bursty_peer_keeps_a_steady_cap() { + let caps = caps_while_full(60, |second, _| { + if (second as u32).is_multiple_of(2) { + 4e6 + } else { + 0.0 + } + }); + + let settled = &caps[caps.len() / 2..]; + let (low, high) = (settled.iter().min(), settled.iter().max()); + assert!( + low.zip(high).is_some_and(|(&low, &high)| low > 2_500_000 && high < 6_000_000), + "{low:?} to {high:?}" + ); + } + + /// A peer that got less work than its cap shows what it was asked for, not what it can + /// serve. Its cap neither falls nor grows then, and bulk work finds it where it was. + #[test_case(4 * 1024, 4 * 1024, 1000; "a trickle of small replies")] + #[test_case(0, 0, 1000; "no work")] + #[test_case(MIN_CAP_BYTES / 2, 0, 40; "one reply in the making")] + fn a_peer_below_its_cap_keeps_it(in_flight: u64, per_tick: u64, ticks: usize) { + let delivered = (5e6 * FLOW_TICK.as_secs_f64()) as u64; + let mut limiter = ByteLimiter::new(0); + let mut bytes = 0; + let mut cap = MIN_CAP_BYTES; + for _ in 0..40 { + bytes += delivered; + cap = limiter.tick(bytes, cap, false); + } + let earned = cap; + + for _ in 0..ticks { + bytes += per_tick; + assert_eq!(limiter.tick(bytes, in_flight, false), earned); + } + for _ in 0..40 { + bytes += delivered; + cap = limiter.tick(bytes, cap, false); + assert!(cap as f64 > earned as f64 * 0.9, "{cap} fell from {earned}"); + } + } + + #[test] + fn a_stall_of_our_link_leaves_the_cap_alone() { + let delivered = (5e6 * FLOW_TICK.as_secs_f64()) as u64; + let mut limiter = ByteLimiter::new(0); + let mut bytes = 0; + let mut cap = MIN_CAP_BYTES; + for _ in 0..40 { + bytes += delivered; + cap = limiter.tick(bytes, cap, false); + } + let earned = cap; + + for _ in 0..40 { + cap = limiter.tick(bytes, cap, true); + } + assert_eq!(cap, earned); + assert!( + limiter.tick(bytes + delivered, cap, false) > earned / 2, + "the rate is still known" + ); + } } diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index 5d63fa243..2ed082179 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -36,7 +36,7 @@ use tokio_util::either::Either; use tokio_util::sync::CancellationToken; use crate::network::bans::Ban; -use crate::network::costs::{as_sent, Ledger}; +use crate::network::costs::{as_sent, Ledger, HEADERS2_RESP_BYTES}; use crate::network::manager::{InboundMessage, NetworkState}; use crate::network::transport::{encrypt_message, TransportPreference, V2Decoder}; use crate::{error::NetworkResult, NetworkError}; @@ -89,7 +89,7 @@ const V2_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); const WRITE_TIMEOUT: Duration = Duration::from_secs(5); const PING_INTERVAL: Duration = Duration::from_secs(120); /// A new peer's in-flight byte cap until the controller has measured it. -pub const INITIAL_CAP_BYTES: u64 = 256 * 1024; +pub const INITIAL_CAP_BYTES: u64 = 2 * HEADERS2_RESP_BYTES; /// Counts bytes read, globally and per peer, to size the in-flight budgets from /// measured throughput. @@ -260,6 +260,9 @@ pub struct PeerHandle { cap: Arc, /// Bytes/s this peer serves while bandwidth bound, 0 until measured. serving_rate: Arc, + /// Bytes/s this peer served over the last two seconds, 0 until it delivered anything. + /// A rough stand-in for `serving_rate` while that is still being measured. + served: Arc, headers2: bool, /// Child of the global shutdown; cancelling it stops the reader and closes the socket. token: CancellationToken, @@ -278,6 +281,10 @@ impl PeerHandle { self.serving_rate.load(Ordering::Relaxed) } + pub(super) fn served(&self) -> u64 { + self.served.load(Ordering::Relaxed) + } + pub fn in_flight_bytes(&self) -> u64 { self.ledger.in_flight_bytes() } @@ -389,6 +396,10 @@ impl ConnectedPeer { self.handle.serving_rate.store(bytes_per_sec, Ordering::Relaxed); } + pub(crate) fn set_served(&self, bytes_per_sec: u64) { + self.handle.served.store(bytes_per_sec, Ordering::Relaxed); + } + /// A handshaken connection `id` over a loopback socket, with no reader running. /// The far end discards what it reads, so sends succeed. #[cfg(test)] @@ -406,6 +417,7 @@ impl ConnectedPeer { ledger: Arc::new(Ledger::new(Arc::new(ResponseCosts::new()))), cap: Arc::new(AtomicU64::new(INITIAL_CAP_BYTES)), serving_rate: Arc::new(AtomicU64::new(0)), + served: Arc::new(AtomicU64::new(0)), headers2: false, token: CancellationToken::new(), }, @@ -644,6 +656,7 @@ impl DisconnectedPeer { ledger, cap, serving_rate: Arc::new(AtomicU64::new(0)), + served: Arc::new(AtomicU64::new(0)), headers2, token, },