diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index 0fbbff9c2..eb440f361 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -1,5 +1,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; -use std::sync::atomic::{AtomicI64, AtomicUsize, Ordering}; +use std::mem; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicI64, AtomicU64, AtomicUsize, Ordering}; use std::time::Duration; use dashcore::network::message::NetworkMessage; @@ -25,6 +27,9 @@ const HEDGE_OVERDUE: Duration = Duration::from_secs(10); /// rate the request was routed by. A peer's queue can hold many seconds of work, and a copy /// sent while the batch merely waits in it is downloaded twice. const HEDGE_QUEUE_FACTOR: f64 = 2.0; +/// A peer that refused a request is not asked for it again for this long. A node whose +/// filter index is still catching up refuses only briefly, so the refusal must not last. +const REFUSAL_HELD: Duration = Duration::from_secs(10); const MIN_IN_FLIGHT_PER_CLASS: usize = 4; const MSG_CLASSES: usize = 5; const DRAIN_PRIORITY: [MsgClass; MSG_CLASSES] = @@ -63,8 +68,7 @@ pub fn request_keys(msg: &NetworkMessage) -> Vec { } enum ReqState { - /// Attempts so far. - Queued(u32), + Queued(Retry), OnWire(Box), /// The whole reply was received and waits for its manager to release the key. Arrived, @@ -83,10 +87,32 @@ struct OnWire { sent: Instant, /// How long after `sent` the request is overdue although its peer keeps delivering. overdue_after: Duration, + /// Rises with every message sent, in send order, which is the order peers answer in. + order: u64, msg: NetworkMessage, height: u32, hedge: Option, + retry: Retry, +} + +/// What earlier attempts of a request taught. +#[derive(Clone, Default)] +struct Retry { attempts: u32, + /// Peers that let the request expire, skipped it or sent `notfound`, and when. + refused: Vec<(SocketAddr, Instant)>, +} + +impl Retry { + fn refuse(&mut self, addr: SocketAddr) { + self.refused.retain(|(a, _)| *a != addr); + self.refused.push((addr, Instant::now())); + } + + /// The peers that refused the request within [`REFUSAL_HELD`]. + fn refusers(&self) -> impl Iterator + '_ { + self.refused.iter().filter(|(_, at)| at.elapsed() < REFUSAL_HELD).map(|(addr, _)| *addr) + } } /// The request queue plus the registry of every keyed request in play, queued, on the @@ -94,6 +120,7 @@ struct OnWire { pub struct Broker { queue: MsgQueue, requests: Mutex>, + sends: AtomicU64, } impl Broker { @@ -101,6 +128,7 @@ impl Broker { Self { queue: MsgQueue::new(), requests: Mutex::new(HashMap::new()), + sends: AtomicU64::new(0), } } @@ -124,7 +152,7 @@ impl Broker { return; } for key in keys { - reqs.insert(key, ReqState::Queued(0)); + reqs.insert(key, ReqState::Queued(Retry::default())); } } self.queue.push(height, msg).await; @@ -160,7 +188,13 @@ impl Broker { self.queue.pop_n(self.queue.len()).await; } - async fn requeue(&self, matches: impl Fn(&RequestKey, &OnWire) -> bool) -> usize { + /// Re-queues the on-wire requests `matches` selects. With `refused`, their peers count as + /// having refused them. + async fn requeue( + &self, + matches: impl Fn(&RequestKey, &OnWire) -> bool, + refused: bool, + ) -> usize { let retried: Vec<(u32, NetworkMessage)> = { let mut reqs = self.requests.lock().await; let keys: Vec = reqs @@ -173,8 +207,18 @@ impl Broker { keys.into_iter() .filter_map(|key| match reqs.remove(&key) { Some(ReqState::OnWire(o)) => { - reqs.insert(key, ReqState::Queued(o.attempts)); - Some((o.height, o.msg)) + let OnWire { + peer, + msg, + height, + mut retry, + .. + } = *o; + if refused { + retry.refuse(peer.addr()); + } + reqs.insert(key, ReqState::Queued(retry)); + Some((height, msg)) } _ => None, }) @@ -195,7 +239,7 @@ impl Broker { } } } - self.requeue(|_, o| gone.contains(&o.peer)).await + self.requeue(|_, o| gone.contains(&o.peer), false).await } pub async fn not_found(&self, peer: PeerId, inv: &[Inventory]) { @@ -207,7 +251,7 @@ impl Broker { }) .collect(); if !keys.is_empty() { - self.requeue(|k, o| o.peer == peer && keys.contains(k)).await; + self.requeue(|k, o| o.peer == peer && keys.contains(k), true).await; } } @@ -216,21 +260,57 @@ impl Broker { /// it. A request put back in the queue is settled by a copy from any peer, and its message /// dropped. The key stays registered until released, so a re-declare before then sends /// nothing, and a manager that cannot use the reply must release the key. + /// + /// Peers answer in request order, so a reply from the carrier proves it skipped every + /// earlier request it holds that also settles on arrival. Those are retried elsewhere. pub async fn arrived(&self, peer: PeerId, key: RequestKey) { let mut reqs = self.requests.lock().await; - match reqs.get(&key) { - Some(ReqState::OnWire(o)) if o.peer == peer || o.hedge == Some(peer) => {} + let skipped: HashSet = match reqs.get(&key) { + Some(ReqState::OnWire(o)) if o.peer == peer => reqs + .iter() + .filter_map(|(earlier, state)| match state { + ReqState::OnWire(e) + if e.peer == peer && e.order < o.order && settles_on_arrival(earlier) => + { + Some(earlier.clone()) + } + _ => None, + }) + .collect(), + Some(ReqState::OnWire(o)) if o.hedge == Some(peer) => HashSet::new(), Some(ReqState::Queued(_)) => { reqs.remove(&key); self.drop_queued(&key, &reqs).await; + HashSet::new() } _ => return, - } + }; reqs.insert(key, ReqState::Arrived); drop(reqs); + if !skipped.is_empty() { + tracing::debug!( + target: "dash_spv::network", + "{} skipped {skipped:?}, retrying elsewhere", + peer.addr(), + ); + self.requeue(|k, _| skipped.contains(k), true).await; + } self.wake(); } + /// Whether a request is queued that every peer at `addrs` refused within + /// [`REFUSAL_HELD`], so only another peer can serve it now. + pub async fn refused_by_all(&self, addrs: &[SocketAddr]) -> bool { + !addrs.is_empty() + && self.requests.lock().await.values().any(|state| match state { + ReqState::Queued(retry) => { + let refusers: Vec = retry.refusers().collect(); + addrs.iter().all(|a| refusers.contains(a)) + } + _ => false, + }) + } + /// Peers answer in order, so any answer is progress on all of its requests. pub async fn progress(&self, peer: PeerId) { self.refresh(&[peer], true).await; @@ -279,21 +359,21 @@ impl Broker { } else { continue; }; - if o.attempts < KICKING_ATTEMPTS { + if o.retry.attempts < KICKING_ATTEMPTS { let since = culprits.entry(o.peer).or_insert(silent_since); *since = (*since).min(silent_since); } else { tracing::debug!( target: "dash_spv::network", "{key:?} unanswered after {} attempts, retrying without kicking {}", - o.attempts, + o.retry.attempts, o.peer, ); unanswerable.insert(key.clone()); } } if !unanswerable.is_empty() { - self.requeue(|k, _| unanswerable.contains(k)).await; + self.requeue(|k, _| unanswerable.contains(k), true).await; } culprits } @@ -319,6 +399,16 @@ impl Broker { } } let msgs = self.queue.pop_guaranteed(budget, &guaranteed, |m| costs.cost(m)).await; + let refused: HashMap> = { + let reqs = self.requests.lock().await; + msgs.iter() + .flat_map(|(_, msg)| request_keys(msg)) + .filter_map(|key| match reqs.get(&key) { + Some(ReqState::Queued(retry)) => Some((key, retry.refusers().collect())), + _ => None, + }) + .collect() + }; let mut sent = 0; // Popped but unsent goes back: its key is registered, so a dropped message // would stall its pipeline. @@ -328,8 +418,16 @@ impl Broker { for (height, msg) in msgs.by_ref() { // `peers` can be a snapshot taken before the timeout monitor closed one of them. let open = peers.iter().filter(|p| !p.is_closed() && p.in_flight_bytes() < p.cap()); - let Some(peer) = choose_peer(open, classify(&msg)) else { + let keys = request_keys(&msg); + let willing = open.clone().filter(|p| { + !keys.iter().any(|k| refused.get(k).is_some_and(|r| r.contains(&p.addr()))) + }); + let Some(peer) = choose_peer(willing, classify(&msg)) else { unsent.push((height, msg)); + // Every peer with room refused this one, so the next may still go out. + if open.clone().next().is_some() { + continue; + } break; }; if peer.send(&msg).await.is_ok() { @@ -343,29 +441,35 @@ impl Broker { } } unsent.extend(msgs); - self.queue.push_front_all(unsent).await; + // No wake: the router is the one waiting, and a wake for a refused message would spin it. + self.queue.restore(unsent).await; if !on_wire.is_empty() { let mut reqs = self.requests.lock().await; for (height, msg, peer, overdue_after) in on_wire { + let now = Instant::now(); + // One order for all keys of a message: its peer answers them in inventory order. + let order = self.sends.fetch_add(1, Ordering::Relaxed); for key in request_keys(&msg) { if let Some(slot) = reqs.get_mut(&key) { - let attempts = match slot { - ReqState::Queued(attempts) => *attempts, - ReqState::OnWire(o) => o.attempts, + let mut retry = match slot { + ReqState::Queued(retry) => mem::take(retry), + ReqState::OnWire(o) => mem::take(&mut o.retry), // The reply to a requeued request can arrive while it is resent. ReqState::Arrived => continue, }; + retry.attempts += 1; *slot = ReqState::OnWire(Box::new(OnWire { peer: peer.id(), msg: msg.clone(), height, - last_progress: Instant::now(), - last_answer: Instant::now(), - sent: Instant::now(), + last_progress: now, + last_answer: now, + sent: now, overdue_after, + order, hedge: None, - attempts: attempts + 1, + retry, })); } } @@ -462,10 +566,14 @@ impl Broker { last_answer: Instant::now(), sent: Instant::now(), overdue_after: HEDGE_OVERDUE, + order: self.sends.fetch_add(1, Ordering::Relaxed), msg: msg.clone(), height: 0, hedge: None, - attempts: 1, + retry: Retry { + attempts: 1, + refused: Vec::new(), + }, }; reqs.insert(key, ReqState::OnWire(Box::new(on_wire))); } @@ -497,6 +605,12 @@ impl OnWire { } } +/// Whether the reply to a request with `key` settles it on arrival. Only for those does a +/// later reply from the same peer prove that peer skipped the request. +fn settles_on_arrival(key: &RequestKey) -> bool { + matches!(key, RequestKey::Block(_) | RequestKey::CFilters(_) | RequestKey::CfHeaders(_)) +} + /// Returns the on-wire request of the lowest height among the keys `wanted` selects. fn lowest_on_wire( reqs: &HashMap, @@ -702,13 +816,18 @@ impl MsgQueue { if msgs.is_empty() { return; } + self.restore(msgs).await; + self.notify.notify_one(); + } + + /// Puts `msgs` back at the front of their queues without waking the router. + async fn restore(&self, msgs: Vec<(u32, NetworkMessage)>) { let n = msgs.len(); for (height, msg) in msgs.into_iter().rev() { let key = (height, self.front_seq.fetch_sub(1, Ordering::SeqCst)); self.queue(classify(&msg)).lock().await.insert(key, msg); } self.len.fetch_add(n, Ordering::SeqCst); - self.notify.notify_one(); } } @@ -1017,6 +1136,72 @@ mod tests { assert_eq!(broker.hedge_filters(&peers[..1]).await, None, "no other peer to ask"); } + enum Refusal { + /// The peer answered a later request first. + Skipped, + /// The request expired at the peer after it was already retried once. + Expired, + NotFound, + } + + /// Peer 1 has far more room, so only its refusal sends the retry to peer 2. Once both + /// refused, the request waits for another peer while the rest of the queue still goes out. + #[test_case(Refusal::Skipped; "a skipped request")] + #[test_case(Refusal::Expired; "an expired request")] + #[test_case(Refusal::NotFound; "a block not found")] + #[tokio::test(start_paused = true)] + async fn a_refused_request_is_retried_at_another_peer(refusal: Refusal) { + let broker = Broker::new(); + let costs = ResponseCosts::new(); + let first = ConnectedPeer::for_test(conn(1, 0)).await; + let second = ConnectedPeer::for_test(conn(2, 0)).await; + first.set_cap(1 << 30); + let both = [first.handle(), second.handle()]; + let key = RequestKey::Block(BlockHash::dummy(1)); + + broker.enqueue(1, get_block(1)).await; + broker.enqueue(2, get_block(2)).await; + assert_eq!(broker.route(&both[..1], &costs).await, 2); + match refusal { + Refusal::Skipped => { + broker.arrived(first.id(), RequestKey::Block(BlockHash::dummy(2))).await; + } + Refusal::Expired => { + broker.requeue_from(&HashSet::from([first.id()])).await; + assert_eq!(broker.route(&both[..1], &costs).await, 2); + tokio::time::advance(Duration::from_millis(1)).await; + assert!(broker.expire(|_| Duration::ZERO).await.is_empty(), "no kick"); + } + Refusal::NotFound => broker.not_found(first.id(), &[block(1)]).await, + } + assert_eq!(broker.state_of(&key).await, Some("queued")); + + let retried = if matches!(refusal, Refusal::Expired) { + 2 + } else { + 1 + }; + assert_eq!(broker.route(&both, &costs).await, retried); + assert_eq!(broker.carrier_of(&key).await, Some(second.id())); + assert!(!broker.refused_by_all(&[first.addr(), second.addr()]).await); + + broker.not_found(second.id(), &[block(1)]).await; + broker.enqueue(3, get_block(3)).await; + assert_eq!(broker.route(&both, &costs).await, 1, "only the other request goes out"); + assert_eq!(broker.state_of(&key).await, Some("queued")); + assert!(broker.refused_by_all(&[first.addr(), second.addr()]).await); + assert!(!broker.refused_by_all(&[first.addr(), second.addr(), conn(3, 0).addr()]).await); + + broker.notified().await; + assert_eq!(broker.route(&both, &costs).await, 0); + let woken = tokio::time::timeout(Duration::from_secs(1), broker.notified()).await; + assert!(woken.is_err(), "a request no peer will take must not spin the router"); + + tokio::time::advance(REFUSAL_HELD).await; + assert!(!broker.refused_by_all(&[first.addr(), second.addr()]).await); + assert_eq!(broker.route(&both, &costs).await, 1, "a refusal does not last"); + } + /// Peer 2 is idle and unmeasured, peer 3 measured and busy. Returns the peer asked for /// the second copy. #[test_case(get_cfilters(0), false => Some(3); "a filter batch goes to the measured peer")] diff --git a/dash-spv/src/network/costs.rs b/dash-spv/src/network/costs.rs index c089c4aac..01fdda77b 100644 --- a/dash-spv/src/network/costs.rs +++ b/dash-spv/src/network/costs.rs @@ -76,6 +76,13 @@ impl Reply { } } + /// Whether a peer sends this reply for every such request it serves and never unasked. + /// Peers answer in request order, so a reply of this kind proves that every earlier + /// request of this kind still awaited was skipped. + fn in_order(&self) -> bool { + matches!(self, Reply::CfHeaders(_) | Reply::CFilters(_) | Reply::Block(_)) + } + fn of_item(item: &Inventory) -> Option { match item { Inventory::Block(hash) => Some(Reply::Block(*hash)), @@ -230,9 +237,10 @@ impl Ledger { return; }; awaited[i].bytes += size; - if awaited[i].reply != Reply::CFilters(filter.block_hash) { + let stop = Reply::CFilters(filter.block_hash); + let Some(i) = awaited.iter().position(|a| a.reply == stop) else { return; - } + }; i } msg => { @@ -246,7 +254,22 @@ impl Ledger { i } }; - let a = awaited.remove(i); + let mut a = awaited.remove(i); + if a.reply.in_order() { + let mut idx = 0; + awaited.retain(|earlier| { + let skipped = idx < i && earlier.reply.in_order(); + idx += 1; + if skipped { + self.release(earlier.cost); + // A skipped batch got the bytes of the batch streamed after it. + if matches!(earlier.reply, Reply::CFilters(_)) { + a.bytes += earlier.bytes; + } + } + !skipped + }); + } self.release(a.cost); self.costs.record(&a.reply, a.bytes); let ns = a.sent.elapsed().as_nanos() as u64; @@ -265,14 +288,58 @@ impl Ledger { #[cfg(test)] mod tests { use super::*; + use dashcore::hash_types::FilterHeader; use dashcore::network::message_blockdata::GetHeadersMessage; + use dashcore::network::message_filter::{CFHeaders, CFilter, GetCFHeaders, GetCFilters}; use dashcore::network::message_headers2::Headers2Message; + use dashcore::{Block, Header}; use test_case::test_case; fn locator() -> GetHeadersMessage { GetHeadersMessage::new(vec![BlockHash::dummy(1)], BlockHash::dummy(0)) } + fn get_cfheaders(stop: u32) -> NetworkMessage { + NetworkMessage::GetCFHeaders(GetCFHeaders { + filter_type: 0, + start_height: 0, + stop_hash: BlockHash::dummy(stop), + }) + } + + fn cfheaders(stop: u32) -> NetworkMessage { + NetworkMessage::CFHeaders(CFHeaders { + filter_type: 0, + stop_hash: BlockHash::dummy(stop), + previous_filter_header: FilterHeader::all_zeros(), + filter_hashes: Vec::new(), + }) + } + + fn get_cfilters(stop: u32) -> NetworkMessage { + NetworkMessage::GetCFilters(GetCFilters { + filter_type: 0, + start_height: 0, + stop_hash: BlockHash::dummy(stop), + }) + } + + fn cfilter(block: u32) -> NetworkMessage { + NetworkMessage::CFilter(CFilter { + filter_type: 0, + block_hash: BlockHash::dummy(block), + filter: Vec::new(), + }) + } + + fn get_block(n: u32) -> NetworkMessage { + NetworkMessage::GetData(vec![Inventory::Block(Header::dummy(n).block_hash())]) + } + + fn block(n: u32) -> NetworkMessage { + NetworkMessage::Block(Block::dummy(n, Vec::new())) + } + fn headers2() -> NetworkMessage { NetworkMessage::Headers2(Headers2Message { headers: Vec::new(), @@ -314,4 +381,26 @@ mod tests { ledger.settle(&reply, charged).await; assert_eq!(ledger.in_flight_bytes(), in_flight); } + + /// The peer skipped the first request of each kind, so the reply to the second settles + /// both. The header request is not of a kind always answered in order, so it stays. + #[test_case(get_cfheaders(1), get_cfheaders(2), cfheaders(2); "filter headers")] + #[test_case(get_cfilters(1), get_cfilters(2), cfilter(2); "filters")] + #[test_case(get_block(1), get_block(2), block(2); "blocks")] + #[tokio::test] + async fn a_reply_settles_the_requests_its_peer_skipped( + skipped: NetworkMessage, + served: NetworkMessage, + reply: NetworkMessage, + ) { + let costs = Arc::new(ResponseCosts::new()); + let ledger = Ledger::new(costs.clone()); + let headers = NetworkMessage::GetHeaders2(locator()); + ledger.charge(&headers).await; + ledger.charge(&skipped).await; + ledger.charge(&served).await; + + ledger.settle(&reply, 100).await; + assert_eq!(ledger.in_flight_bytes(), costs.cost(&headers)); + } } diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 9cec3eec7..fb07bd3f9 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -926,6 +926,8 @@ impl Exploration { /// Why a peer of a full set is to be replaced. #[derive(Clone, Copy)] enum Replace { + /// A peer of a set where every peer refused a queued request. + Refused(SocketAddr), Slow(SocketAddr), Crowded(SocketAddr), /// The slowest of a set that serves slowly as a whole, rotated out to explore, and @@ -936,12 +938,16 @@ enum Replace { impl Replace { fn addr(self) -> SocketAddr { match self { - Replace::Slow(addr) | Replace::Crowded(addr) | Replace::Rotated(addr, _) => addr, + Replace::Refused(addr) + | Replace::Slow(addr) + | Replace::Crowded(addr) + | Replace::Rotated(addr, _) => addr, } } fn reason(self) -> &'static str { match self { + Replace::Refused(_) => "refusing", Replace::Slow(_) => "slow", Replace::Crowded(_) => "crowded", Replace::Rotated(..) => "rotated", @@ -951,7 +957,7 @@ impl Replace { /// Which candidates may replace the peer. fn pick(self) -> Pick { match self { - Replace::Slow(_) => Pick::Any, + Replace::Refused(_) | Replace::Slow(_) => Pick::Any, Replace::Crowded(_) => Pick::NewGroups, Replace::Rotated(..) => Pick::Faster, } @@ -1033,11 +1039,20 @@ impl Supervisor { // or an untried address is worth a swap even when no peer looks slow. let has_work = self.state.broker.len() > 0; let explore = deficit == 0 && has_work && !stalled; + // A request every peer refused is served by none of them, however long it waits. + let addrs: Vec = peers.iter().map(|p| p.addr()).collect(); + let refused = self.state.broker.refused_by_all(&addrs).await; let replace = if deficit == 0 && cooled { - (!stalled) - .then(|| underserving_peer(&peers)) + refused + .then(|| slowest_peer(peers.iter())) .flatten() - .map(|slow| Replace::Slow(slow.addr)) + .map(Replace::Refused) + .or_else(|| { + (!stalled) + .then(|| underserving_peer(&peers)) + .flatten() + .map(|slow| Replace::Slow(slow.addr)) + }) .or_else(|| { (explore && probe_cooled) .then(|| crowded_peer(&peers)) @@ -1302,14 +1317,20 @@ fn rotation_peer(peers: &[ConnectedPeer]) -> Result<(SocketAddr, u64), &'static } /// The slowest peer of the most crowded network group, if any group holds more than one -/// peer. An unmeasured peer counts as slowest, and a higher handshake ping breaks ties. +/// peer. fn crowded_peer(peers: &[ConnectedPeer]) -> Option { let mut groups: HashMap> = HashMap::new(); for peer in peers { groups.entry(NetGroup::of(peer.addr())).or_default().push(peer); } let crowded = groups.into_values().filter(|group| group.len() > 1).max_by_key(Vec::len)?; - crowded.into_iter().min_by_key(|p| (p.serving_rate(), Reverse(lag_key(p)))).map(|p| p.addr()) + slowest_peer(crowded.into_iter()) +} + +/// The slowest of `peers`. An unmeasured peer counts as slowest, and a higher handshake +/// ping breaks ties. +fn slowest_peer<'a>(peers: impl Iterator) -> Option { + peers.min_by_key(|p| (p.serving_rate(), Reverse(lag_key(p)))).map(|p| p.addr()) } fn spawn_peer_supervisor(state: Arc, shutdown: CancellationToken) -> JoinHandle<()> { @@ -1935,6 +1956,8 @@ mod tests { enum Spread { Work, + /// A block every peer sent `notfound` for waits in the queue. + Refused, NoWork, Stalled, Exclusive, @@ -1993,7 +2016,15 @@ mod tests { let admitted = state.connected.admit(peer, &session, state.max_peers, None).await; assert!(matches!(admitted, Admission::Joined)); } - if !matches!(spread, Spread::NoWork) { + if matches!(spread, Spread::Refused) { + state.broker.enqueue(0, NetworkMessage::GetData(vec![block(1)])).await; + let handles: Vec = + state.connected.read().await.iter().map(|p| p.handle()).collect(); + for one in handles.chunks(1) { + assert_eq!(state.broker.route(one, &state.costs).await, 1); + state.broker.not_found(one[0].id(), &[block(1)]).await; + } + } else if !matches!(spread, Spread::NoWork) { state.broker.enqueue(0, get_cfilters(0)).await; } if matches!(spread, Spread::Stalled) { @@ -2075,6 +2106,7 @@ mod tests { #[test_case([120_000, 100_000, 110_000], Spread::Stalled, 1 => Vec::::new(); "not while the link is stalled")] #[test_case([120_000, 100_000, 110_000], Spread::Exclusive, 1 => Vec::::new(); "configured peers are kept")] #[test_case([3_000_000, 2_500_000, 2_800_000], Spread::Work, 1 => Vec::::new(); "a healthy set is not churned")] + #[test_case([3_000_000, 2_500_000, 2_800_000], Spread::Refused, 1 => vec![1]; "a healthy set that refused a request swaps its slowest")] #[tokio::test] async fn a_slow_set_rotates_its_slowest_peer( rates: [u64; 3],