From 707faf6b770887e8a6018dd2d64e9861b382b288 Mon Sep 17 00:00:00 2001 From: xdustinface Date: Sun, 8 Nov 2026 17:48:25 +1000 Subject: [PATCH] fix(dash-spv): spread peers across network groups Most testnet seeds sit in 68.67.122.0/24, so a uniform sample keeps a client inside that block. When the block is congested every peer is slow alike and `underserving_peer` sees nothing to compare. - `PeerDiscoverer::get` orders candidates by `NetGroup`, a /16 for IPv4 and a /32 for IPv6: groups with no connected peer first, one address per group in turn, then the connected groups. It takes the connected and banned addresses, so a banned group cannot shrink the batch. Exclusive peers are not grouped. - While work is queued and the link is not stalled, `repair_round` swaps the slowest peer of the largest group with two or more peers for an address of a group the set lacks, at most one probe per `SWAP_COOLDOWN`. Every newly learned `addr` wakes the supervisor. Configured peers are never swapped for their group. - A node lacking a required service is banned with `Ban::MissingServices` for as long as a misbehaving one, so its group falls through to its next address instead of redialing it every round. Gossiped `addr` and `addrv2` entries are learned only if they advertise every required service. - The supervisor re-checks the set on every `FILL_TICK`. It slept on the timer only while the set was short or crowded, so a full set waited for the next unrelated wake: a peer whose rate became known after the last round was never judged, and a slow peer flagged while `SWAP_COOLDOWN` held the swap back was never re-checked, since the flow controller wakes the supervisor only once per slow peer. A round with nothing to do returns before probing. `a_running_supervisor_swaps_a_slow_peer_once_it_is_measured` drives the real loop and fails with the old gate. - `NetworkState` is built apart from its tasks, so tests can drive the supervisor on a state of their own. --- dash-spv/src/client/config.rs | 6 + dash-spv/src/network/bans.rs | 21 ++ dash-spv/src/network/discovery.rs | 246 +++++++++++++++++--- dash-spv/src/network/manager.rs | 367 ++++++++++++++++++++++++++---- dash-spv/src/network/peer.rs | 1 + 5 files changed, 572 insertions(+), 69 deletions(-) diff --git a/dash-spv/src/client/config.rs b/dash-spv/src/client/config.rs index 90eb7043c..a83914462 100644 --- a/dash-spv/src/client/config.rs +++ b/dash-spv/src/client/config.rs @@ -170,6 +170,12 @@ impl ClientConfig { self } + /// Whether only the configured peers are dialed. Configuring any peer replaces + /// discovery, with or without `restrict_to_configured_peers`. + pub(crate) fn exclusive_peers(&self) -> bool { + !self.peers.is_empty() || self.restrict_to_configured_peers + } + /// Set storage path. pub fn with_storage_path(mut self, path: impl Into) -> Self { self.storage_path = path.into(); diff --git a/dash-spv/src/network/bans.rs b/dash-spv/src/network/bans.rs index 956585378..acd0024dd 100644 --- a/dash-spv/src/network/bans.rs +++ b/dash-spv/src/network/bans.rs @@ -12,6 +12,8 @@ const MISBEHAVIOUR_BAN: Duration = Duration::from_secs(24 * 60 * 60); pub enum Ban { DialFailed, Kicked, + /// Lacks a service the client needs, which does not change within a session. + MissingServices, Misbehaved, } @@ -43,6 +45,7 @@ impl Bans { MAX_DIAL_BACKOFF.min(Duration::from_secs(1 << (*fails).min(6))) } Ban::Kicked => KICK_BAN, + Ban::MissingServices => MISBEHAVIOUR_BAN, Ban::Misbehaved => { tracing::warn!(target: "dash_spv::network", "banning {} for misbehaving", addr); MISBEHAVIOUR_BAN @@ -66,3 +69,21 @@ impl Bans { .collect() } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A node lacking a service the client needs stays banned past the longest dial + /// backoff. + #[tokio::test(start_paused = true)] + async fn missing_services_outlast_the_dial_backoff() { + let bans = Bans::new([]); + let addr: SocketAddr = "46.224.41.173:19999".parse().expect("an address"); + bans.ban(addr, Ban::MissingServices).await; + + tokio::time::advance(MAX_DIAL_BACKOFF * 10).await; + + assert!(bans.banned().await.contains(&addr)); + } +} diff --git a/dash-spv/src/network/discovery.rs b/dash-spv/src/network/discovery.rs index 0ecdfa8a1..26ac476f4 100644 --- a/dash-spv/src/network/discovery.rs +++ b/dash-spv/src/network/discovery.rs @@ -11,13 +11,16 @@ //! //! Results from both sources are merged and deduplicated. Addresses gossiped by //! connected peers are folded in later via [`PeerDiscoverer::learn`]. +//! +//! Most masternodes of a network can sit in one address block, so [`PeerDiscoverer::get`] +//! spreads its picks across [`NetGroup`]s instead of sampling addresses uniformly. -use std::collections::HashMap; -use std::net::SocketAddr; +use std::collections::{HashMap, HashSet}; +use std::net::{IpAddr, SocketAddr}; use std::time::{SystemTime, UNIX_EPOCH}; use dashcore::Network; -use rand::seq::{IndexedRandom, SliceRandom}; +use rand::seq::SliceRandom; use crate::network::peer::DisconnectedPeer; use crate::ClientConfig; @@ -28,11 +31,35 @@ const MAX_ADDR_AGE: u32 = 7 * 24 * 60 * 60; const MAX_ADDR_FUTURE: u32 = 10 * 60; const MAX_PROBED: usize = 32; +/// The address block a peer belongs to, a /16 for IPv4 and a /32 for IPv6. Peers of one +/// block tend to share an operator and an uplink, so they slow down together. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub(crate) enum NetGroup { + V4(u8, u8), + V6(u16, u16), +} + +impl NetGroup { + pub(crate) fn of(addr: SocketAddr) -> Self { + match addr.ip().to_canonical() { + IpAddr::V4(ip) => { + let [a, b, ..] = ip.octets(); + NetGroup::V4(a, b) + } + IpAddr::V6(ip) => { + let [a, b, ..] = ip.segments(); + NetGroup::V6(a, b) + } + } + } +} + pub struct PeerDiscoverer { network: Network, // Empty means "discover" from the compiled-in seeds, then DNS. fixed: Vec, - restrict_to_configured_peers: bool, + /// Only the configured peers are dialed, whatever their network groups. + exclusive: bool, /// Resolved once and kept. Not consumed as handed out: the reconnector comes back /// on every drop, and a drained pool would leave the client nothing to reconnect to. discovered: Option>, @@ -46,7 +73,7 @@ impl PeerDiscoverer { PeerDiscoverer { network: config.network, fixed: config.peers.clone(), - restrict_to_configured_peers: config.restrict_to_configured_peers, + exclusive: config.exclusive_peers(), discovered: None, learned: HashMap::new(), probed: HashMap::new(), @@ -54,7 +81,7 @@ impl PeerDiscoverer { } pub fn learn(&mut self, addresses: impl IntoIterator) -> usize { - if !self.fixed.is_empty() || self.restrict_to_configured_peers { + if self.exclusive { return 0; } let now = SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |d| d.as_secs() as u32); @@ -94,19 +121,22 @@ impl PeerDiscoverer { } } - /// Up to `count` addresses to try: the best probed first, then a random sample. - pub async fn get(&mut self, count: usize) -> Vec { - let mut best: Vec<(SocketAddr, u32)> = self.probed.drain().collect(); - best.sort_by_key(|(_, lag)| *lag); - self.probed.extend(best.split_off(count.min(best.len()))); - let mut out: Vec = - best.into_iter().map(|(addr, _)| DisconnectedPeer::new(addr, self.network)).collect(); - let count = count - out.len(); - - let pool = if !self.fixed.is_empty() { - self.fixed.clone() - } else if self.restrict_to_configured_peers { + /// Up to `count` addresses to dial, none of them `connected` or `banned`. The best + /// probed come first, then a random sample. Unless the peers are exclusive, this order + /// spreads across network groups, see [`spread_across_groups`]. With `new_groups_only` + /// no address from a group of the `connected` peers is returned, and no configured one. + pub async fn get( + &mut self, + count: usize, + connected: &HashSet, + banned: &HashSet, + new_groups_only: bool, + ) -> Vec { + if self.exclusive && new_groups_only { return Vec::new(); + } + let mut pool = if self.exclusive { + self.fixed.clone() } else { if self.discovered.is_none() { let found = Self::discover(self.network).await; @@ -114,16 +144,22 @@ impl PeerDiscoverer { } let mut pool = self.discovered.clone().expect("just set"); pool.extend(self.learned.keys()); - pool.sort(); - pool.dedup(); pool }; - - out.extend( - pool.choose_multiple(&mut rand::rng(), count) - .map(|addr| DisconnectedPeer::new(*addr, self.network)), - ); - out + pool.extend(self.probed.keys()); + pool.sort(); + pool.dedup(); + pool.retain(|addr| !connected.contains(addr) && !banned.contains(addr)); + pool.shuffle(&mut rand::rng()); + pool.sort_by_key(|addr| self.probed.get(addr).copied().unwrap_or(u32::MAX)); + if !self.exclusive { + pool = spread_across_groups(pool, connected, new_groups_only); + } + pool.truncate(count); + for addr in &pool { + self.probed.remove(addr); + } + pool.into_iter().map(|addr| DisconnectedPeer::new(addr, self.network)).collect() } async fn discover(network: Network) -> Vec { @@ -163,9 +199,167 @@ impl PeerDiscoverer { } } +/// Reorders `pool` so groups without a `connected` peer come first, one address per +/// group in turn. Addresses of a group keep their order in `pool`. The groups of the +/// `connected` peers follow, so a network of few groups still fills, unless +/// `new_groups_only` drops them. +fn spread_across_groups( + pool: Vec, + connected: &HashSet, + new_groups_only: bool, +) -> Vec { + let covered: HashSet = connected.iter().map(|addr| NetGroup::of(*addr)).collect(); + let mut turns: HashMap = HashMap::new(); + let mut ranked: Vec<(bool, usize, SocketAddr)> = pool + .into_iter() + .map(|addr| { + let group = NetGroup::of(addr); + let turn = turns.entry(group).or_default(); + *turn += 1; + (covered.contains(&group), *turn, addr) + }) + .filter(|(is_covered, ..)| !(new_groups_only && *is_covered)) + .collect(); + ranked.sort_by_key(|&(is_covered, turn, _)| (is_covered, turn)); + ranked.into_iter().map(|(.., addr)| addr).collect() +} + #[cfg(test)] mod tests { use super::*; + use test_case::test_case; + + fn addr(s: &str) -> SocketAddr { + s.parse().expect("a socket address") + } + + /// A regtest discoverer, which has no seeds, that learned `addresses`. + fn discoverer(addresses: &[&str]) -> PeerDiscoverer { + let mut discoverer = PeerDiscoverer::new(&ClientConfig::regtest()); + let now = SystemTime::now().duration_since(UNIX_EPOCH).expect("after epoch").as_secs(); + discoverer.learn(addresses.iter().map(|a| (addr(a), now as u32))); + discoverer + } + + fn set(addresses: &[&str]) -> HashSet { + addresses.iter().map(|a| addr(a)).collect() + } + + fn set_of(addresses: &[SocketAddr]) -> HashSet { + addresses.iter().copied().collect() + } + + async fn picks( + discoverer: &mut PeerDiscoverer, + count: usize, + connected: &[&str], + new_groups_only: bool, + ) -> Vec { + let connected = set(connected); + let peers = discoverer.get(count, &connected, &HashSet::new(), new_groups_only).await; + peers.iter().map(|p| p.addr()).collect() + } + + #[test_case("68.67.122.1:1", "68.67.200.2:2" => true; "ipv4 in one /16")] + #[test_case("68.67.122.1:1", "68.68.122.1:1" => false; "ipv4 across /16s")] + #[test_case("[2a0e:b781:55d1:f32e::]:1", "[2a0e:b781:1::1]:1" => true; "ipv6 in one /32")] + #[test_case("[2a0e:b781::1]:1", "[2a0e:b782::1]:1" => false; "ipv6 across /32s")] + #[test_case("[::ffff:68.67.122.1]:1", "68.67.0.1:1" => true; "a mapped ipv4 is ipv4")] + #[test_case("127.0.0.1:1", "127.0.0.1:2" => true; "loopback peers share a group")] + fn net_groups(a: &str, b: &str) -> bool { + NetGroup::of(addr(a)) == NetGroup::of(addr(b)) + } + + /// A crowded block must not crowd out the only address of another block. + #[tokio::test] + async fn dials_a_group_not_yet_connected() { + let block: Vec = (1..=50).map(|n| format!("68.67.122.{n}:1")).collect(); + let mut addresses: Vec<&str> = block.iter().map(String::as_str).collect(); + addresses.push("206.245.167.63:1"); + let mut discoverer = discoverer(&addresses); + + let got = picks(&mut discoverer, 1, &["68.67.122.1:1"], false).await; + + assert_eq!(got, [addr("206.245.167.63:1")]); + } + + /// A batch for an empty set takes one address per group before it takes a second one. + #[tokio::test] + async fn a_batch_spreads_over_the_groups() { + let mut discoverer = discoverer(&[ + "68.67.122.1:1", + "68.67.122.2:1", + "68.67.122.3:1", + "206.245.167.63:1", + "85.209.243.1:1", + ]); + + let got = picks(&mut discoverer, 3, &[], false).await; + + let groups: HashSet = got.iter().map(|a| NetGroup::of(*a)).collect(); + assert_eq!(groups.len(), 3, "got {got:?}"); + } + + /// Regtest peers are all loopback: a single group must still fill the set. + #[tokio::test] + async fn falls_back_to_a_connected_group() { + let mut discoverer = discoverer(&["127.0.0.1:1", "127.0.0.1:2", "127.0.0.1:3"]); + + let got = picks(&mut discoverer, 2, &["127.0.0.1:1"], false).await; + + assert_eq!(set_of(&got), set(&["127.0.0.1:2", "127.0.0.1:3"])); + } + + #[tokio::test] + async fn new_groups_only_never_falls_back() { + let mut discoverer = discoverer(&["68.67.122.1:1", "68.67.122.2:1"]); + + assert!(picks(&mut discoverer, 2, &["68.67.122.1:1"], true).await.is_empty()); + } + + /// A probed address waits its group's turn, but goes first within that turn. One not + /// returned stays remembered. + #[tokio::test] + async fn a_probed_address_keeps_its_priority_within_the_groups() { + let mut discoverer = + discoverer(&["68.67.122.2:1", "68.67.122.3:1", "206.245.167.63:1", "206.245.167.64:1"]); + discoverer.probed(addr("68.67.122.2:1"), 10); + discoverer.probed(addr("206.245.167.64:1"), 50); + + let got = picks(&mut discoverer, 1, &["68.67.122.1:1"], false).await; + assert_eq!(got, [addr("206.245.167.64:1")]); + assert!(discoverer.probed.contains_key(&addr("68.67.122.2:1"))); + assert!(!discoverer.probed.contains_key(&addr("206.245.167.64:1"))); + + let got = picks(&mut discoverer, 1, &["206.245.167.63:1"], false).await; + assert_eq!(got, [addr("68.67.122.2:1")]); + } + + #[tokio::test] + async fn skips_connected_and_banned_addresses() { + let mut discoverer = discoverer(&["68.67.122.1:1", "206.245.167.63:1", "85.209.243.1:1"]); + + let got = + discoverer.get(3, &set(&["68.67.122.1:1"]), &set(&["206.245.167.63:1"]), false).await; + + let got: Vec = got.iter().map(|p| p.addr()).collect(); + assert_eq!(got, [addr("85.209.243.1:1")]); + } + + /// Configured peers are dialed whatever their groups, the best probed first, and + /// never offered in place of a connected peer for its group. + #[tokio::test] + async fn exclusive_peers_are_not_grouped() { + let mut config = ClientConfig::regtest(); + config.peers = vec![addr("68.67.122.1:1"), addr("68.67.122.2:1"), addr("206.245.167.63:1")]; + let mut discoverer = PeerDiscoverer::new(&config); + discoverer.probed(addr("68.67.122.2:1"), 10); + + let got = picks(&mut discoverer, 1, &["206.245.167.64:1"], false).await; + assert_eq!(got, [addr("68.67.122.2:1")]); + + assert!(picks(&mut discoverer, 3, &["206.245.167.64:1"], true).await.is_empty()); + } #[tokio::test] async fn test_dns_discovery_testnet_returns_embedded_when_dns_fails() { diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index e04281b14..afdf1780f 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1,4 +1,5 @@ use std::{ + cmp::Reverse, collections::{HashMap, HashSet}, net::SocketAddr, sync::{ @@ -30,7 +31,8 @@ const CONNECT_CHUNK: usize = 16; const BAD_LAG_MULTIPLIER: u32 = 3; const BAD_LAG_FLOOR_MS: u32 = 1000; -/// Shortest time between two slow-peer swaps. +/// Shortest time between two swaps, and between two probes for a network group the +/// set lacks. const SWAP_COOLDOWN: Duration = Duration::from_secs(60); /// How often the supervisor re-checks a below-cap set and probes to keep filling. @@ -78,7 +80,7 @@ use crate::network::TransportPreference; use crate::{ network::{ costs::ResponseCosts, - discovery::PeerDiscoverer, + discovery::{NetGroup, PeerDiscoverer}, peer::{ConnectedPeer, DisconnectedPeer, PeerEvent, PeerHandle, PeerId, INITIAL_CAP_BYTES}, }, ClientConfig, @@ -228,8 +230,10 @@ impl Drop for PeerNetworkManager { } } -impl PeerNetworkManager { - pub async fn new(config: &ClientConfig) -> Self { +impl NetworkState { + /// The state of a network with no task running yet, and the receiver of its + /// peer events. + fn new(config: &ClientConfig) -> (Self, UnboundedReceiver) { let max_peers = config.max_peers.max(1) as usize; // NETWORK is unconditional: the block pipeline asks for full blocks at // arbitrary historical heights (and the gap-limit rescan far more so), which @@ -248,7 +252,7 @@ impl PeerNetworkManager { } let (inbound, inbound_rx) = mpsc::unbounded_channel(); - let state = Arc::new(NetworkState { + let state = NetworkState { connected: PeerSet::new(30), discoverer: Mutex::new(PeerDiscoverer::new(config)), bans: Bans::new(config.peers.iter().copied()), @@ -263,7 +267,15 @@ impl PeerNetworkManager { user_agent: config.user_agent.clone(), transport_preference: config.transport_preference, link_stalled: AtomicBool::new(false), - }); + }; + (state, inbound_rx) + } +} + +impl PeerNetworkManager { + pub async fn new(config: &ClientConfig) -> Self { + let (state, inbound_rx) = NetworkState::new(config); + let state = Arc::new(state); let shutdown = CancellationToken::new(); spawn_pump(state.clone(), inbound_rx, shutdown.clone()); @@ -739,6 +751,8 @@ fn lag_key(peer: &ConnectedPeer) -> u32 { /// bad peer is taken only when the set would otherwise be empty. /// - At cap it waits to be woken by a lost or underserving peer, then probes for a /// replacement. No periodic improvement pass: each probe costs a full handshake. +/// - While two peers share a network group and the client has work, it also probes for +/// a group the set lacks, see [`crowded_peer`]. /// - Probed-but-unused peers are closed and remembered in the discoverer so /// later rounds reconnect the best without probing blindly. struct Supervisor { @@ -748,33 +762,47 @@ struct Supervisor { best_lag_ms: AtomicU32, /// When a slow peer was last swapped out, for [`SWAP_COOLDOWN`]. last_swap: Mutex>, + /// When candidates were last probed for a network group the set lacks. + last_spread: Mutex>, +} + +/// Why a peer of a full set is to be replaced. +#[derive(Clone, Copy)] +enum Replace { + Slow(SocketAddr), + Crowded(SocketAddr), } impl Supervisor { + fn new(state: Arc, shutdown: CancellationToken) -> Self { + Supervisor { + state, + shutdown, + best_lag_ms: AtomicU32::new(u32::MAX), + last_swap: Mutex::new(None), + last_spread: Mutex::new(None), + } + } + async fn run(self) { loop { self.repair_round().await; - let short = self.state.connected.read().await.len() < self.state.max_peers; - + // Also re-checked on a timer: work starting, a peer getting measured or a + // cooldown passing wakes nobody. A round with nothing to do probes nothing. tokio::select! { _ = self.shutdown.cancelled() => break, _ = self.state.peer_wake.notified() => {} - _ = tokio::time::sleep(FILL_TICK), if short => {} + _ = tokio::time::sleep(FILL_TICK) => {} } } } - async fn next_candidates(&self, want: usize) -> Vec { - let mut out = self.state.discoverer.lock().await.get(want).await; + async fn next_candidates(&self, want: usize, new_groups_only: bool) -> Vec { let live: HashSet = self.state.connected.read().await.iter().map(|p| p.addr()).collect(); let banned = self.state.bans.banned().await; - let mut seen = HashSet::new(); - out.retain(|p| { - !live.contains(&p.addr()) && !banned.contains(&p.addr()) && seen.insert(p.addr()) - }); - out + self.state.discoverer.lock().await.get(want, &live, &banned, new_groups_only).await } /// Closes probed-but-unused peers and remembers them for the next round. @@ -787,32 +815,52 @@ impl Supervisor { } /// One pass at whatever the peer set needs: fill a deficit, replace a peer that is - /// underserving the rest, or both. + /// underserving the rest or crowding its network group, or both. async fn repair_round(&self) { - let (deficit, slow) = { + let (deficit, replace) = { let peers = self.state.connected.read().await; let deficit = self.state.max_peers.saturating_sub(peers.len()); // Only look for a laggard once full: below cap every arrival is wanted anyway. // Each swap re-sends the work the old peer held, so at most one per cooldown. let cooled = self.last_swap.lock().await.is_none_or(|t| t.elapsed() >= SWAP_COOLDOWN); + let spread_cooled = + self.last_spread.lock().await.is_none_or(|t| t.elapsed() >= SWAP_COOLDOWN); let stalled = self.state.link_stalled.load(Ordering::Relaxed); - let slow = (deficit == 0 && cooled && !stalled) - .then(|| underserving_peer(&peers)) - .flatten() - .map(|slow| slow.addr); - (deficit, slow) + // Peers of one group slow down together, so with work waiting on them a peer + // from another group is worth a swap even when no peer looks slow. + let has_work = self.state.broker.len() > 0; + let replace = if deficit == 0 && cooled { + (!stalled) + .then(|| underserving_peer(&peers)) + .flatten() + .map(|slow| Replace::Slow(slow.addr)) + .or_else(|| { + (spread_cooled && has_work && !stalled) + .then(|| crowded_peer(&peers)) + .flatten() + .map(Replace::Crowded) + }) + } else { + None + }; + (deficit, replace) }; - if deficit == 0 && slow.is_none() { + if deficit == 0 && replace.is_none() { return; } // Scale the probe to the need: a few candidates to find one replacement, a // full chunk when several slots are open and sync is waiting on them. let want = (deficit.max(1) + DIAL_MARGIN).min(CONNECT_CHUNK); - let batch = self.next_candidates(want).await; + let crowded = matches!(replace, Some(Replace::Crowded(_))); + let batch = self.next_candidates(want, crowded).await; if batch.is_empty() { return; } + if crowded { + *self.last_spread.lock().await = Some(Instant::now()); + } + let slow = replace.map(|(Replace::Slow(addr) | Replace::Crowded(addr))| addr); // Take each peer as soon as ITS handshake lands rather than waiting for the batch, // so one unreachable address doesn't delay sync. Completion order is itself a @@ -867,9 +915,15 @@ impl Supervisor { Admission::Joined => accepted += 1, Admission::Swapped(old) => { replaced = Some(new_addr); + let reason = if crowded { + "crowded" + } else { + "slow" + }; tracing::info!( target: "dash_spv::network", - "peer supervisor: swapped out slow {} for {}", + "peer supervisor: swapped out {} {} for {}", + reason, old.addr(), new_addr, ); @@ -963,16 +1017,19 @@ fn underserving_peer(peers: &[ConnectedPeer]) -> Option { }) } +/// The slowest peer of the most crowded network group, if any group holds more than one +/// peer. An unmeasured peer counts as slowest, and a higher handshake ping breaks ties. +fn crowded_peer(peers: &[ConnectedPeer]) -> Option { + let mut groups: HashMap> = HashMap::new(); + for peer in peers { + groups.entry(NetGroup::of(peer.addr())).or_default().push(peer); + } + let crowded = groups.into_values().filter(|group| group.len() > 1).max_by_key(Vec::len)?; + crowded.into_iter().min_by_key(|p| (p.serving_rate(), Reverse(lag_key(p)))).map(|p| p.addr()) +} + fn spawn_peer_supervisor(state: Arc, shutdown: CancellationToken) -> JoinHandle<()> { - tokio::spawn( - Supervisor { - state, - shutdown, - best_lag_ms: AtomicU32::new(u32::MAX), - last_swap: Mutex::new(None), - } - .run(), - ) + tokio::spawn(Supervisor::new(state, shutdown).run()) } /// What the timeout monitor reads of a connected peer each tick. @@ -1210,6 +1267,7 @@ fn spawn_pump( peer_wake, max_peers, discoverer, + required_services, .. } = &*state; // Per-peer received-message counter to check load balance across peers. @@ -1293,13 +1351,14 @@ fn spawn_pump( } // Addr gossip feeds the discovery pool, otherwise it stays frozen at the seeds and // one DNS lookup. Both spellings: a peer that ignored `sendaddrv2` answers - // `getaddr` with legacy `addr`. + // `getaddr` with legacy `addr`. An address whose services lack what we need + // would only be dropped after its handshake. let gossiped: Option> = match &*msg { NetworkMessage::AddrV2(addrs) if addrs.len() <= MAX_ADDR_PER_MESSAGE => { Some( addrs .iter() - .filter(|a| a.services.has(ServiceFlags::NETWORK)) + .filter(|a| a.services.has(*required_services)) .filter_map(|a| Some((a.socket_addr().ok()?, a.time))) .collect(), ) @@ -1307,7 +1366,7 @@ fn spawn_pump( NetworkMessage::Addr(addrs) if addrs.len() <= MAX_ADDR_PER_MESSAGE => Some( addrs .iter() - .filter(|(_, a)| a.services.has(ServiceFlags::NETWORK)) + .filter(|(_, a)| a.services.has(*required_services)) .filter_map(|(time, a)| Some((a.socket_addr().ok()?, *time))) .collect(), ), @@ -1323,10 +1382,8 @@ fn spawn_pump( learned, addr ); - // Fresh candidates are what a short set is waiting for. - if connected.read().await.len() < *max_peers { - peer_wake.notify_one(); - } + // Fresh candidates are what a short or crowded set is waiting for. + peer_wake.notify_one(); } } @@ -1407,10 +1464,18 @@ mod tests { use super::*; use crate::client::ClientConfig; use crate::test_utils::test_socket_address; + use dashcore::consensus::encode::serialize; + use dashcore::network::address::Address; + use dashcore::network::message::{RawNetworkMessage, RawNetworkMessageCodec}; use dashcore::network::message_blockdata::Inventory; use dashcore::network::message_filter::{CFilter, GetCFilters}; - use dashcore::BlockHash; + use dashcore::network::message_network::VersionMessage; + use dashcore::{BlockHash, Network}; + use std::time::{SystemTime, UNIX_EPOCH}; use test_case::test_case; + use tokio::io::AsyncWriteExt; + use tokio::net::{TcpListener, TcpStream}; + use tokio_util::codec::FramedRead; fn get_cfilters(start: u32) -> NetworkMessage { NetworkMessage::GetCFilters(GetCFilters { @@ -1524,6 +1589,222 @@ mod tests { peers.iter().position(|p| p.addr() == slow.addr) } + /// A regtest node on loopback that completes the handshake and answers pings. + async fn mock_node() -> SocketAddr { + mock_node_serving( + ServiceFlags::NETWORK | ServiceFlags::COMPACT_FILTERS | ServiceFlags::BLOOM, + ) + .await + } + + /// A mock node that advertises `services`. + async fn mock_node_serving(services: ServiceFlags) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind loopback"); + let addr = listener.local_addr().expect("bound address"); + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(serve_handshake(stream, services)); + } + }); + addr + } + + async fn serve_handshake(stream: TcpStream, services: ServiceFlags) { + let (read_half, mut write_half) = stream.into_split(); + let mut reader = FramedRead::new(read_half, RawNetworkMessageCodec); + let local: SocketAddr = "127.0.0.1:0".parse().expect("an address"); + let now = SystemTime::now().duration_since(UNIX_EPOCH).expect("after epoch").as_secs(); + while let Some(Ok(raw)) = reader.next().await { + let replies = match raw.payload { + NetworkMessage::Version(_) => vec![ + NetworkMessage::Version(VersionMessage::new( + services, + now as i64, + Address::new(&local, ServiceFlags::NONE), + Address::new(&local, services), + 1, + "/mock/".to_owned(), + 0, + false, + [0; 32], + )), + NetworkMessage::Verack, + ], + NetworkMessage::Ping(nonce) => vec![NetworkMessage::Pong(nonce)], + _ => vec![], + }; + for payload in replies { + let raw = RawNetworkMessage { + magic: Network::Regtest.magic(), + payload, + }; + if write_half.write_all(&serialize(&raw)).await.is_err() { + return; + } + } + } + } + + enum Spread { + Work, + NoWork, + Stalled, + Exclusive, + } + + /// A full set inside one network group swaps its slowest peer for a peer of a group + /// it lacks. Only while work waits and the link flows, and never for configured peers. + /// No network task runs, so nothing but the supervisor changes the set. + #[test_case(Spread::Work => Some(1); "the slowest of the crowded group goes")] + #[test_case(Spread::NoWork => None; "not at the tip with no work")] + #[test_case(Spread::Stalled => None; "not while the link is stalled")] + #[test_case(Spread::Exclusive => None; "configured peers are kept")] + #[tokio::test] + async fn a_crowded_set_swaps_toward_a_missing_group(spread: Spread) -> Option { + let node = mock_node().await; + let mut config = + ClientConfig::regtest().with_transport_preference(TransportPreference::V1Only); + if matches!(spread, Spread::Exclusive) { + config.peers = vec![node]; + } + let (state, _inbound) = NetworkState::new(&config); + let state = Arc::new(state); + let now = SystemTime::now().duration_since(UNIX_EPOCH).expect("after epoch").as_secs(); + state.discoverer.lock().await.learn([(node, now as u32)]); + let session = CancellationToken::new(); + + let crowded: Vec = + (1..=3).map(|n| SocketAddr::from(([68, 67, 122, n], 19999))).collect(); + for (addr, rate) in crowded.iter().zip([1_000_000, 800_000, 900_000]) { + let peer = ConnectedPeer::for_test(PeerId::for_test(*addr, 0)).await; + peer.set_serving_rate(rate); + let admitted = state.connected.admit(peer, &session, state.max_peers, None).await; + assert!(matches!(admitted, Admission::Joined)); + } + if !matches!(spread, Spread::NoWork) { + state.broker.enqueue(0, get_cfilters(0)).await; + } + if matches!(spread, Spread::Stalled) { + state.link_stalled.store(true, Ordering::Relaxed); + } + + Supervisor::new(state.clone(), session.clone()).repair_round().await; + + let live: Vec = state.connected.read().await.iter().map(|p| p.addr()).collect(); + assert_eq!(live.len(), 3); + let gone = crowded.iter().position(|addr| !live.contains(addr)); + assert_eq!(gone.is_some(), live.contains(&node), "got {live:?}"); + gone + } + + /// The supervisor swaps out a slow peer from its own loop once the rates are known, + /// without anything waking it. Each peer sits in its own network group, so only the + /// timer re-checks the full set. + #[tokio::test] + async fn a_running_supervisor_swaps_a_slow_peer_once_it_is_measured() { + let node = mock_node().await; + let config = ClientConfig::regtest().with_transport_preference(TransportPreference::V1Only); + let (state, _inbound) = NetworkState::new(&config); + let state = Arc::new(state); + let now = SystemTime::now().duration_since(UNIX_EPOCH).expect("after epoch").as_secs(); + state.discoverer.lock().await.learn([(node, now as u32)]); + let session = CancellationToken::new(); + + let peers: Vec = [[68, 67, 122, 1], [206, 245, 167, 63], [85, 209, 243, 1]] + .into_iter() + .map(|ip| SocketAddr::from((ip, 19999))) + .collect(); + for addr in &peers { + let peer = ConnectedPeer::for_test(PeerId::for_test(*addr, 0)).await; + let admitted = state.connected.admit(peer, &session, state.max_peers, None).await; + assert!(matches!(admitted, Admission::Joined)); + } + + let running = tokio::spawn(Supervisor::new(state.clone(), session.clone()).run()); + tokio::time::sleep(Duration::from_millis(200)).await; + for peer in state.connected.read().await.iter() { + let slow = peer.addr() == peers[1]; + peer.set_serving_rate(if slow { + 100_000 + } else { + 1_000_000 + }); + } + let deadline = Instant::now() + FILL_TICK * 2; + while Instant::now() < deadline + && !state.connected.read().await.iter().any(|p| p.addr() == node) + { + tokio::time::sleep(Duration::from_millis(50)).await; + } + session.cancel(); + running.await.expect("the supervisor stops"); + + let live: Vec = state.connected.read().await.iter().map(|p| p.addr()).collect(); + assert!(live.contains(&node) && !live.contains(&peers[1]), "got {live:?}"); + } + + /// Only a group of two or more peers is crowded, and its slowest peer goes, even when + /// a peer alone in its group is slower. + #[test_case(&[[68, 67, 122, 1], [206, 245, 167, 63], [85, 209, 243, 1]] => None; "one peer per group")] + #[test_case(&[[68, 67, 122, 1], [68, 67, 9, 2], [206, 245, 167, 63]] => Some(1); "the slower of a pair")] + #[tokio::test] + async fn crowded_peer_is_the_slowest_of_a_shared_group(ips: &[[u8; 4]]) -> Option { + let mut peers = Vec::new(); + for (ip, rate) in ips.iter().zip([900_000, 800_000, 100_000]) { + let peer = ConnectedPeer::for_test(PeerId::for_test((*ip, 19999).into(), 0)).await; + peer.set_serving_rate(rate); + peers.push(peer); + } + let crowded = crowded_peer(&peers)?; + peers.iter().position(|p| p.addr() == crowded) + } + + /// A node that cannot serve compact filters is banned on the spot, not only backed + /// off like a failed dial, so it does not take a candidate slot round after round. + #[tokio::test] + async fn a_node_lacking_required_services_is_banned() { + let node = mock_node_serving(ServiceFlags::NETWORK | ServiceFlags::BLOOM).await; + let config = ClientConfig::regtest().with_transport_preference(TransportPreference::V1Only); + let (state, _inbound) = NetworkState::new(&config); + let state = Arc::new(state); + + let dialed = DisconnectedPeer::new(node, Network::Regtest) + .connect(state.clone(), CancellationToken::new()) + .await; + + assert!(dialed.is_err()); + assert!(state.bans.banned().await.contains(&node)); + } + + /// Gossiped addresses whose service bits lack what the client needs are not learned. + #[tokio::test] + async fn gossip_without_required_services_is_not_learned() { + let net = PeerNetworkManager::new(&ClientConfig::regtest()).await; + let full = ServiceFlags::NETWORK | ServiceFlags::COMPACT_FILTERS | ServiceFlags::BLOOM; + let serves = SocketAddr::from(([62, 60, 244, 174], 19999)); + let lacks = SocketAddr::from(([46, 224, 41, 173], 19999)); + let now = SystemTime::now().duration_since(UNIX_EPOCH).expect("after epoch").as_secs(); + let addr = NetworkMessage::Addr(vec![ + (now as u32, Address::new(&serves, full)), + (now as u32, Address::new(&lacks, ServiceFlags::NETWORK | ServiceFlags::BLOOM)), + ]); + net.state + .inbound + .send(PeerEvent::Message(conn(1, 0), addr.into())) + .expect("the pump is running"); + tokio::time::sleep(Duration::from_millis(100)).await; + + let known = net + .state + .discoverer + .lock() + .await + .get(10, &HashSet::new(), &HashSet::new(), false) + .await; + let known: Vec = known.iter().map(|p| p.addr()).collect(); + assert_eq!(known, [serves]); + } + #[test] fn headers_carry_the_hashes_computed_while_decompressing_them() { let header = Header::dummy(1); diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index 2ed082179..284db20a8 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -588,6 +588,7 @@ impl DisconnectedPeer { "dropping {}: lacks required services {} (advertises {})", self.addr, required_services, version.services ); + state.bans.ban(self.addr, Ban::MissingServices).await; return Err(NetworkError::ConnectionFailed(format!( "peer {} lacks required services {}", self.addr, required_services