diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index a3c1a41af..695d31d5b 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -12,6 +12,10 @@ use crate::network::costs::ResponseCosts; use crate::network::peer::{PeerHandle, PeerId}; const KICKING_ATTEMPTS: u32 = 2; +/// How many timeouts a request lives on bytes of a message still arriving, without an +/// answer. A big answer on a slow link must fit, but a message that never finishes must +/// not keep its request alive forever. +const ANSWER_TIMEOUTS: u32 = 6; const HEDGE_AFTER: Duration = Duration::from_secs(2); const MIN_IN_FLIGHT_PER_CLASS: usize = 4; const MSG_CLASSES: usize = 5; @@ -25,8 +29,8 @@ pub enum RequestKey { Headers(BlockHash), /// Stop hash. CfHeaders(BlockHash), - /// Start height. - CFilters(u32), + /// Stop hash. + CFilters(BlockHash), MnListDiff(BlockHash), Block(BlockHash), } @@ -37,7 +41,7 @@ pub fn request_keys(msg: &NetworkMessage) -> Vec { m.locator_hashes.first().map(|h| RequestKey::Headers(*h)).into_iter().collect() } NetworkMessage::GetCFHeaders(m) => vec![RequestKey::CfHeaders(m.stop_hash)], - NetworkMessage::GetCFilters(m) => vec![RequestKey::CFilters(m.start_height)], + NetworkMessage::GetCFilters(m) => vec![RequestKey::CFilters(m.stop_hash)], NetworkMessage::GetMnListD(m) => vec![RequestKey::MnListDiff(m.block_hash)], NetworkMessage::GetData(inv) => inv .iter() @@ -54,7 +58,7 @@ enum ReqState { /// Attempts so far. Queued(u32), OnWire(Box), - /// The block was received and waits for its manager to release the key. + /// The whole reply was received and waits for its manager to release the key. Arrived, } @@ -62,8 +66,12 @@ struct OnWire { /// The connection, not the address: a late close of an old connection must not /// pull back the work of a new one to the same address. peer: PeerId, - /// Refreshed by every piece of a streaming response. + /// Refreshed by every answer and by bytes of a message still arriving, since a big + /// answer can take longer than a timeout to arrive. last_progress: Instant, + /// Refreshed only by answers, so a block still arriving slowly can be hedged and a + /// message that never finishes cannot keep the request alive forever. + last_answer: Instant, msg: NetworkMessage, height: u32, hedge: Option, @@ -166,7 +174,16 @@ impl Broker { n } + /// Re-queues the requests of the `gone` peers. A request they only hedged stays with its + /// peer and may be hedged again, since the copy is lost. pub async fn requeue_from(&self, gone: &HashSet) -> usize { + for state in self.requests.lock().await.values_mut() { + if let ReqState::OnWire(o) = state { + if o.hedge.is_some_and(|hedge| gone.contains(&hedge)) { + o.hedge = None; + } + } + } self.requeue(|_, o| gone.contains(&o.peer)).await } @@ -183,12 +200,12 @@ impl Broker { } } - /// Settled on arrival, not when processed: blocks wait in height order, and that - /// wait is not the peer's. The first copy of a hedged block settles it. A request - /// put back in the queue is settled by a copy from any peer, and its message dropped. - /// The key stays registered until released, so a re-declare before then sends nothing. - pub async fn block_arrived(&self, peer: PeerId, hash: BlockHash) { - let key = RequestKey::Block(hash); + /// Settles `key` on arrival of its whole reply from `peer`, not when processed, since + /// the wait for a manager is not the peer's. The first copy of a hedged request settles + /// it. A request put back in the queue is settled by a copy from any peer, and its message + /// dropped. The key stays registered until released, so a re-declare before then sends + /// nothing, and a manager that cannot use the reply must release the key. + pub async fn arrived(&self, peer: PeerId, key: RequestKey) { let mut reqs = self.requests.lock().await; match reqs.get(&key) { Some(ReqState::OnWire(o)) if o.peer == peer || o.hedge == Some(peer) => {} @@ -205,31 +222,55 @@ impl Broker { /// Peers answer in order, so any answer is progress on all of its requests. pub async fn progress(&self, peer: PeerId) { + self.refresh(&[peer], true).await; + } + + /// Bytes of a message still arriving from any of `peers` are progress on their requests. + pub async fn receiving(&self, peers: &[PeerId]) { + if !peers.is_empty() { + self.refresh(peers, false).await; + } + } + + async fn refresh(&self, peers: &[PeerId], answered: bool) { let now = Instant::now(); for state in self.requests.lock().await.values_mut() { if let ReqState::OnWire(o) = state { - if o.peer == peer { + if peers.contains(&o.peer) { o.last_progress = now; + if answered { + o.last_answer = now; + } } } } } - /// Returns the peers to kick for a stalled request. A request that already - /// stalled [`KICKING_ATTEMPTS`] times is retried without blaming its peer. - pub async fn expire(&self, timeout: impl Fn(&PeerId) -> Duration) -> HashSet { + /// Returns the peers to kick for a stalled request, each with the earliest instant + /// its stalled requests are blamed from. A request stalls after a timeout without + /// progress, or after [`ANSWER_TIMEOUTS`] timeouts without an answer. A request that + /// already stalled [`KICKING_ATTEMPTS`] times is retried without blaming its peer. + pub async fn expire(&self, timeout: impl Fn(&PeerId) -> Duration) -> HashMap { let now = Instant::now(); - let mut culprits = HashSet::new(); + let mut culprits: HashMap = HashMap::new(); let mut unanswerable = HashSet::new(); for (key, state) in self.requests.lock().await.iter() { let ReqState::OnWire(o) = state else { continue; }; - if now.duration_since(o.last_progress) <= timeout(&o.peer) { + let timeout = timeout(&o.peer); + let silent_since = if now.duration_since(o.last_progress) > timeout { + o.last_progress + } else if now.duration_since(o.last_answer) > timeout * ANSWER_TIMEOUTS { + // Blamed from the answer, not the recent bytes, so those bytes show our + // link works and the peer is not held as a link stall. + o.last_answer + } else { continue; - } + }; if o.attempts < KICKING_ATTEMPTS { - culprits.insert(o.peer); + let since = culprits.entry(o.peer).or_insert(silent_since); + *since = (*since).min(silent_since); } else { tracing::debug!( target: "dash_spv::network", @@ -282,7 +323,8 @@ impl Broker { let mut on_wire: Vec<(u32, NetworkMessage, PeerHandle)> = Vec::new(); let mut msgs = msgs.into_iter(); for (height, msg) in msgs.by_ref() { - let open = peers.iter().filter(|p| p.in_flight_bytes() < p.cap()); + // `peers` can be a snapshot taken before the timeout monitor closed one of them. + let open = peers.iter().filter(|p| !p.is_closed() && p.in_flight_bytes() < p.cap()); let Some(peer) = choose_peer(open, classify(&msg)) else { unsent.push((height, msg)); break; @@ -306,7 +348,7 @@ impl Broker { let attempts = match slot { ReqState::Queued(attempts) => *attempts, ReqState::OnWire(o) => o.attempts, - // The block of a requeued request can arrive while it is resent. + // The reply to a requeued request can arrive while it is resent. ReqState::Arrived => continue, }; *slot = ReqState::OnWire(Box::new(OnWire { @@ -314,6 +356,7 @@ impl Broker { msg: msg.clone(), height, last_progress: Instant::now(), + last_answer: Instant::now(), hedge: None, attempts: attempts + 1, })); @@ -334,30 +377,55 @@ impl Broker { sent } - /// Endgame: asks a second peer for the lowest block on the wire once its peer delivered + /// Endgame: asks a second peer for the lowest block on the wire once its peer answered /// nothing for [`HEDGE_AFTER`], and returns its height. Blocks are processed in height /// order, so only the lowest one holds up progress and a second copy of any other is wasted. pub async fn hedge(&self, peers: &[PeerHandle]) -> Option { let (key, msg, height, peer) = { let reqs = self.requests.lock().await; - let (key, o) = reqs - .iter() - .filter_map(|(key, state)| match state { - ReqState::OnWire(o) if matches!(key, RequestKey::Block(_)) => { - Some((key, o.as_ref())) - } - _ => None, - }) - .min_by_key(|(_, o)| o.height)?; - if o.hedge.is_some() || o.last_progress.elapsed() < HEDGE_AFTER { + let (key, o) = lowest_on_wire(&reqs, |key| matches!(key, RequestKey::Block(_)))?; + if o.hedge.is_some() || o.last_answer.elapsed() < HEDGE_AFTER { return None; } let peer = peers .iter() - .filter(|p| p.id() != o.peer && p.in_flight_bytes() < p.cap()) + .filter(|p| !p.is_closed() && p.id() != o.peer && p.in_flight_bytes() < p.cap()) .max_by_key(|p| p.cap().saturating_sub(p.in_flight_bytes()))?; (key.clone(), o.msg.clone(), o.height, peer.clone()) }; + self.send_hedge(key, msg, height, peer).await + } + + /// Asks a second peer for the lowest filter batch not yet delivered once its peer has + /// answered nothing for [`HEDGE_AFTER`], and returns its height. Filters are stored in + /// height order, so that batch holds back everything above it however much is downloaded. + /// One queued lower goes out next, so it is no reason to hedge the batch above it. + pub async fn hedge_filters(&self, peers: &[PeerHandle]) -> Option { + let (key, msg, height, peer) = { + let reqs = self.requests.lock().await; + let (key, o) = lowest_on_wire(&reqs, |key| matches!(key, RequestKey::CFilters(_)))?; + if o.hedge.is_some() + || o.last_answer.elapsed() < HEDGE_AFTER + || self.queue.lowest_height(MsgClass::CFilters).await.is_some_and(|h| h < o.height) + { + return None; + } + // Bulk download keeps every peer at its cap, so a cap would rule out all of them. + // Like a header request, the second copy goes where it is answered soonest. + let others = peers.iter().filter(|p| p.id() != o.peer && !p.is_closed()); + let peer = choose_peer(others, MsgClass::Headers)?; + (key.clone(), o.msg.clone(), o.height, peer.clone()) + }; + self.send_hedge(key, msg, height, peer).await + } + + async fn send_hedge( + &self, + key: RequestKey, + msg: NetworkMessage, + height: u32, + peer: PeerHandle, + ) -> Option { peer.send(&msg).await.ok()?; match self.requests.lock().await.get_mut(&key) { Some(ReqState::OnWire(o)) if o.hedge.is_none() => { @@ -375,6 +443,7 @@ impl Broker { let on_wire = OnWire { peer, last_progress: Instant::now(), + last_answer: Instant::now(), msg: msg.clone(), height: 0, hedge: None, @@ -384,6 +453,14 @@ impl Broker { } } + #[cfg(test)] + pub async fn carrier_of(&self, key: &RequestKey) -> Option { + match self.requests.lock().await.get(key) { + Some(ReqState::OnWire(o)) => Some(o.peer), + _ => None, + } + } + #[cfg(test)] pub async fn state_of(&self, key: &RequestKey) -> Option<&'static str> { self.requests.lock().await.get(key).map(|s| match s { @@ -394,6 +471,19 @@ impl Broker { } } +/// Returns the on-wire request of the lowest height among the keys `wanted` selects. +fn lowest_on_wire( + reqs: &HashMap, + wanted: impl Fn(&RequestKey) -> bool, +) -> Option<(&RequestKey, &OnWire)> { + reqs.iter() + .filter_map(|(key, state)| match state { + ReqState::OnWire(o) if wanted(key) => Some((key, o.as_ref())), + _ => None, + }) + .min_by_key(|(_, o)| o.height) +} + #[derive(Clone, Copy, PartialEq, Eq, Debug)] enum MsgClass { Other, @@ -493,6 +583,10 @@ impl MsgQueue { } } + async fn lowest_height(&self, class: MsgClass) -> Option { + self.queue(class).lock().await.first_key_value().map(|(&(height, _), _)| height) + } + async fn pop_n(&self, n: usize) -> Vec<(u32, NetworkMessage)> { self.pop_guaranteed(n as u64, &[0; MSG_CLASSES], |_| 1).await } @@ -587,7 +681,7 @@ mod tests { NetworkMessage::GetCFilters(GetCFilters { filter_type: 0, start_height: start, - stop_hash: BlockHash::dummy(9), + stop_hash: BlockHash::dummy(start), }) } @@ -630,7 +724,7 @@ mod tests { #[test_case(get_headers(1) => vec![RequestKey::Headers(BlockHash::dummy(1))]; "getheaders")] #[test_case(get_cfheaders(2) => vec![RequestKey::CfHeaders(BlockHash::dummy(2))]; "getcfheaders")] - #[test_case(get_cfilters(100) => vec![RequestKey::CFilters(100)]; "getcfilters")] + #[test_case(get_cfilters(100) => vec![RequestKey::CFilters(BlockHash::dummy(100))]; "getcfilters")] #[test_case(get_mnlistdiff(3) => vec![RequestKey::MnListDiff(BlockHash::dummy(4))]; "getmnlistd")] #[test_case(NetworkMessage::GetData(vec![block(1), tx(7), block(2)]) => vec![RequestKey::Block(BlockHash::dummy(1)), RequestKey::Block(BlockHash::dummy(2))]; @@ -664,10 +758,29 @@ mod tests { } else { "on_wire" }; - assert_eq!(broker.state_of(&RequestKey::CFilters(height)).await, Some(want)); + assert_eq!( + broker.state_of(&RequestKey::CFilters(BlockHash::dummy(height))).await, + Some(want) + ); } } + #[tokio::test] + async fn requeued_work_skips_a_closed_peer_still_listed() { + let broker = Broker::new(); + let gone = ConnectedPeer::for_test(conn(1, 0)).await; + let other = ConnectedPeer::for_test(conn(2, 0)).await; + broker.mark_on_wire(&get_cfilters(0), gone.id()).await; + gone.close(); + broker.requeue_from(&HashSet::from([gone.id()])).await; + + // Only the charge matters here: the loopback has no reader to answer it. + broker.route(&[gone.handle(), other.handle()], u64::MAX, &ResponseCosts::new()).await; + + assert_eq!(gone.in_flight_bytes(), 0); + assert!(other.in_flight_bytes() > 0); + } + #[tokio::test] async fn a_requeued_request_dedups_until_its_response_arrives() { let broker = Broker::new(); @@ -675,26 +788,34 @@ mod tests { broker.mark_on_wire(&get_cfilters(0), peer).await; broker.requeue_from(&HashSet::from([peer])).await; broker.queue.pop_n(1).await; - assert_eq!(broker.state_of(&RequestKey::CFilters(0)).await, Some("queued")); + assert_eq!( + broker.state_of(&RequestKey::CFilters(BlockHash::dummy(0))).await, + Some("queued") + ); broker.enqueue(0, get_cfilters(0)).await; assert_eq!(broker.len(), 0); - broker.release(&RequestKey::CFilters(0)).await; - assert_eq!(broker.state_of(&RequestKey::CFilters(0)).await, None); + broker.release(&RequestKey::CFilters(BlockHash::dummy(0))).await; + assert_eq!(broker.state_of(&RequestKey::CFilters(BlockHash::dummy(0))).await, None); broker.enqueue(0, get_cfilters(0)).await; assert_eq!(broker.len(), 1); } /// A block at a slow peer gets a second copy only once it is the lowest on the wire and - /// its peer stopped delivering. + /// its peer stopped delivering. Peer 3 has the most room but was kicked after the + /// snapshot of `peers` was taken, so it never gets the copy. #[tokio::test(start_paused = true)] async fn hedge_asks_another_peer_only_for_the_lowest_stalled_block() { let broker = Broker::new(); let costs = ResponseCosts::new(); + let kicked = ConnectedPeer::for_test(conn(3, 0)).await; + kicked.set_cap(u64::MAX); + kicked.close(); let peers = [ ConnectedPeer::for_test(conn(1, 0)).await.handle(), ConnectedPeer::for_test(conn(2, 0)).await.handle(), + kicked.handle(), ]; let (slow, idle) = (&peers[..1], peers[1].id()); let get_block = |n: u32| NetworkMessage::GetData(vec![block(n)]); @@ -727,33 +848,35 @@ mod tests { tokio::time::advance(HEDGE_AFTER / 2).await; assert_eq!(broker.hedge(&peers).await, None, "the slow peer is still delivering"); + broker.receiving(&[slow[0].id()]).await; tokio::time::advance(HEDGE_AFTER / 2).await; - assert_eq!(broker.hedge(&peers).await, Some(10)); + assert_eq!(broker.hedge(&peers).await, Some(10), "a block still arriving is hedged"); assert_eq!(broker.hedge(&peers).await, None, "later blocks wait for the lowest"); assert_eq!(hedged_blocks().await, [10]); - broker.block_arrived(idle, BlockHash::dummy(10)).await; + broker.arrived(idle, RequestKey::Block(BlockHash::dummy(10))).await; assert_eq!(broker.hedge(&peers).await, Some(11)); assert_eq!(hedged_blocks().await, [11]); assert_eq!(broker.hedge(slow).await, None, "no other peer to ask"); } + #[test_case(NetworkMessage::GetData(vec![block(1)]), RequestKey::Block(BlockHash::dummy(1)); "a block")] + #[test_case(get_cfilters(1), RequestKey::CFilters(BlockHash::dummy(1)); "a filter batch")] + #[test_case(get_cfheaders(1), RequestKey::CfHeaders(BlockHash::dummy(1)); "a filter header batch")] #[tokio::test] - async fn an_arrived_block_dedups_until_released() { + async fn an_arrived_reply_dedups_until_released(request: NetworkMessage, key: RequestKey) { let broker = Broker::new(); let id = conn(1, 0); - let getdata = NetworkMessage::GetData(vec![block(1)]); - let key = RequestKey::Block(BlockHash::dummy(1)); + let peer = ConnectedPeer::for_test(id).await; - broker.enqueue(0, getdata.clone()).await; - assert_eq!(broker.queue.pop_n(1).await.len(), 1); - broker.mark_on_wire(&getdata, id).await; - broker.block_arrived(conn(2, 0), BlockHash::dummy(1)).await; + broker.enqueue(0, request.clone()).await; + assert_eq!(broker.route(&[peer.handle()], u64::MAX, &ResponseCosts::new()).await, 1); + broker.arrived(conn(2, 0), key.clone()).await; assert_eq!(broker.state_of(&key).await, Some("on_wire"), "only its carrier settles it"); - broker.block_arrived(id, BlockHash::dummy(1)).await; + broker.arrived(id, key.clone()).await; assert_eq!(broker.state_of(&key).await, Some("arrived")); - broker.enqueue(0, getdata.clone()).await; + broker.enqueue(0, request.clone()).await; assert_eq!(broker.len(), 0, "a re-declare before the release must not fetch it again"); assert_eq!(broker.requeue_from(&HashSet::from([id])).await, 0); assert!(broker.expire(|_| Duration::ZERO).await.is_empty()); @@ -761,29 +884,110 @@ mod tests { broker.release(&key).await; assert_eq!(broker.state_of(&key).await, None); - broker.enqueue(0, getdata).await; + broker.enqueue(0, request).await; assert_eq!(broker.len(), 1); } + #[test_case(NetworkMessage::GetData(vec![block(1)]), RequestKey::Block(BlockHash::dummy(1)); "a block")] + #[test_case(get_cfilters(1), RequestKey::CFilters(BlockHash::dummy(1)); "a filter batch")] #[tokio::test] - async fn a_requeued_block_is_settled_by_a_copy_from_any_peer() { + async fn a_requeued_request_is_settled_by_a_copy_from_any_peer( + request: NetworkMessage, + key: RequestKey, + ) { let broker = Broker::new(); let carrier = conn(1, 0); - let getdata = NetworkMessage::GetData(vec![block(1)]); - let key = RequestKey::Block(BlockHash::dummy(1)); - broker.mark_on_wire(&getdata, carrier).await; + broker.mark_on_wire(&request, carrier).await; assert_eq!(broker.requeue_from(&HashSet::from([carrier])).await, 1); assert_eq!(broker.state_of(&key).await, Some("queued")); - broker.block_arrived(conn(2, 0), BlockHash::dummy(1)).await; + broker.arrived(conn(2, 0), key.clone()).await; assert_eq!(broker.state_of(&key).await, Some("arrived")); - assert_eq!(broker.len(), 0, "the queued getdata must not go out after the block arrived"); + assert_eq!(broker.len(), 0, "the queued request must not go out after its reply arrived"); - broker.enqueue(0, getdata).await; + broker.enqueue(0, request).await; assert_eq!(broker.len(), 0); } + /// Peers at their caps must not stop it: the batch blocks the store, and every bulk + /// download keeps them there. Peer 2 is at its cap with the batch that completes first. + #[tokio::test(start_paused = true)] + async fn hedge_filters_asks_another_peer_once_for_the_lowest_missing_batch() { + let broker = Broker::new(); + let costs = ResponseCosts::new(); + let slow = ConnectedPeer::for_test(conn(1, 0)).await; + let idle = ConnectedPeer::for_test(conn(2, 0)).await; + let peers = [slow.handle(), idle.handle()]; + let lowest = RequestKey::CFilters(BlockHash::dummy(1000)); + + slow.set_cap(1 << 30); + for start in [1000, 2000] { + broker.enqueue(start, get_cfilters(start)).await; + } + assert_eq!(broker.route(&peers[..1], u64::MAX, &costs).await, 2); + broker.enqueue(3000, get_cfilters(3000)).await; + assert_eq!(broker.route(&peers[1..], u64::MAX, &costs).await, 1); + broker.arrived(idle.id(), RequestKey::CFilters(BlockHash::dummy(3000))).await; + idle.set_cap(idle.in_flight_bytes().max(1)); + + assert_eq!(broker.hedge_filters(&peers).await, None, "it has not waited long enough"); + tokio::time::advance(HEDGE_AFTER).await; + + broker.enqueue(500, get_cfilters(500)).await; + assert_eq!(broker.hedge_filters(&peers).await, None, "the lowest one is queued"); + broker.queue.pop_n(1).await; + broker.release(&RequestKey::CFilters(BlockHash::dummy(500))).await; + + assert_eq!(broker.hedge_filters(&peers).await, Some(1000)); + assert_eq!(broker.hedge_filters(&peers).await, None, "a batch is hedged once"); + assert_eq!(broker.hedge_filters(&peers[..1]).await, None); + + broker.arrived(idle.id(), lowest.clone()).await; + assert_eq!(broker.state_of(&lowest).await, Some("arrived"), "the first copy settles it"); + assert_eq!(broker.hedge_filters(&peers).await, Some(2000)); + assert_eq!(broker.hedge_filters(&peers[..1]).await, None, "no other peer to ask"); + } + + #[tokio::test(start_paused = true)] + async fn a_hedge_lost_with_its_peer_is_sent_again() { + let broker = Broker::new(); + let holder = ConnectedPeer::for_test(conn(1, 0)).await.handle(); + let first = ConnectedPeer::for_test(conn(2, 0)).await.handle(); + let second = ConnectedPeer::for_test(conn(3, 0)).await.handle(); + broker.mark_on_wire(&get_cfilters(0), holder.id()).await; + tokio::time::advance(HEDGE_AFTER).await; + + assert_eq!(broker.hedge_filters(&[holder.clone(), first.clone()]).await, Some(0)); + assert_eq!(broker.hedge_filters(&[holder.clone(), second.clone()]).await, None); + + assert_eq!(broker.requeue_from(&HashSet::from([first.id()])).await, 0); + let key = RequestKey::CFilters(BlockHash::dummy(0)); + assert_eq!(broker.carrier_of(&key).await, Some(holder.id()), "the request stays put"); + assert_eq!(broker.hedge_filters(&[holder, second]).await, Some(0)); + } + + #[test_case(false, Some(0); "a silent peer")] + #[test_case(true, None; "a peer still delivering")] + #[tokio::test(start_paused = true)] + async fn hedge_filters_asks_another_peer_only_for_a_batch_stuck_at_a_silent_one( + delivering: bool, + hedged: Option, + ) { + let broker = Broker::new(); + let slow = ConnectedPeer::for_test(conn(1, 0)).await; + let idle = ConnectedPeer::for_test(conn(2, 0)).await; + broker.mark_on_wire(&get_cfilters(0), conn(1, 0)).await; + + tokio::time::advance(HEDGE_AFTER / 2).await; + if delivering { + broker.progress(conn(1, 0)).await; + } + tokio::time::advance(HEDGE_AFTER / 2).await; + + assert_eq!(broker.hedge_filters(&[slow.handle(), idle.handle()]).await, hedged); + } + #[test_case(get_mnlistdiff(3), vec![RequestKey::MnListDiff(BlockHash::dummy(4))], 0; "its only key")] #[test_case(NetworkMessage::GetData(vec![block(1), block(2)]), vec![RequestKey::Block(BlockHash::dummy(1))], 1; "one of its keys")] #[test_case(NetworkMessage::GetData(vec![block(1), block(2)]), vec![RequestKey::Block(BlockHash::dummy(1)), RequestKey::Block(BlockHash::dummy(2))], 0; "all of its keys")] diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index e96e8eba1..baec0ffef 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -2,7 +2,7 @@ use std::{ collections::{HashMap, HashSet, VecDeque}, net::SocketAddr, sync::{ - atomic::{AtomicU32, AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, Arc, }, time::Duration, @@ -64,6 +64,11 @@ const MAX_ADDR_PER_MESSAGE: usize = 1000; /// How often the timeout monitor scans the outstanding-request registry. const TIMEOUT_CHECK: Duration = Duration::from_secs(1); +/// How many request timeouts a silent peer is held while no peer proves our link +/// works, counted from its silence or from our link's return if later. With the 10 s +/// base timeout that is 60 s, enough for the 40 s outages seen on testnet. +const LINK_STALL_TIMEOUTS: u32 = 6; + use crate::error::{NetworkError, NetworkResult}; use crate::network::bans::{Ban, Bans}; use crate::network::broker::{Broker, RequestKey}; @@ -208,6 +213,8 @@ pub struct NetworkState { pub required_services: ServiceFlags, pub user_agent: String, pub transport_preference: TransportPreference, + /// Whether the timeout monitor sees our own link stalled. + link_stalled: AtomicBool, } pub struct PeerNetworkManager { @@ -257,6 +264,7 @@ impl PeerNetworkManager { required_services, user_agent: config.user_agent.clone(), transport_preference: config.transport_preference, + link_stalled: AtomicBool::new(false), }); let shutdown = CancellationToken::new(); @@ -610,19 +618,27 @@ impl PeerFlow { } } - /// One tick with the peer's cumulative bytes received, its bytes in flight now and - /// whether the client's own cap holds back work. Returns its new cap. - fn tick(&mut self, bytes: u64, in_flight: u64, link_full: bool) -> u64 { + /// One tick with the peer's cumulative bytes received, its bytes in flight now, + /// whether the client's own cap holds back work and whether our link is stalled. + /// Returns its new cap. + fn tick(&mut self, bytes: u64, in_flight: u64, link_full: bool, link_stalled: bool) -> u64 { // The router sends a peer work until it reaches its cap or the link cap binds. // Short of both, its rate shows how much it was asked for, not its bandwidth. let bound = in_flight > 0 && (in_flight >= self.limiter.cap as u64 || link_full); let cap = self.limiter.tick(bytes, in_flight, CAP_STEP_BYTES_PER_SEC); - if bound { + self.sample(self.limiter.rate, bound, link_stalled); + cap + } + + /// Folds `rate` into the serving rate if the peer had work. Ticks are skipped once the + /// timeout monitor sees our own link stalled, since such a stall says nothing about + /// the peer and would drag its average for 30 s. + fn sample(&mut self, rate: f64, has_work: bool, link_stalled: bool) { + if has_work && !link_stalled { self.serving_ticks += 1; let alpha = 1.0 / self.serving_ticks.min(SERVING_SAMPLE_TICKS) as f64; - self.serving_rate += alpha * (self.limiter.rate - self.serving_rate); + self.serving_rate += alpha * (rate - self.serving_rate); } - cap } /// Returns the peer's serving rate in bytes/s, 0 until measured. @@ -648,6 +664,7 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) peer_wake, broker, costs, + link_stalled, .. } = &*state; let mut global = ByteLimiter::new(MIN_CAP_BYTES, bytes.load(Ordering::Relaxed)); @@ -676,9 +693,10 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) let in_flights: Vec = g.iter().map(ConnectedPeer::in_flight_bytes).collect(); let total_in_flight: u64 = in_flights.iter().sum(); let link_full = total_in_flight >= global_cap.load(Ordering::Relaxed); + let stalled = link_stalled.load(Ordering::Relaxed); for (p, &in_flight) in g.iter().zip(&in_flights) { let flow = peers.entry(p.id()).or_insert_with(|| PeerFlow::new(p.bytes_read())); - p.set_cap(flow.tick(p.bytes_read(), in_flight, link_full)); + p.set_cap(flow.tick(p.bytes_read(), in_flight, link_full, stalled)); p.set_serving_rate(flow.serving_rate()); } @@ -742,8 +760,9 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) } // Only the transition wakes the supervisor, so a peer with no replacement - // is not re-probed every tick. - let underserving = underserving_peer(&g); + // is not re-probed every tick. A stall of our own link drags every rate down + // unevenly, so nobody is underserving during one. + let underserving = (!stalled).then(|| underserving_peer(&g)).flatten(); let handles: Vec = g.iter().map(|p| p.handle()).collect(); drop(g); if broker.len() == 0 { @@ -751,6 +770,9 @@ fn spawn_flow_controller(state: Arc, shutdown: CancellationToken) tracing::debug!(target: "dash_spv::network", "endgame: hedged block {}", height); } } + if let Some(height) = broker.hedge_filters(&handles).await { + tracing::debug!(target: "dash_spv::network", "hedged filter batch {}", height); + } if let Some(slow) = underserving.filter(|slow| Some(slow.addr) != last_underserving) { tracing::debug!( target: "dash_spv::network", @@ -840,7 +862,8 @@ impl Supervisor { // Only look for a laggard once full: below cap every arrival is wanted anyway. // Each swap re-sends the work the old peer held, so at most one per cooldown. let cooled = self.last_swap.lock().await.is_none_or(|t| t.elapsed() >= SWAP_COOLDOWN); - let slow = (deficit == 0 && cooled) + let stalled = self.state.link_stalled.load(Ordering::Relaxed); + let slow = (deficit == 0 && cooled && !stalled) .then(|| underserving_peer(&peers)) .flatten() .map(|slow| slow.addr); @@ -1019,17 +1042,146 @@ fn spawn_peer_supervisor(state: Arc, shutdown: CancellationToken) ) } -/// Time requests out and evict the peers that stalled on them. -/// -/// A peer is a culprit when it has in-flight work and no bytes have arrived from it -/// for a full [`REQUEST_TIMEOUT`] since it was given that work, or when one of its -/// requests made no progress for that long. Liveness is per peer, not per request -/// age: a healthy peer draining a deep backlog keeps sending bytes even while its -/// oldest request waits. -/// -/// A culprit is kicked immediately and ALL its on-wire requests are re-queued for -/// other peers; leaving it connected would just route the freed work back to it. -/// The supervisor refills the set. +/// What the timeout monitor reads of a connected peer each tick. +struct Observed { + id: PeerId, + bytes: u64, + /// Whether part of a message has arrived and the rest has not. + partial: bool, + in_flight: u64, + timeout: Duration, +} + +/// What the timeout monitor knows of a connected peer. +struct Liveness { + bytes: u64, + /// When the peer last sent bytes or was idle. + owing_since: Instant, +} + +/// Decides which peers [`spawn_timeout_monitor`] kicks. A peer silent with work is held +/// while no peer proves our link works, but for at most [`LINK_STALL_TIMEOUTS`] +/// timeouts, and silent peers are kicked one per timeout, the one silent the longest first. +#[derive(Default)] +struct StallJudge { + liveness: HashMap, + /// When some peer's bytes last grew. A new peer's first counter is not growth: its + /// handshake may predate a stall of our link, so it proves nothing about the link now. + last_heard: Option, + /// The last kick of a silent peer, which paces the next. + last_kick: Option, + /// Whether our own link may be stalled: the last judgement held a silent peer because + /// no peer proved our link works. + link_stalled: bool, + /// Whether some held peer has had no witness since its silence began, so the next + /// bytes from any peer show our link is back. + link_down: bool, + /// When some peer first received after our link went down. + link_back: Option, +} + +impl StallJudge { + /// Records the peers' byte counters and returns the peers that received bytes of a + /// message still arriving. Small unrelated messages arrive whole, so only these bytes + /// are progress on a request. + fn observe(&mut self, now: Instant, peers: &[Observed]) -> Vec { + self.liveness.retain(|id, _| peers.iter().any(|p| p.id == *id)); + let mut streaming = Vec::new(); + for p in peers { + let entry = self.liveness.entry(p.id).or_insert(Liveness { + bytes: p.bytes, + owing_since: now, + }); + let grew = p.bytes > entry.bytes; + if grew { + if self.link_down { + self.link_back = Some(now); + } + entry.bytes = p.bytes; + self.last_heard = Some(now); + if p.partial { + streaming.push(p.id); + } + } + // An idle peer owes no bytes, so its silence before it gets work + // must not count against its first request. + if grew || p.in_flight == 0 { + entry.owing_since = now; + } + } + streaming + } + + /// Returns the peers to kick and how many culprits are spared. `culprits` maps the + /// peers of stalled requests to when their silence began. + fn judge( + &mut self, + now: Instant, + peers: &[Observed], + mut culprits: HashMap, + ) -> (HashSet, usize) { + self.link_stalled = false; + self.link_down = false; + // Judged on the peer's own counters, not just the registry: streaming in-flight + // bytes are freed on the peer separately, so a wedged peer can sit at its cap + // with no registry entries. Pending work plus a byte counter + // frozen for REQUEST_TIMEOUT means stuck, whatever the registry thinks. + for p in peers.iter().filter(|p| p.in_flight > 0) { + let Some(owing_since) = self.liveness.get(&p.id).map(|l| l.owing_since) else { + continue; + }; + if now.duration_since(owing_since) > p.timeout { + let since = culprits.entry(p.id).or_insert(owing_since); + *since = (*since).min(owing_since); + } + } + if culprits.is_empty() { + return (HashSet::new(), 0); + } + + let mut held = 0; + let mut silent: Vec<(Instant, &Observed)> = Vec::new(); + for p in peers { + let Some(&since) = culprits.get(&p.id) else { + continue; + }; + // Silence is blamed only once any peer, this one included, received a full + // timeout into it, which proves our link works. Silence before the link came + // back does not count, so the peer's own connection gets a timeout to resume. + let witnessed_from = + |from: Instant| self.last_heard.is_some_and(|heard| heard > from + p.timeout); + let blamed_from = self.link_back.map_or(since, |back| back.max(since)); + if witnessed_from(blamed_from) + || now.duration_since(blamed_from) > p.timeout * LINK_STALL_TIMEOUTS + { + silent.push((since, p)); + } else { + self.link_down |= !witnessed_from(since); + held += 1; + } + } + self.link_stalled = held > 0; + // A silent peer may still be our link failing it, so the one silent the longest + // goes, and one kick per timeout lets such a stall cost a single peer, whatever + // share of the set it is. + silent.sort_by_key(|&(since, p)| (since, p.id.addr())); + let due = silent.first().is_some_and(|(_, worst)| { + self.last_kick.is_none_or(|at| now.duration_since(at) > worst.timeout) + }); + if due { + self.last_kick = Some(now); + } + // A culprit that already left is not spared, so its requests are still re-queued. + let mut kicked: HashSet = + culprits.into_keys().filter(|id| peers.iter().all(|p| p.id != *id)).collect(); + kicked.extend(silent.first().filter(|_| due).map(|(_, p)| p.id)); + (kicked, held + silent.len() - usize::from(due)) + } +} + +/// Every [`TIMEOUT_CHECK`], kicks the peers [`StallJudge`] finds stalled on requests +/// with no progress for a [`REQUEST_TIMEOUT`], re-queues all their on-wire requests for +/// other peers and wakes the supervisor to refill the set. fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) -> JoinHandle<()> { tokio::spawn(async move { let NetworkState { @@ -1038,11 +1190,11 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) peer_wake, max_peers, bans, + link_stalled, .. } = &*state; let mut ticker = tokio::time::interval(TIMEOUT_CHECK); - // Per-peer liveness: (bytes read, when the peer last sent bytes or was idle). - let mut progress: HashMap = HashMap::new(); + let mut judge = StallJudge::default(); loop { tokio::select! { _ = shutdown.cancelled() => break, @@ -1050,58 +1202,46 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) } let now = Instant::now(); - - // Judged on the peer's own counters, not just the registry: streaming in-flight - // bytes are freed on the peer separately, so a wedged peer can sit at its cap - // with no registry entries. Pending work plus a byte counter - // frozen for REQUEST_TIMEOUT means stuck, whatever the registry thinks. - let mut culprits: HashSet = { - let peers = connected.read().await; - let live: HashSet = peers.iter().map(|p| p.id()).collect(); - progress.retain(|addr, _| live.contains(addr)); - let mut culprits = HashSet::new(); - for peer in peers.iter() { - let addr = peer.id(); - let bytes = peer.bytes_read(); - let in_flight = peer.in_flight_bytes(); - let entry = progress.entry(addr).or_insert((bytes, now)); - // An idle peer owes no bytes, so its silence before it gets work - // must not count against its first request. - if bytes > entry.0 || in_flight == 0 { - *entry = (bytes, now); - } - if in_flight > 0 && now.duration_since(entry.1) > request_timeout(peer.lag_ms()) - { - culprits.insert(addr); - } - } - culprits - }; - // A request that stopped making progress also condemns its peer: the - // per-peer byte check above sees unrelated traffic and judges it healthy. - let timeouts: HashMap = connected + let peers: Vec = connected .read() .await .iter() - .map(|p| (p.id(), request_timeout(p.lag_ms()))) + .map(|p| Observed { + id: p.id(), + bytes: p.bytes_read(), + partial: p.partial_message(), + in_flight: p.in_flight_bytes(), + timeout: request_timeout(p.lag_ms()), + }) .collect(); - culprits.extend( - broker.expire(|peer| timeouts.get(peer).copied().unwrap_or(REQUEST_TIMEOUT)).await, - ); - + broker.receiving(&judge.observe(now, &peers)).await; + // A request with no progress for a timeout also condemns its peer: the + // per-peer byte check sees unrelated traffic and judges it healthy. + let timeout = |id: &PeerId| { + peers.iter().find(|p| p.id == *id).map_or(REQUEST_TIMEOUT, |p| p.timeout) + }; + let (culprits, spared) = judge.judge(now, &peers, broker.expire(timeout).await); + link_stalled.store(judge.link_stalled, Ordering::Relaxed); if culprits.is_empty() { + if spared > 0 { + tracing::debug!( + target: "dash_spv::network", + "request timeout: kicked none, spared {}", + spared, + ); + } continue; } - // Re-queue every on-wire request of a culprit (fresh ones too: the connection is - // going away), keeping keys as Queued so de-dup holds. - let retried = broker.requeue_from(&culprits).await; - - // Drop culprits still in the set (some may already be gone). + // Drop culprits still in the set (some may already be gone) before their work + // is re-queued, so the router cannot hand it back to them. let (kicked, remaining) = connected.remove(&culprits).await; for peer in &kicked { peer.close(); } + // Re-queue every on-wire request of a culprit (fresh ones too: the connection is + // going away), keeping keys as Queued so de-dup holds. + let retried = broker.requeue_from(&culprits).await; let dropped = kicked.len(); for peer in &culprits { bans.ban(peer.addr(), Ban::Kicked).await; @@ -1113,9 +1253,10 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) tracing::warn!( target: "dash_spv::network", - "request timeout: kicked {} peer(s) {:?}, retried {} request(s)", + "request timeout: kicked {} peer(s) {:?}, spared {}, retried {} request(s)", dropped, culprits, + spared, retried, ); broker.wake(); @@ -1160,12 +1301,22 @@ fn spawn_pump( broker.not_found(peer, inv).await; } - // A block is settled on arrival, not when the wallet gets to it: blocks - // are processed in height order, so one can wait long behind a gap, and - // charging that wait to the peer reads as a queue and shrinks its cap. - // The first copy of a hedged block settles it: the other is a cancel. - if let NetworkMessage::Block(block) = &*msg { - broker.block_arrived(peer, block.block_hash()).await; + // A request is settled on arrival, not when its manager gets to it: blocks + // wait in height order behind a gap and filters in a long backlog, and + // charging that wait to the peer reads as a stall. The last `cfilter` of a + // reply is the one for the stop hash that keys the request. + let arrived = match &*msg { + NetworkMessage::Block(block) => Some(RequestKey::Block(block.block_hash())), + NetworkMessage::CFilter(filter) => { + Some(RequestKey::CFilters(filter.block_hash)) + } + NetworkMessage::CFHeaders(headers) => { + Some(RequestKey::CfHeaders(headers.stop_hash)) + } + _ => None, + }; + if let Some(key) = arrived { + broker.arrived(peer, key).await; } // Peers answer in order, so any answer means the peer is working @@ -1247,8 +1398,8 @@ fn spawn_pump( } let mt = MessageType::from_cmd(msg.cmd()); - // Early router wake on inbound traffic (`release_request` notifies again for the - // real capacity). `cfilter` is skipped: only the last of ~1000 frees anything. + // Early router wake on inbound traffic. `cfilter` is skipped: only the last + // of ~1000 frees anything, and `arrived` wakes the router for it. if mt != Some(MessageType::CFilter) { broker.wake(); } @@ -1332,7 +1483,7 @@ mod tests { NetworkMessage::GetCFilters(GetCFilters { filter_type: 0, start_height: start, - stop_hash: BlockHash::dummy(9), + stop_hash: BlockHash::dummy(start), }) } @@ -1350,6 +1501,13 @@ mod tests { PeerNetworkManager::new(&config).await } + /// Lets the spawned tasks run what the last time advance made due. + async fn settle() { + for _ in 0..4 { + tokio::task::yield_now().await; + } + } + /// The events received until none arrives for a while. async fn drain(events: &mut broadcast::Receiver) -> Vec { let mut received = Vec::new(); @@ -1378,7 +1536,7 @@ mod tests { fn tick(&mut self, delivered: u64, in_flight: u64, link_full: bool) { self.bytes += delivered; - self.cap = self.flow.tick(self.bytes, in_flight, link_full); + self.cap = self.flow.tick(self.bytes, in_flight, link_full, false); } /// Bulk download at `rate` bytes/s. The router keeps the peer at its own cap, or @@ -1488,7 +1646,14 @@ mod tests { let mut late = None; match departure { Departure::Drop => disconnect_all(), - Departure::Kick => tokio::time::advance(REQUEST_TIMEOUT + TIMEOUT_CHECK * 2).await, + Departure::Kick => { + // Silent past the bound, the peers go one per timeout. + for _ in 0..2 { + tokio::time::advance(REQUEST_TIMEOUT * LINK_STALL_TIMEOUTS + TIMEOUT_CHECK) + .await; + settle().await; + } + } Departure::Stop => { net.stop().await; // A handshake that completed while `stop` ran. @@ -1544,40 +1709,110 @@ mod tests { assert_eq!(net.connected_count().await, 0); } - #[test_case(None, true; "a silent request")] - #[test_case(Some(1), false; "the carrier streaming keeps it alive")] - #[test_case(Some(2), true; "another peer streaming is not progress")] + /// Counts `n` bytes as read from connected peer `peer`, leaving a message unfinished + /// if `partial`. + async fn receive(net: &PeerNetworkManager, peer: u8, n: u64, partial: bool) { + for p in net.state.connected.read().await.iter() { + if p.id() == conn(peer, 0) { + p.receive_for_test(n, partial); + } + } + } + + enum Feed { + Nothing, + /// A complete answer from this peer every second. + Answers(u8), + /// More bytes of one long message from this peer every second. + Streams(u8), + /// Small complete messages, such as pings, from this peer every second. + Chatters(u8), + } + + /// Peer 1 carries a request for `secs` seconds while idle peer 2 keeps receiving. + #[test_case(Feed::Nothing, 11, true; "a silent request")] + #[test_case(Feed::Answers(1), 20, false; "the carrier answering keeps it alive")] + #[test_case(Feed::Answers(2), 11, true; "another peer answering is not progress")] + #[test_case(Feed::Streams(1), 55, false; "the carrier streaming one long answer keeps it alive")] + #[test_case(Feed::Streams(1), 61, true; "the carrier streaming one answer past the bound stalls")] + #[test_case(Feed::Streams(2), 11, true; "another peer streaming is not progress")] + #[test_case(Feed::Chatters(1), 11, true; "the carrier sending only small messages stalls")] #[tokio::test(start_paused = true)] - async fn the_timeout_monitor_requeues_stalled_requests(streamer: Option, stalls: bool) { + async fn the_timeout_monitor_requeues_stalled_requests(feed: Feed, secs: u64, stalls: bool) { let net = broker().await; + let session = net.session.lock().await.clone(); + for n in 1..=2 { + let peer = ConnectedPeer::for_test(conn(n, 0)).await; + let admitted = + net.state.connected.admit(peer, &session, net.state.max_peers, None).await; + assert!(matches!(admitted, Admission::Joined)); + } net.state.broker.mark_on_wire(&get_cfilters(0), conn(1, 0)).await; - // Two gaps that each stay under the deadline but together exceed it. - for _ in 0..2 { - tokio::time::advance(REQUEST_TIMEOUT - TIMEOUT_CHECK).await; - tokio::task::yield_now().await; - if let Some(peer) = streamer { - let piece = NetworkMessage::CFilter(CFilter { - filter_type: 0, - block_hash: BlockHash::dummy(1), - filter: vec![0u8; 4], - }); - net.state - .inbound - .send(PeerEvent::Message(conn(peer, 0), piece.into())) - .expect("the pump is running"); + for _ in 0..secs { + tokio::time::advance(TIMEOUT_CHECK).await; + receive(&net, 2, 100, false).await; + match feed { + Feed::Nothing => {} + Feed::Answers(peer) => { + let piece = NetworkMessage::CFilter(CFilter { + filter_type: 0, + block_hash: BlockHash::dummy(1), + filter: vec![0u8; 4], + }); + net.state + .inbound + .send(PeerEvent::Message(conn(peer, 0), piece.into())) + .expect("the pump is running"); + } + Feed::Streams(peer) => receive(&net, peer, 1000, true).await, + Feed::Chatters(peer) => receive(&net, peer, 100, false).await, } - tokio::task::yield_now().await; - tokio::task::yield_now().await; + settle().await; } - let want = if stalls { - "queued" - } else { - "on_wire" - }; - assert_eq!(net.state.broker.state_of(&RequestKey::CFilters(0)).await, Some(want)); - assert_eq!(net.state.broker.len(), stalls as usize); + // A stalled request is re-queued and may already be routed to peer 2. + let carrier = net.state.broker.carrier_of(&RequestKey::CFilters(BlockHash::dummy(0))).await; + assert_eq!(carrier == Some(conn(1, 0)), !stalls, "carrier {carrier:?}"); + } + + /// A kick re-sends what the peer still owes, not a batch that it delivered and that + /// waits for the filters manager. + #[tokio::test(start_paused = true)] + async fn a_kick_requeues_no_delivered_filter_batch() { + let net = broker().await; + let session = net.session.lock().await.clone(); + for n in 1..=2 { + let peer = ConnectedPeer::for_test(conn(n, 0)).await; + let admitted = + net.state.connected.admit(peer, &session, net.state.max_peers, None).await; + assert!(matches!(admitted, Admission::Joined)); + } + let (delivered, owed) = (get_cfilters(0), get_cfilters(1000)); + net.state.broker.mark_on_wire(&delivered, conn(1, 0)).await; + net.state.broker.mark_on_wire(&owed, conn(1, 0)).await; + let last = NetworkMessage::CFilter(CFilter { + filter_type: 0, + block_hash: BlockHash::dummy(0), + filter: vec![0u8; 4], + }); + net.state + .inbound + .send(PeerEvent::Message(conn(1, 0), last.into())) + .expect("the pump is running"); + settle().await; + + for _ in 0..12 { + tokio::time::advance(TIMEOUT_CHECK).await; + receive(&net, 2, 100, false).await; + settle().await; + } + + let broker = &net.state.broker; + let owed = RequestKey::CFilters(BlockHash::dummy(1000)); + assert_ne!(broker.carrier_of(&owed).await, Some(conn(1, 0)), "the peer must be kicked"); + let delivered = RequestKey::CFilters(BlockHash::dummy(0)); + assert_eq!(broker.state_of(&delivered).await, Some("arrived")); } /// A peer that was silent while idle is timed from the request it is given, and @@ -1586,27 +1821,170 @@ mod tests { async fn the_timeout_monitor_times_an_idle_peer_from_its_first_request() { let net = broker().await; let session = net.session.lock().await.clone(); - let id = conn(1, 0); - let peer = ConnectedPeer::for_test(id).await; + let peer = ConnectedPeer::for_test(conn(1, 0)).await; let handle = peer.handle(); - let admitted = net.state.connected.admit(peer, &session, net.state.max_peers, None).await; - assert!(matches!(admitted, Admission::Joined)); - - for _ in 0..3 { - tokio::time::advance(REQUEST_TIMEOUT).await; - tokio::task::yield_now().await; + for peer in [peer, ConnectedPeer::for_test(conn(2, 0)).await] { + let admitted = + net.state.connected.admit(peer, &session, net.state.max_peers, None).await; + assert!(matches!(admitted, Admission::Joined)); } + + // The second peer keeps receiving, so a stall is not blamed on our link. + let run = |secs: u64| { + let net = &net; + async move { + for _ in 0..secs { + tokio::time::advance(TIMEOUT_CHECK).await; + receive(net, 2, 100, false).await; + settle().await; + } + } + }; + run(REQUEST_TIMEOUT.as_secs() * 3).await; // Only the charge matters here: the loopback has no reader to answer it. let _ = handle.send(&get_cfilters(0)).await; assert!(handle.in_flight_bytes() > 0); - tokio::time::advance(TIMEOUT_CHECK * 2).await; - tokio::task::yield_now().await; - assert_eq!(net.connected_count().await, 1, "kicked right after its first request"); + run(2).await; + assert_eq!(net.connected_count().await, 2, "kicked right after its first request"); + + run(REQUEST_TIMEOUT.as_secs()).await; + assert_eq!(net.connected_count().await, 1, "a peer silent with work stays connected"); + } + + /// What a peer of the set does, second by second. + #[derive(Clone, Copy)] + enum Role { + /// Holds a request the registry tracks and receives bytes until this second. + Registered(u64), + /// Owes bytes the registry does not track and receives bytes until this second. + Charged(u64), + /// Idle, and receives bytes from this second on. + Delivers(u64), + /// Connects at this second, idle, and receives nothing. + Joins(u64), + /// Owes bytes. Receives until the time in the first field and again from the time + /// in the second, both in seconds. + Resumes(u64, u64), + /// Idle, and receives bytes every this many seconds. + Whispers(u64), + /// Holds a request the registry tracks and receives small whole messages from + /// this second on. + Chatters(u64), + } + + /// Peer `n + 1` plays `roles[n]` for `secs` seconds. `link_stalled` is whether the + /// judge then suspects our own link. + #[test_case(&[Role::Registered(0); 3], 11, &[1, 2, 3], true; "a stall of every peer at once is our link")] + #[test_case(&[Role::Registered(0); 3], 41, &[1, 2, 3], true; "a 40 s silence of every peer kicks nobody")] + #[test_case(&[Role::Registered(0); 3], 71, &[2, 3], false; "every peer silent past the bound loses the one silent the longest")] + #[test_case(&[Role::Registered(0); 3], 72, &[3], false; "every peer silent past the bound loses one per timeout")] + #[test_case(&[Role::Charged(0); 3], 25, &[1, 2, 3], true; "a long stall of every peer is still our link")] + #[test_case(&[Role::Registered(1), Role::Registered(2), Role::Registered(3)], 25, &[1, 2, 3], true; "peers going silent a second apart in an outage stay")] + #[test_case(&[Role::Charged(1), Role::Charged(2), Role::Charged(3)], 25, &[1, 2, 3], true; "peers owing bytes and going silent a second apart stay")] + #[test_case(&[Role::Delivers(0), Role::Registered(0), Role::Delivers(0)], 11, &[1, 3], false; "a lone silent request among delivering peers is kicked")] + #[test_case(&[Role::Delivers(0), Role::Charged(0), Role::Delivers(0)], 12, &[1, 3], false; "a lone peer owing bytes among delivering peers is kicked")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(0)], 11, &[2, 3], false; "two stalls beside a delivering peer kick one")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(0)], 22, &[2], false; "the second stall goes a timeout later")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(1)], 12, &[2, 3], false; "stalls a second apart beside a delivering peer go one per timeout")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(2)], 14, &[2, 3], false; "stalls two seconds apart beside a delivering peer go one per timeout")] + #[test_case(&[Role::Registered(0), Role::Delivers(0), Role::Registered(1), Role::Joins(11)], 12, &[2, 3, 4], false; "a replacement joining does not unpace the next stall")] + #[test_case(&[Role::Registered(0), Role::Delivers(13), Role::Registered(1)], 24, &[2, 3], false; "a late witness kicks the peer silent the longest a timeout later")] + #[test_case(&[Role::Resumes(5, 20), Role::Resumes(5, 20), Role::Charged(5)], 25, &[1, 2, 3], true; "a peer still silent when the link is back gets a timeout")] + #[test_case(&[Role::Resumes(5, 20), Role::Resumes(5, 20), Role::Charged(5)], 31, &[1, 2], false; "a peer silent a timeout after the link is back is kicked")] + #[test_case(&[Role::Charged(0), Role::Whispers(3)], 13, &[2], false; "a lone peer owing bytes beside an idle one is no link stall")] + #[test_case(&[Role::Registered(0)], 41, &[1], true; "a sole silent peer stays through a 40 s outage")] + #[test_case(&[Role::Registered(0)], 61, &[], false; "a sole silent peer is kicked past the bound")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Joins(12)], 13, &[1, 2, 3], true; "a peer joining during an outage proves nothing")] + #[test_case(&[Role::Charged(0), Role::Delivers(30)], 15, &[1, 2], true; "a lone peer owing bytes beside a quiet idle one is held")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 35, &[1, 2, 3], true; "an idle peer receiving first after an outage gives the held peers a timeout")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 36, &[2, 3], false; "held peers silent a timeout after an idle peer receives go")] + #[test_case(&[Role::Registered(0), Role::Registered(0), Role::Delivers(25)], 47, &[3], false; "held peers silent after an idle peer receives go one per timeout")] + #[test_case(&[Role::Charged(0), Role::Whispers(15)], 30, &[2], false; "an idle peer's later bytes do not restart a held peer's timeout")] + #[test_case(&[Role::Resumes(0, 55), Role::Charged(0)], 65, &[1, 2], true; "a peer silent through a 55 s outage gets a timeout after it")] + #[test_case(&[Role::Resumes(0, 55), Role::Charged(0)], 66, &[1], false; "a peer silent a timeout after a 55 s outage is kicked")] + #[test_case(&[Role::Chatters(0)], 11, &[], false; "a sole peer receiving only small messages is kicked")] + #[test_case(&[Role::Chatters(25); 3], 36, &[2, 3], false; "peers receiving only small messages after an outage go one per timeout")] + fn the_stall_judge_kicks_one_peer_per_shared_stall( + roles: &[Role], + secs: u64, + kept: &[u8], + link_stalled: bool, + ) { + // Every received byte but a chatter's is part of an answer still arriving. + let peer = |n: u8, role: Role| Observed { + id: conn(n, 0), + bytes: 0, + partial: !matches!(role, Role::Chatters(_)), + in_flight: matches!(role, Role::Charged(_) | Role::Resumes(..)).into(), + timeout: REQUEST_TIMEOUT, + }; + let start = Instant::now(); + let mut peers: Vec = (1..) + .zip(roles) + .filter(|(_, role)| !matches!(role, Role::Joins(_))) + .map(|(n, &role)| peer(n, role)) + .collect(); + // The last progress of each registered request. + let mut requests: HashMap = (1..) + .zip(roles) + .filter(|(_, role)| matches!(role, Role::Registered(_) | Role::Chatters(_))) + .map(|(n, _)| (conn(n, 0), start)) + .collect(); + let mut judge = StallJudge::default(); + + for second in 1..=secs { + for (n, &role) in (1..).zip(roles) { + let receives = match role { + Role::Registered(until) | Role::Charged(until) => second <= until, + Role::Delivers(from) | Role::Chatters(from) => second >= from, + Role::Resumes(until, from) => second <= until || second >= from, + Role::Whispers(period) => second.is_multiple_of(period), + Role::Joins(at) => { + if second == at { + peers.push(peer(n, role)); + } + false + } + }; + if let Some(p) = peers.iter_mut().find(|p| receives && p.id == conn(n, 0)) { + p.bytes += 100; + } + } + let now = start + Duration::from_secs(second); + for id in judge.observe(now, &peers) { + if let Some(progress) = requests.get_mut(&id) { + *progress = now; + } + } + let stalled = requests + .iter() + .filter(|(_, &progress)| now.duration_since(progress) > REQUEST_TIMEOUT) + .map(|(&id, &progress)| (id, progress)) + .collect(); + let (kicked, _) = judge.judge(now, &peers, stalled); + assert!(kicked.len() <= 1, "second {second} kicked {kicked:?}"); + peers.retain(|p| !kicked.contains(&p.id)); + requests.retain(|id, _| !kicked.contains(id)); + } + + let left: Vec = peers.iter().map(|p| p.id).collect(); + let kept: Vec = kept.iter().map(|&n| conn(n, 0)).collect(); + assert_eq!(left, kept); + assert_eq!(judge.link_stalled, link_stalled); + } - tokio::time::advance(REQUEST_TIMEOUT).await; - tokio::task::yield_now().await; - assert_eq!(net.connected_count().await, 0, "a peer silent with work stays connected"); + /// A peer serves 1 MB/s with work for three ticks, then nothing for one. + #[test_case(false, true => 750_000; "a stall of its own drags its serving rate")] + #[test_case(true, true => 1_000_000; "a stall of our link does not")] + #[test_case(false, false => 1_000_000; "idle ticks do not")] + fn the_serving_rate_averages_ticks_with_work(link_stalled: bool, has_work: bool) -> u64 { + let mut flow = PeerFlow::new(0); + for _ in 0..3 { + flow.sample(1e6, true, false); + } + flow.sample(0.0, has_work, link_stalled); + flow.serving_rate as u64 } #[test_case(&[&[MessageType::Inv], &[MessageType::Inv]], 0, &[1, 1]; "every subscriber of the type")] diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index 850ac5bee..5d63fa243 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -17,7 +17,7 @@ use dashcore::{ use futures::lock::Mutex; use rand::Rng; use std::net::SocketAddr; -use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; #[cfg(test)] @@ -137,6 +137,23 @@ impl Decoder for SizedCodec { } } +/// Frames messages and records whether the bytes of an unfinished one are buffered. +struct TrackedCodec { + codec: SizedCodec, + partial: Arc, +} + +impl Decoder for TrackedCodec { + type Item = (RawNetworkMessage, u64); + type Error = encode::Error; + + fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { + let frame = self.codec.decode(src); + self.partial.store(!src.is_empty(), Ordering::Relaxed); + frame + } +} + /// The socket's write half, framing each message for the connection's transport. struct PeerWriter { half: OwnedWriteHalf, @@ -172,7 +189,7 @@ impl PeerWriter { /// the peer's version packet. type LinkReader = Either, OwnedReadHalf>; -type PeerReader = FramedRead, SizedCodec>; +type PeerReader = FramedRead, TrackedCodec>; /// A TCP connection with its transport negotiated, ready for the version handshake. struct Link { @@ -269,6 +286,10 @@ impl PeerHandle { self.cap.load(Ordering::Relaxed) } + pub(crate) fn is_closed(&self) -> bool { + self.token.is_cancelled() + } + pub async fn send(&self, msg: &NetworkMessage) -> NetworkResult<()> { // The sync layer sends a plain `getheaders`, since it does not know the peer. // The reader decompresses `Headers2` back to `Headers`. @@ -300,6 +321,7 @@ pub struct ConnectedPeer { version: VersionMessage, lag_ms: AtomicU32, bytes: Arc, + partial: Arc, } pub struct DisconnectedPeer { @@ -342,6 +364,11 @@ impl ConnectedPeer { self.bytes.load(Ordering::Relaxed) } + /// Whether part of a message has arrived and the rest has not. + pub fn partial_message(&self) -> bool { + self.partial.load(Ordering::Relaxed) + } + pub fn in_flight_bytes(&self) -> u64 { self.handle.in_flight_bytes() } @@ -385,8 +412,17 @@ impl ConnectedPeer { version: build_version(id.addr(), "test"), lag_ms: AtomicU32::new(1), bytes: Arc::new(AtomicU64::new(0)), + partial: Arc::new(AtomicBool::new(false)), } } + + /// Counts `n` bytes as read from the socket, as the reader does, leaving a message + /// unfinished if `partial`. + #[cfg(test)] + pub(crate) fn receive_for_test(&self, n: u64, partial: bool) { + self.bytes.fetch_add(n, Ordering::Relaxed); + self.partial.store(partial, Ordering::Relaxed); + } } impl DisconnectedPeer { @@ -481,13 +517,17 @@ impl DisconnectedPeer { 1 }; let peer_bytes = Arc::new(AtomicU64::new(0)); + let partial = Arc::new(AtomicBool::new(false)); let mut reader = FramedRead::new( CountingReader { inner: reader, state: state.clone(), peer_bytes: peer_bytes.clone(), }, - codec, + TrackedCodec { + codec, + partial: partial.clone(), + }, ); handshake_send( @@ -610,6 +650,7 @@ impl DisconnectedPeer { version, lag_ms: AtomicU32::new(lag_ms), bytes: peer_bytes, + partial, }) } } @@ -752,6 +793,8 @@ async fn handshake_send(writer: &mut PeerWriter, payload: NetworkMessage) -> Net mod tests { use std::io::{Error as IoError, ErrorKind}; + use test_case::test_case; + use super::*; fn io_error(kind: ErrorKind) -> ProtocolError { @@ -789,4 +832,25 @@ mod tests { wire_agent(&format!("{}é{}", "a".repeat(MAX_USER_AGENT_LEN - 3), "b".repeat(10))); assert_eq!(straddling, format!("/{}/", "a".repeat(MAX_USER_AGENT_LEN - 3))); } + + /// Whether the codec reports an unfinished message after the first `n` bytes of two + /// 32-byte pings arrive. + #[test_case(1 => true; "part of a header")] + #[test_case(24 => true; "a header without its payload")] + #[test_case(32 => false; "a whole ping")] + #[test_case(40 => true; "a whole ping and part of the next")] + fn the_codec_tracks_an_unfinished_message(n: usize) -> bool { + let magic = Network::Regtest.magic(); + let ping = encode::serialize(&RawNetworkMessage { + magic, + payload: NetworkMessage::Ping(1), + }); + let mut codec = TrackedCodec { + codec: SizedCodec::V1, + partial: Arc::new(AtomicBool::new(false)), + }; + let mut src = BytesMut::from(&[ping.clone(), ping].concat()[..n]); + while codec.decode(&mut src).unwrap().is_some() {} + codec.partial.load(Ordering::Relaxed) + } } diff --git a/dash-spv/src/sync/filter_headers/manager.rs b/dash-spv/src/sync/filter_headers/manager.rs index 229f7da03..244db94a5 100644 --- a/dash-spv/src/sync/filter_headers/manager.rs +++ b/dash-spv/src/sync/filter_headers/manager.rs @@ -388,7 +388,7 @@ impl std::fmt::Debug #[cfg(test)] mod tests { use super::*; - use crate::network::MessageType; + use crate::network::{MessageType, RequestKey}; use crate::storage::{ DiskStorageManager, PersistentBlockHeaderStorage, PersistentFilterHeaderStorage, StorageManager, @@ -801,6 +801,29 @@ mod tests { assert_eq!(filter_tip(&storage).await, Some(3999)); } + /// The broker settles a request when its reply arrives, so a reply the pipeline does + /// not take must still release the key or a later declaration of the batch sends nothing. + #[test_case(0, false; "an empty reply")] + #[test_case(2000, true; "a reply reaching a pipeline dropped by a disconnect")] + #[tokio::test] + async fn test_cfheaders_reply_matching_nothing_releases_its_key( + count: u32, + disconnected: bool, + ) { + let storage = DiskStorageManager::with_temp_dir().await.unwrap(); + let hashes = store_block_headers(&storage, 0, 2000).await; + let (mut manager, network) = start_sync(&storage, 1999).await; + if disconnected { + manager.on_disconnect(); + } + + let reply = cfheaders(FilterHeader::all_zeros(), hashes[1999], count); + let (stored, _) = deliver(&mut manager, &network, reply).await; + + assert!(stored.is_empty()); + assert_eq!(network.released_keys().await, vec![RequestKey::CfHeaders(hashes[1999])]); + } + #[tokio::test] async fn test_buffered_cfheaders_batch_not_connecting_is_requeued_when_ready() { let storage = DiskStorageManager::with_temp_dir().await.unwrap(); diff --git a/dash-spv/src/sync/filter_headers/sync_manager.rs b/dash-spv/src/sync/filter_headers/sync_manager.rs index aaa12346c..a9216b605 100644 --- a/dash-spv/src/sync/filter_headers/sync_manager.rs +++ b/dash-spv/src/sync/filter_headers/sync_manager.rs @@ -46,9 +46,11 @@ impl SyncManager for FilterHeade msg: InboundMessage, network: &Arc, ) -> SyncResult> { + // Released whatever comes of the reply: a reply that matches nothing must not leave + // its key settled, and a batch still wanted is declared again. if let NetworkMessage::CFHeaders(cfheaders) = &*msg { + network.release_request(RequestKey::CfHeaders(cfheaders.stop_hash)).await; if self.pipeline.receive_below(cfheaders) { - network.release_request(RequestKey::CfHeaders(cfheaders.stop_hash)).await; return self.store_lower_batches().await; } } @@ -63,8 +65,6 @@ impl SyncManager for FilterHeade return Ok(vec![]); }; - network.release_request(RequestKey::CfHeaders(cfheaders.stop_hash)).await; - let mut events = Vec::new(); // Try to receive (may buffer if out of order) diff --git a/dash-spv/src/sync/filters/manager.rs b/dash-spv/src/sync/filters/manager.rs index 12d08f954..9be3795b3 100644 --- a/dash-spv/src/sync/filters/manager.rs +++ b/dash-spv/src/sync/filters/manager.rs @@ -13,8 +13,8 @@ use dashcore::ScriptBuf; use super::batch::FiltersBatch; use super::block_match_tracker::{BlockMatchTracker, BlockTrackResult}; use super::pipeline::FiltersPipeline; -use crate::error::SyncResult; -use crate::network::NetworkManager; +use crate::error::{SyncError, SyncResult}; +use crate::network::{NetworkManager, RequestKey}; use crate::storage::{BlockHeaderStorage, FilterHeaderStorage, FilterStorage}; use crate::sync::filters::util::get_prev_filter_header; use crate::sync::{FiltersProgress, SyncEvent, SyncManager, SyncState}; @@ -251,11 +251,15 @@ impl) { self.active_batches.clear(); self.tracker.clear(); self.pending_batches.clear(); + for stop_hash in self.filter_pipeline.wanted_stops() { + network.release_request(RequestKey::CFilters(*stop_hash)).await; + } self.filter_pipeline = FiltersPipeline::new(); } @@ -496,6 +500,43 @@ impl SyncResult<()> { + // Load filter headers for verification + let filter_headers = self + .filter_header_storage + .read() + .await + .load_filter_headers(batch.start_height()..batch.end_height() + 1) + .await?; + + let filter_headers_map: HashMap = filter_headers + .into_iter() + .enumerate() + .map(|(idx, header)| (batch.start_height() + idx as u32, header)) + .collect(); + + let filter_header_storage = self.filter_header_storage.read().await; + let prev_filter_header = + get_prev_filter_header(&*filter_header_storage, batch.start_height()).await?; + drop(filter_header_storage); + + let validator = FilterValidator::new(); + let validation_input = FilterValidationInput { + filters: batch.filters(), + expected_headers: &filter_headers_map, + prev_filter_header, + }; + validator.validate(validation_input)?; + + // Store verified filters to disk + let mut filter_storage = self.filter_storage.write().await; + for (key, filter) in batch.filters() { + filter_storage.store_filter(key.height(), &filter.content).await?; + } + Ok(()) + } + /// Store completed filter batches to disk and do speculative matching. /// This is decoupled from block processing - we store and match as fast as possible. pub(super) async fn store_and_match_batches(&mut self) -> SyncResult> { @@ -540,44 +581,27 @@ impl = filter_headers - .into_iter() - .enumerate() - .map(|(idx, header)| (batch.start_height() + idx as u32, header)) - .collect(); - - let filter_header_storage = self.filter_header_storage.read().await; - let prev_filter_header = - get_prev_filter_header(&*filter_header_storage, batch.start_height()).await?; - drop(filter_header_storage); - - let validator = FilterValidator::new(); - let validation_input = FilterValidationInput { - filters: batch.filters(), - expected_headers: &filter_headers_map, - prev_filter_header, - }; - validator.validate(validation_input)?; - - // Store verified filters to disk - let mut filter_storage = self.filter_storage.write().await; - for (key, filter) in batch.filters() { - filter_storage.store_filter(key.height(), &filter.content).await?; + let (start, end) = (batch.start_height(), batch.end_height()); + match self.verify_and_store(&batch).await { + Ok(()) => events.push(SyncEvent::FiltersStored { + start_height: start, + end_height: end, + }), + Err(SyncError::Validation(reason)) => { + tracing::warn!( + "Rejected filter batch {}-{}, requesting it again: {}", + start, + end, + reason + ); + self.filter_pipeline.requeue_rejected(start, end); + break; + } + Err(e) => { + self.filter_pipeline.requeue_rejected(start, end); + return Err(e); + } } - drop(filter_storage); - - events.push(SyncEvent::FiltersStored { - start_height: batch.start_height(), - end_height: batch.end_height(), - }); } // === Load filters into all active batches that overlap === @@ -1362,6 +1386,7 @@ mod tests { MockWallet, MockWalletState, MultiMockWallet, MOCK_WALLET_ID, }; use std::net::SocketAddr; + use test_case::test_case; type TestFiltersManager = FiltersManager< PersistentBlockHeaderStorage, @@ -3155,7 +3180,8 @@ mod tests { manager.filter_pipeline.init(2000, 2999); assert!(!manager.is_idle()); - manager.reset_for_rescan(); + let requests: Arc = Arc::new(MockNetworkManager::new()); + manager.reset_for_rescan(&requests).await; assert!(manager.is_idle()); } @@ -3832,8 +3858,10 @@ mod tests { /// block makes `extend_target` re-request every filter from height 1 and /// wedges the in-order store loop on a `next_batch_to_store` of 0, freezing /// `committed_height` one below tip. + #[test_case(false; "the first copy is valid")] + #[test_case(true; "a rejected copy is requested again")] #[tokio::test] - async fn test_synced_boot_then_boundary_block_commits() { + async fn test_synced_boot_then_boundary_block_commits(rejected_first: bool) { let (mut manager, headers, boundary_filter) = setup_synced_manager_at_tip().await; // Fully-synced boot leaves the manager in its default state. @@ -3890,6 +3918,19 @@ mod tests { block_hash: headers[101].block_hash(), filter: boundary_filter.content.clone(), }; + if rejected_first { + let corrupt = CFilter { + filter: vec![0xFF; 4], + ..cfilter.clone() + }; + manager + .handle_message(peer, NetworkMessage::CFilter(corrupt).into(), &requests) + .await + .unwrap(); + mock.clear_sent().await; + manager.tick(&requests).await.unwrap(); + assert_eq!(drain_getcfilters(&mock).await, 1, "the rejected batch must return"); + } manager .handle_message(peer, NetworkMessage::CFilter(cfilter).into(), &requests) .await @@ -4041,7 +4082,14 @@ mod tests { // MockWallet defaults to synced_height=0, so wallets_behind(100) = {MOCK_WALLET_ID}. assert_eq!(manager.wallet.read().await.synced_height(), 0); - let requests: Arc = Arc::new(MockNetworkManager::new()); + let mock = Arc::new(MockNetworkManager::new()); + let requests: Arc = mock.clone(); + let header_storage = manager.header_storage.clone(); + manager + .filter_pipeline + .send_pending(&requests, &*header_storage.read().await) + .await + .unwrap(); // Sanity: the pre-populated stale processed record is present, so // `track` for the same wallet would short-circuit to AlreadyProcessed. @@ -4059,6 +4107,11 @@ mod tests { // Old in-flight state was cleared and a fresh batch was created at scan_start=0. assert!(!manager.active_batches.contains_key(&101)); assert!(manager.active_batches.contains_key(&0)); + assert_eq!( + mock.released_keys().await, + vec![RequestKey::CFilters(headers[200].block_hash())], + "a dropped batch must not leave its request registered" + ); // The stale pre-populated record was wiped by `reset_for_rescan`: // a fresh `track` for the same wallet now returns `NewlyTracked`. assert!(matches!( @@ -4215,11 +4268,48 @@ mod tests { *headers[10].hash(), &filter.content ), - Some(200_000), + Some(*headers[10].hash()), "a tick re-requested filters that were still in flight" ); } + /// The broker settles a request when the last filter of its reply arrives, so + /// only the manager can tell that the reply was not enough. + #[tokio::test] + async fn a_reply_that_leaves_its_batch_incomplete_is_requested_again() { + let mut manager = create_anchored_test_manager(200_000, 10).await; + manager.filter_storage.write().await.clear_filters().await.unwrap(); + manager.progress.update_stored_height(0); + + let mock = Arc::new(MockNetworkManager::new()); + let requests: Arc = mock.clone(); + manager.handle_new_filter_headers(200_010, &requests).await.unwrap(); + assert_eq!(drain_getcfilters(&mock).await, 1); + + let stop_hash = + *manager.header_storage.read().await.get_header(200_010).await.unwrap().unwrap().hash(); + let last_filter = |block_hash| { + NetworkMessage::CFilter(CFilter { + filter_type: 0, + block_hash, + filter: vec![0; 4], + }) + .into() + }; + let peer: SocketAddr = "127.0.0.1:19999".parse().unwrap(); + + manager.handle_message(peer, last_filter(stop_hash), &requests).await.unwrap(); + assert_eq!(mock.released_keys().await, vec![RequestKey::CFilters(stop_hash)]); + + manager.tick(&requests).await.unwrap(); + assert_eq!(drain_getcfilters(&mock).await, 1, "the incomplete batch must return"); + + // A stop hash that no stored header resolves cannot reach the pipeline. + let unknown = dashcore::BlockHash::dummy(7); + assert!(manager.handle_message(peer, last_filter(unknown), &requests).await.is_err()); + assert_eq!(mock.released_keys().await.last(), Some(&RequestKey::CFilters(unknown))); + } + /// A wallet added at runtime reports `synced_height = 0` regardless of what /// the client anchored at, so raising the birth height at construction does /// not cover this path. It must converge, not rescan forever. @@ -4350,7 +4440,8 @@ mod tests { // Sanity check: the full-reset path (`reset_for_rescan`) does wipe // everything — the two paths must remain distinct. - manager.reset_for_rescan(); + let requests: Arc = Arc::new(MockNetworkManager::new()); + manager.reset_for_rescan(&requests).await; assert!(manager.active_batches.is_empty()); assert!(manager.pending_batches.is_empty()); assert!(manager.tracker.is_empty()); diff --git a/dash-spv/src/sync/filters/pipeline.rs b/dash-spv/src/sync/filters/pipeline.rs index 1b36f3c27..9731189fc 100644 --- a/dash-spv/src/sync/filters/pipeline.rs +++ b/dash-spv/src/sync/filters/pipeline.rs @@ -197,14 +197,15 @@ impl FiltersPipeline { /// Handle a received CFilter message with filter data. /// /// Buffers the filter data for batch verification and wallet matching. - /// Returns `Some(batch_start)` (the `GetCFilters` request key) when a batch - /// completes, `None` otherwise. + /// Returns the stop hash of the batch's `GetCFilters` request when that request is over: + /// the batch completed, or its last filter arrived and earlier ones are missing. Peers + /// answer in order, so nothing more will come and the request must be declared again. pub(super) fn receive_with_data( &mut self, height: u32, block_hash: BlockHash, filter_data: &[u8], - ) -> Option { + ) -> Option { // Find which batch this filter belongs to let batch_start = self.find_batch_for_height(height)?; @@ -226,7 +227,17 @@ impl FiltersPipeline { expected ); } - return None; + return if height == tracker.end_height() { + tracing::warn!( + "Filter batch {} ended incomplete with {}/{} filters, requesting it again", + batch_start, + received, + expected + ); + self.batch_stops.get(&batch_start).copied() + } else { + None + }; } let end_height = tracker.end_height(); @@ -235,7 +246,7 @@ impl FiltersPipeline { self.batch_trackers.get_mut(&batch_start).map(|t| t.take_filters()).unwrap_or_default(); self.batch_trackers.remove(&batch_start); - self.batch_stops.remove(&batch_start); + let stop_hash = self.batch_stops.remove(&batch_start); for key in filters.keys() { self.block_heights.remove(key.hash()); } @@ -249,7 +260,18 @@ impl FiltersPipeline { let batch = FiltersBatch::new(batch_start, end_height, filters); self.completed_batches.insert(batch); - Some(batch_start) + stop_hash + } + + /// Returns the stop hash of every batch still wanted, the keys of its `GetCFilters` requests. + pub(super) fn wanted_stops(&self) -> impl Iterator { + self.batch_stops.values() + } + + /// Wants the batch `start_height..=end_height` again. A rejected batch left the pipeline + /// when it completed, and without it the store order waits for it forever. + pub(super) fn requeue_rejected(&mut self, start_height: u32, end_height: u32) { + self.batch_trackers.insert(start_height, BatchTracker::new(end_height)); } /// Returns the start height of the wanted batch that contains `height`. @@ -397,20 +419,22 @@ mod tests { } #[test] - fn test_receive_batch_completion_returns_batch_start() { + fn test_receive_batch_completion_returns_the_stop_hash() { let mut pipeline = FiltersPipeline::new(); pipeline.target_height = 2; pipeline.batch_trackers.insert(0, BatchTracker::new(2)); + let stop_hash = Header::dummy(2).block_hash(); + pipeline.batch_stops.insert(0, stop_hash); - let mut completed_at = None; + let mut completed_with = None; for h in 0..=2 { let hash = Header::dummy(h).block_hash(); - if let Some(start) = pipeline.receive_with_data(h, hash, &dummy_filter_data(h)) { - completed_at = Some(start); + if let Some(stop) = pipeline.receive_with_data(h, hash, &dummy_filter_data(h)) { + completed_with = Some(stop); } } - assert_eq!(completed_at, Some(0)); + assert_eq!(completed_with, Some(stop_hash)); assert!(pipeline.batch_trackers.is_empty()); assert!(!pipeline.batch_stops.contains_key(&0)); assert_eq!(pipeline.completed_batches.len(), 1); diff --git a/dash-spv/src/sync/filters/sync_manager.rs b/dash-spv/src/sync/filters/sync_manager.rs index 7f6fef22d..e14012ba0 100644 --- a/dash-spv/src/sync/filters/sync_manager.rs +++ b/dash-spv/src/sync/filters/sync_manager.rs @@ -120,9 +120,13 @@ impl< // Find height for this filter let height = - self.filter_pipeline.resolve_height(&cfilter.block_hash, &self.header_storage).await?; + self.filter_pipeline.resolve_height(&cfilter.block_hash, &self.header_storage).await; + if !matches!(height, Ok(Some(_))) { + // This may be the last filter of its batch, which the pipeline will never see. + network.release_request(RequestKey::CFilters(cfilter.block_hash)).await; + } - let Some(h) = height else { + let Some(h) = height? else { tracing::warn!( block_hash = %cfilter.block_hash, peer = %peer, @@ -136,13 +140,13 @@ impl< }; // Buffer filter in pipeline - let batch_completed = + let request_over = self.filter_pipeline.receive_with_data(h, cfilter.block_hash, &cfilter.filter); // One `getcfilters` is answered only once its whole batch has arrived, so // individual `cfilter`s are never answered on their own. - if let Some(batch_start) = batch_completed { - network.release_request(RequestKey::CFilters(batch_start)).await; + if let Some(stop_hash) = request_over { + network.release_request(RequestKey::CFilters(stop_hash)).await; } // No `send_pending` here: re-declaring per `cfilter` would re-scan every @@ -241,7 +245,7 @@ impl< committed, restart_at ); - self.reset_for_rescan(); + self.reset_for_rescan(network).await; self.progress.update_committed_height(stale_min_synced); return self.start_download(network).await; } diff --git a/dash-spv/src/test_utils/network.rs b/dash-spv/src/test_utils/network.rs index 9281b65c0..2768bfb50 100644 --- a/dash-spv/src/test_utils/network.rs +++ b/dash-spv/src/test_utils/network.rs @@ -215,7 +215,7 @@ mod tests { NetworkMessage::GetCFilters(GetCFilters { filter_type: 0, start_height: start, - stop_hash: BlockHash::dummy(9), + stop_hash: BlockHash::dummy(start), }) } @@ -243,7 +243,7 @@ mod tests { mock.send(get_cfilters(0)).await; assert_eq!(mock.sent_messages().await.len(), 1); - mock.release_request(RequestKey::CFilters(0)).await; + mock.release_request(RequestKey::CFilters(BlockHash::dummy(0))).await; assert_eq!(mock.requests_in_play().await, 0); mock.send(get_cfilters(0)).await; diff --git a/dash-spv/src/test_utils/node.rs b/dash-spv/src/test_utils/node.rs index e49c1d82c..cdf81c0e0 100644 --- a/dash-spv/src/test_utils/node.rs +++ b/dash-spv/src/test_utils/node.rs @@ -16,6 +16,8 @@ use std::process::ExitStatus; use std::sync::atomic::{AtomicU16, Ordering}; use std::time::Duration; use tokio::process::Child; +#[cfg(unix)] +use tokio::process::Command; use tokio::time::{sleep, timeout}; use super::fs_helpers::{copy_dir, copy_to_retain_dir}; @@ -683,6 +685,25 @@ impl DashCoreNode { tracing::info!("Disconnected peer {}", addr); } + /// Freezes dashd with `SIGSTOP`, or resumes it with `SIGCONT`. Its sockets stay open, so + /// its peers see a silent connection rather than a disconnect. + #[cfg(unix)] + pub async fn set_paused(&self, paused: bool) { + let pid = self.process.as_ref().and_then(Child::id).expect("dashd is not running"); + let signal = if paused { + "-STOP" + } else { + "-CONT" + }; + let status = Command::new("kill") + .args([signal, &pid.to_string()]) + .status() + .await + .expect("failed to run kill"); + assert!(status.success(), "kill {signal} {pid} failed: {status}"); + tracing::info!("Set paused={} on dashd", paused); + } + /// Enable or disable all P2P network activity on this node. pub fn set_network_active(&self, active: bool) { let client = self.rpc_client(); diff --git a/dash-spv/tests/dashd_sync/main.rs b/dash-spv/tests/dashd_sync/main.rs index a22afc967..52ba1e26f 100644 --- a/dash-spv/tests/dashd_sync/main.rs +++ b/dash-spv/tests/dashd_sync/main.rs @@ -11,4 +11,7 @@ mod tests_mempool; mod tests_multi_wallet; mod tests_node_start; mod tests_restart; +// SIGSTOP freezes dashd, and Windows has no equivalent. +#[cfg(unix)] +mod tests_stall; mod tests_transaction; diff --git a/dash-spv/tests/dashd_sync/tests_stall.rs b/dash-spv/tests/dashd_sync/tests_stall.rs new file mode 100644 index 000000000..9e75ad6b4 --- /dev/null +++ b/dash-spv/tests/dashd_sync/tests_stall.rs @@ -0,0 +1,55 @@ +use std::time::Duration; + +use dash_spv::network::NetworkEvent; +use dash_spv::test_utils::{TestChain, SYNC_TIMEOUT}; +use dash_spv::SyncEvent; +use tokio::sync::broadcast::error::TryRecvError; + +use super::helpers::{is_progress_event, wait_for_sync}; +use super::setup::TestContext; + +/// Verify a peer that goes silent mid-sync, as when our own link stalls, stays connected. +/// +/// `SIGSTOP` freezes dashd with its socket open for well over a request timeout, but +/// under the bound for holding a silent peer. The client must keep its only peer through +/// the freeze, and sync must complete once dashd resumes. +#[tokio::test] +async fn test_sync_keeps_a_peer_frozen_mid_sync() { + let Some(ctx) = TestContext::new(TestChain::Full).await else { + return; + }; + let mut client_handle = ctx.spawn_new_client().await; + + let mut progress_events = 0; + tokio::time::timeout(SYNC_TIMEOUT, async { + while progress_events < 5 { + match client_handle.sync_event_receiver.recv().await { + Ok(ref event) if is_progress_event(event) => progress_events += 1, + Ok(SyncEvent::SyncComplete { + .. + }) => panic!("sync completed before dashd could be frozen"), + Ok(_) => {} + Err(e) => panic!("sync events lost: {e}"), + } + } + }) + .await + .expect("Timeout waiting for sync progress"); + + ctx.dashd.node.set_paused(true).await; + tokio::time::sleep(Duration::from_secs(30)).await; + ctx.dashd.node.set_paused(false).await; + + wait_for_sync(&mut client_handle.progress_receiver, ctx.dashd.initial_height).await; + loop { + match client_handle.network_event_receiver.try_recv() { + Ok(NetworkEvent::PeerDisconnected(addr)) => panic!("peer {addr} was dropped"), + Ok(_) => {} + Err(TryRecvError::Empty) => break, + Err(e) => panic!("network events lost: {e}"), + } + } + + client_handle.stop().await; + ctx.assert_synced(&client_handle.client.progress().await).await; +}