diff --git a/dash-spv/src/network/broker.rs b/dash-spv/src/network/broker.rs index f111931bd..7cbbb3ca9 100644 --- a/dash-spv/src/network/broker.rs +++ b/dash-spv/src/network/broker.rs @@ -12,6 +12,10 @@ use crate::network::costs::ResponseCosts; use crate::network::peer::{PeerHandle, PeerId}; const KICKING_ATTEMPTS: u32 = 2; +/// How many timeouts a request lives on bytes of a message still arriving, without an +/// answer. A big answer on a slow link must fit, but a message that never finishes must +/// not keep its request alive forever. +const ANSWER_TIMEOUTS: u32 = 6; const HEDGE_AFTER: Duration = Duration::from_secs(2); const MIN_IN_FLIGHT_PER_CLASS: usize = 4; const MSG_CLASSES: usize = 5; @@ -62,8 +66,12 @@ struct OnWire { /// The connection, not the address: a late close of an old connection must not /// pull back the work of a new one to the same address. peer: PeerId, - /// Refreshed by every piece of a streaming response. + /// Refreshed by every answer and by bytes of a message still arriving, since a big + /// answer can take longer than a timeout to arrive. last_progress: Instant, + /// Refreshed only by answers, so a block still arriving slowly can be hedged and a + /// message that never finishes cannot keep the request alive forever. + last_answer: Instant, msg: NetworkMessage, height: u32, hedge: Option, @@ -205,18 +213,33 @@ impl Broker { /// Peers answer in order, so any answer is progress on all of its requests. pub async fn progress(&self, peer: PeerId) { + self.refresh(&[peer], true).await; + } + + /// Bytes of a message still arriving from any of `peers` are progress on their requests. + pub async fn receiving(&self, peers: &[PeerId]) { + if !peers.is_empty() { + self.refresh(peers, false).await; + } + } + + async fn refresh(&self, peers: &[PeerId], answered: bool) { let now = Instant::now(); for state in self.requests.lock().await.values_mut() { if let ReqState::OnWire(o) = state { - if o.peer == peer { + if peers.contains(&o.peer) { o.last_progress = now; + if answered { + o.last_answer = now; + } } } } } - /// Returns the peers to kick for a stalled request. A request that already - /// stalled [`KICKING_ATTEMPTS`] times is retried without blaming its peer. + /// Returns the peers to kick for a stalled request. A request stalls after a timeout + /// without progress, or after [`ANSWER_TIMEOUTS`] timeouts without an answer. A request + /// that already stalled [`KICKING_ATTEMPTS`] times is retried without blaming its peer. pub async fn expire(&self, timeout: impl Fn(&PeerId) -> Duration) -> HashSet { let now = Instant::now(); let mut culprits = HashSet::new(); @@ -225,7 +248,10 @@ impl Broker { let ReqState::OnWire(o) = state else { continue; }; - if now.duration_since(o.last_progress) <= timeout(&o.peer) { + let timeout = timeout(&o.peer); + if now.duration_since(o.last_progress) <= timeout + && now.duration_since(o.last_answer) <= timeout * ANSWER_TIMEOUTS + { continue; } if o.attempts < KICKING_ATTEMPTS { @@ -317,6 +343,7 @@ impl Broker { msg: msg.clone(), height, last_progress: Instant::now(), + last_answer: Instant::now(), hedge: None, attempts: attempts + 1, })); @@ -337,7 +364,7 @@ impl Broker { sent } - /// Endgame: asks a second peer for the lowest block on the wire once its peer delivered + /// Endgame: asks a second peer for the lowest block on the wire once its peer answered /// nothing for [`HEDGE_AFTER`], and returns its height. Blocks are processed in height /// order, so only the lowest one holds up progress and a second copy of any other is wasted. pub async fn hedge(&self, peers: &[PeerHandle]) -> Option { @@ -352,7 +379,7 @@ impl Broker { _ => None, }) .min_by_key(|(_, o)| o.height)?; - if o.hedge.is_some() || o.last_progress.elapsed() < HEDGE_AFTER { + if o.hedge.is_some() || o.last_answer.elapsed() < HEDGE_AFTER { return None; } let peer = peers @@ -378,6 +405,7 @@ impl Broker { let on_wire = OnWire { peer, last_progress: Instant::now(), + last_answer: Instant::now(), msg: msg.clone(), height: 0, hedge: None, @@ -387,6 +415,14 @@ impl Broker { } } + #[cfg(test)] + pub async fn carrier_of(&self, key: &RequestKey) -> Option { + match self.requests.lock().await.get(key) { + Some(ReqState::OnWire(o)) => Some(o.peer), + _ => None, + } + } + #[cfg(test)] pub async fn state_of(&self, key: &RequestKey) -> Option<&'static str> { self.requests.lock().await.get(key).map(|s| match s { @@ -766,8 +802,9 @@ mod tests { tokio::time::advance(HEDGE_AFTER / 2).await; assert_eq!(broker.hedge(&peers).await, None, "the slow peer is still delivering"); + broker.receiving(&[slow[0].id()]).await; tokio::time::advance(HEDGE_AFTER / 2).await; - assert_eq!(broker.hedge(&peers).await, Some(10)); + assert_eq!(broker.hedge(&peers).await, Some(10), "a block still arriving is hedged"); assert_eq!(broker.hedge(&peers).await, None, "later blocks wait for the lowest"); assert_eq!(hedged_blocks().await, [10]); diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 90e479c6f..107075df5 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1055,19 +1055,26 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) // bytes are freed on the peer separately, so a wedged peer can sit at its cap // with no registry entries. Pending work plus a byte counter // frozen for REQUEST_TIMEOUT means stuck, whatever the registry thinks. - let mut culprits: HashSet = { + let (mut culprits, streaming) = { let peers = connected.read().await; let live: HashSet = peers.iter().map(|p| p.id()).collect(); progress.retain(|addr, _| live.contains(addr)); let mut culprits = HashSet::new(); + let mut streaming = Vec::new(); for peer in peers.iter() { let addr = peer.id(); let bytes = peer.bytes_read(); let in_flight = peer.in_flight_bytes(); let entry = progress.entry(addr).or_insert((bytes, now)); + let grew = bytes > entry.0; + // Small unrelated messages arrive whole, so only bytes of a message + // still arriving are progress on a request. + if grew && peer.partial_message() { + streaming.push(addr); + } // An idle peer owes no bytes, so its silence before it gets work // must not count against its first request. - if bytes > entry.0 || in_flight == 0 { + if grew || in_flight == 0 { *entry = (bytes, now); } if in_flight > 0 && now.duration_since(entry.1) > request_timeout(peer.lag_ms()) @@ -1075,8 +1082,9 @@ fn spawn_timeout_monitor(state: Arc, shutdown: CancellationToken) culprits.insert(addr); } } - culprits + (culprits, streaming) }; + broker.receiving(&streaming).await; // A request that stopped making progress also condemns its peer: the // per-peer byte check above sees unrelated traffic and judges it healthy. let timeouts: HashMap = connected @@ -1323,10 +1331,13 @@ mod tests { use super::*; use crate::client::ClientConfig; use crate::test_utils::test_socket_address; + use dashcore::consensus::encode::serialize; + use dashcore::network::message::RawNetworkMessage; use dashcore::network::message_blockdata::Inventory; use dashcore::network::message_filter::{CFilter, GetCFilters}; - use dashcore::BlockHash; + use dashcore::{BlockHash, Network}; use test_case::test_case; + use tokio::io::AsyncWriteExt; use tokio::task; use tokio::time; @@ -1352,6 +1363,13 @@ mod tests { PeerNetworkManager::new(&config).await } + /// Lets the spawned tasks run what the last time advance made due. + async fn settle() { + for _ in 0..4 { + tokio::task::yield_now().await; + } + } + /// The events received until none arrives for a while. async fn drain(events: &mut broadcast::Receiver) -> Vec { let mut received = Vec::new(); @@ -1546,40 +1564,80 @@ mod tests { assert_eq!(net.connected_count().await, 0); } - #[test_case(None, true; "a silent request")] - #[test_case(Some(1), false; "the carrier streaming keeps it alive")] - #[test_case(Some(2), true; "another peer streaming is not progress")] + enum Feed { + Nothing, + /// A complete answer from this peer every second. + Answers(u8), + /// More bytes of one long message from this peer every second. + Streams(u8), + /// Small complete messages, such as pings, from this peer every second. + Chatters(u8), + } + + /// Peer 1 carries a request for `secs` seconds beside idle peer 2. + #[test_case(Feed::Nothing, 11, true; "a silent request")] + #[test_case(Feed::Answers(1), 20, false; "the carrier answering keeps it alive")] + #[test_case(Feed::Answers(2), 11, true; "another peer answering is not progress")] + #[test_case(Feed::Streams(1), 55, false; "the carrier streaming one long answer keeps it alive")] + #[test_case(Feed::Streams(1), 61, true; "the carrier streaming one answer past the bound stalls")] + #[test_case(Feed::Streams(2), 11, true; "another peer streaming is not progress")] + #[test_case(Feed::Chatters(1), 11, true; "the carrier sending only small messages stalls")] #[tokio::test(start_paused = true)] - async fn the_timeout_monitor_requeues_stalled_requests(streamer: Option, stalls: bool) { + async fn the_timeout_monitor_requeues_stalled_requests(feed: Feed, secs: u64, stalls: bool) { let net = broker().await; + let session = net.session.lock().await.clone(); + let mut far_ends = Vec::new(); + for n in 1..=2 { + let (peer, far) = ConnectedPeer::for_test_reading(conn(n, 0), net.state.clone()).await; + far_ends.push(far); + let admitted = + net.state.connected.admit(peer, &session, net.state.max_peers, None).await; + assert!(matches!(admitted, Admission::Joined)); + } net.state.broker.mark_on_wire(&get_cfilters(0), conn(1, 0)).await; - - // Two gaps that each stay under the deadline but together exceed it. - for _ in 0..2 { - tokio::time::advance(REQUEST_TIMEOUT - TIMEOUT_CHECK).await; - tokio::task::yield_now().await; - if let Some(peer) = streamer { - let piece = NetworkMessage::CFilter(CFilter { - filter_type: 0, - block_hash: BlockHash::dummy(1), - filter: vec![0u8; 4], - }); - net.state - .inbound - .send(PeerEvent::Message(conn(peer, 0), piece.into())) - .expect("the pump is running"); + let framed = |payload| { + serialize(&RawNetworkMessage { + magic: Network::Regtest.magic(), + payload, + }) + }; + let long_answer = framed(NetworkMessage::CFilter(CFilter { + filter_type: 0, + block_hash: BlockHash::dummy(1), + filter: vec![0; 100_000], + })); + let ping = framed(NetworkMessage::Ping(1)); + + for second in 0..secs as usize { + tokio::time::advance(TIMEOUT_CHECK).await; + match feed { + Feed::Nothing => {} + Feed::Answers(peer) => { + let piece = NetworkMessage::CFilter(CFilter { + filter_type: 0, + block_hash: BlockHash::dummy(1), + filter: vec![0u8; 4], + }); + net.state + .inbound + .send(PeerEvent::Message(conn(peer, 0), piece.into())) + .expect("the pump is running"); + } + // A kicked peer's socket is closed, so a write after the kick fails. + Feed::Streams(peer) => { + let piece = &long_answer[second * 1000..(second + 1) * 1000]; + let _ = far_ends[peer as usize - 1].write_all(piece).await; + } + Feed::Chatters(peer) => { + let _ = far_ends[peer as usize - 1].write_all(&ping).await; + } } - tokio::task::yield_now().await; - tokio::task::yield_now().await; + settle().await; } - let want = if stalls { - "queued" - } else { - "on_wire" - }; - assert_eq!(net.state.broker.state_of(&RequestKey::CFilters(0)).await, Some(want)); - assert_eq!(net.state.broker.len(), stalls as usize); + // A stalled request is re-queued and may already be routed to peer 2. + let carrier = net.state.broker.carrier_of(&RequestKey::CFilters(0)).await; + assert_eq!(carrier == Some(conn(1, 0)), !stalls, "carrier {carrier:?}"); } /// A timeout kick re-queues the stalled peer's work only once that peer is closed and out diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index c68a35692..7da8a886b 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -17,7 +17,7 @@ use dashcore::{ use futures::lock::Mutex; use rand::Rng; use std::net::SocketAddr; -use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; #[cfg(test)] @@ -137,6 +137,23 @@ impl Decoder for SizedCodec { } } +/// Frames messages and records whether the bytes of an unfinished one are buffered. +struct TrackedCodec { + codec: SizedCodec, + partial: Arc, +} + +impl Decoder for TrackedCodec { + type Item = (RawNetworkMessage, u64); + type Error = encode::Error; + + fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { + let frame = self.codec.decode(src); + self.partial.store(!src.is_empty(), Ordering::Relaxed); + frame + } +} + /// The socket's write half, framing each message for the connection's transport. struct PeerWriter { half: OwnedWriteHalf, @@ -172,7 +189,30 @@ impl PeerWriter { /// the peer's version packet. type LinkReader = Either, OwnedReadHalf>; -type PeerReader = FramedRead, SizedCodec>; +type PeerReader = FramedRead, TrackedCodec>; + +/// Returns a connection's reader with the counters the peer reads: the bytes read from +/// the socket, and whether the bytes of an unfinished message are buffered. +fn peer_reader( + reader: LinkReader, + codec: SizedCodec, + state: Arc, +) -> (PeerReader, Arc, Arc) { + let peer_bytes = Arc::new(AtomicU64::new(0)); + let partial = Arc::new(AtomicBool::new(false)); + let reader = FramedRead::new( + CountingReader { + inner: reader, + state, + peer_bytes: peer_bytes.clone(), + }, + TrackedCodec { + codec, + partial: partial.clone(), + }, + ); + (reader, peer_bytes, partial) +} /// A TCP connection with its transport negotiated, ready for the version handshake. struct Link { @@ -304,6 +344,7 @@ pub struct ConnectedPeer { version: VersionMessage, lag_ms: AtomicU32, bytes: Arc, + partial: Arc, } pub struct DisconnectedPeer { @@ -346,6 +387,11 @@ impl ConnectedPeer { self.bytes.load(Ordering::Relaxed) } + /// Whether part of a message has arrived and the rest has not. + pub fn partial_message(&self) -> bool { + self.partial.load(Ordering::Relaxed) + } + pub fn in_flight_bytes(&self) -> u64 { self.handle.in_flight_bytes() } @@ -370,29 +416,73 @@ impl ConnectedPeer { /// The far end discards what it reads, so sends succeed. #[cfg(test)] pub(crate) async fn for_test(id: PeerId) -> Self { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind loopback"); - let addr = listener.local_addr().expect("bound address"); - let stream = TcpStream::connect(addr).await.expect("connect loopback"); - let (mut far, _) = listener.accept().await.expect("accept loopback"); + let (link, mut far) = test_link().await; tokio::spawn(async move { io::copy(&mut far, &mut io::sink()).await }); - let link = Link::v1(stream, Network::Regtest.magic()); + let ledger = Arc::new(Ledger::new(Arc::new(ResponseCosts::new()))); + Self::from_test_parts( + id, + Arc::new(Mutex::new(link.writer)), + ledger, + (Arc::new(AtomicU64::new(0)), Arc::new(AtomicBool::new(false))), + CancellationToken::new(), + ) + } + + /// A handshaken connection `id` over a loopback socket, its reader started as + /// `connect` starts it. Returns the far end's write half, which feeds the reader. + #[cfg(test)] + pub(crate) async fn for_test_reading( + id: PeerId, + state: Arc, + ) -> (Self, OwnedWriteHalf) { + let (link, far) = test_link().await; + let (mut far_read, far_write) = far.into_split(); + tokio::spawn(async move { io::copy(&mut far_read, &mut io::sink()).await }); + let (reader, bytes, partial) = peer_reader(link.reader, link.codec, state.clone()); + let writer = Arc::new(Mutex::new(link.writer)); + let ledger = Arc::new(Ledger::new(state.costs.clone())); + let token = CancellationToken::new(); + let magic = Network::Regtest.magic(); + spawn_reader(id, magic, reader, writer.clone(), state, token.clone(), ledger.clone()); + (Self::from_test_parts(id, writer, ledger, (bytes, partial), token), far_write) + } + + #[cfg(test)] + fn from_test_parts( + id: PeerId, + writer: Arc>, + ledger: Arc, + (bytes, partial): (Arc, Arc), + token: CancellationToken, + ) -> Self { ConnectedPeer { handle: PeerHandle { id, - writer: Arc::new(Mutex::new(link.writer)), - ledger: Arc::new(Ledger::new(Arc::new(ResponseCosts::new()))), + writer, + ledger, cap: Arc::new(AtomicU64::new(INITIAL_CAP_BYTES)), serving_rate: Arc::new(AtomicU64::new(0)), headers2: false, - token: CancellationToken::new(), + token, }, version: build_version(id.addr(), "test"), lag_ms: AtomicU32::new(1), - bytes: Arc::new(AtomicU64::new(0)), + bytes, + partial, } } } +/// A V1 link over a loopback socket, and the socket's far end. +#[cfg(test)] +async fn test_link() -> (Link, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind loopback"); + let addr = listener.local_addr().expect("bound address"); + let stream = TcpStream::connect(addr).await.expect("connect loopback"); + let (far, _) = listener.accept().await.expect("accept loopback"); + (Link::v1(stream, Network::Regtest.magic()), far) +} + impl DisconnectedPeer { pub fn new(addr: SocketAddr, network: Network) -> Self { DisconnectedPeer { @@ -484,15 +574,7 @@ impl DisconnectedPeer { } else { 1 }; - let peer_bytes = Arc::new(AtomicU64::new(0)); - let mut reader = FramedRead::new( - CountingReader { - inner: reader, - state: state.clone(), - peer_bytes: peer_bytes.clone(), - }, - codec, - ); + let (mut reader, peer_bytes, partial) = peer_reader(reader, codec, state.clone()); handshake_send( &mut writer, @@ -614,6 +696,7 @@ impl DisconnectedPeer { version, lag_ms: AtomicU32::new(lag_ms), bytes: peer_bytes, + partial, }) } } @@ -756,6 +839,8 @@ async fn handshake_send(writer: &mut PeerWriter, payload: NetworkMessage) -> Net mod tests { use std::io::{Error as IoError, ErrorKind}; + use test_case::test_case; + use super::*; fn io_error(kind: ErrorKind) -> ProtocolError { @@ -793,4 +878,25 @@ mod tests { wire_agent(&format!("{}é{}", "a".repeat(MAX_USER_AGENT_LEN - 3), "b".repeat(10))); assert_eq!(straddling, format!("/{}/", "a".repeat(MAX_USER_AGENT_LEN - 3))); } + + /// Whether the codec reports an unfinished message after the first `n` bytes of two + /// 32-byte pings arrive. + #[test_case(1 => true; "part of a header")] + #[test_case(24 => true; "a header without its payload")] + #[test_case(32 => false; "a whole ping")] + #[test_case(40 => true; "a whole ping and part of the next")] + fn the_codec_tracks_an_unfinished_message(n: usize) -> bool { + let magic = Network::Regtest.magic(); + let ping = encode::serialize(&RawNetworkMessage { + magic, + payload: NetworkMessage::Ping(1), + }); + let mut codec = TrackedCodec { + codec: SizedCodec::V1, + partial: Arc::new(AtomicBool::new(false)), + }; + let mut src = BytesMut::from(&[ping.clone(), ping].concat()[..n]); + while codec.decode(&mut src).unwrap().is_some() {} + codec.partial.load(Ordering::Relaxed) + } }