Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 45 additions & 8 deletions dash-spv/src/network/broker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ use crate::network::costs::ResponseCosts;
use crate::network::peer::{PeerHandle, PeerId};

const KICKING_ATTEMPTS: u32 = 2;
/// How many timeouts a request lives on bytes of a message still arriving, without an
/// answer. A big answer on a slow link must fit, but a message that never finishes must
/// not keep its request alive forever.
const ANSWER_TIMEOUTS: u32 = 6;
const HEDGE_AFTER: Duration = Duration::from_secs(2);
const MIN_IN_FLIGHT_PER_CLASS: usize = 4;
const MSG_CLASSES: usize = 5;
Expand Down Expand Up @@ -62,8 +66,12 @@ struct OnWire {
/// The connection, not the address: a late close of an old connection must not
/// pull back the work of a new one to the same address.
peer: PeerId,
/// Refreshed by every piece of a streaming response.
/// Refreshed by every answer and by bytes of a message still arriving, since a big
/// answer can take longer than a timeout to arrive.
last_progress: Instant,
/// Refreshed only by answers, so a block still arriving slowly can be hedged and a
/// message that never finishes cannot keep the request alive forever.
last_answer: Instant,
msg: NetworkMessage,
height: u32,
hedge: Option<PeerId>,
Expand Down Expand Up @@ -205,18 +213,33 @@ impl Broker {

/// Peers answer in order, so any answer is progress on all of its requests.
pub async fn progress(&self, peer: PeerId) {
self.refresh(&[peer], true).await;
}

/// Bytes of a message still arriving from any of `peers` are progress on their requests.
pub async fn receiving(&self, peers: &[PeerId]) {
if !peers.is_empty() {
self.refresh(peers, false).await;
}
}

async fn refresh(&self, peers: &[PeerId], answered: bool) {
let now = Instant::now();
for state in self.requests.lock().await.values_mut() {
if let ReqState::OnWire(o) = state {
if o.peer == peer {
if peers.contains(&o.peer) {
o.last_progress = now;
if answered {
o.last_answer = now;
}
}
}
}
}

/// Returns the peers to kick for a stalled request. A request that already
/// stalled [`KICKING_ATTEMPTS`] times is retried without blaming its peer.
/// Returns the peers to kick for a stalled request. A request stalls after a timeout
/// without progress, or after [`ANSWER_TIMEOUTS`] timeouts without an answer. A request
/// that already stalled [`KICKING_ATTEMPTS`] times is retried without blaming its peer.
pub async fn expire(&self, timeout: impl Fn(&PeerId) -> Duration) -> HashSet<PeerId> {
let now = Instant::now();
let mut culprits = HashSet::new();
Expand All @@ -225,7 +248,10 @@ impl Broker {
let ReqState::OnWire(o) = state else {
continue;
};
if now.duration_since(o.last_progress) <= timeout(&o.peer) {
let timeout = timeout(&o.peer);
if now.duration_since(o.last_progress) <= timeout
&& now.duration_since(o.last_answer) <= timeout * ANSWER_TIMEOUTS
{
continue;
}
if o.attempts < KICKING_ATTEMPTS {
Expand Down Expand Up @@ -317,6 +343,7 @@ impl Broker {
msg: msg.clone(),
height,
last_progress: Instant::now(),
last_answer: Instant::now(),
hedge: None,
attempts: attempts + 1,
}));
Expand All @@ -337,7 +364,7 @@ impl Broker {
sent
}

/// Endgame: asks a second peer for the lowest block on the wire once its peer delivered
/// Endgame: asks a second peer for the lowest block on the wire once its peer answered
/// nothing for [`HEDGE_AFTER`], and returns its height. Blocks are processed in height
/// order, so only the lowest one holds up progress and a second copy of any other is wasted.
pub async fn hedge(&self, peers: &[PeerHandle]) -> Option<u32> {
Expand All @@ -352,7 +379,7 @@ impl Broker {
_ => None,
})
.min_by_key(|(_, o)| o.height)?;
if o.hedge.is_some() || o.last_progress.elapsed() < HEDGE_AFTER {
if o.hedge.is_some() || o.last_answer.elapsed() < HEDGE_AFTER {
return None;
}
let peer = peers
Expand All @@ -378,6 +405,7 @@ impl Broker {
let on_wire = OnWire {
peer,
last_progress: Instant::now(),
last_answer: Instant::now(),
msg: msg.clone(),
height: 0,
hedge: None,
Expand All @@ -387,6 +415,14 @@ impl Broker {
}
}

#[cfg(test)]
pub async fn carrier_of(&self, key: &RequestKey) -> Option<PeerId> {
match self.requests.lock().await.get(key) {
Some(ReqState::OnWire(o)) => Some(o.peer),
_ => None,
}
}

#[cfg(test)]
pub async fn state_of(&self, key: &RequestKey) -> Option<&'static str> {
self.requests.lock().await.get(key).map(|s| match s {
Expand Down Expand Up @@ -766,8 +802,9 @@ mod tests {
tokio::time::advance(HEDGE_AFTER / 2).await;
assert_eq!(broker.hedge(&peers).await, None, "the slow peer is still delivering");

broker.receiving(&[slow[0].id()]).await;
tokio::time::advance(HEDGE_AFTER / 2).await;
assert_eq!(broker.hedge(&peers).await, Some(10));
assert_eq!(broker.hedge(&peers).await, Some(10), "a block still arriving is hedged");
assert_eq!(broker.hedge(&peers).await, None, "later blocks wait for the lowest");
assert_eq!(hedged_blocks().await, [10]);

Expand Down
122 changes: 90 additions & 32 deletions dash-spv/src/network/manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1055,28 +1055,36 @@ fn spawn_timeout_monitor(state: Arc<NetworkState>, shutdown: CancellationToken)
// bytes are freed on the peer separately, so a wedged peer can sit at its cap
// with no registry entries. Pending work plus a byte counter
// frozen for REQUEST_TIMEOUT means stuck, whatever the registry thinks.
let mut culprits: HashSet<PeerId> = {
let (mut culprits, streaming) = {
let peers = connected.read().await;
let live: HashSet<PeerId> = peers.iter().map(|p| p.id()).collect();
progress.retain(|addr, _| live.contains(addr));
let mut culprits = HashSet::new();
let mut streaming = Vec::new();
for peer in peers.iter() {
let addr = peer.id();
let bytes = peer.bytes_read();
let in_flight = peer.in_flight_bytes();
let entry = progress.entry(addr).or_insert((bytes, now));
let grew = bytes > entry.0;
// Small unrelated messages arrive whole, so only bytes of a message
// still arriving are progress on a request.
if grew && peer.partial_message() {
streaming.push(addr);
}
// An idle peer owes no bytes, so its silence before it gets work
// must not count against its first request.
if bytes > entry.0 || in_flight == 0 {
if grew || in_flight == 0 {
*entry = (bytes, now);
}
if in_flight > 0 && now.duration_since(entry.1) > request_timeout(peer.lag_ms())
{
culprits.insert(addr);
}
}
culprits
(culprits, streaming)
};
broker.receiving(&streaming).await;
// A request that stopped making progress also condemns its peer: the
// per-peer byte check above sees unrelated traffic and judges it healthy.
let timeouts: HashMap<PeerId, Duration> = connected
Expand Down Expand Up @@ -1323,10 +1331,13 @@ mod tests {
use super::*;
use crate::client::ClientConfig;
use crate::test_utils::test_socket_address;
use dashcore::consensus::encode::serialize;
use dashcore::network::message::RawNetworkMessage;
use dashcore::network::message_blockdata::Inventory;
use dashcore::network::message_filter::{CFilter, GetCFilters};
use dashcore::BlockHash;
use dashcore::{BlockHash, Network};
use test_case::test_case;
use tokio::io::AsyncWriteExt;
use tokio::task;
use tokio::time;

Expand All @@ -1352,6 +1363,13 @@ mod tests {
PeerNetworkManager::new(&config).await
}

/// Lets the spawned tasks run what the last time advance made due.
async fn settle() {
for _ in 0..4 {
tokio::task::yield_now().await;
}
}

/// The events received until none arrives for a while.
async fn drain(events: &mut broadcast::Receiver<NetworkEvent>) -> Vec<NetworkEvent> {
let mut received = Vec::new();
Expand Down Expand Up @@ -1546,40 +1564,80 @@ mod tests {
assert_eq!(net.connected_count().await, 0);
}

#[test_case(None, true; "a silent request")]
#[test_case(Some(1), false; "the carrier streaming keeps it alive")]
#[test_case(Some(2), true; "another peer streaming is not progress")]
enum Feed {
Nothing,
/// A complete answer from this peer every second.
Answers(u8),
/// More bytes of one long message from this peer every second.
Streams(u8),
/// Small complete messages, such as pings, from this peer every second.
Chatters(u8),
}

/// Peer 1 carries a request for `secs` seconds beside idle peer 2.
#[test_case(Feed::Nothing, 11, true; "a silent request")]
#[test_case(Feed::Answers(1), 20, false; "the carrier answering keeps it alive")]
#[test_case(Feed::Answers(2), 11, true; "another peer answering is not progress")]
#[test_case(Feed::Streams(1), 55, false; "the carrier streaming one long answer keeps it alive")]
#[test_case(Feed::Streams(1), 61, true; "the carrier streaming one answer past the bound stalls")]
#[test_case(Feed::Streams(2), 11, true; "another peer streaming is not progress")]
#[test_case(Feed::Chatters(1), 11, true; "the carrier sending only small messages stalls")]
#[tokio::test(start_paused = true)]
async fn the_timeout_monitor_requeues_stalled_requests(streamer: Option<u8>, stalls: bool) {
async fn the_timeout_monitor_requeues_stalled_requests(feed: Feed, secs: u64, stalls: bool) {
let net = broker().await;
let session = net.session.lock().await.clone();
let mut far_ends = Vec::new();
for n in 1..=2 {
let (peer, far) = ConnectedPeer::for_test_reading(conn(n, 0), net.state.clone()).await;
far_ends.push(far);
let admitted =
net.state.connected.admit(peer, &session, net.state.max_peers, None).await;
assert!(matches!(admitted, Admission::Joined));
}
net.state.broker.mark_on_wire(&get_cfilters(0), conn(1, 0)).await;

// Two gaps that each stay under the deadline but together exceed it.
for _ in 0..2 {
tokio::time::advance(REQUEST_TIMEOUT - TIMEOUT_CHECK).await;
tokio::task::yield_now().await;
if let Some(peer) = streamer {
let piece = NetworkMessage::CFilter(CFilter {
filter_type: 0,
block_hash: BlockHash::dummy(1),
filter: vec![0u8; 4],
});
net.state
.inbound
.send(PeerEvent::Message(conn(peer, 0), piece.into()))
.expect("the pump is running");
let framed = |payload| {
serialize(&RawNetworkMessage {
magic: Network::Regtest.magic(),
payload,
})
};
let long_answer = framed(NetworkMessage::CFilter(CFilter {
filter_type: 0,
block_hash: BlockHash::dummy(1),
filter: vec![0; 100_000],
}));
let ping = framed(NetworkMessage::Ping(1));

for second in 0..secs as usize {
tokio::time::advance(TIMEOUT_CHECK).await;
match feed {
Feed::Nothing => {}
Feed::Answers(peer) => {
let piece = NetworkMessage::CFilter(CFilter {
filter_type: 0,
block_hash: BlockHash::dummy(1),
filter: vec![0u8; 4],
});
net.state
.inbound
.send(PeerEvent::Message(conn(peer, 0), piece.into()))
.expect("the pump is running");
}
// A kicked peer's socket is closed, so a write after the kick fails.
Feed::Streams(peer) => {
let piece = &long_answer[second * 1000..(second + 1) * 1000];
let _ = far_ends[peer as usize - 1].write_all(piece).await;
}
Feed::Chatters(peer) => {
let _ = far_ends[peer as usize - 1].write_all(&ping).await;
}
}
tokio::task::yield_now().await;
tokio::task::yield_now().await;
settle().await;
}

let want = if stalls {
"queued"
} else {
"on_wire"
};
assert_eq!(net.state.broker.state_of(&RequestKey::CFilters(0)).await, Some(want));
assert_eq!(net.state.broker.len(), stalls as usize);
// A stalled request is re-queued and may already be routed to peer 2.
let carrier = net.state.broker.carrier_of(&RequestKey::CFilters(0)).await;
assert_eq!(carrier == Some(conn(1, 0)), !stalls, "carrier {carrier:?}");
}

/// A timeout kick re-queues the stalled peer's work only once that peer is closed and out
Expand Down
Loading
Loading