Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions book/src/sdk/builder-pattern.md
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ const DEFAULT_REQUEST_SETTINGS: RequestSettings = RequestSettings {
retries: Some(3),
timeout: None,
ban_failed_address: None,
ban_busy_address: None,
connect_timeout: None,
max_decoding_message_size: None,
};
Expand Down
4 changes: 3 additions & 1 deletion packages/rs-dapi-client/src/address_list.rs
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,9 @@ impl AddressList {
/// place, falling back to evicted members if there are too few standbys.
///
/// This is the failover path for callers that disable banning
/// ([RequestSettings::ban_failed_address](crate::RequestSettings)): a
/// ([RequestSettings::ban_failed_address](crate::RequestSettings)), and for a
/// busy node a request asks not to ban
/// ([RequestSettings::ban_busy_address](crate::RequestSettings)): a
/// failing node yields its slot's traffic to an available standby even
/// when it is never banned.
pub fn evict_from_rotation(&self, address: &Address) {
Expand Down
20 changes: 16 additions & 4 deletions packages/rs-dapi-client/src/dapi_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,13 @@ impl CanRetry for DapiClientError {
_ => None,
}
}

fn is_node_busy(&self) -> bool {
match self {
DapiClientError::Transport(te) => te.is_node_busy(),
_ => false,
}
}
}

/// Serialization of [DapiClientError].
Expand Down Expand Up @@ -231,7 +238,11 @@ pub fn update_address_ban_status<R, E>(
Err(error) => {
if error.can_retry() {
if let Some(address) = error.address.as_ref() {
if applied_settings.ban_failed_address {
// A node that says it is busy is working, and a request can ask that it
// is not banned for saying so.
let ban = applied_settings.ban_failed_address
&& (applied_settings.ban_busy_address || !error.is_node_busy());
if ban {
let reason = Some(error.to_string());
let period_opt = error.rate_limit_ban_duration();
let banned = match period_opt {
Expand Down Expand Up @@ -266,14 +277,14 @@ pub fn update_address_ban_status<R, E>(
);
}
} else {
// Banning is disabled for this request, but failover
// must still move traffic away from the failing node:
// The request does not ban a node for this failure, but
// failover must still move traffic away from the failing node:
// drop it from the sticky rotation, ban state untouched.
address_list.evict_from_rotation(address);
tracing::debug!(
?error,
?address,
"banning is disabled; evicted address {address} from rotation due to the error"
"not banning for this error; evicted address {address} from rotation"
);
}
} else {
Expand Down Expand Up @@ -302,6 +313,7 @@ mod tests {
timeout: Duration::from_secs(10),
retries: 5,
ban_failed_address: ban,
ban_busy_address: true,
max_decoding_message_size: None,
#[cfg(not(target_arch = "wasm32"))]
ca_certificate: None,
Expand Down
4 changes: 4 additions & 0 deletions packages/rs-dapi-client/src/executor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ impl<E: CanRetry> CanRetry for ExecutionError<E> {
fn rate_limit_ban_duration(&self) -> Option<std::time::Duration> {
self.inner.rate_limit_ban_duration()
}

fn is_node_busy(&self) -> bool {
self.inner.is_node_busy()
}
}

/// Request execution response.
Expand Down
9 changes: 9 additions & 0 deletions packages/rs-dapi-client/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,15 @@ pub trait CanRetry {
None
}

/// Returns true if the node answered that it is busy or rate limited (gRPC
/// `ResourceExhausted`), or that what was asked is not ready yet (gRPC `Unavailable`
/// with a `retry-after` header): it is working, and only cannot serve this for now.
/// A request can ask that a node is not banned for it, see
/// [RequestSettings::ban_busy_address].
fn is_node_busy(&self) -> bool {
false
}

/// Get boolean flag that indicates if the error is retryable.
///
/// Deprecated in favor of [CanRetry::can_retry].
Expand Down
26 changes: 26 additions & 0 deletions packages/rs-dapi-client/src/request_settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ const DEFAULT_CONNECT_TIMEOUT: Option<Duration> = None;
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
const DEFAULT_RETRIES: usize = 5;
const DEFAULT_BAN_FAILED_ADDRESS: bool = true;
const DEFAULT_BAN_BUSY_ADDRESS: bool = true;

/// DAPI request settings.
///
Expand Down Expand Up @@ -43,6 +44,16 @@ pub struct RequestSettings {
pub retries: Option<usize>,
/// Ban DAPI address if node not responded or responded with error.
pub ban_failed_address: Option<bool>,
/// Ban a DAPI address whose node answered that it is busy or rate limited (gRPC
/// `ResourceExhausted`), or that what was asked is not ready yet (gRPC `Unavailable`
/// with a `retry-after` header).
///
/// Such a node is working. A request to a method with a tight limit of its own sets this
/// to `false`: the request then moves to another node, and this one stays available to
/// every other request. Every other failure is still banned as
/// [ban_failed_address](Self::ban_failed_address) says, which also turns this off when
/// it is `false`.
pub ban_busy_address: Option<bool>,
/// Maximum gRPC response size in bytes (decoding limit).
pub max_decoding_message_size: Option<usize>,
}
Expand All @@ -56,6 +67,7 @@ impl RequestSettings {
timeout: None,
retries: None,
ban_failed_address: None,
ban_busy_address: None,
max_decoding_message_size: None,
}
}
Expand All @@ -69,6 +81,7 @@ impl RequestSettings {
timeout: rhs.timeout.or(self.timeout),
retries: rhs.retries.or(self.retries),
ban_failed_address: rhs.ban_failed_address.or(self.ban_failed_address),
ban_busy_address: rhs.ban_busy_address.or(self.ban_busy_address),
max_decoding_message_size: rhs
.max_decoding_message_size
.or(self.max_decoding_message_size),
Expand All @@ -84,6 +97,7 @@ impl RequestSettings {
ban_failed_address: self
.ban_failed_address
.unwrap_or(DEFAULT_BAN_FAILED_ADDRESS),
ban_busy_address: self.ban_busy_address.unwrap_or(DEFAULT_BAN_BUSY_ADDRESS),
max_decoding_message_size: self.max_decoding_message_size,
#[cfg(not(target_arch = "wasm32"))]
ca_certificate: None,
Expand All @@ -108,6 +122,9 @@ pub struct AppliedRequestSettings {
pub retries: usize,
/// Ban DAPI address if node not responded or responded with error.
pub ban_failed_address: bool,
/// Ban a DAPI address whose node answered that it is busy or rate limited; see
/// [RequestSettings::ban_busy_address].
pub ban_busy_address: bool,
/// Maximum gRPC response size in bytes (decoding limit).
pub max_decoding_message_size: Option<usize>,
/// Certificate Authority certificate to use for verifying the server's certificate.
Expand Down Expand Up @@ -180,6 +197,7 @@ impl AppliedRequestSettings {
timeout: _,
retries: _,
ban_failed_address: _,
ban_busy_address: _,
max_decoding_message_size,
#[cfg(not(target_arch = "wasm32"))]
ca_certificate,
Expand Down Expand Up @@ -233,6 +251,7 @@ mod tests {
retries: Some(3),
connect_timeout: Some(Duration::from_secs(2)),
ban_failed_address: Some(true),
ban_busy_address: Some(false),
max_decoding_message_size: Some(1024),
};

Expand All @@ -242,6 +261,7 @@ mod tests {
retries: None,
connect_timeout: None,
ban_failed_address: None,
ban_busy_address: None,
max_decoding_message_size: None,
};

Expand All @@ -250,6 +270,7 @@ mod tests {
assert_eq!(result.retries, Some(3)); // preserved from base
assert_eq!(result.connect_timeout, Some(Duration::from_secs(2))); // preserved
assert_eq!(result.ban_failed_address, Some(true)); // preserved
assert_eq!(result.ban_busy_address, Some(false)); // preserved
assert_eq!(result.max_decoding_message_size, Some(1024)); // preserved
}

Expand All @@ -260,6 +281,7 @@ mod tests {
retries: Some(3),
connect_timeout: None,
ban_failed_address: None,
ban_busy_address: None,
max_decoding_message_size: None,
};

Expand All @@ -277,6 +299,7 @@ mod tests {
assert_eq!(applied.timeout, Duration::from_secs(10));
assert_eq!(applied.retries, 5);
assert!(applied.ban_failed_address);
assert!(applied.ban_busy_address);
assert!(applied.max_decoding_message_size.is_none());
}

Expand All @@ -287,6 +310,7 @@ mod tests {
timeout: Some(Duration::from_secs(30)),
retries: Some(10),
ban_failed_address: Some(false),
ban_busy_address: Some(false),
max_decoding_message_size: Some(4096),
};

Expand All @@ -295,6 +319,7 @@ mod tests {
assert_eq!(applied.timeout, Duration::from_secs(30));
assert_eq!(applied.retries, 10);
assert!(!applied.ban_failed_address);
assert!(!applied.ban_busy_address);
assert_eq!(applied.max_decoding_message_size, Some(4096));
}

Expand Down Expand Up @@ -392,6 +417,7 @@ mod tests {
timeout: Some(Duration::from_secs(30)),
retries: Some(1),
ban_failed_address: Some(false),
ban_busy_address: Some(false),
..RequestSettings::default()
}
.finalize();
Expand Down
6 changes: 6 additions & 0 deletions packages/rs-dapi-client/src/transport.rs
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,12 @@ impl CanRetry for TransportError {
TransportError::Grpc(status) => status.rate_limit_ban_duration(),
}
}

fn is_node_busy(&self) -> bool {
match self {
TransportError::Grpc(status) => status.is_node_busy(),
}
}
}

/// Serialization of [TransportError].
Expand Down
21 changes: 19 additions & 2 deletions packages/rs-dapi-client/src/transport/grpc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,19 @@ impl CanRetry for dapi_grpc::tonic::Status {
secs.clamp(MIN_RATE_LIMIT_BAN_SECS, MAX_RATE_LIMIT_BAN_SECS),
))
}

fn is_node_busy(&self) -> bool {
use dapi_grpc::tonic::Code;

match self.code() {
Code::ResourceExhausted => true,
// A node that asks to be tried again later is working: what was asked is not
// ready yet, such as the results of a block rs-dapi's node has just stored.
// A node that is down, or a proxy in front of one, sends no such header.
Code::Unavailable => self.metadata().contains_key("retry-after"),
_ => false,
}
}
}

/// Macro to implement the `TransportRequest` trait for a given request type, response type, client type, and settings.
Expand Down Expand Up @@ -563,6 +576,7 @@ impl_transport_request_grpc!(
timeout: Some(Duration::from_secs(30)),
retries: Some(3),
ban_failed_address: None,
ban_busy_address: None,
connect_timeout: None,
max_decoding_message_size: None,
},
Expand Down Expand Up @@ -800,7 +814,8 @@ impl_transport_request_grpc!(
timeout: Some(LARGE_RESPONSE_TIMEOUT),
// rs-dapi refuses this method when too many are in progress. A busy node is not a
// failed one, and a ban would take it away from the client's other requests too.
ban_failed_address: Some(false),
// A node that fails in any other way is still banned.
ban_busy_address: Some(false),
..RequestSettings::default()
},
get_owner_transactions_in_block
Expand All @@ -818,7 +833,8 @@ impl_transport_request_grpc!(
timeout: Some(LARGE_RESPONSE_TIMEOUT),
// The gateway limits this method far below every other one. A ban lasts until the
// limit resets and would take the node away from the client's other requests too.
ban_failed_address: Some(false),
// A node that fails in any other way is still banned.
ban_busy_address: Some(false),
..RequestSettings::default()
Comment thread
QuantumExplorer marked this conversation as resolved.
Resolved
},
get_blocks
Expand Down Expand Up @@ -1008,6 +1024,7 @@ impl_transport_request_grpc!(
RequestSettings {
timeout: Some(STREAMING_TIMEOUT),
ban_failed_address: None,
ban_busy_address: None,
connect_timeout: None,
retries: None,
max_decoding_message_size: None,
Expand Down
Loading
Loading