diff --git a/book/src/sdk/builder-pattern.md b/book/src/sdk/builder-pattern.md index a3034f6cbf4..6ff6e817ebe 100644 --- a/book/src/sdk/builder-pattern.md +++ b/book/src/sdk/builder-pattern.md @@ -271,6 +271,7 @@ const DEFAULT_REQUEST_SETTINGS: RequestSettings = RequestSettings { retries: Some(3), timeout: None, ban_failed_address: None, + ban_busy_address: None, connect_timeout: None, max_decoding_message_size: None, }; diff --git a/packages/rs-dapi-client/src/address_list.rs b/packages/rs-dapi-client/src/address_list.rs index 037abc98f1e..45a0456b34c 100644 --- a/packages/rs-dapi-client/src/address_list.rs +++ b/packages/rs-dapi-client/src/address_list.rs @@ -504,7 +504,9 @@ impl AddressList { /// place, falling back to evicted members if there are too few standbys. /// /// This is the failover path for callers that disable banning - /// ([RequestSettings::ban_failed_address](crate::RequestSettings)): a + /// ([RequestSettings::ban_failed_address](crate::RequestSettings)), and for a + /// busy node a request asks not to ban + /// ([RequestSettings::ban_busy_address](crate::RequestSettings)): a /// failing node yields its slot's traffic to an available standby even /// when it is never banned. pub fn evict_from_rotation(&self, address: &Address) { diff --git a/packages/rs-dapi-client/src/dapi_client.rs b/packages/rs-dapi-client/src/dapi_client.rs index d4b69e4dbf3..2d0fe426245 100644 --- a/packages/rs-dapi-client/src/dapi_client.rs +++ b/packages/rs-dapi-client/src/dapi_client.rs @@ -88,6 +88,13 @@ impl CanRetry for DapiClientError { _ => None, } } + + fn is_node_busy(&self) -> bool { + match self { + DapiClientError::Transport(te) => te.is_node_busy(), + _ => false, + } + } } /// Serialization of [DapiClientError]. @@ -231,7 +238,11 @@ pub fn update_address_ban_status( Err(error) => { if error.can_retry() { if let Some(address) = error.address.as_ref() { - if applied_settings.ban_failed_address { + // A node that says it is busy is working, and a request can ask that it + // is not banned for saying so. + let ban = applied_settings.ban_failed_address + && (applied_settings.ban_busy_address || !error.is_node_busy()); + if ban { let reason = Some(error.to_string()); let period_opt = error.rate_limit_ban_duration(); let banned = match period_opt { @@ -266,14 +277,14 @@ pub fn update_address_ban_status( ); } } else { - // Banning is disabled for this request, but failover - // must still move traffic away from the failing node: + // The request does not ban a node for this failure, but + // failover must still move traffic away from the failing node: // drop it from the sticky rotation, ban state untouched. address_list.evict_from_rotation(address); tracing::debug!( ?error, ?address, - "banning is disabled; evicted address {address} from rotation due to the error" + "not banning for this error; evicted address {address} from rotation" ); } } else { @@ -302,6 +313,7 @@ mod tests { timeout: Duration::from_secs(10), retries: 5, ban_failed_address: ban, + ban_busy_address: true, max_decoding_message_size: None, #[cfg(not(target_arch = "wasm32"))] ca_certificate: None, diff --git a/packages/rs-dapi-client/src/executor.rs b/packages/rs-dapi-client/src/executor.rs index 443460d0fcf..79e3da57101 100644 --- a/packages/rs-dapi-client/src/executor.rs +++ b/packages/rs-dapi-client/src/executor.rs @@ -100,6 +100,10 @@ impl CanRetry for ExecutionError { fn rate_limit_ban_duration(&self) -> Option { self.inner.rate_limit_ban_duration() } + + fn is_node_busy(&self) -> bool { + self.inner.is_node_busy() + } } /// Request execution response. diff --git a/packages/rs-dapi-client/src/lib.rs b/packages/rs-dapi-client/src/lib.rs index 2f77b2f7fcc..e511055fc86 100644 --- a/packages/rs-dapi-client/src/lib.rs +++ b/packages/rs-dapi-client/src/lib.rs @@ -104,6 +104,15 @@ pub trait CanRetry { None } + /// Returns true if the node answered that it is busy or rate limited (gRPC + /// `ResourceExhausted`), or that what was asked is not ready yet (gRPC `Unavailable` + /// with a `retry-after` header): it is working, and only cannot serve this for now. + /// A request can ask that a node is not banned for it, see + /// [RequestSettings::ban_busy_address]. + fn is_node_busy(&self) -> bool { + false + } + /// Get boolean flag that indicates if the error is retryable. /// /// Deprecated in favor of [CanRetry::can_retry]. diff --git a/packages/rs-dapi-client/src/request_settings.rs b/packages/rs-dapi-client/src/request_settings.rs index 49b6be33a6d..1d5a99ac29e 100644 --- a/packages/rs-dapi-client/src/request_settings.rs +++ b/packages/rs-dapi-client/src/request_settings.rs @@ -11,6 +11,7 @@ const DEFAULT_CONNECT_TIMEOUT: Option = None; const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); const DEFAULT_RETRIES: usize = 5; const DEFAULT_BAN_FAILED_ADDRESS: bool = true; +const DEFAULT_BAN_BUSY_ADDRESS: bool = true; /// DAPI request settings. /// @@ -43,6 +44,16 @@ pub struct RequestSettings { pub retries: Option, /// Ban DAPI address if node not responded or responded with error. pub ban_failed_address: Option, + /// Ban a DAPI address whose node answered that it is busy or rate limited (gRPC + /// `ResourceExhausted`), or that what was asked is not ready yet (gRPC `Unavailable` + /// with a `retry-after` header). + /// + /// Such a node is working. A request to a method with a tight limit of its own sets this + /// to `false`: the request then moves to another node, and this one stays available to + /// every other request. Every other failure is still banned as + /// [ban_failed_address](Self::ban_failed_address) says, which also turns this off when + /// it is `false`. + pub ban_busy_address: Option, /// Maximum gRPC response size in bytes (decoding limit). pub max_decoding_message_size: Option, } @@ -56,6 +67,7 @@ impl RequestSettings { timeout: None, retries: None, ban_failed_address: None, + ban_busy_address: None, max_decoding_message_size: None, } } @@ -69,6 +81,7 @@ impl RequestSettings { timeout: rhs.timeout.or(self.timeout), retries: rhs.retries.or(self.retries), ban_failed_address: rhs.ban_failed_address.or(self.ban_failed_address), + ban_busy_address: rhs.ban_busy_address.or(self.ban_busy_address), max_decoding_message_size: rhs .max_decoding_message_size .or(self.max_decoding_message_size), @@ -84,6 +97,7 @@ impl RequestSettings { ban_failed_address: self .ban_failed_address .unwrap_or(DEFAULT_BAN_FAILED_ADDRESS), + ban_busy_address: self.ban_busy_address.unwrap_or(DEFAULT_BAN_BUSY_ADDRESS), max_decoding_message_size: self.max_decoding_message_size, #[cfg(not(target_arch = "wasm32"))] ca_certificate: None, @@ -108,6 +122,9 @@ pub struct AppliedRequestSettings { pub retries: usize, /// Ban DAPI address if node not responded or responded with error. pub ban_failed_address: bool, + /// Ban a DAPI address whose node answered that it is busy or rate limited; see + /// [RequestSettings::ban_busy_address]. + pub ban_busy_address: bool, /// Maximum gRPC response size in bytes (decoding limit). pub max_decoding_message_size: Option, /// Certificate Authority certificate to use for verifying the server's certificate. @@ -180,6 +197,7 @@ impl AppliedRequestSettings { timeout: _, retries: _, ban_failed_address: _, + ban_busy_address: _, max_decoding_message_size, #[cfg(not(target_arch = "wasm32"))] ca_certificate, @@ -233,6 +251,7 @@ mod tests { retries: Some(3), connect_timeout: Some(Duration::from_secs(2)), ban_failed_address: Some(true), + ban_busy_address: Some(false), max_decoding_message_size: Some(1024), }; @@ -242,6 +261,7 @@ mod tests { retries: None, connect_timeout: None, ban_failed_address: None, + ban_busy_address: None, max_decoding_message_size: None, }; @@ -250,6 +270,7 @@ mod tests { assert_eq!(result.retries, Some(3)); // preserved from base assert_eq!(result.connect_timeout, Some(Duration::from_secs(2))); // preserved assert_eq!(result.ban_failed_address, Some(true)); // preserved + assert_eq!(result.ban_busy_address, Some(false)); // preserved assert_eq!(result.max_decoding_message_size, Some(1024)); // preserved } @@ -260,6 +281,7 @@ mod tests { retries: Some(3), connect_timeout: None, ban_failed_address: None, + ban_busy_address: None, max_decoding_message_size: None, }; @@ -277,6 +299,7 @@ mod tests { assert_eq!(applied.timeout, Duration::from_secs(10)); assert_eq!(applied.retries, 5); assert!(applied.ban_failed_address); + assert!(applied.ban_busy_address); assert!(applied.max_decoding_message_size.is_none()); } @@ -287,6 +310,7 @@ mod tests { timeout: Some(Duration::from_secs(30)), retries: Some(10), ban_failed_address: Some(false), + ban_busy_address: Some(false), max_decoding_message_size: Some(4096), }; @@ -295,6 +319,7 @@ mod tests { assert_eq!(applied.timeout, Duration::from_secs(30)); assert_eq!(applied.retries, 10); assert!(!applied.ban_failed_address); + assert!(!applied.ban_busy_address); assert_eq!(applied.max_decoding_message_size, Some(4096)); } @@ -392,6 +417,7 @@ mod tests { timeout: Some(Duration::from_secs(30)), retries: Some(1), ban_failed_address: Some(false), + ban_busy_address: Some(false), ..RequestSettings::default() } .finalize(); diff --git a/packages/rs-dapi-client/src/transport.rs b/packages/rs-dapi-client/src/transport.rs index 57b1a997d1d..5c9227b6efe 100644 --- a/packages/rs-dapi-client/src/transport.rs +++ b/packages/rs-dapi-client/src/transport.rs @@ -177,6 +177,12 @@ impl CanRetry for TransportError { TransportError::Grpc(status) => status.rate_limit_ban_duration(), } } + + fn is_node_busy(&self) -> bool { + match self { + TransportError::Grpc(status) => status.is_node_busy(), + } + } } /// Serialization of [TransportError]. diff --git a/packages/rs-dapi-client/src/transport/grpc.rs b/packages/rs-dapi-client/src/transport/grpc.rs index 28882a05465..24a438c5148 100644 --- a/packages/rs-dapi-client/src/transport/grpc.rs +++ b/packages/rs-dapi-client/src/transport/grpc.rs @@ -226,6 +226,19 @@ impl CanRetry for dapi_grpc::tonic::Status { secs.clamp(MIN_RATE_LIMIT_BAN_SECS, MAX_RATE_LIMIT_BAN_SECS), )) } + + fn is_node_busy(&self) -> bool { + use dapi_grpc::tonic::Code; + + match self.code() { + Code::ResourceExhausted => true, + // A node that asks to be tried again later is working: what was asked is not + // ready yet, such as the results of a block rs-dapi's node has just stored. + // A node that is down, or a proxy in front of one, sends no such header. + Code::Unavailable => self.metadata().contains_key("retry-after"), + _ => false, + } + } } /// Macro to implement the `TransportRequest` trait for a given request type, response type, client type, and settings. @@ -563,6 +576,7 @@ impl_transport_request_grpc!( timeout: Some(Duration::from_secs(30)), retries: Some(3), ban_failed_address: None, + ban_busy_address: None, connect_timeout: None, max_decoding_message_size: None, }, @@ -800,7 +814,8 @@ impl_transport_request_grpc!( timeout: Some(LARGE_RESPONSE_TIMEOUT), // rs-dapi refuses this method when too many are in progress. A busy node is not a // failed one, and a ban would take it away from the client's other requests too. - ban_failed_address: Some(false), + // A node that fails in any other way is still banned. + ban_busy_address: Some(false), ..RequestSettings::default() }, get_owner_transactions_in_block @@ -818,7 +833,8 @@ impl_transport_request_grpc!( timeout: Some(LARGE_RESPONSE_TIMEOUT), // The gateway limits this method far below every other one. A ban lasts until the // limit resets and would take the node away from the client's other requests too. - ban_failed_address: Some(false), + // A node that fails in any other way is still banned. + ban_busy_address: Some(false), ..RequestSettings::default() }, get_blocks @@ -1008,6 +1024,7 @@ impl_transport_request_grpc!( RequestSettings { timeout: Some(STREAMING_TIMEOUT), ban_failed_address: None, + ban_busy_address: None, connect_timeout: None, retries: None, max_decoding_message_size: None, diff --git a/packages/rs-dapi-client/tests/rate_limit_ban.rs b/packages/rs-dapi-client/tests/rate_limit_ban.rs index 8f1d08d8733..d4009f71aa3 100644 --- a/packages/rs-dapi-client/tests/rate_limit_ban.rs +++ b/packages/rs-dapi-client/tests/rate_limit_ban.rs @@ -12,6 +12,8 @@ use rs_dapi_client::{ update_address_ban_status, Address, AddressList, CanRetry, DapiClient, DapiClientError, DapiRequestExecutor, ExecutionError, ExecutionResult, RequestSettings, }; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; use std::time::Duration; fn make_address() -> rs_dapi_client::Address { @@ -24,6 +26,7 @@ fn applied_settings(ban: bool) -> AppliedRequestSettings { timeout: Duration::from_secs(10), retries: 5, ban_failed_address: ban, + ban_busy_address: true, max_decoding_message_size: None, #[cfg(not(target_arch = "wasm32"))] ca_certificate: None, @@ -461,15 +464,228 @@ async fn rate_limited_node_banned_for_advertised_window_via_execute() { ); } -/// A node refuses these when its limits are reached, which says nothing about its health. -#[test] -fn should_not_ban_a_node_for_failing_a_block_query() { - assert_eq!( - GetBlocksRequest::SETTINGS_OVERRIDES.ban_failed_address, - Some(false) - ); - assert_eq!( - GetOwnerTransactionsInBlockRequest::SETTINGS_OVERRIDES.ban_failed_address, - Some(false) +/// A failed answer from the one node of a list, judged under `settings`. +fn judge(status: dapi_grpc::tonic::Status, settings: &AppliedRequestSettings) -> AddressList { + let mut address_list = AddressList::new(); + let address = make_address(); + address_list.add(address.clone()); + + let result: ExecutionResult = Err(ExecutionError { + possibly_accepted: false, + inner: DapiClientError::Transport(TransportError::Grpc(status)), + retries: 0, + address: Some(address), + }); + update_address_ban_status(&address_list, &result, settings); + + address_list +} + +fn rate_limited(reset_secs: &str) -> dapi_grpc::tonic::Status { + let mut status = dapi_grpc::tonic::Status::resource_exhausted("429"); + status.metadata_mut().insert( + "ratelimit-reset", + MetadataValue::try_from(reset_secs).unwrap(), ); + status +} + +/// What rs-dapi answers for a block whose results its Tenderdash has not stored yet. +fn results_not_ready() -> dapi_grpc::tonic::Status { + let mut status = + dapi_grpc::tonic::Status::unavailable("the results of block 7 are not available yet"); + status + .metadata_mut() + .insert("retry-after", MetadataValue::from(1u64)); + status +} + +/// A node refuses a block query when its limits are reached, which says nothing about its +/// health. Any other failure does. +#[test] +fn should_ban_a_node_that_fails_a_block_query_but_not_one_that_is_busy() { + for settings in [ + GetBlocksRequest::SETTINGS_OVERRIDES, + GetOwnerTransactionsInBlockRequest::SETTINGS_OVERRIDES, + ] { + let settings = settings.finalize(); + let node = make_address(); + + let busy = dapi_grpc::tonic::Status::resource_exhausted("too many requests in progress"); + assert!(!judge(busy, &settings).is_banned(&node)); + assert!(!judge(rate_limited("3600"), &settings).is_banned(&node)); + assert!(!judge(results_not_ready(), &settings).is_banned(&node)); + + let down = dapi_grpc::tonic::Status::unavailable("connection refused"); + assert!(judge(down, &settings).is_banned(&node)); + let slow = dapi_grpc::tonic::Status::deadline_exceeded("no answer in time"); + assert!(judge(slow, &settings).is_banned(&node)); + } +} + +#[test] +fn should_ban_a_busy_node_unless_the_request_asks_otherwise() { + let node = make_address(); + let busy = || dapi_grpc::tonic::Status::resource_exhausted("429"); + + // Every other request keeps today's behaviour + let default = RequestSettings::default().finalize(); + assert!(judge(busy(), &default).is_banned(&node)); + + // Turning bans off turns them off for a busy node too + let no_bans = RequestSettings { + ban_failed_address: Some(false), + ban_busy_address: Some(true), + ..RequestSettings::default() + } + .finalize(); + assert!(!judge(busy(), &no_bans).is_banned(&node)); + let down = dapi_grpc::tonic::Status::unavailable("connection refused"); + assert!(!judge(down, &no_bans).is_banned(&node)); +} + +#[test] +fn should_tell_a_busy_node_through_every_error_wrapper() { + let busy = || dapi_grpc::tonic::Status::resource_exhausted("429"); + assert!(busy().is_node_busy()); + assert!(TransportError::Grpc(busy()).is_node_busy()); + assert!(DapiClientError::Transport(TransportError::Grpc(busy())).is_node_busy()); + assert!(ExecutionError { + possibly_accepted: false, + inner: DapiClientError::Transport(TransportError::Grpc(busy())), + retries: 0, + address: None, + } + .is_node_busy()); + + // A node that asks to be tried again later is working too + assert!(results_not_ready().is_node_busy()); + assert!(DapiClientError::Transport(TransportError::Grpc(results_not_ready())).is_node_busy()); + + assert!(!dapi_grpc::tonic::Status::unavailable("down").is_node_busy()); + assert!(!DapiClientError::NoAvailableAddresses.is_node_busy()); +} + +const FAILING_PORT: u16 = 10311; +const HEALTHY_PORT: u16 = 10312; + +/// Send a request that does not ban a busy node `times` times to a list of two nodes, one of +/// which always answers with `failure`. Returns the client and how often that node was asked. +async fn ask_two_nodes( + failure: fn() -> dapi_grpc::tonic::Status, + times: usize, +) -> (DapiClient, usize) { + let address_list: AddressList = + format!("http://127.0.0.1:{FAILING_PORT},http://127.0.0.1:{HEALTHY_PORT}") + .parse() + .expect("valid address list"); + let request = ScriptedRequest::new(move |uri| { + if uri.port_u16() == Some(FAILING_PORT) { + Err(TransportError::Grpc(failure())) + } else { + Ok(common::FakeResponse) + } + }); + let settings = RequestSettings { + ban_busy_address: Some(false), + ..RequestSettings::default() + }; + let client = DapiClient::new(address_list, RequestSettings::default()); + + for _ in 0..times { + client + .execute(request.clone(), settings) + .await + .expect("the healthy node answers"); + } + + let asked = request + .hit_uris + .lock() + .unwrap() + .iter() + .filter(|uri| uri.port_u16() == Some(FAILING_PORT)) + .count(); + (client, asked) +} + +/// With only two nodes the failed one would come straight back into rotation if it were +/// only moved out of it, so it has to be banned. +#[tokio::test] +async fn should_stop_asking_a_failed_node_of_a_short_list() { + let failing: Address = format!("http://127.0.0.1:{FAILING_PORT}") + .parse() + .expect("valid address"); + + let (client, asked) = ask_two_nodes(|| dapi_grpc::tonic::Status::unavailable("down"), 10).await; + + assert!(client.address_list().is_banned(&failing)); + assert_eq!(asked, 1, "a banned node is not asked again"); +} + +#[tokio::test] +async fn should_keep_a_busy_node_of_a_short_list_available() { + let busy: Address = format!("http://127.0.0.1:{FAILING_PORT}") + .parse() + .expect("valid address"); + + let (client, asked) = ask_two_nodes(|| rate_limited("3600"), 10).await; + + assert!(!client.address_list().is_banned(&busy)); + assert!(asked > 1, "a busy node is asked again later"); +} + +/// The newest block can be without its results for a moment. A client with a single node +/// has to get its answer from that node once they are stored, so the node must not be banned. +#[tokio::test] +async fn should_retry_one_node_until_its_block_results_are_ready() { + let node: Address = "http://127.0.0.1:10321".parse().expect("valid address"); + let address_list: AddressList = "http://127.0.0.1:10321" + .parse() + .expect("valid address list"); + + let attempts = Arc::new(AtomicUsize::new(0)); + let seen = Arc::clone(&attempts); + let request = ScriptedRequest::new(move |_uri| { + if seen.fetch_add(1, Ordering::SeqCst) == 0 { + Err(TransportError::Grpc(results_not_ready())) + } else { + Ok(common::FakeResponse) + } + }); + let client = DapiClient::new(address_list, RequestSettings::default()); + + client + .execute(request, GetBlocksRequest::SETTINGS_OVERRIDES) + .await + .expect("the node answers once the results are stored"); + + assert_eq!(attempts.load(Ordering::SeqCst), 2); + assert!(!client.address_list().is_banned(&node)); +} + +/// The same single node, down: without the header its answer is a failure, and it is banned. +#[tokio::test] +async fn should_ban_one_node_that_is_unavailable_without_saying_when_to_retry() { + let node: Address = "http://127.0.0.1:10322".parse().expect("valid address"); + let address_list: AddressList = "http://127.0.0.1:10322" + .parse() + .expect("valid address list"); + let request = ScriptedRequest::new(|_uri| { + Err(TransportError::Grpc(dapi_grpc::tonic::Status::unavailable( + "connection refused", + ))) + }); + let client = DapiClient::new(address_list, RequestSettings::default()); + + let error = client + .execute(request, GetBlocksRequest::SETTINGS_OVERRIDES) + .await + .expect_err("the only node is down"); + + assert!(matches!( + error.inner, + DapiClientError::NoAvailableAddressesToRetry(_) + )); + assert!(client.address_list().is_banned(&node)); } diff --git a/packages/rs-dapi/src/services/platform_service/get_blocks.rs b/packages/rs-dapi/src/services/platform_service/get_blocks.rs index f81a6ab490a..1c8648479f0 100644 --- a/packages/rs-dapi/src/services/platform_service/get_blocks.rs +++ b/packages/rs-dapi/src/services/platform_service/get_blocks.rs @@ -11,7 +11,8 @@ use chrono::DateTime; use dapi_grpc::platform::v0::get_blocks_response::get_blocks_response_v0::{Block, Transaction}; use dapi_grpc::platform::v0::get_blocks_response::{self, GetBlocksResponseV0}; use dapi_grpc::platform::v0::{GetBlocksRequest, GetBlocksResponse, get_blocks_request}; -use dapi_grpc::tonic::{Request, Response}; +use dapi_grpc::tonic::metadata::MetadataValue; +use dapi_grpc::tonic::{Request, Response, Status}; use drive_proof_verifier::block_proof::transaction_hash; use drive_proof_verifier::types::blocks::MAX_BLOCKS_PER_REQUEST; @@ -158,6 +159,11 @@ fn read_block( Ok(Some(built)) } +/// Seconds after which a block whose results were not stored yet is worth asking for again, +/// sent as the `retry-after` header. Tenderdash stores the results right after the block, so +/// this is the smallest value the header can carry. +const RESULTS_RETRY_AFTER_SECS: u64 = 1; + /// The results of a block this node holds. Tenderdash stores a block before its results, so /// the newest block can be without them for a moment. That is answered as `Unavailable`, /// which a client retries, where a height the node lacks is `NotFound`. @@ -167,13 +173,24 @@ pub(super) fn held_block_results( ) -> DAPIResult> { match results { Ok(results) => Ok(results.txs_results), - Err(error) if error.is_missing_tenderdash_height() => Err(DapiError::Unavailable(format!( - "the results of block {height} are not available yet" - ))), + Err(error) if error.is_missing_tenderdash_height() => Err(results_not_stored_yet(height)), Err(error) => Err(error), } } +/// The answer for a block whose results are not stored yet. The node is working, and the +/// `retry-after` header says so: the SDK reads it and retries without banning the node, as +/// it would for any other `Unavailable` answer. +fn results_not_stored_yet(height: u64) -> DapiError { + let mut status = Status::unavailable(format!( + "the results of block {height} are not available yet" + )); + status + .metadata_mut() + .insert("retry-after", MetadataValue::from(RESULTS_RETRY_AFTER_SECS)); + DapiError::Status(status) +} + /// Validate the request, returning the heights it asks for and whether to prove them. fn parse_request(request: &GetBlocksRequest) -> DAPIResult<(RangeInclusive, bool)> { let Some(get_blocks_request::Version::V0(v0)) = &request.version else { @@ -592,9 +609,12 @@ mod tests { ) }); - let error = result.unwrap_err(); - assert!(matches!(&error, DapiError::Unavailable(message) if message.contains("block 6"))); - assert_eq!(error.to_status().code(), Code::Unavailable); + let status = result.unwrap_err().to_status(); + assert_eq!(status.code(), Code::Unavailable); + assert!(status.message().contains("block 6")); + // The SDK tells this answer from a node that is down by the header + let retry_after = status.metadata().get("retry-after").unwrap(); + assert_eq!(retry_after.to_str().unwrap(), "1"); } #[test] diff --git a/packages/rs-platform-wallet/src/wallet/asset_lock/sync/locate.rs b/packages/rs-platform-wallet/src/wallet/asset_lock/sync/locate.rs index bc01f25c954..52c3c4b89af 100644 --- a/packages/rs-platform-wallet/src/wallet/asset_lock/sync/locate.rs +++ b/packages/rs-platform-wallet/src/wallet/asset_lock/sync/locate.rs @@ -106,6 +106,7 @@ const LOCATE_REQUEST_SETTINGS: RequestSettings = RequestSettings { timeout: Some(Duration::from_secs(5)), retries: Some(1), ban_failed_address: None, + ban_busy_address: None, max_decoding_message_size: None, }; @@ -117,6 +118,7 @@ const BLOCK_REQUEST_SETTINGS: RequestSettings = RequestSettings { timeout: Some(Duration::from_secs(10)), retries: Some(1), ban_failed_address: None, + ban_busy_address: None, max_decoding_message_size: Some(4 << 20), }; diff --git a/packages/rs-sdk/src/sdk.rs b/packages/rs-sdk/src/sdk.rs index 4302688f2f1..95281eacaa3 100644 --- a/packages/rs-sdk/src/sdk.rs +++ b/packages/rs-sdk/src/sdk.rs @@ -171,6 +171,7 @@ const DEFAULT_REQUEST_SETTINGS: RequestSettings = RequestSettings { retries: Some(3), timeout: None, ban_failed_address: None, + ban_busy_address: None, connect_timeout: None, max_decoding_message_size: None, }; diff --git a/packages/wasm-sdk/src/settings.rs b/packages/wasm-sdk/src/settings.rs index b08f609fb53..06dfdaf787a 100644 --- a/packages/wasm-sdk/src/settings.rs +++ b/packages/wasm-sdk/src/settings.rs @@ -71,6 +71,8 @@ impl From for RequestSettings { timeout: input.timeout_ms.map(Duration::from_millis), connect_timeout: input.connect_timeout_ms.map(Duration::from_millis), ban_failed_address: input.ban_failed_address, + // Not a setting of the JavaScript SDK: each request keeps its own default. + ban_busy_address: None, max_decoding_message_size: input.max_decoding_message_size.map(|v| v as usize), } }