From 5f75eba40ec0911ca838b03d7f22f840ba93ef6b Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 21 Oct 2026 11:21:10 -0500 Subject: [PATCH 01/25] refactor(sdk): move system contract lookup out of the trusted provider The system data contract lookup and the Platform activation heights are moved verbatim from TrustedHttpContextProvider into a crate-private system_contracts module, so a second context provider in this crate can serve the same contracts without wrapping the trusted one. Behaviour of TrustedHttpContextProvider is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../src/lib.rs | 1 + .../src/provider.rs | 217 +---------------- .../src/system_contracts.rs | 229 ++++++++++++++++++ 3 files changed, 237 insertions(+), 210 deletions(-) create mode 100644 packages/rs-sdk-trusted-context-provider/src/system_contracts.rs diff --git a/packages/rs-sdk-trusted-context-provider/src/lib.rs b/packages/rs-sdk-trusted-context-provider/src/lib.rs index 6512ed5140a..05fd054c4a5 100644 --- a/packages/rs-sdk-trusted-context-provider/src/lib.rs +++ b/packages/rs-sdk-trusted-context-provider/src/lib.rs @@ -11,6 +11,7 @@ pub mod error; pub mod provider; mod quorum_refresh; +mod system_contracts; pub mod types; pub use error::TrustedContextProviderError; diff --git a/packages/rs-sdk-trusted-context-provider/src/provider.rs b/packages/rs-sdk-trusted-context-provider/src/provider.rs index aa8d94ef27b..46e26cb9c79 100644 --- a/packages/rs-sdk-trusted-context-provider/src/provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/provider.rs @@ -3,6 +3,7 @@ use crate::get_quorum_base_url; use crate::quorum_refresh::{ contained, sleep, FetchedQuorums, QuorumRefreshes, QUORUM_LIST_REQUEST_TIMEOUT, }; +use crate::system_contracts::{platform_activation_height, system_data_contract}; use crate::types::{PreviousQuorumsResponse, QuorumData, QuorumsResponse}; use arc_swap::ArcSwap; @@ -14,32 +15,6 @@ use dpp::prelude::{CoreBlockHeight, DataContract, Identifier}; type QuorumHash = [u8; 32]; use dpp::dashcore::Network; use dpp::data_contract::TokenConfiguration; -#[cfg(any( - feature = "dpns-contract", - feature = "dashpay-contract", - feature = "withdrawals-contract", - feature = "wallet-utils-contract", - feature = "token-history-contract", - feature = "keywords-contract", - feature = "document-history-contract", - feature = "app-connect-contract", - feature = "moderation-charters-contract", - feature = "decision-proposals-contract", - feature = "all-system-contracts" -))] -use dpp::system_data_contracts::{load_system_data_contract, SystemDataContract}; -#[cfg(any(feature = "app-connect-contract", feature = "all-system-contracts"))] -use dpp::version::feature_initial_protocol_versions::APP_CONNECT_CONTRACT_INITIAL_PROTOCOL_VERSION; -#[cfg(any( - feature = "decision-proposals-contract", - feature = "all-system-contracts" -))] -use dpp::version::feature_initial_protocol_versions::DECISION_PROPOSALS_CONTRACT_INITIAL_PROTOCOL_VERSION; -#[cfg(any( - feature = "moderation-charters-contract", - feature = "all-system-contracts" -))] -use dpp::version::feature_initial_protocol_versions::MODERATION_CHARTERS_CONTRACT_INITIAL_PROTOCOL_VERSION; use dpp::version::PlatformVersion; use futures::future::{BoxFuture, FutureExt}; @@ -1045,184 +1020,8 @@ impl ContextProvider for TrustedHttpContextProvider { } drop(known); - // Check if this is a system data contract and the corresponding feature is enabled - #[cfg(any( - feature = "dpns-contract", - feature = "dashpay-contract", - feature = "withdrawals-contract", - feature = "wallet-utils-contract", - feature = "token-history-contract", - feature = "keywords-contract", - feature = "document-history-contract", - feature = "app-connect-contract", - feature = "moderation-charters-contract", - feature = "decision-proposals-contract", - feature = "all-system-contracts" - ))] - { - // Check each system contract if its feature is enabled - #[cfg(any(feature = "dpns-contract", feature = "all-system-contracts"))] - if *id == SystemDataContract::DPNS.id() { - return load_system_data_contract(SystemDataContract::DPNS, platform_version) - .map(|contract| Some(Arc::new(contract))) - .map_err(|e| { - ContextProviderError::Generic(format!( - "Failed to load DPNS contract: {}", - e - )) - }); - } - - #[cfg(any(feature = "dashpay-contract", feature = "all-system-contracts"))] - if *id == SystemDataContract::Dashpay.id() { - return load_system_data_contract(SystemDataContract::Dashpay, platform_version) - .map(|contract| Some(Arc::new(contract))) - .map_err(|e| { - ContextProviderError::Generic(format!( - "Failed to load Dashpay contract: {}", - e - )) - }); - } - - #[cfg(any(feature = "withdrawals-contract", feature = "all-system-contracts"))] - if *id == SystemDataContract::Withdrawals.id() { - return load_system_data_contract( - SystemDataContract::Withdrawals, - platform_version, - ) - .map(|contract| Some(Arc::new(contract))) - .map_err(|e| { - ContextProviderError::Generic(format!( - "Failed to load Withdrawals contract: {}", - e - )) - }); - } - - #[cfg(any(feature = "wallet-utils-contract", feature = "all-system-contracts"))] - if *id == SystemDataContract::WalletUtils.id() { - return load_system_data_contract( - SystemDataContract::WalletUtils, - platform_version, - ) - .map(|contract| Some(Arc::new(contract))) - .map_err(|e| { - ContextProviderError::Generic(format!( - "Failed to load WalletUtils contract: {}", - e - )) - }); - } - - #[cfg(any(feature = "token-history-contract", feature = "all-system-contracts"))] - if *id == SystemDataContract::TokenHistory.id() { - return load_system_data_contract( - SystemDataContract::TokenHistory, - platform_version, - ) - .map(|contract| Some(Arc::new(contract))) - .map_err(|e| { - ContextProviderError::Generic(format!( - "Failed to load TokenHistory contract: {}", - e - )) - }); - } - - #[cfg(any(feature = "keywords-contract", feature = "all-system-contracts"))] - if *id == SystemDataContract::KeywordSearch.id() { - return load_system_data_contract( - SystemDataContract::KeywordSearch, - platform_version, - ) - .map(|contract| Some(Arc::new(contract))) - .map_err(|e| { - ContextProviderError::Generic(format!( - "Failed to load KeywordSearch contract: {}", - e - )) - }); - } - - #[cfg(any( - feature = "document-history-contract", - feature = "all-system-contracts" - ))] - if *id == SystemDataContract::DocumentHistory.id() { - return load_system_data_contract( - SystemDataContract::DocumentHistory, - platform_version, - ) - .map(|contract| Some(Arc::new(contract))) - .map_err(|e| { - ContextProviderError::Generic(format!( - "Failed to load DocumentHistory contract: {}", - e - )) - }); - } - - #[cfg(any(feature = "app-connect-contract", feature = "all-system-contracts"))] - // Below protocol version 14 the app-connect contract is - // absent, so the lookup falls through to the fallback provider (or `None`). - if *id == SystemDataContract::AppConnect.id() - && platform_version.protocol_version - >= APP_CONNECT_CONTRACT_INITIAL_PROTOCOL_VERSION - { - return load_system_data_contract(SystemDataContract::AppConnect, platform_version) - .map(|contract| Some(Arc::new(contract))) - .map_err(|e| { - ContextProviderError::Generic(format!( - "Failed to load AppConnect contract: {}", - e - )) - }); - } - - #[cfg(any( - feature = "moderation-charters-contract", - feature = "all-system-contracts" - ))] - // Below protocol version 14 the moderation charters contract is absent too. - if *id == SystemDataContract::ModerationCharters.id() - && platform_version.protocol_version - >= MODERATION_CHARTERS_CONTRACT_INITIAL_PROTOCOL_VERSION - { - return load_system_data_contract( - SystemDataContract::ModerationCharters, - platform_version, - ) - .map(|contract| Some(Arc::new(contract))) - .map_err(|e| { - ContextProviderError::Generic(format!( - "Failed to load ModerationCharters contract: {}", - e - )) - }); - } - - #[cfg(any( - feature = "decision-proposals-contract", - feature = "all-system-contracts" - ))] - // Below protocol version 15 the decision proposals contract is absent. - if *id == SystemDataContract::DecisionProposals.id() - && platform_version.protocol_version - >= DECISION_PROPOSALS_CONTRACT_INITIAL_PROTOCOL_VERSION - { - return load_system_data_contract( - SystemDataContract::DecisionProposals, - platform_version, - ) - .map(|contract| Some(Arc::new(contract))) - .map_err(|e| { - ContextProviderError::Generic(format!( - "Failed to load DecisionProposals contract: {}", - e - )) - }); - } + if let Some(contract) = system_data_contract(id, platform_version)? { + return Ok(Some(contract)); } // If not found in known contracts or system contracts, delegate to fallback provider if available @@ -1261,12 +1060,7 @@ impl ContextProvider for TrustedHttpContextProvider { } fn get_platform_activation_height(&self) -> Result { - // Return the L1 locked height for each network - match self.network { - Network::Mainnet => Ok(2132092), // Mainnet L1 locked height - Network::Testnet => Ok(1090319), // Testnet L1 locked height - Network::Devnet | Network::Regtest => Ok(1), // Devnet/Regtest activation height - } + Ok(platform_activation_height(self.network)) } } @@ -2492,6 +2286,7 @@ mod tests { #[test] fn should_serve_app_connect_only_from_protocol_14() { use dpp::data_contract::accessors::v0::DataContractV0Getters; + use dpp::system_data_contracts::SystemDataContract; use dpp::version::PlatformVersion; // A numeric loopback URL avoids DNS; contract lookups make no HTTP requests. @@ -2524,6 +2319,7 @@ mod tests { #[test] fn should_serve_moderation_charters_only_from_protocol_14() { use dpp::data_contract::accessors::v0::DataContractV0Getters; + use dpp::system_data_contracts::SystemDataContract; use dpp::version::PlatformVersion; // A numeric loopback URL avoids DNS; contract lookups make no HTTP requests. @@ -2556,6 +2352,7 @@ mod tests { #[test] fn should_serve_decision_proposals_only_from_protocol_15() { use dpp::data_contract::accessors::v0::DataContractV0Getters; + use dpp::system_data_contracts::SystemDataContract; use dpp::version::PlatformVersion; // A numeric loopback URL avoids DNS; contract lookups make no HTTP requests. diff --git a/packages/rs-sdk-trusted-context-provider/src/system_contracts.rs b/packages/rs-sdk-trusted-context-provider/src/system_contracts.rs new file mode 100644 index 00000000000..c96ed4d5d6e --- /dev/null +++ b/packages/rs-sdk-trusted-context-provider/src/system_contracts.rs @@ -0,0 +1,229 @@ +//! System data contracts and activation heights a context provider serves +//! without asking any source. + +use dash_context_provider::ContextProviderError; +use dpp::dashcore::Network; +use dpp::prelude::{CoreBlockHeight, DataContract, Identifier}; +#[cfg(any( + feature = "dpns-contract", + feature = "dashpay-contract", + feature = "withdrawals-contract", + feature = "wallet-utils-contract", + feature = "token-history-contract", + feature = "keywords-contract", + feature = "document-history-contract", + feature = "app-connect-contract", + feature = "moderation-charters-contract", + feature = "decision-proposals-contract", + feature = "all-system-contracts" +))] +use dpp::system_data_contracts::{load_system_data_contract, SystemDataContract}; +#[cfg(any(feature = "app-connect-contract", feature = "all-system-contracts"))] +use dpp::version::feature_initial_protocol_versions::APP_CONNECT_CONTRACT_INITIAL_PROTOCOL_VERSION; +#[cfg(any( + feature = "decision-proposals-contract", + feature = "all-system-contracts" +))] +use dpp::version::feature_initial_protocol_versions::DECISION_PROPOSALS_CONTRACT_INITIAL_PROTOCOL_VERSION; +#[cfg(any( + feature = "moderation-charters-contract", + feature = "all-system-contracts" +))] +use dpp::version::feature_initial_protocol_versions::MODERATION_CHARTERS_CONTRACT_INITIAL_PROTOCOL_VERSION; +use dpp::version::PlatformVersion; +use std::sync::Arc; + +/// The system data contract with this id, if its cargo feature is enabled and +/// it exists at `platform_version`; `Ok(None)` otherwise. +#[cfg_attr( + not(any( + feature = "dpns-contract", + feature = "dashpay-contract", + feature = "withdrawals-contract", + feature = "wallet-utils-contract", + feature = "token-history-contract", + feature = "keywords-contract", + feature = "document-history-contract", + feature = "app-connect-contract", + feature = "moderation-charters-contract", + feature = "decision-proposals-contract", + feature = "all-system-contracts" + )), + allow(unused_variables) +)] +pub(crate) fn system_data_contract( + id: &Identifier, + platform_version: &PlatformVersion, +) -> Result>, ContextProviderError> { + // Check if this is a system data contract and the corresponding feature is enabled + #[cfg(any( + feature = "dpns-contract", + feature = "dashpay-contract", + feature = "withdrawals-contract", + feature = "wallet-utils-contract", + feature = "token-history-contract", + feature = "keywords-contract", + feature = "document-history-contract", + feature = "app-connect-contract", + feature = "moderation-charters-contract", + feature = "decision-proposals-contract", + feature = "all-system-contracts" + ))] + { + // Check each system contract if its feature is enabled + #[cfg(any(feature = "dpns-contract", feature = "all-system-contracts"))] + if *id == SystemDataContract::DPNS.id() { + return load_system_data_contract(SystemDataContract::DPNS, platform_version) + .map(|contract| Some(Arc::new(contract))) + .map_err(|e| { + ContextProviderError::Generic(format!("Failed to load DPNS contract: {}", e)) + }); + } + + #[cfg(any(feature = "dashpay-contract", feature = "all-system-contracts"))] + if *id == SystemDataContract::Dashpay.id() { + return load_system_data_contract(SystemDataContract::Dashpay, platform_version) + .map(|contract| Some(Arc::new(contract))) + .map_err(|e| { + ContextProviderError::Generic(format!("Failed to load Dashpay contract: {}", e)) + }); + } + + #[cfg(any(feature = "withdrawals-contract", feature = "all-system-contracts"))] + if *id == SystemDataContract::Withdrawals.id() { + return load_system_data_contract(SystemDataContract::Withdrawals, platform_version) + .map(|contract| Some(Arc::new(contract))) + .map_err(|e| { + ContextProviderError::Generic(format!( + "Failed to load Withdrawals contract: {}", + e + )) + }); + } + + #[cfg(any(feature = "wallet-utils-contract", feature = "all-system-contracts"))] + if *id == SystemDataContract::WalletUtils.id() { + return load_system_data_contract(SystemDataContract::WalletUtils, platform_version) + .map(|contract| Some(Arc::new(contract))) + .map_err(|e| { + ContextProviderError::Generic(format!( + "Failed to load WalletUtils contract: {}", + e + )) + }); + } + + #[cfg(any(feature = "token-history-contract", feature = "all-system-contracts"))] + if *id == SystemDataContract::TokenHistory.id() { + return load_system_data_contract(SystemDataContract::TokenHistory, platform_version) + .map(|contract| Some(Arc::new(contract))) + .map_err(|e| { + ContextProviderError::Generic(format!( + "Failed to load TokenHistory contract: {}", + e + )) + }); + } + + #[cfg(any(feature = "keywords-contract", feature = "all-system-contracts"))] + if *id == SystemDataContract::KeywordSearch.id() { + return load_system_data_contract(SystemDataContract::KeywordSearch, platform_version) + .map(|contract| Some(Arc::new(contract))) + .map_err(|e| { + ContextProviderError::Generic(format!( + "Failed to load KeywordSearch contract: {}", + e + )) + }); + } + + #[cfg(any( + feature = "document-history-contract", + feature = "all-system-contracts" + ))] + if *id == SystemDataContract::DocumentHistory.id() { + return load_system_data_contract( + SystemDataContract::DocumentHistory, + platform_version, + ) + .map(|contract| Some(Arc::new(contract))) + .map_err(|e| { + ContextProviderError::Generic(format!( + "Failed to load DocumentHistory contract: {}", + e + )) + }); + } + + #[cfg(any(feature = "app-connect-contract", feature = "all-system-contracts"))] + // Below protocol version 14 the app-connect contract is + // absent, so the lookup falls through to the fallback provider (or `None`). + if *id == SystemDataContract::AppConnect.id() + && platform_version.protocol_version >= APP_CONNECT_CONTRACT_INITIAL_PROTOCOL_VERSION + { + return load_system_data_contract(SystemDataContract::AppConnect, platform_version) + .map(|contract| Some(Arc::new(contract))) + .map_err(|e| { + ContextProviderError::Generic(format!( + "Failed to load AppConnect contract: {}", + e + )) + }); + } + + #[cfg(any( + feature = "moderation-charters-contract", + feature = "all-system-contracts" + ))] + // Below protocol version 14 the moderation charters contract is absent too. + if *id == SystemDataContract::ModerationCharters.id() + && platform_version.protocol_version + >= MODERATION_CHARTERS_CONTRACT_INITIAL_PROTOCOL_VERSION + { + return load_system_data_contract( + SystemDataContract::ModerationCharters, + platform_version, + ) + .map(|contract| Some(Arc::new(contract))) + .map_err(|e| { + ContextProviderError::Generic(format!( + "Failed to load ModerationCharters contract: {}", + e + )) + }); + } + + #[cfg(any( + feature = "decision-proposals-contract", + feature = "all-system-contracts" + ))] + // Below protocol version 15 the decision proposals contract is absent. + if *id == SystemDataContract::DecisionProposals.id() + && platform_version.protocol_version + >= DECISION_PROPOSALS_CONTRACT_INITIAL_PROTOCOL_VERSION + { + return load_system_data_contract( + SystemDataContract::DecisionProposals, + platform_version, + ) + .map(|contract| Some(Arc::new(contract))) + .map_err(|e| { + ContextProviderError::Generic(format!( + "Failed to load DecisionProposals contract: {}", + e + )) + }); + } + } + + Ok(None) +} + +/// The Core height Platform activated at (the L1 locked height) on `network`. +pub(crate) fn platform_activation_height(network: Network) -> CoreBlockHeight { + match network { + Network::Mainnet => 2132092, // Mainnet L1 locked height + Network::Testnet => 1090319, // Testnet L1 locked height + Network::Devnet | Network::Regtest => 1, // Devnet/Regtest activation height + } +} From 0323275cca43610fd9e1efcc72fc23212a438f8e Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 21 Oct 2026 11:29:33 -0500 Subject: [PATCH 02/25] feat(sdk): add opt-in verified HTTP context provider TrustedHttpContextProvider takes Platform quorum keys from an HTTP service on trust. VerifiedHttpContextProvider instead proves each key with a Core quorum proof chain (DIP 175) from a checkpoint, using dash-core-proof, and treats the quorum service as an untrusted relay. Applications opt in through SdkBuilder::with_context_provider; nothing changes by default. - Embedded mainnet (2,550,000) and testnet (1,560,000) checkpoints, or an application-supplied one; a test recomputes each embedded checkpoint from its block header, coinbase and Merkle branch. - Keys are cached per quorum and served for 600 blocks past the target that proved them; past that the quorum is proven again from the newest verified state. - Every fetch failure is QuorumSourceUnavailable, never Ok(None): an inclusion proof cannot show absence, so a relay must not be able to get honest nodes banned. - Fetches are shared per quorum, at most two quorums are proven at once, failed quorums and failing sources cool down, a fetch has a 70 s deadline and at most 8 requests, redirects are refused and bodies are capped at 1 MiB. A failure tied to an unverified height only affects requests near that height. - Browsers fetch through web_sys with redirect "error", no-store and omitted credentials, abort on deadline or drop, and stream the body through wasm-streams; the wasm32 tree stays free of hyper/rustls/tower/mio. Co-Authored-By: Claude Opus 5.5 --- .../rs-packages-no-workflows.yml | 1 + .github/package-filters/rs-packages.yml | 1 + .github/workflows/tests.yml | 1 + Cargo.lock | 5 + .../Cargo.toml | 19 + .../rs-sdk-trusted-context-provider/README.md | 88 ++ .../checkpoints/mainnet.json | 7 + .../checkpoints/testnet.json | 7 + .../src/lib.rs | 13 + .../src/proof_fetches.rs | 981 +++++++++++++++ .../src/proof_transport.rs | 325 +++++ .../src/quorum_refresh.rs | 20 +- .../src/verified_provider.rs | 1112 +++++++++++++++++ .../tests/checkpoints.rs | 63 + .../data/checkpoint-evidence-mainnet.json | 11 + .../data/checkpoint-evidence-testnet.json | 9 + .../tests/data/proof-mainnet.json | 18 + .../tests/data/proof-testnet.json | 18 + 18 files changed, 2693 insertions(+), 6 deletions(-) create mode 100644 packages/rs-sdk-trusted-context-provider/checkpoints/mainnet.json create mode 100644 packages/rs-sdk-trusted-context-provider/checkpoints/testnet.json create mode 100644 packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs create mode 100644 packages/rs-sdk-trusted-context-provider/src/proof_transport.rs create mode 100644 packages/rs-sdk-trusted-context-provider/src/verified_provider.rs create mode 100644 packages/rs-sdk-trusted-context-provider/tests/checkpoints.rs create mode 100644 packages/rs-sdk-trusted-context-provider/tests/data/checkpoint-evidence-mainnet.json create mode 100644 packages/rs-sdk-trusted-context-provider/tests/data/checkpoint-evidence-testnet.json create mode 100644 packages/rs-sdk-trusted-context-provider/tests/data/proof-mainnet.json create mode 100644 packages/rs-sdk-trusted-context-provider/tests/data/proof-testnet.json diff --git a/.github/package-filters/rs-packages-no-workflows.yml b/.github/package-filters/rs-packages-no-workflows.yml index 5a696583b99..8a7ee8904f3 100644 --- a/.github/package-filters/rs-packages-no-workflows.yml +++ b/.github/package-filters/rs-packages-no-workflows.yml @@ -125,6 +125,7 @@ rs-sdk-trusted-context-provider: &sdk_trusted_context_provider - packages/rs-sdk-trusted-context-provider/** - *dash_async - *context_provider + - *core_proof - *dpp dapi-grpc: &dapi_grpc diff --git a/.github/package-filters/rs-packages.yml b/.github/package-filters/rs-packages.yml index 5228992762f..8e20f274c56 100644 --- a/.github/package-filters/rs-packages.yml +++ b/.github/package-filters/rs-packages.yml @@ -149,6 +149,7 @@ rs-sdk-trusted-context-provider: &sdk_trusted_context_provider - packages/rs-sdk-trusted-context-provider/** - *dash_async - *context_provider + - *core_proof - *dpp dapi-grpc: &dapi_grpc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 78d566cd103..f380da6a3d8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -361,6 +361,7 @@ jobs: - packages/keyword-search-contract/** - packages/masternode-reward-shares-contract/** - packages/rs-context-provider/** + - packages/rs-core-proof/** - packages/rs-dapi-client/** - packages/rs-dash-async/** - packages/rs-dash-platform-macros/** diff --git a/Cargo.lock b/Cargo.lock index 2c7312e8878..d0f8a2486cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6536,10 +6536,12 @@ dependencies = [ "arc-swap", "dash-async", "dash-context-provider", + "dash-core-proof", "dpp", "futures", "gloo-timers", "hex", + "js-sys", "lru", "reqwest 0.12.28", "serde", @@ -6548,7 +6550,10 @@ dependencies = [ "tokio", "tracing", "url", + "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", + "web-sys", "web-time", ] diff --git a/packages/rs-sdk-trusted-context-provider/Cargo.toml b/packages/rs-sdk-trusted-context-provider/Cargo.toml index f9f2602d355..4a6ab6715e4 100644 --- a/packages/rs-sdk-trusted-context-provider/Cargo.toml +++ b/packages/rs-sdk-trusted-context-provider/Cargo.toml @@ -10,6 +10,8 @@ description = "Trusted HTTP-based context provider for Dash Platform SDK" dash-async = { path = "../rs-dash-async" } dash-context-provider = { path = "../rs-context-provider" } +# Verifies the quorum proofs behind `VerifiedHttpContextProvider`. +dash-core-proof = { path = "../rs-core-proof" } dpp = { path = "../rs-dpp", default-features = false, features = [ "dash-sdk-features", "bls-signatures", @@ -65,6 +67,23 @@ decision-proposals-contract = ["dpp/decision-proposals-contract"] # Runs the browser's non-`Send` quorum fetches and timers on the local executor. wasm-bindgen-futures = { version = "0.4.58" } gloo-timers = { version = "0.3.0", features = ["futures"] } +# Browser `fetch` for proof requests: unlike reqwest on wasm32 it can refuse +# redirects, abort, and bound a body while streaming it. +js-sys = "0.3.85" +wasm-bindgen = "0.2.108" +wasm-streams = "0.5" +web-sys = { version = "0.3.85", features = [ + "AbortController", + "AbortSignal", + "Headers", + "ReadableStream", + "Request", + "RequestCache", + "RequestCredentials", + "RequestInit", + "RequestRedirect", + "Response", +] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Waits out the gap between two refreshes of the quorum lists. diff --git a/packages/rs-sdk-trusted-context-provider/README.md b/packages/rs-sdk-trusted-context-provider/README.md index 78aecf3c068..ddf1c79d69c 100644 --- a/packages/rs-sdk-trusted-context-provider/README.md +++ b/packages/rs-sdk-trusted-context-provider/README.md @@ -117,6 +117,94 @@ let sdk = SdkBuilder::new(AddressList::default()) .build()?; ``` +## Verified Quorum Keys (opt-in) + +`VerifiedHttpContextProvider` trusts no HTTP endpoint for quorum keys. It +starts from a checkpoint, a ChainLocked Core block and the masternode and +quorum Merkle roots its coinbase commits to, and accepts a Platform quorum key +only after a Core quorum proof chain (DIP 175, `DASHNC02`) verifies from that +checkpoint and opens the quorum's commitment. Relays that serve the proofs are +untrusted byte sources: a relay that lies, stalls or fails can delay requests, +but cannot make the SDK accept a forged key or ban an honest node. + +```rust +use dash_sdk::SdkBuilder; +use dash_sdk::dapi_client::AddressList; +use dpp::dashcore::Network; +use rs_sdk_trusted_context_provider::VerifiedHttpContextProvider; +use std::num::NonZeroUsize; + +// Embedded checkpoint, network's public quorum service as the relay. +let provider = VerifiedHttpContextProvider::new( + Network::Testnet, + NonZeroUsize::new(100).expect("cache size"), +)?; + +let sdk = SdkBuilder::new(AddressList::default()) + .with_context_provider(provider) + .build()?; +``` + +Your own checkpoint and relays (1 to 8, `https` except on loopback): + +```rust +let checkpoint = VerifiedHttpContextProvider::embedded_checkpoint(Network::Testnet)?; +let provider = VerifiedHttpContextProvider::with_checkpoint( + Network::Testnet, + checkpoint, + vec!["https://relay.example.org".to_string()], + NonZeroUsize::new(100).expect("cache size"), +)?; + +// Later: persist the newest verified state and start from it next time. +let state_json = serde_json::to_string(&provider.latest_verified_state())?; +``` + +Behaviour: + +- Mainnet and testnet only. Each provider POSTs to `/proofs` with the + checkpoint hash, a height, the quorum hash and type, and `nodeCount: 0`. +- A proven key is served for 600 blocks past the target block that proved the + quorum active, then proven again. Failures of any kind (timeouts, HTTP + errors, oversize or invalid proofs, proofs of other quorums) are reported as + an unavailable quorum source, so the SDK tries another node without banning + the one that named the quorum. A quorum of another type than the network's + Platform quorum type is rejected as invalid. +- Fetches of one quorum are shared, at most two quorums are proven at once, + failed quorums and failing sources cool down, and a fetch gives up after + 70 s. Redirects are refused and responses are capped at 1 MiB. +- Data contracts come from the known-contracts cache and the system contracts + enabled by cargo features; token configurations from the known cache. +- The public mainnet quorum service currently answers `/proofs` with HTTP 403. + `VerifiedHttpContextProvider::new(Network::Mainnet, ..)` builds, but its + fetches fail until the service serves proofs; pass working mainnet relays + to `with_checkpoint` meanwhile. + +### Trust model + +The checkpoint is trust configuration, like a hard-coded block hash: whoever +chooses it chooses the chain. Beyond it, the provider trusts the ChainLock +quorums the proof chain passes through, as Core does. A valid proof shows a +quorum was mined, not that its target is the newest ChainLock; how fresh a +response must be is the SDK's policy. + +### Refreshing the embedded checkpoints + +`checkpoints/.json` hold Core `ProofState` JSON (hashes in RPC display +order). To move one to a newer block `H`: + +1. Pick a height with `chainlock: true` in `getblock` on two independent Core + nodes, and confirm `getblockhash H` and the block hash on a block explorer + agree. +2. Write the state from the block: `network` (0 mainnet, 1 testnet), `height`, + `block_hash`, and the coinbase payload's `merkleRootMNList` and + `merkleRootQuorums` as `masternode_root` and `quorum_root`. +3. Update `tests/data/checkpoint-evidence-.json`: the raw header + (`getblockheader false`), the raw coinbase transaction, and the + coinbase's Merkle branch (sibling hashes in wire order). The + `checkpoints` test recomputes the block hash, the header's Merkle root and + both roots from this evidence. + ## Implementation Details The `TrustedHttpContextProvider` implements the `ContextProvider` trait and provides: diff --git a/packages/rs-sdk-trusted-context-provider/checkpoints/mainnet.json b/packages/rs-sdk-trusted-context-provider/checkpoints/mainnet.json new file mode 100644 index 00000000000..12c322d7d29 --- /dev/null +++ b/packages/rs-sdk-trusted-context-provider/checkpoints/mainnet.json @@ -0,0 +1,7 @@ +{ + "network": 0, + "height": 2550000, + "block_hash": "000000000000000759db5eed70bf470171e7738b85765b395c1a75029c9683ce", + "masternode_root": "1d9ee9f97d3c393c1825c58cec162ffd66efe6a25f24bc4142f68c05b1df1750", + "quorum_root": "1b790dc5600ccfe0f8167412e6b751de8cbac34bcab37ed5feebbb56c117b540" +} diff --git a/packages/rs-sdk-trusted-context-provider/checkpoints/testnet.json b/packages/rs-sdk-trusted-context-provider/checkpoints/testnet.json new file mode 100644 index 00000000000..f6144c818e6 --- /dev/null +++ b/packages/rs-sdk-trusted-context-provider/checkpoints/testnet.json @@ -0,0 +1,7 @@ +{ + "network": 1, + "height": 1560000, + "block_hash": "00000215d91d5f3cdfda9de376171ce7eebfeb5adbabc012e8d59501ba07a35e", + "masternode_root": "335e0734bf2efdc117e639e0db433d230042a061b5b17da8462429cf3d11dd80", + "quorum_root": "d1a9305a96f477a0f86be90f2ee4af1085027d5305fd769f2c406dd353114a43" +} diff --git a/packages/rs-sdk-trusted-context-provider/src/lib.rs b/packages/rs-sdk-trusted-context-provider/src/lib.rs index 05fd054c4a5..fb7ecc88e91 100644 --- a/packages/rs-sdk-trusted-context-provider/src/lib.rs +++ b/packages/rs-sdk-trusted-context-provider/src/lib.rs @@ -7,15 +7,28 @@ //! - **Mainnet**: Uses `https://quorums.mainnet.networks.dash.org/` //! - **Testnet**: Uses `https://quorums.testnet.networks.dash.org/` //! - **Devnet**: Uses `https://quorums..networks.dash.org/` +//! +//! ## Verified quorum keys +//! +//! [`VerifiedHttpContextProvider`] is an opt-in alternative for mainnet and +//! testnet that trusts no HTTP endpoint: it proves every Platform quorum key +//! from a checkpoint with Core's DIP 175 proofs, and treats the quorum +//! service as an untrusted relay. It lives in this crate because it serves +//! the same applications, talks to the same quorum service and shares the +//! system-contract and HTTP plumbing; the name of the crate predates it. pub mod error; +mod proof_fetches; +mod proof_transport; pub mod provider; mod quorum_refresh; mod system_contracts; pub mod types; +pub mod verified_provider; pub use error::TrustedContextProviderError; pub use provider::TrustedHttpContextProvider; +pub use verified_provider::{VerifiedContextProviderError, VerifiedHttpContextProvider}; use dpp::dashcore::Network; diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs new file mode 100644 index 00000000000..79df2098cb6 --- /dev/null +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -0,0 +1,981 @@ +//! Proving Platform quorum keys with proofs fetched from relays. +//! +//! A key is believed only after a `DASHNC02` bootstrap proof, checked from a +//! state this provider already trusts, opens the quorum's commitment under a +//! ChainLocked quorum root. Relays are untrusted byte sources: one that lies, +//! stalls or fails costs time, bounded here, but never changes a key. +//! +//! A proven key is immutable: under ChainLocks each quorum has one mined +//! commitment. What expires is the evidence that the quorum is still active. +//! A key proven by a target at height `T` is served for heights up to +//! `T + window`, where `window` is how long a Platform quorum stays in the +//! active set. The height comes from the response being verified and is not +//! authenticated, so the window is only a trigger to prove again, not a +//! security boundary. +//! +//! The SDK treats any error from a fetch as "the source gave no answer", and +//! never bans a node for it. A fetch therefore never reports a quorum as +//! absent: an inclusion proof cannot show absence, and a relay's "not found" +//! is unauthenticated. + +use crate::proof_transport::{PostError, ProofTransport, Reply}; +#[cfg(target_arch = "wasm32")] +use crate::quorum_refresh::on_local_executor; +#[cfg(not(target_arch = "wasm32"))] +use crate::quorum_refresh::panic_message; +#[cfg(not(target_arch = "wasm32"))] +use crate::quorum_refresh::sleep; +use dash_context_provider::ContextProviderError; +use dash_core_proof::bootstrap::{self, RecordKind}; +use dash_core_proof::{parse_commitment, rpc_hex, State}; +use futures::future::{self, select, BoxFuture, Either, FutureExt, Shared}; +use futures::stream::FuturesUnordered; +use futures::StreamExt; +#[cfg(target_arch = "wasm32")] +use gloo_timers::future::sleep; +use lru::LruCache; +use std::collections::{BTreeMap, HashMap}; +use std::future::Future; +use std::num::NonZeroUsize; +#[cfg(not(target_arch = "wasm32"))] +use std::panic::AssertUnwindSafe; +use std::pin::pin; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::Duration; +use url::Url; +use web_time::Instant; + +/// A quorum's BLS public key. +pub(crate) type QuorumKey = [u8; 48]; + +/// A quorum: its LLMQ type and its hash in the order the SDK uses (Core's RPC +/// display order). +type QuorumId = (u32, [u8; 32]); + +/// Most requests one fetch sends, across sources and retries. +const MAX_REQUESTS_PER_FETCH: usize = 8; +/// Most requests of one fetch in flight at once: the first and one hedge. +const MAX_REQUESTS_IN_FLIGHT: usize = 2; +/// Most distinct quorums being proven at once. Each costs a relay a Core +/// call, and the quorums come from unverified responses. +const MAX_QUORUMS_IN_FLIGHT: usize = 2; +/// Most quorums with a running or recently failed fetch remembered. +const MAX_FETCH_SLOTS: usize = 64; +/// Verified states kept besides the checkpoint, to start later proofs from. +const MAX_RECENT_STATES: usize = 16; +/// The retry delay after a relay's 502 doubles this many times. +const MAX_RETRY_DOUBLINGS: u32 = 3; +/// How much of an error response's body goes into the failure reason. +const MAX_REASON_BODY_CHARS: usize = 120; + +/// Deadlines, delays and cooldowns of fetches. +#[derive(Clone, Debug)] +pub(crate) struct Timing { + /// Overall deadline of one fetch. + pub(crate) deadline: Duration, + /// Deadline of one request. Relays give Core 60 s for a proof. + pub(crate) request_timeout: Duration, + /// How long a request may go unanswered before the next source is asked + /// as well. + pub(crate) hedge_after: Duration, + /// First delay before asking again after a relay's 502. + pub(crate) first_retry: Duration, + /// First wait before a quorum whose fetch failed is fetched again. + pub(crate) quorum_cooldown: Duration, + /// Longest such wait. + pub(crate) max_quorum_cooldown: Duration, + /// First wait before a failing source is asked again. + pub(crate) source_cooldown: Duration, + /// Longest such wait. + pub(crate) max_source_cooldown: Duration, +} + +impl Default for Timing { + fn default() -> Self { + Self { + deadline: Duration::from_secs(70), + request_timeout: Duration::from_secs(65), + hedge_after: Duration::from_secs(5), + first_retry: Duration::from_secs(1), + quorum_cooldown: Duration::from_secs(5), + max_quorum_cooldown: Duration::from_secs(60), + source_cooldown: Duration::from_secs(10), + max_source_cooldown: Duration::from_secs(120), + } + } +} + +/// `first`, doubled for every failure after the first, capped at `max`. +fn backoff(first: Duration, max: Duration, failures: u32) -> Duration { + let doublings = failures.saturating_sub(1).min(16); + first.saturating_mul(1 << doublings).min(max) +} + +/// A proven key and the highest target height that proved it. +struct Cached { + key: QuorumKey, + active_at: u32, +} + +type SharedFetch = Shared>>; + +/// The latest fetch of one quorum. +enum Slot { + Running { + id: u64, + height: u32, + fetch: SharedFetch, + }, + Failed { + at: Instant, + failures: u32, + height: u32, + }, +} + +#[derive(Default)] +struct SourceHealth { + failed_until: Option, + failures: u32, +} + +/// Everything fetches change, behind one lock that is never held across an +/// await. +struct Proven { + keys: LruCache, + /// The checkpoint and the most recent verified targets, by height. + states: BTreeMap, + fetches: HashMap, + sources: Vec, + next_fetch_id: u64, +} + +/// What to ask a relay for: a proof from `checkpoint` to a target at or above +/// `height`, where 0 asks for Core's latest ChainLock. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Ask { + checkpoint: State, + height: u32, +} + +/// Choose what to ask a relay for when proving a quorum for `height`. +/// +/// The proof starts from the newest verified state below `height`, so it is +/// as short as it can be, and asks for a target at `height`. Once a state +/// past the checkpoint is verified, the checkpoint is never used again: a +/// later quorum is proven from the latest state, asking for the tip. +/// `alternate` flips between asking for `height` and for the tip, which a +/// relay that could not serve one may still serve. +fn choose_ask( + states: &BTreeMap, + checkpoint_height: u32, + height: u32, + alternate: bool, +) -> Option { + let (_, latest) = states.last_key_value()?; + let ask = match states.range(..height).next_back() { + Some((&below, state)) if below != checkpoint_height || latest.height == below => Ask { + checkpoint: state.clone(), + height, + }, + _ => Ask { + checkpoint: latest.clone(), + height: 0, + }, + }; + Some(if alternate && ask.height != 0 { + Ask { height: 0, ..ask } + } else { + ask + }) +} + +/// How one answer from a relay counts. +enum Verdict { + Proven, + /// The relay could not serve the proof yet; ask again after `after`, for + /// the other height. + Retry { + reason: String, + after: Duration, + }, + /// The source failed and cools down. + SourceFailed(String), + /// A valid proof of the relay's latest ChainLock does not reach the + /// height: asking again cannot help until the chain grows. + BeyondTip(String), +} + +/// Proves quorum keys and caches them, for a provider and its clones. +pub(crate) struct ProofFetches { + quorum_type: u32, + window: u32, + checkpoint: State, + endpoints: Vec, + transport: ProofTransport, + timing: Timing, + proven: Mutex, +} + +impl ProofFetches { + /// Prove keys of `quorum_type` quorums from `checkpoint` with proofs + /// from `endpoints`, serving each for `window` blocks past the target + /// that proved it. + pub(crate) fn new( + quorum_type: u32, + window: u32, + checkpoint: State, + endpoints: Vec, + cache_size: NonZeroUsize, + transport: ProofTransport, + timing: Timing, + ) -> Self { + let proven = Proven { + keys: LruCache::new(cache_size), + states: BTreeMap::from([(checkpoint.height, checkpoint.clone())]), + fetches: HashMap::new(), + sources: endpoints.iter().map(|_| SourceHealth::default()).collect(), + next_fetch_id: 0, + }; + Self { + quorum_type, + window, + checkpoint, + endpoints, + transport, + timing, + proven: Mutex::new(proven), + } + } + + /// The checkpoint proofs start from until a later state is verified. + pub(crate) fn checkpoint(&self) -> &State { + &self.checkpoint + } + + /// The highest verified state, or the checkpoint. + pub(crate) fn latest_state(&self) -> State { + self.lock() + .states + .last_key_value() + .map_or_else(|| self.checkpoint.clone(), |(_, state)| state.clone()) + } + + /// How many quorum keys are cached. + pub(crate) fn cached_quorum_count(&self) -> usize { + self.lock().keys.len() + } + + /// Whether this provider ever proves the key of this quorum for this + /// height: only Platform quorums, at heights Core can have. + pub(crate) fn provable(&self, quorum_type: u32, height: u32) -> bool { + quorum_type == self.quorum_type && height <= i32::MAX as u32 + } + + /// The cached key of a quorum, if it was proven active recently enough + /// for `height`. + pub(crate) fn lookup( + &self, + quorum_type: u32, + quorum_hash: [u8; 32], + height: u32, + ) -> Result { + if !self.provable(quorum_type, height) { + return Err(ContextProviderError::InvalidQuorum(format!( + "quorum type {quorum_type} at core height {height} is not a Platform quorum this \ + provider verifies (type {})", + self.quorum_type + ))); + } + let mut proven = self.lock(); + match proven.keys.get(&(quorum_type, quorum_hash)) { + Some(cached) if height <= cached.active_at.saturating_add(self.window) => { + Ok(cached.key) + } + Some(cached) => Err(ContextProviderError::QuorumSourceUnavailable(format!( + "quorum {} was proven active at core height {}, not yet for {height}", + hex::encode(quorum_hash), + cached.active_at + ))), + None => Err(ContextProviderError::QuorumSourceUnavailable(format!( + "quorum {} is not proven yet", + hex::encode(quorum_hash) + ))), + } + } + + /// Prove a quorum's key for `height`, or join the fetch already proving + /// it, and return it. Every failure is + /// [`ContextProviderError::QuorumSourceUnavailable`]; this never reports + /// the quorum as absent. + pub(crate) async fn fetch( + self: Arc, + quorum_type: u32, + quorum_hash: [u8; 32], + height: u32, + ) -> Result { + if let Ok(key) = self.lookup(quorum_type, quorum_hash, height) { + return Ok(key); + } + let fetch = self + .join_or_start(quorum_type, quorum_hash, height) + .map_err(ContextProviderError::QuorumSourceUnavailable)?; + fetch + .await + .map_err(ContextProviderError::QuorumSourceUnavailable)?; + // A fetch this one joined may have proven the quorum only for a lower + // height; the lookup then reports the source as unavailable. + self.lookup(quorum_type, quorum_hash, height) + } + + fn lock(&self) -> MutexGuard<'_, Proven> { + // Every update under the lock completes before it is released. + self.proven.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// The running fetch of this quorum, or a new one if the limits allow. + /// + /// A fetch or a failure counts only for heights within the validity + /// window of its own, since the height comes from an unverified response: + /// a bogus height must not hold back, or cool down, a request for an + /// honest one. + fn join_or_start( + self: &Arc, + quorum_type: u32, + quorum_hash: [u8; 32], + height: u32, + ) -> Result { + let quorum = (quorum_type, quorum_hash); + let near = |other: u32| other.abs_diff(height) <= self.window; + let now = Instant::now(); + let forgotten_after = self.timing.max_quorum_cooldown.saturating_mul(2); + let mut proven = self.lock(); + // A fetch every waiter dropped is not running natively; it is + // replaced rather than joined or counted. + proven.fetches.retain(|_, slot| match slot { + Slot::Running { fetch, .. } => fetch.strong_count() != Some(1), + Slot::Failed { at, .. } => now.saturating_duration_since(*at) < forgotten_after, + }); + // A fetch that finished since the caller's lookup has nothing to add. + if let Some(cached) = proven.keys.peek(&quorum) { + if height <= cached.active_at.saturating_add(self.window) { + return Ok(future::ready(Ok(())).boxed().shared()); + } + } + let failures = match proven.fetches.get(&quorum) { + Some(Slot::Running { + fetch, + height: running, + .. + }) if near(*running) => { + return Ok(fetch.clone()); + } + Some(Slot::Running { + height: running, .. + }) => { + return Err(format!( + "quorum {} is being proven for core height {running}; retry", + hex::encode(quorum_hash) + )); + } + Some(Slot::Failed { + at, + failures, + height: failed, + }) if near(*failed) => { + let cooldown = backoff( + self.timing.quorum_cooldown, + self.timing.max_quorum_cooldown, + *failures, + ); + let wait = cooldown.saturating_sub(now.saturating_duration_since(*at)); + if !wait.is_zero() { + return Err(format!( + "quorum {} was asked for too recently, retry in {} s", + hex::encode(quorum_hash), + wait.as_millis().div_ceil(1000) + )); + } + *failures + } + Some(Slot::Failed { .. }) | None => 0, + }; + let running = proven + .fetches + .values() + .filter(|slot| matches!(slot, Slot::Running { .. })) + .count(); + if running >= MAX_QUORUMS_IN_FLIGHT { + return Err(format!( + "{running} other quorums are being proven; not starting another" + )); + } + if !proven.fetches.contains_key(&quorum) && proven.fetches.len() >= MAX_FETCH_SLOTS { + return Err("too many quorums were asked for recently".to_string()); + } + proven.next_fetch_id += 1; + let id = proven.next_fetch_id; + // Built under the lock, so it must do nothing until it is polled. + let fetch = Arc::clone(self).start(quorum, height, id, failures); + proven.fetches.insert( + quorum, + Slot::Running { + id, + height, + fetch: fetch.clone(), + }, + ); + Ok(fetch) + } + + /// A fetch every waiter of this quorum shares. + fn start( + self: Arc, + quorum: QuorumId, + height: u32, + id: u64, + failures: u32, + ) -> SharedFetch { + async move { + let prover = Arc::clone(&self); + let result = contained(move || async move { + let result = prover.prove(quorum, height).await; + // Recorded here too, so a browser fetch that outlives its + // waiters still frees its slot. + prover.finish(quorum, height, id, failures, &result); + result + }) + .await; + // Covers a fetch that panicked; a no-op otherwise. + self.finish(quorum, height, id, failures, &result); + result + } + .boxed() + .shared() + } + + /// Record how fetch `id` of a quorum ended, if its slot is still its own. + fn finish( + &self, + quorum: QuorumId, + height: u32, + id: u64, + failures: u32, + result: &Result<(), String>, + ) { + let mut proven = self.lock(); + let ours = matches!( + proven.fetches.get(&quorum), + Some(Slot::Running { id: running, .. }) if *running == id + ); + if ours && result.is_ok() { + proven.fetches.remove(&quorum); + } else if ours { + let failed = Slot::Failed { + at: Instant::now(), + failures: failures.saturating_add(1), + height, + }; + proven.fetches.insert(quorum, failed); + } + } + + /// Ask the sources for a proof of `quorum` until one verifies, the + /// request budget is spent or the deadline passes. + async fn prove(&self, quorum: QuorumId, height: u32) -> Result<(), String> { + let started = Instant::now(); + let deadline = started + self.timing.deadline; + let mut requests = FuturesUnordered::new(); + let mut asked = vec![0u32; self.endpoints.len()]; + let mut busy = Vec::new(); + let mut sent = 0; + let mut retries = 0; + let mut alternate = false; + let mut next_start = started; + let mut failures = Vec::new(); + loop { + let now = Instant::now(); + if now >= deadline { + failures.push("deadline passed".to_string()); + break; + } + let mut wake = deadline; + if requests.len() < MAX_REQUESTS_IN_FLIGHT && sent < MAX_REQUESTS_PER_FETCH { + if now < next_start { + wake = next_start.min(deadline); + } else if let Some(source) = self.pick_source(now, &asked, &busy) { + let ask = choose_ask( + &self.lock().states, + self.checkpoint.height, + height, + alternate, + ) + .unwrap_or_else(|| Ask { + checkpoint: self.checkpoint.clone(), + height, + }); + let body = request_body(&ask, quorum); + let remaining = deadline.saturating_duration_since(now); + let timeout = self.timing.request_timeout.min(remaining); + // A request cut short by the deadline says nothing about + // its source. + let cut_short = timeout < self.timing.request_timeout; + asked[source] += 1; + busy.push(source); + sent += 1; + next_start = now + self.timing.hedge_after; + let endpoint = &self.endpoints[source]; + requests.push(async move { + let reply = self.transport.post(endpoint, body, timeout).await; + (source, ask, now, cut_short, reply) + }); + continue; + } else if requests.is_empty() { + failures.push("every source is cooling down after failures".to_string()); + break; + } + } else if requests.is_empty() { + break; + } + + let wait = wake.saturating_duration_since(now); + if requests.is_empty() { + sleep(wait).await; + continue; + } + let answered = match select(requests.next(), pin!(sleep(wait))).await { + Either::Left((Some(answered), _)) => answered, + Either::Left((None, _)) | Either::Right(_) => continue, + }; + let (source, ask, asked_at, cut_short, reply) = answered; + busy.retain(|busy| *busy != source); + let origin = self.endpoints[source].origin().ascii_serialization(); + let elapsed = asked_at.elapsed().as_millis(); + let verdict = match reply { + Err(PostError::Failed(reason)) if cut_short => Verdict::Retry { + reason, + after: Duration::ZERO, + }, + reply => self.judge(quorum, height, &ask, reply, retries), + }; + match verdict { + Verdict::Proven => { + self.source_answered(source); + tracing::debug!( + quorum_type = quorum.0, + quorum_hash = %hex::encode(quorum.1), + height, + %origin, + elapsed_ms = elapsed, + "proved quorum key" + ); + return Ok(()); + } + Verdict::Retry { reason, after } => { + failures.push(format!("{origin}: {reason} after {elapsed} ms")); + retries += 1; + alternate = !alternate; + next_start = Instant::now() + after; + } + Verdict::SourceFailed(reason) => { + failures.push(format!("{origin}: {reason} after {elapsed} ms")); + self.source_failed(source); + next_start = Instant::now(); + } + Verdict::BeyondTip(reason) => { + failures.push(format!("{origin}: {reason}")); + break; + } + } + } + let reason = format!( + "could not prove quorum {} of type {} for core height {height} within {} ms: {}", + hex::encode(quorum.1), + quorum.0, + started.elapsed().as_millis(), + failures.join("; ") + ); + tracing::warn!(%reason, "quorum proof fetch failed"); + Err(reason) + } + + /// Judge a relay's answer, publishing the key if it proves one. + /// + /// Only what an honest relay never does cools a source down: failed or + /// oversized transfers, redirects, error statuses other than 502, and + /// proofs that do not verify. A valid proof that does not serve the + /// request may be the fault of the unverified height or quorum, so it is + /// retried instead. + fn judge( + &self, + quorum: QuorumId, + height: u32, + ask: &Ask, + reply: Result, + retries: u32, + ) -> Verdict { + let retry = |reason: String| Verdict::Retry { + reason, + after: self + .timing + .first_retry + .saturating_mul(1 << retries.min(MAX_RETRY_DOUBLINGS)), + }; + let reply = match reply { + Ok(reply) => reply, + Err(PostError::TooLarge) => { + return Verdict::SourceFailed("answered with more than a proof can be".to_string()) + } + Err(PostError::Failed(reason)) => return Verdict::SourceFailed(reason), + }; + let body = &reply.body; + match reply.status { + 200 => {} + // The relay's Core could not serve this proof (yet): worth asking + // again, for the other height. + 502 => return retry(format!("HTTP 502{}", body_excerpt(body))), + // Anything else, 503 (workers busy) included, cools the source down. + status => return Verdict::SourceFailed(format!("HTTP {status}{}", body_excerpt(body))), + } + let verified = match bootstrap::verify(body, &ask.checkpoint, 0) { + Ok(verified) => verified, + Err(error) => return Verdict::SourceFailed(format!("invalid proof: {error}")), + }; + let target = verified.state().height; + let needed = height.saturating_sub(self.window); + if target < needed && ask.height == 0 { + return Verdict::BeyondTip(format!( + "its latest ChainLock, at core height {target}, is below the {needed} that core \ + height {height} needs" + )); + } + if target < needed { + return retry(format!( + "proof ends at core height {target}, below {needed}" + )); + } + let mut keys = Vec::new(); + // EvoNode records are not asked for; any that come back are ignored. + for record in verified.records() { + if record.kind != RecordKind::Quorum { + continue; + } + match parse_commitment(record.leaf) { + Ok(commitment) => { + let mut hash = commitment.quorum_hash; + hash.reverse(); + if (u32::from(commitment.quorum_type), hash) == quorum { + keys.push(commitment.public_key); + } + } + // Records of other quorums do not matter; one that names the + // requested quorum must parse. + Err(error) if names(record.leaf, quorum) => { + return retry(format!("unparsable record of the quorum: {error}")); + } + Err(_) => {} + } + } + let [key] = keys.as_slice() else { + return retry(format!( + "proof opens the requested quorum {} times, not once", + keys.len() + )); + }; + match self.publish(quorum, *key, verified.state()) { + Ok(()) => Verdict::Proven, + Err(reason) => retry(reason), + } + } + + /// Cache a proven key and remember the state that proved it. + fn publish(&self, quorum: QuorumId, key: QuorumKey, target: &State) -> Result<(), String> { + let mut proven = self.lock(); + match proven.keys.get_mut(&quorum) { + Some(cached) if cached.key != key => { + tracing::error!( + quorum_type = quorum.0, + quorum_hash = %hex::encode(quorum.1), + cached = %hex::encode(cached.key), + proven = %hex::encode(key), + "a proof opened a different key for a quorum already proven; keeping the first" + ); + return Err("opens a different key than one already proven".to_string()); + } + Some(cached) => cached.active_at = cached.active_at.max(target.height), + None => { + proven.keys.put( + quorum, + Cached { + key, + active_at: target.height, + }, + ); + } + } + match proven.states.get(&target.height) { + Some(known) if known != target => tracing::error!( + height = target.height, + known = %rpc_hex(&known.block_hash), + proven = %rpc_hex(&target.block_hash), + "two proofs verified different blocks at one height; keeping the first" + ), + Some(_) => {} + None => { + proven.states.insert(target.height, target.clone()); + // One state is added at a time, so one removal keeps the bound. + if proven.states.len() > MAX_RECENT_STATES + 1 { + let oldest = proven + .states + .keys() + .copied() + .find(|height| *height != self.checkpoint.height); + if let Some(height) = oldest { + proven.states.remove(&height); + } + } + } + } + Ok(()) + } + + /// The source to ask next: a healthy one with no request of this fetch + /// in flight, asked the fewest times so far. + fn pick_source(&self, now: Instant, asked: &[u32], busy: &[usize]) -> Option { + let proven = self.lock(); + proven + .sources + .iter() + .enumerate() + .filter(|(source, health)| { + !busy.contains(source) && health.failed_until.is_none_or(|until| until <= now) + }) + .min_by_key(|(source, _)| asked[*source]) + .map(|(source, _)| source) + } + + fn source_failed(&self, source: usize) { + let first = self.timing.source_cooldown; + let max = self.timing.max_source_cooldown; + if let Some(health) = self.lock().sources.get_mut(source) { + health.failures = health.failures.saturating_add(1); + health.failed_until = Some(Instant::now() + backoff(first, max, health.failures)); + } + } + + fn source_answered(&self, source: usize) { + if let Some(health) = self.lock().sources.get_mut(source) { + *health = SourceHealth::default(); + } + } +} + +/// Whether a commitment's leading bytes (version, type, hash) name `quorum`. +fn names(leaf: &[u8], (quorum_type, quorum_hash): QuorumId) -> bool { + let mut wire = quorum_hash; + wire.reverse(); + leaf.get(2).map(|t| u32::from(*t)) == Some(quorum_type) && leaf.get(3..35) == Some(&wire[..]) +} + +/// The JSON a relay's `/proofs` takes. Relays reject unknown fields. +fn request_body(ask: &Ask, (quorum_type, quorum_hash): QuorumId) -> String { + serde_json::json!({ + "checkpoint": rpc_hex(&ask.checkpoint.block_hash), + "height": ask.height, + "quorumHash": hex::encode(quorum_hash), + "llmqType": quorum_type, + "nodeCount": 0, + }) + .to_string() +} + +/// The start of an error body, for the failure reason. +fn body_excerpt(body: &[u8]) -> String { + let text: String = String::from_utf8_lossy(body) + .chars() + .filter(|c| !c.is_control()) + .take(MAX_REASON_BODY_CHARS) + .collect(); + if text.is_empty() { + text + } else { + format!(": {text}") + } +} + +/// Run a fetch so that a panic becomes a failed fetch instead of being raised +/// again in every waiter of the shared future. +#[cfg(not(target_arch = "wasm32"))] +async fn contained(prove: F) -> Result<(), String> +where + F: FnOnce() -> Fut, + Fut: Future>, +{ + match AssertUnwindSafe(async move { prove().await }) + .catch_unwind() + .await + { + Ok(result) => result, + Err(payload) => { + let message = panic_message(payload.as_ref()); + tracing::error!(%message, "quorum proof fetch panicked"); + Err(format!("quorum proof fetch panicked: {message}")) + } + } +} + +/// Run a fetch on the local executor, where browser requests and timers have +/// to run, so it finishes even if every waiter goes away. +#[cfg(target_arch = "wasm32")] +async fn contained(prove: F) -> Result<(), String> +where + F: FnOnce() -> Fut + Send + 'static, + Fut: Future> + 'static, +{ + on_local_executor(prove) + .await + .unwrap_or_else(|| Err("quorum proof task was dropped".to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + use dash_core_proof::ProofNetwork; + + fn state(height: u32) -> State { + State { + network: ProofNetwork::Testnet, + height, + block_hash: [height as u8; 32], + masternode_root: [1; 32], + quorum_root: [2; 32], + } + } + + fn states(heights: &[u32]) -> BTreeMap { + heights + .iter() + .map(|height| (*height, state(*height))) + .collect() + } + + fn ask(from: u32, height: u32) -> Option { + Some(Ask { + checkpoint: state(from), + height, + }) + } + + #[test] + fn should_ask_from_the_newest_state_below_the_height_but_never_fall_back_to_the_checkpoint() { + let cases = [ + // (verified heights, height, alternate, expected, why) + (&[100][..], 150, false, ask(100, 150), "checkpoint below"), + (&[100], 150, true, ask(100, 0), "alternate asks for the tip"), + (&[100], 100, false, ask(100, 0), "nothing below: the tip"), + ( + &[100], + 50, + false, + ask(100, 0), + "height under the checkpoint", + ), + (&[100, 200], 250, false, ask(200, 250), "later state below"), + ( + &[100, 200], + 150, + false, + ask(200, 0), + "no downgrade to checkpoint", + ), + ( + &[100, 200, 300], + 250, + false, + ask(200, 250), + "verified state below", + ), + ( + &[100, 200, 300], + 250, + true, + ask(200, 0), + "alternate from it", + ), + ]; + for (heights, height, alternate, expected, why) in cases { + assert_eq!( + choose_ask(&states(heights), 100, height, alternate), + expected, + "{why}" + ); + } + } + + #[test] + fn should_double_cooldowns_up_to_their_cap() { + let (first, max) = (Duration::from_secs(5), Duration::from_secs(60)); + let waits: Vec<_> = (1..=6) + .map(|failures| backoff(first, max, failures).as_secs()) + .collect(); + assert_eq!(waits, [5, 10, 20, 40, 60, 60]); + assert_eq!(backoff(first, max, u32::MAX), max); + } + + fn fetches() -> ProofFetches { + ProofFetches::new( + 6, + 600, + state(100), + vec![Url::parse("https://relay.example").expect("a URL")], + NonZeroUsize::new(4).expect("non-zero"), + ProofTransport::new().expect("a client"), + Timing::default(), + ) + } + + #[test] + fn should_keep_the_first_key_proven_for_a_quorum() { + let fetches = fetches(); + let quorum = (6, [9; 32]); + fetches + .publish(quorum, [1; 48], &state(200)) + .expect("first proof"); + assert!(fetches.publish(quorum, [2; 48], &state(300)).is_err()); + assert_eq!(fetches.lookup(6, [9; 32], 800).ok(), Some([1; 48])); + assert!( + fetches.lookup(6, [9; 32], 801).is_err(), + "the conflicting proof must not extend the window" + ); + assert_eq!(fetches.latest_state(), state(200)); + } + + #[test] + fn should_never_lower_a_quorums_active_height() { + let fetches = fetches(); + let quorum = (6, [9; 32]); + fetches + .publish(quorum, [1; 48], &state(300)) + .expect("later"); + fetches + .publish(quorum, [1; 48], &state(200)) + .expect("earlier"); + assert_eq!(fetches.lookup(6, [9; 32], 900).ok(), Some([1; 48])); + assert_eq!(fetches.latest_state(), state(300)); + } + + #[test] + fn should_keep_the_checkpoint_and_the_most_recent_states() { + let fetches = fetches(); + for height in 101..=140 { + fetches + .publish((6, [height as u8; 32]), [1; 48], &state(height)) + .expect("a proof"); + } + let proven = fetches.lock(); + let heights: Vec<_> = proven.states.keys().copied().collect(); + assert_eq!(heights.len(), MAX_RECENT_STATES + 1); + assert_eq!(heights[0], 100, "the checkpoint stays"); + assert_eq!(heights[1], 140 - MAX_RECENT_STATES as u32 + 1); + } +} diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_transport.rs b/packages/rs-sdk-trusted-context-provider/src/proof_transport.rs new file mode 100644 index 00000000000..e41747f2b75 --- /dev/null +++ b/packages/rs-sdk-trusted-context-provider/src/proof_transport.rs @@ -0,0 +1,325 @@ +//! HTTP POSTs to proof relays. +//! +//! A relay is an untrusted byte source: whatever it returns is verified +//! against a checkpoint before anything is believed. The transport only has +//! to keep a relay from wasting the caller's time or memory, so every request +//! has a deadline, redirects are refused, and a body is read only up to +//! [`MAX_PROOF_BYTES`]. +//! +//! Native builds use one `reqwest` client. Browser builds call `fetch` +//! directly, because `reqwest` on wasm32 can neither refuse redirects nor +//! bound a body without buffering it whole; those futures are not `Send`, so +//! the caller runs them on the local executor. + +use dash_core_proof::MAX_PROOF_BYTES; +use futures::{Stream, StreamExt}; +use std::pin::pin; +use std::time::Duration; +use url::Url; + +#[cfg(not(target_arch = "wasm32"))] +use futures::stream::try_unfold; +#[cfg(not(target_arch = "wasm32"))] +use reqwest::header::CONTENT_TYPE; +#[cfg(not(target_arch = "wasm32"))] +use reqwest::redirect::Policy; +#[cfg(not(target_arch = "wasm32"))] +use std::error::Error as StdError; + +#[cfg(target_arch = "wasm32")] +use futures::future::{select, Either}; +#[cfg(target_arch = "wasm32")] +use gloo_timers::future::sleep; +#[cfg(target_arch = "wasm32")] +use js_sys::{Error as JsError, Function, Promise, Reflect, Uint8Array}; +#[cfg(target_arch = "wasm32")] +use wasm_bindgen::{JsCast, JsValue}; +#[cfg(target_arch = "wasm32")] +use wasm_bindgen_futures::JsFuture; +#[cfg(target_arch = "wasm32")] +use web_sys::{ + AbortController, Headers, Request, RequestCache, RequestCredentials, RequestInit, + RequestRedirect, Response, +}; + +/// How long connecting to a relay may take. +#[cfg(not(target_arch = "wasm32"))] +const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); + +/// A relay's answer: its status and at most [`MAX_PROOF_BYTES`] of body. +#[derive(Debug)] +pub(crate) struct Reply { + pub(crate) status: u16, + pub(crate) body: Vec, +} + +/// Why a POST got no usable answer. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum PostError { + /// The request failed or timed out before a complete answer arrived. + Failed(String), + /// The body is larger than a proof can be. + TooLarge, +} + +/// Read a body chunk by chunk, refusing one larger than [`MAX_PROOF_BYTES`] +/// before buffering past the limit. `declared` is the body length the +/// response announced, if any. +async fn read_capped(declared: Option, chunks: S) -> Result, PostError> +where + S: Stream>, + C: AsRef<[u8]>, +{ + if declared.is_some_and(|length| length > MAX_PROOF_BYTES as u64) { + return Err(PostError::TooLarge); + } + let mut chunks = pin!(chunks); + let mut body = Vec::new(); + while let Some(chunk) = chunks.next().await { + let chunk = chunk?; + let chunk = chunk.as_ref(); + if chunk.len() > MAX_PROOF_BYTES - body.len() { + return Err(PostError::TooLarge); + } + body.extend_from_slice(chunk); + } + Ok(body) +} + +/// Sends proof requests to relays. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) struct ProofTransport { + client: reqwest::Client, +} + +#[cfg(not(target_arch = "wasm32"))] +impl ProofTransport { + /// Build the client every request of a provider shares. + pub(crate) fn new() -> Result { + let client = reqwest::Client::builder() + .redirect(Policy::none()) + .connect_timeout(CONNECT_TIMEOUT) + .user_agent(concat!("DashSDK-verified/", env!("CARGO_PKG_VERSION"))) + .build()?; + Ok(Self { client }) + } + + /// POST a JSON `body` to `url`, giving up after `timeout`. + pub(crate) async fn post( + &self, + url: &Url, + body: String, + timeout: Duration, + ) -> Result { + let failed = |error: reqwest::Error| PostError::Failed(with_causes(&error)); + let response = self + .client + .post(url.clone()) + .header(CONTENT_TYPE, "application/json") + .body(body) + .timeout(timeout) + .send() + .await + .map_err(failed)?; + let status = response.status().as_u16(); + let declared = response.content_length(); + let chunks = try_unfold(response, move |mut response| async move { + Ok(response + .chunk() + .await + .map_err(failed)? + .map(|chunk| (chunk, response))) + }); + let body = read_capped(declared, chunks).await?; + Ok(Reply { status, body }) + } +} + +/// An error with every cause it carries, which `Display` alone leaves out. +#[cfg(not(target_arch = "wasm32"))] +fn with_causes(error: &dyn StdError) -> String { + let mut message = error.to_string(); + let mut source = error.source(); + while let Some(cause) = source { + message.push_str(": "); + message.push_str(&cause.to_string()); + source = cause.source(); + } + message +} + +/// Sends proof requests to relays with the browser's `fetch`. +#[cfg(target_arch = "wasm32")] +pub(crate) struct ProofTransport; + +#[cfg(target_arch = "wasm32")] +impl ProofTransport { + /// Nothing to build: every request calls the global `fetch`. + pub(crate) fn new() -> Result { + Ok(Self) + } + + /// POST a JSON `body` to `url`, giving up after `timeout`. + /// + /// The request refuses redirects, bypasses the HTTP cache and sends no + /// credentials. It is aborted when the deadline passes or the returned + /// future is dropped. Must run on the local executor. + pub(crate) async fn post( + &self, + url: &Url, + body: String, + timeout: Duration, + ) -> Result { + let controller = AbortController::new().map_err(js_failure)?; + let _abort = AbortOnDrop(controller.clone()); + let exchange = pin!(browser_post(url, body, &controller)); + let deadline = pin!(sleep(timeout)); + match select(exchange, deadline).await { + Either::Left((reply, _)) => reply, + Either::Right(_) => Err(PostError::Failed(format!( + "no answer within {} ms", + timeout.as_millis() + ))), + } + } +} + +/// Aborts a browser request when dropped, so an abandoned request does not +/// keep downloading. +#[cfg(target_arch = "wasm32")] +struct AbortOnDrop(AbortController); + +#[cfg(target_arch = "wasm32")] +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.abort(); + } +} + +#[cfg(target_arch = "wasm32")] +fn js_failure(error: JsValue) -> PostError { + PostError::Failed(match error.dyn_ref::() { + Some(error) => format!( + "{}: {}", + String::from(error.name()), + String::from(error.message()) + ), + None => error + .as_string() + .unwrap_or_else(|| "fetch failed".to_string()), + }) +} + +/// The browser half of [`ProofTransport::post`], without the deadline. +/// +/// `fetch` is looked up on the global object so the same code runs in a +/// window, a worker and Node.js. +#[cfg(target_arch = "wasm32")] +async fn browser_post( + url: &Url, + body: String, + controller: &AbortController, +) -> Result { + let headers = Headers::new().map_err(js_failure)?; + headers + .set("Content-Type", "application/json") + .map_err(js_failure)?; + let init = RequestInit::new(); + init.set_method("POST"); + init.set_body(&JsValue::from_str(&body)); + init.set_headers(&headers); + init.set_redirect(RequestRedirect::Error); + init.set_cache(RequestCache::NoStore); + init.set_credentials(RequestCredentials::Omit); + init.set_signal(Some(&controller.signal())); + let request = Request::new_with_str_and_init(url.as_str(), &init).map_err(js_failure)?; + + let global = js_sys::global(); + let fetch: Function = Reflect::get(&global, &JsValue::from_str("fetch")) + .map_err(js_failure)? + .dyn_into() + .map_err(|_| PostError::Failed("no global fetch".to_string()))?; + let promise: Promise = fetch + .call1(&global, &request) + .map_err(js_failure)? + .dyn_into() + .map_err(|_| PostError::Failed("fetch did not return a promise".to_string()))?; + let response: Response = JsFuture::from(promise) + .await + .map_err(js_failure)? + .dyn_into() + .map_err(|_| PostError::Failed("fetch did not return a response".to_string()))?; + + let status = response.status(); + let declared = response + .headers() + .get("Content-Length") + .ok() + .flatten() + .and_then(|length| length.trim().parse::().ok()); + let Some(raw) = response.body() else { + return Ok(Reply { + status, + body: Vec::new(), + }); + }; + let chunks = wasm_streams::ReadableStream::from_raw(raw) + .try_into_stream() + .map_err(|_| PostError::Failed("response body is locked".to_string()))? + .map(|chunk| { + let chunk: Uint8Array = chunk + .map_err(js_failure)? + .dyn_into() + .map_err(|_| PostError::Failed("a body chunk is not bytes".to_string()))?; + // Refuse an oversized chunk before copying it out of JavaScript. + if chunk.length() as usize > MAX_PROOF_BYTES { + return Err(PostError::TooLarge); + } + Ok(chunk.to_vec()) + }); + let body = read_capped(declared, chunks).await?; + Ok(Reply { status, body }) +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::stream; + + fn chunks(sizes: &[usize]) -> impl Stream, PostError>> { + stream::iter( + sizes + .iter() + .map(|size| Ok(vec![7; *size])) + .collect::>(), + ) + } + + #[tokio::test] + async fn should_read_a_body_up_to_the_proof_limit() { + let body = read_capped(None, chunks(&[MAX_PROOF_BYTES - 1, 1])) + .await + .expect("a body at the limit"); + assert_eq!(body.len(), MAX_PROOF_BYTES); + } + + #[tokio::test] + async fn should_refuse_a_body_past_the_proof_limit_without_buffering_it() { + let endless = stream::repeat_with(|| Ok::<_, PostError>(vec![0; 4096])); + assert_eq!(read_capped(None, endless).await, Err(PostError::TooLarge)); + assert_eq!( + read_capped(None, chunks(&[MAX_PROOF_BYTES + 1])).await, + Err(PostError::TooLarge) + ); + } + + #[tokio::test] + async fn should_refuse_a_declared_length_past_the_proof_limit_before_reading() { + let unreadable = + stream::once(async { Err::, _>(PostError::Failed("read".to_string())) }); + assert_eq!( + read_capped(Some(MAX_PROOF_BYTES as u64 + 1), unreadable).await, + Err(PostError::TooLarge) + ); + } +} diff --git a/packages/rs-sdk-trusted-context-provider/src/quorum_refresh.rs b/packages/rs-sdk-trusted-context-provider/src/quorum_refresh.rs index 034d50092d3..e1b5a3bac7b 100644 --- a/packages/rs-sdk-trusted-context-provider/src/quorum_refresh.rs +++ b/packages/rs-sdk-trusted-context-provider/src/quorum_refresh.rs @@ -15,6 +15,8 @@ use crate::types::QuorumData; use futures::future::{BoxFuture, FutureExt, Shared}; +#[cfg(not(target_arch = "wasm32"))] +use std::any::Any; use std::future::Future; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; @@ -271,17 +273,23 @@ where { Ok(fetched) => fetched, Err(payload) => { - let message = payload - .downcast_ref::<&str>() - .map(|message| message.to_string()) - .or_else(|| payload.downcast_ref::().cloned()) - .unwrap_or_default(); + let message = panic_message(payload.as_ref()); tracing::error!(%message, "quorum list refresh panicked"); FetchedQuorums::failed(&format!("quorum refresh panicked: {message}")) } } } +/// The message a panic was raised with, if it has one. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn panic_message(payload: &(dyn Any + Send)) -> String { + payload + .downcast_ref::<&str>() + .map(|message| message.to_string()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_default() +} + /// Run a refresh as a `Send` future. /// /// Browser fetches are not `Send`, so the refresh is built and run on the @@ -317,7 +325,7 @@ pub(crate) async fn sleep(duration: Duration) { /// transport. `None` if the task is dropped before it finishes, which happens /// only when the executor goes away. #[cfg(target_arch = "wasm32")] -async fn on_local_executor(make: F) -> Option +pub(crate) async fn on_local_executor(make: F) -> Option where F: FnOnce() -> Fut + Send + 'static, Fut: Future + 'static, diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs new file mode 100644 index 00000000000..970b45598d6 --- /dev/null +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -0,0 +1,1112 @@ +//! A context provider whose Platform quorum keys are proven, not trusted. +//! +//! [`VerifiedHttpContextProvider`] starts from a checkpoint: a ChainLocked +//! Core block and the quorum and masternode Merkle roots its coinbase commits +//! to. The checkpoint is trust configuration, embedded in this crate per +//! network or supplied by the application. When the SDK meets a Platform +//! quorum it has no key for, the provider asks a relay for a `DASHNC02` proof +//! (DIP 175) and believes the key only if the proof verifies from a state it +//! already trusts. Relays are untrusted byte sources; a lying relay can delay +//! requests but cannot make the SDK accept a forged key or ban honest nodes. + +use crate::get_quorum_base_url; +use crate::proof_fetches::{ProofFetches, Timing}; +use crate::proof_transport::ProofTransport; +use crate::system_contracts::{platform_activation_height, system_data_contract}; +use dash_context_provider::{ContextProvider, ContextProviderError, QuorumKeyFuture}; +use dash_core_proof::{ProofNetwork, State}; +use dpp::dashcore::sml::llmq_type::network::NetworkLLMQExt; +use dpp::dashcore::Network; +use dpp::data_contract::accessors::v0::DataContractV0Getters; +use dpp::data_contract::TokenConfiguration; +use dpp::prelude::{CoreBlockHeight, DataContract, Identifier}; +use dpp::version::PlatformVersion; +use std::collections::HashMap; +use std::num::NonZeroUsize; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use thiserror::Error; +use url::{Host, Url}; + +/// Most relays one provider asks. +const MAX_SOURCES: usize = 8; + +/// The embedded mainnet checkpoint, Core `ProofState` JSON. +const MAINNET_CHECKPOINT: &str = include_str!("../checkpoints/mainnet.json"); +/// The embedded testnet checkpoint, Core `ProofState` JSON. +const TESTNET_CHECKPOINT: &str = include_str!("../checkpoints/testnet.json"); + +/// Why a [`VerifiedHttpContextProvider`] could not be built. +#[derive(Debug, Error)] +pub enum VerifiedContextProviderError { + /// Only mainnet and testnet have the fixed ChainLock quorum types the + /// proof format relies on. + #[error("unsupported network for verified quorum keys: {0:?}")] + UnsupportedNetwork(Network), + + /// The checkpoint is malformed, outside the supported era or for another + /// network. + #[error("invalid checkpoint: {0}")] + InvalidCheckpoint(String), + + /// A relay URL is not one the provider may ask. + #[error("invalid proof source: {0}")] + InvalidSource(String), + + /// The HTTP client could not be built. + #[error("HTTP client error: {0}")] + Http(#[from] reqwest::Error), +} + +/// A [`ContextProvider`] that serves Platform quorum keys only after proving +/// them with DIP 175 proofs from a checkpoint. +/// +/// Opt in with `SdkBuilder::with_context_provider` or +/// `Sdk::set_context_provider`. Clones share their caches. +/// +/// Quorum keys: +/// * [`get_quorum_public_key`](ContextProvider::get_quorum_public_key) serves +/// a cached key if the quorum was proven active within a validity window of +/// the requested height (600 blocks on mainnet and testnet). Otherwise it +/// reports the source as unavailable, and the SDK calls +/// [`fetch_quorum_public_key`](ContextProvider::fetch_quorum_public_key). +/// * The fetch POSTs to each source's `/proofs`, verifies the proof from the +/// newest verified state, caches the key and returns it. Any failure +/// (timeouts, HTTP errors, oversize or invalid proofs, a proof of another +/// quorum) is reported as an unavailable source, so the SDK moves to +/// another node without banning the one that named the quorum. +/// * A quorum of a type other than the network's Platform type is rejected +/// as invalid and never fetched. +/// +/// Fetches are shared per quorum, at most two quorums are proven at once, +/// failed quorums and failing sources cool down, and each fetch has a 70 s +/// deadline. Freshness of responses is the SDK's concern, not the +/// provider's. +/// +/// The public mainnet relay currently answers `/proofs` with HTTP 403, so on +/// mainnet [`new`](Self::new) builds but fetches fail until a relay serves +/// proofs; pass working sources to [`with_checkpoint`](Self::with_checkpoint). +/// +/// Data contracts and token configurations come from the known caches below +/// and the system contracts enabled by cargo features; nothing else is +/// fetched. +#[derive(Clone)] +pub struct VerifiedHttpContextProvider { + inner: Arc, +} + +struct Inner { + network: Network, + sources: Vec, + fetches: Arc, + known_contracts: Mutex>>, + known_token_configurations: Mutex>, +} + +/// Lock a cache; every update under these locks completes before release. +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +fn proof_network(network: Network) -> Result { + match network { + Network::Mainnet => Ok(ProofNetwork::Mainnet), + Network::Testnet => Ok(ProofNetwork::Testnet), + other => Err(VerifiedContextProviderError::UnsupportedNetwork(other)), + } +} + +/// A checkpoint a proof can start from. +fn validated(checkpoint: State) -> Result { + checkpoint + .validate() + .map_err(|error| VerifiedContextProviderError::InvalidCheckpoint(error.to_string()))?; + Ok(checkpoint) +} + +/// Parse a relay base URL. It must be absolute `https`, or `http` to a +/// loopback host, and carry no credentials, query or fragment. The host is +/// not resolved. +fn parse_source(text: &str) -> Result { + let invalid = + |why: &str| VerifiedContextProviderError::InvalidSource(format!("'{text}' {why}")); + let url = Url::parse(text).map_err(|error| invalid(&format!("is not a URL: {error}")))?; + if !url.username().is_empty() || url.password().is_some() { + return Err(invalid("must not carry credentials")); + } + if url.query().is_some() || url.fragment().is_some() { + return Err(invalid("must not carry a query or fragment")); + } + let loopback = match url.host() { + Some(Host::Ipv4(address)) => address.is_loopback(), + Some(Host::Ipv6(address)) => address.is_loopback(), + Some(Host::Domain(domain)) => domain.eq_ignore_ascii_case("localhost"), + None => return Err(invalid("has no host")), + }; + match url.scheme() { + "https" => Ok(url), + "http" if loopback => Ok(url), + "http" => Err(invalid("must use https unless it is a loopback address")), + _ => Err(invalid("must use https")), + } +} + +/// The `/proofs` endpoint under a relay base URL. +fn proofs_endpoint(source: &Url) -> Result { + let mut endpoint = source.clone(); + endpoint + .path_segments_mut() + .map_err(|()| { + VerifiedContextProviderError::InvalidSource(format!("'{source}' cannot be a base")) + })? + .pop_if_empty() + .push("proofs"); + Ok(endpoint) +} + +impl VerifiedHttpContextProvider { + /// A provider for mainnet or testnet that starts from the embedded + /// checkpoint and asks the network's public quorum service. Makes no + /// network request. + pub fn new( + network: Network, + cache_size: NonZeroUsize, + ) -> Result { + let checkpoint = Self::embedded_checkpoint(network)?; + let source = get_quorum_base_url(network, None) + .map_err(|_| VerifiedContextProviderError::UnsupportedNetwork(network))?; + Self::with_checkpoint(network, checkpoint, vec![source], cache_size) + } + + /// A provider that starts from `checkpoint` and asks `sources`, 1 to 8 + /// relay base URLs whose `/proofs` endpoint serves Core's + /// `getquorumproofchain` bootstrap proofs. + /// + /// The checkpoint is trust configuration: a ChainLocked block the + /// application checked independently, or a state this provider verified + /// earlier ([`latest_verified_state`](Self::latest_verified_state)). + /// Sources must be `https`, except on loopback hosts. Hosts are not + /// resolved here. + pub fn with_checkpoint( + network: Network, + checkpoint: State, + sources: Vec, + cache_size: NonZeroUsize, + ) -> Result { + Self::build(network, checkpoint, sources, cache_size, Timing::default()) + } + + fn build( + network: Network, + checkpoint: State, + sources: Vec, + cache_size: NonZeroUsize, + timing: Timing, + ) -> Result { + let expected = proof_network(network)?; + let checkpoint = validated(checkpoint)?; + if checkpoint.network != expected { + return Err(VerifiedContextProviderError::InvalidCheckpoint(format!( + "checkpoint is for {:?}, provider for {network:?}", + checkpoint.network + ))); + } + if !(1..=MAX_SOURCES).contains(&sources.len()) { + return Err(VerifiedContextProviderError::InvalidSource(format!( + "{} sources given, expected 1 to {MAX_SOURCES}", + sources.len() + ))); + } + let sources = sources + .iter() + .map(|source| parse_source(source)) + .collect::, _>>()?; + if let Some(repeated) = sources + .iter() + .enumerate() + .find_map(|(index, source)| sources[..index].contains(source).then_some(source)) + { + return Err(VerifiedContextProviderError::InvalidSource(format!( + "'{repeated}' is given twice" + ))); + } + let endpoints = sources + .iter() + .map(proofs_endpoint) + .collect::, _>>()?; + + let platform_type = network.platform_type(); + // A Platform quorum signs while it is among the newest + // `signing_active_quorum_count` quorums, and one is formed every DKG + // interval; one more interval covers a quorum mined late in it. + let params = platform_type.params(); + let window = params + .dkg_params + .interval + .saturating_mul(params.signing_active_quorum_count.saturating_add(1)); + let fetches = ProofFetches::new( + u32::from(u8::from(platform_type)), + window, + checkpoint, + endpoints, + cache_size, + ProofTransport::new()?, + timing, + ); + Ok(Self { + inner: Arc::new(Inner { + network, + sources, + fetches: Arc::new(fetches), + known_contracts: Mutex::new(HashMap::new()), + known_token_configurations: Mutex::new(HashMap::new()), + }), + }) + } + + /// The checkpoint this crate embeds for `network`, validated. + pub fn embedded_checkpoint(network: Network) -> Result { + let json = match proof_network(network)? { + ProofNetwork::Mainnet => MAINNET_CHECKPOINT, + ProofNetwork::Testnet => TESTNET_CHECKPOINT, + }; + serde_json::from_str(json) + .map_err(|error| VerifiedContextProviderError::InvalidCheckpoint(error.to_string())) + .and_then(validated) + } + + /// The checkpoint the provider was built with. + pub fn checkpoint(&self) -> &State { + self.inner.fetches.checkpoint() + } + + /// The highest state a proof has verified, or the checkpoint. An + /// application may persist it and pass it back as the checkpoint, so + /// later proofs are shorter. + pub fn latest_verified_state(&self) -> State { + self.inner.fetches.latest_state() + } + + /// The relay base URLs the provider asks. + pub fn sources(&self) -> &[Url] { + &self.inner.sources + } + + /// How many proven quorum keys are cached. + pub fn cached_quorum_count(&self) -> usize { + self.inner.fetches.cached_quorum_count() + } + + /// Set known contracts that will be served immediately. + pub fn with_known_contracts(self, contracts: Vec) -> Self { + self.add_known_contracts(contracts); + self + } + + /// Add a data contract to the known contracts cache. + pub fn add_known_contract(&self, contract: DataContract) { + lock(&self.inner.known_contracts).insert(contract.id(), Arc::new(contract)); + } + + /// Add data contracts to the known contracts cache. + pub fn add_known_contracts(&self, contracts: Vec) { + let mut known = lock(&self.inner.known_contracts); + for contract in contracts { + known.insert(contract.id(), Arc::new(contract)); + } + } + + /// Insert a contract only when its version is strictly newer, returning + /// the retained entry. Equal versions preserve the existing body. + pub fn add_known_contract_if_newer(&self, contract: DataContract) -> Arc { + let mut known = lock(&self.inner.known_contracts); + let id = contract.id(); + if let Some(existing) = known.get(&id) { + if existing.version() >= contract.version() { + return Arc::clone(existing); + } + } + let contract = Arc::new(contract); + known.insert(id, Arc::clone(&contract)); + contract + } + + /// A data contract from the known contracts cache. + pub fn get_known_contract(&self, id: &Identifier) -> Option> { + lock(&self.inner.known_contracts).get(id).cloned() + } + + /// Remove a data contract from the known contracts cache, returning + /// whether it was there. + pub fn remove_known_contract(&self, id: &Identifier) -> bool { + lock(&self.inner.known_contracts).remove(id).is_some() + } + + /// Add a token configuration to the known token configurations cache. + pub fn add_known_token_configuration(&self, token_id: Identifier, config: TokenConfiguration) { + lock(&self.inner.known_token_configurations).insert(token_id, config); + } + + /// Add token configurations to the known token configurations cache. + pub fn add_known_token_configurations(&self, configs: Vec<(Identifier, TokenConfiguration)>) { + lock(&self.inner.known_token_configurations).extend(configs); + } +} + +impl ContextProvider for VerifiedHttpContextProvider { + fn get_data_contract( + &self, + id: &Identifier, + platform_version: &PlatformVersion, + ) -> Result>, ContextProviderError> { + if let Some(contract) = self.get_known_contract(id) { + return Ok(Some(contract)); + } + system_data_contract(id, platform_version) + } + + fn register_data_contract(&self, contract: Arc) { + lock(&self.inner.known_contracts).insert(contract.id(), contract); + } + + fn get_token_configuration( + &self, + token_id: &Identifier, + ) -> Result, ContextProviderError> { + Ok(lock(&self.inner.known_token_configurations) + .get(token_id) + .cloned()) + } + + fn get_quorum_public_key( + &self, + quorum_type: u32, + quorum_hash: [u8; 32], + core_chain_locked_height: u32, + ) -> Result<[u8; 48], ContextProviderError> { + self.inner + .fetches + .lookup(quorum_type, quorum_hash, core_chain_locked_height) + } + + fn fetch_quorum_public_key( + &self, + quorum_type: u32, + quorum_hash: [u8; 32], + core_chain_locked_height: u32, + ) -> Option { + if !self + .inner + .fetches + .provable(quorum_type, core_chain_locked_height) + { + return None; + } + let fetches = Arc::clone(&self.inner.fetches); + // Never `Ok(None)`: a proof can only show that a quorum exists, so a + // failure to prove one says nothing about the node that named it. + Some(Box::pin(async move { + fetches + .fetch(quorum_type, quorum_hash, core_chain_locked_height) + .await + .map(Some) + })) + } + + fn get_platform_activation_height(&self) -> Result { + Ok(platform_activation_height(self.inner.network)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use dash_core_proof::bootstrap; + use dash_core_proof::{parse_commitment, rpc_hex, MAX_PROOF_BYTES}; + use futures::future::join_all; + use serde_json::{json, Value}; + use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; + use std::net::{TcpListener, TcpStream}; + use std::thread; + use std::time::{Duration, Instant}; + + /// A real `getquorumproofchain` result and the quorum it opens. + struct Fixture { + network: Network, + checkpoint: State, + bootstrap: Vec, + target: State, + quorum_type: u32, + quorum_hash: [u8; 32], + key: [u8; 48], + } + + fn fixture(network: Network) -> Fixture { + let json = match network { + Network::Mainnet => include_str!("../tests/data/proof-mainnet.json"), + _ => include_str!("../tests/data/proof-testnet.json"), + }; + let value: Value = serde_json::from_str(json).expect("fixture JSON"); + let state = |key: &str| serde_json::from_value::(value[key].clone()).expect(key); + let bootstrap = hex::decode(value["bootstrap_hex"].as_str().expect("bootstrap_hex")) + .expect("bootstrap hex"); + let checkpoint = state("checkpoint"); + let verified = bootstrap::verify(&bootstrap, &checkpoint, 0).expect("fixture verifies"); + let commitment = parse_commitment(verified.records()[0].leaf).expect("a commitment"); + let mut quorum_hash = commitment.quorum_hash; + quorum_hash.reverse(); + Fixture { + network, + checkpoint, + target: state("target"), + bootstrap, + quorum_type: u32::from(commitment.quorum_type), + quorum_hash, + key: commitment.public_key, + } + } + + /// What the mock relay does with one connection. + enum Answer { + Reply { + status: u16, + headers: Vec<(&'static str, String)>, + body: Vec, + }, + /// A chunked body of `chunks` chunks of `size` bytes. + Chunked { size: usize, chunks: usize }, + /// Read the request, then hold the connection open without answering. + Hang(Duration), + } + + fn reply(status: u16, body: Vec) -> Answer { + Answer::Reply { + status, + headers: Vec::new(), + body, + } + } + + /// A relay on a loopback port that answers connections in order and + /// records each request body. It stops after its answers, or when no + /// connection comes for a while. + struct MockRelay { + url: String, + requests: Arc>>, + } + + impl MockRelay { + fn spawn(answers: Vec) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock relay"); + listener + .set_nonblocking(true) + .expect("make the mock relay bounded"); + let url = format!("http://{}", listener.local_addr().expect("relay address")); + let requests = Arc::new(Mutex::new(Vec::new())); + let recorded = Arc::clone(&requests); + thread::spawn(move || { + for answer in answers { + let Some(mut stream) = accept(&listener) else { + return; + }; + let body = read_request(&stream); + lock(&recorded).push(body); + match answer { + Answer::Reply { + status, + headers, + body, + } => { + let mut head = format!( + "HTTP/1.1 {status} Mock\r\nConnection: close\r\n\ + Content-Type: application/octet-stream\r\n" + ); + if !headers.iter().any(|(name, _)| *name == "Content-Length") { + head.push_str(&format!("Content-Length: {}\r\n", body.len())); + } + for (name, value) in headers { + head.push_str(&format!("{name}: {value}\r\n")); + } + head.push_str("\r\n"); + // The client may hang up early; that is its right. + let _ = stream.write_all(head.as_bytes()); + let _ = stream.write_all(&body); + } + Answer::Chunked { size, chunks } => { + let _ = stream.write_all( + b"HTTP/1.1 200 OK\r\nConnection: close\r\n\ + Transfer-Encoding: chunked\r\n\r\n", + ); + let chunk = vec![0u8; size]; + for _ in 0..chunks { + let written = stream + .write_all(format!("{size:x}\r\n").as_bytes()) + .and_then(|()| stream.write_all(&chunk)) + .and_then(|()| stream.write_all(b"\r\n")); + if written.is_err() { + break; + } + } + let _ = stream.write_all(b"0\r\n\r\n"); + } + Answer::Hang(duration) => { + thread::spawn(move || { + thread::sleep(duration); + drop(stream); + }); + } + } + } + }); + Self { url, requests } + } + + fn requests(&self) -> Vec { + lock(&self.requests).clone() + } + } + + fn accept(listener: &TcpListener) -> Option { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + match listener.accept() { + Ok((stream, _)) => { + // An accepted socket inherits non-blocking mode on BSDs. + stream + .set_nonblocking(false) + .expect("make the request blocking"); + return Some(stream); + } + Err(error) + if error.kind() == ErrorKind::WouldBlock && Instant::now() < deadline => + { + thread::sleep(Duration::from_millis(5)); + } + Err(_) => return None, + } + } + } + + /// Read one request and return its JSON body. + fn read_request(stream: &TcpStream) -> Value { + let mut reader = BufReader::new(stream.try_clone().expect("clone the request stream")); + let mut length = 0; + loop { + let mut line = String::new(); + reader.read_line(&mut line).expect("read a request line"); + if line == "\r\n" || line.is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + length = value.trim().parse().expect("a content length"); + } + } + } + let mut body = vec![0; length]; + reader.read_exact(&mut body).expect("read the request body"); + serde_json::from_slice(&body).expect("a JSON request body") + } + + fn cache_size() -> NonZeroUsize { + NonZeroUsize::new(8).expect("non-zero") + } + + /// Short retry delays; everything else as in production. + fn quick() -> Timing { + Timing { + first_retry: Duration::from_millis(5), + ..Timing::default() + } + } + + fn provider_for( + f: &Fixture, + sources: Vec, + timing: Timing, + ) -> VerifiedHttpContextProvider { + VerifiedHttpContextProvider::build( + f.network, + f.checkpoint.clone(), + sources, + cache_size(), + timing, + ) + .expect("a provider") + } + + async fn fetch( + provider: &VerifiedHttpContextProvider, + quorum_type: u32, + quorum_hash: [u8; 32], + height: u32, + ) -> Result, ContextProviderError> { + provider + .fetch_quorum_public_key(quorum_type, quorum_hash, height) + .expect("a Platform quorum is fetched") + .await + } + + fn unavailable_reason(result: Result, ContextProviderError>) -> String { + match result { + Err(ContextProviderError::QuorumSourceUnavailable(reason)) => reason, + other => panic!("expected an unavailable source, got {other:?}"), + } + } + + #[tokio::test] + async fn should_prove_cache_and_return_keys_from_real_relay_proofs() { + for network in [Network::Mainnet, Network::Testnet] { + let f = fixture(network); + let relay = MockRelay::spawn(vec![reply(200, f.bootstrap.clone())]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let height = f.target.height; + assert!(matches!( + provider.get_quorum_public_key(f.quorum_type, f.quorum_hash, height), + Err(ContextProviderError::QuorumSourceUnavailable(_)) + )); + + let key = fetch(&provider, f.quorum_type, f.quorum_hash, height).await; + assert_eq!(key.expect("a proven key"), Some(f.key), "{network:?}"); + assert_eq!( + provider + .get_quorum_public_key(f.quorum_type, f.quorum_hash, height) + .expect("cached"), + f.key + ); + assert_eq!(provider.cached_quorum_count(), 1); + assert_eq!(provider.latest_verified_state(), f.target); + assert_eq!( + relay.requests(), + [json!({ + "checkpoint": rpc_hex(&f.checkpoint.block_hash), + "height": height, + "quorumHash": hex::encode(f.quorum_hash), + "llmqType": f.quorum_type, + "nodeCount": 0, + })] + ); + } + } + + #[test] + fn should_open_the_mainnet_platform_quorum_named_in_rpc_order() { + let f = fixture(Network::Mainnet); + assert_eq!(f.quorum_type, 4); + assert_eq!( + hex::encode(f.quorum_hash), + "000000000000001e3234155b7645fdd5af10c59bd1507ee6d75ded0ae4eab979" + ); + assert_eq!(fixture(Network::Testnet).quorum_type, 6); + } + + #[tokio::test] + async fn should_serve_a_key_inside_the_window_and_prove_it_again_past_it() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn(vec![ + reply(200, f.bootstrap.clone()), + reply(500, b"later".to_vec()), + ]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let proven = f.target.height; + fetch(&provider, f.quorum_type, f.quorum_hash, proven) + .await + .expect("proven"); + + assert!(provider + .get_quorum_public_key(f.quorum_type, f.quorum_hash, proven + 600) + .is_ok()); + assert!(provider + .get_quorum_public_key(f.quorum_type, f.quorum_hash, 0) + .is_ok()); + let past = proven + 601; + assert!(matches!( + provider.get_quorum_public_key(f.quorum_type, f.quorum_hash, past), + Err(ContextProviderError::QuorumSourceUnavailable(_)) + )); + + // The new proof starts from the verified target, not the checkpoint. + let reason = unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, past).await); + assert!(reason.contains("HTTP 500: later"), "{reason}"); + let requests = relay.requests(); + assert_eq!(requests[1]["checkpoint"], rpc_hex(&f.target.block_hash)); + assert_eq!(requests[1]["height"], past); + assert!( + provider + .get_quorum_public_key(f.quorum_type, f.quorum_hash, proven + 600) + .is_ok(), + "a failed proof does not take back an earlier one" + ); + } + + #[tokio::test] + async fn should_send_one_request_for_concurrent_fetches_of_a_quorum() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn(vec![reply(200, f.bootstrap.clone())]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let fetches = + (0..16).map(|_| fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height)); + for key in join_all(fetches).await { + assert_eq!(key.expect("a proven key"), Some(f.key)); + } + assert_eq!(relay.requests().len(), 1); + } + + #[tokio::test] + async fn should_give_up_on_a_relay_that_hangs_before_the_deadline() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn(vec![Answer::Hang(Duration::from_secs(5))]); + let timing = Timing { + deadline: Duration::from_millis(800), + request_timeout: Duration::from_millis(400), + ..quick() + }; + let provider = provider_for(&f, vec![relay.url.clone()], timing); + let started = Instant::now(); + let reason = unavailable_reason( + fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await, + ); + assert!(started.elapsed() < Duration::from_secs(5), "{reason}"); + assert!(reason.contains(&relay.url), "names the source: {reason}"); + assert_eq!(provider.cached_quorum_count(), 0); + } + + #[tokio::test] + async fn should_cool_down_a_refused_quorum_and_a_failing_source() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn(vec![reply(403, b"forbidden".to_vec())]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let reason = unavailable_reason( + fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await, + ); + assert!(reason.contains("HTTP 403: forbidden"), "{reason}"); + + let again = unavailable_reason( + fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await, + ); + assert!(again.contains("retry in 5 s"), "{again}"); + let other = + unavailable_reason(fetch(&provider, f.quorum_type, [0x11; 32], f.target.height).await); + assert!(other.contains("cooling down"), "{other}"); + assert_eq!(relay.requests().len(), 1, "cooling down sends nothing"); + } + + #[tokio::test] + async fn should_report_a_refused_connection_as_an_unavailable_source() { + let f = fixture(Network::Testnet); + let closed = TcpListener::bind("127.0.0.1:0").expect("bind"); + let url = format!("http://{}", closed.local_addr().expect("address")); + drop(closed); + let provider = provider_for(&f, vec![url], quick()); + let started = Instant::now(); + unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await); + assert!(started.elapsed() < Duration::from_secs(10)); + } + + #[tokio::test] + async fn should_ask_for_the_tip_after_a_bad_gateway() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn(vec![ + reply(502, b"Requested quorum is not in the target root".to_vec()), + reply(200, f.bootstrap.clone()), + ]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let height = f.target.height; + let key = fetch(&provider, f.quorum_type, f.quorum_hash, height).await; + assert_eq!(key.expect("a proven key"), Some(f.key)); + let heights: Vec<_> = relay + .requests() + .iter() + .map(|r| r["height"].clone()) + .collect(); + assert_eq!(heights, [json!(height), json!(0)]); + } + + #[tokio::test] + async fn should_not_let_a_bogus_height_hold_back_an_honest_one() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn(vec![ + reply(502, b"height beyond the tip".to_vec()), + reply(200, f.bootstrap.clone()), + reply(200, f.bootstrap.clone()), + ]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let bogus = i32::MAX as u32; + let reason = + unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, bogus).await); + assert!(reason.contains("latest ChainLock"), "{reason}"); + + // Neither the quorum nor the relay cools down for the bogus height. + let honest = f.target.height; + let key = fetch(&provider, f.quorum_type, f.quorum_hash, honest).await; + assert_eq!(key.expect("the honest height is proven"), Some(f.key)); + let heights: Vec<_> = relay + .requests() + .iter() + .map(|r| r["height"].clone()) + .collect(); + assert_eq!(heights, [json!(bogus), json!(0), json!(honest)]); + } + + #[tokio::test] + async fn should_bound_requests_to_a_relay_that_keeps_failing() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn((0..12).map(|_| reply(502, Vec::new())).collect()); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let reason = unavailable_reason( + fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await, + ); + assert!(reason.contains("HTTP 502"), "{reason}"); + assert_eq!(relay.requests().len(), 8); + } + + #[tokio::test] + async fn should_refuse_oversized_answers() { + let f = fixture(Network::Testnet); + let declared = Answer::Reply { + status: 200, + headers: vec![("Content-Length", (MAX_PROOF_BYTES + 1).to_string())], + body: Vec::new(), + }; + let chunked = Answer::Chunked { + size: 64 * 1024, + chunks: MAX_PROOF_BYTES / (64 * 1024) + 2, + }; + for answer in [declared, chunked] { + let relay = MockRelay::spawn(vec![answer]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let reason = unavailable_reason( + fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await, + ); + assert!(reason.contains("more than a proof can be"), "{reason}"); + assert_eq!(provider.cached_quorum_count(), 0); + } + } + + #[tokio::test] + async fn should_reject_a_proof_with_a_flipped_byte() { + let f = fixture(Network::Testnet); + let mut corrupt = f.bootstrap.clone(); + let middle = corrupt.len() / 2; + corrupt[middle] ^= 1; + let relay = MockRelay::spawn(vec![reply(200, corrupt)]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let reason = unavailable_reason( + fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await, + ); + assert!(reason.contains("invalid proof"), "{reason}"); + assert_eq!(provider.cached_quorum_count(), 0); + assert_eq!(provider.latest_verified_state(), f.checkpoint); + } + + #[tokio::test] + async fn should_cache_nothing_from_a_proof_of_another_quorum() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn(vec![reply(200, f.bootstrap.clone())]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let reason = + unavailable_reason(fetch(&provider, f.quorum_type, [0x11; 32], f.target.height).await); + assert!( + reason.contains("opens the requested quorum 0 times"), + "{reason}" + ); + assert_eq!(provider.cached_quorum_count(), 0); + assert_eq!(provider.latest_verified_state(), f.checkpoint); + } + + #[tokio::test] + async fn should_not_follow_redirects() { + let f = fixture(Network::Testnet); + let target = MockRelay::spawn(vec![reply(200, f.bootstrap.clone())]); + let redirect = MockRelay::spawn(vec![Answer::Reply { + status: 302, + headers: vec![("Location", format!("{}/proofs", target.url))], + body: Vec::new(), + }]); + let provider = provider_for(&f, vec![redirect.url.clone()], quick()); + let reason = unavailable_reason( + fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await, + ); + assert!(reason.contains("HTTP 302"), "{reason}"); + assert_eq!(redirect.requests().len(), 1); + assert!(target.requests().is_empty(), "the redirect was followed"); + assert_eq!(provider.cached_quorum_count(), 0); + } + + #[tokio::test] + async fn should_not_start_proving_a_third_quorum_at_once() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn(vec![ + Answer::Hang(Duration::from_secs(5)), + Answer::Hang(Duration::from_secs(5)), + ]); + let timing = Timing { + deadline: Duration::from_millis(500), + request_timeout: Duration::from_millis(500), + ..quick() + }; + let provider = provider_for(&f, vec![relay.url.clone()], timing); + let height = f.target.height; + let results = + join_all([1u8, 2, 3].map(|seed| fetch(&provider, f.quorum_type, [seed; 32], height))) + .await; + let reasons: Vec<_> = results.into_iter().map(unavailable_reason).collect(); + assert!( + reasons[2].contains("other quorums are being proven"), + "{reasons:?}" + ); + } + + #[test] + fn should_reject_quorums_it_never_proves() { + let f = fixture(Network::Testnet); + let provider = provider_for(&f, vec!["https://relay.example".to_string()], quick()); + for (quorum_type, height) in [(4, 1), (1, 1), (f.quorum_type, i32::MAX as u32 + 1)] { + assert!(provider + .fetch_quorum_public_key(quorum_type, [1; 32], height) + .is_none()); + assert!(matches!( + provider.get_quorum_public_key(quorum_type, [1; 32], height), + Err(ContextProviderError::InvalidQuorum(_)) + )); + } + } + + fn with_sources( + sources: &[&str], + ) -> Result { + let checkpoint = VerifiedHttpContextProvider::embedded_checkpoint(Network::Testnet) + .expect("embedded checkpoint"); + VerifiedHttpContextProvider::with_checkpoint( + Network::Testnet, + checkpoint, + sources.iter().map(|source| source.to_string()).collect(), + cache_size(), + ) + } + + #[test] + fn should_accept_only_https_sources_or_plain_http_on_loopback() { + for good in [ + "https://quorums.testnet.networks.dash.org", + "https://relay.example/base/", + "http://127.0.0.1:8080", + "http://127.1.2.3", + "http://[::1]:9000", + "http://localhost:1234", + // Hosts are not resolved when the provider is built. + "https://does-not-resolve.invalid", + ] { + assert!(with_sources(&[good]).is_ok(), "{good}"); + } + for bad in [ + "http://relay.example", + "http://10.0.0.1", + "ftp://relay.example", + "https://user:secret@relay.example", + "https://relay.example/?key=1", + "https://relay.example/#top", + "relay.example", + "data:text/plain,proof", + ] { + assert!( + matches!( + with_sources(&[bad]), + Err(VerifiedContextProviderError::InvalidSource(_)) + ), + "{bad}" + ); + } + assert!(with_sources(&[]).is_err()); + let many: Vec<_> = (0..9) + .map(|n| format!("https://relay{n}.example")) + .collect(); + let many: Vec<_> = many.iter().map(String::as_str).collect(); + assert!(with_sources(&many).is_err()); + assert!(with_sources(&many[..8]).is_ok()); + assert!( + with_sources(&["https://relay.example", "https://relay.example/"]).is_err(), + "the same relay twice" + ); + } + + #[test] + fn should_post_to_proofs_under_each_source() { + for (source, endpoint) in [ + ("https://relay.example", "https://relay.example/proofs"), + ("https://relay.example/", "https://relay.example/proofs"), + ( + "https://relay.example/a/b", + "https://relay.example/a/b/proofs", + ), + ("https://relay.example/a/", "https://relay.example/a/proofs"), + ] { + let url = parse_source(source).expect("a source"); + assert_eq!( + proofs_endpoint(&url).expect("an endpoint").as_str(), + endpoint + ); + } + } + + #[test] + fn should_build_only_for_mainnet_and_testnet_with_a_matching_checkpoint() { + for network in [Network::Devnet, Network::Regtest] { + assert!(matches!( + VerifiedHttpContextProvider::new(network, cache_size()), + Err(VerifiedContextProviderError::UnsupportedNetwork(_)) + )); + } + let mainnet = VerifiedHttpContextProvider::embedded_checkpoint(Network::Mainnet) + .expect("mainnet checkpoint"); + assert_eq!(mainnet.height, 2_550_000); + let mismatched = VerifiedHttpContextProvider::with_checkpoint( + Network::Testnet, + mainnet.clone(), + vec!["https://relay.example".to_string()], + cache_size(), + ); + assert!(matches!( + mismatched, + Err(VerifiedContextProviderError::InvalidCheckpoint(_)) + )); + let mut empty = mainnet; + empty.quorum_root = [0; 32]; + assert!(matches!( + VerifiedHttpContextProvider::with_checkpoint( + Network::Mainnet, + empty, + vec!["https://relay.example".to_string()], + cache_size(), + ), + Err(VerifiedContextProviderError::InvalidCheckpoint(_)) + )); + + let testnet = VerifiedHttpContextProvider::new(Network::Testnet, cache_size()) + .expect("a testnet provider without network access"); + assert_eq!(testnet.checkpoint().height, 1_560_000); + assert_eq!(testnet.latest_verified_state(), *testnet.checkpoint()); + assert_eq!( + testnet.sources()[0].as_str(), + "https://quorums.testnet.networks.dash.org/" + ); + } + + #[test] + fn should_serve_known_contracts_and_token_configurations_only() { + let f = fixture(Network::Testnet); + let provider = provider_for(&f, vec!["https://relay.example".to_string()], quick()); + let id = Identifier::from([7; 32]); + assert!(provider + .get_data_contract(&id, PlatformVersion::latest()) + .expect("a lookup") + .is_none()); + assert!(provider + .get_token_configuration(&id) + .expect("a lookup") + .is_none()); + assert_eq!( + provider.get_platform_activation_height().expect("a height"), + 1_090_319 + ); + } +} diff --git a/packages/rs-sdk-trusted-context-provider/tests/checkpoints.rs b/packages/rs-sdk-trusted-context-provider/tests/checkpoints.rs new file mode 100644 index 00000000000..66a552dea17 --- /dev/null +++ b/packages/rs-sdk-trusted-context-provider/tests/checkpoints.rs @@ -0,0 +1,63 @@ +//! The embedded checkpoints are what the chain says they are. +//! +//! `tests/data/checkpoint-evidence-.json` holds the checkpoint +//! block's raw 80-byte header, its raw coinbase transaction and the coinbase's +//! Merkle branch (sibling hashes in wire order). The test recomputes the block +//! hash from the header, the header's Merkle root from the coinbase, and the +//! masternode and quorum roots from the coinbase payload, so a checkpoint file +//! cannot be edited without the evidence also being forged. +use dash_core_proof::{coinbase_roots, header_hash, sha256d, MerklePath}; +use dpp::dashcore::Network; +use rs_sdk_trusted_context_provider::VerifiedHttpContextProvider; +use serde_json::Value; + +fn bytes(value: &Value) -> Vec { + hex::decode(value.as_str().expect("a hex string")).expect("hex") +} + +#[test] +fn should_embed_checkpoints_that_match_their_block_header_and_coinbase() { + for (network, evidence) in [ + ( + Network::Mainnet, + include_str!("data/checkpoint-evidence-mainnet.json"), + ), + ( + Network::Testnet, + include_str!("data/checkpoint-evidence-testnet.json"), + ), + ] { + let checkpoint = + VerifiedHttpContextProvider::embedded_checkpoint(network).expect("a valid checkpoint"); + let evidence: Value = serde_json::from_str(evidence).expect("evidence JSON"); + + let header: [u8; 80] = bytes(&evidence["header"]) + .try_into() + .expect("an 80-byte header"); + assert_eq!(header_hash(&header), checkpoint.block_hash, "{network:?}"); + + let coinbase = bytes(&evidence["coinbase"]); + let path = &evidence["coinbase_path"]; + let siblings = path["siblings"] + .as_array() + .expect("siblings") + .iter() + .map(|sibling| bytes(sibling).try_into().expect("a 32-byte hash")) + .collect(); + let path = MerklePath { + index: path["index"].as_u64().expect("an index") as u32, + count: path["count"].as_u64().expect("a count") as u32, + siblings, + }; + assert_eq!(path.index, 0, "the coinbase is the first transaction"); + let merkle_root: [u8; 32] = header[36..68].try_into().expect("32 bytes"); + path.verify(sha256d(&coinbase), &merkle_root) + .expect("the coinbase is in the checkpoint block"); + + assert_eq!( + coinbase_roots(&coinbase, checkpoint.height).expect("a coinbase payload"), + (checkpoint.masternode_root, checkpoint.quorum_root), + "{network:?}" + ); + } +} diff --git a/packages/rs-sdk-trusted-context-provider/tests/data/checkpoint-evidence-mainnet.json b/packages/rs-sdk-trusted-context-provider/tests/data/checkpoint-evidence-mainnet.json new file mode 100644 index 00000000000..b350574480a --- /dev/null +++ b/packages/rs-sdk-trusted-context-provider/tests/data/checkpoint-evidence-mainnet.json @@ -0,0 +1,11 @@ +{ + "header": "00000020feaa87b095f769cac0f1ed9bfce345a01bb7414475f3914b15000000000000002675755b75e820f55a08c881245c671633df4bdcef0fd93410c6a15bf75b1ea7de13c36a45a81a194a867dbb", + "coinbase": "03000500010000000000000000000000000000000000000000000000000000000000000000ffffffff2703f0e82604de13c36a08000079215e9300000fe4b883e5bda9e7a59ee4bb99e9b1bc04f09f909f21f982b403cf0d7302000000001976a91464d4d4127bbce4b888bb4af080fa4de863b02b3f88ac1b6fc10200000000016a51ba9704000000001976a914afffeed7edf24a1e22ae2074aef3742eeef2241288ac606cf823af0300f0e826005017dfb1058cf64241bc245fa2e6ef66fd2f16ec8cc525183c393c7df9e99e1d40b517c156bbebfed57eb3ca4bc3ba8cde51b7e6127416f8e0cf0c60c50d791b00ac8dab951a2688eab2175241b79513d56bf4f72fc4ade0c6fd0897a824b24cba3fa18430d47ca8bf923af4d7f4d4ae9c0310c278a4d3630605176c44e70f1cdd9d1464830c8f9655b62713244e69640a711d1e02d539a3fe685a0f4ebbbc533c4ab87bb79a030000", + "coinbase_path": { + "index": 0, + "count": 2, + "siblings": [ + "3c8bcbed42498f6050292360c1c5508abcd6a185e611f53e276daa6b04910c0c" + ] + } +} diff --git a/packages/rs-sdk-trusted-context-provider/tests/data/checkpoint-evidence-testnet.json b/packages/rs-sdk-trusted-context-provider/tests/data/checkpoint-evidence-testnet.json new file mode 100644 index 00000000000..393af29e14c --- /dev/null +++ b/packages/rs-sdk-trusted-context-provider/tests/data/checkpoint-evidence-testnet.json @@ -0,0 +1,9 @@ +{ + "header": "00000020f7d6ef3f979b46d113f3abdcc63ddd363778903ce2c5e5893bf883cd200000002ebbe01f8fc668a09da849e138e8ce05969f69529f98d43a995289c4b88134990286b56a165d051e38500700", + "coinbase": "03000500010000000000000000000000000000000000000000000000000000000000000000ffffffff016affffffff03b84b8c03000000001976a914b489115851ca07a26a5ad8bac3cec3c7dbebd83188ac2ed5fd0300000000016af80da706000000001976a914c69a0bda7daaae481be8def95e5f347a1d00a4b488ac00000000af0300c0cd170080dd113dcf292446a87db1b561a04200233d43dbe039e617c1fd2ebf34075e33434a1153d36d402c9f76fd05537d028510afe42e0fe96bf8a077f4965a30a9d1008c62ca98828c6842f0c2a1a9d904f482c9c20e6cd32239b9912ea93142a26b69420a31ffba488e8850a1eb7c07200d2d02e0d9f6b2d4934f618e420ccb323b312be33e77900c46bf7a1ad799febf6c03fffcd753e77637015d212a7be43a479f5961de996d210000", + "coinbase_path": { + "index": 0, + "count": 1, + "siblings": [] + } +} diff --git a/packages/rs-sdk-trusted-context-provider/tests/data/proof-mainnet.json b/packages/rs-sdk-trusted-context-provider/tests/data/proof-mainnet.json new file mode 100644 index 00000000000..781fa108839 --- /dev/null +++ b/packages/rs-sdk-trusted-context-provider/tests/data/proof-mainnet.json @@ -0,0 +1,18 @@ +{ + "checkpoint": { + "network": 0, + "height": 2557564, + "block_hash": "0000000000000016cab4d83ebe0b97aa4c185a54e3e3b1b71d81089b42a25084", + "masternode_root": "7408f665b08384f080708f1f1d63f619856525ec2e774a582a25a3de87ee1b59", + "quorum_root": "1961ef5b35e9f918419758ede42f7ecca013b30777741b185b35a3519b7c65ee" + }, + "proof_hex": "444153484e433032007c0627008450a2429b08811db7b1e3e3545a184caa970bbe3ed8b4ca1600000000000000591bee87dea3252a584a772eec25658519f6631d1f8f7080f08483b065f60874ee657c9b51a3355b181b747707b313a0cc7e2fe4ed58974118f9e9355bef61199d010000030002d923a53e111baa0c675393690cf8fadaadf9c10e990f1f6f0400000000000000fd9001ff7ffffffffeff7fffdffffffefffffffffffffffffffffffffefffbf6defffffffffffffffffffffffffffffffffdfffffffd9001fffffffffffffffffffffffffffffffffffffffffffffffffffffffffeffffffffffffffffffffffffffffffffffffffffff880f756820607f3f6dbf0921045583eb6db631842f834b17880242f4013b46c3fd6ce7a9f72b2f790a07c23306d0452fd489e53ae3bdce8ac6092a028b039e8efbb647b503297383a4f284ff9d578968915fddc3f719edb0f5bd23a6954084d335f2f98a1de3574ffb8aad38be4ccd2f4e36af901b366a177ca5b32cb0c506ee04affca3b38371e3fd48d7fe965d238e3bdfba27ae7abd9133e3762f559eddd02ebaf29f43ac34a38f84a07346ba9840a320309e24d4c488e4d976fa551d081018b283a288e4a64854ef715b343bfe5b9399625c33e6f35b61967bf676341e4b0f66db9c247f89df6b15c2a07b51627ffa7787c55fed99891d723c2236c243c5d8fa3a94f2a772b9073c16b9c36633b4450000005800000007d2c7a47f66b8a11f462c52ee54df1a43b79721023fd435600946f20b4fd45efa0bc20a5ac8942f2d479c17f169e0f0effa1bb5f14c19339a9f43d3fd2f143b27629b8ef72a511f1c28c5aba1266c90312d034850dae60ae54ce540b8c8709c85141dc7ff8c020361437e9f251c5d13796c9d69cd87d3144a8bdce4d14371621439fa2c65c4142bdf0d20f11d02e95837591ef8a7f2618d2cef6919a6d6e2f7b4d5e8cd8a477a166fcfd1e348a13f15204e668acfbd3f925ece8f25da284e4f9ac54f580e5a7f86fa55318850e3927b9bd9d693046506bbc4bc49e6bb0d4acb9702003507270000000020b0bc2fe6eaa443d6b12462cef9b23696a0f645a77e34bb3a260000000000000044e937cdae9e2372cdaf59ad090f376c3325ae5e3fa99a37365f1ccad950cfd09ab9d56a0d5d3519ea1c6d1ea19bd1937caa70ec8b51f40909e16efa31d19562cbaa15632488b59436e4a6b495c24e2d9453d0ff044a43921809143d0273c4d4799d0463e47b652b8e16a7ef13c306c61b2d738ad59b22372952475b32145e99dd00da1f9e9474bea29d5e01b001000003000600000000000000fda301010035072700030002d84784d6a005526733639c02104a316f4397df6a2124f8803100000000000000fd9001ffffbfffffe7ffffffffffefeffffffffff7fffffbffffffffffffffffffffff79fff9ffffffffffffffbffffffdffffffdffd9001ffffffffffefffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff91f425e7e7465f13504d6257068fddb843eba17ffe4502f1d318668b086ba4227ec3b19d6cdd8fa0dce22a6103f4236be787023398d5dcbd4bac6822f2bf35828067c53f6b90bcb7f01f7280c882d8fa92d7c87c5d40c7c5047d6e6aa64ef24f24f90d48774b79f300bc39a2cd2d4a97ca2926720fd3c903008d1c88b5ca4d4a160a8e8afd9eafac5766481d1ebbeb0773d0f8b7ce7c1bdeb2dc799023d50f877cae8115bbc98342ab8b7d18f7d4e47e966c2b340d71bebec4e2903f762510a47afc333ca47183bf136eca5da05f047f487c54da0818b5a7dbe71df7de798758101912801d1a3fe485645cee567e15dcf8ade6e9bc0bc108555160b3356d8c05ed349ec3d7bcda8390d82a89b468e91e010000000f000000045100c6cefdfad63ae0381043d71b226fc5dac30ba73f9c308c562ac59ee6023a2c5021731cd0ea6f077e31f3d40a6fb5e105432c0935300d40fdce2c5e5bc666b183bcac2cf430e7b808f8cf9901f7cdd458285daf10f3e96179437c5a753315eebb053ae09bb4013e2d126cc9e030203060f504e97dff6feccd609a388e74fd0000b50b2700000000204977537a3c2aab47dc5f765b38aa2eab4ebe419009f77f8f0300000000000000a95ababc8c49b5ce030bfb6e8e0fe063f2beaeacc84d3d0a1f95da1eb793a4115279d86a54951219083b259b8013a228dc9d9971874680967afc828d25a92d82e7a06559d1d0bcf888cced6af995c54e52eec0cf3f5e55ec50b9e1190ea400999cd874ae0547c54e9b09e4787537ab255a913dca120db300139868558f0bfdba1ea5779d79153b83c8f055cdb001000003000600000000000000fda3010100b50b27000300024f53faafb819a9d6b8348b3ff4c7b025bc9f4d02ea4ba1ab2900000000000000fd9001effffffbff7fffffffffffedff7ffffffffffffff7fffffffffffffffffffffffffffffffffffffffdfdffffffffffdffffffd9001ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffa9fcfab24895d15f57819734dfa331fbd22ea19cb3017ee7a47a576e7ae2ea1b57777846d72e6f1a20baba62701342231effc0d73e1850da0c3bd605856000f85fac323a5b3afc70131e5a06a9137523aa18b388492cc61a1b340e0711175562fbcceeb2196ee535bec72f683928b918f285c6e70716beb6820bd287d167c5750498eaddd323a33c0ece707d3a0bce8d6552d90c570867a51625cf9eb6e63a083914ea026cd30d4562460ef3f838d9fc94ef40870fb35c37e78384effadf1f59544c8d73fc698c637a7ececa9aa3c94e913ce51dcc952a1c0809f3ae06bdfd6d02b879455a7b053a341a5a1d22c925a37e68d7775ee67954f6a723e22f2b0f43215dd011d9a4f2857531b9cf20cb9cf6010000001e0000000585729fb23c80a52d14980febf8ff6c7764b10bdf023e80701f9b6550159d0f116ed5765aa56d1071f844485533f8707ed166893a9f663f0e26c0abd64659c6b77b625a7f0f4fb1ac9c809b10bc3d7105b914b391ef9094f01547a927bb3c5f6405f107e6f0e8489f4236aa94ffb01e43495ca7a810700958caec3c06c177cd2f2dd10fe47d7f78d9c1be81c7c4fd3be60b9bcc6d44aeaffd2361bcfda52e8a580000580c27000000002090fe54ef03175d17f065d081fc32060331b4eec2402e3a1c090000000000000012a0a0bbbb0d4a7644cfbda0231ca084261bf23d4997f08f6120a13a6d3d2446f0e0d86a0d161c19461a068dad236cbb9d9d85ba09013e44a62395a51ffe01b4af2726d4d6fcf250c153a7723fe78bb3a6085b9e1647502e464b68b109440b7e041439a039e73848c31b8fb4e717006598fe07ea9a7dc844cb01c322f2d820697ab21ba38220d0e80f38b2b45801000003000500010000000000000000000000000000000000000000000000000000000000000000ffffffff2703580c2704f0e0d86a0800004e00815108000fe4b883e5bda9e7a59ee4bb99e9b1bc04f09f909f004e81b80300ad7302000000001976a91464d4d4127bbce4b888bb4af080fa4de863b02b3f88ac1b6fc10200000000016ae4979904000000001976a914df08e5b0877bc3bf686e92c5fb21efd03b52f30288acb0ecb825af0300580c27006a583e3f51328edb8d225ba017800e67eb90023a172d809f2c50ffa45232d4ea5062daf169d830c39ec0e0187f517acb62d2031aae9639e4ce5f6e76e6fcb35200b7a957a2e6b00349712f203621a86f49fdcbae576eaf477c0d014c936bdb2cbced25a351a74ff27ea4b81626475b1505072c26cc4aff5f80a1701bb0b5b71474c8acdef833400843302923349dcd9798ffcd800665c0dc52ae74b28ac644b927774c828f57030000000000004600000007a4667324472ddc99d9416ad6ce53ea409c8b3a585a2edd4a3a33a5288d4b7d247b811bdc6ac8aed651f4d23b69ddb60696050ba50d65b523093bc170501cc69f0d702252a228be032e60fb25a498c97c0dda6556cc4ceb996a67ae50bd529737743469aeafd5665857625f280cd00227dde497e137da0a68d578cf2ab7133f36a365afbbb2a7deb9837044a09f420444d488c600faa289e7c78b72f2f58359b22e47e02e1b8313f02f587469ca73e63e32780004dd709eaf3c5b361a3538def1721ceac9e3a5fedd45622d2cadfef2eca71acee5d1d81e44fc9d52619df82fd3", + "bootstrap_hex": "f80b0000444153484e433032007c0627008450a2429b08811db7b1e3e3545a184caa970bbe3ed8b4ca1600000000000000591bee87dea3252a584a772eec25658519f6631d1f8f7080f08483b065f60874ee657c9b51a3355b181b747707b313a0cc7e2fe4ed58974118f9e9355bef61199d010000030002d923a53e111baa0c675393690cf8fadaadf9c10e990f1f6f0400000000000000fd9001ff7ffffffffeff7fffdffffffefffffffffffffffffffffffffefffbf6defffffffffffffffffffffffffffffffffdfffffffd9001fffffffffffffffffffffffffffffffffffffffffffffffffffffffffeffffffffffffffffffffffffffffffffffffffffff880f756820607f3f6dbf0921045583eb6db631842f834b17880242f4013b46c3fd6ce7a9f72b2f790a07c23306d0452fd489e53ae3bdce8ac6092a028b039e8efbb647b503297383a4f284ff9d578968915fddc3f719edb0f5bd23a6954084d335f2f98a1de3574ffb8aad38be4ccd2f4e36af901b366a177ca5b32cb0c506ee04affca3b38371e3fd48d7fe965d238e3bdfba27ae7abd9133e3762f559eddd02ebaf29f43ac34a38f84a07346ba9840a320309e24d4c488e4d976fa551d081018b283a288e4a64854ef715b343bfe5b9399625c33e6f35b61967bf676341e4b0f66db9c247f89df6b15c2a07b51627ffa7787c55fed99891d723c2236c243c5d8fa3a94f2a772b9073c16b9c36633b4450000005800000007d2c7a47f66b8a11f462c52ee54df1a43b79721023fd435600946f20b4fd45efa0bc20a5ac8942f2d479c17f169e0f0effa1bb5f14c19339a9f43d3fd2f143b27629b8ef72a511f1c28c5aba1266c90312d034850dae60ae54ce540b8c8709c85141dc7ff8c020361437e9f251c5d13796c9d69cd87d3144a8bdce4d14371621439fa2c65c4142bdf0d20f11d02e95837591ef8a7f2618d2cef6919a6d6e2f7b4d5e8cd8a477a166fcfd1e348a13f15204e668acfbd3f925ece8f25da284e4f9ac54f580e5a7f86fa55318850e3927b9bd9d693046506bbc4bc49e6bb0d4acb9702003507270000000020b0bc2fe6eaa443d6b12462cef9b23696a0f645a77e34bb3a260000000000000044e937cdae9e2372cdaf59ad090f376c3325ae5e3fa99a37365f1ccad950cfd09ab9d56a0d5d3519ea1c6d1ea19bd1937caa70ec8b51f40909e16efa31d19562cbaa15632488b59436e4a6b495c24e2d9453d0ff044a43921809143d0273c4d4799d0463e47b652b8e16a7ef13c306c61b2d738ad59b22372952475b32145e99dd00da1f9e9474bea29d5e01b001000003000600000000000000fda301010035072700030002d84784d6a005526733639c02104a316f4397df6a2124f8803100000000000000fd9001ffffbfffffe7ffffffffffefeffffffffff7fffffbffffffffffffffffffffff79fff9ffffffffffffffbffffffdffffffdffd9001ffffffffffefffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff91f425e7e7465f13504d6257068fddb843eba17ffe4502f1d318668b086ba4227ec3b19d6cdd8fa0dce22a6103f4236be787023398d5dcbd4bac6822f2bf35828067c53f6b90bcb7f01f7280c882d8fa92d7c87c5d40c7c5047d6e6aa64ef24f24f90d48774b79f300bc39a2cd2d4a97ca2926720fd3c903008d1c88b5ca4d4a160a8e8afd9eafac5766481d1ebbeb0773d0f8b7ce7c1bdeb2dc799023d50f877cae8115bbc98342ab8b7d18f7d4e47e966c2b340d71bebec4e2903f762510a47afc333ca47183bf136eca5da05f047f487c54da0818b5a7dbe71df7de798758101912801d1a3fe485645cee567e15dcf8ade6e9bc0bc108555160b3356d8c05ed349ec3d7bcda8390d82a89b468e91e010000000f000000045100c6cefdfad63ae0381043d71b226fc5dac30ba73f9c308c562ac59ee6023a2c5021731cd0ea6f077e31f3d40a6fb5e105432c0935300d40fdce2c5e5bc666b183bcac2cf430e7b808f8cf9901f7cdd458285daf10f3e96179437c5a753315eebb053ae09bb4013e2d126cc9e030203060f504e97dff6feccd609a388e74fd0000b50b2700000000204977537a3c2aab47dc5f765b38aa2eab4ebe419009f77f8f0300000000000000a95ababc8c49b5ce030bfb6e8e0fe063f2beaeacc84d3d0a1f95da1eb793a4115279d86a54951219083b259b8013a228dc9d9971874680967afc828d25a92d82e7a06559d1d0bcf888cced6af995c54e52eec0cf3f5e55ec50b9e1190ea400999cd874ae0547c54e9b09e4787537ab255a913dca120db300139868558f0bfdba1ea5779d79153b83c8f055cdb001000003000600000000000000fda3010100b50b27000300024f53faafb819a9d6b8348b3ff4c7b025bc9f4d02ea4ba1ab2900000000000000fd9001effffffbff7fffffffffffedff7ffffffffffffff7fffffffffffffffffffffffffffffffffffffffdfdffffffffffdffffffd9001ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffa9fcfab24895d15f57819734dfa331fbd22ea19cb3017ee7a47a576e7ae2ea1b57777846d72e6f1a20baba62701342231effc0d73e1850da0c3bd605856000f85fac323a5b3afc70131e5a06a9137523aa18b388492cc61a1b340e0711175562fbcceeb2196ee535bec72f683928b918f285c6e70716beb6820bd287d167c5750498eaddd323a33c0ece707d3a0bce8d6552d90c570867a51625cf9eb6e63a083914ea026cd30d4562460ef3f838d9fc94ef40870fb35c37e78384effadf1f59544c8d73fc698c637a7ececa9aa3c94e913ce51dcc952a1c0809f3ae06bdfd6d02b879455a7b053a341a5a1d22c925a37e68d7775ee67954f6a723e22f2b0f43215dd011d9a4f2857531b9cf20cb9cf6010000001e0000000585729fb23c80a52d14980febf8ff6c7764b10bdf023e80701f9b6550159d0f116ed5765aa56d1071f844485533f8707ed166893a9f663f0e26c0abd64659c6b77b625a7f0f4fb1ac9c809b10bc3d7105b914b391ef9094f01547a927bb3c5f6405f107e6f0e8489f4236aa94ffb01e43495ca7a810700958caec3c06c177cd2f2dd10fe47d7f78d9c1be81c7c4fd3be60b9bcc6d44aeaffd2361bcfda52e8a580000580c27000000002090fe54ef03175d17f065d081fc32060331b4eec2402e3a1c090000000000000012a0a0bbbb0d4a7644cfbda0231ca084261bf23d4997f08f6120a13a6d3d2446f0e0d86a0d161c19461a068dad236cbb9d9d85ba09013e44a62395a51ffe01b4af2726d4d6fcf250c153a7723fe78bb3a6085b9e1647502e464b68b109440b7e041439a039e73848c31b8fb4e717006598fe07ea9a7dc844cb01c322f2d820697ab21ba38220d0e80f38b2b45801000003000500010000000000000000000000000000000000000000000000000000000000000000ffffffff2703580c2704f0e0d86a0800004e00815108000fe4b883e5bda9e7a59ee4bb99e9b1bc04f09f909f004e81b80300ad7302000000001976a91464d4d4127bbce4b888bb4af080fa4de863b02b3f88ac1b6fc10200000000016ae4979904000000001976a914df08e5b0877bc3bf686e92c5fb21efd03b52f30288acb0ecb825af0300580c27006a583e3f51328edb8d225ba017800e67eb90023a172d809f2c50ffa45232d4ea5062daf169d830c39ec0e0187f517acb62d2031aae9639e4ce5f6e76e6fcb35200b7a957a2e6b00349712f203621a86f49fdcbae576eaf477c0d014c936bdb2cbced25a351a74ff27ea4b81626475b1505072c26cc4aff5f80a1701bb0b5b71474c8acdef833400843302923349dcd9798ffcd800665c0dc52ae74b28ac644b927774c828f57030000000000004600000007a4667324472ddc99d9416ad6ce53ea409c8b3a585a2edd4a3a33a5288d4b7d247b811bdc6ac8aed651f4d23b69ddb60696050ba50d65b523093bc170501cc69f0d702252a228be032e60fb25a498c97c0dda6556cc4ceb996a67ae50bd529737743469aeafd5665857625f280cd00227dde497e137da0a68d578cf2ab7133f36a365afbbb2a7deb9837044a09f420444d488c600faa289e7c78b72f2f58359b22e47e02e1b8313f02f587469ca73e63e32780004dd709eaf3c5b361a3538def1721ceac9e3a5fedd45622d2cadfef2eca71acee5d1d81e44fc9d52619df82fd303004f01000003000479b9eae40aed5dd7e67e50d19bc510afd5fd45765b1534321e0000000000000064ffffffffffffffffffffffff0f64ffffffffffffffffffffffff0f8445e70c9d22d2a3f174091ece2e7a9bfadab486562c7ae5266c4e3266e3d0da0c875b089558d239662b6cdde1ca4415c432b35c68ffe88a6238529dddabba0f9f4c749db4570623688160b6a0b6facc87eea4095b32faecafcd7da68b9195ea213933aa976febb4cd5286ee20faacfe412f350d9d7b398f120f03a0784d439415fb2afb1e086da59bacb04daf9256fbf02b59b9f59280f9942c323b89eed607af9a0775be153e169381c9dbbb8072978392e1d283ba41fa42ee5e7499517323d5b9bbfc816d7b836d3a54d1826e68a121c9958d81d242a3d5fd54c87ba4ab1c19f6be9e8627468c53732fbec137a9c872821efb4ca144cfe4c1a154ac9e653d12f6b382fd23cf3863b9e11ea73a2e2d0c000000580000000740617cb0095a2b6a6f2db5c14ddf4850244f905d25093567732404c38a84349ec5d83501f757cfa9eb70ca5ef33655ce297f03bae755df956421f18e58b6589823de39ac37c6377bacb54fb4a7cd27eff5ac1841e7ba401dcb861ac268c4a5ecf254c70de0e22b565b147e783817c0f12551b6daa9e3552a7c2154a27a4ac621bb1adfc61baa5d03ad5792462ed456675f72a128f733ac6e5da9d85003eb2b1ef55e3c13e0d72d72efa6419987a9548c4f3e66da0d9eaa69e5c979d37fb59ccc71ce9268f2c3458a9192d3c1faa1ef261eafd5a25a1c3dfadbe7f31d11f4abee01af00000001b2dde1b42a20d00d0f860f34e76e7484c703fcc941d8efac00b4c034fb79ea742f4688978c2f4721388e39a5fd476d59ef506ac0e8ec6f200000000000000000000000000000000000ffffadd4e85a270fa2ea825c2b13c6c1ed98941e701627a260ffa92fc5528e0f16427d73a2b129668172ce387fb38af9b5b3f6947cab6cef9eb8167d9cb1b6a6cf3eefebea75a38c8ac951a6010100bb01d9ac437d0b83c5251b4b096ae6fd97d81efff74a11000000c60b00000c07c171fa22a893014fa5d3fc4cf0c507affb40353d1c71bd563bfdeb8ab99867cc441bf36b87028e48ac7f0f15538384eea71d115bf7678efcd0ab078307988d855282369662ed7a3c6b85a7e889d0f13a48aac5b57a54e6b8574745977ba98db2e7325b617c833a0ef6ffd5687ff1fa028779668656c7a70316de6de3d8c7fe866b1be43fdf41dc7c2ce3ce6930391464dfc174f9795dab87db3cc18bd9344b0eff61c696da314aa7707453a2edf5189794c1a5775867975beac0e26fe4579437dec36ea6863e42103cee0a10d8fee182104d374ab915d3a9d6a504e060150a6aa5cec14ee36470051a58066ac081236e4d169b6236997301721fcffa11446ead3a3e4f39550e8131438dfe55f614893f61609813205860f09dea4531636400da00c5fcd658962006ea9ea7dacfd94690fadad9d233b9321344b944f73f871448feed702bc4daa1d08f18ecafaef078716029fcbd3afb004a07ce45013878550aea84f9dcdde6bb7617549ea2fb5782ab60c6da0552bea95ded3f70c8c7168401af000000021fb986e0ee3134ed19cba704f04b2479c0aa3bc05e09a994c667d1d2df5dcbe58f25eab055b62e1a39b86073db371d37de0adc1790c3700f0000000000000000000000000000000000ffff22d2ee43270f875cb8c7bcaabebdc4ee34cff249808761d95a3f8a0298df56c35324d8a07f44b163158093d9d005ef3680ac352524b7403227f73fd804b7e4d657912f7bcc496c79fa54010100bb018cb97997a418f4814a63d3564b9574a39343796816000000c60b00000c25d718d7ffa7622cc0d1185fe7ddf3c665096ec7b26bcc8aae32094b4cb39cdd2f5c6af3f104e93f1dded2e6dd9aee5a270862e42578846053c42da3525307a2fe39ea427e0a39abc3e434f4bb9f0a3a1abd7d2c373d476b4b3ac79f06152c44b2e7325b617c833a0ef6ffd5687ff1fa028779668656c7a70316de6de3d8c7fe866b1be43fdf41dc7c2ce3ce6930391464dfc174f9795dab87db3cc18bd9344b0eff61c696da314aa7707453a2edf5189794c1a5775867975beac0e26fe4579437dec36ea6863e42103cee0a10d8fee182104d374ab915d3a9d6a504e060150a6aa5cec14ee36470051a58066ac081236e4d169b6236997301721fcffa11446ead3a3e4f39550e8131438dfe55f614893f61609813205860f09dea4531636400da00c5fcd658962006ea9ea7dacfd94690fadad9d233b9321344b944f73f871448feed702bc4daa1d08f18ecafaef078716029fcbd3afb004a07ce45013878550aea84f9dcdde6bb7617549ea2fb5782ab60c6da0552bea95ded3f70c8c71684", + "target": { + "network": 0, + "height": 2559064, + "block_hash": "0000000000000010f0455b6c18e43218b6a5af379f6b5c25e84970943d774d23", + "masternode_root": "ead43252a4ff502c9f802d173a0290eb670e8017a05b228ddb8e32513f3e586a", + "quorum_root": "52b3fce6766e5fcee43996ae1a03d262cb7a517f18e0c09ec330d869f1da6250" + } +} diff --git a/packages/rs-sdk-trusted-context-provider/tests/data/proof-testnet.json b/packages/rs-sdk-trusted-context-provider/tests/data/proof-testnet.json new file mode 100644 index 00000000000..7b22d55764c --- /dev/null +++ b/packages/rs-sdk-trusted-context-provider/tests/data/proof-testnet.json @@ -0,0 +1,18 @@ +{ + "checkpoint": { + "network": 1, + "height": 1574768, + "block_hash": "0000006648a17e1ab1c4c5745d80fdb6d69bff32fd93ad4fc60baad624e1bb62", + "masternode_root": "d19783ddd57d2cdcf8cc2552a82859f062faab7c4b675b870762bd0848b5af8c", + "quorum_root": "5e6becb125aed06017530637a1423d87a27b468bef6865405d619b20e8dcf2c5" + }, + "proof_hex": "444153484e433032017007180062bbe124d6aa0bc64fad93fd32ff9bd6b6fd805d74c5c4b11a7ea148660000008cafb54808bd6207875b674b7cabfa62f05928a85225ccf8dc2c7dd5dd8397d1c5f2dce8209b615d406568ef8b467ba2873d42a13706531760d0ae25b1ec6b5e430100000300010d95087124bef7460a981cd5900accd968de3ef62bcc4beb15e12a057000000032ffffffffffff0332ffffffffffff03b8ec962444fda60eb92aa98921ea8f30beae27e4df2b29b0e7034f3ec07d707930f7b4270bbdb1555945c95b5c83e83cc5a2061f4a1264f4ffe8dc96a3225fb0d5801a573c664faf8185832c13949665a1c29c6782e2327382da0477368180966d8edd7fad10524186dc4237d6a113cd999dfece70f7ca7ef7e2044780caa5ca02185d175ad1a7a8c95e816cb007f087c5af457c50c17e5e9387c15374077844f22a6b82364d41867ab09a809d11bb3f8a089a200c80aafa2249ea7258b787ceeea7038e99e264c0eec5821c758efc840f0c5299ffe06a6e609b9be24abe91b60e3c5cc7193d299126852deb9db67519a302979b611418e1f3c3644dcf60c2c1b3c1509c7586fc36ce1ed1ec5029a855590000006d00000007cf9f697dec52933af5dfe5b6384677f03483ee40b86548fee10cfbaa39e645704ebfb6ebef37d76f9e165b2ed9fdf7d836c48b048ed3afc645d102ad7f5c7ceba0de3ff4a1134f4d584a2b4ec04ebb269d9341798d7a42c7f153d6d2e70432a5239d4140cada4d980599d8ad36d5c4cde999553e552644ce3473579fabbd22b0b573feb9ee12066cc7d5c58d452f3c93348ad4566c4a3bf5334761ef683ff287e32c60f601702009cb148cf8540cd3895c1a749af84146ae3235aadbeb1ee86f7e06521e6a8fde85c191e6c1a959afa67f3584f614896e58d41d9b289c27dfd804004c08180000000020ed36421d595231d47ca154e473b7fe295c8887ee8ef2792832fbe1217600000042b8a8e30a7ceffbbf223fc7996e4bc500f1466cf4acd2bf35ffa300cdb7552aa11cd56a3671081e4bf40700b25838566ff7148e704894657f8a57245c1948c64b2331343c7bd17586a93721a08395bf340bb6f946537a4785c7391614ef0ac4c98af0e4c5354305906a8fac3a39158d240e42711affe2b2effc50238b45e2ddb8b9fb307681fb3ee79b08f35601000003000600000000000000fd490101004c081800030001002ac52efade4141ddad6eace8848d62bd0e9eab08d78d318c9a67b54100000032ffffffffffff0332ffffffffffff038c2b695848a83578c91f93abe2a82cb49b87c3808331a4b71af0e64aa067bb12cf4f7b39b6360b3c6f761f87fed867747e4abfc1e7516f79fc57529bda60a0a9672b3fe391d72343f6e436e997d420cd900ebe0078752340e408f4675f8be88dfafcc4b7b6bc1d8bc42d67f5d77a22c388c43365475d36af3f82a80489cae366122921c6979d0a9f5240f39fd735ddfcca1cac6e2a57a89999166a1ed53ee3cb941dcf293630fb4c0d87a93db32dcf19aad2b734dc5f0e2ce1cd98fb5c5ba260c57e209e6dcbdf94b9d3204257b72be796b8b06cd6d1e0e881b4ec89476741c816ca5ef8ce05ffd3fb68a99e56dbb1d5857b35bc1ad38a0b0abb56d2fb5713be9bad91fd723bfd43c844f0ad2960c3fc01000000090000000407ad1c43e625e57a92eb8bfc3405b96a2d1d191bd769965d25247496d76c483c01f48e3d0bd279c582a9e4e7e19025f947e9c9837b63b4bc31a8c9a0dcf487e4ae91136e6f21ab70c073ac0903175496d030aff12c8b3eb3c89821664618cbff9c979ccd160f37b0e8f98f37a8544e6f52f430347d6b4a3c10fcbe0e1e35b2cb00002d0a180000000020af0df340c2ca8fa70d16a30c0ac2a816d3b889c94b8a8df305885f99810000003effd237bb0f2c93a6eb426841d028c84d930a76d90e68b44649f09de4b4f059a524d66abaf2001ea0790300b37d1b0eb7b82b866de51573f7052f27b7b47227a05aae6135a033a680e91e9fca7ac0ee63788e711d3f58f2022b8f00081b1d2a0e5282a2aba993bf0c8e260d06bd3cf3b2246446b50255b920ad0aff470544a796308c5edcd3190a488dfd205601000003000600000000000000fd490101002b0a1800030001d63e1a2e552b5e587be5b601a46db1d658507be5b5f335c6f94ccbd58700000032ffffffffffff0332ffffffffffff03a89c1a59186276c2e8d0bbe16c7b1cc1ca33e1e01999fcb03743a8b379d40096102006a9033a9968d4446f9276fe0e6493af4d30b5915f3eb55180859c827e0f0fb1410a7bc21b1ec33680b86b09be48a714ce03012396db7fdf8657a203457617321bde81d6cffdaa2632a67f678cf2f4d1c379efe48137594aa2634398901f0799500ab92d564dc535cc0a5218a04ea27607721d1d8d5f904899e07f6b10319aad0818f476797a77108a6b3b19e69cae8b8a7a39ae74cbb8caa70deb45cc1dbd3146b9c7028fdbedfb31b81ab137b8767660a7156a25f5793f1335a1fde3650807563eb4a466936a9fbb5afdb7ed7faab2d455e9703ed88a949f24b5c240001938d0d9e2f38891c39e53731293d54e0100000004000000021fdb2c1f0572f6b450d3248ab684916e34ddf253cb2f227c3ed6afd59614116a0fe0ff86eabf34e96a60ca95062c93fc92e5063a89bd82cb679580cec2011788020000000020c8d425146fb9576a709eba4d5b8e687d60a3fb1145097802f21e53391a00000058b2a810fb0e2cc6529886b841746408274af7835e8c167097526379444c60f43223d66a49fd001e9fda020000000020e0817029310a6efdfd2dfaf4c748690b55084c4c6276c0513fedc5c339000000cc50ab9c888f36a173ca0d35ca143aa15d8e5a6bbb89ff4b56b260bd41025bab0624d66a75fc001e6cba0200ab0b180000000020a317d9052d2b8fc82951e4d6a2f8ab5ecc9bff910db7a0b394921971bf000000afbcfc3b9e8ef8f75ef65bb655608adf3e759f741e1e6b70a6f3bd24f90170a901f2d66afc5d011ed73a050090269ef2eefee16cc69dd6ac4b847ffb515804e55a5b721623ada9ebf1f5760508f521a8800f323ae6bdf3470e5ea7de06320d3f4fcd86fa3c519e10abf8a2f9b3f398fcbbb3e0cbbb4212210d8b8ec0fa1db2c04318de595a992ae6ca2e39c15601000003000600000000000000fd49010100ab0b18000300019e3221848c14752808d3be14a37d3bf6186efe8b04137bd3cf75d9b22d00000032ffffffffffff0332ffffffffffff0397b1d1bb8e9bf6ece8600d3da9d2e0cf185f0e1b0e0176d6a2f1734049e8ae0280a79c58e2ad2f941a3b372471ba19479472988c79245e291b1cb7112518da65a2501f40cb006acee652b3af9804bf1db99794fab1d74869619491e4aba703cd18361d4561b3ae6703131bfb49fbbbc555c2b75ed40471109af490a80b3f14bb045c6516800ab685fa00db1c8bbee6b0fd705a56d90114b0498d4f89ec5c9599361ea17cccc72509f5680807f80448e4ad5809608497ddb3ee9d199e006bfb648b7d4c10bc208ba3022560e05c364a0ead7be8e15ca51cabaaa6ef3bed8d15ce0681b262d52374b93432b48058012a2044c2b17a5e3816bc96f53f0e99a8ea7d16ce5c1fbfe21ac3e12456d4d1db2adf010000000600000003fb24f952450bb40a953581cea905bd97df4dc187e180a4ba45085148b99b3983f58e336499890b90ec37aea130a0f75478bd0270788d63e1961f47a8b644aff12f2cd56720a13a04b8e3936892a980c7ab70619d9c6e6d6c34ab58e9288165c60000be0d180000000020752292f6f953563fa944234551f944106e5c217827253dba0000231ef0000000a1dd2c9ccd046790ebe38555e289154a967f7aaabea8f0a20fa596a11ceb67ab4f0ed86a1b75011ecf740400808e90bd2060e079b20f865feea40131f378333eecca5918c67817f96ef2fa8490785c003bce0b6b0d3ef4b49b93a8b90ab2027affd42e74f674d18701531a923e3c566c6ff732e4c1dd22a3c68b16e6903474512863f22172313336a46d205e5601000003000600000000000000fd49010100bb0d18000300013524c22745ca56b62cc5956e49bc9bcfcd725b821b2dbec797a807ea4100000032ffffffffffff0332ffffffffffff03a4854d5b1c3ff8e6012c3a3a1d34d12c77d542a07e893ef582c48ea8d2e9f0412b53f01ff29bc64cfdd7ee9a0bcaacdc5e0a6603d6aaf5141c612e16066d94e4cad096560fddcc90d1353044ebe8a1fd85406b3e75cd6b3debab754f4f8552b1f37062c988edef4931960b9469ca3fac0efbb1fde3f5e45801243e752134c6270ea886e5676fa46941f29f35a87942ff1b93f4f5cc5092df754142a1605965a56bab1050884c9e77570dabc5cc2fd5f9b1f94b81435bdb39d3bbc66034cede162febd77e1e1105cb4cc2a879e0baac4b94f2ec7ae06cc889e9fcbe6cdf43fb4f144fe18ac2f0e9f85e4f2fb232b000377744acc40147db2a098b7dbc55319c45ce6cdee01ca275c54c64d9a11665cf1801000000040000000276de27fa656a35dbb32e7cd9f1f7a15479eb3d3800380528fba0e07f0071442e98bf09a6736601b1a05c3f8416b5c2cedd6383652975d4c4f263ee3e2cba915b030000000020085663f228b88ea38af5180fd36e13e489c9373c646cc4825a1b9b4c5f00000061d1b9bae9061efc4d1d0526b56f3dfeba5257fa227326d0bdbbf1ee9a723f632c0dd86a1879011e96e70100000000208b487ce47e6a317bad93be933847b822553f8d240e8d74601684b4c5b3000000ad0101428b9020a7aa726db48e6e50fb1d4f4b85daf807b0c1fc6a52ca8454cf390dd86a3673011ef0c50100000000209dc63ad77910d9fabcdf23be8126b5c6a3a51ec2acc526eb57d7766a230000006a281b8c67908ebd83d47341673c93f27fa198ab66d55dc3034536d72c508b9b4b0dd86a0a7b011e0cba0300400f1800000000203e6023289fa4ebf4338f76a2c9da0741e8084a6d02e7efface6cff60020000005615dc62efe815c5b0fa2bcf8af7594c1b02124f6c53759a39b2e050ad4e467417e1d86a5ee1001efcac0e00b46a4edf1b6b8eecc2f0245d2f22c51c739c6ca3aadb3a58ff186bb7ec7f55d96545fb3e531c8d5240f668e3f312827603674df95713354777f9729b451637a141489a1e87c34b1059abee83cc04bcba976127b15dd3512c558c3f2b5d0c55e43201000003000500010000000000000000000000000000000000000000000000000000000000000000ffffffff016affffffff03b84b8c03000000001976a914b489115851ca07a26a5ad8bac3cec3c7dbebd83188ac2ed5fd0300000000016af80da706000000001976a914c69a0bda7daaae481be8def95e5f347a1d00a4b488ac00000000af0300400f1800df4dea512efab716a7ae84e315e3d45fe3c05a18eaed64058d9a8c1966667328e1f2b2b43f667274b3e8f488f24dc9f7fa611149dc36e2324826b951d5be895001aa08742e5f391c309351c0f45dd708fa7bac12a5d3b47f6d8e3d84547dab096f39268c2e64c7cca868a503f908bf0c7d08f18e05dc3658522ae25124561bce8ae0e5d7f85456d6ebdc7b9817a0c73ed26055550e0a2ede3482e990e227fcf1fbf4ac23a22d220000000000000100000000", + "bootstrap_hex": "e60f0000444153484e433032017007180062bbe124d6aa0bc64fad93fd32ff9bd6b6fd805d74c5c4b11a7ea148660000008cafb54808bd6207875b674b7cabfa62f05928a85225ccf8dc2c7dd5dd8397d1c5f2dce8209b615d406568ef8b467ba2873d42a13706531760d0ae25b1ec6b5e430100000300010d95087124bef7460a981cd5900accd968de3ef62bcc4beb15e12a057000000032ffffffffffff0332ffffffffffff03b8ec962444fda60eb92aa98921ea8f30beae27e4df2b29b0e7034f3ec07d707930f7b4270bbdb1555945c95b5c83e83cc5a2061f4a1264f4ffe8dc96a3225fb0d5801a573c664faf8185832c13949665a1c29c6782e2327382da0477368180966d8edd7fad10524186dc4237d6a113cd999dfece70f7ca7ef7e2044780caa5ca02185d175ad1a7a8c95e816cb007f087c5af457c50c17e5e9387c15374077844f22a6b82364d41867ab09a809d11bb3f8a089a200c80aafa2249ea7258b787ceeea7038e99e264c0eec5821c758efc840f0c5299ffe06a6e609b9be24abe91b60e3c5cc7193d299126852deb9db67519a302979b611418e1f3c3644dcf60c2c1b3c1509c7586fc36ce1ed1ec5029a855590000006d00000007cf9f697dec52933af5dfe5b6384677f03483ee40b86548fee10cfbaa39e645704ebfb6ebef37d76f9e165b2ed9fdf7d836c48b048ed3afc645d102ad7f5c7ceba0de3ff4a1134f4d584a2b4ec04ebb269d9341798d7a42c7f153d6d2e70432a5239d4140cada4d980599d8ad36d5c4cde999553e552644ce3473579fabbd22b0b573feb9ee12066cc7d5c58d452f3c93348ad4566c4a3bf5334761ef683ff287e32c60f601702009cb148cf8540cd3895c1a749af84146ae3235aadbeb1ee86f7e06521e6a8fde85c191e6c1a959afa67f3584f614896e58d41d9b289c27dfd804004c08180000000020ed36421d595231d47ca154e473b7fe295c8887ee8ef2792832fbe1217600000042b8a8e30a7ceffbbf223fc7996e4bc500f1466cf4acd2bf35ffa300cdb7552aa11cd56a3671081e4bf40700b25838566ff7148e704894657f8a57245c1948c64b2331343c7bd17586a93721a08395bf340bb6f946537a4785c7391614ef0ac4c98af0e4c5354305906a8fac3a39158d240e42711affe2b2effc50238b45e2ddb8b9fb307681fb3ee79b08f35601000003000600000000000000fd490101004c081800030001002ac52efade4141ddad6eace8848d62bd0e9eab08d78d318c9a67b54100000032ffffffffffff0332ffffffffffff038c2b695848a83578c91f93abe2a82cb49b87c3808331a4b71af0e64aa067bb12cf4f7b39b6360b3c6f761f87fed867747e4abfc1e7516f79fc57529bda60a0a9672b3fe391d72343f6e436e997d420cd900ebe0078752340e408f4675f8be88dfafcc4b7b6bc1d8bc42d67f5d77a22c388c43365475d36af3f82a80489cae366122921c6979d0a9f5240f39fd735ddfcca1cac6e2a57a89999166a1ed53ee3cb941dcf293630fb4c0d87a93db32dcf19aad2b734dc5f0e2ce1cd98fb5c5ba260c57e209e6dcbdf94b9d3204257b72be796b8b06cd6d1e0e881b4ec89476741c816ca5ef8ce05ffd3fb68a99e56dbb1d5857b35bc1ad38a0b0abb56d2fb5713be9bad91fd723bfd43c844f0ad2960c3fc01000000090000000407ad1c43e625e57a92eb8bfc3405b96a2d1d191bd769965d25247496d76c483c01f48e3d0bd279c582a9e4e7e19025f947e9c9837b63b4bc31a8c9a0dcf487e4ae91136e6f21ab70c073ac0903175496d030aff12c8b3eb3c89821664618cbff9c979ccd160f37b0e8f98f37a8544e6f52f430347d6b4a3c10fcbe0e1e35b2cb00002d0a180000000020af0df340c2ca8fa70d16a30c0ac2a816d3b889c94b8a8df305885f99810000003effd237bb0f2c93a6eb426841d028c84d930a76d90e68b44649f09de4b4f059a524d66abaf2001ea0790300b37d1b0eb7b82b866de51573f7052f27b7b47227a05aae6135a033a680e91e9fca7ac0ee63788e711d3f58f2022b8f00081b1d2a0e5282a2aba993bf0c8e260d06bd3cf3b2246446b50255b920ad0aff470544a796308c5edcd3190a488dfd205601000003000600000000000000fd490101002b0a1800030001d63e1a2e552b5e587be5b601a46db1d658507be5b5f335c6f94ccbd58700000032ffffffffffff0332ffffffffffff03a89c1a59186276c2e8d0bbe16c7b1cc1ca33e1e01999fcb03743a8b379d40096102006a9033a9968d4446f9276fe0e6493af4d30b5915f3eb55180859c827e0f0fb1410a7bc21b1ec33680b86b09be48a714ce03012396db7fdf8657a203457617321bde81d6cffdaa2632a67f678cf2f4d1c379efe48137594aa2634398901f0799500ab92d564dc535cc0a5218a04ea27607721d1d8d5f904899e07f6b10319aad0818f476797a77108a6b3b19e69cae8b8a7a39ae74cbb8caa70deb45cc1dbd3146b9c7028fdbedfb31b81ab137b8767660a7156a25f5793f1335a1fde3650807563eb4a466936a9fbb5afdb7ed7faab2d455e9703ed88a949f24b5c240001938d0d9e2f38891c39e53731293d54e0100000004000000021fdb2c1f0572f6b450d3248ab684916e34ddf253cb2f227c3ed6afd59614116a0fe0ff86eabf34e96a60ca95062c93fc92e5063a89bd82cb679580cec2011788020000000020c8d425146fb9576a709eba4d5b8e687d60a3fb1145097802f21e53391a00000058b2a810fb0e2cc6529886b841746408274af7835e8c167097526379444c60f43223d66a49fd001e9fda020000000020e0817029310a6efdfd2dfaf4c748690b55084c4c6276c0513fedc5c339000000cc50ab9c888f36a173ca0d35ca143aa15d8e5a6bbb89ff4b56b260bd41025bab0624d66a75fc001e6cba0200ab0b180000000020a317d9052d2b8fc82951e4d6a2f8ab5ecc9bff910db7a0b394921971bf000000afbcfc3b9e8ef8f75ef65bb655608adf3e759f741e1e6b70a6f3bd24f90170a901f2d66afc5d011ed73a050090269ef2eefee16cc69dd6ac4b847ffb515804e55a5b721623ada9ebf1f5760508f521a8800f323ae6bdf3470e5ea7de06320d3f4fcd86fa3c519e10abf8a2f9b3f398fcbbb3e0cbbb4212210d8b8ec0fa1db2c04318de595a992ae6ca2e39c15601000003000600000000000000fd49010100ab0b18000300019e3221848c14752808d3be14a37d3bf6186efe8b04137bd3cf75d9b22d00000032ffffffffffff0332ffffffffffff0397b1d1bb8e9bf6ece8600d3da9d2e0cf185f0e1b0e0176d6a2f1734049e8ae0280a79c58e2ad2f941a3b372471ba19479472988c79245e291b1cb7112518da65a2501f40cb006acee652b3af9804bf1db99794fab1d74869619491e4aba703cd18361d4561b3ae6703131bfb49fbbbc555c2b75ed40471109af490a80b3f14bb045c6516800ab685fa00db1c8bbee6b0fd705a56d90114b0498d4f89ec5c9599361ea17cccc72509f5680807f80448e4ad5809608497ddb3ee9d199e006bfb648b7d4c10bc208ba3022560e05c364a0ead7be8e15ca51cabaaa6ef3bed8d15ce0681b262d52374b93432b48058012a2044c2b17a5e3816bc96f53f0e99a8ea7d16ce5c1fbfe21ac3e12456d4d1db2adf010000000600000003fb24f952450bb40a953581cea905bd97df4dc187e180a4ba45085148b99b3983f58e336499890b90ec37aea130a0f75478bd0270788d63e1961f47a8b644aff12f2cd56720a13a04b8e3936892a980c7ab70619d9c6e6d6c34ab58e9288165c60000be0d180000000020752292f6f953563fa944234551f944106e5c217827253dba0000231ef0000000a1dd2c9ccd046790ebe38555e289154a967f7aaabea8f0a20fa596a11ceb67ab4f0ed86a1b75011ecf740400808e90bd2060e079b20f865feea40131f378333eecca5918c67817f96ef2fa8490785c003bce0b6b0d3ef4b49b93a8b90ab2027affd42e74f674d18701531a923e3c566c6ff732e4c1dd22a3c68b16e6903474512863f22172313336a46d205e5601000003000600000000000000fd49010100bb0d18000300013524c22745ca56b62cc5956e49bc9bcfcd725b821b2dbec797a807ea4100000032ffffffffffff0332ffffffffffff03a4854d5b1c3ff8e6012c3a3a1d34d12c77d542a07e893ef582c48ea8d2e9f0412b53f01ff29bc64cfdd7ee9a0bcaacdc5e0a6603d6aaf5141c612e16066d94e4cad096560fddcc90d1353044ebe8a1fd85406b3e75cd6b3debab754f4f8552b1f37062c988edef4931960b9469ca3fac0efbb1fde3f5e45801243e752134c6270ea886e5676fa46941f29f35a87942ff1b93f4f5cc5092df754142a1605965a56bab1050884c9e77570dabc5cc2fd5f9b1f94b81435bdb39d3bbc66034cede162febd77e1e1105cb4cc2a879e0baac4b94f2ec7ae06cc889e9fcbe6cdf43fb4f144fe18ac2f0e9f85e4f2fb232b000377744acc40147db2a098b7dbc55319c45ce6cdee01ca275c54c64d9a11665cf1801000000040000000276de27fa656a35dbb32e7cd9f1f7a15479eb3d3800380528fba0e07f0071442e98bf09a6736601b1a05c3f8416b5c2cedd6383652975d4c4f263ee3e2cba915b030000000020085663f228b88ea38af5180fd36e13e489c9373c646cc4825a1b9b4c5f00000061d1b9bae9061efc4d1d0526b56f3dfeba5257fa227326d0bdbbf1ee9a723f632c0dd86a1879011e96e70100000000208b487ce47e6a317bad93be933847b822553f8d240e8d74601684b4c5b3000000ad0101428b9020a7aa726db48e6e50fb1d4f4b85daf807b0c1fc6a52ca8454cf390dd86a3673011ef0c50100000000209dc63ad77910d9fabcdf23be8126b5c6a3a51ec2acc526eb57d7766a230000006a281b8c67908ebd83d47341673c93f27fa198ab66d55dc3034536d72c508b9b4b0dd86a0a7b011e0cba0300400f1800000000203e6023289fa4ebf4338f76a2c9da0741e8084a6d02e7efface6cff60020000005615dc62efe815c5b0fa2bcf8af7594c1b02124f6c53759a39b2e050ad4e467417e1d86a5ee1001efcac0e00b46a4edf1b6b8eecc2f0245d2f22c51c739c6ca3aadb3a58ff186bb7ec7f55d96545fb3e531c8d5240f668e3f312827603674df95713354777f9729b451637a141489a1e87c34b1059abee83cc04bcba976127b15dd3512c558c3f2b5d0c55e43201000003000500010000000000000000000000000000000000000000000000000000000000000000ffffffff016affffffff03b84b8c03000000001976a914b489115851ca07a26a5ad8bac3cec3c7dbebd83188ac2ed5fd0300000000016af80da706000000001976a914c69a0bda7daaae481be8def95e5f347a1d00a4b488ac00000000af0300400f1800df4dea512efab716a7ae84e315e3d45fe3c05a18eaed64058d9a8c1966667328e1f2b2b43f667274b3e8f488f24dc9f7fa611149dc36e2324826b951d5be895001aa08742e5f391c309351c0f45dd708fa7bac12a5d3b47f6d8e3d84547dab096f39268c2e64c7cca868a503f908bf0c7d08f18e05dc3658522ae25124561bce8ae0e5d7f85456d6ebdc7b9817a0c73ed26055550e0a2ede3482e990e227fcf1fbf4ac23a22d22000000000000010000000003003d010000030006f64aa6f2cf18889e8ff4704f6bbd50a7ab80f3f1fc6eab984e691f381b00000019ffffff0119ffffff01b813d560b9a73db7b01cfe80ee7ab20a4af7ccda4b94a0cd8ceda8f5cb0fa604760430ef8c644e0f823e089aa7839f9393e0215b2e2b69a88310ffe95ac90099be1d2ece9704bd99a3fd0a3ba8d948e387351b53935ce873abf06103066bcd2f6c1f10f343ce5baf4ab97db6dd483b914ca27ef04636641bcba6c5a5c7aa053903bfacfe8a3317e4fd2be691d6301e814b0c563b9baff75ee551967e41c8bfc8ea07d468ad3123eeb5bd52fb397ae8c788da739f2edf77707516ad03022a0abb3fb74623fe36a82f3036c7cfe6a8d40f426fceded4783cd87b43bc45160e9dfe0d6f09ba2f353ee0797334ea74ce21e967c0efc0ac3984ec0a314650ed1dc4f746f146742eb342b95e7466426d5b532f0a0000006d000000071b4109128f361787885d43e827477232ca77d1253201f14b6d9bfcbb36b5ca29296f2a6e10f6b5d257bdf26f29fd098fbefcf317d43ac45bf9b9a912d6990f973e1af01f8e5eae6165af402ae54256702b3ef04e7bd38397bb341c593a62bcad03ca107596c89ccecb5e5cefdb493a11d62c7f7c171d9b0d9a4f223b5237c9112862024cffad39318b66bbcef991aaa9c5b6862e41310b0410c050a61bda1be2b1fea91d5e2ee314113d4bfcf58b80aa6d0f7a5fa959582db28ee87945740044ef7f65c97d261116cdd5dfa7a5203904d3c6a4a6a625a412384925657abdc78b01af0000000a5b7f1478902ecab28b3404f5e79e9020acb9abd31a6e13928c2450d42684b6adf56996e04e36430af272b8c0b1ed1dd57b6146775532c53634ecb5b600000000000000000000000000ffff44437a064e1fb8a150e6472bc281b51537dda3d077264786f207d05c4ec8d314b5883e32b14b15077bf39ccf5d92543a6fd0ee23897802e543987843fbd195e84ff3a28637ab2231d98b010100a30559b537a5cf3ed95e48d8c006d53de9e049dbf9a1120000008a01000009de4fceb4b2b1ab89bc498f4caca25247813558f17e26eb6c6054ef28688906fe592122fffeb92cbd77956c86811e91b0ed60d5a072dbb3d402c90b4361af3f81d70256b9757f6707488b92e518d616a9cf223aa571cd66e73e7c1236f85dc66ff3a1f817422025f9894edeabb8639ebb57d968a3f5fd3bc5ceb7c661cc34273033306f00e1e3878289f50ae34e3d72691aff06fe77a52080f7a479fcdb79263e8c1ed9f00b2c64da2453cd9bc9de4c250c6fd40a0fc9b18b9fe62408dc14f29ce668b31b1e605ea286913109d87371338c9d545551119f821a939657e6f2491632c3ff1a6584536b4196c735f8331a47844337e0cb68c59388a1ea289e724443d00c1109ec72111ce417de8c49fea97279ee87434bc1f85707010cdbda71b22301af00000012c8d756322b07e0cf2f73ad46f5f25bf91c8b7749419d02a6ccca8da60dc488f5364ad36ce7990bab58abc9f2d480903668268322b7e5099d089ccb3d00000000000000000000000000ffff55d1f3064e1f89d3ff18d870c0b6c9605d986c2de92f44c48032408e183a1d268bad77570e30a6d057a026119d9d044c1ce29b518e91d69372e5829f2cc1c3a9882e466ae9761dfc40e5010100a3053fab9f8566e701ad9bdc75f6f0fe7aaba6008f061d0000008a01000009b1f4342c3ae24aaaae59a75c67fc9ad1e62d520ae3f9a11459aa73f7e8fb753170ff09a84bf2b4e29d54d9c346bee0a9498548bdfb78fef626caa32545a6a4d4bbca76f29274761340c446e9e7d3c8940d296a7abb8f4d4d315a19fe11dd5bdd23a95704aa8cbd08f66e671b27e3782a5f5d8c5cc4f625ae1654b06e3888daaa33306f00e1e3878289f50ae34e3d72691aff06fe77a52080f7a479fcdb79263e8c1ed9f00b2c64da2453cd9bc9de4c250c6fd40a0fc9b18b9fe62408dc14f29ce668b31b1e605ea286913109d87371338c9d545551119f821a939657e6f2491632c3ff1a6584536b4196c735f8331a47844337e0cb68c59388a1ea289e724443d00c1109ec72111ce417de8c49fea97279ee87434bc1f85707010cdbda71b223", + "target": { + "network": 1, + "height": 1576768, + "block_hash": "00000021a51acdce0ba4c5f1bff3fd121143e7ab958f9f30bd1de4856b449285", + "masternode_root": "28736666198c9a8d0564edea185ac0e35fd4e315e384aea716b7fa2e51ea4ddf", + "quorum_root": "5089bed551b9264832e236dc491161faf7c94df288f4e8b37472663fb4b2f2e1" + } +} From 313a4a22cd0e9f550f67307232b6b9a5e70aa551 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 21 Oct 2026 12:02:27 -0500 Subject: [PATCH 03/25] fix(sdk): keep verified quorum fetches available under bad input Review of the verified provider found ways unverified input or normal relay load could make it unavailable without affecting soundness: - A fake quorum cost up to 8 requests and ~39 s of retries while holding one of the two proving places. A source that cannot serve the quorum for both the height and the tip is now left out of the fetch, so a fake quorum costs two requests per source, and a fetch only waiting out a backoff does not hold a place. - Fast failures could fill the 64-entry slot table and lock new quorums out. The oldest failure is now evicted, and a fetch that sent nothing (every source cooling down) records no failure. - One lagging or lying relay whose tip proof falls short ended the whole fetch; now only that relay is left out. - A 503 means the relay's workers are busy, normal under load: it is retried with backoff instead of cooling the relay down. - Every hedge counted as cut short by the deadline, so refused or failed connections never cooled their source; now only a request whose shortened timer fired is. - On wasm32 a fetch keeps running on the local executor after its waiters leave, so it is no longer replaced as abandoned. - Error bodies that look like markup are left out of reasons, a state conflict is checked before a key is cached, and the height asked next follows the answer that just came back. Co-Authored-By: Claude Opus 5.5 --- .../src/proof_fetches.rs | 540 ++++++++++++------ .../src/verified_provider.rs | 119 +++- 2 files changed, 480 insertions(+), 179 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index 79df2098cb6..41ca0a3ee0d 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -27,7 +27,7 @@ use crate::quorum_refresh::panic_message; use crate::quorum_refresh::sleep; use dash_context_provider::ContextProviderError; use dash_core_proof::bootstrap::{self, RecordKind}; -use dash_core_proof::{parse_commitment, rpc_hex, State}; +use dash_core_proof::{parse_commitment, rpc_hex, Error as ProofError, State}; use futures::future::{self, select, BoxFuture, Either, FutureExt, Shared}; use futures::stream::FuturesUnordered; use futures::StreamExt; @@ -40,6 +40,7 @@ use std::num::NonZeroUsize; #[cfg(not(target_arch = "wasm32"))] use std::panic::AssertUnwindSafe; use std::pin::pin; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::time::Duration; use url::Url; @@ -50,7 +51,7 @@ pub(crate) type QuorumKey = [u8; 48]; /// A quorum: its LLMQ type and its hash in the order the SDK uses (Core's RPC /// display order). -type QuorumId = (u32, [u8; 32]); +pub(crate) type QuorumId = (u32, [u8; 32]); /// Most requests one fetch sends, across sources and retries. const MAX_REQUESTS_PER_FETCH: usize = 8; @@ -63,8 +64,14 @@ const MAX_QUORUMS_IN_FLIGHT: usize = 2; const MAX_FETCH_SLOTS: usize = 64; /// Verified states kept besides the checkpoint, to start later proofs from. const MAX_RECENT_STATES: usize = 16; -/// The retry delay after a relay's 502 doubles this many times. +/// The delay before asking a busy relay again doubles this many times. const MAX_RETRY_DOUBLINGS: u32 = 3; +/// Answers a source may give that it cannot serve the quorum before a fetch +/// stops asking it: one for the height and one for the tip. +const MAX_CANNOT_SERVE: u32 = 2; +/// How `bootstrap::verify` reports a target below the minimum height, which +/// it checks before any signature. +const STALE_TARGET: ProofError = ProofError("stale target"); /// How much of an error response's body goes into the failure reason. const MAX_REASON_BODY_CHARS: usize = 120; @@ -78,7 +85,8 @@ pub(crate) struct Timing { /// How long a request may go unanswered before the next source is asked /// as well. pub(crate) hedge_after: Duration, - /// First delay before asking again after a relay's 502. + /// Delay before asking a relay again after it could not serve a quorum, + /// and the first delay after it was busy. pub(crate) first_retry: Duration, /// First wait before a quorum whose fetch failed is fetched again. pub(crate) quorum_cooldown: Duration, @@ -124,6 +132,8 @@ enum Slot { Running { id: u64, height: u32, + /// Set while the fetch only waits out a relay's backoff. + waiting: Arc, fetch: SharedFetch, }, Failed { @@ -193,19 +203,65 @@ fn choose_ask( /// How one answer from a relay counts. enum Verdict { Proven, - /// The relay could not serve the proof yet; ask again after `after`, for - /// the other height. - Retry { - reason: String, - after: Duration, - }, - /// The source failed and cools down. + /// The relay cannot serve this quorum for the height asked: a 502, or a + /// valid proof that does not serve the request. Asked again once, for the + /// other height, after a delay; then left out of the fetch. + CannotServe(String), + /// The relay's workers are busy: ask again after a backoff. + Busy(String), + /// The source failed and cools down for every fetch. SourceFailed(String), - /// A valid proof of the relay's latest ChainLock does not reach the - /// height: asking again cannot help until the chain grows. + /// A proof of the relay's latest ChainLock does not reach the height: + /// the relay is left out of this fetch. BeyondTip(String), } +/// One fetch: which quorum, for which height, and its slot. +struct Attempt { + quorum: QuorumId, + height: u32, + id: u64, + /// Earlier failures of this quorum, for its cooldown. + failures: u32, + waiting: Arc, +} + +/// What one fetch knows about one source. +#[derive(Clone)] +struct SourceAttempts { + asked: u32, + in_flight: bool, + ready_at: Instant, + cannot_serve: u32, + busy: u32, + /// Ask for the tip rather than the height next time. + ask_tip: bool, + dropped: bool, +} + +impl SourceAttempts { + fn new(now: Instant) -> Self { + Self { + asked: 0, + in_flight: false, + ready_at: now, + cannot_serve: 0, + busy: 0, + ask_tip: false, + dropped: false, + } + } +} + +/// The source a fetch asks next. +enum Pick { + Ready(usize), + /// None is ready before this time. + At(Instant), + /// None is left: all are cooling down, left out or already asked. + Nothing, +} + /// Proves quorum keys and caches them, for a provider and its clones. pub(crate) struct ProofFetches { quorum_type: u32, @@ -350,10 +406,13 @@ impl ProofFetches { let now = Instant::now(); let forgotten_after = self.timing.max_quorum_cooldown.saturating_mul(2); let mut proven = self.lock(); - // A fetch every waiter dropped is not running natively; it is - // replaced rather than joined or counted. proven.fetches.retain(|_, slot| match slot { - Slot::Running { fetch, .. } => fetch.strong_count() != Some(1), + // A native fetch every waiter dropped no longer runs, so it is + // replaced rather than joined or counted. A browser fetch runs on + // the local executor until it frees its own slot. + Slot::Running { fetch, .. } => { + cfg!(target_arch = "wasm32") || fetch.strong_count() != Some(1) + } Slot::Failed { at, .. } => now.saturating_duration_since(*at) < forgotten_after, }); // A fetch that finished since the caller's lookup has nothing to add. @@ -367,9 +426,7 @@ impl ProofFetches { fetch, height: running, .. - }) if near(*running) => { - return Ok(fetch.clone()); - } + }) if near(*running) => return Ok(fetch.clone()), Some(Slot::Running { height: running, .. }) => { @@ -400,10 +457,12 @@ impl ProofFetches { } Some(Slot::Failed { .. }) | None => 0, }; + // A fetch only waiting out a relay's backoff sends nothing, so it + // does not hold one of the places. let running = proven .fetches .values() - .filter(|slot| matches!(slot, Slot::Running { .. })) + .filter(|slot| matches!(slot, Slot::Running { waiting, .. } if !waiting.load(Ordering::Relaxed))) .count(); if running >= MAX_QUORUMS_IN_FLIGHT { return Err(format!( @@ -411,17 +470,39 @@ impl ProofFetches { )); } if !proven.fetches.contains_key(&quorum) && proven.fetches.len() >= MAX_FETCH_SLOTS { - return Err("too many quorums were asked for recently".to_string()); + // Forget the oldest failure rather than lock new quorums out. + let oldest = proven + .fetches + .iter() + .filter_map(|(quorum, slot)| match slot { + Slot::Failed { at, .. } => Some((*at, *quorum)), + Slot::Running { .. } => None, + }) + .min_by_key(|(at, _)| *at) + .map(|(_, quorum)| quorum); + match oldest { + Some(oldest) => proven.fetches.remove(&oldest), + None => return Err("too many quorums are being proven".to_string()), + }; } proven.next_fetch_id += 1; let id = proven.next_fetch_id; + let waiting = Arc::new(AtomicBool::new(false)); + let attempt = Attempt { + quorum, + height, + id, + failures, + waiting: Arc::clone(&waiting), + }; // Built under the lock, so it must do nothing until it is polled. - let fetch = Arc::clone(self).start(quorum, height, id, failures); + let fetch = Arc::clone(self).start(attempt); proven.fetches.insert( quorum, Slot::Running { id, height, + waiting, fetch: fetch.clone(), }, ); @@ -429,69 +510,62 @@ impl ProofFetches { } /// A fetch every waiter of this quorum shares. - fn start( - self: Arc, - quorum: QuorumId, - height: u32, - id: u64, - failures: u32, - ) -> SharedFetch { + fn start(self: Arc, attempt: Attempt) -> SharedFetch { + let attempt = Arc::new(attempt); async move { let prover = Arc::clone(&self); + let inner = Arc::clone(&attempt); let result = contained(move || async move { - let result = prover.prove(quorum, height).await; + let (result, asked) = prover.prove(&inner).await; // Recorded here too, so a browser fetch that outlives its // waiters still frees its slot. - prover.finish(quorum, height, id, failures, &result); + prover.finish(&inner, &result, asked); result }) .await; // Covers a fetch that panicked; a no-op otherwise. - self.finish(quorum, height, id, failures, &result); + self.finish(&attempt, &result, true); result } .boxed() .shared() } - /// Record how fetch `id` of a quorum ended, if its slot is still its own. - fn finish( - &self, - quorum: QuorumId, - height: u32, - id: u64, - failures: u32, - result: &Result<(), String>, - ) { + /// Record how a fetch ended, if its slot is still its own. A failure + /// without a single request asked nothing, so it does not cool the + /// quorum down. + fn finish(&self, attempt: &Attempt, result: &Result<(), String>, asked: bool) { let mut proven = self.lock(); let ours = matches!( - proven.fetches.get(&quorum), - Some(Slot::Running { id: running, .. }) if *running == id + proven.fetches.get(&attempt.quorum), + Some(Slot::Running { id, .. }) if *id == attempt.id ); - if ours && result.is_ok() { - proven.fetches.remove(&quorum); - } else if ours { + if !ours { + return; + } + if result.is_err() && asked { let failed = Slot::Failed { at: Instant::now(), - failures: failures.saturating_add(1), - height, + failures: attempt.failures.saturating_add(1), + height: attempt.height, }; - proven.fetches.insert(quorum, failed); + proven.fetches.insert(attempt.quorum, failed); + } else { + proven.fetches.remove(&attempt.quorum); } } - /// Ask the sources for a proof of `quorum` until one verifies, the - /// request budget is spent or the deadline passes. - async fn prove(&self, quorum: QuorumId, height: u32) -> Result<(), String> { + /// Ask the sources for a proof of a quorum until one verifies, the + /// request budget is spent, every source is out or the deadline passes. + /// Also returns whether any request was sent. + async fn prove(&self, attempt: &Attempt) -> (Result<(), String>, bool) { + let (quorum, height) = (attempt.quorum, attempt.height); let started = Instant::now(); let deadline = started + self.timing.deadline; let mut requests = FuturesUnordered::new(); - let mut asked = vec![0u32; self.endpoints.len()]; - let mut busy = Vec::new(); + let mut sources = vec![SourceAttempts::new(started); self.endpoints.len()]; let mut sent = 0; - let mut retries = 0; - let mut alternate = false; - let mut next_start = started; + let mut hedge_at = started; let mut failures = Vec::new(); loop { let now = Instant::now(); @@ -501,38 +575,42 @@ impl ProofFetches { } let mut wake = deadline; if requests.len() < MAX_REQUESTS_IN_FLIGHT && sent < MAX_REQUESTS_PER_FETCH { - if now < next_start { - wake = next_start.min(deadline); - } else if let Some(source) = self.pick_source(now, &asked, &busy) { - let ask = choose_ask( - &self.lock().states, - self.checkpoint.height, - height, - alternate, - ) - .unwrap_or_else(|| Ask { - checkpoint: self.checkpoint.clone(), - height, - }); - let body = request_body(&ask, quorum); - let remaining = deadline.saturating_duration_since(now); - let timeout = self.timing.request_timeout.min(remaining); - // A request cut short by the deadline says nothing about - // its source. - let cut_short = timeout < self.timing.request_timeout; - asked[source] += 1; - busy.push(source); - sent += 1; - next_start = now + self.timing.hedge_after; - let endpoint = &self.endpoints[source]; - requests.push(async move { - let reply = self.transport.post(endpoint, body, timeout).await; - (source, ask, now, cut_short, reply) - }); - continue; - } else if requests.is_empty() { - failures.push("every source is cooling down after failures".to_string()); - break; + if !requests.is_empty() && now < hedge_at { + wake = hedge_at; + } else { + match self.pick_source(now, &sources) { + Pick::Ready(source) => { + let ask = choose_ask( + &self.lock().states, + self.checkpoint.height, + height, + sources[source].ask_tip, + ) + .unwrap_or_else(|| Ask { + checkpoint: self.checkpoint.clone(), + height, + }); + let body = request_body(&ask, quorum); + let remaining = deadline.saturating_duration_since(now); + let timeout = self.timing.request_timeout.min(remaining); + sources[source].asked += 1; + sources[source].in_flight = true; + sent += 1; + hedge_at = now + self.timing.hedge_after; + let endpoint = &self.endpoints[source]; + requests.push(async move { + let reply = self.transport.post(endpoint, body, timeout).await; + (source, ask, now, timeout, reply) + }); + continue; + } + Pick::At(at) => wake = at.min(deadline), + Pick::Nothing if requests.is_empty() => { + failures.push("no source left to ask".to_string()); + break; + } + Pick::Nothing => {} + } } } else if requests.is_empty() { break; @@ -540,23 +618,30 @@ impl ProofFetches { let wait = wake.saturating_duration_since(now); if requests.is_empty() { + attempt.waiting.store(true, Ordering::Relaxed); sleep(wait).await; + attempt.waiting.store(false, Ordering::Relaxed); continue; } let answered = match select(requests.next(), pin!(sleep(wait))).await { Either::Left((Some(answered), _)) => answered, Either::Left((None, _)) | Either::Right(_) => continue, }; - let (source, ask, asked_at, cut_short, reply) = answered; - busy.retain(|busy| *busy != source); + let (source, ask, asked_at, timeout, reply) = answered; let origin = self.endpoints[source].origin().ascii_serialization(); - let elapsed = asked_at.elapsed().as_millis(); + let elapsed = asked_at.elapsed(); + let state = &mut sources[source]; + state.in_flight = false; + let note = + |reason: &str| format!("{origin}: {reason} after {} ms", elapsed.as_millis()); + // A request the deadline cut short says nothing about its source. + let cut_short = timeout < self.timing.request_timeout && elapsed >= timeout; let verdict = match reply { - Err(PostError::Failed(reason)) if cut_short => Verdict::Retry { - reason, - after: Duration::ZERO, - }, - reply => self.judge(quorum, height, &ask, reply, retries), + Err(PostError::Failed(reason)) if cut_short => { + failures.push(note(&reason)); + continue; + } + reply => self.judge(quorum, height, &ask, reply), }; match verdict { Verdict::Proven => { @@ -566,25 +651,40 @@ impl ProofFetches { quorum_hash = %hex::encode(quorum.1), height, %origin, - elapsed_ms = elapsed, + elapsed_ms = elapsed.as_millis(), "proved quorum key" ); - return Ok(()); + return (Ok(()), true); + } + Verdict::CannotServe(reason) => { + state.cannot_serve += 1; + // Next time, ask this source for the other height. + state.ask_tip = ask.height != 0; + if state.cannot_serve >= MAX_CANNOT_SERVE { + state.dropped = true; + failures.push(note(&format!("{reason}; not asked again"))); + } else { + state.ready_at = Instant::now() + self.timing.first_retry; + failures.push(note(&reason)); + } } - Verdict::Retry { reason, after } => { - failures.push(format!("{origin}: {reason} after {elapsed} ms")); - retries += 1; - alternate = !alternate; - next_start = Instant::now() + after; + Verdict::Busy(reason) => { + state.busy += 1; + let max = self + .timing + .first_retry + .saturating_mul(1 << MAX_RETRY_DOUBLINGS); + state.ready_at = + Instant::now() + backoff(self.timing.first_retry, max, state.busy); + failures.push(note(&reason)); } Verdict::SourceFailed(reason) => { - failures.push(format!("{origin}: {reason} after {elapsed} ms")); self.source_failed(source); - next_start = Instant::now(); + failures.push(note(&reason)); } Verdict::BeyondTip(reason) => { - failures.push(format!("{origin}: {reason}")); - break; + state.dropped = true; + failures.push(note(&reason)); } } } @@ -596,31 +696,22 @@ impl ProofFetches { failures.join("; ") ); tracing::warn!(%reason, "quorum proof fetch failed"); - Err(reason) + (Err(reason), sent > 0) } /// Judge a relay's answer, publishing the key if it proves one. /// /// Only what an honest relay never does cools a source down: failed or - /// oversized transfers, redirects, error statuses other than 502, and - /// proofs that do not verify. A valid proof that does not serve the - /// request may be the fault of the unverified height or quorum, so it is - /// retried instead. + /// oversized transfers, redirects, error statuses other than 502 and + /// 503, and proofs that do not verify. A valid proof that does not serve + /// the request may be the fault of the unverified height or quorum hash. fn judge( &self, quorum: QuorumId, height: u32, ask: &Ask, reply: Result, - retries: u32, ) -> Verdict { - let retry = |reason: String| Verdict::Retry { - reason, - after: self - .timing - .first_retry - .saturating_mul(1 << retries.min(MAX_RETRY_DOUBLINGS)), - }; let reply = match reply { Ok(reply) => reply, Err(PostError::TooLarge) => { @@ -631,29 +722,27 @@ impl ProofFetches { let body = &reply.body; match reply.status { 200 => {} - // The relay's Core could not serve this proof (yet): worth asking - // again, for the other height. - 502 => return retry(format!("HTTP 502{}", body_excerpt(body))), - // Anything else, 503 (workers busy) included, cools the source down. + // The relay's Core cannot serve this quorum for this height. + 502 => return Verdict::CannotServe(format!("HTTP 502{}", body_excerpt(body))), + // The relay's workers are busy, which is normal under load. + 503 => return Verdict::Busy(format!("HTTP 503{}", body_excerpt(body))), status => return Verdict::SourceFailed(format!("HTTP {status}{}", body_excerpt(body))), } - let verified = match bootstrap::verify(body, &ask.checkpoint, 0) { + let needed = height.saturating_sub(self.window); + // The target height is checked before any signature, so a bogus + // height costs no BLS work. Only this source is affected if it lies. + let verified = match bootstrap::verify(body, &ask.checkpoint, needed) { Ok(verified) => verified, + Err(error) if error == STALE_TARGET && ask.height == 0 => { + return Verdict::BeyondTip(format!( + "its latest ChainLock is below the core height {needed} that {height} needs" + )); + } + Err(error) if error == STALE_TARGET => { + return Verdict::CannotServe(format!("proof ends below core height {needed}")); + } Err(error) => return Verdict::SourceFailed(format!("invalid proof: {error}")), }; - let target = verified.state().height; - let needed = height.saturating_sub(self.window); - if target < needed && ask.height == 0 { - return Verdict::BeyondTip(format!( - "its latest ChainLock, at core height {target}, is below the {needed} that core \ - height {height} needs" - )); - } - if target < needed { - return retry(format!( - "proof ends at core height {target}, below {needed}" - )); - } let mut keys = Vec::new(); // EvoNode records are not asked for; any that come back are ignored. for record in verified.records() { @@ -671,26 +760,45 @@ impl ProofFetches { // Records of other quorums do not matter; one that names the // requested quorum must parse. Err(error) if names(record.leaf, quorum) => { - return retry(format!("unparsable record of the quorum: {error}")); + return Verdict::CannotServe(format!( + "unparsable record of the quorum: {error}" + )); } Err(_) => {} } } let [key] = keys.as_slice() else { - return retry(format!( + return Verdict::CannotServe(format!( "proof opens the requested quorum {} times, not once", keys.len() )); }; match self.publish(quorum, *key, verified.state()) { Ok(()) => Verdict::Proven, - Err(reason) => retry(reason), + Err(reason) => Verdict::CannotServe(reason), } } - /// Cache a proven key and remember the state that proved it. - fn publish(&self, quorum: QuorumId, key: QuorumKey, target: &State) -> Result<(), String> { + /// Cache a proven key and remember the state that proved it. Nothing is + /// cached if either conflicts with what is already known. + pub(crate) fn publish( + &self, + quorum: QuorumId, + key: QuorumKey, + target: &State, + ) -> Result<(), String> { let mut proven = self.lock(); + if let Some(known) = proven.states.get(&target.height) { + if known != target { + tracing::error!( + height = target.height, + known = %rpc_hex(&known.block_hash), + proven = %rpc_hex(&target.block_hash), + "two proofs verified different blocks at one height; keeping the first" + ); + return Err("proves a different block than one already verified".to_string()); + } + } match proven.keys.get_mut(&quorum) { Some(cached) if cached.key != key => { tracing::error!( @@ -713,45 +821,51 @@ impl ProofFetches { ); } } - match proven.states.get(&target.height) { - Some(known) if known != target => tracing::error!( - height = target.height, - known = %rpc_hex(&known.block_hash), - proven = %rpc_hex(&target.block_hash), - "two proofs verified different blocks at one height; keeping the first" - ), - Some(_) => {} - None => { - proven.states.insert(target.height, target.clone()); - // One state is added at a time, so one removal keeps the bound. - if proven.states.len() > MAX_RECENT_STATES + 1 { - let oldest = proven - .states - .keys() - .copied() - .find(|height| *height != self.checkpoint.height); - if let Some(height) = oldest { - proven.states.remove(&height); - } + if proven + .states + .insert(target.height, target.clone()) + .is_none() + { + // One state is added at a time, so one removal keeps the bound. + if proven.states.len() > MAX_RECENT_STATES + 1 { + let oldest = proven + .states + .keys() + .copied() + .find(|height| *height != self.checkpoint.height); + if let Some(height) = oldest { + proven.states.remove(&height); } } } Ok(()) } - /// The source to ask next: a healthy one with no request of this fetch - /// in flight, asked the fewest times so far. - fn pick_source(&self, now: Instant, asked: &[u32], busy: &[usize]) -> Option { + /// The source to ask next: one that is not cooling down, left out of + /// this fetch or already asked, and is ready; among those the one asked + /// the fewest times. + fn pick_source(&self, now: Instant, sources: &[SourceAttempts]) -> Pick { let proven = self.lock(); - proven - .sources + let candidates: Vec<_> = sources .iter() + .zip(&proven.sources) .enumerate() - .filter(|(source, health)| { - !busy.contains(source) && health.failed_until.is_none_or(|until| until <= now) + .filter(|(_, (attempts, health))| { + !attempts.dropped + && !attempts.in_flight + && health.failed_until.is_none_or(|until| until <= now) }) - .min_by_key(|(source, _)| asked[*source]) - .map(|(source, _)| source) + .map(|(source, (attempts, _))| (source, attempts)) + .collect(); + let ready = candidates + .iter() + .filter(|(_, attempts)| attempts.ready_at <= now) + .min_by_key(|(_, attempts)| attempts.asked); + match (ready, candidates.iter().map(|(_, a)| a.ready_at).min()) { + (Some((source, _)), _) => Pick::Ready(*source), + (None, Some(at)) => Pick::At(at), + (None, None) => Pick::Nothing, + } } fn source_failed(&self, source: usize) { @@ -789,8 +903,12 @@ fn request_body(ask: &Ask, (quorum_type, quorum_hash): QuorumId) -> String { .to_string() } -/// The start of an error body, for the failure reason. +/// The start of a plain-text error body, for the failure reason. Markup, +/// such as a CDN's error page, is left out. fn body_excerpt(body: &[u8]) -> String { + if body.contains(&b'<') { + return String::new(); + } let text: String = String::from_utf8_lossy(body) .chars() .filter(|c| !c.is_control()) @@ -934,6 +1052,80 @@ mod tests { ) } + fn running_id(fetches: &ProofFetches, quorum: QuorumId) -> Option { + match fetches.lock().fetches.get(&quorum) { + Some(Slot::Running { id, .. }) => Some(*id), + _ => None, + } + } + + #[test] + fn should_evict_the_oldest_failure_when_the_slot_table_is_full() { + let fetches = Arc::new(fetches()); + let now = Instant::now(); + let earlier = now.checked_sub(Duration::from_secs(1)).unwrap_or(now); + { + let mut proven = fetches.lock(); + for n in 0..MAX_FETCH_SLOTS { + let at = if n == 0 { earlier } else { now }; + let failed = Slot::Failed { + at, + failures: 1, + height: 150, + }; + proven.fetches.insert((6, [n as u8; 32]), failed); + } + } + let _fetch = fetches + .join_or_start(6, [0xff; 32], 150) + .expect("a new quorum is not locked out"); + let proven = fetches.lock(); + assert_eq!(proven.fetches.len(), MAX_FETCH_SLOTS); + assert!( + !proven.fetches.contains_key(&(6, [0; 32])), + "the oldest goes" + ); + } + + #[test] + fn should_join_a_fetch_only_for_a_height_it_covers() { + let fetches = Arc::new(fetches()); + let _first = fetches.join_or_start(6, [1; 32], 150).expect("starts"); + let id = running_id(&fetches, (6, [1; 32])); + let _joined = fetches.join_or_start(6, [1; 32], 750).expect("joins"); + assert_eq!(running_id(&fetches, (6, [1; 32])), id); + let far = fetches.join_or_start(6, [1; 32], 751).map(|_| ()); + assert!( + far.as_ref() + .is_err_and(|reason| reason.contains("being proven for core height 150")), + "{far:?}" + ); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn should_replace_a_fetch_every_waiter_dropped() { + let fetches = Arc::new(fetches()); + let first = fetches.join_or_start(6, [1; 32], 150).expect("starts"); + let id = running_id(&fetches, (6, [1; 32])); + drop(first); + let _second = fetches.join_or_start(6, [1; 32], 150).expect("restarts"); + let replaced = running_id(&fetches, (6, [1; 32])); + assert!(replaced.is_some() && replaced != id); + } + + #[test] + fn should_not_count_fetches_waiting_out_a_backoff_against_the_limit() { + let fetches = Arc::new(fetches()); + let _first = fetches.join_or_start(6, [1; 32], 150).expect("first"); + let _second = fetches.join_or_start(6, [2; 32], 150).expect("second"); + assert!(fetches.join_or_start(6, [3; 32], 150).is_err()); + if let Some(Slot::Running { waiting, .. }) = fetches.lock().fetches.get(&(6, [1; 32])) { + waiting.store(true, Ordering::Relaxed); + } + assert!(fetches.join_or_start(6, [3; 32], 150).is_ok()); + } + #[test] fn should_keep_the_first_key_proven_for_a_quorum() { let fetches = fetches(); diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index 970b45598d6..ba70d5c3209 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -786,7 +786,7 @@ mod tests { assert!(again.contains("retry in 5 s"), "{again}"); let other = unavailable_reason(fetch(&provider, f.quorum_type, [0x11; 32], f.target.height).await); - assert!(other.contains("cooling down"), "{other}"); + assert!(other.contains("no source left"), "{other}"); assert_eq!(relay.requests().len(), 1, "cooling down sends nothing"); } @@ -848,15 +848,124 @@ mod tests { } #[tokio::test] - async fn should_bound_requests_to_a_relay_that_keeps_failing() { + async fn should_stop_asking_relays_that_cannot_serve_a_fake_quorum() { let f = fixture(Network::Testnet); - let relay = MockRelay::spawn((0..12).map(|_| reply(502, Vec::new())).collect()); + let gateway = || (0..6).map(|_| reply(502, Vec::new())).collect::>(); + let relays = [MockRelay::spawn(gateway()), MockRelay::spawn(gateway())]; + let sources = relays.iter().map(|relay| relay.url.clone()).collect(); + let provider = provider_for(&f, sources, quick()); + let started = Instant::now(); + let reason = + unavailable_reason(fetch(&provider, f.quorum_type, [0x11; 32], f.target.height).await); + assert!(reason.contains("not asked again"), "{reason}"); + assert!(started.elapsed() < Duration::from_secs(20), "{reason}"); + for relay in &relays { + let heights: Vec<_> = relay + .requests() + .iter() + .map(|r| r["height"].clone()) + .collect(); + assert_eq!(heights, [json!(f.target.height), json!(0)], "{reason}"); + } + } + + #[tokio::test] + async fn should_hedge_to_another_source_while_one_hangs() { + let f = fixture(Network::Testnet); + let hung = MockRelay::spawn(vec![Answer::Hang(Duration::from_secs(20))]); + let answering = MockRelay::spawn(vec![reply(200, f.bootstrap.clone())]); + let timing = Timing { + hedge_after: Duration::from_millis(200), + ..quick() + }; + let sources = vec![hung.url.clone(), answering.url.clone()]; + let provider = provider_for(&f, sources, timing); + let key = fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await; + assert_eq!(key.expect("the second source proves it"), Some(f.key)); + assert_eq!(hung.requests().len(), 1); + assert_eq!(answering.requests().len(), 1); + } + + #[tokio::test] + async fn should_prove_with_another_source_after_one_fails() { + let f = fixture(Network::Testnet); + let failing = MockRelay::spawn(vec![reply(403, Vec::new())]); + let answering = MockRelay::spawn(vec![reply(200, f.bootstrap.clone())]); + let sources = vec![failing.url.clone(), answering.url.clone()]; + let provider = provider_for(&f, sources, quick()); + let key = fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await; + assert_eq!(key.expect("the second source proves it"), Some(f.key)); + assert_eq!(failing.requests().len(), 1); + assert_eq!(answering.requests().len(), 1); + } + + #[tokio::test] + async fn should_ask_the_other_sources_when_one_is_behind() { + let f = fixture(Network::Testnet); + let behind = || vec![reply(502, Vec::new()), reply(200, f.bootstrap.clone())]; + let relays = [MockRelay::spawn(behind()), MockRelay::spawn(behind())]; + let sources = relays.iter().map(|relay| relay.url.clone()).collect(); + let provider = provider_for(&f, sources, quick()); + let reason = unavailable_reason( + fetch(&provider, f.quorum_type, f.quorum_hash, i32::MAX as u32).await, + ); + // Each relay's tip proof falls short; the first does not end the fetch. + assert_eq!(reason.matches("latest ChainLock").count(), 2, "{reason}"); + for relay in &relays { + assert_eq!(relay.requests().len(), 2, "{reason}"); + } + } + + #[tokio::test] + async fn should_retry_a_busy_relay_without_cooling_it_down() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn(vec![ + reply(503, b"busy".to_vec()), + reply(200, f.bootstrap.clone()), + ]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let key = fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await; + assert_eq!(key.expect("proven after the retry"), Some(f.key)); + assert_eq!(relay.requests().len(), 2); + } + + #[tokio::test] + async fn should_cool_down_a_relay_that_times_out_upstream() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn(vec![reply(504, Vec::new())]); let provider = provider_for(&f, vec![relay.url.clone()], quick()); let reason = unavailable_reason( fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await, ); - assert!(reason.contains("HTTP 502"), "{reason}"); - assert_eq!(relay.requests().len(), 8); + assert!(reason.contains("HTTP 504"), "{reason}"); + let other = + unavailable_reason(fetch(&provider, f.quorum_type, [0x11; 32], f.target.height).await); + assert!(other.contains("no source left"), "{other}"); + assert_eq!(relay.requests().len(), 1); + } + + #[tokio::test] + async fn should_keep_the_first_key_when_a_proof_opens_another() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn(vec![reply(200, f.bootstrap.clone())]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let quorum = (f.quorum_type, f.quorum_hash); + // A key proven long ago, too long for the requested height. + provider + .inner + .fetches + .publish(quorum, [1; 48], &f.checkpoint) + .expect("an earlier proof"); + let reason = unavailable_reason( + fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await, + ); + assert!(reason.contains("different key"), "{reason}"); + let cached = + provider.get_quorum_public_key(f.quorum_type, f.quorum_hash, f.checkpoint.height); + assert_eq!(cached.expect("still cached"), [1; 48]); + assert!(provider + .get_quorum_public_key(f.quorum_type, f.quorum_hash, f.target.height) + .is_err()); } #[tokio::test] From abab15aa711b09718ae4a9ad550b22125c795c19 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 21 Oct 2026 12:03:41 -0500 Subject: [PATCH 04/25] docs(sdk): document verified provider limits and cache sizing Explain how a checkpoint ages and why long-lived applications should persist the latest verified state, what happens to unproven quorums if Platform halts, the cache size that avoids re-proving, and describe both providers in the README's implementation details. Co-Authored-By: Claude Opus 5.5 --- .../rs-sdk-trusted-context-provider/README.md | 50 ++++++++++++++++--- .../src/verified_provider.rs | 22 ++++++-- 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/README.md b/packages/rs-sdk-trusted-context-provider/README.md index ddf1c79d69c..72b925a2a69 100644 --- a/packages/rs-sdk-trusted-context-provider/README.md +++ b/packages/rs-sdk-trusted-context-provider/README.md @@ -170,9 +170,15 @@ Behaviour: an unavailable quorum source, so the SDK tries another node without banning the one that named the quorum. A quorum of another type than the network's Platform quorum type is rejected as invalid. -- Fetches of one quorum are shared, at most two quorums are proven at once, - failed quorums and failing sources cool down, and a fetch gives up after - 70 s. Redirects are refused and responses are capped at 1 MiB. +- Fetches of one quorum are shared and at most two quorums are proven at + once. A relay that cannot serve a quorum for the height and for its tip is + not asked again in that fetch; a busy relay (503) is retried after a + backoff; relays that fail otherwise, and quorums whose fetch failed, cool + down. A fetch gives up after 70 s. Redirects are refused and responses are + capped at 1 MiB. +- The cache should hold at least the active Platform quorums (24 on mainnet + and testnet); 32 or more avoids proving keys again just because they were + evicted. - Data contracts come from the known-contracts cache and the system contracts enabled by cargo features; token configurations from the known cache. - The public mainnet quorum service currently answers `/proofs` with HTTP 403. @@ -183,11 +189,32 @@ Behaviour: ### Trust model The checkpoint is trust configuration, like a hard-coded block hash: whoever -chooses it chooses the chain. Beyond it, the provider trusts the ChainLock -quorums the proof chain passes through, as Core does. A valid proof shows a +chooses it chooses the chain. Beyond it, the provider trusts that no ChainLock +quorum active at the checkpoint or mined after it ever signs dishonestly, +retired ones included, as `dash-core-proof` documents. A valid proof shows a quorum was mined, not that its target is the newest ChainLock; how fresh a response must be is the SDK's policy. +### Checkpoint shelf life + +A proof covers every ChainLock quorum rotation between its starting state and +its target, so it grows as the checkpoint ages, and a relay has to keep the +blocks back to it. Each SDK release can move the embedded checkpoints forward. +A long-lived application should persist `latest_verified_state()` and pass it +back to `with_checkpoint` on the next start, so its proofs stay short. + +### Limitations + +- Keys are proven for the Core height Platform reports in its responses. If + Platform halts, that height stops while Core moves on: a quorum not cached + before the halt can be proven only while a relay can still serve a proof + within 600 blocks of that height whose quorum root contains the quorum, and + retired quorums leave Core's quorum root. Such quorums may stay unprovable + until Platform resumes. +- An inclusion proof cannot show that a quorum does not exist, so the + provider never reports a quorum as absent; nodes naming fake quorums are + skipped by the SDK, not banned. + ### Refreshing the embedded checkpoints `checkpoints/.json` hold Core `ProofState` JSON (hashes in RPC display @@ -207,7 +234,18 @@ order). To move one to a newer block `H`: ## Implementation Details -The `TrustedHttpContextProvider` implements the `ContextProvider` trait and provides: +Both providers implement the `ContextProvider` trait. + +`VerifiedHttpContextProvider` provides: + +1. **Quorum Public Keys**: proven from its checkpoint with DIP 175 proofs from + its relays, and kept in an LRU cache +2. **Data Contracts**: pre-loaded or registered known contracts, then the + system contracts enabled by cargo features, otherwise `None` +3. **Token Configurations**: pre-loaded known configurations, otherwise `None` +4. **Platform Activation Height**: hardcoded values for each network + +`TrustedHttpContextProvider` provides: 1. **Quorum Public Keys**: Fetched from trusted HTTP endpoints with LRU caching 2. **Data Contracts**: diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index ba70d5c3209..fe1dc13b25e 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -78,9 +78,17 @@ pub enum VerifiedContextProviderError { /// as invalid and never fetched. /// /// Fetches are shared per quorum, at most two quorums are proven at once, -/// failed quorums and failing sources cool down, and each fetch has a 70 s -/// deadline. Freshness of responses is the SDK's concern, not the -/// provider's. +/// relays that cannot serve a quorum are left out of its fetch, failed +/// quorums and failing relays cool down, and each fetch has a 70 s deadline. +/// Freshness of responses is the SDK's concern, not the provider's. +/// +/// Keys are proven for the Core height Platform reports. If Platform halts, +/// that height stops while Core moves on, and a quorum not cached before the +/// halt may become unprovable once it leaves Core's quorum root. +/// +/// A proof grows with the age of its starting state. Long-lived applications +/// should persist [`latest_verified_state`](Self::latest_verified_state) and +/// pass it back as the checkpoint on their next start. /// /// The public mainnet relay currently answers `/proofs` with HTTP 403, so on /// mainnet [`new`](Self::new) builds but fetches fail until a relay serves @@ -167,6 +175,10 @@ impl VerifiedHttpContextProvider { /// A provider for mainnet or testnet that starts from the embedded /// checkpoint and asks the network's public quorum service. Makes no /// network request. + /// + /// `cache_size` is how many proven quorum keys are kept. It should be at + /// least the number of active Platform quorums (24 on mainnet and + /// testnet); 32 or more avoids proving keys again after eviction. pub fn new( network: Network, cache_size: NonZeroUsize, @@ -184,8 +196,8 @@ impl VerifiedHttpContextProvider { /// The checkpoint is trust configuration: a ChainLocked block the /// application checked independently, or a state this provider verified /// earlier ([`latest_verified_state`](Self::latest_verified_state)). - /// Sources must be `https`, except on loopback hosts. Hosts are not - /// resolved here. + /// Sources must be `https`, except on loopback hosts, and distinct. Hosts + /// are not resolved here. `cache_size` is as for [`new`](Self::new). pub fn with_checkpoint( network: Network, checkpoint: State, From 5ebe53d0c421ae298d3ee5a87d48f065317bbbc9 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 21 Oct 2026 12:13:49 -0500 Subject: [PATCH 05/25] fix(sdk): bound verified quorum fetches that resume after a backoff The two-quorum limit was only checked when a fetch started, so fetches waiting out a relay's backoff let new ones start and then all sent again together. A fetch now holds one of two places only while it has requests to send: it gives the place up while it waits and must take one again before asking a relay, waiting within its deadline if none is free. Also: - a fetch that sent nothing keeps the quorum's earlier failure count instead of resetting its cooldown; - failure reasons say when every source is cooling down or out of the fetch, and when the request budget is spent; - waits are at least 1 ms, since browser timers round shorter ones to zero. Tests cover a lagging relay left out while another proves the key, a refused hedge near the deadline cooling its source while the request the deadline cut short does not, and fetches giving up and retaking their place. Co-Authored-By: Claude Opus 5.5 --- .../rs-sdk-trusted-context-provider/README.md | 5 +- .../src/proof_fetches.rs | 151 +++++++++++++----- .../src/verified_provider.rs | 78 ++++++++- .../tests/data/proof-testnet-no-links.json | 18 +++ 4 files changed, 208 insertions(+), 44 deletions(-) create mode 100644 packages/rs-sdk-trusted-context-provider/tests/data/proof-testnet-no-links.json diff --git a/packages/rs-sdk-trusted-context-provider/README.md b/packages/rs-sdk-trusted-context-provider/README.md index 72b925a2a69..aaf10ebd34a 100644 --- a/packages/rs-sdk-trusted-context-provider/README.md +++ b/packages/rs-sdk-trusted-context-provider/README.md @@ -170,8 +170,9 @@ Behaviour: an unavailable quorum source, so the SDK tries another node without banning the one that named the quorum. A quorum of another type than the network's Platform quorum type is rejected as invalid. -- Fetches of one quorum are shared and at most two quorums are proven at - once. A relay that cannot serve a quorum for the height and for its tip is +- Fetches of one quorum are shared, and requests for at most two quorums + are in flight at once (a fetch waiting out a backoff gives its place up). + A relay that cannot serve a quorum for the height and for its tip is not asked again in that fetch; a busy relay (503) is retried after a backoff; relays that fail otherwise, and quorums whose fetch failed, cool down. A fetch gives up after 70 s. Redirects are refused and responses are diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index 41ca0a3ee0d..61878e2675a 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -69,6 +69,9 @@ const MAX_RETRY_DOUBLINGS: u32 = 3; /// Answers a source may give that it cannot serve the quorum before a fetch /// stops asking it: one for the height and one for the tip. const MAX_CANNOT_SERVE: u32 = 2; +/// How long a fetch waits before trying again to take a place to prove its +/// quorum. +const PLACE_RETRY: Duration = Duration::from_millis(250); /// How `bootstrap::verify` reports a target below the minimum height, which /// it checks before any signature. const STALE_TARGET: ProofError = ProofError("stale target"); @@ -132,8 +135,8 @@ enum Slot { Running { id: u64, height: u32, - /// Set while the fetch only waits out a relay's backoff. - waiting: Arc, + /// Set while the fetch holds one of the places to prove a quorum. + holding: Arc, fetch: SharedFetch, }, Failed { @@ -221,9 +224,11 @@ struct Attempt { quorum: QuorumId, height: u32, id: u64, - /// Earlier failures of this quorum, for its cooldown. + /// Earlier failures of this quorum, for its cooldown, and when the last + /// one was. failures: u32, - waiting: Arc, + failed_at: Option, + holding: Arc, } /// What one fetch knows about one source. @@ -421,7 +426,7 @@ impl ProofFetches { return Ok(future::ready(Ok(())).boxed().shared()); } } - let failures = match proven.fetches.get(&quorum) { + let (failures, failed_at) = match proven.fetches.get(&quorum) { Some(Slot::Running { fetch, height: running, @@ -453,17 +458,11 @@ impl ProofFetches { wait.as_millis().div_ceil(1000) )); } - *failures + (*failures, Some(*at)) } - Some(Slot::Failed { .. }) | None => 0, + Some(Slot::Failed { .. }) | None => (0, None), }; - // A fetch only waiting out a relay's backoff sends nothing, so it - // does not hold one of the places. - let running = proven - .fetches - .values() - .filter(|slot| matches!(slot, Slot::Running { waiting, .. } if !waiting.load(Ordering::Relaxed))) - .count(); + let running = places_taken(&proven); if running >= MAX_QUORUMS_IN_FLIGHT { return Err(format!( "{running} other quorums are being proven; not starting another" @@ -487,13 +486,14 @@ impl ProofFetches { } proven.next_fetch_id += 1; let id = proven.next_fetch_id; - let waiting = Arc::new(AtomicBool::new(false)); + let holding = Arc::new(AtomicBool::new(true)); let attempt = Attempt { quorum, height, id, failures, - waiting: Arc::clone(&waiting), + failed_at, + holding: Arc::clone(&holding), }; // Built under the lock, so it must do nothing until it is polled. let fetch = Arc::clone(self).start(attempt); @@ -502,7 +502,7 @@ impl ProofFetches { Slot::Running { id, height, - waiting, + holding, fetch: fetch.clone(), }, ); @@ -533,8 +533,9 @@ impl ProofFetches { /// Record how a fetch ended, if its slot is still its own. A failure /// without a single request asked nothing, so it does not cool the - /// quorum down. + /// quorum down further; the earlier failures are kept. fn finish(&self, attempt: &Attempt, result: &Result<(), String>, asked: bool) { + attempt.holding.store(false, Ordering::Relaxed); let mut proven = self.lock(); let ours = matches!( proven.fetches.get(&attempt.quorum), @@ -543,16 +544,40 @@ impl ProofFetches { if !ours { return; } - if result.is_err() && asked { - let failed = Slot::Failed { - at: Instant::now(), - failures: attempt.failures.saturating_add(1), - height: attempt.height, - }; - proven.fetches.insert(attempt.quorum, failed); - } else { - proven.fetches.remove(&attempt.quorum); + let failed = match (result, asked, attempt.failed_at) { + (Ok(()), _, _) => None, + (Err(_), true, _) => Some((Instant::now(), attempt.failures.saturating_add(1))), + (Err(_), false, Some(at)) => Some((at, attempt.failures)), + (Err(_), false, None) => None, + }; + match failed { + Some((at, failures)) => { + let failed = Slot::Failed { + at, + failures, + height: attempt.height, + }; + proven.fetches.insert(attempt.quorum, failed); + } + None => { + proven.fetches.remove(&attempt.quorum); + } + } + } + + /// Take one of the places to prove a quorum, unless the fetch holds one + /// already. A fetch gives its place up while it only waits, so the limit + /// holds for fetches that send. + fn take_place(&self, attempt: &Attempt) -> bool { + if attempt.holding.load(Ordering::Relaxed) { + return true; + } + let proven = self.lock(); + let free = places_taken(&proven) < MAX_QUORUMS_IN_FLIGHT; + if free { + attempt.holding.store(true, Ordering::Relaxed); } + free } /// Ask the sources for a proof of a quorum until one verifies, the @@ -567,6 +592,7 @@ impl ProofFetches { let mut sent = 0; let mut hedge_at = started; let mut failures = Vec::new(); + let mut waited_for_place = false; loop { let now = Instant::now(); if now >= deadline { @@ -579,6 +605,13 @@ impl ProofFetches { wake = hedge_at; } else { match self.pick_source(now, &sources) { + Pick::Ready(_) if !self.take_place(attempt) => { + if !waited_for_place { + waited_for_place = true; + failures.push("waited for other quorums' proofs".to_string()); + } + wake = (now + PLACE_RETRY).min(deadline); + } Pick::Ready(source) => { let ask = choose_ask( &self.lock().states, @@ -606,21 +639,27 @@ impl ProofFetches { } Pick::At(at) => wake = at.min(deadline), Pick::Nothing if requests.is_empty() => { - failures.push("no source left to ask".to_string()); + failures.push( + "every source is cooling down or out of this fetch".to_string(), + ); break; } Pick::Nothing => {} } } } else if requests.is_empty() { + failures.push(format!("all {MAX_REQUESTS_PER_FETCH} requests used")); break; } - let wait = wake.saturating_duration_since(now); + // Browser timers round short waits down to zero. + let wait = wake + .saturating_duration_since(now) + .max(Duration::from_millis(1)); if requests.is_empty() { - attempt.waiting.store(true, Ordering::Relaxed); + // Nothing is in flight: let another quorum be proven meanwhile. + attempt.holding.store(false, Ordering::Relaxed); sleep(wait).await; - attempt.waiting.store(false, Ordering::Relaxed); continue; } let answered = match select(requests.next(), pin!(sleep(wait))).await { @@ -868,6 +907,15 @@ impl ProofFetches { } } + /// Which sources are cooling down. + #[cfg(test)] + pub(crate) fn cooling(&self) -> Vec { + let now = Instant::now(); + let proven = self.lock(); + let cooling = |health: &SourceHealth| health.failed_until.is_some_and(|until| until > now); + proven.sources.iter().map(cooling).collect() + } + fn source_failed(&self, source: usize) { let first = self.timing.source_cooldown; let max = self.timing.max_source_cooldown; @@ -884,6 +932,17 @@ impl ProofFetches { } } +/// How many fetches hold one of the places to prove a quorum. +fn places_taken(proven: &Proven) -> usize { + proven + .fetches + .values() + .filter( + |slot| matches!(slot, Slot::Running { holding, .. } if holding.load(Ordering::Relaxed)), + ) + .count() +} + /// Whether a commitment's leading bytes (version, type, hash) name `quorum`. fn names(leaf: &[u8], (quorum_type, quorum_hash): QuorumId) -> bool { let mut wire = quorum_hash; @@ -1115,15 +1174,35 @@ mod tests { } #[test] - fn should_not_count_fetches_waiting_out_a_backoff_against_the_limit() { + fn should_let_only_fetches_holding_a_place_send() { let fetches = Arc::new(fetches()); let _first = fetches.join_or_start(6, [1; 32], 150).expect("first"); let _second = fetches.join_or_start(6, [2; 32], 150).expect("second"); assert!(fetches.join_or_start(6, [3; 32], 150).is_err()); - if let Some(Slot::Running { waiting, .. }) = fetches.lock().fetches.get(&(6, [1; 32])) { - waiting.store(true, Ordering::Relaxed); - } - assert!(fetches.join_or_start(6, [3; 32], 150).is_ok()); + let holding = |quorum: QuorumId| match fetches.lock().fetches.get(&quorum) { + Some(Slot::Running { holding, .. }) => Some(Arc::clone(holding)), + _ => None, + }; + // The first gives its place up while it waits out a backoff. + let first = holding((6, [1; 32])).expect("running"); + first.store(false, Ordering::Relaxed); + let _third = fetches + .join_or_start(6, [3; 32], 150) + .expect("a place is free"); + // Back from the wait, the first must wait for a place again. + let attempt = Attempt { + quorum: (6, [1; 32]), + height: 150, + id: 1, + failures: 0, + failed_at: None, + holding: first, + }; + assert!(!fetches.take_place(&attempt)); + holding((6, [2; 32])) + .expect("running") + .store(false, Ordering::Relaxed); + assert!(fetches.take_place(&attempt)); } #[test] diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index fe1dc13b25e..7fba4502a04 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -77,8 +77,8 @@ pub enum VerifiedContextProviderError { /// * A quorum of a type other than the network's Platform type is rejected /// as invalid and never fetched. /// -/// Fetches are shared per quorum, at most two quorums are proven at once, -/// relays that cannot serve a quorum are left out of its fetch, failed +/// Fetches are shared per quorum, requests for at most two quorums are in +/// flight at once, relays that cannot serve a quorum are left out of its fetch, failed /// quorums and failing relays cool down, and each fetch has a 70 s deadline. /// Freshness of responses is the SDK's concern, not the provider's. /// @@ -488,6 +488,9 @@ mod tests { Chunked { size: usize, chunks: usize }, /// Read the request, then hold the connection open without answering. Hang(Duration), + /// Answer 200 with this body after a delay, without holding up + /// later connections. + Late(Duration, Vec), } fn reply(status: u16, body: Vec) -> Answer { @@ -566,6 +569,18 @@ mod tests { drop(stream); }); } + Answer::Late(delay, body) => { + thread::spawn(move || { + thread::sleep(delay); + let head = format!( + "HTTP/1.1 200 OK\r\nConnection: close\r\n\ + Content-Length: {}\r\n\r\n", + body.len() + ); + let _ = stream.write_all(head.as_bytes()); + let _ = stream.write_all(&body); + }); + } } } }); @@ -798,7 +813,7 @@ mod tests { assert!(again.contains("retry in 5 s"), "{again}"); let other = unavailable_reason(fetch(&provider, f.quorum_type, [0x11; 32], f.target.height).await); - assert!(other.contains("no source left"), "{other}"); + assert!(other.contains("cooling down"), "{other}"); assert_eq!(relay.requests().len(), 1, "cooling down sends nothing"); } @@ -912,7 +927,40 @@ mod tests { } #[tokio::test] - async fn should_ask_the_other_sources_when_one_is_behind() { + async fn should_leave_out_only_the_source_that_is_behind() { + let f = fixture(Network::Testnet); + // A proof from another checkpoint whose target is two blocks below + // the fixture's: its height is judged before anything else. + let behind: Value = + serde_json::from_str(include_str!("../tests/data/proof-testnet-no-links.json")) + .expect("fixture JSON"); + let behind = hex::decode(behind["bootstrap_hex"].as_str().expect("bootstrap_hex")) + .expect("bootstrap hex"); + let lagging = MockRelay::spawn(vec![reply(502, Vec::new()), reply(200, behind)]); + let proving = MockRelay::spawn(vec![Answer::Late( + Duration::from_millis(500), + f.bootstrap.clone(), + )]); + let timing = Timing { + hedge_after: Duration::from_millis(50), + ..quick() + }; + let provider = provider_for(&f, vec![lagging.url.clone(), proving.url.clone()], timing); + // The fixture's target is exactly the lowest one this height accepts. + let height = f.target.height + 600; + let key = fetch(&provider, f.quorum_type, f.quorum_hash, height).await; + assert_eq!(key.expect("the other source proves it"), Some(f.key)); + let heights: Vec<_> = lagging + .requests() + .iter() + .map(|r| r["height"].clone()) + .collect(); + assert_eq!(heights, [json!(height), json!(0)]); + assert_eq!(proving.requests().len(), 1); + } + + #[tokio::test] + async fn should_end_a_fetch_once_every_source_is_behind() { let f = fixture(Network::Testnet); let behind = || vec![reply(502, Vec::new()), reply(200, f.bootstrap.clone())]; let relays = [MockRelay::spawn(behind()), MockRelay::spawn(behind())]; @@ -921,11 +969,29 @@ mod tests { let reason = unavailable_reason( fetch(&provider, f.quorum_type, f.quorum_hash, i32::MAX as u32).await, ); - // Each relay's tip proof falls short; the first does not end the fetch. assert_eq!(reason.matches("latest ChainLock").count(), 2, "{reason}"); for relay in &relays { assert_eq!(relay.requests().len(), 2, "{reason}"); } + assert_eq!(provider.inner.fetches.cooling(), [false, false]); + } + + #[tokio::test] + async fn should_cool_down_a_hedged_source_that_refuses_near_the_deadline() { + let f = fixture(Network::Testnet); + let hung = MockRelay::spawn(vec![Answer::Hang(Duration::from_secs(20))]); + let closed = TcpListener::bind("127.0.0.1:0").expect("bind"); + let refused = format!("http://{}", closed.local_addr().expect("address")); + drop(closed); + let timing = Timing { + deadline: Duration::from_secs(2), + hedge_after: Duration::from_millis(100), + ..quick() + }; + let provider = provider_for(&f, vec![hung.url.clone(), refused], timing); + unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await); + // The hung request was cut short by the deadline; the refusal was not. + assert_eq!(provider.inner.fetches.cooling(), [false, true]); } #[tokio::test] @@ -952,7 +1018,7 @@ mod tests { assert!(reason.contains("HTTP 504"), "{reason}"); let other = unavailable_reason(fetch(&provider, f.quorum_type, [0x11; 32], f.target.height).await); - assert!(other.contains("no source left"), "{other}"); + assert!(other.contains("cooling down"), "{other}"); assert_eq!(relay.requests().len(), 1); } diff --git a/packages/rs-sdk-trusted-context-provider/tests/data/proof-testnet-no-links.json b/packages/rs-sdk-trusted-context-provider/tests/data/proof-testnet-no-links.json new file mode 100644 index 00000000000..ea28d635005 --- /dev/null +++ b/packages/rs-sdk-trusted-context-provider/tests/data/proof-testnet-no-links.json @@ -0,0 +1,18 @@ +{ + "checkpoint": { + "network": 1, + "height": 1576616, + "block_hash": "0000000d3730f7d79f266391a3404636ae7d53e9ac1535ee629dac1fd01eec6d", + "masternode_root": "28736666198c9a8d0564edea185ac0e35fd4e315e384aea716b7fa2e51ea4ddf", + "quorum_root": "e5cf38b3f4aa4fdc1ce08bb90ae297f78f2659c4d043d0042b309c6d05d5fd38" + }, + "proof_hex": "444153484e43303201a80e18006dec1ed01fac9d62ee3515ace9537dae364640a39163269fd7f730370d000000df4dea512efab716a7ae84e315e3d45fe3c05a18eaed64058d9a8c196666732838fdd5056d9c302b04d043d0c459268ff797e20ab98be01cdc4faaf4b338cfe543010000030001f94e8097514b57020d7d908d4966051535dd12a0a2dae7a9905a011a3f00000032ffffffffffff0332ffffffffffff0394be2be0233ebc598fc641982e613dbcfc5877aa602f159cfcdd29be1d0a4a750813459b63f0c405ab34014781357ad94259bb4abbda8f3466fbc42967c84bd5ecc4f796b75e34f47b54b72af075dcb6989a99e1b1da140f77edbb04e036598b52ef4ccec10d99cb31ec779d4d850868537b8ce4494d98ca6a5b679069b106e90c5e1ef210b68295a3995975c2361e0b3bc86948068e2c8e51a879f9bbcba402835698407cbaafb9f020706a4fa29c48a1b285d4b64cb837366d873e0437fd8ec7a401ae35920c29967f38c1ace4239be38ed28d308311ff6ff3b96e9b5150b904574da93dd7c4b210ce1654c695bc2ceeb7d5fd275813decf9a56e972ab7db74aea3bccb739de7b6e9e273b171516a32b0000006d000000074c097f54a9994db8255d509566490589f0277a0d42afaf64c7ff8f096797bc3c2ef0779342efb9820f53577e9fc79b59c393f204a9c2b1745e48392b1eeefd09cd4b4a67dd4c74bb8021815dd277c5a2a1974f699ad28c2e44072874c9a120c4e48a1ecf7705668ce329a18c7f5bba9f14f494830288571b82b7138534c6929f089b14beaa068ac85c1443a97aa6009c8ac175de5461cf2034a094f3ebb1d4c4d3e23116ae003d38d61f14af6299de389397805ca4ecbd65a2aab7c9c5f99be77f818b449cef8aab960ee6848b4968a0c24f7b45e0f1c105622379b1cb8c083300003e0f180000000020373732036aa2b418bda3ad8f22a62152d4ef9d550b49c0ebf261604f34000000ec6c85972aba478981a19233b3514ef541976a4e8a6d0450dcd8d7277a685538e4e0d86ad4e7001ebff50500aa08742e5f391c309351c0f45dd708fa7bac12a5d3b47f6d8e3d84547dab096f39268c2e64c7cca868a503f908bf0c7d08f18e05dc3658522ae25124561bce8ae0e5d7f85456d6ebdc7b9817a0c73ed26055550e0a2ede3482e990e227fcf1fb3201000003000500010000000000000000000000000000000000000000000000000000000000000000ffffffff016affffffff03b84b8c03000000001976a914b489115851ca07a26a5ad8bac3cec3c7dbebd83188ac2ed5fd0300000000016af80da706000000001976a914c69a0bda7daaae481be8def95e5f347a1d00a4b488ac00000000af03003e0f1800df4dea512efab716a7ae84e315e3d45fe3c05a18eaed64058d9a8c1966667328e1f2b2b43f667274b3e8f488f24dc9f7fa611149dc36e2324826b951d5be8950008539584f69b9a56090c811d246a1571fdf8edda332425d51db17f9d421c77b1d7f7d610a9314032a18b73989ff8b82f70a06172ae5b1daae0b858225b6c85fabaee79909e71158e102f7207eee5ca3783b7a4cabe981fb525e072d1751e0ba789802289a2d220000000000000100000000", + "bootstrap_hex": "92040000444153484e43303201a80e18006dec1ed01fac9d62ee3515ace9537dae364640a39163269fd7f730370d000000df4dea512efab716a7ae84e315e3d45fe3c05a18eaed64058d9a8c196666732838fdd5056d9c302b04d043d0c459268ff797e20ab98be01cdc4faaf4b338cfe543010000030001f94e8097514b57020d7d908d4966051535dd12a0a2dae7a9905a011a3f00000032ffffffffffff0332ffffffffffff0394be2be0233ebc598fc641982e613dbcfc5877aa602f159cfcdd29be1d0a4a750813459b63f0c405ab34014781357ad94259bb4abbda8f3466fbc42967c84bd5ecc4f796b75e34f47b54b72af075dcb6989a99e1b1da140f77edbb04e036598b52ef4ccec10d99cb31ec779d4d850868537b8ce4494d98ca6a5b679069b106e90c5e1ef210b68295a3995975c2361e0b3bc86948068e2c8e51a879f9bbcba402835698407cbaafb9f020706a4fa29c48a1b285d4b64cb837366d873e0437fd8ec7a401ae35920c29967f38c1ace4239be38ed28d308311ff6ff3b96e9b5150b904574da93dd7c4b210ce1654c695bc2ceeb7d5fd275813decf9a56e972ab7db74aea3bccb739de7b6e9e273b171516a32b0000006d000000074c097f54a9994db8255d509566490589f0277a0d42afaf64c7ff8f096797bc3c2ef0779342efb9820f53577e9fc79b59c393f204a9c2b1745e48392b1eeefd09cd4b4a67dd4c74bb8021815dd277c5a2a1974f699ad28c2e44072874c9a120c4e48a1ecf7705668ce329a18c7f5bba9f14f494830288571b82b7138534c6929f089b14beaa068ac85c1443a97aa6009c8ac175de5461cf2034a094f3ebb1d4c4d3e23116ae003d38d61f14af6299de389397805ca4ecbd65a2aab7c9c5f99be77f818b449cef8aab960ee6848b4968a0c24f7b45e0f1c105622379b1cb8c083300003e0f180000000020373732036aa2b418bda3ad8f22a62152d4ef9d550b49c0ebf261604f34000000ec6c85972aba478981a19233b3514ef541976a4e8a6d0450dcd8d7277a685538e4e0d86ad4e7001ebff50500aa08742e5f391c309351c0f45dd708fa7bac12a5d3b47f6d8e3d84547dab096f39268c2e64c7cca868a503f908bf0c7d08f18e05dc3658522ae25124561bce8ae0e5d7f85456d6ebdc7b9817a0c73ed26055550e0a2ede3482e990e227fcf1fb3201000003000500010000000000000000000000000000000000000000000000000000000000000000ffffffff016affffffff03b84b8c03000000001976a914b489115851ca07a26a5ad8bac3cec3c7dbebd83188ac2ed5fd0300000000016af80da706000000001976a914c69a0bda7daaae481be8def95e5f347a1d00a4b488ac00000000af03003e0f1800df4dea512efab716a7ae84e315e3d45fe3c05a18eaed64058d9a8c1966667328e1f2b2b43f667274b3e8f488f24dc9f7fa611149dc36e2324826b951d5be8950008539584f69b9a56090c811d246a1571fdf8edda332425d51db17f9d421c77b1d7f7d610a9314032a18b73989ff8b82f70a06172ae5b1daae0b858225b6c85fabaee79909e71158e102f7207eee5ca3783b7a4cabe981fb525e072d1751e0ba789802289a2d22000000000000010000000003003d010000030006f64aa6f2cf18889e8ff4704f6bbd50a7ab80f3f1fc6eab984e691f381b00000019ffffff0119ffffff01b813d560b9a73db7b01cfe80ee7ab20a4af7ccda4b94a0cd8ceda8f5cb0fa604760430ef8c644e0f823e089aa7839f9393e0215b2e2b69a88310ffe95ac90099be1d2ece9704bd99a3fd0a3ba8d948e387351b53935ce873abf06103066bcd2f6c1f10f343ce5baf4ab97db6dd483b914ca27ef04636641bcba6c5a5c7aa053903bfacfe8a3317e4fd2be691d6301e814b0c563b9baff75ee551967e41c8bfc8ea07d468ad3123eeb5bd52fb397ae8c788da739f2edf77707516ad03022a0abb3fb74623fe36a82f3036c7cfe6a8d40f426fceded4783cd87b43bc45160e9dfe0d6f09ba2f353ee0797334ea74ce21e967c0efc0ac3984ec0a314650ed1dc4f746f146742eb342b95e7466426d5b532f0a0000006d000000071b4109128f361787885d43e827477232ca77d1253201f14b6d9bfcbb36b5ca29296f2a6e10f6b5d257bdf26f29fd098fbefcf317d43ac45bf9b9a912d6990f973e1af01f8e5eae6165af402ae54256702b3ef04e7bd38397bb341c593a62bcad03ca107596c89ccecb5e5cefdb493a11d62c7f7c171d9b0d9a4f223b5237c9112862024cffad39318b66bbcef991aaa9c5b6862e41310b0410c050a61bda1be2b1fea91d5e2ee314113d4bfcf58b80aa6d0f7a5fa959582db28ee87945740044ef7f65c97d261116cdd5dfa7a5203904d3c6a4a6a625a412384925657abdc78b01af0000000a5b7f1478902ecab28b3404f5e79e9020acb9abd31a6e13928c2450d42684b6adf56996e04e36430af272b8c0b1ed1dd57b6146775532c53634ecb5b600000000000000000000000000ffff44437a064e1fb8a150e6472bc281b51537dda3d077264786f207d05c4ec8d314b5883e32b14b15077bf39ccf5d92543a6fd0ee23897802e543987843fbd195e84ff3a28637ab2231d98b010100a30559b537a5cf3ed95e48d8c006d53de9e049dbf9a1120000008a01000009de4fceb4b2b1ab89bc498f4caca25247813558f17e26eb6c6054ef28688906fe592122fffeb92cbd77956c86811e91b0ed60d5a072dbb3d402c90b4361af3f81d70256b9757f6707488b92e518d616a9cf223aa571cd66e73e7c1236f85dc66ff3a1f817422025f9894edeabb8639ebb57d968a3f5fd3bc5ceb7c661cc34273033306f00e1e3878289f50ae34e3d72691aff06fe77a52080f7a479fcdb79263e8c1ed9f00b2c64da2453cd9bc9de4c250c6fd40a0fc9b18b9fe62408dc14f29ce668b31b1e605ea286913109d87371338c9d545551119f821a939657e6f2491632c3ff1a6584536b4196c735f8331a47844337e0cb68c59388a1ea289e724443d00c1109ec72111ce417de8c49fea97279ee87434bc1f85707010cdbda71b22301af00000012c8d756322b07e0cf2f73ad46f5f25bf91c8b7749419d02a6ccca8da60dc488f5364ad36ce7990bab58abc9f2d480903668268322b7e5099d089ccb3d00000000000000000000000000ffff55d1f3064e1f89d3ff18d870c0b6c9605d986c2de92f44c48032408e183a1d268bad77570e30a6d057a026119d9d044c1ce29b518e91d69372e5829f2cc1c3a9882e466ae9761dfc40e5010100a3053fab9f8566e701ad9bdc75f6f0fe7aaba6008f061d0000008a01000009b1f4342c3ae24aaaae59a75c67fc9ad1e62d520ae3f9a11459aa73f7e8fb753170ff09a84bf2b4e29d54d9c346bee0a9498548bdfb78fef626caa32545a6a4d4bbca76f29274761340c446e9e7d3c8940d296a7abb8f4d4d315a19fe11dd5bdd23a95704aa8cbd08f66e671b27e3782a5f5d8c5cc4f625ae1654b06e3888daaa33306f00e1e3878289f50ae34e3d72691aff06fe77a52080f7a479fcdb79263e8c1ed9f00b2c64da2453cd9bc9de4c250c6fd40a0fc9b18b9fe62408dc14f29ce668b31b1e605ea286913109d87371338c9d545551119f821a939657e6f2491632c3ff1a6584536b4196c735f8331a47844337e0cb68c59388a1ea289e724443d00c1109ec72111ce417de8c49fea97279ee87434bc1f85707010cdbda71b223", + "target": { + "network": 1, + "height": 1576766, + "block_hash": "0000004d3052033c5aafbbecf1c247137c7b0879845c0f13a97bd704339885fb", + "masternode_root": "28736666198c9a8d0564edea185ac0e35fd4e315e384aea716b7fa2e51ea4ddf", + "quorum_root": "5089bed551b9264832e236dc491161faf7c94df288f4e8b37472663fb4b2f2e1" + } +} From 549ab99002961c04227cc43e56bc03770540cf57 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 21 Oct 2026 12:29:35 -0500 Subject: [PATCH 06/25] fix(sdk): free proof places held by abandoned fetches A native fetch whose callers all dropped is never polled again, but it kept counting toward the two places for proving quorums until a later start pruned its slot. A fetch resuming after a backoff could then wait out its whole deadline and have its quorum's cooldown doubled. Fetches that no longer run are now left out of the count. A fetch that runs out of time while other quorums hold the places no longer counts as a failure of its quorum, and failure reasons drop invisible formatting characters such as bidirectional overrides. Tests cover a fetch waiting for a place after a 503 while two others hold them (never more than two requests open at the relay), an abandoned fetch freeing its place, and a fetch that sent nothing keeping its quorum's earlier failures. Co-Authored-By: Claude Opus 5.5 --- .../src/proof_fetches.rs | 107 ++++++++++-- .../src/verified_provider.rs | 152 ++++++++++++++++-- 2 files changed, 233 insertions(+), 26 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index 61878e2675a..58e18f4ef13 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -412,12 +412,8 @@ impl ProofFetches { let forgotten_after = self.timing.max_quorum_cooldown.saturating_mul(2); let mut proven = self.lock(); proven.fetches.retain(|_, slot| match slot { - // A native fetch every waiter dropped no longer runs, so it is - // replaced rather than joined or counted. A browser fetch runs on - // the local executor until it frees its own slot. - Slot::Running { fetch, .. } => { - cfg!(target_arch = "wasm32") || fetch.strong_count() != Some(1) - } + // A fetch that no longer runs is replaced, not joined. + Slot::Running { fetch, .. } => !abandoned(fetch), Slot::Failed { at, .. } => now.saturating_duration_since(*at) < forgotten_after, }); // A fetch that finished since the caller's lookup has nothing to add. @@ -582,7 +578,8 @@ impl ProofFetches { /// Ask the sources for a proof of a quorum until one verifies, the /// request budget is spent, every source is out or the deadline passes. - /// Also returns whether any request was sent. + /// Also returns whether the failure counts against the quorum: whether + /// any request was sent, and the fetch did not end waiting for a place. async fn prove(&self, attempt: &Attempt) -> (Result<(), String>, bool) { let (quorum, height) = (attempt.quorum, attempt.height); let started = Instant::now(); @@ -593,6 +590,7 @@ impl ProofFetches { let mut hedge_at = started; let mut failures = Vec::new(); let mut waited_for_place = false; + let mut waiting_for_place = false; loop { let now = Instant::now(); if now >= deadline { @@ -606,6 +604,7 @@ impl ProofFetches { } else { match self.pick_source(now, &sources) { Pick::Ready(_) if !self.take_place(attempt) => { + waiting_for_place = true; if !waited_for_place { waited_for_place = true; failures.push("waited for other quorums' proofs".to_string()); @@ -626,6 +625,7 @@ impl ProofFetches { let body = request_body(&ask, quorum); let remaining = deadline.saturating_duration_since(now); let timeout = self.timing.request_timeout.min(remaining); + waiting_for_place = false; sources[source].asked += 1; sources[source].in_flight = true; sent += 1; @@ -735,7 +735,9 @@ impl ProofFetches { failures.join("; ") ); tracing::warn!(%reason, "quorum proof fetch failed"); - (Err(reason), sent > 0) + // Running out of time while other quorums held the places says + // nothing about this one. + (Err(reason), sent > 0 && !waiting_for_place) } /// Judge a relay's answer, publishing the key if it proves one. @@ -937,12 +939,22 @@ fn places_taken(proven: &Proven) -> usize { proven .fetches .values() - .filter( - |slot| matches!(slot, Slot::Running { holding, .. } if holding.load(Ordering::Relaxed)), - ) + .filter(|slot| match slot { + Slot::Running { holding, fetch, .. } => { + holding.load(Ordering::Relaxed) && !abandoned(fetch) + } + Slot::Failed { .. } => false, + }) .count() } +/// Whether every waiter of a fetch has gone, so that it no longer runs. A +/// native fetch runs only while polled; a browser fetch runs on the local +/// executor until it frees its own slot. +fn abandoned(fetch: &SharedFetch) -> bool { + !cfg!(target_arch = "wasm32") && fetch.strong_count() == Some(1) +} + /// Whether a commitment's leading bytes (version, type, hash) name `quorum`. fn names(leaf: &[u8], (quorum_type, quorum_hash): QuorumId) -> bool { let mut wire = quorum_hash; @@ -962,6 +974,19 @@ fn request_body(ask: &Ask, (quorum_type, quorum_hash): QuorumId) -> String { .to_string() } +/// Invisible formatting characters, such as bidirectional overrides, that +/// could make a logged reason read differently from what it says. +fn is_invisible_format(c: char) -> bool { + matches!( + c, + '\u{200B}'..='\u{200F}' + | '\u{202A}'..='\u{202E}' + | '\u{2060}'..='\u{2064}' + | '\u{2066}'..='\u{2069}' + | '\u{FEFF}' + ) +} + /// The start of a plain-text error body, for the failure reason. Markup, /// such as a CDN's error page, is left out. fn body_excerpt(body: &[u8]) -> String { @@ -970,7 +995,7 @@ fn body_excerpt(body: &[u8]) -> String { } let text: String = String::from_utf8_lossy(body) .chars() - .filter(|c| !c.is_control()) + .filter(|c| !c.is_control() && !is_invisible_format(*c)) .take(MAX_REASON_BODY_CHARS) .collect(); if text.is_empty() { @@ -1205,6 +1230,64 @@ mod tests { assert!(fetches.take_place(&attempt)); } + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn should_not_count_a_fetch_every_waiter_dropped_against_the_places() { + let fetches = Arc::new(fetches()); + let first = fetches.join_or_start(6, [1; 32], 150).expect("first"); + let _second = fetches.join_or_start(6, [2; 32], 150).expect("second"); + drop(first); + let second = match fetches.lock().fetches.get(&(6, [2; 32])) { + Some(Slot::Running { holding, id, .. }) => Attempt { + quorum: (6, [2; 32]), + height: 150, + id: *id, + failures: 0, + failed_at: None, + holding: Arc::clone(holding), + }, + _ => panic!("the second fetch is running"), + }; + // The second waited out a backoff and wants to send again. + second.holding.store(false, Ordering::Relaxed); + assert!(fetches.take_place(&second)); + } + + #[test] + fn should_keep_earlier_failures_when_a_fetch_sent_nothing() { + let fetches = Arc::new(fetches()); + let quorum = (6, [1; 32]); + let at = Instant::now(); + let _fetch = fetches.join_or_start(6, [1; 32], 150).expect("starts"); + let id = running_id(&fetches, quorum).expect("running"); + let attempt = Attempt { + quorum, + height: 150, + id, + failures: 3, + failed_at: Some(at), + holding: Arc::new(AtomicBool::new(true)), + }; + fetches.finish(&attempt, &Err("nothing sent".to_string()), false); + match fetches.lock().fetches.get(&quorum) { + Some(Slot::Failed { + at: kept, + failures: 3, + .. + }) => assert_eq!(*kept, at), + _ => panic!("the earlier failures are kept"), + }; + } + + #[test] + fn should_leave_markup_and_invisible_characters_out_of_reasons() { + assert_eq!(body_excerpt(b"403"), ""); + assert_eq!( + body_excerpt("busy\u{202E}yrt\u{200B}\n".as_bytes()), + ": busyyrt" + ); + } + #[test] fn should_keep_the_first_key_proven_for_a_quorum() { let fetches = fetches(); diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index 7fba4502a04..b6e9b5a368e 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -78,9 +78,9 @@ pub enum VerifiedContextProviderError { /// as invalid and never fetched. /// /// Fetches are shared per quorum, requests for at most two quorums are in -/// flight at once, relays that cannot serve a quorum are left out of its fetch, failed -/// quorums and failing relays cool down, and each fetch has a 70 s deadline. -/// Freshness of responses is the SDK's concern, not the provider's. +/// flight at once, relays that cannot serve a quorum are left out of its fetch, +/// failed quorums and failing relays cool down, and each fetch has a 70 s +/// deadline. Freshness of responses is the SDK's concern, not the provider's. /// /// Keys are proven for the Core height Platform reports. If Platform halts, /// that height stops while Core moves on, and a quorum not cached before the @@ -436,8 +436,10 @@ mod tests { use dash_core_proof::{parse_commitment, rpc_hex, MAX_PROOF_BYTES}; use futures::future::join_all; use serde_json::{json, Value}; + use std::collections::VecDeque; use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; use std::net::{TcpListener, TcpStream}; + use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; use std::thread; use std::time::{Duration, Instant}; @@ -488,9 +490,9 @@ mod tests { Chunked { size: usize, chunks: usize }, /// Read the request, then hold the connection open without answering. Hang(Duration), - /// Answer 200 with this body after a delay, without holding up - /// later connections. - Late(Duration, Vec), + /// Answer with this status and body after a delay, without holding + /// up later connections. + Late(Duration, u16, Vec), } fn reply(status: u16, body: Vec) -> Answer { @@ -506,11 +508,44 @@ mod tests { /// connection comes for a while. struct MockRelay { url: String, - requests: Arc>>, + requests: Arc>>, + most_open: Arc, + } + + /// Counts a connection as open until dropped. + struct Open(Arc); + + impl Drop for Open { + fn drop(&mut self) { + self.0.fetch_sub(1, AtomicOrdering::SeqCst); + } } impl MockRelay { + /// Answer connections in the order they arrive. fn spawn(answers: Vec) -> Self { + let count = answers.len(); + let mut answers = answers.into_iter(); + Self::serve(count, move |_| answers.next()) + } + + /// Answer each request from the queue of the quorum it names. + fn spawn_by_quorum(queues: Vec<([u8; 32], Vec)>) -> Self { + let count = queues.iter().map(|(_, answers)| answers.len()).sum(); + let mut queues: HashMap> = queues + .into_iter() + .map(|(hash, answers)| (hex::encode(hash), answers.into())) + .collect(); + Self::serve(count, move |request| { + let hash = request["quorumHash"].as_str()?; + queues.get_mut(hash)?.pop_front() + }) + } + + fn serve( + count: usize, + mut answer_for: impl FnMut(&Value) -> Option + Send + 'static, + ) -> Self { let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock relay"); listener .set_nonblocking(true) @@ -518,13 +553,23 @@ mod tests { let url = format!("http://{}", listener.local_addr().expect("relay address")); let requests = Arc::new(Mutex::new(Vec::new())); let recorded = Arc::clone(&requests); + let open = Arc::new(AtomicUsize::new(0)); + let most_open = Arc::new(AtomicUsize::new(0)); + let most = Arc::clone(&most_open); thread::spawn(move || { - for answer in answers { + for _ in 0..count { let Some(mut stream) = accept(&listener) else { return; }; + let now_open = open.fetch_add(1, AtomicOrdering::SeqCst) + 1; + most.fetch_max(now_open, AtomicOrdering::SeqCst); + let guard = Open(Arc::clone(&open)); let body = read_request(&stream); - lock(&recorded).push(body); + let answer = answer_for(&body); + lock(&recorded).push((Instant::now(), body)); + let Some(answer) = answer else { + continue; + }; match answer { Answer::Reply { status, @@ -567,28 +612,52 @@ mod tests { thread::spawn(move || { thread::sleep(duration); drop(stream); + drop(guard); }); } - Answer::Late(delay, body) => { + Answer::Late(delay, status, body) => { thread::spawn(move || { thread::sleep(delay); let head = format!( - "HTTP/1.1 200 OK\r\nConnection: close\r\n\ + "HTTP/1.1 {status} Mock\r\nConnection: close\r\n\ Content-Length: {}\r\n\r\n", body.len() ); let _ = stream.write_all(head.as_bytes()); let _ = stream.write_all(&body); + drop(stream); + drop(guard); }); } } } }); - Self { url, requests } + Self { + url, + requests, + most_open, + } } fn requests(&self) -> Vec { - lock(&self.requests).clone() + lock(&self.requests) + .iter() + .map(|(_, body)| body.clone()) + .collect() + } + + /// When each request for `quorum_hash` arrived. + fn arrivals(&self, quorum_hash: [u8; 32]) -> Vec { + let hash = hex::encode(quorum_hash); + lock(&self.requests) + .iter() + .filter(|(_, body)| body["quorumHash"] == hash.as_str()) + .map(|(at, _)| *at) + .collect() + } + + fn most_open(&self) -> usize { + self.most_open.load(AtomicOrdering::SeqCst) } } @@ -939,6 +1008,7 @@ mod tests { let lagging = MockRelay::spawn(vec![reply(502, Vec::new()), reply(200, behind)]); let proving = MockRelay::spawn(vec![Answer::Late( Duration::from_millis(500), + 200, f.bootstrap.clone(), )]); let timing = Timing { @@ -977,7 +1047,61 @@ mod tests { } #[tokio::test] - async fn should_cool_down_a_hedged_source_that_refuses_near_the_deadline() { + async fn should_wait_for_a_place_before_asking_again_after_a_backoff() { + let f = fixture(Network::Testnet); + let (b, c) = ([0x21; 32], [0x22; 32]); + let slow_gateway = || { + vec![ + Answer::Late(Duration::from_millis(1500), 502, Vec::new()), + reply(502, Vec::new()), + ] + }; + let relay = MockRelay::spawn_by_quorum(vec![ + ( + f.quorum_hash, + vec![reply(503, Vec::new()), reply(200, f.bootstrap.clone())], + ), + (b, slow_gateway()), + (c, slow_gateway()), + ]); + let timing = Timing { + first_retry: Duration::from_millis(300), + deadline: Duration::from_secs(10), + ..Timing::default() + }; + let provider = provider_for(&f, vec![relay.url.clone()], timing); + let height = f.target.height; + let first = fetch(&provider, f.quorum_type, f.quorum_hash, height); + let later = async { + // B and C start while the first quorum waits out the 503. + tokio::time::sleep(Duration::from_millis(100)).await; + futures::join!( + fetch(&provider, f.quorum_type, b, height), + fetch(&provider, f.quorum_type, c, height) + ) + }; + let (key, (b_result, c_result)) = futures::join!(first, later); + assert_eq!(key.expect("proven once a place is free"), Some(f.key)); + assert!(b_result.is_err() && c_result.is_err()); + + let first_asked = relay.arrivals(f.quorum_hash); + let b_asked = relay.arrivals(b); + assert_eq!(first_asked.len(), 2); + // Without the limit the retry would follow the 503 after 300 ms; it + // came only once B and C had their answers, 1.5 s after asking. + assert!( + first_asked[1] >= b_asked[0] + Duration::from_millis(1400), + "asked again too early" + ); + assert!( + relay.most_open() <= 2, + "{} requests open at once", + relay.most_open() + ); + } + + #[tokio::test] + async fn should_cool_down_a_source_that_fails_early_on_a_request_the_deadline_shortened() { let f = fixture(Network::Testnet); let hung = MockRelay::spawn(vec![Answer::Hang(Duration::from_secs(20))]); let closed = TcpListener::bind("127.0.0.1:0").expect("bind"); From 28b0d85d239142d9c1d8b3dd74fc333c5e7475e5 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 21 Oct 2026 12:34:58 -0500 Subject: [PATCH 07/25] refactor(sdk): simplify the verified context provider Consolidate duplicated test helpers: request heights of a mock relay, the refused-connection URL, and the proof fetch Attempt fixture. Co-Authored-By: Claude Opus 5.5 --- .../src/proof_fetches.rs | 36 +++++++------- .../src/verified_provider.rs | 48 ++++++++----------- 2 files changed, 36 insertions(+), 48 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index 58e18f4ef13..f159adfe09c 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -1143,6 +1143,17 @@ mod tests { } } + fn attempt(quorum: QuorumId, id: u64, holding: Arc) -> Attempt { + Attempt { + quorum, + height: 150, + id, + failures: 0, + failed_at: None, + holding, + } + } + #[test] fn should_evict_the_oldest_failure_when_the_slot_table_is_full() { let fetches = Arc::new(fetches()); @@ -1215,14 +1226,7 @@ mod tests { .join_or_start(6, [3; 32], 150) .expect("a place is free"); // Back from the wait, the first must wait for a place again. - let attempt = Attempt { - quorum: (6, [1; 32]), - height: 150, - id: 1, - failures: 0, - failed_at: None, - holding: first, - }; + let attempt = attempt((6, [1; 32]), 1, first); assert!(!fetches.take_place(&attempt)); holding((6, [2; 32])) .expect("running") @@ -1238,14 +1242,9 @@ mod tests { let _second = fetches.join_or_start(6, [2; 32], 150).expect("second"); drop(first); let second = match fetches.lock().fetches.get(&(6, [2; 32])) { - Some(Slot::Running { holding, id, .. }) => Attempt { - quorum: (6, [2; 32]), - height: 150, - id: *id, - failures: 0, - failed_at: None, - holding: Arc::clone(holding), - }, + Some(Slot::Running { holding, id, .. }) => { + attempt((6, [2; 32]), *id, Arc::clone(holding)) + } _ => panic!("the second fetch is running"), }; // The second waited out a backoff and wants to send again. @@ -1261,12 +1260,9 @@ mod tests { let _fetch = fetches.join_or_start(6, [1; 32], 150).expect("starts"); let id = running_id(&fetches, quorum).expect("running"); let attempt = Attempt { - quorum, - height: 150, - id, failures: 3, failed_at: Some(at), - holding: Arc::new(AtomicBool::new(true)), + ..attempt(quorum, id, Arc::new(AtomicBool::new(true))) }; fetches.finish(&attempt, &Err("nothing sent".to_string()), false); match fetches.lock().fetches.get(&quorum) { diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index b6e9b5a368e..eb02ecd40c4 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -656,6 +656,14 @@ mod tests { .collect() } + /// The `height` of each request, in arrival order. + fn heights(&self) -> Vec { + self.requests() + .iter() + .map(|body| body["height"].clone()) + .collect() + } + fn most_open(&self) -> usize { self.most_open.load(AtomicOrdering::SeqCst) } @@ -703,6 +711,12 @@ mod tests { serde_json::from_slice(&body).expect("a JSON request body") } + /// The URL of a loopback port nothing listens on. + fn refused_url() -> String { + let closed = TcpListener::bind("127.0.0.1:0").expect("bind"); + format!("http://{}", closed.local_addr().expect("address")) + } + fn cache_size() -> NonZeroUsize { NonZeroUsize::new(8).expect("non-zero") } @@ -889,10 +903,7 @@ mod tests { #[tokio::test] async fn should_report_a_refused_connection_as_an_unavailable_source() { let f = fixture(Network::Testnet); - let closed = TcpListener::bind("127.0.0.1:0").expect("bind"); - let url = format!("http://{}", closed.local_addr().expect("address")); - drop(closed); - let provider = provider_for(&f, vec![url], quick()); + let provider = provider_for(&f, vec![refused_url()], quick()); let started = Instant::now(); unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await); assert!(started.elapsed() < Duration::from_secs(10)); @@ -909,11 +920,7 @@ mod tests { let height = f.target.height; let key = fetch(&provider, f.quorum_type, f.quorum_hash, height).await; assert_eq!(key.expect("a proven key"), Some(f.key)); - let heights: Vec<_> = relay - .requests() - .iter() - .map(|r| r["height"].clone()) - .collect(); + let heights = relay.heights(); assert_eq!(heights, [json!(height), json!(0)]); } @@ -935,11 +942,7 @@ mod tests { let honest = f.target.height; let key = fetch(&provider, f.quorum_type, f.quorum_hash, honest).await; assert_eq!(key.expect("the honest height is proven"), Some(f.key)); - let heights: Vec<_> = relay - .requests() - .iter() - .map(|r| r["height"].clone()) - .collect(); + let heights = relay.heights(); assert_eq!(heights, [json!(bogus), json!(0), json!(honest)]); } @@ -956,11 +959,7 @@ mod tests { assert!(reason.contains("not asked again"), "{reason}"); assert!(started.elapsed() < Duration::from_secs(20), "{reason}"); for relay in &relays { - let heights: Vec<_> = relay - .requests() - .iter() - .map(|r| r["height"].clone()) - .collect(); + let heights = relay.heights(); assert_eq!(heights, [json!(f.target.height), json!(0)], "{reason}"); } } @@ -1020,11 +1019,7 @@ mod tests { let height = f.target.height + 600; let key = fetch(&provider, f.quorum_type, f.quorum_hash, height).await; assert_eq!(key.expect("the other source proves it"), Some(f.key)); - let heights: Vec<_> = lagging - .requests() - .iter() - .map(|r| r["height"].clone()) - .collect(); + let heights = lagging.heights(); assert_eq!(heights, [json!(height), json!(0)]); assert_eq!(proving.requests().len(), 1); } @@ -1104,15 +1099,12 @@ mod tests { async fn should_cool_down_a_source_that_fails_early_on_a_request_the_deadline_shortened() { let f = fixture(Network::Testnet); let hung = MockRelay::spawn(vec![Answer::Hang(Duration::from_secs(20))]); - let closed = TcpListener::bind("127.0.0.1:0").expect("bind"); - let refused = format!("http://{}", closed.local_addr().expect("address")); - drop(closed); let timing = Timing { deadline: Duration::from_secs(2), hedge_after: Duration::from_millis(100), ..quick() }; - let provider = provider_for(&f, vec![hung.url.clone(), refused], timing); + let provider = provider_for(&f, vec![hung.url.clone(), refused_url()], timing); unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await); // The hung request was cut short by the deadline; the refusal was not. assert_eq!(provider.inner.fetches.cooling(), [false, true]); From 6435232c2579da02538d5fc1b88f6bd9e08f10e6 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 21 Oct 2026 13:12:29 -0500 Subject: [PATCH 08/25] refactor(sdk): tell stale proofs apart by error kind dash-core-proof now exposes Error::kind(), so the verified provider no longer compares the verifier's message text to recognise a stale target. Co-Authored-By: Claude Opus 5.5 --- .../rs-sdk-trusted-context-provider/src/proof_fetches.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index f159adfe09c..cd2683a92de 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -27,7 +27,7 @@ use crate::quorum_refresh::panic_message; use crate::quorum_refresh::sleep; use dash_context_provider::ContextProviderError; use dash_core_proof::bootstrap::{self, RecordKind}; -use dash_core_proof::{parse_commitment, rpc_hex, Error as ProofError, State}; +use dash_core_proof::{parse_commitment, rpc_hex, ErrorKind, State}; use futures::future::{self, select, BoxFuture, Either, FutureExt, Shared}; use futures::stream::FuturesUnordered; use futures::StreamExt; @@ -72,9 +72,6 @@ const MAX_CANNOT_SERVE: u32 = 2; /// How long a fetch waits before trying again to take a place to prove its /// quorum. const PLACE_RETRY: Duration = Duration::from_millis(250); -/// How `bootstrap::verify` reports a target below the minimum height, which -/// it checks before any signature. -const STALE_TARGET: ProofError = ProofError("stale target"); /// How much of an error response's body goes into the failure reason. const MAX_REASON_BODY_CHARS: usize = 120; @@ -774,12 +771,12 @@ impl ProofFetches { // height costs no BLS work. Only this source is affected if it lies. let verified = match bootstrap::verify(body, &ask.checkpoint, needed) { Ok(verified) => verified, - Err(error) if error == STALE_TARGET && ask.height == 0 => { + Err(error) if error.kind() == ErrorKind::StaleTarget && ask.height == 0 => { return Verdict::BeyondTip(format!( "its latest ChainLock is below the core height {needed} that {height} needs" )); } - Err(error) if error == STALE_TARGET => { + Err(error) if error.kind() == ErrorKind::StaleTarget => { return Verdict::CannotServe(format!("proof ends below core height {needed}")); } Err(error) => return Verdict::SourceFailed(format!("invalid proof: {error}")), From 5c605ec5654df0788438984623afc7a8e5decee7 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 21 Oct 2026 16:15:50 -0500 Subject: [PATCH 09/25] fix(sdk): release abandoned proof fetches and keep honest heights moving Review of the verified provider found three ways it could leak or lock out honest lookups: - A running fetch's slot held a strong handle to the shared fetch, which holds the provider's state: if every caller left while a request was pending, the state, its caches and the HTTP client were never freed. Slots now hold a weak handle. Natively a fetch lives only while callers poll it; in a browser a local task drives it to its deadline. - A fetch at a bogus, unverified height turned away honest lookups of the same quorum at heights more than a window away. Fetches are now kept per quorum and height: a far height runs its own fetch, subject to the two proving places, while near heights still share one. - Fetches waiting out a backoff stayed in the 64-entry table, so live waiting fetches could fill it and reject an honest quorum. The limit now applies only to remembered failures, oldest forgotten first; running fetches are bounded by their callers. Also: records that fail to parse are skipped without reading their bytes here, both providers share one error-cause helper, and the browser transport has wasm-bindgen tests (request options, streamed body, unbounded and over-declared bodies, abort at the deadline) that run under Node.js; tokio is a native-only dev-dependency. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + .../Cargo.toml | 6 +- .../src/error.rs | 13 + .../src/proof_fetches.rs | 364 ++++++++++-------- .../src/proof_transport.rs | 147 ++++++- .../src/provider.rs | 16 +- .../src/quorum_refresh.rs | 4 +- .../src/verified_provider.rs | 54 ++- 8 files changed, 404 insertions(+), 201 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d0f8a2486cd..d62ca846d07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6552,6 +6552,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-bindgen-test", "wasm-streams", "web-sys", "web-time", diff --git a/packages/rs-sdk-trusted-context-provider/Cargo.toml b/packages/rs-sdk-trusted-context-provider/Cargo.toml index 4a6ab6715e4..443651c9d15 100644 --- a/packages/rs-sdk-trusted-context-provider/Cargo.toml +++ b/packages/rs-sdk-trusted-context-provider/Cargo.toml @@ -95,5 +95,9 @@ reqwest = { version = "0.12", features = ["json"] } [target.'cfg(target_os = "android")'.dependencies] reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-webpki-roots"] } -[dev-dependencies] +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] tokio = { version = "1.40", features = ["macros", "rt-multi-thread"] } + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies] +# Runs the browser transport's tests under Node.js or a browser. +wasm-bindgen-test = "0.3.58" diff --git a/packages/rs-sdk-trusted-context-provider/src/error.rs b/packages/rs-sdk-trusted-context-provider/src/error.rs index 91fe0f98211..de409e0b7de 100644 --- a/packages/rs-sdk-trusted-context-provider/src/error.rs +++ b/packages/rs-sdk-trusted-context-provider/src/error.rs @@ -1,3 +1,4 @@ +use std::error::Error as StdError; use thiserror::Error; #[derive(Error, Debug)] @@ -29,3 +30,15 @@ pub enum TrustedContextProviderError { #[error("Unsupported network: {0}")] UnsupportedNetwork(String), } + +/// An error with every cause it carries, which `Display` alone leaves out. +pub(crate) fn with_causes(error: &dyn StdError) -> String { + let mut message = error.to_string(); + let mut source = error.source(); + while let Some(cause) = source { + message.push_str(": "); + message.push_str(&cause.to_string()); + source = cause.source(); + } + message +} diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index cd2683a92de..1e7aff4bc53 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -28,13 +28,13 @@ use crate::quorum_refresh::sleep; use dash_context_provider::ContextProviderError; use dash_core_proof::bootstrap::{self, RecordKind}; use dash_core_proof::{parse_commitment, rpc_hex, ErrorKind, State}; -use futures::future::{self, select, BoxFuture, Either, FutureExt, Shared}; +use futures::future::{self, select, BoxFuture, Either, FutureExt, Shared, WeakShared}; use futures::stream::FuturesUnordered; use futures::StreamExt; #[cfg(target_arch = "wasm32")] use gloo_timers::future::sleep; use lru::LruCache; -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; use std::future::Future; use std::num::NonZeroUsize; #[cfg(not(target_arch = "wasm32"))] @@ -60,8 +60,9 @@ const MAX_REQUESTS_IN_FLIGHT: usize = 2; /// Most distinct quorums being proven at once. Each costs a relay a Core /// call, and the quorums come from unverified responses. const MAX_QUORUMS_IN_FLIGHT: usize = 2; -/// Most quorums with a running or recently failed fetch remembered. -const MAX_FETCH_SLOTS: usize = 64; +/// Most recently failed fetches remembered for their cooldowns. Running +/// fetches are not counted: each has a caller waiting on it. +const MAX_FAILED_FETCHES: usize = 64; /// Verified states kept besides the checkpoint, to start later proofs from. const MAX_RECENT_STATES: usize = 16; /// The delay before asking a busy relay again doubles this many times. @@ -125,16 +126,20 @@ struct Cached { active_at: u32, } -type SharedFetch = Shared>>; +type FetchFuture = BoxFuture<'static, Result<(), String>>; +type SharedFetch = Shared; -/// The latest fetch of one quorum. +/// A running or recently failed fetch of a quorum, for one height. enum Slot { Running { id: u64, height: u32, /// Set while the fetch holds one of the places to prove a quorum. holding: Arc, - fetch: SharedFetch, + /// Weak, because the fetch holds the provider's state: only its + /// waiters keep it alive. Natively it runs only while they poll it; + /// in a browser a local task drives it to its deadline. + fetch: WeakShared, }, Failed { at: Instant, @@ -155,7 +160,7 @@ struct Proven { keys: LruCache, /// The checkpoint and the most recent verified targets, by height. states: BTreeMap, - fetches: HashMap, + fetches: Vec<(QuorumId, Slot)>, sources: Vec, next_fetch_id: u64, } @@ -291,7 +296,7 @@ impl ProofFetches { let proven = Proven { keys: LruCache::new(cache_size), states: BTreeMap::from([(checkpoint.height, checkpoint.clone())]), - fetches: HashMap::new(), + fetches: Vec::new(), sources: endpoints.iter().map(|_| SourceHealth::default()).collect(), next_fetch_id: 0, }; @@ -391,12 +396,13 @@ impl ProofFetches { self.proven.lock().unwrap_or_else(PoisonError::into_inner) } - /// The running fetch of this quorum, or a new one if the limits allow. + /// The running fetch of this quorum for a height near this one, or a + /// new one if the limits allow. /// /// A fetch or a failure counts only for heights within the validity /// window of its own, since the height comes from an unverified response: - /// a bogus height must not hold back, or cool down, a request for an - /// honest one. + /// a fetch or failure at a bogus height must not hold back, or cool down, + /// a request for an honest one, which runs its own fetch. fn join_or_start( self: &Arc, quorum_type: u32, @@ -408,9 +414,9 @@ impl ProofFetches { let now = Instant::now(); let forgotten_after = self.timing.max_quorum_cooldown.saturating_mul(2); let mut proven = self.lock(); - proven.fetches.retain(|_, slot| match slot { - // A fetch that no longer runs is replaced, not joined. - Slot::Running { fetch, .. } => !abandoned(fetch), + proven.fetches.retain(|(_, slot)| match slot { + // A fetch every waiter dropped no longer runs. + Slot::Running { fetch, .. } => fetch.upgrade().is_some(), Slot::Failed { at, .. } => now.saturating_duration_since(*at) < forgotten_after, }); // A fetch that finished since the caller's lookup has nothing to add. @@ -419,64 +425,55 @@ impl ProofFetches { return Ok(future::ready(Ok(())).boxed().shared()); } } - let (failures, failed_at) = match proven.fetches.get(&quorum) { - Some(Slot::Running { + let joinable = proven.fetches.iter().find_map(|(other, slot)| match slot { + Slot::Running { fetch, height: running, .. - }) if near(*running) => return Ok(fetch.clone()), - Some(Slot::Running { - height: running, .. - }) => { + } if *other == quorum && near(*running) => fetch.upgrade(), + _ => None, + }); + if let Some(fetch) = joinable { + return Ok(fetch); + } + let earlier = proven + .fetches + .iter() + .filter_map(|(other, slot)| match slot { + Slot::Failed { + at, + failures, + height: failed, + } if *other == quorum && near(*failed) => Some((*at, *failures)), + _ => None, + }) + .max_by_key(|(_, failures)| *failures); + if let Some((at, failures)) = earlier { + let cooldown = backoff( + self.timing.quorum_cooldown, + self.timing.max_quorum_cooldown, + failures, + ); + let wait = cooldown.saturating_sub(now.saturating_duration_since(at)); + if !wait.is_zero() { return Err(format!( - "quorum {} is being proven for core height {running}; retry", - hex::encode(quorum_hash) + "quorum {} was asked for too recently, retry in {} s", + hex::encode(quorum_hash), + wait.as_millis().div_ceil(1000) )); } - Some(Slot::Failed { - at, - failures, - height: failed, - }) if near(*failed) => { - let cooldown = backoff( - self.timing.quorum_cooldown, - self.timing.max_quorum_cooldown, - *failures, - ); - let wait = cooldown.saturating_sub(now.saturating_duration_since(*at)); - if !wait.is_zero() { - return Err(format!( - "quorum {} was asked for too recently, retry in {} s", - hex::encode(quorum_hash), - wait.as_millis().div_ceil(1000) - )); - } - (*failures, Some(*at)) - } - Some(Slot::Failed { .. }) | None => (0, None), - }; + } let running = places_taken(&proven); if running >= MAX_QUORUMS_IN_FLIGHT { return Err(format!( "{running} other quorums are being proven; not starting another" )); } - if !proven.fetches.contains_key(&quorum) && proven.fetches.len() >= MAX_FETCH_SLOTS { - // Forget the oldest failure rather than lock new quorums out. - let oldest = proven - .fetches - .iter() - .filter_map(|(quorum, slot)| match slot { - Slot::Failed { at, .. } => Some((*at, *quorum)), - Slot::Running { .. } => None, - }) - .min_by_key(|(at, _)| *at) - .map(|(_, quorum)| quorum); - match oldest { - Some(oldest) => proven.fetches.remove(&oldest), - None => return Err("too many quorums are being proven".to_string()), - }; - } + // The new fetch carries the earlier failures near its height on. + proven.fetches.retain(|(other, slot)| { + !(*other == quorum + && matches!(slot, Slot::Failed { height: failed, .. } if near(*failed))) + }); proven.next_fetch_id += 1; let id = proven.next_fetch_id; let holding = Arc::new(AtomicBool::new(true)); @@ -484,21 +481,28 @@ impl ProofFetches { quorum, height, id, - failures, - failed_at, + failures: earlier.map_or(0, |(_, failures)| failures), + failed_at: earlier.map(|(at, _)| at), holding: Arc::clone(&holding), }; // Built under the lock, so it must do nothing until it is polled. let fetch = Arc::clone(self).start(attempt); - proven.fetches.insert( + let Some(weak) = fetch.downgrade() else { + return Ok(fetch); + }; + proven.fetches.push(( quorum, Slot::Running { id, height, holding, - fetch: fetch.clone(), + fetch: weak, }, - ); + )); + // A browser fetch runs to its deadline even if every caller leaves, + // so its slot stays joinable meanwhile. + #[cfg(target_arch = "wasm32")] + wasm_bindgen_futures::spawn_local(fetch.clone().map(drop)); Ok(fetch) } @@ -530,30 +534,41 @@ impl ProofFetches { fn finish(&self, attempt: &Attempt, result: &Result<(), String>, asked: bool) { attempt.holding.store(false, Ordering::Relaxed); let mut proven = self.lock(); - let ours = matches!( - proven.fetches.get(&attempt.quorum), - Some(Slot::Running { id, .. }) if *id == attempt.id - ); - if !ours { + let Some(index) = proven + .fetches + .iter() + .position(|(_, slot)| matches!(slot, Slot::Running { id, .. } if *id == attempt.id)) + else { return; - } + }; let failed = match (result, asked, attempt.failed_at) { (Ok(()), _, _) => None, (Err(_), true, _) => Some((Instant::now(), attempt.failures.saturating_add(1))), (Err(_), false, Some(at)) => Some((at, attempt.failures)), (Err(_), false, None) => None, }; - match failed { - Some((at, failures)) => { - let failed = Slot::Failed { - at, - failures, - height: attempt.height, - }; - proven.fetches.insert(attempt.quorum, failed); - } - None => { - proven.fetches.remove(&attempt.quorum); + let Some((at, failures)) = failed else { + proven.fetches.swap_remove(index); + return; + }; + proven.fetches[index].1 = Slot::Failed { + at, + failures, + height: attempt.height, + }; + // Forget the oldest failure rather than remember without bound. + let failed: Vec<_> = proven + .fetches + .iter() + .enumerate() + .filter_map(|(index, (_, slot))| match slot { + Slot::Failed { at, .. } => Some((*at, index)), + Slot::Running { .. } => None, + }) + .collect(); + if failed.len() > MAX_FAILED_FETCHES { + if let Some((_, oldest)) = failed.into_iter().min() { + proven.fetches.swap_remove(oldest); } } } @@ -787,22 +802,14 @@ impl ProofFetches { if record.kind != RecordKind::Quorum { continue; } - match parse_commitment(record.leaf) { - Ok(commitment) => { - let mut hash = commitment.quorum_hash; - hash.reverse(); - if (u32::from(commitment.quorum_type), hash) == quorum { - keys.push(commitment.public_key); - } - } - // Records of other quorums do not matter; one that names the - // requested quorum must parse. - Err(error) if names(record.leaf, quorum) => { - return Verdict::CannotServe(format!( - "unparsable record of the quorum: {error}" - )); + // A record that does not parse opens no key; if it was the + // requested quorum's, the count below is short. + if let Ok(commitment) = parse_commitment(record.leaf) { + let mut hash = commitment.quorum_hash; + hash.reverse(); + if (u32::from(commitment.quorum_type), hash) == quorum { + keys.push(commitment.public_key); } - Err(_) => {} } } let [key] = keys.as_slice() else { @@ -907,7 +914,7 @@ impl ProofFetches { } /// Which sources are cooling down. - #[cfg(test)] + #[cfg(all(test, not(target_arch = "wasm32")))] pub(crate) fn cooling(&self) -> Vec { let now = Instant::now(); let proven = self.lock(); @@ -935,30 +942,17 @@ impl ProofFetches { fn places_taken(proven: &Proven) -> usize { proven .fetches - .values() - .filter(|slot| match slot { + .iter() + .filter(|(_, slot)| match slot { + // A fetch every waiter dropped no longer runs. Slot::Running { holding, fetch, .. } => { - holding.load(Ordering::Relaxed) && !abandoned(fetch) + holding.load(Ordering::Relaxed) && fetch.upgrade().is_some() } Slot::Failed { .. } => false, }) .count() } -/// Whether every waiter of a fetch has gone, so that it no longer runs. A -/// native fetch runs only while polled; a browser fetch runs on the local -/// executor until it frees its own slot. -fn abandoned(fetch: &SharedFetch) -> bool { - !cfg!(target_arch = "wasm32") && fetch.strong_count() == Some(1) -} - -/// Whether a commitment's leading bytes (version, type, hash) name `quorum`. -fn names(leaf: &[u8], (quorum_type, quorum_hash): QuorumId) -> bool { - let mut wire = quorum_hash; - wire.reverse(); - leaf.get(2).map(|t| u32::from(*t)) == Some(quorum_type) && leaf.get(3..35) == Some(&wire[..]) -} - /// The JSON a relay's `/proofs` takes. Relays reject unknown fields. fn request_body(ask: &Ask, (quorum_type, quorum_hash): QuorumId) -> String { serde_json::json!({ @@ -1133,11 +1127,29 @@ mod tests { ) } - fn running_id(fetches: &ProofFetches, quorum: QuorumId) -> Option { - match fetches.lock().fetches.get(&quorum) { - Some(Slot::Running { id, .. }) => Some(*id), - _ => None, - } + fn running_id(fetches: &ProofFetches, quorum: QuorumId, height: u32) -> Option { + fetches + .lock() + .fetches + .iter() + .find_map(|(other, slot)| match slot { + Slot::Running { id, height: h, .. } if *other == quorum && *h == height => { + Some(*id) + } + _ => None, + }) + } + + fn holding(fetches: &ProofFetches, quorum: QuorumId) -> Arc { + fetches + .lock() + .fetches + .iter() + .find_map(|(other, slot)| match slot { + Slot::Running { holding, .. } if *other == quorum => Some(Arc::clone(holding)), + _ => None, + }) + .expect("a running fetch") } fn attempt(quorum: QuorumId, id: u64, holding: Arc) -> Attempt { @@ -1151,47 +1163,72 @@ mod tests { } } + fn failed(at: Instant) -> Slot { + Slot::Failed { + at, + failures: 1, + height: 150, + } + } + #[test] - fn should_evict_the_oldest_failure_when_the_slot_table_is_full() { + fn should_forget_the_oldest_failure_past_the_limit() { let fetches = Arc::new(fetches()); let now = Instant::now(); let earlier = now.checked_sub(Duration::from_secs(1)).unwrap_or(now); { let mut proven = fetches.lock(); - for n in 0..MAX_FETCH_SLOTS { - let at = if n == 0 { earlier } else { now }; - let failed = Slot::Failed { - at, - failures: 1, - height: 150, - }; - proven.fetches.insert((6, [n as u8; 32]), failed); + proven.fetches.push(((6, [0; 32]), failed(earlier))); + for n in 1..MAX_FAILED_FETCHES { + proven.fetches.push(((6, [n as u8; 32]), failed(now))); } } - let _fetch = fetches - .join_or_start(6, [0xff; 32], 150) - .expect("a new quorum is not locked out"); + let _fetch = fetches.join_or_start(6, [0xff; 32], 150).expect("starts"); + let id = running_id(&fetches, (6, [0xff; 32]), 150).expect("running"); + let failure = Err("failed".to_string()); + fetches.finish( + &attempt((6, [0xff; 32]), id, holding(&fetches, (6, [0xff; 32]))), + &failure, + true, + ); let proven = fetches.lock(); - assert_eq!(proven.fetches.len(), MAX_FETCH_SLOTS); + assert_eq!(proven.fetches.len(), MAX_FAILED_FETCHES); assert!( - !proven.fetches.contains_key(&(6, [0; 32])), + !proven + .fetches + .iter() + .any(|(quorum, _)| *quorum == (6, [0; 32])), "the oldest goes" ); } #[test] - fn should_join_a_fetch_only_for_a_height_it_covers() { + fn should_admit_a_quorum_while_many_fetches_wait_out_backoffs() { + let fetches = Arc::new(fetches()); + let mut waiting = Vec::new(); + for n in 0..MAX_FAILED_FETCHES + 8 { + let fetch = fetches + .join_or_start(6, [n as u8; 32], 150) + .expect("a place is free"); + // Each gives its place up while it waits out a relay's backoff. + holding(&fetches, (6, [n as u8; 32])).store(false, Ordering::Relaxed); + waiting.push(fetch); + } + assert!(fetches.join_or_start(6, [0xff; 32], 150).is_ok()); + } + + #[test] + fn should_run_a_separate_fetch_for_a_height_far_from_the_running_one() { let fetches = Arc::new(fetches()); let _first = fetches.join_or_start(6, [1; 32], 150).expect("starts"); - let id = running_id(&fetches, (6, [1; 32])); + let id = running_id(&fetches, (6, [1; 32]), 150); let _joined = fetches.join_or_start(6, [1; 32], 750).expect("joins"); - assert_eq!(running_id(&fetches, (6, [1; 32])), id); - let far = fetches.join_or_start(6, [1; 32], 751).map(|_| ()); - assert!( - far.as_ref() - .is_err_and(|reason| reason.contains("being proven for core height 150")), - "{far:?}" - ); + assert_eq!(running_id(&fetches, (6, [1; 32]), 150), id); + assert_eq!(running_id(&fetches, (6, [1; 32]), 750), None); + let _far = fetches + .join_or_start(6, [1; 32], 751) + .expect("its own fetch"); + assert!(running_id(&fetches, (6, [1; 32]), 751).is_some()); } #[cfg(not(target_arch = "wasm32"))] @@ -1199,10 +1236,10 @@ mod tests { fn should_replace_a_fetch_every_waiter_dropped() { let fetches = Arc::new(fetches()); let first = fetches.join_or_start(6, [1; 32], 150).expect("starts"); - let id = running_id(&fetches, (6, [1; 32])); + let id = running_id(&fetches, (6, [1; 32]), 150); drop(first); let _second = fetches.join_or_start(6, [1; 32], 150).expect("restarts"); - let replaced = running_id(&fetches, (6, [1; 32])); + let replaced = running_id(&fetches, (6, [1; 32]), 150); assert!(replaced.is_some() && replaced != id); } @@ -1212,12 +1249,8 @@ mod tests { let _first = fetches.join_or_start(6, [1; 32], 150).expect("first"); let _second = fetches.join_or_start(6, [2; 32], 150).expect("second"); assert!(fetches.join_or_start(6, [3; 32], 150).is_err()); - let holding = |quorum: QuorumId| match fetches.lock().fetches.get(&quorum) { - Some(Slot::Running { holding, .. }) => Some(Arc::clone(holding)), - _ => None, - }; // The first gives its place up while it waits out a backoff. - let first = holding((6, [1; 32])).expect("running"); + let first = holding(&fetches, (6, [1; 32])); first.store(false, Ordering::Relaxed); let _third = fetches .join_or_start(6, [3; 32], 150) @@ -1225,9 +1258,7 @@ mod tests { // Back from the wait, the first must wait for a place again. let attempt = attempt((6, [1; 32]), 1, first); assert!(!fetches.take_place(&attempt)); - holding((6, [2; 32])) - .expect("running") - .store(false, Ordering::Relaxed); + holding(&fetches, (6, [2; 32])).store(false, Ordering::Relaxed); assert!(fetches.take_place(&attempt)); } @@ -1238,12 +1269,8 @@ mod tests { let first = fetches.join_or_start(6, [1; 32], 150).expect("first"); let _second = fetches.join_or_start(6, [2; 32], 150).expect("second"); drop(first); - let second = match fetches.lock().fetches.get(&(6, [2; 32])) { - Some(Slot::Running { holding, id, .. }) => { - attempt((6, [2; 32]), *id, Arc::clone(holding)) - } - _ => panic!("the second fetch is running"), - }; + let id = running_id(&fetches, (6, [2; 32]), 150).expect("running"); + let second = attempt((6, [2; 32]), id, holding(&fetches, (6, [2; 32]))); // The second waited out a backoff and wants to send again. second.holding.store(false, Ordering::Relaxed); assert!(fetches.take_place(&second)); @@ -1255,19 +1282,22 @@ mod tests { let quorum = (6, [1; 32]); let at = Instant::now(); let _fetch = fetches.join_or_start(6, [1; 32], 150).expect("starts"); - let id = running_id(&fetches, quorum).expect("running"); + let id = running_id(&fetches, quorum, 150).expect("running"); let attempt = Attempt { failures: 3, failed_at: Some(at), ..attempt(quorum, id, Arc::new(AtomicBool::new(true))) }; fetches.finish(&attempt, &Err("nothing sent".to_string()), false); - match fetches.lock().fetches.get(&quorum) { - Some(Slot::Failed { - at: kept, - failures: 3, - .. - }) => assert_eq!(*kept, at), + match fetches.lock().fetches.first() { + Some(( + _, + Slot::Failed { + at: kept, + failures: 3, + .. + }, + )) => assert_eq!(*kept, at), _ => panic!("the earlier failures are kept"), }; } diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_transport.rs b/packages/rs-sdk-trusted-context-provider/src/proof_transport.rs index e41747f2b75..e775b319bee 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_transport.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_transport.rs @@ -17,14 +17,14 @@ use std::pin::pin; use std::time::Duration; use url::Url; +#[cfg(not(target_arch = "wasm32"))] +use crate::error::with_causes; #[cfg(not(target_arch = "wasm32"))] use futures::stream::try_unfold; #[cfg(not(target_arch = "wasm32"))] use reqwest::header::CONTENT_TYPE; #[cfg(not(target_arch = "wasm32"))] use reqwest::redirect::Policy; -#[cfg(not(target_arch = "wasm32"))] -use std::error::Error as StdError; #[cfg(target_arch = "wasm32")] use futures::future::{select, Either}; @@ -135,19 +135,6 @@ impl ProofTransport { } } -/// An error with every cause it carries, which `Display` alone leaves out. -#[cfg(not(target_arch = "wasm32"))] -fn with_causes(error: &dyn StdError) -> String { - let mut message = error.to_string(); - let mut source = error.source(); - while let Some(cause) = source { - message.push_str(": "); - message.push_str(&cause.to_string()); - source = cause.source(); - } - message -} - /// Sends proof requests to relays with the browser's `fetch`. #[cfg(target_arch = "wasm32")] pub(crate) struct ProofTransport; @@ -281,7 +268,7 @@ async fn browser_post( Ok(Reply { status, body }) } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use super::*; use futures::stream; @@ -323,3 +310,131 @@ mod tests { ); } } + +#[cfg(all(test, target_arch = "wasm32"))] +mod browser_tests { + use super::*; + use wasm_bindgen_test::wasm_bindgen_test; + + /// Replace the global `fetch` with a function of `request`. + fn mock_fetch(body: &str) { + let fetch = Function::new_with_args("request", body); + Reflect::set(&js_sys::global(), &JsValue::from_str("fetch"), &fetch) + .expect("set the global fetch"); + } + + fn seen(field: &str) -> JsValue { + let seen = Reflect::get(&js_sys::global(), &JsValue::from_str("__seen")).expect("seen"); + Reflect::get(&seen, &JsValue::from_str(field)).expect(field) + } + + fn relay() -> Url { + Url::parse("https://relay.test/proofs").expect("a URL") + } + + #[wasm_bindgen_test] + async fn should_post_json_without_redirects_cache_or_credentials_and_stream_the_body() { + mock_fetch( + r#" + globalThis.__seen = { + method: request.method, + redirect: request.redirect, + cache: request.cache, + credentials: request.credentials, + type: request.headers.get("content-type"), + }; + return request.text().then((text) => { + globalThis.__seen.body = text; + const chunks = [new Uint8Array([1, 2]), new Uint8Array([3])]; + const stream = new ReadableStream({ + pull(controller) { + const next = chunks.shift(); + if (next) controller.enqueue(next); else controller.close(); + }, + }); + return new Response(stream, { status: 200 }); + }); + "#, + ); + let reply = ProofTransport + .post( + &relay(), + r#"{"height":1}"#.to_string(), + Duration::from_secs(5), + ) + .await + .expect("a reply"); + assert_eq!((reply.status, reply.body), (200, vec![1, 2, 3])); + for (field, expected) in [ + ("method", "POST"), + ("redirect", "error"), + ("cache", "no-store"), + ("credentials", "omit"), + ("type", "application/json"), + ("body", r#"{"height":1}"#), + ] { + assert_eq!( + seen(field).as_string().as_deref(), + Some(expected), + "{field}" + ); + } + } + + #[wasm_bindgen_test] + async fn should_refuse_an_endless_body_without_a_declared_length() { + mock_fetch( + r#" + const stream = new ReadableStream({ + pull(controller) { controller.enqueue(new Uint8Array(65536)); }, + }); + return Promise.resolve(new Response(stream, { status: 200 })); + "#, + ); + let reply = ProofTransport + .post(&relay(), "{}".to_string(), Duration::from_secs(5)) + .await; + assert_eq!(reply.err(), Some(PostError::TooLarge)); + } + + #[wasm_bindgen_test] + async fn should_refuse_a_declared_length_past_the_limit() { + mock_fetch( + r#" + return Promise.resolve(new Response("", { + status: 200, + headers: { "content-length": "99999999" }, + })); + "#, + ); + let reply = ProofTransport + .post(&relay(), "{}".to_string(), Duration::from_secs(5)) + .await; + assert_eq!(reply.err(), Some(PostError::TooLarge)); + } + + #[wasm_bindgen_test] + async fn should_abort_the_request_at_the_deadline() { + mock_fetch( + r#" + globalThis.__aborted = false; + return new Promise((_, reject) => { + request.signal.addEventListener("abort", () => { + globalThis.__aborted = true; + reject(new Error("aborted")); + }); + }); + "#, + ); + let reply = ProofTransport + .post(&relay(), "{}".to_string(), Duration::from_millis(50)) + .await; + assert!( + matches!(&reply, Err(PostError::Failed(reason)) if reason.contains("no answer within")), + "{reply:?}" + ); + let aborted = + Reflect::get(&js_sys::global(), &JsValue::from_str("__aborted")).expect("the flag"); + assert_eq!(aborted.as_bool(), Some(true)); + } +} diff --git a/packages/rs-sdk-trusted-context-provider/src/provider.rs b/packages/rs-sdk-trusted-context-provider/src/provider.rs index 46e26cb9c79..ba49e1e44c8 100644 --- a/packages/rs-sdk-trusted-context-provider/src/provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/provider.rs @@ -1,4 +1,4 @@ -use crate::error::TrustedContextProviderError; +use crate::error::{with_causes, TrustedContextProviderError}; use crate::get_quorum_base_url; use crate::quorum_refresh::{ contained, sleep, FetchedQuorums, QuorumRefreshes, QUORUM_LIST_REQUEST_TIMEOUT, @@ -86,18 +86,6 @@ pub struct TrustedHttpContextProvider { quorum_refreshes: Arc, } -/// An error with every cause it carries, which `Display` alone leaves out. -fn with_causes(error: &dyn StdError) -> String { - let mut message = error.to_string(); - let mut source = error.source(); - while let Some(cause) = source { - message.push_str(": "); - message.push_str(&cause.to_string()); - source = cause.source(); - } - message -} - #[derive(Debug, Deserialize)] struct MasternodeEntry { address: String, @@ -1064,7 +1052,7 @@ impl ContextProvider for TrustedHttpContextProvider { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use super::*; use crate::quorum_refresh::MIN_GAP; diff --git a/packages/rs-sdk-trusted-context-provider/src/quorum_refresh.rs b/packages/rs-sdk-trusted-context-provider/src/quorum_refresh.rs index e1b5a3bac7b..e85c7029c2d 100644 --- a/packages/rs-sdk-trusted-context-provider/src/quorum_refresh.rs +++ b/packages/rs-sdk-trusted-context-provider/src/quorum_refresh.rs @@ -215,7 +215,7 @@ impl QuorumRefreshes { } /// Make the latest refresh look `by` older, as if that much time had passed. - #[cfg(test)] + #[cfg(all(test, not(target_arch = "wasm32")))] pub(crate) fn age_latest(&self, by: Duration) { if let Some(latest) = self.lock().latest.as_mut() { latest.started -= by; @@ -338,7 +338,7 @@ where receiver.await.ok() } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use super::*; diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index eb02ecd40c4..50e5e523307 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -429,7 +429,7 @@ impl ContextProvider for VerifiedHttpContextProvider { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use super::*; use dash_core_proof::bootstrap; @@ -946,6 +946,58 @@ mod tests { assert_eq!(heights, [json!(bogus), json!(0), json!(honest)]); } + #[tokio::test] + async fn should_prove_an_honest_height_while_a_bogus_one_is_in_flight() { + let f = fixture(Network::Testnet); + let bogus = i32::MAX as u32; + let bootstrap = f.bootstrap.clone(); + let relay = MockRelay::serve(2, move |request| { + Some(if request["height"] == bogus { + Answer::Hang(Duration::from_secs(20)) + } else { + reply(200, bootstrap.clone()) + }) + }); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let stuck = provider + .fetch_quorum_public_key(f.quorum_type, f.quorum_hash, bogus) + .expect("a Platform quorum is fetched"); + let stuck = tokio::spawn(stuck); + let asked = Instant::now(); + while relay.requests().is_empty() && asked.elapsed() < Duration::from_secs(10) { + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_eq!(relay.heights(), [json!(bogus)]); + + let key = fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await; + assert_eq!(key.expect("the honest height is proven"), Some(f.key)); + assert!(!stuck.is_finished(), "the bogus fetch is still waiting"); + stuck.abort(); + } + + #[tokio::test] + async fn should_release_the_provider_when_every_caller_of_a_pending_fetch_leaves() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn(vec![Answer::Hang(Duration::from_secs(20))]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let inner = Arc::downgrade(&provider.inner); + let fetches = Arc::downgrade(&provider.inner.fetches); + let mut pending = provider + .fetch_quorum_public_key(f.quorum_type, f.quorum_hash, f.target.height) + .expect("a Platform quorum is fetched"); + let asked = Instant::now(); + while relay.requests().is_empty() && asked.elapsed() < Duration::from_secs(10) { + let polled = tokio::time::timeout(Duration::from_millis(20), &mut pending).await; + assert!(polled.is_err(), "the relay hangs"); + } + assert_eq!(relay.requests().len(), 1, "the request is in flight"); + + drop(pending); + drop(provider); + assert!(fetches.upgrade().is_none(), "the fetch state is released"); + assert!(inner.upgrade().is_none(), "the provider is released"); + } + #[tokio::test] async fn should_stop_asking_relays_that_cannot_serve_a_fake_quorum() { let f = fixture(Network::Testnet); From 9f7864db8088009205c1104695a9685d846bde4a Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 21 Oct 2026 16:23:04 -0500 Subject: [PATCH 10/25] fix(sdk): keep a quorum's failures when its next fetch is abandoned A fetch took over the earlier failures near its height when it started. If every caller then left, natively the fetch was dropped without finishing and the failures were gone, so the quorum's cooldown started over. The failures now stay until the fetch ends, whose outcome then replaces them. Co-Authored-By: Claude Opus 5.5 --- .../src/proof_fetches.rs | 50 ++++++++++++++++--- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index 1e7aff4bc53..2ebf0940719 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -61,7 +61,8 @@ const MAX_REQUESTS_IN_FLIGHT: usize = 2; /// call, and the quorums come from unverified responses. const MAX_QUORUMS_IN_FLIGHT: usize = 2; /// Most recently failed fetches remembered for their cooldowns. Running -/// fetches are not counted: each has a caller waiting on it. +/// fetches are not counted: natively each has a caller waiting on it, and in +/// a browser each needed a free place to start and ends at its deadline. const MAX_FAILED_FETCHES: usize = 64; /// Verified states kept besides the checkpoint, to start later proofs from. const MAX_RECENT_STATES: usize = 16; @@ -469,11 +470,8 @@ impl ProofFetches { "{running} other quorums are being proven; not starting another" )); } - // The new fetch carries the earlier failures near its height on. - proven.fetches.retain(|(other, slot)| { - !(*other == quorum - && matches!(slot, Slot::Failed { height: failed, .. } if near(*failed))) - }); + // The earlier failures near this height stay until the new fetch + // ends, so a fetch every caller abandons does not erase them. proven.next_fetch_id += 1; let id = proven.next_fetch_id; let holding = Arc::new(AtomicBool::new(true)); @@ -487,6 +485,8 @@ impl ProofFetches { }; // Built under the lock, so it must do nothing until it is polled. let fetch = Arc::clone(self).start(attempt); + // A fetch not yet polled always downgrades; were it done, it would + // need no slot. let Some(weak) = fetch.downgrade() else { return Ok(fetch); }; @@ -534,6 +534,18 @@ impl ProofFetches { fn finish(&self, attempt: &Attempt, result: &Result<(), String>, asked: bool) { attempt.holding.store(false, Ordering::Relaxed); let mut proven = self.lock(); + let ours = |slot: &Slot| matches!(slot, Slot::Running { id, .. } if *id == attempt.id); + if !proven.fetches.iter().any(|(_, slot)| ours(slot)) { + return; + } + // This fetch's outcome replaces the failures it carried on. + let window = self.window; + proven.fetches.retain(|(quorum, slot)| match slot { + Slot::Failed { height, .. } => { + *quorum != attempt.quorum || height.abs_diff(attempt.height) > window + } + Slot::Running { .. } => true, + }); let Some(index) = proven .fetches .iter() @@ -1276,6 +1288,32 @@ mod tests { assert!(fetches.take_place(&second)); } + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn should_keep_earlier_failures_when_every_caller_abandons_the_next_fetch() { + let fetches = Arc::new(fetches()); + let quorum = (6, [1; 32]); + let now = Instant::now(); + let Some(long_ago) = now.checked_sub(Duration::from_secs(11)) else { + return; + }; + let failed = Slot::Failed { + at: long_ago, + failures: 2, + height: 150, + }; + fetches.lock().fetches.push((quorum, failed)); + let abandoned = fetches.join_or_start(6, [1; 32], 150).expect("cooled down"); + drop(abandoned); + let _next = fetches + .join_or_start(6, [1; 32], 150) + .expect("starts again"); + let kept = fetches.lock().fetches.iter().any(|(other, slot)| { + *other == quorum && matches!(slot, Slot::Failed { failures: 2, .. }) + }); + assert!(kept, "the earlier failures survive the abandoned fetch"); + } + #[test] fn should_keep_earlier_failures_when_a_fetch_sent_nothing() { let fetches = Arc::new(fetches()); From 3e66f31d95d42fe4bfabe7db32f5799c656f4f6f Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 26 Oct 2026 11:33:37 -0500 Subject: [PATCH 11/25] fix(sdk): cover joined heights and bound live proof fetches A fetch for height H may prove any target at or above H minus the validity window, which covers heights only up to H. A caller at a higher height that joined it could find its own height uncovered and fail without asking again. A caller now joins a fetch sure to cover its height, or waits for one at a lower height near its own and, if what it proved does not cover, starts a fetch of its own, once. Waiting fetches give their proving place up, so fetches of distinct fake quorums could pile up with their timers and, in browsers, local tasks. At most 16 fetches run at once; past that new quorums are refused at once, while running fetches can still be joined. A fetch now hands its waiters the key it proved and the height that covers, instead of each re-reading a cache a small cache size may already have evicted it from. Co-Authored-By: Claude Opus 5.5 --- .../src/proof_fetches.rs | 221 +++++++++++++----- .../src/verified_provider.rs | 36 +++ 2 files changed, 196 insertions(+), 61 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index 2ebf0940719..90867072186 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -61,9 +61,12 @@ const MAX_REQUESTS_IN_FLIGHT: usize = 2; /// call, and the quorums come from unverified responses. const MAX_QUORUMS_IN_FLIGHT: usize = 2; /// Most recently failed fetches remembered for their cooldowns. Running -/// fetches are not counted: natively each has a caller waiting on it, and in -/// a browser each needed a free place to start and ends at its deadline. +/// fetches have their own bound. const MAX_FAILED_FETCHES: usize = 64; +/// Most fetches running at once, waiting ones included. A fetch gives its +/// proving place up while it waits, so without this bound fetches of fake +/// quorums could pile up; past it, new quorums are refused until one ends. +const MAX_RUNNING_FETCHES: usize = 16; /// Verified states kept besides the checkpoint, to start later proofs from. const MAX_RECENT_STATES: usize = 16; /// The delay before asking a busy relay again doubles this many times. @@ -121,13 +124,19 @@ fn backoff(first: Duration, max: Duration, failures: u32) -> Duration { first.saturating_mul(1 << doublings).min(max) } -/// A proven key and the highest target height that proved it. -struct Cached { +/// A proven key and the highest target height that proved it. The key is +/// served for heights up to `active_at` plus the validity window. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Cached { key: QuorumKey, active_at: u32, } -type FetchFuture = BoxFuture<'static, Result<(), String>>; +/// What a fetch hands every caller waiting on it: the key it proved and the +/// height that covers, so that a caller does not depend on the cache still +/// holding it. +type FetchResult = Result; +type FetchFuture = BoxFuture<'static, FetchResult>; type SharedFetch = Shared; /// A running or recently failed fetch of a quorum, for one height. @@ -208,7 +217,7 @@ fn choose_ask( /// How one answer from a relay counts. enum Verdict { - Proven, + Proven(Cached), /// The relay cannot serve this quorum for the height asked: a 502, or a /// valid proof that does not serve the request. Asked again once, for the /// other height, after a delay; then left out of the fetch. @@ -372,6 +381,10 @@ impl ProofFetches { /// it, and return it. Every failure is /// [`ContextProviderError::QuorumSourceUnavailable`]; this never reports /// the quorum as absent. + /// + /// A caller may first join a fetch for a lower height near its own, which + /// may prove the quorum only up to that height; it then starts a fetch of + /// its own, once. pub(crate) async fn fetch( self: Arc, quorum_type: u32, @@ -381,15 +394,25 @@ impl ProofFetches { if let Ok(key) = self.lookup(quorum_type, quorum_hash, height) { return Ok(key); } - let fetch = self - .join_or_start(quorum_type, quorum_hash, height) - .map_err(ContextProviderError::QuorumSourceUnavailable)?; - fetch - .await - .map_err(ContextProviderError::QuorumSourceUnavailable)?; - // A fetch this one joined may have proven the quorum only for a lower - // height; the lookup then reports the source as unavailable. - self.lookup(quorum_type, quorum_hash, height) + let unavailable = ContextProviderError::QuorumSourceUnavailable; + let mut may_join_lower = true; + loop { + let (fetch, covers) = self + .join_or_start_for(quorum_type, quorum_hash, height, may_join_lower) + .map_err(unavailable)?; + let proven = fetch.await.map_err(unavailable)?; + if height <= proven.active_at.saturating_add(self.window) { + return Ok(proven.key); + } + if covers || !may_join_lower { + return Err(unavailable(format!( + "quorum {} was proven active at core height {}, not for {height}", + hex::encode(quorum_hash), + proven.active_at + ))); + } + may_join_lower = false; + } } fn lock(&self) -> MutexGuard<'_, Proven> { @@ -397,20 +420,40 @@ impl ProofFetches { self.proven.lock().unwrap_or_else(PoisonError::into_inner) } - /// The running fetch of this quorum for a height near this one, or a - /// new one if the limits allow. + /// [`join_or_start_for`](Self::join_or_start_for), joining a fetch for a + /// lower height if there is one. + #[cfg(test)] + fn join_or_start( + self: &Arc, + quorum_type: u32, + quorum_hash: [u8; 32], + height: u32, + ) -> Result<(SharedFetch, bool), String> { + self.join_or_start_for(quorum_type, quorum_hash, height, true) + } + + /// A running fetch of this quorum to wait on, or a new one if the limits + /// allow, and whether its result is sure to cover `height`. + /// + /// A fetch for height `H` proves a target at or above `H` minus the + /// window, so its key is sure to be served for heights up to `H`. A + /// caller joins such a fetch for a height at most a window below its + /// own; with `may_join_lower`, it otherwise joins one for a height at + /// most a window above, and must check what it proved. /// /// A fetch or a failure counts only for heights within the validity /// window of its own, since the height comes from an unverified response: /// a fetch or failure at a bogus height must not hold back, or cool down, /// a request for an honest one, which runs its own fetch. - fn join_or_start( + fn join_or_start_for( self: &Arc, quorum_type: u32, quorum_hash: [u8; 32], height: u32, - ) -> Result { + may_join_lower: bool, + ) -> Result<(SharedFetch, bool), String> { let quorum = (quorum_type, quorum_hash); + let window = self.window; let near = |other: u32| other.abs_diff(height) <= self.window; let now = Instant::now(); let forgotten_after = self.timing.max_quorum_cooldown.saturating_mul(2); @@ -423,19 +466,30 @@ impl ProofFetches { // A fetch that finished since the caller's lookup has nothing to add. if let Some(cached) = proven.keys.peek(&quorum) { if height <= cached.active_at.saturating_add(self.window) { - return Ok(future::ready(Ok(())).boxed().shared()); + return Ok((future::ready(Ok(*cached)).boxed().shared(), true)); } } - let joinable = proven.fetches.iter().find_map(|(other, slot)| match slot { - Slot::Running { - fetch, - height: running, - .. - } if *other == quorum && near(*running) => fetch.upgrade(), - _ => None, - }); - if let Some(fetch) = joinable { - return Ok(fetch); + let joinable = proven + .fetches + .iter() + .filter_map(|(other, slot)| match slot { + Slot::Running { + fetch, + height: running, + .. + } if *other == quorum => { + let covers = height <= *running && *running - height <= window; + let lower = may_join_lower && *running < height && height - *running <= window; + (covers || lower) + .then(|| fetch.upgrade()) + .flatten() + .map(|fetch| (fetch, covers)) + } + _ => None, + }) + .max_by_key(|(_, covers)| *covers); + if let Some(joined) = joinable { + return Ok(joined); } let earlier = proven .fetches @@ -470,6 +524,17 @@ impl ProofFetches { "{running} other quorums are being proven; not starting another" )); } + // Every running slot left after the pruning above is alive. + let live = proven + .fetches + .iter() + .filter(|(_, slot)| matches!(slot, Slot::Running { .. })) + .count(); + if live >= MAX_RUNNING_FETCHES { + return Err(format!( + "{live} quorum fetches are running; not starting another" + )); + } // The earlier failures near this height stay until the new fetch // ends, so a fetch every caller abandons does not erase them. proven.next_fetch_id += 1; @@ -488,7 +553,7 @@ impl ProofFetches { // A fetch not yet polled always downgrades; were it done, it would // need no slot. let Some(weak) = fetch.downgrade() else { - return Ok(fetch); + return Ok((fetch, true)); }; proven.fetches.push(( quorum, @@ -503,7 +568,7 @@ impl ProofFetches { // so its slot stays joinable meanwhile. #[cfg(target_arch = "wasm32")] wasm_bindgen_futures::spawn_local(fetch.clone().map(drop)); - Ok(fetch) + Ok((fetch, true)) } /// A fetch every waiter of this quorum shares. @@ -531,7 +596,7 @@ impl ProofFetches { /// Record how a fetch ended, if its slot is still its own. A failure /// without a single request asked nothing, so it does not cool the /// quorum down further; the earlier failures are kept. - fn finish(&self, attempt: &Attempt, result: &Result<(), String>, asked: bool) { + fn finish(&self, attempt: &Attempt, result: &FetchResult, asked: bool) { attempt.holding.store(false, Ordering::Relaxed); let mut proven = self.lock(); let ours = |slot: &Slot| matches!(slot, Slot::Running { id, .. } if *id == attempt.id); @@ -554,7 +619,7 @@ impl ProofFetches { return; }; let failed = match (result, asked, attempt.failed_at) { - (Ok(()), _, _) => None, + (Ok(_), _, _) => None, (Err(_), true, _) => Some((Instant::now(), attempt.failures.saturating_add(1))), (Err(_), false, Some(at)) => Some((at, attempt.failures)), (Err(_), false, None) => None, @@ -604,7 +669,7 @@ impl ProofFetches { /// request budget is spent, every source is out or the deadline passes. /// Also returns whether the failure counts against the quorum: whether /// any request was sent, and the fetch did not end waiting for a place. - async fn prove(&self, attempt: &Attempt) -> (Result<(), String>, bool) { + async fn prove(&self, attempt: &Attempt) -> (FetchResult, bool) { let (quorum, height) = (attempt.quorum, attempt.height); let started = Instant::now(); let deadline = started + self.timing.deadline; @@ -707,7 +772,7 @@ impl ProofFetches { reply => self.judge(quorum, height, &ask, reply), }; match verdict { - Verdict::Proven => { + Verdict::Proven(proven) => { self.source_answered(source); tracing::debug!( quorum_type = quorum.0, @@ -717,7 +782,7 @@ impl ProofFetches { elapsed_ms = elapsed.as_millis(), "proved quorum key" ); - return (Ok(()), true); + return (Ok(proven), true); } Verdict::CannotServe(reason) => { state.cannot_serve += 1; @@ -831,19 +896,20 @@ impl ProofFetches { )); }; match self.publish(quorum, *key, verified.state()) { - Ok(()) => Verdict::Proven, + Ok(proven) => Verdict::Proven(proven), Err(reason) => Verdict::CannotServe(reason), } } - /// Cache a proven key and remember the state that proved it. Nothing is - /// cached if either conflicts with what is already known. + /// Cache a proven key and remember the state that proved it, and return + /// what the cache now holds for the quorum. Nothing is cached if either + /// conflicts with what is already known. pub(crate) fn publish( &self, quorum: QuorumId, key: QuorumKey, target: &State, - ) -> Result<(), String> { + ) -> Result { let mut proven = self.lock(); if let Some(known) = proven.states.get(&target.height) { if known != target { @@ -856,7 +922,7 @@ impl ProofFetches { return Err("proves a different block than one already verified".to_string()); } } - match proven.keys.get_mut(&quorum) { + let cached = match proven.keys.get_mut(&quorum) { Some(cached) if cached.key != key => { tracing::error!( quorum_type = quorum.0, @@ -867,17 +933,19 @@ impl ProofFetches { ); return Err("opens a different key than one already proven".to_string()); } - Some(cached) => cached.active_at = cached.active_at.max(target.height), + Some(cached) => { + cached.active_at = cached.active_at.max(target.height); + *cached + } None => { - proven.keys.put( - quorum, - Cached { - key, - active_at: target.height, - }, - ); + let cached = Cached { + key, + active_at: target.height, + }; + proven.keys.put(quorum, cached); + cached } - } + }; if proven .states .insert(target.height, target.clone()) @@ -895,7 +963,7 @@ impl ProofFetches { } } } - Ok(()) + Ok(cached) } /// The source to ask next: one that is not cooling down, left out of @@ -1011,10 +1079,10 @@ fn body_excerpt(body: &[u8]) -> String { /// Run a fetch so that a panic becomes a failed fetch instead of being raised /// again in every waiter of the shared future. #[cfg(not(target_arch = "wasm32"))] -async fn contained(prove: F) -> Result<(), String> +async fn contained(prove: F) -> FetchResult where F: FnOnce() -> Fut, - Fut: Future>, + Fut: Future, { match AssertUnwindSafe(async move { prove().await }) .catch_unwind() @@ -1032,10 +1100,10 @@ where /// Run a fetch on the local executor, where browser requests and timers have /// to run, so it finishes even if every waiter goes away. #[cfg(target_arch = "wasm32")] -async fn contained(prove: F) -> Result<(), String> +async fn contained(prove: F) -> FetchResult where F: FnOnce() -> Fut + Send + 'static, - Fut: Future> + 'static, + Fut: Future + 'static, { on_local_executor(prove) .await @@ -1215,10 +1283,10 @@ mod tests { } #[test] - fn should_admit_a_quorum_while_many_fetches_wait_out_backoffs() { + fn should_refuse_new_quorums_while_the_running_fetches_are_full() { let fetches = Arc::new(fetches()); let mut waiting = Vec::new(); - for n in 0..MAX_FAILED_FETCHES + 8 { + for n in 0..MAX_RUNNING_FETCHES { let fetch = fetches .join_or_start(6, [n as u8; 32], 150) .expect("a place is free"); @@ -1226,9 +1294,40 @@ mod tests { holding(&fetches, (6, [n as u8; 32])).store(false, Ordering::Relaxed); waiting.push(fetch); } + let refused = fetches.join_or_start(6, [0xff; 32], 150).map(|_| ()); + assert!( + refused + .as_ref() + .is_err_and(|reason| reason.contains("quorum fetches are running")), + "{refused:?}" + ); + // A running fetch can still be joined. + assert!(fetches.join_or_start(6, [0; 32], 150).is_ok()); + + let id = running_id(&fetches, (6, [1; 32]), 150).expect("running"); + let first = attempt((6, [1; 32]), id, holding(&fetches, (6, [1; 32]))); + fetches.finish(&first, &Err("failed".to_string()), true); assert!(fetches.join_or_start(6, [0xff; 32], 150).is_ok()); } + #[test] + fn should_join_a_lower_fetch_only_when_asked_to_and_say_it_may_not_cover() { + let fetches = Arc::new(fetches()); + let _lower = fetches.join_or_start(6, [1; 32], 150).expect("starts"); + let id = running_id(&fetches, (6, [1; 32]), 150); + let (_, covers) = fetches.join_or_start(6, [1; 32], 100).expect("joins"); + assert!(covers, "a fetch for 150 covers 100"); + let (_, covers) = fetches.join_or_start(6, [1; 32], 400).expect("joins"); + assert!(!covers, "a fetch for 150 may prove only up to 150"); + assert_eq!(running_id(&fetches, (6, [1; 32]), 400), None); + let (_, covers) = fetches + .join_or_start_for(6, [1; 32], 400, false) + .expect("its own fetch"); + assert!(covers); + assert!(running_id(&fetches, (6, [1; 32]), 400).is_some()); + assert_eq!(running_id(&fetches, (6, [1; 32]), 150), id); + } + #[test] fn should_run_a_separate_fetch_for_a_height_far_from_the_running_one() { let fetches = Arc::new(fetches()); @@ -1294,9 +1393,9 @@ mod tests { let fetches = Arc::new(fetches()); let quorum = (6, [1; 32]); let now = Instant::now(); - let Some(long_ago) = now.checked_sub(Duration::from_secs(11)) else { - return; - }; + let long_ago = now + .checked_sub(Duration::from_secs(11)) + .expect("the clock has been running longer than the cooldown"); let failed = Slot::Failed { at: long_ago, failures: 2, diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index 50e5e523307..3c5500ff6f6 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -946,6 +946,42 @@ mod tests { assert_eq!(heights, [json!(bogus), json!(0), json!(honest)]); } + #[tokio::test] + async fn should_prove_again_for_a_joined_height_the_lower_fetch_did_not_cover() { + let f = fixture(Network::Testnet); + let bootstrap = f.bootstrap.clone(); + let mut answered = 0; + let relay = MockRelay::serve(3, move |_| { + answered += 1; + Some(if answered == 1 { + Answer::Late(Duration::from_millis(500), 200, bootstrap.clone()) + } else { + reply(200, bootstrap.clone()) + }) + }); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + // The relay proves the lowest target the lower height accepts. + let lower = f.target.height + 600; + let higher = lower + 1; + let first = fetch(&provider, f.quorum_type, f.quorum_hash, lower); + let second = async { + tokio::time::sleep(Duration::from_millis(100)).await; + fetch(&provider, f.quorum_type, f.quorum_hash, higher).await + }; + let (first, second) = futures::join!(first, second); + assert_eq!(first.expect("the lower height is proven"), Some(f.key)); + // The higher caller waited for the lower fetch instead of asking at + // once, then asked for its own height. The fixture proves nothing + // higher, so its own fetch fails; it does not fail for having joined. + let heights = relay.heights(); + assert_eq!(heights[..2], [json!(lower), json!(higher)], "{heights:?}"); + let reason = unavailable_reason(second); + assert!( + reason.contains(&format!("for core height {higher}")), + "{reason}" + ); + } + #[tokio::test] async fn should_prove_an_honest_height_while_a_bogus_one_is_in_flight() { let f = fixture(Network::Testnet); From b254731208041abae39c6374b4d69864596d5352 Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 26 Oct 2026 11:40:11 -0500 Subject: [PATCH 12/25] fix(sdk): bound each caller's wait and drop relays that stay busy A caller that waited on a fetch for a lower height and then started its own could wait two fetch deadlines. Each caller now has one 70 s deadline across both, after which it reports the source unavailable. A relay answering 503 is not cooled down, so a fetch against one that 503s everything stayed running, and held one of the 16 running slots, until its deadline. A fetch now stops asking a relay after three busy answers in a row; other fetches still ask it. Co-Authored-By: Claude Opus 5.5 --- .../src/proof_fetches.rs | 30 ++++++++++- .../src/verified_provider.rs | 52 +++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index 90867072186..972c91b1bee 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -25,6 +25,8 @@ use crate::quorum_refresh::on_local_executor; use crate::quorum_refresh::panic_message; #[cfg(not(target_arch = "wasm32"))] use crate::quorum_refresh::sleep; +// A `Send` timer on every target, for callers outside the local executor. +use crate::quorum_refresh::sleep as caller_sleep; use dash_context_provider::ContextProviderError; use dash_core_proof::bootstrap::{self, RecordKind}; use dash_core_proof::{parse_commitment, rpc_hex, ErrorKind, State}; @@ -74,6 +76,10 @@ const MAX_RETRY_DOUBLINGS: u32 = 3; /// Answers a source may give that it cannot serve the quorum before a fetch /// stops asking it: one for the height and one for the tip. const MAX_CANNOT_SERVE: u32 = 2; +/// Busy answers in a row after which a fetch stops asking a source. A relay +/// that stays busy is not cooled down for other fetches, but it must not keep +/// one running, and holding a running slot, for its whole deadline. +const MAX_BUSY: u32 = 3; /// How long a fetch waits before trying again to take a place to prove its /// quorum. const PLACE_RETRY: Duration = Duration::from_millis(250); @@ -395,12 +401,25 @@ impl ProofFetches { return Ok(key); } let unavailable = ContextProviderError::QuorumSourceUnavailable; + // One deadline for the caller, across the lower fetch it may wait on + // and the fetch of its own that may follow. + let deadline = Instant::now() + self.timing.deadline; let mut may_join_lower = true; loop { let (fetch, covers) = self .join_or_start_for(quorum_type, quorum_hash, height, may_join_lower) .map_err(unavailable)?; - let proven = fetch.await.map_err(unavailable)?; + let remaining = deadline.saturating_duration_since(Instant::now()); + let proven = match select(fetch, pin!(caller_sleep(remaining))).await { + Either::Left((proven, _)) => proven.map_err(unavailable)?, + Either::Right(_) => { + return Err(unavailable(format!( + "quorum {} was not proven within {} ms", + hex::encode(quorum_hash), + self.timing.deadline.as_millis() + ))); + } + }; if height <= proven.active_at.saturating_add(self.window) { return Ok(proven.key); } @@ -785,6 +804,7 @@ impl ProofFetches { return (Ok(proven), true); } Verdict::CannotServe(reason) => { + state.busy = 0; state.cannot_serve += 1; // Next time, ask this source for the other height. state.ask_tip = ask.height != 0; @@ -796,6 +816,12 @@ impl ProofFetches { failures.push(note(&reason)); } } + Verdict::Busy(reason) if state.busy + 1 >= MAX_BUSY => { + state.dropped = true; + failures.push(note(&format!( + "{reason}; busy {MAX_BUSY} times, not asked again" + ))); + } Verdict::Busy(reason) => { state.busy += 1; let max = self @@ -1312,6 +1338,8 @@ mod tests { #[test] fn should_join_a_lower_fetch_only_when_asked_to_and_say_it_may_not_cover() { + // The fetch for 150 keeps its place, and the one for 400 takes the + // second of the two, so starting it is not refused. let fetches = Arc::new(fetches()); let _lower = fetches.join_or_start(6, [1; 32], 150).expect("starts"); let id = running_id(&fetches, (6, [1; 32]), 150); diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index 3c5500ff6f6..e6d479b6fe5 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -975,6 +975,11 @@ mod tests { // higher, so its own fetch fails; it does not fail for having joined. let heights = relay.heights(); assert_eq!(heights[..2], [json!(lower), json!(higher)], "{heights:?}"); + assert_eq!( + relay.most_open(), + 1, + "the higher caller did not ask at once" + ); let reason = unavailable_reason(second); assert!( reason.contains(&format!("for core height {higher}")), @@ -982,6 +987,38 @@ mod tests { ); } + #[tokio::test] + async fn should_give_a_caller_one_deadline_across_the_fetches_it_waits_on() { + let f = fixture(Network::Testnet); + let bootstrap = f.bootstrap.clone(); + let mut answered = 0; + let relay = MockRelay::serve(2, move |_| { + answered += 1; + Some(if answered == 1 { + Answer::Late(Duration::from_millis(1500), 200, bootstrap.clone()) + } else { + Answer::Hang(Duration::from_secs(20)) + }) + }); + let timing = Timing { + deadline: Duration::from_secs(2), + request_timeout: Duration::from_secs(2), + ..quick() + }; + let provider = provider_for(&f, vec![relay.url.clone()], timing); + let lower = f.target.height + 600; + let first = fetch(&provider, f.quorum_type, f.quorum_hash, lower); + let started = Instant::now(); + let second = fetch(&provider, f.quorum_type, f.quorum_hash, lower + 1); + let (first, second) = futures::join!(first, second); + assert!(first.is_ok()); + // The lower fetch took 1.5 s; the caller's own fetch gets only what + // is left of its 2 s, not another 2 s. + let reason = unavailable_reason(second); + assert!(reason.contains("not proven within"), "{reason}"); + assert!(started.elapsed() < Duration::from_secs(3), "{reason}"); + } + #[tokio::test] async fn should_prove_an_honest_height_while_a_bogus_one_is_in_flight() { let f = fixture(Network::Testnet); @@ -1198,6 +1235,21 @@ mod tests { assert_eq!(provider.inner.fetches.cooling(), [false, true]); } + #[tokio::test] + async fn should_stop_asking_a_relay_that_stays_busy() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn((0..8).map(|_| reply(503, Vec::new())).collect()); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let started = Instant::now(); + let reason = unavailable_reason( + fetch(&provider, f.quorum_type, f.quorum_hash, f.target.height).await, + ); + assert!(reason.contains("not asked again"), "{reason}"); + assert_eq!(relay.requests().len(), 3); + assert!(started.elapsed() < Duration::from_secs(10), "{reason}"); + assert_eq!(provider.inner.fetches.cooling(), [false]); + } + #[tokio::test] async fn should_retry_a_busy_relay_without_cooling_it_down() { let f = fixture(Network::Testnet); From 6aed7560593b76fd7c6fa64352dd9b1afcb4ac54 Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 26 Oct 2026 12:20:22 -0500 Subject: [PATCH 13/25] fix(sdk): let a higher height prove itself after a lower fetch fails A caller that joined a fetch for a lower height returned that fetch's failure without asking for its own height, and the lower failure then held back later requests for every height within a window of it. Since heights come from unverified responses, a lower fetch made to fail could suppress an honest higher one. The caller now starts its own fetch once after a lower fetch fails, as after one that proved too little, within the same deadline; and a failure holds back only the heights its fetch would have covered. In browsers, the timer bounding a caller's wait ran in a local task to its full length even after the fetch finished, one per caller. It now ends, clearing the browser timer, as soon as the caller stops waiting. Co-Authored-By: Claude Opus 5.5 --- .../src/proof_fetches.rs | 65 +++++++++++++++++-- .../src/verified_provider.rs | 34 ++++++++++ 2 files changed, 92 insertions(+), 7 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index 972c91b1bee..a7a228e4a44 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -25,11 +25,13 @@ use crate::quorum_refresh::on_local_executor; use crate::quorum_refresh::panic_message; #[cfg(not(target_arch = "wasm32"))] use crate::quorum_refresh::sleep; -// A `Send` timer on every target, for callers outside the local executor. +#[cfg(not(target_arch = "wasm32"))] use crate::quorum_refresh::sleep as caller_sleep; use dash_context_provider::ContextProviderError; use dash_core_proof::bootstrap::{self, RecordKind}; use dash_core_proof::{parse_commitment, rpc_hex, ErrorKind, State}; +#[cfg(target_arch = "wasm32")] +use futures::channel::oneshot; use futures::future::{self, select, BoxFuture, Either, FutureExt, Shared, WeakShared}; use futures::stream::FuturesUnordered; use futures::StreamExt; @@ -389,8 +391,8 @@ impl ProofFetches { /// the quorum as absent. /// /// A caller may first join a fetch for a lower height near its own, which - /// may prove the quorum only up to that height; it then starts a fetch of - /// its own, once. + /// may prove the quorum only up to that height or fail; it then starts a + /// fetch of its own, once, within the same deadline. pub(crate) async fn fetch( self: Arc, quorum_type: u32, @@ -411,6 +413,11 @@ impl ProofFetches { .map_err(unavailable)?; let remaining = deadline.saturating_duration_since(Instant::now()); let proven = match select(fetch, pin!(caller_sleep(remaining))).await { + // A lower fetch failing says nothing about this height. + Either::Left((Err(_), _)) if !covers && may_join_lower => { + may_join_lower = false; + continue; + } Either::Left((proven, _)) => proven.map_err(unavailable)?, Either::Right(_) => { return Err(unavailable(format!( @@ -473,7 +480,6 @@ impl ProofFetches { ) -> Result<(SharedFetch, bool), String> { let quorum = (quorum_type, quorum_hash); let window = self.window; - let near = |other: u32| other.abs_diff(height) <= self.window; let now = Instant::now(); let forgotten_after = self.timing.max_quorum_cooldown.saturating_mul(2); let mut proven = self.lock(); @@ -497,7 +503,7 @@ impl ProofFetches { height: running, .. } if *other == quorum => { - let covers = height <= *running && *running - height <= window; + let covers = covered(*running, height, window); let lower = may_join_lower && *running < height && height - *running <= window; (covers || lower) .then(|| fetch.upgrade()) @@ -518,7 +524,7 @@ impl ProofFetches { at, failures, height: failed, - } if *other == quorum && near(*failed) => Some((*at, *failures)), + } if *other == quorum && covered(*failed, height, window) => Some((*at, *failures)), _ => None, }) .max_by_key(|(_, failures)| *failures); @@ -626,7 +632,7 @@ impl ProofFetches { let window = self.window; proven.fetches.retain(|(quorum, slot)| match slot { Slot::Failed { height, .. } => { - *quorum != attempt.quorum || height.abs_diff(attempt.height) > window + *quorum != attempt.quorum || !covered(*height, attempt.height, window) } Slot::Running { .. } => true, }); @@ -1044,6 +1050,13 @@ impl ProofFetches { } } +/// Whether a fetch for `requested` is sure to cover `height`: it proves a +/// target at or above `requested` minus the window, whose key is served up to +/// `requested`. A failure counts against the heights it would have covered. +fn covered(requested: u32, height: u32, window: u32) -> bool { + height <= requested && requested - height <= window +} + /// How many fetches hold one of the places to prove a quorum. fn places_taken(proven: &Proven) -> usize { proven @@ -1123,6 +1136,22 @@ where } } +/// Wait for `duration` in a `Send` future, for callers outside the local +/// executor. The browser timer runs in a local task that ends, clearing the +/// timer, as soon as this future is dropped. +#[cfg(target_arch = "wasm32")] +async fn caller_sleep(duration: Duration) { + let (cancel, cancelled) = oneshot::channel::<()>(); + let (fired, done) = oneshot::channel::<()>(); + wasm_bindgen_futures::spawn_local(async move { + if let Either::Left(_) = select(sleep(duration), cancelled).await { + let _ = fired.send(()); + } + }); + let _cancel_on_drop = cancel; + let _ = done.await; +} + /// Run a fetch on the local executor, where browser requests and timers have /// to run, so it finishes even if every waiter goes away. #[cfg(target_arch = "wasm32")] @@ -1308,6 +1337,28 @@ mod tests { ); } + #[test] + fn should_hold_back_only_heights_a_failed_fetch_would_have_covered() { + let fetches = Arc::new(fetches()); + let quorum = (6, [1; 32]); + fetches + .lock() + .fetches + .push((quorum, failed(Instant::now()))); + for height in [150, 100] { + assert!( + fetches.join_or_start(6, [1; 32], height).is_err(), + "{height}" + ); + } + for height in [151, 750] { + assert!( + fetches.join_or_start(6, [1; 32], height).is_ok(), + "{height}" + ); + } + } + #[test] fn should_refuse_new_quorums_while_the_running_fetches_are_full() { let fetches = Arc::new(fetches()); diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index e6d479b6fe5..99fbb6eaf6a 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -987,6 +987,40 @@ mod tests { ); } + #[tokio::test] + async fn should_prove_a_higher_height_after_the_lower_fetch_it_joined_fails() { + let f = fixture(Network::Testnet); + let height = f.target.height; + let lower = height - 100; + let bootstrap = f.bootstrap.clone(); + let relay = MockRelay::serve(3, move |request| { + Some(if request["height"] == height { + reply(200, bootstrap.clone()) + } else { + // The relay cannot serve the lower height, for it or the tip. + Answer::Late(Duration::from_millis(300), 502, Vec::new()) + }) + }); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let first = fetch(&provider, f.quorum_type, f.quorum_hash, lower); + let second = async { + tokio::time::sleep(Duration::from_millis(100)).await; + fetch(&provider, f.quorum_type, f.quorum_hash, height).await + }; + let (first, second) = futures::join!(first, second); + unavailable_reason(first); + assert_eq!( + second.expect("the higher height proves itself"), + Some(f.key) + ); + let heights = relay.heights(); + assert_eq!( + heights, + [json!(lower), json!(0), json!(height)], + "{heights:?}" + ); + } + #[tokio::test] async fn should_give_a_caller_one_deadline_across_the_fetches_it_waits_on() { let f = fixture(Network::Testnet); From edc267047ae1aba852d62f34f27a508e99ba2d41 Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 26 Oct 2026 12:26:17 -0500 Subject: [PATCH 14/25] fix(sdk): keep a quorum's higher failure when a lower fetch fails A failed fetch replaces the failures that covered its height. When a fetch for a lower height failed, the failure it recorded covered only heights up to its own, so the higher failure's cooldown was lost and heights between the two were admitted with no backoff. The recorded failure now keeps the highest height it replaced. Co-Authored-By: Claude Opus 5.5 --- .../src/proof_fetches.rs | 72 ++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index a7a228e4a44..ca04c190348 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -628,8 +628,22 @@ impl ProofFetches { if !proven.fetches.iter().any(|(_, slot)| ours(slot)) { return; } - // This fetch's outcome replaces the failures it carried on. + // This fetch's outcome replaces the failures it carried on. A failure + // recorded in their place keeps the highest height they covered, so + // a fetch for a lower height cannot clear a higher one's cooldown. let window = self.window; + let replaced = proven + .fetches + .iter() + .filter_map(|(quorum, slot)| match slot { + Slot::Failed { height, .. } + if *quorum == attempt.quorum && covered(*height, attempt.height, window) => + { + Some(*height) + } + _ => None, + }) + .max(); proven.fetches.retain(|(quorum, slot)| match slot { Slot::Failed { height, .. } => { *quorum != attempt.quorum || !covered(*height, attempt.height, window) @@ -656,7 +670,7 @@ impl ProofFetches { proven.fetches[index].1 = Slot::Failed { at, failures, - height: attempt.height, + height: replaced.map_or(attempt.height, |height| height.max(attempt.height)), }; // Forget the oldest failure rather than remember without bound. let failed: Vec<_> = proven @@ -1337,6 +1351,60 @@ mod tests { ); } + #[test] + fn should_hold_back_heights_down_to_a_window_below_a_failure() { + let fetches = Arc::new(fetches()); + let failure = Slot::Failed { + at: Instant::now(), + failures: 1, + height: 1000, + }; + fetches.lock().fetches.push(((6, [1; 32]), failure)); + assert!(fetches.join_or_start(6, [1; 32], 400).is_err()); + assert!(fetches.join_or_start(6, [1; 32], 399).is_ok()); + } + + #[test] + fn should_keep_a_higher_failures_reach_when_a_lower_fetch_fails() { + let fetches = Arc::new(fetches()); + let quorum = (6, [1; 32]); + let long_ago = Instant::now() + .checked_sub(Duration::from_secs(70)) + .expect("the clock has been running longer than the cooldown"); + // Past the longest cooldown, but not yet forgotten. + let failure = Slot::Failed { + at: long_ago, + failures: 4, + height: 1000, + }; + fetches.lock().fetches.push((quorum, failure)); + // The cooldown has passed, so a fetch for 500 starts and fails. + let _lower = fetches.join_or_start(6, [1; 32], 500).expect("cooled down"); + let id = running_id(&fetches, quorum, 500).expect("running"); + let lower = Attempt { + height: 500, + failures: 4, + failed_at: Some(long_ago), + ..attempt(quorum, id, holding(&fetches, quorum)) + }; + fetches.finish(&lower, &Err("failed".to_string()), true); + let recorded = fetches + .lock() + .fetches + .iter() + .find_map(|(_, slot)| match slot { + Slot::Failed { + failures, height, .. + } => Some((*failures, *height)), + Slot::Running { .. } => None, + }); + assert_eq!(recorded, Some((5, 1000))); + assert!( + fetches.join_or_start(6, [1; 32], 900).is_err(), + "900 is still held back" + ); + } + #[test] fn should_hold_back_only_heights_a_failed_fetch_would_have_covered() { let fetches = Arc::new(fetches()); From fb1d5b1e7f4cf6b8bdc3d04b45694cc5c3dc776d Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 26 Oct 2026 13:45:27 -0500 Subject: [PATCH 15/25] test(sdk): check that a browser caller's deadline fires The browser timer that bounds a caller's wait was reported as never firing. It does fire: its local task sends on the sender of the channel whose receiver the caller awaits. Two wasm-bindgen tests now pin it down: the timer wakes a caller racing a future that never resolves (against a fallback browser timer, so a broken deadline fails rather than leaves the test pending), and a fetch whose relay never answers ends as an unavailable source. Co-Authored-By: Claude Opus 5.5 --- .../src/proof_fetches.rs | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index ca04c190348..f8df6018508 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -1640,3 +1640,58 @@ mod tests { assert_eq!(heights[1], 140 - MAX_RECENT_STATES as u32 + 1); } } + +#[cfg(all(test, target_arch = "wasm32"))] +mod browser_tests { + use super::*; + use dash_core_proof::ProofNetwork; + use js_sys::{Function, Reflect}; + use wasm_bindgen::JsValue; + use wasm_bindgen_test::wasm_bindgen_test; + + #[wasm_bindgen_test] + async fn should_end_a_callers_wait_at_its_deadline() { + // A browser timer of the test's own, so a deadline that never fires + // fails the test instead of leaving it pending. + let fallback = pin!(sleep(Duration::from_secs(5))); + let deadline = pin!(caller_sleep(Duration::from_millis(50))); + let woke = select(fallback, deadline).await; + assert!(matches!(woke, Either::Right(_)), "the deadline fired"); + } + + #[wasm_bindgen_test] + async fn should_report_a_relay_that_never_answers_as_unavailable() { + let fetch = Function::new_with_args("request", "return new Promise(() => {});"); + Reflect::set(&js_sys::global(), &JsValue::from_str("fetch"), &fetch) + .expect("set the global fetch"); + let checkpoint = State { + network: ProofNetwork::Testnet, + height: 1_000_000, + block_hash: [1; 32], + masternode_root: [2; 32], + quorum_root: [3; 32], + }; + let timing = Timing { + deadline: Duration::from_millis(300), + request_timeout: Duration::from_millis(300), + ..Timing::default() + }; + let fetches = Arc::new(ProofFetches::new( + 6, + 600, + checkpoint, + vec![Url::parse("https://relay.test/proofs").expect("a URL")], + NonZeroUsize::new(4).expect("non-zero"), + ProofTransport::new().expect("a transport"), + timing, + )); + let result = fetches.fetch(6, [7; 32], 1_000_100).await; + assert!( + matches!( + result, + Err(ContextProviderError::QuorumSourceUnavailable(_)) + ), + "{result:?}" + ); + } +} From 2b5b54895133c48a577fb3c7daa95887c172d28e Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 28 Oct 2026 23:50:10 -0500 Subject: [PATCH 16/25] fix(sdk): serve a quorum key only for heights its proofs bracket A verified target above the requested Core height, such as a tip proof, was accepted as evidence for that height, and a key was served for 600 blocks past the target that proved it. A quorum in the quorum root at a later target may have been mined after the height, so that proved nothing about it. Platform quorums (mainnet type 4, testnet type 6) do not rotate: the active set at a height is the newest quorums mined by it, so a quorum enters when mined and never returns once it leaves. A quorum opened under the quorum roots of two verified targets was therefore active at every height between them. The cache now keeps, per quorum, the lowest and highest verified targets that opened it, and serves the key for a height only inside that interval. The 600-block serving window is gone. - With evidence at or below the height, the relay is asked for its tip (alternating with the height), which also covers the next heights. - Otherwise the height itself is asked for, from a verified state below it, the checkpoint included. A valid target above the height is kept as evidence but counts as a refusal, and that source is asked next for a target up to one DKG interval (24 blocks) below the height. - A target below the height asked for (or below the height, for a tip) is refused before any signature is checked. - A caller waiting on another height's fetch starts its own once if the result does not bracket its height. A failed fetch now cools down only its own height, since proving another height asks for other targets. - The mock relay in the tests ignores connections that close without a request, which made one test flaky. Co-Authored-By: Claude Opus 5.5 --- .../rs-sdk-trusted-context-provider/README.md | 48 +- .../src/proof_fetches.rs | 546 +++++++++++------- .../src/verified_provider.rs | 362 ++++++++++-- 3 files changed, 682 insertions(+), 274 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/README.md b/packages/rs-sdk-trusted-context-provider/README.md index aaf10ebd34a..3aeffd0103a 100644 --- a/packages/rs-sdk-trusted-context-provider/README.md +++ b/packages/rs-sdk-trusted-context-provider/README.md @@ -164,19 +164,28 @@ Behaviour: - Mainnet and testnet only. Each provider POSTs to `/proofs` with the checkpoint hash, a height, the quorum hash and type, and `nodeCount: 0`. -- A proven key is served for 600 blocks past the target block that proved the - quorum active, then proven again. Failures of any kind (timeouts, HTTP - errors, oversize or invalid proofs, proofs of other quorums) are reported as - an unavailable quorum source, so the SDK tries another node without banning - the one that named the quorum. A quorum of another type than the network's - Platform quorum type is rejected as invalid. +- A key is returned for a Core height only when verified proofs bracket that + height: among the targets whose quorum roots opened the quorum, one is at or + below the height and one at or above it. Failures of any kind (timeouts, + HTTP errors, oversize or invalid proofs, proofs of other quorums) are + reported as an unavailable quorum source, so the SDK tries another node + without banning the one that named the quorum. A quorum of another type + than the network's Platform quorum type is rejected as invalid. +- Serving costs about one small proof per new Core height per quorum in use. + The first proof of a quorum asks for the height itself; Core usually + answers with a target exactly there, since Platform's Core height is + ChainLocked. Above a proven height the relay is usually asked for its + latest ChainLock, which also covers the next few heights Platform reports. + A relay that answers with a target above the height, because the height + has no ChainLock of its own, is asked again for a target up to one DKG + interval (24 blocks) below it; the later target is kept as evidence. - Fetches of one quorum are shared, and requests for at most two quorums are in flight at once (a fetch waiting out a backoff gives its place up). - A relay that cannot serve a quorum for the height and for its tip is - not asked again in that fetch; a busy relay (503) is retried after a - backoff; relays that fail otherwise, and quorums whose fetch failed, cool - down. A fetch gives up after 70 s. Redirects are refused and responses are - capped at 1 MiB. + A relay that cannot serve a quorum for two asks (the height and its tip, + or the height and just below it) is not asked again in that fetch; a busy + relay (503) is retried after a backoff; relays that fail otherwise cool + down, and so does a quorum at a height whose fetch failed. A fetch gives up after 70 s. + Redirects are refused and responses are capped at 1 MiB. - The cache should hold at least the active Platform quorums (24 on mainnet and testnet); 32 or more avoids proving keys again just because they were evicted. @@ -189,6 +198,13 @@ Behaviour: ### Trust model +A bracket proves the quorum was active at the height. Platform quorums +(mainnet type 4, testnet type 6) do not rotate: the active set at a height is +the newest quorums mined by it, so a quorum enters the set when it is mined +and never returns once it leaves. A quorum in the quorum roots of two verified +targets was therefore active at every height between them. All targets are +proven from one checkpoint through ChainLocks, so they lie on one chain. + The checkpoint is trust configuration, like a hard-coded block hash: whoever chooses it chooses the chain. Beyond it, the provider trusts that no ChainLock quorum active at the checkpoint or mined after it ever signs dishonestly, @@ -207,11 +223,11 @@ back to `with_checkpoint` on the next start, so its proofs stay short. ### Limitations - Keys are proven for the Core height Platform reports in its responses. If - Platform halts, that height stops while Core moves on: a quorum not cached - before the halt can be proven only while a relay can still serve a proof - within 600 blocks of that height whose quorum root contains the quorum, and - retired quorums leave Core's quorum root. Such quorums may stay unprovable - until Platform resumes. + Platform halts, that height stops while Core moves on. A quorum not cached + before the halt is still proven with a proof whose target is that height, + as long as the relay's Core has that ChainLock. +- A height at or below the checkpoint cannot be proven, since every proof + starts from it. - An inclusion proof cannot show that a quorum does not exist, so the provider never reports a quorum as absent; nodes naming fake quorums are skipped by the SDK, not banned. diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index f8df6018508..196f2984c50 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -6,12 +6,14 @@ //! stalls or fails costs time, bounded here, but never changes a key. //! //! A proven key is immutable: under ChainLocks each quorum has one mined -//! commitment. What expires is the evidence that the quorum is still active. -//! A key proven by a target at height `T` is served for heights up to -//! `T + window`, where `window` is how long a Platform quorum stays in the -//! active set. The height comes from the response being verified and is not -//! authenticated, so the window is only a trigger to prove again, not a -//! security boundary. +//! commitment. What a proof adds is evidence of when the quorum was active. +//! Platform quorums do not rotate: the active set at a height is the newest +//! quorums mined by it, so a quorum enters the set when it is mined and never +//! returns once it leaves. A quorum opened under the quorum roots of two +//! verified targets was therefore active at every height between them. A key +//! is served for a Core height only when verified targets bracket it. Every +//! target is proven from one checkpoint through ChainLocks, so all of them lie +//! on one chain. //! //! The SDK treats any error from a fetch as "the source gave no answer", and //! never bans a node for it. A fetch therefore never reports a quorum as @@ -76,12 +78,18 @@ const MAX_RECENT_STATES: usize = 16; /// The delay before asking a busy relay again doubles this many times. const MAX_RETRY_DOUBLINGS: u32 = 3; /// Answers a source may give that it cannot serve the quorum before a fetch -/// stops asking it: one for the height and one for the tip. +/// stops asking it: one for each of the two heights [`choose_ask`] alternates +/// between. const MAX_CANNOT_SERVE: u32 = 2; /// Busy answers in a row after which a fetch stops asking a source. A relay /// that stays busy is not cooled down for other fetches, but it must not keep /// one running, and holding a running slot, for its whole deadline. const MAX_BUSY: u32 = 3; +/// One DKG interval of the Platform quorum types (mainnet type 4, testnet +/// type 6). A relay asked for a height may answer with a later target when +/// the height has no ChainLock of its own; it is then asked for a target at +/// most this far below the height, which with the later one brackets it. +pub(crate) const LOWER_ASK_BLOCKS: u32 = 24; /// How long a fetch waits before trying again to take a place to prove its /// quorum. const PLACE_RETRY: Duration = Duration::from_millis(250); @@ -132,17 +140,26 @@ fn backoff(first: Duration, max: Duration, failures: u32) -> Duration { first.saturating_mul(1 << doublings).min(max) } -/// A proven key and the highest target height that proved it. The key is -/// served for heights up to `active_at` plus the validity window. +/// A proven key and the lowest and highest verified targets whose quorum root +/// opened it. The quorum was active at every height from `lo` to `hi`, and the +/// key is served for those heights only. #[derive(Clone, Copy, Debug)] pub(crate) struct Cached { key: QuorumKey, - active_at: u32, + lo: u32, + hi: u32, +} + +impl Cached { + /// Whether the proofs behind this key bracket `height`. + fn brackets(&self, height: u32) -> bool { + (self.lo..=self.hi).contains(&height) + } } /// What a fetch hands every caller waiting on it: the key it proved and the -/// height that covers, so that a caller does not depend on the cache still -/// holding it. +/// heights its proofs bracket, so that a caller does not depend on the cache +/// still holding it. type FetchResult = Result; type FetchFuture = BoxFuture<'static, FetchResult>; type SharedFetch = Shared; @@ -193,33 +210,46 @@ struct Ask { /// Choose what to ask a relay for when proving a quorum for `height`. /// -/// The proof starts from the newest verified state below `height`, so it is -/// as short as it can be, and asks for a target at `height`. Once a state -/// past the checkpoint is verified, the checkpoint is never used again: a -/// later quorum is proven from the latest state, asking for the tip. -/// `alternate` flips between asking for `height` and for the tip, which a -/// relay that could not serve one may still serve. +/// `extending` says the cache holds the quorum from a target at or below +/// `height`, so any target at or above `height` brackets it. The relay is then +/// asked for its tip, which also covers the next heights Platform reports, or, +/// with `alternate`, for `height`. The proof starts from the newest verified +/// state below `height`, so it is as short as it can be, but once a state +/// past the checkpoint is verified, neither is asked for from the checkpoint +/// again: the tip is asked for from the latest state instead. +/// +/// Otherwise a target at or below `height` is needed: exactly at it if +/// nothing is known yet, or anywhere up to it if a later target opened the +/// quorum. Only a state below `height` can lead there, the checkpoint +/// included. The relay is asked for `height`, or, with `alternate`, for a +/// target at most [`LOWER_ASK_BLOCKS`] below it. There is no such ask when no +/// verified state is below `height`. fn choose_ask( states: &BTreeMap, checkpoint_height: u32, height: u32, + extending: bool, alternate: bool, ) -> Option { + let below = states.range(..height).next_back(); + if !extending { + let (&from, state) = below?; + let lowest = height.saturating_sub(LOWER_ASK_BLOCKS).max(from + 1); + return Some(Ask { + checkpoint: state.clone(), + height: if alternate { lowest } else { height }, + }); + } let (_, latest) = states.last_key_value()?; - let ask = match states.range(..height).next_back() { - Some((&below, state)) if below != checkpoint_height || latest.height == below => Ask { + Some(match below { + Some((&from, state)) if from != checkpoint_height || latest.height == from => Ask { checkpoint: state.clone(), - height, + height: if alternate { height } else { 0 }, }, _ => Ask { checkpoint: latest.clone(), height: 0, }, - }; - Some(if alternate && ask.height != 0 { - Ask { height: 0, ..ask } - } else { - ask }) } @@ -227,8 +257,9 @@ fn choose_ask( enum Verdict { Proven(Cached), /// The relay cannot serve this quorum for the height asked: a 502, or a - /// valid proof that does not serve the request. Asked again once, for the - /// other height, after a delay; then left out of the fetch. + /// valid proof that does not serve the request, such as one whose target + /// does not bracket the height. Asked again once, for the other height, + /// after a delay; then left out of the fetch. CannotServe(String), /// The relay's workers are busy: ask again after a backoff. Busy(String), @@ -259,8 +290,9 @@ struct SourceAttempts { ready_at: Instant, cannot_serve: u32, busy: u32, - /// Ask for the tip rather than the height next time. - ask_tip: bool, + /// Ask for the other height next time: the tip rather than the height, + /// or a lower height rather than the height. See [`choose_ask`]. + alternate: bool, dropped: bool, } @@ -272,12 +304,21 @@ impl SourceAttempts { ready_at: now, cannot_serve: 0, busy: 0, - ask_tip: false, + alternate: false, dropped: false, } } } +/// What a fetch does next, given what the cache holds for its quorum. +enum NextAsk { + Ask(Ask), + /// The quorum is proven for the height already. + Proven(Cached), + /// No proof can bracket the height. + Unreachable, +} + /// The source a fetch asks next. enum Pick { Ready(usize), @@ -300,8 +341,8 @@ pub(crate) struct ProofFetches { impl ProofFetches { /// Prove keys of `quorum_type` quorums from `checkpoint` with proofs - /// from `endpoints`, serving each for `window` blocks past the target - /// that proved it. + /// from `endpoints`. A caller may wait on a running fetch of a quorum for + /// a height at most `window` from its own. pub(crate) fn new( quorum_type: u32, window: u32, @@ -353,8 +394,7 @@ impl ProofFetches { quorum_type == self.quorum_type && height <= i32::MAX as u32 } - /// The cached key of a quorum, if it was proven active recently enough - /// for `height`. + /// The cached key of a quorum, if verified proofs bracket `height`. pub(crate) fn lookup( &self, quorum_type: u32, @@ -370,13 +410,12 @@ impl ProofFetches { } let mut proven = self.lock(); match proven.keys.get(&(quorum_type, quorum_hash)) { - Some(cached) if height <= cached.active_at.saturating_add(self.window) => { - Ok(cached.key) - } + Some(cached) if cached.brackets(height) => Ok(cached.key), Some(cached) => Err(ContextProviderError::QuorumSourceUnavailable(format!( - "quorum {} was proven active at core height {}, not yet for {height}", + "quorum {} was proven active at core heights {} to {}, not yet at {height}", hex::encode(quorum_hash), - cached.active_at + cached.lo, + cached.hi ))), None => Err(ContextProviderError::QuorumSourceUnavailable(format!( "quorum {} is not proven yet", @@ -390,9 +429,10 @@ impl ProofFetches { /// [`ContextProviderError::QuorumSourceUnavailable`]; this never reports /// the quorum as absent. /// - /// A caller may first join a fetch for a lower height near its own, which - /// may prove the quorum only up to that height or fail; it then starts a - /// fetch of its own, once, within the same deadline. + /// A caller may first join a fetch of the quorum for another height near + /// its own, whose proofs may not bracket the caller's height, or which may + /// fail; it then starts a fetch of its own, once, within the same + /// deadline. pub(crate) async fn fetch( self: Arc, quorum_type: u32, @@ -403,19 +443,19 @@ impl ProofFetches { return Ok(key); } let unavailable = ContextProviderError::QuorumSourceUnavailable; - // One deadline for the caller, across the lower fetch it may wait on + // One deadline for the caller, across the other fetch it may wait on // and the fetch of its own that may follow. let deadline = Instant::now() + self.timing.deadline; - let mut may_join_lower = true; + let mut may_join_other = true; loop { - let (fetch, covers) = self - .join_or_start_for(quorum_type, quorum_hash, height, may_join_lower) + let (fetch, own) = self + .join_or_start_for(quorum_type, quorum_hash, height, may_join_other) .map_err(unavailable)?; let remaining = deadline.saturating_duration_since(Instant::now()); let proven = match select(fetch, pin!(caller_sleep(remaining))).await { - // A lower fetch failing says nothing about this height. - Either::Left((Err(_), _)) if !covers && may_join_lower => { - may_join_lower = false; + // Another height's fetch failing says nothing about this one. + Either::Left((Err(_), _)) if !own && may_join_other => { + may_join_other = false; continue; } Either::Left((proven, _)) => proven.map_err(unavailable)?, @@ -427,17 +467,18 @@ impl ProofFetches { ))); } }; - if height <= proven.active_at.saturating_add(self.window) { + if proven.brackets(height) { return Ok(proven.key); } - if covers || !may_join_lower { + if own || !may_join_other { return Err(unavailable(format!( - "quorum {} was proven active at core height {}, not for {height}", + "quorum {} was proven active at core heights {} to {}, not at {height}", hex::encode(quorum_hash), - proven.active_at + proven.lo, + proven.hi ))); } - may_join_lower = false; + may_join_other = false; } } @@ -446,8 +487,8 @@ impl ProofFetches { self.proven.lock().unwrap_or_else(PoisonError::into_inner) } - /// [`join_or_start_for`](Self::join_or_start_for), joining a fetch for a - /// lower height if there is one. + /// [`join_or_start_for`](Self::join_or_start_for), joining a fetch for + /// another height near this one if there is one. #[cfg(test)] fn join_or_start( self: &Arc, @@ -459,24 +500,24 @@ impl ProofFetches { } /// A running fetch of this quorum to wait on, or a new one if the limits - /// allow, and whether its result is sure to cover `height`. + /// allow, and whether it is the fetch for `height` itself, whose result + /// is final for the caller. /// - /// A fetch for height `H` proves a target at or above `H` minus the - /// window, so its key is sure to be served for heights up to `H`. A - /// caller joins such a fetch for a height at most a window below its - /// own; with `may_join_lower`, it otherwise joins one for a height at - /// most a window above, and must check what it proved. + /// A fetch succeeds only once its proofs bracket its own height, so only + /// a fetch for `height` itself is sure to serve the caller. With + /// `may_join_other`, a caller otherwise joins a fetch for a height at + /// most a window from its own, and must check what it proved. /// - /// A fetch or a failure counts only for heights within the validity - /// window of its own, since the height comes from an unverified response: - /// a fetch or failure at a bogus height must not hold back, or cool down, - /// a request for an honest one, which runs its own fetch. + /// A failure cools the quorum down only for its own height. Proving + /// another height asks for other targets, and the height comes from an + /// unverified response: a failure at a bogus height must not hold back a + /// request for an honest one, which runs its own fetch. fn join_or_start_for( self: &Arc, quorum_type: u32, quorum_hash: [u8; 32], height: u32, - may_join_lower: bool, + may_join_other: bool, ) -> Result<(SharedFetch, bool), String> { let quorum = (quorum_type, quorum_hash); let window = self.window; @@ -490,7 +531,7 @@ impl ProofFetches { }); // A fetch that finished since the caller's lookup has nothing to add. if let Some(cached) = proven.keys.peek(&quorum) { - if height <= cached.active_at.saturating_add(self.window) { + if cached.brackets(height) { return Ok((future::ready(Ok(*cached)).boxed().shared(), true)); } } @@ -503,16 +544,16 @@ impl ProofFetches { height: running, .. } if *other == quorum => { - let covers = covered(*running, height, window); - let lower = may_join_lower && *running < height && height - *running <= window; - (covers || lower) + let own = *running == height; + let near = running.abs_diff(height) <= window; + (own || (may_join_other && near)) .then(|| fetch.upgrade()) .flatten() - .map(|fetch| (fetch, covers)) + .map(|fetch| (fetch, own)) } _ => None, }) - .max_by_key(|(_, covers)| *covers); + .max_by_key(|(_, own)| *own); if let Some(joined) = joinable { return Ok(joined); } @@ -524,7 +565,7 @@ impl ProofFetches { at, failures, height: failed, - } if *other == quorum && covered(*failed, height, window) => Some((*at, *failures)), + } if *other == quorum && *failed == height => Some((*at, *failures)), _ => None, }) .max_by_key(|(_, failures)| *failures); @@ -560,8 +601,8 @@ impl ProofFetches { "{live} quorum fetches are running; not starting another" )); } - // The earlier failures near this height stay until the new fetch - // ends, so a fetch every caller abandons does not erase them. + // The earlier failures at this height stay until the new fetch ends, + // so a fetch every caller abandons does not erase them. proven.next_fetch_id += 1; let id = proven.next_fetch_id; let holding = Arc::new(AtomicBool::new(true)); @@ -628,26 +669,9 @@ impl ProofFetches { if !proven.fetches.iter().any(|(_, slot)| ours(slot)) { return; } - // This fetch's outcome replaces the failures it carried on. A failure - // recorded in their place keeps the highest height they covered, so - // a fetch for a lower height cannot clear a higher one's cooldown. - let window = self.window; - let replaced = proven - .fetches - .iter() - .filter_map(|(quorum, slot)| match slot { - Slot::Failed { height, .. } - if *quorum == attempt.quorum && covered(*height, attempt.height, window) => - { - Some(*height) - } - _ => None, - }) - .max(); + // This fetch's outcome replaces the failures it carried on. proven.fetches.retain(|(quorum, slot)| match slot { - Slot::Failed { height, .. } => { - *quorum != attempt.quorum || !covered(*height, attempt.height, window) - } + Slot::Failed { height, .. } => *quorum != attempt.quorum || *height != attempt.height, Slot::Running { .. } => true, }); let Some(index) = proven @@ -670,7 +694,7 @@ impl ProofFetches { proven.fetches[index].1 = Slot::Failed { at, failures, - height: replaced.map_or(attempt.height, |height| height.max(attempt.height)), + height: attempt.height, }; // Forget the oldest failure rather than remember without bound. let failed: Vec<_> = proven @@ -740,16 +764,19 @@ impl ProofFetches { wake = (now + PLACE_RETRY).min(deadline); } Pick::Ready(source) => { - let ask = choose_ask( - &self.lock().states, - self.checkpoint.height, - height, - sources[source].ask_tip, - ) - .unwrap_or_else(|| Ask { - checkpoint: self.checkpoint.clone(), - height, - }); + let ask = match self.next_ask(quorum, height, sources[source].alternate) + { + NextAsk::Ask(ask) => ask, + // Another fetch proved it meanwhile. + NextAsk::Proven(proven) => return (Ok(proven), sent > 0), + NextAsk::Unreachable => { + failures.push(format!( + "no verified state is below core height {height}, so \ + no proof can reach it" + )); + break; + } + }; let body = request_body(&ask, quorum); let remaining = deadline.saturating_duration_since(now); let timeout = self.timing.request_timeout.min(remaining); @@ -827,7 +854,7 @@ impl ProofFetches { state.busy = 0; state.cannot_serve += 1; // Next time, ask this source for the other height. - state.ask_tip = ask.height != 0; + state.alternate = !state.alternate; if state.cannot_serve >= MAX_CANNOT_SERVE { state.dropped = true; failures.push(note(&format!("{reason}; not asked again"))); @@ -881,6 +908,10 @@ impl ProofFetches { /// oversized transfers, redirects, error statuses other than 502 and /// 503, and proofs that do not verify. A valid proof that does not serve /// the request may be the fault of the unverified height or quorum hash. + /// + /// A valid proof whose target does not bracket `height` together with + /// the earlier proofs is still published, as evidence for later asks, but + /// does not prove the key for `height`. fn judge( &self, quorum: QuorumId, @@ -904,14 +935,15 @@ impl ProofFetches { 503 => return Verdict::Busy(format!("HTTP 503{}", body_excerpt(body))), status => return Verdict::SourceFailed(format!("HTTP {status}{}", body_excerpt(body))), } - let needed = height.saturating_sub(self.window); - // The target height is checked before any signature, so a bogus + // A target below the height asked for, or below `height` for the + // tip, can never help. It is refused before any signature, so a bogus // height costs no BLS work. Only this source is affected if it lies. + let needed = if ask.height == 0 { height } else { ask.height }; let verified = match bootstrap::verify(body, &ask.checkpoint, needed) { Ok(verified) => verified, Err(error) if error.kind() == ErrorKind::StaleTarget && ask.height == 0 => { return Verdict::BeyondTip(format!( - "its latest ChainLock is below the core height {needed} that {height} needs" + "its latest ChainLock is below core height {height}" )); } Err(error) if error.kind() == ErrorKind::StaleTarget => { @@ -942,14 +974,41 @@ impl ProofFetches { )); }; match self.publish(quorum, *key, verified.state()) { - Ok(proven) => Verdict::Proven(proven), + Ok(proven) if proven.brackets(height) => Verdict::Proven(proven), + Ok(proven) => Verdict::CannotServe(format!( + "proof target {} leaves the quorum proven active at core heights {} to {}, \ + not at {height}", + verified.state().height, + proven.lo, + proven.hi + )), Err(reason) => Verdict::CannotServe(reason), } } - /// Cache a proven key and remember the state that proved it, and return - /// what the cache now holds for the quorum. Nothing is cached if either - /// conflicts with what is already known. + /// What to ask a source next for this quorum, given what the cache holds + /// for it now. + fn next_ask(&self, quorum: QuorumId, height: u32, alternate: bool) -> NextAsk { + let proven = self.lock(); + let cached = proven.keys.peek(&quorum); + if let Some(cached) = cached.filter(|cached| cached.brackets(height)) { + return NextAsk::Proven(*cached); + } + let extending = cached.is_some_and(|cached| cached.lo <= height); + choose_ask( + &proven.states, + self.checkpoint.height, + height, + extending, + alternate, + ) + .map_or(NextAsk::Unreachable, NextAsk::Ask) + } + + /// Cache a proven key, widening the heights it is known active at to the + /// target that opened it, and remember the target, and return what the + /// cache now holds for the quorum. Nothing is cached if either conflicts + /// with what is already known. pub(crate) fn publish( &self, quorum: QuorumId, @@ -980,13 +1039,15 @@ impl ProofFetches { return Err("opens a different key than one already proven".to_string()); } Some(cached) => { - cached.active_at = cached.active_at.max(target.height); + cached.lo = cached.lo.min(target.height); + cached.hi = cached.hi.max(target.height); *cached } None => { let cached = Cached { key, - active_at: target.height, + lo: target.height, + hi: target.height, }; proven.keys.put(quorum, cached); cached @@ -1064,13 +1125,6 @@ impl ProofFetches { } } -/// Whether a fetch for `requested` is sure to cover `height`: it proves a -/// target at or above `requested` minus the window, whose key is served up to -/// `requested`. A failure counts against the heights it would have covered. -fn covered(requested: u32, height: u32, window: u32) -> bool { - height <= requested && requested - height <= window -} - /// How many fetches hold one of the places to prove a quorum. fn places_taken(proven: &Proven) -> usize { proven @@ -1209,20 +1263,18 @@ mod tests { } #[test] - fn should_ask_from_the_newest_state_below_the_height_but_never_fall_back_to_the_checkpoint() { + fn should_ask_for_the_tip_when_extending_without_falling_back_to_the_checkpoint() { let cases = [ // (verified heights, height, alternate, expected, why) - (&[100][..], 150, false, ask(100, 150), "checkpoint below"), - (&[100], 150, true, ask(100, 0), "alternate asks for the tip"), - (&[100], 100, false, ask(100, 0), "nothing below: the tip"), + (&[100][..], 150, false, ask(100, 0), "the tip, from below"), ( &[100], - 50, - false, - ask(100, 0), - "height under the checkpoint", + 150, + true, + ask(100, 150), + "alternate asks for the height", ), - (&[100, 200], 250, false, ask(200, 250), "later state below"), + (&[100], 100, false, ask(100, 0), "nothing below: the latest"), ( &[100, 200], 150, @@ -1230,24 +1282,65 @@ mod tests { ask(200, 0), "no downgrade to checkpoint", ), + ( + &[100, 200], + 150, + true, + ask(200, 0), + "not for the height either", + ), ( &[100, 200, 300], 250, false, - ask(200, 250), + ask(200, 0), "verified state below", ), ( &[100, 200, 300], 250, true, - ask(200, 0), + ask(200, 250), "alternate from it", ), ]; for (heights, height, alternate, expected, why) in cases { assert_eq!( - choose_ask(&states(heights), 100, height, alternate), + choose_ask(&states(heights), 100, height, true, alternate), + expected, + "{why}" + ); + } + } + + #[test] + fn should_ask_for_the_height_or_just_below_it_from_a_state_below_it() { + let lower = 150 - LOWER_ASK_BLOCKS; + let cases = [ + // (verified heights, height, alternate, expected, why) + (&[100][..], 150, false, ask(100, 150), "the height"), + (&[100], 150, true, ask(100, lower), "alternate asks lower"), + (&[100], 110, true, ask(100, 101), "not below the start"), + ( + &[100, 200], + 150, + false, + ask(100, 150), + "only the checkpoint is below", + ), + ( + &[100, 200, 300], + 250, + false, + ask(200, 250), + "verified state below", + ), + (&[100], 100, false, None, "nothing below"), + (&[100], 50, true, None, "under the checkpoint"), + ]; + for (heights, height, alternate, expected, why) in cases { + assert_eq!( + choose_ask(&states(heights), 100, height, false, alternate), expected, "{why}" ); @@ -1352,81 +1445,58 @@ mod tests { } #[test] - fn should_hold_back_heights_down_to_a_window_below_a_failure() { + fn should_hold_back_only_the_height_a_fetch_failed_for() { let fetches = Arc::new(fetches()); - let failure = Slot::Failed { - at: Instant::now(), - failures: 1, - height: 1000, - }; - fetches.lock().fetches.push(((6, [1; 32]), failure)); - assert!(fetches.join_or_start(6, [1; 32], 400).is_err()); - assert!(fetches.join_or_start(6, [1; 32], 399).is_ok()); + fetches + .lock() + .fetches + .push(((6, [1; 32]), failed(Instant::now()))); + assert!(fetches.join_or_start(6, [1; 32], 150).is_err()); + // Another height asks for other targets, so it runs its own fetch. + for height in [149, 151] { + assert!( + fetches.join_or_start_for(6, [1; 32], height, false).is_ok(), + "{height}" + ); + } } #[test] - fn should_keep_a_higher_failures_reach_when_a_lower_fetch_fails() { + fn should_keep_another_heights_failure_when_a_fetch_fails() { let fetches = Arc::new(fetches()); let quorum = (6, [1; 32]); - let long_ago = Instant::now() - .checked_sub(Duration::from_secs(70)) - .expect("the clock has been running longer than the cooldown"); - // Past the longest cooldown, but not yet forgotten. let failure = Slot::Failed { - at: long_ago, + at: Instant::now(), failures: 4, height: 1000, }; fetches.lock().fetches.push((quorum, failure)); - // The cooldown has passed, so a fetch for 500 starts and fails. - let _lower = fetches.join_or_start(6, [1; 32], 500).expect("cooled down"); + let _lower = fetches.join_or_start(6, [1; 32], 500).expect("starts"); let id = running_id(&fetches, quorum, 500).expect("running"); let lower = Attempt { height: 500, - failures: 4, - failed_at: Some(long_ago), ..attempt(quorum, id, holding(&fetches, quorum)) }; fetches.finish(&lower, &Err("failed".to_string()), true); - let recorded = fetches + let mut recorded: Vec<_> = fetches .lock() .fetches .iter() - .find_map(|(_, slot)| match slot { + .filter_map(|(_, slot)| match slot { Slot::Failed { failures, height, .. } => Some((*failures, *height)), Slot::Running { .. } => None, - }); - assert_eq!(recorded, Some((5, 1000))); + }) + .collect(); + recorded.sort(); + assert_eq!(recorded, [(1, 500), (4, 1000)]); assert!( - fetches.join_or_start(6, [1; 32], 900).is_err(), - "900 is still held back" + fetches.join_or_start(6, [1; 32], 1000).is_err(), + "1000 is still held back" ); } - #[test] - fn should_hold_back_only_heights_a_failed_fetch_would_have_covered() { - let fetches = Arc::new(fetches()); - let quorum = (6, [1; 32]); - fetches - .lock() - .fetches - .push((quorum, failed(Instant::now()))); - for height in [150, 100] { - assert!( - fetches.join_or_start(6, [1; 32], height).is_err(), - "{height}" - ); - } - for height in [151, 750] { - assert!( - fetches.join_or_start(6, [1; 32], height).is_ok(), - "{height}" - ); - } - } - #[test] fn should_refuse_new_quorums_while_the_running_fetches_are_full() { let fetches = Arc::new(fetches()); @@ -1456,23 +1526,30 @@ mod tests { } #[test] - fn should_join_a_lower_fetch_only_when_asked_to_and_say_it_may_not_cover() { + fn should_join_another_heights_fetch_only_when_asked_to_and_say_it_is_not_its_own() { // The fetch for 150 keeps its place, and the one for 400 takes the // second of the two, so starting it is not refused. let fetches = Arc::new(fetches()); - let _lower = fetches.join_or_start(6, [1; 32], 150).expect("starts"); + let _first = fetches.join_or_start(6, [1; 32], 150).expect("starts"); let id = running_id(&fetches, (6, [1; 32]), 150); - let (_, covers) = fetches.join_or_start(6, [1; 32], 100).expect("joins"); - assert!(covers, "a fetch for 150 covers 100"); - let (_, covers) = fetches.join_or_start(6, [1; 32], 400).expect("joins"); - assert!(!covers, "a fetch for 150 may prove only up to 150"); - assert_eq!(running_id(&fetches, (6, [1; 32]), 400), None); - let (_, covers) = fetches + let (_, own) = fetches.join_or_start(6, [1; 32], 150).expect("joins"); + assert!(own, "the fetch for 150 is final for 150"); + for height in [100, 400] { + let (_, own) = fetches.join_or_start(6, [1; 32], height).expect("joins"); + assert!(!own, "a fetch for 150 may not bracket {height}"); + assert_eq!(running_id(&fetches, (6, [1; 32]), height), None); + } + let (_, own) = fetches .join_or_start_for(6, [1; 32], 400, false) .expect("its own fetch"); - assert!(covers); + assert!(own); assert!(running_id(&fetches, (6, [1; 32]), 400).is_some()); assert_eq!(running_id(&fetches, (6, [1; 32]), 150), id); + // Without joining others, a caller still joins its own height's fetch. + let (_, own) = fetches + .join_or_start_for(6, [1; 32], 150, false) + .expect("joins its own height"); + assert!(own); } #[test] @@ -1603,26 +1680,95 @@ mod tests { .publish(quorum, [1; 48], &state(200)) .expect("first proof"); assert!(fetches.publish(quorum, [2; 48], &state(300)).is_err()); - assert_eq!(fetches.lookup(6, [9; 32], 800).ok(), Some([1; 48])); + assert_eq!(fetches.lookup(6, [9; 32], 200).ok(), Some([1; 48])); assert!( - fetches.lookup(6, [9; 32], 801).is_err(), - "the conflicting proof must not extend the window" + fetches.lookup(6, [9; 32], 201).is_err(), + "the conflicting proof must not widen the interval" ); assert_eq!(fetches.latest_state(), state(200)); } #[test] - fn should_never_lower_a_quorums_active_height() { + fn should_serve_a_key_only_at_heights_its_targets_bracket() { let fetches = fetches(); let quorum = (6, [9; 32]); - fetches + let served = |heights: &[u32]| { + heights + .iter() + .map(|height| fetches.lookup(6, [9; 32], *height).is_ok()) + .collect::>() + }; + let cached = fetches .publish(quorum, [1; 48], &state(300)) + .expect("first"); + assert_eq!((cached.lo, cached.hi), (300, 300)); + assert_eq!( + served(&[0, 299, 300, 301, 900]), + [false, false, true, false, false] + ); + // A later target widens upward, an earlier one downward; neither is + // ever narrowed. + let cached = fetches + .publish(quorum, [1; 48], &state(350)) .expect("later"); - fetches + assert_eq!((cached.lo, cached.hi), (300, 350)); + let cached = fetches .publish(quorum, [1; 48], &state(200)) .expect("earlier"); - assert_eq!(fetches.lookup(6, [9; 32], 900).ok(), Some([1; 48])); - assert_eq!(fetches.latest_state(), state(300)); + assert_eq!((cached.lo, cached.hi), (200, 350)); + let cached = fetches + .publish(quorum, [1; 48], &state(250)) + .expect("inside"); + assert_eq!((cached.lo, cached.hi), (200, 350)); + assert_eq!( + served(&[199, 200, 275, 350, 351]), + [false, true, true, true, false] + ); + assert_eq!(fetches.latest_state(), state(350)); + } + + #[test] + fn should_refuse_two_blocks_at_one_height() { + let fetches = fetches(); + fetches + .publish((6, [9; 32]), [1; 48], &state(200)) + .expect("first"); + let other = State { + block_hash: [0xee; 32], + ..state(200) + }; + assert!(fetches.publish((6, [8; 32]), [2; 48], &other).is_err()); + assert!(fetches.lookup(6, [8; 32], 200).is_err()); + } + + #[test] + fn should_tell_what_to_ask_from_the_quorums_interval() { + let fetches = fetches(); + let quorum = (6, [9; 32]); + let next = |height: u32, alternate: bool| match fetches.next_ask(quorum, height, alternate) + { + NextAsk::Ask(ask) => Some(ask), + NextAsk::Proven(_) => None, + NextAsk::Unreachable => panic!("reachable"), + }; + // Nothing known: the height itself, or just below it. + assert_eq!(next(400, false), ask(100, 400)); + assert_eq!(next(400, true), ask(100, 400 - LOWER_ASK_BLOCKS)); + fetches + .publish(quorum, [1; 48], &state(300)) + .expect("a target"); + // Above the interval: the tip, which brackets with its low end. + assert_eq!(next(400, false), ask(300, 0)); + assert_eq!(next(400, true), ask(300, 400)); + // Inside it: nothing to ask. + assert_eq!(next(300, false), None); + // Below it: a target at or below the height, from a state below it. + assert_eq!(next(250, false), ask(100, 250)); + assert_eq!(next(250, true), ask(100, 250 - LOWER_ASK_BLOCKS)); + assert!(matches!( + fetches.next_ask(quorum, 100, false), + NextAsk::Unreachable + )); } #[test] diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index 99fbb6eaf6a..b7c788f02a4 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -65,26 +65,46 @@ pub enum VerifiedContextProviderError { /// /// Quorum keys: /// * [`get_quorum_public_key`](ContextProvider::get_quorum_public_key) serves -/// a cached key if the quorum was proven active within a validity window of -/// the requested height (600 blocks on mainnet and testnet). Otherwise it -/// reports the source as unavailable, and the SDK calls +/// a cached key for a Core height only when verified proofs bracket that +/// height: the lowest and highest targets whose quorum roots opened the +/// quorum are at and below, and at and above it. Otherwise it reports the +/// source as unavailable, and the SDK calls /// [`fetch_quorum_public_key`](ContextProvider::fetch_quorum_public_key). -/// * The fetch POSTs to each source's `/proofs`, verifies the proof from the -/// newest verified state, caches the key and returns it. Any failure -/// (timeouts, HTTP errors, oversize or invalid proofs, a proof of another -/// quorum) is reported as an unavailable source, so the SDK moves to -/// another node without banning the one that named the quorum. +/// * The fetch POSTs to each source's `/proofs`, verifies the proof from a +/// verified state below the height, caches the key and returns it once the +/// targets bracket the height. Any failure (timeouts, HTTP errors, oversize +/// or invalid proofs, a proof of another quorum) is reported as an +/// unavailable source, so the SDK moves to another node without banning +/// the one that named the quorum. /// * A quorum of a type other than the network's Platform type is rejected /// as invalid and never fetched. /// +/// A bracket proves the quorum was active at the height. Platform quorums do +/// not rotate: the active set at a height is the newest quorums mined by it, +/// so a quorum enters when it is mined and never returns once it leaves. A +/// quorum in the quorum roots of two targets was therefore active at every +/// height between them. All targets are proven from one checkpoint through +/// ChainLocks, so they lie on one chain. +/// +/// Serving costs about one small proof per new Core height per quorum in use. +/// The first proof asks for the height itself, which Core usually answers +/// with a target exactly there, since Platform's Core height is ChainLocked. +/// Later heights above a proven one usually take a proof of the relay's +/// latest ChainLock, which also covers the next few heights Platform reports. +/// When a relay answers with a target above the height, because that height +/// has no ChainLock of its own, the target is kept as evidence and the relay +/// is asked again for a target up to one DKG interval below the height. +/// /// Fetches are shared per quorum, requests for at most two quorums are in /// flight at once, relays that cannot serve a quorum are left out of its fetch, -/// failed quorums and failing relays cool down, and each fetch has a 70 s -/// deadline. Freshness of responses is the SDK's concern, not the provider's. +/// failing relays and quorums at a height whose fetch failed cool down, and +/// each fetch has a 70 s deadline. Freshness of responses is the SDK's +/// concern, not the provider's. /// /// Keys are proven for the Core height Platform reports. If Platform halts, -/// that height stops while Core moves on, and a quorum not cached before the -/// halt may become unprovable once it leaves Core's quorum root. +/// that height stops while Core moves on; a quorum not cached before the halt +/// is still proven with a proof whose target is that height, as long as the +/// relay's Core still has that ChainLock. /// /// A proof grows with the age of its starting state. Long-lived applications /// should persist [`latest_verified_state`](Self::latest_verified_state) and @@ -247,9 +267,12 @@ impl VerifiedHttpContextProvider { .collect::, _>>()?; let platform_type = network.platform_type(); - // A Platform quorum signs while it is among the newest - // `signing_active_quorum_count` quorums, and one is formed every DKG - // interval; one more interval covers a quorum mined late in it. + // How far apart two heights may be for a running or failed fetch of a + // quorum at one to count for the other: about how long a Platform + // quorum stays active. It is among the newest + // `signing_active_quorum_count` quorums, one is formed every DKG + // interval, and one more interval covers a quorum mined late in it. + // This bounds requests; it never decides which key is served. let params = platform_type.params(); let window = params .dkg_params @@ -432,6 +455,7 @@ impl ContextProvider for VerifiedHttpContextProvider { #[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use super::*; + use crate::proof_fetches::LOWER_ASK_BLOCKS; use dash_core_proof::bootstrap; use dash_core_proof::{parse_commitment, rpc_hex, MAX_PROOF_BYTES}; use futures::future::join_all; @@ -459,6 +483,19 @@ mod tests { Network::Mainnet => include_str!("../tests/data/proof-mainnet.json"), _ => include_str!("../tests/data/proof-testnet.json"), }; + fixture_from(network, json) + } + + /// A second real testnet proof, from another checkpoint to a target two + /// blocks below the testnet fixture's, which opens the same quorum. + fn earlier_fixture() -> Fixture { + fixture_from( + Network::Testnet, + include_str!("../tests/data/proof-testnet-no-links.json"), + ) + } + + fn fixture_from(network: Network, json: &str) -> Fixture { let value: Value = serde_json::from_str(json).expect("fixture JSON"); let state = |key: &str| serde_json::from_value::(value[key].clone()).expect(key); let bootstrap = hex::decode(value["bootstrap_hex"].as_str().expect("bootstrap_hex")) @@ -557,14 +594,19 @@ mod tests { let most_open = Arc::new(AtomicUsize::new(0)); let most = Arc::clone(&most_open); thread::spawn(move || { - for _ in 0..count { + let mut answered = 0; + while answered < count { let Some(mut stream) = accept(&listener) else { return; }; let now_open = open.fetch_add(1, AtomicOrdering::SeqCst) + 1; most.fetch_max(now_open, AtomicOrdering::SeqCst); let guard = Open(Arc::clone(&open)); - let body = read_request(&stream); + // A connection closed before it sent a request is not one. + let Some(body) = read_request(&stream) else { + continue; + }; + answered += 1; let answer = answer_for(&body); lock(&recorded).push((Instant::now(), body)); let Some(answer) = answer else { @@ -690,9 +732,14 @@ mod tests { } } - /// Read one request and return its JSON body. - fn read_request(stream: &TcpStream) -> Value { + /// Read one request and return its JSON body, or nothing if the + /// connection closed before a request arrived. + fn read_request(stream: &TcpStream) -> Option { let mut reader = BufReader::new(stream.try_clone().expect("clone the request stream")); + let mut first = String::new(); + if reader.read_line(&mut first).unwrap_or(0) == 0 { + return None; + } let mut length = 0; loop { let mut line = String::new(); @@ -708,7 +755,7 @@ mod tests { } let mut body = vec![0; length]; reader.read_exact(&mut body).expect("read the request body"); - serde_json::from_slice(&body).expect("a JSON request body") + Some(serde_json::from_slice(&body).expect("a JSON request body")) } /// The URL of a loopback port nothing listens on. @@ -756,6 +803,16 @@ mod tests { .await } + /// Stands in for an earlier proof that opened the quorum at `state`, so + /// that a fetch above it asks for the tip. + fn prove_at(provider: &VerifiedHttpContextProvider, f: &Fixture, state: &State) { + provider + .inner + .fetches + .publish((f.quorum_type, f.quorum_hash), f.key, state) + .expect("earlier evidence"); + } + fn unavailable_reason(result: Result, ContextProviderError>) -> String { match result { Err(ContextProviderError::QuorumSourceUnavailable(reason)) => reason, @@ -810,7 +867,7 @@ mod tests { } #[tokio::test] - async fn should_serve_a_key_inside_the_window_and_prove_it_again_past_it() { + async fn should_serve_a_key_only_at_its_proven_height_and_ask_for_the_tip_above_it() { let f = fixture(Network::Testnet); let relay = MockRelay::spawn(vec![ reply(200, f.bootstrap.clone()), @@ -822,32 +879,145 @@ mod tests { .await .expect("proven"); - assert!(provider - .get_quorum_public_key(f.quorum_type, f.quorum_hash, proven + 600) - .is_ok()); - assert!(provider - .get_quorum_public_key(f.quorum_type, f.quorum_hash, 0) - .is_ok()); - let past = proven + 601; - assert!(matches!( - provider.get_quorum_public_key(f.quorum_type, f.quorum_hash, past), - Err(ContextProviderError::QuorumSourceUnavailable(_)) - )); + for height in [0, proven - 1, proven + 1, proven + 600] { + assert!( + matches!( + provider.get_quorum_public_key(f.quorum_type, f.quorum_hash, height), + Err(ContextProviderError::QuorumSourceUnavailable(_)) + ), + "{height}" + ); + } - // The new proof starts from the verified target, not the checkpoint. - let reason = unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, past).await); + // Above the proven height any later target brackets it, so the + // relay is asked for its tip, from the verified target. + let above = proven + 1; + let reason = + unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, above).await); assert!(reason.contains("HTTP 500: later"), "{reason}"); let requests = relay.requests(); assert_eq!(requests[1]["checkpoint"], rpc_hex(&f.target.block_hash)); - assert_eq!(requests[1]["height"], past); + assert_eq!(requests[1]["height"], 0); assert!( provider - .get_quorum_public_key(f.quorum_type, f.quorum_hash, proven + 600) + .get_quorum_public_key(f.quorum_type, f.quorum_hash, proven) .is_ok(), "a failed proof does not take back an earlier one" ); } + #[tokio::test] + async fn should_bracket_a_height_between_two_real_targets_and_serve_only_inside_them() { + let f = fixture(Network::Testnet); + let earlier = earlier_fixture(); + assert_eq!( + (earlier.quorum_type, earlier.quorum_hash, earlier.key), + (f.quorum_type, f.quorum_hash, f.key) + ); + let (lo, hi) = (earlier.target.height, f.target.height); + assert_eq!(hi - lo, 2); + let relay = MockRelay::spawn(vec![ + reply(200, f.bootstrap.clone()), + reply(200, earlier.bootstrap.clone()), + reply(502, Vec::new()), + ]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let key = fetch(&provider, f.quorum_type, f.quorum_hash, hi).await; + assert_eq!(key.expect("proven at its target"), Some(f.key)); + // The state the earlier proof starts from, as if another quorum's + // proof had verified it. Both fixtures come from one chain. + provider + .inner + .fetches + .publish((f.quorum_type, [0xaa; 32]), [2; 48], &earlier.checkpoint) + .expect("another quorum's proof"); + + // Below the proven height only a target at or below it brackets it, + // so it is asked for from a state below it. + let key = fetch(&provider, f.quorum_type, f.quorum_hash, lo).await; + assert_eq!(key.expect("bracketed by the two targets"), Some(f.key)); + let requests = relay.requests(); + assert_eq!( + requests[1]["checkpoint"], + rpc_hex(&earlier.checkpoint.block_hash) + ); + assert_eq!(requests[1]["height"], lo); + + // Served at every height between the targets, none asked for. + for height in lo..=hi { + let cached = provider.get_quorum_public_key(f.quorum_type, f.quorum_hash, height); + assert_eq!(cached.expect("bracketed"), f.key, "{height}"); + } + for height in [lo - 1, hi + 1] { + assert!( + provider + .get_quorum_public_key(f.quorum_type, f.quorum_hash, height) + .is_err(), + "{height}" + ); + } + // Above the interval the relay is asked for its tip. + unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, hi + 1).await); + let requests = relay.requests(); + assert_eq!(requests[2]["checkpoint"], rpc_hex(&f.target.block_hash)); + assert_eq!(requests[2]["height"], 0); + } + + #[tokio::test] + async fn should_not_serve_a_height_below_a_later_target_and_ask_that_source_lower() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn(vec![ + reply(200, f.bootstrap.clone()), + reply(502, Vec::new()), + ]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + // The relay answers with a target two blocks above the height. + let height = f.target.height - 2; + let reason = + unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, height).await); + assert!(reason.contains(&format!("not at {height}")), "{reason}"); + assert_eq!( + relay.heights(), + [json!(height), json!(height - LOWER_ASK_BLOCKS)] + ); + assert!(provider + .get_quorum_public_key(f.quorum_type, f.quorum_hash, height) + .is_err()); + // The later target is kept as evidence, for its own height. + assert_eq!( + provider + .get_quorum_public_key(f.quorum_type, f.quorum_hash, f.target.height) + .expect("proven at the target"), + f.key + ); + assert_eq!(provider.latest_verified_state(), f.target); + } + + #[tokio::test] + async fn should_serve_a_height_above_earlier_evidence_from_a_tip_proof_and_nothing_below_it() { + let f = fixture(Network::Testnet); + let tip = earlier_fixture(); + let relay = MockRelay::spawn(vec![reply(200, tip.bootstrap.clone())]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let (lo, hi) = (tip.checkpoint.height, tip.target.height); + prove_at(&provider, &f, &tip.checkpoint); + let height = hi - 10; + let key = fetch(&provider, f.quorum_type, f.quorum_hash, height).await; + assert_eq!(key.expect("bracketed by the tip"), Some(f.key)); + assert_eq!(relay.heights(), [json!(0)]); + assert_eq!( + relay.requests()[0]["checkpoint"], + rpc_hex(&tip.checkpoint.block_hash) + ); + let served = |height: u32| { + provider + .get_quorum_public_key(f.quorum_type, f.quorum_hash, height) + .is_ok() + }; + assert!(served(lo) && served(hi)); + assert!(!served(lo - 1) && !served(hi + 1)); + } + #[tokio::test] async fn should_send_one_request_for_concurrent_fetches_of_a_quorum() { let f = fixture(Network::Testnet); @@ -910,7 +1080,7 @@ mod tests { } #[tokio::test] - async fn should_ask_for_the_tip_after_a_bad_gateway() { + async fn should_ask_just_below_the_height_after_a_bad_gateway() { let f = fixture(Network::Testnet); let relay = MockRelay::spawn(vec![ reply(502, b"Requested quorum is not in the target root".to_vec()), @@ -921,7 +1091,23 @@ mod tests { let key = fetch(&provider, f.quorum_type, f.quorum_hash, height).await; assert_eq!(key.expect("a proven key"), Some(f.key)); let heights = relay.heights(); - assert_eq!(heights, [json!(height), json!(0)]); + assert_eq!(heights, [json!(height), json!(height - LOWER_ASK_BLOCKS)]); + } + + #[tokio::test] + async fn should_ask_for_the_height_after_the_tip_gives_a_bad_gateway() { + let f = fixture(Network::Testnet); + let earlier = earlier_fixture(); + let relay = MockRelay::spawn(vec![ + reply(502, b"busy elsewhere".to_vec()), + reply(200, earlier.bootstrap.clone()), + ]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + prove_at(&provider, &f, &earlier.checkpoint); + let height = earlier.target.height; + let key = fetch(&provider, f.quorum_type, f.quorum_hash, height).await; + assert_eq!(key.expect("a proven key"), Some(f.key)); + assert_eq!(relay.heights(), [json!(0), json!(height)]); } #[tokio::test] @@ -936,14 +1122,15 @@ mod tests { let bogus = i32::MAX as u32; let reason = unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, bogus).await); - assert!(reason.contains("latest ChainLock"), "{reason}"); + assert!(reason.contains("proof ends below core height"), "{reason}"); // Neither the quorum nor the relay cools down for the bogus height. let honest = f.target.height; let key = fetch(&provider, f.quorum_type, f.quorum_hash, honest).await; assert_eq!(key.expect("the honest height is proven"), Some(f.key)); let heights = relay.heights(); - assert_eq!(heights, [json!(bogus), json!(0), json!(honest)]); + let lower = bogus - LOWER_ASK_BLOCKS; + assert_eq!(heights, [json!(bogus), json!(lower), json!(honest)]); } #[tokio::test] @@ -960,8 +1147,7 @@ mod tests { }) }); let provider = provider_for(&f, vec![relay.url.clone()], quick()); - // The relay proves the lowest target the lower height accepts. - let lower = f.target.height + 600; + let lower = f.target.height; let higher = lower + 1; let first = fetch(&provider, f.quorum_type, f.quorum_hash, lower); let second = async { @@ -971,10 +1157,15 @@ mod tests { let (first, second) = futures::join!(first, second); assert_eq!(first.expect("the lower height is proven"), Some(f.key)); // The higher caller waited for the lower fetch instead of asking at - // once, then asked for its own height. The fixture proves nothing - // higher, so its own fetch fails; it does not fail for having joined. + // once, then asked for the tip from the target that fetch verified. + // The fixture starts from the checkpoint, so its own fetch fails; it + // does not fail for having joined. let heights = relay.heights(); - assert_eq!(heights[..2], [json!(lower), json!(higher)], "{heights:?}"); + assert_eq!(heights[..2], [json!(lower), json!(0)], "{heights:?}"); + assert_eq!( + relay.requests()[1]["checkpoint"], + rpc_hex(&f.target.block_hash) + ); assert_eq!( relay.most_open(), 1, @@ -1014,13 +1205,55 @@ mod tests { Some(f.key) ); let heights = relay.heights(); + let lowest = lower - LOWER_ASK_BLOCKS; assert_eq!( heights, - [json!(lower), json!(0), json!(height)], + [json!(lower), json!(lowest), json!(height)], "{heights:?}" ); } + #[tokio::test] + async fn should_ask_for_its_own_height_after_a_joined_higher_fetch_does_not_bracket_it() { + let f = fixture(Network::Testnet); + let bootstrap = f.bootstrap.clone(); + let mut answered = 0; + let relay = MockRelay::serve(3, move |_| { + answered += 1; + Some(match answered { + 1 => Answer::Late(Duration::from_millis(500), 200, bootstrap.clone()), + 2 => reply(200, bootstrap.clone()), + _ => reply(502, Vec::new()), + }) + }); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let higher = f.target.height; + let lower = higher - 2; + let first = fetch(&provider, f.quorum_type, f.quorum_hash, higher); + let second = async { + tokio::time::sleep(Duration::from_millis(100)).await; + fetch(&provider, f.quorum_type, f.quorum_hash, lower).await + }; + let (first, second) = futures::join!(first, second); + assert_eq!(first.expect("the higher height is proven"), Some(f.key)); + // The lower caller waited for the higher fetch, whose target does not + // bracket its height, then asked for its own height and just below. + let reason = unavailable_reason(second); + assert!( + reason.contains(&format!("for core height {lower}")), + "{reason}" + ); + let lowest = lower - LOWER_ASK_BLOCKS; + assert_eq!( + relay.heights(), + [json!(higher), json!(lower), json!(lowest)] + ); + assert_eq!(relay.most_open(), 1, "the lower caller did not ask at once"); + assert!(provider + .get_quorum_public_key(f.quorum_type, f.quorum_hash, lower) + .is_err()); + } + #[tokio::test] async fn should_give_a_caller_one_deadline_across_the_fetches_it_waits_on() { let f = fixture(Network::Testnet); @@ -1040,7 +1273,7 @@ mod tests { ..quick() }; let provider = provider_for(&f, vec![relay.url.clone()], timing); - let lower = f.target.height + 600; + let lower = f.target.height; let first = fetch(&provider, f.quorum_type, f.quorum_hash, lower); let started = Instant::now(); let second = fetch(&provider, f.quorum_type, f.quorum_hash, lower + 1); @@ -1117,9 +1350,10 @@ mod tests { unavailable_reason(fetch(&provider, f.quorum_type, [0x11; 32], f.target.height).await); assert!(reason.contains("not asked again"), "{reason}"); assert!(started.elapsed() < Duration::from_secs(20), "{reason}"); + let lower = f.target.height - LOWER_ASK_BLOCKS; for relay in &relays { let heights = relay.heights(); - assert_eq!(heights, [json!(f.target.height), json!(0)], "{reason}"); + assert_eq!(heights, [json!(f.target.height), json!(lower)], "{reason}"); } } @@ -1163,7 +1397,7 @@ mod tests { .expect("fixture JSON"); let behind = hex::decode(behind["bootstrap_hex"].as_str().expect("bootstrap_hex")) .expect("bootstrap hex"); - let lagging = MockRelay::spawn(vec![reply(502, Vec::new()), reply(200, behind)]); + let lagging = MockRelay::spawn(vec![reply(200, behind)]); let proving = MockRelay::spawn(vec![Answer::Late( Duration::from_millis(500), 200, @@ -1174,28 +1408,32 @@ mod tests { ..quick() }; let provider = provider_for(&f, vec![lagging.url.clone(), proving.url.clone()], timing); + // Evidence at the checkpoint's height, which only a stand-in can give, + // so that both relays are asked for their tip from the checkpoint. + prove_at(&provider, &f, &f.checkpoint); // The fixture's target is exactly the lowest one this height accepts. - let height = f.target.height + 600; + let height = f.target.height; let key = fetch(&provider, f.quorum_type, f.quorum_hash, height).await; assert_eq!(key.expect("the other source proves it"), Some(f.key)); - let heights = lagging.heights(); - assert_eq!(heights, [json!(height), json!(0)]); + assert_eq!(lagging.heights(), [json!(0)]); assert_eq!(proving.requests().len(), 1); } #[tokio::test] async fn should_end_a_fetch_once_every_source_is_behind() { let f = fixture(Network::Testnet); - let behind = || vec![reply(502, Vec::new()), reply(200, f.bootstrap.clone())]; + let earlier = earlier_fixture(); + let behind = || vec![reply(200, earlier.bootstrap.clone())]; let relays = [MockRelay::spawn(behind()), MockRelay::spawn(behind())]; let sources = relays.iter().map(|relay| relay.url.clone()).collect(); let provider = provider_for(&f, sources, quick()); - let reason = unavailable_reason( - fetch(&provider, f.quorum_type, f.quorum_hash, i32::MAX as u32).await, - ); + prove_at(&provider, &f, &earlier.checkpoint); + let height = earlier.target.height + 1; + let reason = + unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, height).await); assert_eq!(reason.matches("latest ChainLock").count(), 2, "{reason}"); for relay in &relays { - assert_eq!(relay.requests().len(), 2, "{reason}"); + assert_eq!(relay.heights(), [json!(0)], "{reason}"); } assert_eq!(provider.inner.fetches.cooling(), [false, false]); } @@ -1318,7 +1556,7 @@ mod tests { let relay = MockRelay::spawn(vec![reply(200, f.bootstrap.clone())]); let provider = provider_for(&f, vec![relay.url.clone()], quick()); let quorum = (f.quorum_type, f.quorum_hash); - // A key proven long ago, too long for the requested height. + // A key proven earlier, at a height below the requested one. provider .inner .fetches @@ -1433,6 +1671,14 @@ mod tests { ); } + #[test] + fn should_ask_lower_by_one_dkg_interval_of_the_platform_quorums() { + for network in [Network::Mainnet, Network::Testnet] { + let interval = network.platform_type().params().dkg_params.interval; + assert_eq!(interval, LOWER_ASK_BLOCKS, "{network:?}"); + } + } + #[test] fn should_reject_quorums_it_never_proves() { let f = fixture(Network::Testnet); From aa340487355eece851314ea3f5e919716f946554 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 28 Oct 2026 23:56:47 -0500 Subject: [PATCH 17/25] fix(sdk): serve a proven quorum key for one DKG interval past its highest target Drive moves off its validator quorum only at the end of the first block whose Core height drops that quorum from the active set (validator_set_update_v2 in rs-drive-abci's block_end/validator_set_update/v2/mod.rs). That block reports a Core height at which Core has just rotated its signing quorum out, so under the strict lo <= H <= hi rule its responses could never be verified. A key is now served for Core height H when lo <= H <= hi + 24, where 24 is one DKG interval of the Platform quorum types (DKG_INTERVAL_BLOCKS, which also sets the lowered ask). That proves the quorum was mined by H and active at some height in [H - 24, H]. A target above H is still never evidence for H. A Core-height jump of more than 24 between two Platform blocks leaves that one block unverifiable until the next one; it is reported as an unavailable source, so no node is banned. - One predicate, Cached::serves, decides every lookup, fetch result, joined result, ready path, next ask and verdict. - The tip is asked for only once H is more than 24 past the highest target, and a tip below H - 24 is refused before any signature. - Both ask paths alternate with a target at most 24 below H (kept above the starting state). Any such target that opens the quorum serves H by itself, including on the rotation block, where asking for H or the tip could only return targets the retired quorum is no longer in. - A proof is needed about once per 24 Core blocks per quorum in use. Co-Authored-By: Claude Opus 5.5 --- .../rs-sdk-trusted-context-provider/README.md | 61 +++-- .../src/proof_fetches.rs | 217 +++++++++--------- .../src/verified_provider.rs | 173 +++++++++----- 3 files changed, 265 insertions(+), 186 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/README.md b/packages/rs-sdk-trusted-context-provider/README.md index 3aeffd0103a..295d4f4d9e1 100644 --- a/packages/rs-sdk-trusted-context-provider/README.md +++ b/packages/rs-sdk-trusted-context-provider/README.md @@ -164,25 +164,28 @@ Behaviour: - Mainnet and testnet only. Each provider POSTs to `/proofs` with the checkpoint hash, a height, the quorum hash and type, and `nodeCount: 0`. -- A key is returned for a Core height only when verified proofs bracket that - height: among the targets whose quorum roots opened the quorum, one is at or - below the height and one at or above it. Failures of any kind (timeouts, - HTTP errors, oversize or invalid proofs, proofs of other quorums) are - reported as an unavailable quorum source, so the SDK tries another node - without banning the one that named the quorum. A quorum of another type - than the network's Platform quorum type is rejected as invalid. -- Serving costs about one small proof per new Core height per quorum in use. +- A key is returned for a Core height `H` only when verified proofs show that + the quorum was mined by `H` and active at some height in `[H - 24, H]`: + among the targets whose quorum roots opened the quorum, one is at or below + `H` and one at or above `H - 24` (see the trust model below). Failures of + any kind (timeouts, HTTP errors, oversize or invalid proofs, proofs of + other quorums) are reported as an unavailable quorum source, so the SDK + tries another node without banning the one that named the quorum. A + quorum of another type than the network's Platform quorum type is + rejected as invalid. +- Serving costs about one small proof per 24 Core blocks per quorum in use. The first proof of a quorum asks for the height itself; Core usually answers with a target exactly there, since Platform's Core height is - ChainLocked. Above a proven height the relay is usually asked for its - latest ChainLock, which also covers the next few heights Platform reports. - A relay that answers with a target above the height, because the height - has no ChainLock of its own, is asked again for a target up to one DKG - interval (24 blocks) below it; the later target is kept as evidence. + ChainLocked. Past one DKG interval (24 blocks) above the highest proven + target, the relay is usually asked for its latest ChainLock, which serves + the next 24 heights or more. A relay that answers with a target above the + height, because the height has no ChainLock of its own, is asked again for + a target up to 24 blocks below it, which serves the height by itself; the + later target is kept as evidence. - Fetches of one quorum are shared, and requests for at most two quorums are in flight at once (a fetch waiting out a backoff gives its place up). - A relay that cannot serve a quorum for two asks (the height and its tip, - or the height and just below it) is not asked again in that fetch; a busy + A relay that cannot serve a quorum for two asks (the tip or the height, + then a height 24 blocks below it) is not asked again in that fetch; a busy relay (503) is retried after a backoff; relays that fail otherwise cool down, and so does a quorum at a height whose fetch failed. A fetch gives up after 70 s. Redirects are refused and responses are capped at 1 MiB. @@ -198,12 +201,23 @@ Behaviour: ### Trust model -A bracket proves the quorum was active at the height. Platform quorums -(mainnet type 4, testnet type 6) do not rotate: the active set at a height is -the newest quorums mined by it, so a quorum enters the set when it is mined -and never returns once it leaves. A quorum in the quorum roots of two verified +What a served key proves: the quorum was mined by the response's Core height +`H`, and active at some height in `[H - 24, H]`. Platform quorums (mainnet +type 4, testnet type 6) do not rotate: the active set at a height is the +newest quorums mined by it, so a quorum enters the set when it is mined and +never returns once it leaves. A quorum in the quorum roots of two verified targets was therefore active at every height between them. All targets are -proven from one checkpoint through ChainLocks, so they lie on one chain. +proven from one checkpoint through ChainLocks, so they lie on one chain. A +target above `H` is never evidence that the quorum existed at `H`. + +The 24 blocks, one DKG interval of the Platform quorum types, cover how Drive +rotates validator sets. It moves off its quorum only at the end of the first +block whose Core height drops that quorum from the active set +(`validator_set_update_v2` in +`packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/v2/mod.rs`), +so that block is still signed by a quorum Core has just rotated out at the +height it reports. The Core height can advance between consecutive Platform +blocks, and the rotation applies only after the block. The checkpoint is trust configuration, like a hard-coded block hash: whoever chooses it chooses the chain. Beyond it, the provider trusts that no ChainLock @@ -228,6 +242,13 @@ back to `with_checkpoint` on the next start, so its proofs stay short. as long as the relay's Core has that ChainLock. - A height at or below the checkpoint cannot be proven, since every proof starts from it. +- If the Core height jumps by more than 24 blocks between two Platform blocks, + for example on the first block after a long idle period or a halt, that one + block's responses cannot be verified until the next Platform block. The + provider reports the source as unavailable, so no node is banned. +- A node may name a retired quorum at an old height; it is proven from a + state below that height, possibly the checkpoint. That is bounded by the + same fetch limits and costs only cache churn, not safety. - An inclusion proof cannot show that a quorum does not exist, so the provider never reports a quorum as absent; nodes naming fake quorums are skipped by the SDK, not banned. diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index 196f2984c50..54503a6ecc5 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -10,11 +10,21 @@ //! Platform quorums do not rotate: the active set at a height is the newest //! quorums mined by it, so a quorum enters the set when it is mined and never //! returns once it leaves. A quorum opened under the quorum roots of two -//! verified targets was therefore active at every height between them. A key -//! is served for a Core height only when verified targets bracket it. Every +//! verified targets was therefore active at every height between them. Every //! target is proven from one checkpoint through ChainLocks, so all of them lie //! on one chain. //! +//! A key is served for a Core height `H` when a verified target at or below +//! `H` opened the quorum, so it was mined by `H`, and one at or above `H` +//! minus one DKG interval did too, so it was active at some height in that +//! last interval up to `H`. A target above `H` is never evidence that the +//! quorum existed at `H`. The grace above the highest target is for Drive, +//! which rotates its validator set only at the end of the first block whose +//! Core height drops the current quorum +//! (`rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/v2/mod.rs`): +//! that block reports a height at which Core has just rotated its signing +//! quorum out. +//! //! The SDK treats any error from a fetch as "the source gave no answer", and //! never bans a node for it. A fetch therefore never reports a quorum as //! absent: an inclusion proof cannot show absence, and a relay's "not found" @@ -78,7 +88,7 @@ const MAX_RECENT_STATES: usize = 16; /// The delay before asking a busy relay again doubles this many times. const MAX_RETRY_DOUBLINGS: u32 = 3; /// Answers a source may give that it cannot serve the quorum before a fetch -/// stops asking it: one for each of the two heights [`choose_ask`] alternates +/// stops asking it: one for each of the two asks [`choose_ask`] alternates /// between. const MAX_CANNOT_SERVE: u32 = 2; /// Busy answers in a row after which a fetch stops asking a source. A relay @@ -86,10 +96,12 @@ const MAX_CANNOT_SERVE: u32 = 2; /// one running, and holding a running slot, for its whole deadline. const MAX_BUSY: u32 = 3; /// One DKG interval of the Platform quorum types (mainnet type 4, testnet -/// type 6). A relay asked for a height may answer with a later target when -/// the height has no ChainLock of its own; it is then asked for a target at -/// most this far below the height, which with the later one brackets it. -pub(crate) const LOWER_ASK_BLOCKS: u32 = 24; +/// type 6), in Core blocks. A key is served this far above the highest +/// target that opened it, for the block on which Drive rotates away from a +/// quorum Core has just retired. A relay asked for a height may answer with +/// a later target when the height has no ChainLock of its own; it is then +/// asked for a target at most this far below the height, which serves it. +pub(crate) const DKG_INTERVAL_BLOCKS: u32 = 24; /// How long a fetch waits before trying again to take a place to prove its /// quorum. const PLACE_RETRY: Duration = Duration::from_millis(250); @@ -141,8 +153,8 @@ fn backoff(first: Duration, max: Duration, failures: u32) -> Duration { } /// A proven key and the lowest and highest verified targets whose quorum root -/// opened it. The quorum was active at every height from `lo` to `hi`, and the -/// key is served for those heights only. +/// opened it. The quorum was active at every height from `lo` to `hi`. The +/// key is served from `lo` to one DKG interval past `hi`. #[derive(Clone, Copy, Debug)] pub(crate) struct Cached { key: QuorumKey, @@ -151,14 +163,21 @@ pub(crate) struct Cached { } impl Cached { - /// Whether the proofs behind this key bracket `height`. - fn brackets(&self, height: u32) -> bool { - (self.lo..=self.hi).contains(&height) + /// The highest Core height the key is served for. + fn last_served(&self) -> u32 { + self.hi.saturating_add(DKG_INTERVAL_BLOCKS) + } + + /// Whether the key is served for `height`: the quorum was mined by it and + /// active within one DKG interval before it. Every check of a cached or + /// fetched key goes through this. + fn serves(&self, height: u32) -> bool { + (self.lo..=self.last_served()).contains(&height) } } /// What a fetch hands every caller waiting on it: the key it proved and the -/// heights its proofs bracket, so that a caller does not depend on the cache +/// heights its proofs serve, so that a caller does not depend on the cache /// still holding it. type FetchResult = Result; type FetchFuture = BoxFuture<'static, FetchResult>; @@ -210,20 +229,24 @@ struct Ask { /// Choose what to ask a relay for when proving a quorum for `height`. /// +/// With `alternate`, the relay is asked for a target at most +/// [`DKG_INTERVAL_BLOCKS`] below `height`, kept above the state the proof +/// starts from. Any such target that opens the quorum serves `height`, even +/// on the block where Drive still signs with a quorum Core has just retired. +/// /// `extending` says the cache holds the quorum from a target at or below -/// `height`, so any target at or above `height` brackets it. The relay is then -/// asked for its tip, which also covers the next heights Platform reports, or, -/// with `alternate`, for `height`. The proof starts from the newest verified -/// state below `height`, so it is as short as it can be, but once a state -/// past the checkpoint is verified, neither is asked for from the checkpoint -/// again: the tip is asked for from the latest state instead. +/// `height` that is more than one DKG interval old, so any target at or above +/// `height` minus that interval serves it. The relay is then asked for its +/// tip, which also serves the next heights Platform reports, or the lower +/// target. The proof starts from the newest verified state below `height`, +/// so it is as short as it can be, but once a state past the checkpoint is +/// verified, neither is asked for from the checkpoint again: the tip is asked +/// for from the latest state instead. /// -/// Otherwise a target at or below `height` is needed: exactly at it if -/// nothing is known yet, or anywhere up to it if a later target opened the -/// quorum. Only a state below `height` can lead there, the checkpoint -/// included. The relay is asked for `height`, or, with `alternate`, for a -/// target at most [`LOWER_ASK_BLOCKS`] below it. There is no such ask when no -/// verified state is below `height`. +/// Otherwise a target at or below `height` is needed: the relay is asked for +/// `height`, or the lower target. Only a state below `height` can lead there, +/// the checkpoint included. There is no such ask when no verified state is +/// below `height`. fn choose_ask( states: &BTreeMap, checkpoint_height: u32, @@ -232,19 +255,20 @@ fn choose_ask( alternate: bool, ) -> Option { let below = states.range(..height).next_back(); + // The lowest target that can serve `height`, above the state asked from. + let lowest = |from: u32| height.saturating_sub(DKG_INTERVAL_BLOCKS).max(from + 1); if !extending { let (&from, state) = below?; - let lowest = height.saturating_sub(LOWER_ASK_BLOCKS).max(from + 1); return Some(Ask { checkpoint: state.clone(), - height: if alternate { lowest } else { height }, + height: if alternate { lowest(from) } else { height }, }); } let (_, latest) = states.last_key_value()?; Some(match below { Some((&from, state)) if from != checkpoint_height || latest.height == from => Ask { checkpoint: state.clone(), - height: if alternate { height } else { 0 }, + height: if alternate { lowest(from) } else { 0 }, }, _ => Ask { checkpoint: latest.clone(), @@ -258,8 +282,8 @@ enum Verdict { Proven(Cached), /// The relay cannot serve this quorum for the height asked: a 502, or a /// valid proof that does not serve the request, such as one whose target - /// does not bracket the height. Asked again once, for the other height, - /// after a delay; then left out of the fetch. + /// is above the height. Asked again once, for the other height, after a + /// delay; then left out of the fetch. CannotServe(String), /// The relay's workers are busy: ask again after a backoff. Busy(String), @@ -290,8 +314,8 @@ struct SourceAttempts { ready_at: Instant, cannot_serve: u32, busy: u32, - /// Ask for the other height next time: the tip rather than the height, - /// or a lower height rather than the height. See [`choose_ask`]. + /// Ask for the lower height next time, rather than the tip or the + /// height. See [`choose_ask`]. alternate: bool, dropped: bool, } @@ -315,7 +339,7 @@ enum NextAsk { Ask(Ask), /// The quorum is proven for the height already. Proven(Cached), - /// No proof can bracket the height. + /// No proof can serve the height. Unreachable, } @@ -394,7 +418,7 @@ impl ProofFetches { quorum_type == self.quorum_type && height <= i32::MAX as u32 } - /// The cached key of a quorum, if verified proofs bracket `height`. + /// The cached key of a quorum, if verified proofs serve `height`. pub(crate) fn lookup( &self, quorum_type: u32, @@ -410,12 +434,12 @@ impl ProofFetches { } let mut proven = self.lock(); match proven.keys.get(&(quorum_type, quorum_hash)) { - Some(cached) if cached.brackets(height) => Ok(cached.key), + Some(cached) if cached.serves(height) => Ok(cached.key), Some(cached) => Err(ContextProviderError::QuorumSourceUnavailable(format!( - "quorum {} was proven active at core heights {} to {}, not yet at {height}", + "quorum {} is proven for core heights {} to {}, not yet for {height}", hex::encode(quorum_hash), cached.lo, - cached.hi + cached.last_served() ))), None => Err(ContextProviderError::QuorumSourceUnavailable(format!( "quorum {} is not proven yet", @@ -430,7 +454,7 @@ impl ProofFetches { /// the quorum as absent. /// /// A caller may first join a fetch of the quorum for another height near - /// its own, whose proofs may not bracket the caller's height, or which may + /// its own, whose proofs may not serve the caller's height, or which may /// fail; it then starts a fetch of its own, once, within the same /// deadline. pub(crate) async fn fetch( @@ -467,15 +491,15 @@ impl ProofFetches { ))); } }; - if proven.brackets(height) { + if proven.serves(height) { return Ok(proven.key); } if own || !may_join_other { return Err(unavailable(format!( - "quorum {} was proven active at core heights {} to {}, not at {height}", + "quorum {} is proven for core heights {} to {}, not for {height}", hex::encode(quorum_hash), proven.lo, - proven.hi + proven.last_served() ))); } may_join_other = false; @@ -503,7 +527,7 @@ impl ProofFetches { /// allow, and whether it is the fetch for `height` itself, whose result /// is final for the caller. /// - /// A fetch succeeds only once its proofs bracket its own height, so only + /// A fetch succeeds only once its proofs serve its own height, so only /// a fetch for `height` itself is sure to serve the caller. With /// `may_join_other`, a caller otherwise joins a fetch for a height at /// most a window from its own, and must check what it proved. @@ -531,7 +555,7 @@ impl ProofFetches { }); // A fetch that finished since the caller's lookup has nothing to add. if let Some(cached) = proven.keys.peek(&quorum) { - if cached.brackets(height) { + if cached.serves(height) { return Ok((future::ready(Ok(*cached)).boxed().shared(), true)); } } @@ -909,9 +933,9 @@ impl ProofFetches { /// 503, and proofs that do not verify. A valid proof that does not serve /// the request may be the fault of the unverified height or quorum hash. /// - /// A valid proof whose target does not bracket `height` together with - /// the earlier proofs is still published, as evidence for later asks, but - /// does not prove the key for `height`. + /// A valid proof whose target, together with the earlier proofs, does not + /// serve `height` is still published, as evidence for later asks, but does + /// not prove the key for `height`. fn judge( &self, quorum: QuorumId, @@ -935,15 +959,19 @@ impl ProofFetches { 503 => return Verdict::Busy(format!("HTTP 503{}", body_excerpt(body))), status => return Verdict::SourceFailed(format!("HTTP {status}{}", body_excerpt(body))), } - // A target below the height asked for, or below `height` for the - // tip, can never help. It is refused before any signature, so a bogus - // height costs no BLS work. Only this source is affected if it lies. - let needed = if ask.height == 0 { height } else { ask.height }; + // A target below the height asked for, or more than one DKG interval + // below `height` for the tip, can never help. It is refused before any + // signature, so a bogus height costs no BLS work. Only this source is + // affected if it lies. + let needed = match ask.height { + 0 => height.saturating_sub(DKG_INTERVAL_BLOCKS), + asked => asked, + }; let verified = match bootstrap::verify(body, &ask.checkpoint, needed) { Ok(verified) => verified, Err(error) if error.kind() == ErrorKind::StaleTarget && ask.height == 0 => { return Verdict::BeyondTip(format!( - "its latest ChainLock is below core height {height}" + "its latest ChainLock is below core height {needed}, which {height} needs" )); } Err(error) if error.kind() == ErrorKind::StaleTarget => { @@ -974,13 +1002,13 @@ impl ProofFetches { )); }; match self.publish(quorum, *key, verified.state()) { - Ok(proven) if proven.brackets(height) => Verdict::Proven(proven), + Ok(proven) if proven.serves(height) => Verdict::Proven(proven), Ok(proven) => Verdict::CannotServe(format!( - "proof target {} leaves the quorum proven active at core heights {} to {}, \ - not at {height}", + "proof target {} leaves the quorum proven for core heights {} to {}, not for \ + {height}", verified.state().height, proven.lo, - proven.hi + proven.last_served() )), Err(reason) => Verdict::CannotServe(reason), } @@ -991,7 +1019,7 @@ impl ProofFetches { fn next_ask(&self, quorum: QuorumId, height: u32, alternate: bool) -> NextAsk { let proven = self.lock(); let cached = proven.keys.peek(&quorum); - if let Some(cached) = cached.filter(|cached| cached.brackets(height)) { + if let Some(cached) = cached.filter(|cached| cached.serves(height)) { return NextAsk::Proven(*cached); } let extending = cached.is_some_and(|cached| cached.lo <= height); @@ -1264,45 +1292,17 @@ mod tests { #[test] fn should_ask_for_the_tip_when_extending_without_falling_back_to_the_checkpoint() { + let lower = 250 - DKG_INTERVAL_BLOCKS; let cases = [ // (verified heights, height, alternate, expected, why) (&[100][..], 150, false, ask(100, 0), "the tip, from below"), - ( - &[100], - 150, - true, - ask(100, 150), - "alternate asks for the height", - ), + (&[100], 150, true, ask(100, 126), "alternate asks lower"), (&[100], 100, false, ask(100, 0), "nothing below: the latest"), - ( - &[100, 200], - 150, - false, - ask(200, 0), - "no downgrade to checkpoint", - ), - ( - &[100, 200], - 150, - true, - ask(200, 0), - "not for the height either", - ), - ( - &[100, 200, 300], - 250, - false, - ask(200, 0), - "verified state below", - ), - ( - &[100, 200, 300], - 250, - true, - ask(200, 250), - "alternate from it", - ), + (&[100, 200], 150, false, ask(200, 0), "no downgrade"), + (&[100, 200], 150, true, ask(200, 0), "not for lower either"), + (&[100, 200, 300], 250, false, ask(200, 0), "state below"), + (&[100, 200, 300], 250, true, ask(200, lower), "alternate"), + (&[100, 240], 250, true, ask(240, 241), "above the start"), ]; for (heights, height, alternate, expected, why) in cases { assert_eq!( @@ -1315,7 +1315,7 @@ mod tests { #[test] fn should_ask_for_the_height_or_just_below_it_from_a_state_below_it() { - let lower = 150 - LOWER_ASK_BLOCKS; + let lower = 150 - DKG_INTERVAL_BLOCKS; let cases = [ // (verified heights, height, alternate, expected, why) (&[100][..], 150, false, ask(100, 150), "the height"), @@ -1536,7 +1536,7 @@ mod tests { assert!(own, "the fetch for 150 is final for 150"); for height in [100, 400] { let (_, own) = fetches.join_or_start(6, [1; 32], height).expect("joins"); - assert!(!own, "a fetch for 150 may not bracket {height}"); + assert!(!own, "a fetch for 150 may not serve {height}"); assert_eq!(running_id(&fetches, (6, [1; 32]), height), None); } let (_, own) = fetches @@ -1680,16 +1680,17 @@ mod tests { .publish(quorum, [1; 48], &state(200)) .expect("first proof"); assert!(fetches.publish(quorum, [2; 48], &state(300)).is_err()); - assert_eq!(fetches.lookup(6, [9; 32], 200).ok(), Some([1; 48])); + let last = 200 + DKG_INTERVAL_BLOCKS; + assert_eq!(fetches.lookup(6, [9; 32], last).ok(), Some([1; 48])); assert!( - fetches.lookup(6, [9; 32], 201).is_err(), + fetches.lookup(6, [9; 32], last + 1).is_err(), "the conflicting proof must not widen the interval" ); assert_eq!(fetches.latest_state(), state(200)); } #[test] - fn should_serve_a_key_only_at_heights_its_targets_bracket() { + fn should_serve_a_key_from_its_lowest_target_to_one_interval_past_its_highest() { let fetches = fetches(); let quorum = (6, [9; 32]); let served = |heights: &[u32]| { @@ -1702,9 +1703,10 @@ mod tests { .publish(quorum, [1; 48], &state(300)) .expect("first"); assert_eq!((cached.lo, cached.hi), (300, 300)); + let grace = DKG_INTERVAL_BLOCKS; assert_eq!( - served(&[0, 299, 300, 301, 900]), - [false, false, true, false, false] + served(&[0, 299, 300, 300 + grace, 301 + grace]), + [false, false, true, true, false] ); // A later target widens upward, an earlier one downward; neither is // ever narrowed. @@ -1721,7 +1723,7 @@ mod tests { .expect("inside"); assert_eq!((cached.lo, cached.hi), (200, 350)); assert_eq!( - served(&[199, 200, 275, 350, 351]), + served(&[199, 200, 275, 350 + grace, 351 + grace]), [false, true, true, true, false] ); assert_eq!(fetches.latest_state(), state(350)); @@ -1753,18 +1755,25 @@ mod tests { }; // Nothing known: the height itself, or just below it. assert_eq!(next(400, false), ask(100, 400)); - assert_eq!(next(400, true), ask(100, 400 - LOWER_ASK_BLOCKS)); + assert_eq!(next(400, true), ask(100, 400 - DKG_INTERVAL_BLOCKS)); fetches .publish(quorum, [1; 48], &state(300)) .expect("a target"); - // Above the interval: the tip, which brackets with its low end. + // More than one interval above it: the tip, which serves with its + // low end. assert_eq!(next(400, false), ask(300, 0)); - assert_eq!(next(400, true), ask(300, 400)); - // Inside it: nothing to ask. + assert_eq!(next(400, true), ask(300, 400 - DKG_INTERVAL_BLOCKS)); + // Inside it, or within one interval above it: nothing to ask. assert_eq!(next(300, false), None); + assert_eq!(next(300 + DKG_INTERVAL_BLOCKS, false), None); + assert_eq!( + next(301 + DKG_INTERVAL_BLOCKS, false), + ask(300, 0), + "past the grace" + ); // Below it: a target at or below the height, from a state below it. assert_eq!(next(250, false), ask(100, 250)); - assert_eq!(next(250, true), ask(100, 250 - LOWER_ASK_BLOCKS)); + assert_eq!(next(250, true), ask(100, 250 - DKG_INTERVAL_BLOCKS)); assert!(matches!( fetches.next_ask(quorum, 100, false), NextAsk::Unreachable diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index b7c788f02a4..19707db5641 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -65,35 +65,50 @@ pub enum VerifiedContextProviderError { /// /// Quorum keys: /// * [`get_quorum_public_key`](ContextProvider::get_quorum_public_key) serves -/// a cached key for a Core height only when verified proofs bracket that -/// height: the lowest and highest targets whose quorum roots opened the -/// quorum are at and below, and at and above it. Otherwise it reports the -/// source as unavailable, and the SDK calls +/// a cached key for a Core height `H` only when verified proofs show that +/// the quorum was mined by `H` and active at some height in `[H - 24, H]`: +/// among the targets whose quorum roots opened the quorum, the lowest is at +/// or below `H` and the highest at or above `H - 24`. Otherwise it reports +/// the source as unavailable, and the SDK calls /// [`fetch_quorum_public_key`](ContextProvider::fetch_quorum_public_key). /// * The fetch POSTs to each source's `/proofs`, verifies the proof from a /// verified state below the height, caches the key and returns it once the -/// targets bracket the height. Any failure (timeouts, HTTP errors, oversize +/// targets serve the height. Any failure (timeouts, HTTP errors, oversize /// or invalid proofs, a proof of another quorum) is reported as an /// unavailable source, so the SDK moves to another node without banning /// the one that named the quorum. /// * A quorum of a type other than the network's Platform type is rejected /// as invalid and never fetched. /// -/// A bracket proves the quorum was active at the height. Platform quorums do -/// not rotate: the active set at a height is the newest quorums mined by it, -/// so a quorum enters when it is mined and never returns once it leaves. A -/// quorum in the quorum roots of two targets was therefore active at every -/// height between them. All targets are proven from one checkpoint through -/// ChainLocks, so they lie on one chain. +/// Why this is evidence: Platform quorums do not rotate. The active set at a +/// height is the newest quorums mined by it, so a quorum enters when it is +/// mined and never returns once it leaves. A quorum in the quorum roots of +/// two targets was therefore active at every height between them. All +/// targets are proven from one checkpoint through ChainLocks, so they lie on +/// one chain. A target above `H` is never evidence that the quorum existed at +/// `H`. /// -/// Serving costs about one small proof per new Core height per quorum in use. +/// The 24 blocks, one DKG interval of the Platform quorum types, cover how +/// Drive rotates validator sets: it moves off its quorum only at the end of +/// the first block whose Core height drops that quorum from the active set +/// (`validator_set_update_v2`, in +/// `rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/v2/mod.rs`). +/// That block reports a Core height at which Core has just rotated its +/// signing quorum out. If the Core height jumps by more than 24 between two +/// Platform blocks, for example on the first block after a long idle period +/// or a halt, that one block's responses cannot be verified until the next +/// Platform block; the provider reports the source as unavailable, so no +/// node is banned. +/// +/// Serving costs about one small proof per 24 Core blocks per quorum in use. /// The first proof asks for the height itself, which Core usually answers /// with a target exactly there, since Platform's Core height is ChainLocked. -/// Later heights above a proven one usually take a proof of the relay's -/// latest ChainLock, which also covers the next few heights Platform reports. +/// Past one interval above the highest target, a proof of the relay's latest +/// ChainLock usually follows, which serves the next 24 heights or more. /// When a relay answers with a target above the height, because that height /// has no ChainLock of its own, the target is kept as evidence and the relay -/// is asked again for a target up to one DKG interval below the height. +/// is asked again for a target up to one DKG interval below the height, +/// which serves it by itself. /// /// Fetches are shared per quorum, requests for at most two quorums are in /// flight at once, relays that cannot serve a quorum are left out of its fetch, @@ -455,7 +470,7 @@ impl ContextProvider for VerifiedHttpContextProvider { #[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use super::*; - use crate::proof_fetches::LOWER_ASK_BLOCKS; + use crate::proof_fetches::DKG_INTERVAL_BLOCKS; use dash_core_proof::bootstrap; use dash_core_proof::{parse_commitment, rpc_hex, MAX_PROOF_BYTES}; use futures::future::join_all; @@ -867,7 +882,7 @@ mod tests { } #[tokio::test] - async fn should_serve_a_key_only_at_its_proven_height_and_ask_for_the_tip_above_it() { + async fn should_serve_a_height_up_to_one_dkg_interval_above_the_highest_target() { let f = fixture(Network::Testnet); let relay = MockRelay::spawn(vec![ reply(200, f.bootstrap.clone()), @@ -879,21 +894,20 @@ mod tests { .await .expect("proven"); - for height in [0, proven - 1, proven + 1, proven + 600] { - assert!( - matches!( - provider.get_quorum_public_key(f.quorum_type, f.quorum_hash, height), - Err(ContextProviderError::QuorumSourceUnavailable(_)) - ), - "{height}" - ); + let last = proven + DKG_INTERVAL_BLOCKS; + for height in [proven, proven + 1, last] { + let cached = provider.get_quorum_public_key(f.quorum_type, f.quorum_hash, height); + assert_eq!(cached.expect("within the grace"), f.key, "{height}"); } + assert!(matches!( + provider.get_quorum_public_key(f.quorum_type, f.quorum_hash, last + 1), + Err(ContextProviderError::QuorumSourceUnavailable(_)) + )); - // Above the proven height any later target brackets it, so the - // relay is asked for its tip, from the verified target. - let above = proven + 1; + // Past the grace any target one interval below the height serves it, + // so the relay is asked for its tip, from the verified target. let reason = - unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, above).await); + unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, last + 1).await); assert!(reason.contains("HTTP 500: later"), "{reason}"); let requests = relay.requests(); assert_eq!(requests[1]["checkpoint"], rpc_hex(&f.target.block_hash)); @@ -907,7 +921,28 @@ mod tests { } #[tokio::test] - async fn should_bracket_a_height_between_two_real_targets_and_serve_only_inside_them() { + async fn should_never_serve_a_height_below_the_lowest_target() { + let f = fixture(Network::Testnet); + let relay = MockRelay::spawn(vec![reply(200, f.bootstrap.clone())]); + let provider = provider_for(&f, vec![relay.url.clone()], quick()); + let lo = f.target.height; + fetch(&provider, f.quorum_type, f.quorum_hash, lo) + .await + .expect("proven"); + assert_eq!(provider.cached_quorum_count(), 1); + for height in [0, f.checkpoint.height, lo - 1] { + assert!( + matches!( + provider.get_quorum_public_key(f.quorum_type, f.quorum_hash, height), + Err(ContextProviderError::QuorumSourceUnavailable(_)) + ), + "{height}" + ); + } + } + + #[tokio::test] + async fn should_serve_heights_from_the_lower_of_two_real_targets_to_a_grace_past_the_higher() { let f = fixture(Network::Testnet); let earlier = earlier_fixture(); assert_eq!( @@ -932,10 +967,10 @@ mod tests { .publish((f.quorum_type, [0xaa; 32]), [2; 48], &earlier.checkpoint) .expect("another quorum's proof"); - // Below the proven height only a target at or below it brackets it, - // so it is asked for from a state below it. + // Below the proven height only a target at or below it serves it, so + // it is asked for from a state below it. let key = fetch(&provider, f.quorum_type, f.quorum_hash, lo).await; - assert_eq!(key.expect("bracketed by the two targets"), Some(f.key)); + assert_eq!(key.expect("proven by the lower target"), Some(f.key)); let requests = relay.requests(); assert_eq!( requests[1]["checkpoint"], @@ -943,12 +978,14 @@ mod tests { ); assert_eq!(requests[1]["height"], lo); - // Served at every height between the targets, none asked for. - for height in lo..=hi { + // Served at every height between the targets, none asked for, and up + // to one DKG interval past the higher one. + let last = hi + DKG_INTERVAL_BLOCKS; + for height in lo..=last { let cached = provider.get_quorum_public_key(f.quorum_type, f.quorum_hash, height); - assert_eq!(cached.expect("bracketed"), f.key, "{height}"); + assert_eq!(cached.expect("served"), f.key, "{height}"); } - for height in [lo - 1, hi + 1] { + for height in [lo - 1, last + 1] { assert!( provider .get_quorum_public_key(f.quorum_type, f.quorum_hash, height) @@ -956,8 +993,8 @@ mod tests { "{height}" ); } - // Above the interval the relay is asked for its tip. - unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, hi + 1).await); + // Past the grace the relay is asked for its tip. + unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, last + 1).await); let requests = relay.requests(); assert_eq!(requests[2]["checkpoint"], rpc_hex(&f.target.block_hash)); assert_eq!(requests[2]["height"], 0); @@ -975,10 +1012,10 @@ mod tests { let height = f.target.height - 2; let reason = unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, height).await); - assert!(reason.contains(&format!("not at {height}")), "{reason}"); + assert!(reason.contains(&format!("not for {height}")), "{reason}"); assert_eq!( relay.heights(), - [json!(height), json!(height - LOWER_ASK_BLOCKS)] + [json!(height), json!(height - DKG_INTERVAL_BLOCKS)] ); assert!(provider .get_quorum_public_key(f.quorum_type, f.quorum_hash, height) @@ -1003,7 +1040,7 @@ mod tests { prove_at(&provider, &f, &tip.checkpoint); let height = hi - 10; let key = fetch(&provider, f.quorum_type, f.quorum_hash, height).await; - assert_eq!(key.expect("bracketed by the tip"), Some(f.key)); + assert_eq!(key.expect("served with the tip"), Some(f.key)); assert_eq!(relay.heights(), [json!(0)]); assert_eq!( relay.requests()[0]["checkpoint"], @@ -1014,8 +1051,9 @@ mod tests { .get_quorum_public_key(f.quorum_type, f.quorum_hash, height) .is_ok() }; - assert!(served(lo) && served(hi)); - assert!(!served(lo - 1) && !served(hi + 1)); + let last = hi + DKG_INTERVAL_BLOCKS; + assert!(served(lo) && served(hi) && served(last)); + assert!(!served(lo - 1) && !served(last + 1)); } #[tokio::test] @@ -1087,15 +1125,19 @@ mod tests { reply(200, f.bootstrap.clone()), ]); let provider = provider_for(&f, vec![relay.url.clone()], quick()); - let height = f.target.height; + // The lower ask's target is below the height and serves it alone. + let height = f.target.height + 10; let key = fetch(&provider, f.quorum_type, f.quorum_hash, height).await; assert_eq!(key.expect("a proven key"), Some(f.key)); let heights = relay.heights(); - assert_eq!(heights, [json!(height), json!(height - LOWER_ASK_BLOCKS)]); + assert_eq!( + heights, + [json!(height), json!(height - DKG_INTERVAL_BLOCKS)] + ); } #[tokio::test] - async fn should_ask_for_the_height_after_the_tip_gives_a_bad_gateway() { + async fn should_ask_one_interval_below_the_height_after_the_tip_gives_a_bad_gateway() { let f = fixture(Network::Testnet); let earlier = earlier_fixture(); let relay = MockRelay::spawn(vec![ @@ -1104,10 +1146,13 @@ mod tests { ]); let provider = provider_for(&f, vec![relay.url.clone()], quick()); prove_at(&provider, &f, &earlier.checkpoint); - let height = earlier.target.height; + // As on the block where Drive still signs with a quorum Core has just + // retired: only a target below the height can open it. + let height = earlier.target.height + 10; let key = fetch(&provider, f.quorum_type, f.quorum_hash, height).await; assert_eq!(key.expect("a proven key"), Some(f.key)); - assert_eq!(relay.heights(), [json!(0), json!(height)]); + let lower = height - DKG_INTERVAL_BLOCKS; + assert_eq!(relay.heights(), [json!(0), json!(lower)]); } #[tokio::test] @@ -1129,7 +1174,7 @@ mod tests { let key = fetch(&provider, f.quorum_type, f.quorum_hash, honest).await; assert_eq!(key.expect("the honest height is proven"), Some(f.key)); let heights = relay.heights(); - let lower = bogus - LOWER_ASK_BLOCKS; + let lower = bogus - DKG_INTERVAL_BLOCKS; assert_eq!(heights, [json!(bogus), json!(lower), json!(honest)]); } @@ -1148,7 +1193,8 @@ mod tests { }); let provider = provider_for(&f, vec![relay.url.clone()], quick()); let lower = f.target.height; - let higher = lower + 1; + // Just past what the lower fetch's target serves. + let higher = lower + DKG_INTERVAL_BLOCKS + 1; let first = fetch(&provider, f.quorum_type, f.quorum_hash, lower); let second = async { tokio::time::sleep(Duration::from_millis(100)).await; @@ -1205,7 +1251,7 @@ mod tests { Some(f.key) ); let heights = relay.heights(); - let lowest = lower - LOWER_ASK_BLOCKS; + let lowest = lower - DKG_INTERVAL_BLOCKS; assert_eq!( heights, [json!(lower), json!(lowest), json!(height)], @@ -1214,7 +1260,7 @@ mod tests { } #[tokio::test] - async fn should_ask_for_its_own_height_after_a_joined_higher_fetch_does_not_bracket_it() { + async fn should_ask_for_its_own_height_after_a_joined_higher_fetch_does_not_serve_it() { let f = fixture(Network::Testnet); let bootstrap = f.bootstrap.clone(); let mut answered = 0; @@ -1236,14 +1282,14 @@ mod tests { }; let (first, second) = futures::join!(first, second); assert_eq!(first.expect("the higher height is proven"), Some(f.key)); - // The lower caller waited for the higher fetch, whose target does not - // bracket its height, then asked for its own height and just below. + // The lower caller waited for the higher fetch, whose target is above + // its height, then asked for its own height and just below. let reason = unavailable_reason(second); assert!( reason.contains(&format!("for core height {lower}")), "{reason}" ); - let lowest = lower - LOWER_ASK_BLOCKS; + let lowest = lower - DKG_INTERVAL_BLOCKS; assert_eq!( relay.heights(), [json!(higher), json!(lower), json!(lowest)] @@ -1276,7 +1322,8 @@ mod tests { let lower = f.target.height; let first = fetch(&provider, f.quorum_type, f.quorum_hash, lower); let started = Instant::now(); - let second = fetch(&provider, f.quorum_type, f.quorum_hash, lower + 1); + let higher = lower + DKG_INTERVAL_BLOCKS + 1; + let second = fetch(&provider, f.quorum_type, f.quorum_hash, higher); let (first, second) = futures::join!(first, second); assert!(first.is_ok()); // The lower fetch took 1.5 s; the caller's own fetch gets only what @@ -1350,7 +1397,7 @@ mod tests { unavailable_reason(fetch(&provider, f.quorum_type, [0x11; 32], f.target.height).await); assert!(reason.contains("not asked again"), "{reason}"); assert!(started.elapsed() < Duration::from_secs(20), "{reason}"); - let lower = f.target.height - LOWER_ASK_BLOCKS; + let lower = f.target.height - DKG_INTERVAL_BLOCKS; for relay in &relays { let heights = relay.heights(); assert_eq!(heights, [json!(f.target.height), json!(lower)], "{reason}"); @@ -1411,8 +1458,9 @@ mod tests { // Evidence at the checkpoint's height, which only a stand-in can give, // so that both relays are asked for their tip from the checkpoint. prove_at(&provider, &f, &f.checkpoint); - // The fixture's target is exactly the lowest one this height accepts. - let height = f.target.height; + // The fixture's target is exactly the lowest one this height accepts, + // one DKG interval below it; the lagging one is two blocks lower. + let height = f.target.height + DKG_INTERVAL_BLOCKS; let key = fetch(&provider, f.quorum_type, f.quorum_hash, height).await; assert_eq!(key.expect("the other source proves it"), Some(f.key)); assert_eq!(lagging.heights(), [json!(0)]); @@ -1428,7 +1476,8 @@ mod tests { let sources = relays.iter().map(|relay| relay.url.clone()).collect(); let provider = provider_for(&f, sources, quick()); prove_at(&provider, &f, &earlier.checkpoint); - let height = earlier.target.height + 1; + // One past what the relays' latest ChainLock serves. + let height = earlier.target.height + DKG_INTERVAL_BLOCKS + 1; let reason = unavailable_reason(fetch(&provider, f.quorum_type, f.quorum_hash, height).await); assert_eq!(reason.matches("latest ChainLock").count(), 2, "{reason}"); @@ -1675,7 +1724,7 @@ mod tests { fn should_ask_lower_by_one_dkg_interval_of_the_platform_quorums() { for network in [Network::Mainnet, Network::Testnet] { let interval = network.platform_type().params().dkg_params.interval; - assert_eq!(interval, LOWER_ASK_BLOCKS, "{network:?}"); + assert_eq!(interval, DKG_INTERVAL_BLOCKS, "{network:?}"); } } From 6236bca5d21d99d9d93c1cc1c33bdc016211d09d Mon Sep 17 00:00:00 2001 From: pasta Date: Thu, 29 Oct 2026 00:23:53 -0500 Subject: [PATCH 18/25] build(docker): put llvm-ar on PATH for wasm32 C dependencies The verified context provider brings rs-x11-hash into the wasm-sdk build through dash-core-proof. Its build script archives the wasm32 object with `llvm-ar` by name, and Alpine installs that only under /usr/lib/llvm*/bin, so the dashmate-helper image failed in `cargo chef cook`. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 08839c2f01b..97e256c4215 100644 --- a/Dockerfile +++ b/Dockerfile @@ -164,6 +164,9 @@ RUN if [[ "$TARGETARCH" == "arm64" ]] ; then export PROTOC_ARCH=aarch_64; else e # Note that CC / CXX can be updated later on (eg. when configuring sccache) RUN rm /usr/bin/cc && \ ln -s /usr/bin/clang /usr/bin/cc +# rs-x11-hash archives its wasm32 build with `llvm-ar`, which Alpine keeps off PATH +RUN ln -s "$(ls -d /usr/lib/llvm*/bin | sort -V | tail -n 1)/llvm-ar" /usr/local/bin/llvm-ar && \ + llvm-ar --version RUN <> /root/env echo "export CC='clang'" >> /root/env From 35841dbe53509e7c9e0eaa52dd49bd47faa0944a Mon Sep 17 00:00:00 2001 From: pasta Date: Thu, 29 Oct 2026 02:30:05 -0500 Subject: [PATCH 19/25] ci: run the trusted context provider's tests The crate's native tests, including the verified provider's proof-fetch state machine against a mock relay, were compiled as a dependency but never run: the nextest package list did not name it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests-rs-workspace.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests-rs-workspace.yml b/.github/workflows/tests-rs-workspace.yml index f10d59a301e..9820153d80c 100644 --- a/.github/workflows/tests-rs-workspace.yml +++ b/.github/workflows/tests-rs-workspace.yml @@ -460,6 +460,7 @@ jobs: --package dash-platform-queries \ --package dash-async \ --package dash-core-proof \ + --package rs-sdk-trusted-context-provider \ --package platform-value \ --package platform-encryption \ --package rs-dapi \ From cda0e61ce7c1862e888a1ae5161aadc09ac538f0 Mon Sep 17 00:00:00 2001 From: pasta Date: Thu, 29 Oct 2026 02:45:09 -0500 Subject: [PATCH 20/25] fix(sdk): reject relays that differ only by a trailing slash Sources were compared as parsed base URLs, so `/a` and `/a/` passed as two relays although both post to `/a/proofs`. Compare the endpoints instead. Co-Authored-By: Claude Opus 5.5 --- .../src/verified_provider.rs | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index 19707db5641..10ba2c56bd0 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -267,19 +267,19 @@ impl VerifiedHttpContextProvider { .iter() .map(|source| parse_source(source)) .collect::, _>>()?; - if let Some(repeated) = sources + let endpoints = sources .iter() - .enumerate() - .find_map(|(index, source)| sources[..index].contains(source).then_some(source)) + .map(proofs_endpoint) + .collect::, _>>()?; + // Compared as endpoints: `/a` and `/a/` are one relay. + if let Some(index) = + (0..endpoints.len()).find(|&index| endpoints[..index].contains(&endpoints[index])) { return Err(VerifiedContextProviderError::InvalidSource(format!( - "'{repeated}' is given twice" + "'{}' is given twice", + sources[index] ))); } - let endpoints = sources - .iter() - .map(proofs_endpoint) - .collect::, _>>()?; let platform_type = network.platform_type(); // How far apart two heights may be for a running or failed fetch of a @@ -1799,6 +1799,10 @@ mod tests { with_sources(&["https://relay.example", "https://relay.example/"]).is_err(), "the same relay twice" ); + assert!( + with_sources(&["https://relay.example/a", "https://relay.example/a/"]).is_err(), + "one relay with and without a trailing slash" + ); } #[test] From 8b28cd253562ff20b43413056137186a0aa55ec9 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 31 Oct 2026 00:11:42 +0000 Subject: [PATCH 21/25] fix(sdk): refuse a verified provider cache smaller than the active quorums The SDK drops the key a fetch returns and verifies the response again, reading the key back from the provider's LRU cache. With a cache smaller than the active Platform quorums, a concurrent fetch of another quorum could evict the key in between, and the successful fetch was reported as an unavailable source. VerifiedHttpContextProvider now fails to build with CacheTooSmall below signing_active_quorum_count (24 on mainnet and testnet), which the docs already recommended. Co-Authored-By: Claude Opus 5.5 --- .../rs-sdk-trusted-context-provider/README.md | 6 +-- .../src/verified_provider.rs | 39 +++++++++++++++++-- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/README.md b/packages/rs-sdk-trusted-context-provider/README.md index 295d4f4d9e1..67212b58059 100644 --- a/packages/rs-sdk-trusted-context-provider/README.md +++ b/packages/rs-sdk-trusted-context-provider/README.md @@ -189,9 +189,9 @@ Behaviour: relay (503) is retried after a backoff; relays that fail otherwise cool down, and so does a quorum at a height whose fetch failed. A fetch gives up after 70 s. Redirects are refused and responses are capped at 1 MiB. -- The cache should hold at least the active Platform quorums (24 on mainnet - and testnet); 32 or more avoids proving keys again just because they were - evicted. +- The cache must hold at least the active Platform quorums (24 on mainnet + and testnet), or the provider is not built; 32 or more avoids proving keys + again just because they were evicted. - Data contracts come from the known-contracts cache and the system contracts enabled by cargo features; token configurations from the known cache. - The public mainnet quorum service currently answers `/proofs` with HTTP 403. diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index 10ba2c56bd0..1d62843f8aa 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -52,6 +52,10 @@ pub enum VerifiedContextProviderError { #[error("invalid proof source: {0}")] InvalidSource(String), + /// The key cache cannot hold every active Platform quorum. + #[error("a cache of {given} quorum keys cannot hold the {minimum} active Platform quorums")] + CacheTooSmall { given: usize, minimum: usize }, + /// The HTTP client could not be built. #[error("HTTP client error: {0}")] Http(#[from] reqwest::Error), @@ -211,9 +215,11 @@ impl VerifiedHttpContextProvider { /// checkpoint and asks the network's public quorum service. Makes no /// network request. /// - /// `cache_size` is how many proven quorum keys are kept. It should be at + /// `cache_size` is how many proven quorum keys are kept. It must be at /// least the number of active Platform quorums (24 on mainnet and - /// testnet); 32 or more avoids proving keys again after eviction. + /// testnet), or building fails with + /// [`CacheTooSmall`](VerifiedContextProviderError::CacheTooSmall); 32 or + /// more avoids proving keys again after eviction. pub fn new( network: Network, cache_size: NonZeroUsize, @@ -282,13 +288,22 @@ impl VerifiedHttpContextProvider { } let platform_type = network.platform_type(); + let params = platform_type.params(); + // The SDK reads a key it just fetched back for a second verification. + // A cache smaller than the active quorums could evict it before then. + let minimum = params.signing_active_quorum_count as usize; + if cache_size.get() < minimum { + return Err(VerifiedContextProviderError::CacheTooSmall { + given: cache_size.get(), + minimum, + }); + } // How far apart two heights may be for a running or failed fetch of a // quorum at one to count for the other: about how long a Platform // quorum stays active. It is among the newest // `signing_active_quorum_count` quorums, one is formed every DKG // interval, and one more interval covers a quorum mined late in it. // This bounds requests; it never decides which key is served. - let params = platform_type.params(); let window = params .dkg_params .interval @@ -780,7 +795,7 @@ mod tests { } fn cache_size() -> NonZeroUsize { - NonZeroUsize::new(8).expect("non-zero") + NonZeroUsize::new(32).expect("non-zero") } /// Short retry delays; everything else as in production. @@ -1824,6 +1839,22 @@ mod tests { } } + #[test] + fn should_refuse_a_cache_smaller_than_the_active_quorums() { + for network in [Network::Mainnet, Network::Testnet] { + let too_small = NonZeroUsize::new(23).expect("non-zero"); + assert!(matches!( + VerifiedHttpContextProvider::new(network, too_small), + Err(VerifiedContextProviderError::CacheTooSmall { + given: 23, + minimum: 24 + }) + )); + let smallest = NonZeroUsize::new(24).expect("non-zero"); + assert!(VerifiedHttpContextProvider::new(network, smallest).is_ok()); + } + } + #[test] fn should_build_only_for_mainnet_and_testnet_with_a_matching_checkpoint() { for network in [Network::Devnet, Network::Regtest] { From 076f3f3c8fd63668e948092e7fa67fabb2ffac2c Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 31 Oct 2026 00:11:42 +0000 Subject: [PATCH 22/25] fix(sdk): ask for a retired quorum's lower target from the checkpoint When a quorum was cached from a target at or below the requested height but did not cover it, and the only verified state left below the height was the checkpoint, both asks went to the relay's tip from the latest state. A quorum that had retired since that height is not in the tip's quorum root, so it could no longer be proven once the states between the checkpoint and the height had been dropped, though the same lookup for an uncached quorum would start from the checkpoint. The tip is still asked for first; the second ask is now for a target up to one DKG interval below the height, from the newest state below it. Co-Authored-By: Claude Opus 5.5 --- .../src/proof_fetches.rs | 59 +++++++++++++++++-- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index 54503a6ecc5..75bb5306f24 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -240,8 +240,10 @@ struct Ask { /// tip, which also serves the next heights Platform reports, or the lower /// target. The proof starts from the newest verified state below `height`, /// so it is as short as it can be, but once a state past the checkpoint is -/// verified, neither is asked for from the checkpoint again: the tip is asked -/// for from the latest state instead. +/// verified, the tip is not asked for from the checkpoint again: it is asked +/// for from the latest state instead. The lower target is still asked for +/// from the checkpoint, so a quorum that has retired since `height` can be +/// proven after the states between the checkpoint and `height` were dropped. /// /// Otherwise a target at or below `height` is needed: the relay is asked for /// `height`, or the lower target. Only a state below `height` can lead there, @@ -266,9 +268,15 @@ fn choose_ask( } let (_, latest) = states.last_key_value()?; Some(match below { + // The tip's quorum root may no longer hold a quorum that has retired + // since `height`, so the lower target is asked for from below. + Some((&from, state)) if alternate => Ask { + checkpoint: state.clone(), + height: lowest(from), + }, Some((&from, state)) if from != checkpoint_height || latest.height == from => Ask { checkpoint: state.clone(), - height: if alternate { lowest(from) } else { 0 }, + height: 0, }, _ => Ask { checkpoint: latest.clone(), @@ -1291,7 +1299,7 @@ mod tests { } #[test] - fn should_ask_for_the_tip_when_extending_without_falling_back_to_the_checkpoint() { + fn should_ask_for_the_tip_from_a_later_state_and_the_lower_target_from_below() { let lower = 250 - DKG_INTERVAL_BLOCKS; let cases = [ // (verified heights, height, alternate, expected, why) @@ -1299,7 +1307,13 @@ mod tests { (&[100], 150, true, ask(100, 126), "alternate asks lower"), (&[100], 100, false, ask(100, 0), "nothing below: the latest"), (&[100, 200], 150, false, ask(200, 0), "no downgrade"), - (&[100, 200], 150, true, ask(200, 0), "not for lower either"), + ( + &[100, 200], + 150, + true, + ask(100, 126), + "lower from the checkpoint", + ), (&[100, 200, 300], 250, false, ask(200, 0), "state below"), (&[100, 200, 300], 250, true, ask(200, lower), "alternate"), (&[100, 240], 250, true, ask(240, 241), "above the start"), @@ -1780,6 +1794,41 @@ mod tests { )); } + #[test] + fn should_ask_for_the_lower_target_from_the_checkpoint_after_states_below_were_dropped() { + // A cache that holds every quorum published here. + let fetches = ProofFetches::new( + 6, + 600, + state(100), + vec![Url::parse("https://relay.example").expect("a URL")], + NonZeroUsize::new(32).expect("non-zero"), + ProofTransport::new().expect("a client"), + Timing::default(), + ); + let quorum = (6, [9; 32]); + fetches + .publish(quorum, [1; 48], &state(120)) + .expect("a target"); + // Later proofs of other quorums push the state at 120 out. + for height in 300..300 + MAX_RECENT_STATES as u32 { + fetches + .publish((6, [height as u8; 32]), [2; 48], &state(height)) + .expect("a proof"); + } + assert!(!fetches.lock().states.contains_key(&120)); + let latest = 300 + MAX_RECENT_STATES as u32 - 1; + let next = |alternate| match fetches.next_ask(quorum, 200, alternate) { + NextAsk::Ask(ask) => ask, + _ => panic!("an ask"), + }; + // The tip first, from the latest state; if the quorum has retired, + // the tip cannot open it, so then a target just below the height, + // from the checkpoint. + assert_eq!(Some(next(false)), ask(latest, 0)); + assert_eq!(Some(next(true)), ask(100, 200 - DKG_INTERVAL_BLOCKS)); + } + #[test] fn should_keep_the_checkpoint_and_the_most_recent_states() { let fetches = fetches(); From 97ad09955f28c1fa286f0a04ad645a020cfd7f87 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 31 Oct 2026 00:26:52 +0000 Subject: [PATCH 23/25] fix(sdk): ask for the lower target from a state below it The second ask for a quorum is for a target one DKG interval below the height, but it started from the newest state below the height and was kept above that state. With a state retained within the interval below the height, a quorum that had retired since the lower target could not be reached, though a proof from an older state to the lower target would serve the height. The second ask now starts from the newest state below the lower target, falling back to the newest state below the height when none is kept, on both the extending and the plain path. Co-Authored-By: Claude Opus 5.5 --- .../src/proof_fetches.rs | 52 +++++++++++-------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index 75bb5306f24..4405bf609dd 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -229,21 +229,20 @@ struct Ask { /// Choose what to ask a relay for when proving a quorum for `height`. /// -/// With `alternate`, the relay is asked for a target at most -/// [`DKG_INTERVAL_BLOCKS`] below `height`, kept above the state the proof -/// starts from. Any such target that opens the quorum serves `height`, even -/// on the block where Drive still signs with a quorum Core has just retired. +/// With `alternate`, the relay is asked for a target [`DKG_INTERVAL_BLOCKS`] +/// below `height`, from the newest state below that target, so a quorum that +/// retired between that target and `height` is reached too. Any such target that opens the +/// quorum serves `height`, even on the block where Drive still signs with a +/// quorum Core has just retired. When no state is kept below that target, +/// the lowest target above the newest state below `height` is asked for. /// /// `extending` says the cache holds the quorum from a target at or below /// `height` that is more than one DKG interval old, so any target at or above /// `height` minus that interval serves it. The relay is then asked for its /// tip, which also serves the next heights Platform reports, or the lower -/// target. The proof starts from the newest verified state below `height`, -/// so it is as short as it can be, but once a state past the checkpoint is -/// verified, the tip is not asked for from the checkpoint again: it is asked -/// for from the latest state instead. The lower target is still asked for -/// from the checkpoint, so a quorum that has retired since `height` can be -/// proven after the states between the checkpoint and `height` were dropped. +/// target. The tip is asked for from the newest verified state below +/// `height`, so the proof is as short as it can be, but not from the +/// checkpoint once a later state is verified: from the latest state instead. /// /// Otherwise a target at or below `height` is needed: the relay is asked for /// `height`, or the lower target. Only a state below `height` can lead there, @@ -257,23 +256,25 @@ fn choose_ask( alternate: bool, ) -> Option { let below = states.range(..height).next_back(); - // The lowest target that can serve `height`, above the state asked from. - let lowest = |from: u32| height.saturating_sub(DKG_INTERVAL_BLOCKS).max(from + 1); + if alternate { + // A quorum retired since `lower` is in no quorum root above it. + let lower = height.saturating_sub(DKG_INTERVAL_BLOCKS); + if let Some((&from, state)) = states.range(..lower).next_back().or(below) { + return Some(Ask { + checkpoint: state.clone(), + height: lower.max(from + 1), + }); + } + } if !extending { - let (&from, state) = below?; + let (_, state) = below?; return Some(Ask { checkpoint: state.clone(), - height: if alternate { lowest(from) } else { height }, + height, }); } let (_, latest) = states.last_key_value()?; Some(match below { - // The tip's quorum root may no longer hold a quorum that has retired - // since `height`, so the lower target is asked for from below. - Some((&from, state)) if alternate => Ask { - checkpoint: state.clone(), - height: lowest(from), - }, Some((&from, state)) if from != checkpoint_height || latest.height == from => Ask { checkpoint: state.clone(), height: 0, @@ -1306,6 +1307,7 @@ mod tests { (&[100][..], 150, false, ask(100, 0), "the tip, from below"), (&[100], 150, true, ask(100, 126), "alternate asks lower"), (&[100], 100, false, ask(100, 0), "nothing below: the latest"), + (&[100], 100, true, ask(100, 0), "nothing below either way"), (&[100, 200], 150, false, ask(200, 0), "no downgrade"), ( &[100, 200], @@ -1316,7 +1318,14 @@ mod tests { ), (&[100, 200, 300], 250, false, ask(200, 0), "state below"), (&[100, 200, 300], 250, true, ask(200, lower), "alternate"), - (&[100, 240], 250, true, ask(240, 241), "above the start"), + (&[100, 240], 250, true, ask(100, lower), "from below it"), + ( + &[100, 150, 190], + 200, + true, + ask(150, 200 - DKG_INTERVAL_BLOCKS), + "a quorum retired by 190", + ), ]; for (heights, height, alternate, expected, why) in cases { assert_eq!( @@ -1335,6 +1344,7 @@ mod tests { (&[100][..], 150, false, ask(100, 150), "the height"), (&[100], 150, true, ask(100, lower), "alternate asks lower"), (&[100], 110, true, ask(100, 101), "not below the start"), + (&[100, 140], 150, true, ask(100, lower), "from below it"), ( &[100, 200], 150, From 34381809005357aed11882a168f45bd82631f170 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 31 Oct 2026 03:23:18 +0000 Subject: [PATCH 24/25] fix(sdk): keep the last proof request for the lower ask A fetch sends at most 8 requests and each source's first one is the height (or the tip); only a source's second request asks one DKG interval below. With 8 sources the first asks used the whole budget, so on the block where Drive still signs with a quorum Core has just retired every relay answered 502 and none was asked the target that proves it, though each alone would have. Once a relay cannot serve the first ask, the last request is now kept for the lower ask until one is sent, whatever the number of sources. A source that cannot serve an ask is next asked the other one, rather than the opposite of its last turn. Co-Authored-By: Claude Opus 5.5 --- .../src/proof_fetches.rs | 25 +++++++++---- .../src/verified_provider.rs | 36 +++++++++++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index 4405bf609dd..7cab68e02bc 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -69,7 +69,10 @@ pub(crate) type QuorumKey = [u8; 48]; /// display order). pub(crate) type QuorumId = (u32, [u8; 32]); -/// Most requests one fetch sends, across sources and retries. +/// Most requests one fetch sends, across sources and retries. Once a relay +/// cannot serve the first ask, the last one is kept for the lower ask (see +/// [`choose_ask`]) until that is sent, so that it is reached with any number +/// of sources. const MAX_REQUESTS_PER_FETCH: usize = 8; /// Most requests of one fetch in flight at once: the first and one hedge. const MAX_REQUESTS_IN_FLIGHT: usize = 2; @@ -772,6 +775,10 @@ impl ProofFetches { let mut requests = FuturesUnordered::new(); let mut sources = vec![SourceAttempts::new(started); self.endpoints.len()]; let mut sent = 0; + // Whether a relay could not serve the first ask, and whether the + // lower ask was sent since. + let mut refused = false; + let mut asked_lower = false; let mut hedge_at = started; let mut failures = Vec::new(); let mut waited_for_place = false; @@ -797,8 +804,12 @@ impl ProofFetches { wake = (now + PLACE_RETRY).min(deadline); } Pick::Ready(source) => { - let ask = match self.next_ask(quorum, height, sources[source].alternate) - { + // With more sources than half the budget, each + // would otherwise get the first ask and none the + // lower one. + let alternate = sources[source].alternate + || refused && !asked_lower && sent + 1 == MAX_REQUESTS_PER_FETCH; + let ask = match self.next_ask(quorum, height, alternate) { NextAsk::Ask(ask) => ask, // Another fetch proved it meanwhile. NextAsk::Proven(proven) => return (Ok(proven), sent > 0), @@ -817,11 +828,12 @@ impl ProofFetches { sources[source].asked += 1; sources[source].in_flight = true; sent += 1; + asked_lower |= alternate; hedge_at = now + self.timing.hedge_after; let endpoint = &self.endpoints[source]; requests.push(async move { let reply = self.transport.post(endpoint, body, timeout).await; - (source, ask, now, timeout, reply) + (source, ask, alternate, now, timeout, reply) }); continue; } @@ -854,7 +866,7 @@ impl ProofFetches { Either::Left((Some(answered), _)) => answered, Either::Left((None, _)) | Either::Right(_) => continue, }; - let (source, ask, asked_at, timeout, reply) = answered; + let (source, ask, alternate, asked_at, timeout, reply) = answered; let origin = self.endpoints[source].origin().ascii_serialization(); let elapsed = asked_at.elapsed(); let state = &mut sources[source]; @@ -886,8 +898,9 @@ impl ProofFetches { Verdict::CannotServe(reason) => { state.busy = 0; state.cannot_serve += 1; + refused |= !alternate; // Next time, ask this source for the other height. - state.alternate = !state.alternate; + state.alternate = !alternate; if state.cannot_serve >= MAX_CANNOT_SERVE { state.dropped = true; failures.push(note(&format!("{reason}; not asked again"))); diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index 1d62843f8aa..7eff160a0b7 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -1170,6 +1170,42 @@ mod tests { assert_eq!(relay.heights(), [json!(0), json!(lower)]); } + #[tokio::test] + async fn should_ask_below_the_height_within_the_budget_of_every_supported_source() { + let f = fixture(Network::Testnet); + let height = f.target.height + 10; + let lower = height - DKG_INTERVAL_BLOCKS; + // Each relay alone proves the quorum: its Core refuses the height, + // as on the block where Drive still signs with a just retired quorum, + // and answers the lower ask. + let relays: Vec<_> = (0..MAX_SOURCES) + .map(|_| { + let bootstrap = f.bootstrap.clone(); + MockRelay::serve(2, move |request| { + Some(if request["height"] == json!(lower) { + reply(200, bootstrap.clone()) + } else { + reply(502, b"Requested quorum is not in the target root".to_vec()) + }) + }) + }) + .collect(); + let sources = relays.iter().map(|relay| relay.url.clone()).collect(); + let provider = provider_for(&f, sources, quick()); + let key = fetch(&provider, f.quorum_type, f.quorum_hash, height).await; + assert_eq!(key.expect("a proven key"), Some(f.key)); + let heights: Vec<_> = relays.iter().flat_map(MockRelay::heights).collect(); + assert_eq!(heights.len(), MAX_SOURCES, "{heights:?}"); + assert_eq!( + heights + .iter() + .filter(|asked| **asked == json!(lower)) + .count(), + 1, + "{heights:?}" + ); + } + #[tokio::test] async fn should_not_let_a_bogus_height_hold_back_an_honest_one() { let f = fixture(Network::Testnet); From ba69ea0ed00dd139c068220d59bdf4ac80b72b41 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 31 Oct 2026 04:29:49 +0000 Subject: [PATCH 25/25] test(sdk): order the proving-place test by its places, not by sleeps The test started B and C 100 ms after the first quorum and relied on that landing before the first quorum's 300 ms retry. An executor delayed past both timers could let the retry go first, failing the test though the limit held. The test now drives the first fetch until its 503 is answered and it gives its place up, then drives only B and C until both hold a place, and only then resumes all three. A native fetch runs only while polled, so the order no longer depends on scheduling. Co-Authored-By: Claude Opus 5.5 --- .../src/proof_fetches.rs | 6 ++++ .../src/verified_provider.rs | 31 ++++++++++++++++--- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs index 7cab68e02bc..0136484e2a9 100644 --- a/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs +++ b/packages/rs-sdk-trusted-context-provider/src/proof_fetches.rs @@ -1150,6 +1150,12 @@ impl ProofFetches { } } + /// How many fetches hold one of the places to prove a quorum. + #[cfg(all(test, not(target_arch = "wasm32")))] + pub(crate) fn places(&self) -> usize { + places_taken(&self.lock()) + } + /// Which sources are cooling down. #[cfg(all(test, not(target_arch = "wasm32")))] pub(crate) fn cooling(&self) -> Vec { diff --git a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs index 7eff160a0b7..a8b066c6804 100644 --- a/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs +++ b/packages/rs-sdk-trusted-context-provider/src/verified_provider.rs @@ -491,8 +491,10 @@ mod tests { use futures::future::join_all; use serde_json::{json, Value}; use std::collections::VecDeque; + use std::future::Future; use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; use std::net::{TcpListener, TcpStream}; + use std::pin::{pin, Pin}; use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; use std::thread; use std::time::{Duration, Instant}; @@ -833,6 +835,18 @@ mod tests { .await } + /// Poll `future` until `done` holds, which must be before it finishes. + async fn drive_until(mut future: Pin<&mut F>, done: impl Fn() -> bool) { + let deadline = Instant::now() + Duration::from_secs(5); + while !done() { + assert!(Instant::now() < deadline, "the condition never held"); + tokio::select! { + _ = future.as_mut() => panic!("finished before the condition held"), + () = tokio::time::sleep(Duration::from_millis(1)) => {} + } + } + } + /// Stands in for an earlier proof that opened the quorum at `state`, so /// that a fetch above it asks for the tip. fn prove_at(provider: &VerifiedHttpContextProvider, f: &Fixture, state: &State) { @@ -1563,15 +1577,22 @@ mod tests { }; let provider = provider_for(&f, vec![relay.url.clone()], timing); let height = f.target.height; - let first = fetch(&provider, f.quorum_type, f.quorum_hash, height); - let later = async { - // B and C start while the first quorum waits out the 503. - tokio::time::sleep(Duration::from_millis(100)).await; + let fetches = &provider.inner.fetches; + let mut first = pin!(fetch(&provider, f.quorum_type, f.quorum_hash, height)); + // The first quorum is answered 503 and gives its place up to wait. + drive_until(first.as_mut(), || { + relay.arrivals(f.quorum_hash).len() == 1 && fetches.places() == 0 + }) + .await; + // B and C take both places while it waits; a native fetch runs only + // while polled, so it cannot ask again meanwhile. + let mut later = pin!(async { futures::join!( fetch(&provider, f.quorum_type, b, height), fetch(&provider, f.quorum_type, c, height) ) - }; + }); + drive_until(later.as_mut(), || fetches.places() == 2).await; let (key, (b_result, c_result)) = futures::join!(first, later); assert_eq!(key.expect("proven once a place is free"), Some(f.key)); assert!(b_result.is_err() && c_result.is_err());