From cfbbd7d77b0df41989c0a9a714aa98066b52eefe Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 9 Oct 2026 20:13:25 +0700 Subject: [PATCH 01/14] fix: retain a rolling window of Platform quorum keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebuild the previous quorum list from sampled Core heights, preserve the active set at the cap, memoize public keys, and explicitly exclude secret shares. Poll Core height changes, expose current height and refresh time, and report stale caches as degraded while retaining last good data. Test would have caught this in CI: ✖ before fix, ✔ after. Seven initial regressions failed against 89a3abd; the expanded original-runtime check failed fourteen cases. The unsafe mining-offset step, invalid shipped config, and pre-activation sampling each failed before their corrections. All 45 tests now pass; clippy and fmt have documented baseline failures. Co-Authored-By: GPT-6.1 Sol --- README.md | 109 ++++++- config.toml | 11 +- src/api.rs | 25 +- src/config.rs | 187 ++++++++++- src/main.rs | 7 + src/masternode_cache.rs | 2 +- src/quorum_cache.rs | 293 ++++++++--------- src/quorum_cache_tests.rs | 643 ++++++++++++++++++++++++++++++++++++++ src/quorum_loader.rs | 390 ++++++++++++++--------- 9 files changed, 1343 insertions(+), 324 deletions(-) create mode 100644 src/quorum_cache_tests.rs diff --git a/README.md b/README.md index 6a65996..22b11eb 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # Quorum List Server -A Rust-based HTTP API server that provides RESTful endpoints for managing Dash LLMQ_25_67 quorum lists via RPC calls to Dash Core. +A Rust-based HTTP API server providing the configured network's Platform quorum lists via RPC calls to Dash Core. ## Features -- RPC-based quorum loading from Dash Core (LLMQ_25_67 quorums for testnet) +- RPC-based Platform quorum loading for mainnet, testnet, devnet and regtest - RESTful API for quorum list management - TOML-based configuration with environment variable fallbacks -- Previous quorum state lookup (configurable block offset) +- Stateless rolling window of active and recently retired quorums - Thread-safe shared state - Cross-platform support (Linux/Windows) - Graceful shutdown handling @@ -15,13 +15,74 @@ A Rust-based HTTP API server that provides RESTful endpoints for managing Dash L ## API Endpoints ### Health Check -- `GET /health` - Server health status +- `GET /health` - Quorum cache freshness status ### Quorum Management - `GET /quorums` - Get all current quorums - `GET /quorums/stats` - Get quorum statistics - `GET /quorums/{hash}` - Get specific quorum by hash -- `GET /previous` - Get quorums from previous blocks (configurable offset) +- `GET /previous` - Get the union of Platform quorums active in the retention window + +### Quorum retention and freshness + +`/quorums` keeps its existing `data` array and adds top-level `height` and +`lastUpdated`. These describe the Core height sampled and the Unix timestamp in +seconds of the successful refresh. Repeated reads and failed active-list refreshes +do not advance them. Entries are ordered oldest to newest. + +`/previous` keeps `{ "success": true, "data": { "height": ..., "quorums": [...] } }`. +Its list is rebuilt from Core on every refresh by taking the union of +`quorum listextended` at the tip, then descending by the configured sample step, +including the lower boundary `max(0, tip - R)`. Here +`R = max(retention_blocks, previous_blocks_offset)`: existing offset settings +retain at least their old coverage. `data.height` is now the lowest sampled +height. No membership history is persisted, so a restart reconstructs the same +window and a refresh removes orphaned quorums after a reorg. + +The Core 24 rc.3 Platform parameters are: + +| Network | LLMQ type | Active count A | DKG interval D | Default step | +| --- | --- | --- | --- | --- | +| mainnet | 4 (`llmq_100_67`) | 24 | 24 | 288 | +| testnet | 6 (`llmq_25_67`) | 24 | 24 | 288 | +| devnet | 107 (`llmq_devnet_platform`) | 4 | 24 | 48 | +| regtest | 106 (`llmq_test_platform`) | 2 | 24 | 24 | + +Quorums become active when their commitments are mined. Core allows mining +offsets 10 through 18 within a DKG cycle, so a run can be as short as `A*D - 8` +blocks. Sampling every `A*D` blocks could miss a quorum. The conservative maximum +step is `(A-1)*D + 1`, allowing any mining offset within a cycle. The default is +`max(1, min(A*D/2, (A-1)*D+1))`. Failed DKGs lengthen active runs. Parameter +overrides must match the connected Core; unsafe steps, zero intervals and zero +cache limits are rejected at startup with an error. See Core's +[parameter table](https://github.com/dashpay/dash/blob/v24.0.0-rc.3/src/llmq/params.h) +and [mined commitment scan](https://github.com/dashpay/dash/blob/v24.0.0-rc.3/src/llmq/quorumsman.cpp#L243-L244). + +The rolling list is ordered oldest to newest, with a default cap of 90 to fit +clients' 100-entry caches. Overflow removes the oldest retired quorums first; +active quorums are always retained, even if their count alone exceeds the cap. +Trimming sacrifices coverage at the old end of the window. Size the retention +and cap together. The 48-block default is suitable for 150-second blocks; +accelerated 10-second devnets typically need 360–720 blocks to cover Core/Platform +lag, a Platform block interval, the client's 31-minute metadata tolerance and +clock skew. + +A bounded public-key memo avoids repeat `quorum info` calls for known hashes. +All such calls explicitly use `includeSkShare=false`. Membership is recomputed, +not taken from the memo. All quorum RPC runs on a serialized blocking worker; +requests only read cached data. RPC sockets have a five-second timeout, refreshes +have a 30-second deadline, and a timed-out worker retains its permit until it +finishes, preventing accumulating background workers. + +The service polls `getblockcount` once per second and reloads when the height +changes. The cache TTL bounds reloads even at an unchanged height (including +same-height reorgs), and a poll interval longer than the TTL is reduced to the +TTL. Both lists are attempted independently and keep their last good data on +failure. An empty active list or invalid quorum/key data is a failed refresh, +never an empty success. `/health` still returns HTTP 200 and the same JSON shape; +it returns `success:false`, `data:null` and a degraded message when either list +has never loaded or its last successful refresh is older than +`cache_ttl_seconds * health_stale_intervals` (three intervals by default). ### Masternode bootstrap metadata @@ -67,6 +128,8 @@ validate the node ID and endpoint fields. Existing clients can continue reading ### config.toml ```toml +network = "testnet" + [server] port = 3000 host = "0.0.0.0" @@ -78,6 +141,15 @@ password = "password" [quorum] previous_blocks_offset = 8 +retention_blocks = 48 +cache_ttl_seconds = 60 +poll_interval_seconds = 1 +max_entries = 90 +key_memo_max_entries = 180 +health_stale_intervals = 3 +# sample_step_blocks = 288 +# active_quorum_count = 24 +# dkg_interval = 24 ``` ### Environment Variables (fallbacks) @@ -86,7 +158,24 @@ previous_blocks_offset = 8 - `DASH_RPC_URL` - RPC endpoint (default: http://127.0.0.1:19998) - `DASH_RPC_USER` - RPC username (default: dashrpc) - `DASH_RPC_PASSWORD` - RPC password (default: password) -- `QUORUM_PREVIOUS_BLOCKS_OFFSET` - Previous blocks offset (default: 8) +- `DASH_NETWORK` - mainnet, testnet, devnet or regtest (default: testnet) + +| TOML `[quorum]` key | Environment fallback | Default | +| --- | --- | --- | +| `previous_blocks_offset` | `QUORUM_PREVIOUS_BLOCKS_OFFSET` | 8; lower bound on retention | +| `retention_blocks` | `QUORUM_RETENTION_BLOCKS` | 48 | +| `cache_ttl_seconds` | `QUORUM_CACHE_TTL_SECONDS` | 60; local dashmate networks can use 1 | +| `sample_step_blocks` | `QUORUM_SAMPLE_STEP_BLOCKS` | Derived from A and D as above | +| `active_quorum_count` | `QUORUM_ACTIVE_QUORUM_COUNT` | Network table above | +| `dkg_interval` | `QUORUM_DKG_INTERVAL` | 24 blocks | +| `max_entries` | `QUORUM_MAX_ENTRIES` | 90; active entries are never trimmed | +| `key_memo_max_entries` | `QUORUM_KEY_MEMO_MAX_ENTRIES` | 180 public keys | +| `poll_interval_seconds` | `QUORUM_POLL_INTERVAL_SECONDS` | 1 | +| `health_stale_intervals` | `QUORUM_HEALTH_STALE_INTERVALS` | 3 | + +A valid TOML file takes precedence over all environment variables. When the +file cannot be loaded, environment variables supply fallbacks. Older TOML files +with only `previous_blocks_offset` continue to work with the new defaults. ## Usage @@ -94,7 +183,7 @@ previous_blocks_offset = 8 # Start the server (uses config.toml) cargo run -# With environment variables (overrides config.toml) +# With environment variables (when config.toml is absent) DASH_RPC_URL="http://192.168.1.100:19998" DASH_RPC_USER="myuser" cargo run # Different port @@ -138,7 +227,7 @@ curl http://localhost:3000/quorums # Get quorum stats curl http://localhost:3000/quorums/stats -# Get previous quorums (8 blocks ago by default) +# Get active and recently retired quorums (48-block window by default) curl http://localhost:3000/previous # Get specific quorum by hash @@ -180,9 +269,9 @@ Example quorum response: ## Architecture - **RPC Integration**: Uses `dashcore-rpc` to communicate with Dash Core -- **LLMQ Type**: Only processes LLMQ_25_67 (type 6) quorums for testnet +- **LLMQ Type**: Processes the configured network's Platform quorum type - **Configuration**: TOML-first with environment variable fallbacks -- **State Management**: Thread-safe Arc> for shared state +- **State Management**: Cached lists and freshness metadata published under one lock - **API Framework**: Built with Axum for async HTTP handling ## Development diff --git a/config.toml b/config.toml index 1c4a8de..4696f6d 100644 --- a/config.toml +++ b/config.toml @@ -1,3 +1,5 @@ +network = "testnet" + [server] port = 8080 host = "0.0.0.0" @@ -9,6 +11,9 @@ password = "password" [quorum] previous_blocks_offset = 8 - -[network] -network = "testnet" \ No newline at end of file +retention_blocks = 48 +cache_ttl_seconds = 60 +poll_interval_seconds = 1 +max_entries = 90 +key_memo_max_entries = 180 +health_stale_intervals = 3 diff --git a/src/api.rs b/src/api.rs index aa67fd2..e1c54e0 100644 --- a/src/api.rs +++ b/src/api.rs @@ -26,6 +26,8 @@ pub struct ApiResponse { pub message: Option, #[serde(rename = "lastUpdated", skip_serializing_if = "Option::is_none")] pub last_updated: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub height: Option, } impl ApiResponse { @@ -35,6 +37,7 @@ impl ApiResponse { data: Some(data), message: None, last_updated: None, + height: None, } } @@ -44,6 +47,7 @@ impl ApiResponse { data: None, message: Some(message), last_updated: None, + height: None, } } } @@ -122,17 +126,26 @@ pub fn create_router(quorum_cache: SharedQuorumCache, config: Config, masternode .layer(CompressionLayer::new()) } -async fn health_check() -> Json> { - Json(ApiResponse::success("Quorum List Server is running".to_string())) +async fn health_check( + State((quorum_cache, _, _)): State<(SharedQuorumCache, SharedConfig, SharedMasternodeCache)>, +) -> Json> { + if quorum_cache.is_healthy().await { + Json(ApiResponse::success("Quorum List Server is running".to_string())) + } else { + Json(ApiResponse::error("Quorum List Server is degraded: quorum refresh is stale or unavailable".to_string())) + } } async fn get_all_quorums( State((quorum_cache, _, _)): State<(SharedQuorumCache, SharedConfig, SharedMasternodeCache)>, ) -> Result>>, StatusCode> { - match quorum_cache.get_quorums().await { - Ok(quorums) => { - let quorums: Vec = quorums.iter().map(|entry| entry.into()).collect(); - Ok(Json(ApiResponse::success(quorums))) + match quorum_cache.get_snapshot().await { + Ok(snapshot) => { + let quorums: Vec = snapshot.quorums.iter().map(|entry| entry.into()).collect(); + let mut response = ApiResponse::success(quorums); + response.height = Some(snapshot.height); + response.last_updated = Some(snapshot.last_updated); + Ok(Json(response)) } Err(e) => Ok(Json(ApiResponse::error(format!("Failed to load quorums: {}", e)))) } diff --git a/src/config.rs b/src/config.rs index 661f493..8ed92d2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -68,6 +68,15 @@ impl Network { Network::Regtest => 2443, } } + + /// Platform LLMQ defaults from Dash Core's llmq/params.h. + pub fn platform_quorum_parameters(&self) -> (u32, u32) { + match self { + Network::Mainnet | Network::Testnet => (24, 24), + Network::Devnet => (4, 24), + Network::Regtest => (2, 24), + } + } } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -110,14 +119,35 @@ pub struct RpcConfig { } #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] pub struct QuorumConfig { pub previous_blocks_offset: u32, - #[serde(default = "default_cache_ttl")] pub cache_ttl_seconds: u64, + pub retention_blocks: u32, + pub sample_step_blocks: Option, + pub active_quorum_count: Option, + pub dkg_interval: Option, + pub max_entries: usize, + pub key_memo_max_entries: usize, + pub poll_interval_seconds: u64, + pub health_stale_intervals: u32, } -fn default_cache_ttl() -> u64 { - 60 +impl Default for QuorumConfig { + fn default() -> Self { + Self { + previous_blocks_offset: 8, + cache_ttl_seconds: 60, + retention_blocks: 48, + sample_step_blocks: None, + active_quorum_count: None, + dkg_interval: None, + max_entries: 90, + key_memo_max_entries: 180, + poll_interval_seconds: 1, + health_stale_intervals: 3, + } + } } impl Default for Config { @@ -132,10 +162,7 @@ impl Default for Config { username: "dashrpc".to_string(), password: "password".to_string(), }, - quorum: QuorumConfig { - previous_blocks_offset: 8, - cache_ttl_seconds: default_cache_ttl(), - }, + quorum: QuorumConfig::default(), network: Network::default(), docker: DockerConfig::default(), } @@ -194,6 +221,33 @@ impl Config { } } + macro_rules! quorum_env { + ($name:literal, $field:ident) => { + if let Ok(value) = std::env::var($name) { + config.quorum.$field = value + .parse() + .unwrap_or_else(|_| panic!("Invalid {}", $name)); + } + }; + ($name:literal, $field:ident, optional) => { + if let Ok(value) = std::env::var($name) { + config.quorum.$field = Some( + value + .parse() + .unwrap_or_else(|_| panic!("Invalid {}", $name)), + ); + } + }; + } + quorum_env!("QUORUM_RETENTION_BLOCKS", retention_blocks); + quorum_env!("QUORUM_SAMPLE_STEP_BLOCKS", sample_step_blocks, optional); + quorum_env!("QUORUM_ACTIVE_QUORUM_COUNT", active_quorum_count, optional); + quorum_env!("QUORUM_DKG_INTERVAL", dkg_interval, optional); + quorum_env!("QUORUM_MAX_ENTRIES", max_entries); + quorum_env!("QUORUM_KEY_MEMO_MAX_ENTRIES", key_memo_max_entries); + quorum_env!("QUORUM_POLL_INTERVAL_SECONDS", poll_interval_seconds); + quorum_env!("QUORUM_HEALTH_STALE_INTERVALS", health_stale_intervals); + if let Ok(network_str) = std::env::var("DASH_NETWORK") { config.network = Network::try_from(network_str.as_str()).unwrap_or_else(|e| panic!("{}", e)); @@ -227,6 +281,42 @@ impl Config { self.network.llmq_type_id() } + pub fn quorum_sample_step(&self) -> Result> { + let (active, interval) = self.network.platform_quorum_parameters(); + let active = self.quorum.active_quorum_count.unwrap_or(active); + let interval = self.quorum.dkg_interval.unwrap_or(interval); + let lifetime = active + .checked_mul(interval) + .filter(|value| *value > 0) + .ok_or( + "Quorum active count and DKG interval must be positive and their product fit u32", + )?; + // Commitment mining offsets can differ within a cycle, shortening an active run. + let max_step = (active - 1) * interval + 1; + let step = self + .quorum + .sample_step_blocks + .unwrap_or((lifetime / 2).min(max_step).max(1)); + if step == 0 || step > max_step { + return Err(format!("Quorum sample step {step} must be between 1 and {max_step}: commitment mining offsets can shorten active runs").into()); + } + if self.quorum.cache_ttl_seconds == 0 + || self.quorum.poll_interval_seconds == 0 + || self.quorum.health_stale_intervals == 0 + || self.quorum.max_entries == 0 + || self.quorum.key_memo_max_entries == 0 + { + return Err("Quorum intervals and cache limits must be positive".into()); + } + Ok(step) + } + + pub fn quorum_retention(&self) -> u32 { + self.quorum + .retention_blocks + .max(self.quorum.previous_blocks_offset) + } + pub fn get_dapi_port(&self) -> u16 { self.network.dapi_port() } @@ -301,4 +391,87 @@ previous_blocks_offset = 8 assert_eq!(config.get_llmq_type(), "llmq_devnet_platform"); assert_eq!(config.get_llmq_type_id(), 107); } + + #[test] + fn should_default_legacy_configs_to_a_safe_rolling_window() { + let quorum: QuorumConfig = toml::from_str("previous_blocks_offset = 8").unwrap(); + assert_eq!(quorum.retention_blocks, 48); + assert_eq!(quorum.cache_ttl_seconds, 60); + assert_eq!(quorum.max_entries, 90); + for (network, expected) in [ + (Network::Mainnet, 288), + (Network::Testnet, 288), + (Network::Devnet, 48), + (Network::Regtest, 24), + ] { + let mut config = Config { + network, + ..Config::default() + }; + assert_eq!(config.quorum_sample_step().unwrap(), expected); + config.quorum.active_quorum_count = Some(1); + assert_eq!(config.quorum_sample_step().unwrap(), 1); + } + } + + #[test] + fn should_reject_invalid_quorum_intervals_and_limits() { + let mut config = Config::default(); + config.quorum.sample_step_blocks = Some(0); + assert!(config.quorum_sample_step().is_err()); + config.quorum.sample_step_blocks = None; + config.quorum.active_quorum_count = Some(0); + assert!(config.quorum_sample_step().is_err()); + config.quorum.active_quorum_count = Some(u32::MAX); + assert!(config.quorum_sample_step().is_err()); + config.quorum.active_quorum_count = None; + config.quorum.cache_ttl_seconds = 0; + assert!(config.quorum_sample_step().is_err()); + } + + #[test] + fn should_preserve_quorum_env_fallbacks() { + let variables = [ + ("QUORUM_PREVIOUS_BLOCKS_OFFSET", "96"), + ("QUORUM_CACHE_TTL_SECONDS", "1"), + ("QUORUM_RETENTION_BLOCKS", "720"), + ("QUORUM_SAMPLE_STEP_BLOCKS", "12"), + ("QUORUM_ACTIVE_QUORUM_COUNT", "4"), + ("QUORUM_DKG_INTERVAL", "24"), + ("QUORUM_MAX_ENTRIES", "80"), + ("QUORUM_KEY_MEMO_MAX_ENTRIES", "160"), + ("QUORUM_POLL_INTERVAL_SECONDS", "2"), + ("QUORUM_HEALTH_STALE_INTERVALS", "5"), + ]; + let original: Vec<_> = variables + .iter() + .map(|(name, _)| (*name, std::env::var_os(name))) + .collect(); + for (name, value) in variables { + std::env::set_var(name, value); + } + let config = Config::load_from_env_or_file("/nonexistent/qls16-config.toml"); + for (name, value) in original { + if let Some(value) = value { + std::env::set_var(name, value); + } else { + std::env::remove_var(name); + } + } + assert_eq!(config.quorum.previous_blocks_offset, 96); + assert_eq!(config.quorum.cache_ttl_seconds, 1); + assert_eq!(config.quorum_retention(), 720); + assert_eq!(config.quorum_sample_step().unwrap(), 12); + assert_eq!(config.quorum.max_entries, 80); + assert_eq!(config.quorum.key_memo_max_entries, 160); + assert_eq!(config.quorum.poll_interval_seconds, 2); + assert_eq!(config.quorum.health_stale_intervals, 5); + } + + #[test] + fn should_load_the_shipped_config_file() { + let config: Config = toml::from_str(include_str!("../config.toml")).unwrap(); + assert_eq!(config.network, Network::Testnet); + assert_eq!(config.quorum_sample_step().unwrap(), 288); + } } diff --git a/src/main.rs b/src/main.rs index ab1cf9f..86ed095 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,9 @@ mod masternode_loader; mod masternode_cache; mod grpc_client; +#[cfg(test)] +mod quorum_cache_tests; + use config::Config; use masternode_cache::MasternodeCache; use quorum_cache::QuorumCache; @@ -21,6 +24,10 @@ async fn main() -> Result<(), Box> { // Load configuration let config = Config::load_from_env_or_file("config.toml"); + if let Err(error) = config.quorum_sample_step() { + eprintln!("Invalid quorum configuration: {error}"); + return Err(std::io::Error::other(error.to_string()).into()); + } println!("Configuration loaded:"); println!(" Server: {}:{}", config.server.host, config.server.port); println!(" RPC: {} (user: {})", config.rpc.url, config.rpc.username); diff --git a/src/masternode_cache.rs b/src/masternode_cache.rs index 70f89bb..3f17ec6 100644 --- a/src/masternode_cache.rs +++ b/src/masternode_cache.rs @@ -323,7 +323,7 @@ mod tests { assert_eq!(result["success"], false); assert!(result.get("lastUpdated").is_none()); let health = response(config, cache, "/health").await; - assert_eq!(health["success"], true); + assert_eq!(health["success"], false); assert!(health.get("lastUpdated").is_none()); } } diff --git a/src/quorum_cache.rs b/src/quorum_cache.rs index fd4940b..dede7e8 100644 --- a/src/quorum_cache.rs +++ b/src/quorum_cache.rs @@ -1,16 +1,27 @@ use crate::config::Config; use crate::quorum_list::QuorumList; -use crate::quorum_loader; -use chrono::Local; -use std::sync::{Arc, RwLock}; +use crate::quorum_loader::{self, KeyMemo, LoadResult, REFRESH_TIMEOUT}; +use std::sync::{Arc, Mutex, RwLock}; use std::time::{Duration, Instant}; -use tokio::sync::Mutex; + +#[derive(Clone)] +pub struct QuorumSnapshot { + pub quorums: QuorumList, + pub height: u32, + pub last_updated: i64, + refreshed: Instant, +} + +#[derive(Default)] +struct CacheState { + current: Option, + previous: Option, +} pub struct QuorumCache { - current_quorums: Arc>>, - previous_quorums: Arc>>, - current_last_update: Arc>>, - previous_last_update: Arc>>, + state: RwLock, + memo: Arc>, + rpc_worker: Arc>, config: Arc, update_interval: Duration, } @@ -19,170 +30,164 @@ impl QuorumCache { pub fn new(config: Config) -> Self { let ttl = config.quorum.cache_ttl_seconds; Self { - current_quorums: Arc::new(RwLock::new(None)), - previous_quorums: Arc::new(RwLock::new(None)), - current_last_update: Arc::new(Mutex::new(None)), - previous_last_update: Arc::new(Mutex::new(None)), + state: RwLock::new(CacheState::default()), + memo: Arc::new(Mutex::new(KeyMemo::default())), + rpc_worker: Arc::new(tokio::sync::Mutex::new(())), config: Arc::new(config), update_interval: Duration::from_secs(ttl), } } - /// Read the cached current quorums. - /// - /// This never contacts Dash Core: the background refresh task is the only - /// writer. Serving a request must not depend on RPC latency. - pub async fn get_quorums( - &self, - ) -> Result> { - let data = self - .current_quorums + /// Reads the list and its metadata together without contacting Core. + pub async fn get_snapshot(&self) -> LoadResult { + self.state .read() - .map_err(|_| "Failed to read quorum cache")?; - Ok(data - .as_ref() - .ok_or("Quorum cache is not populated yet")? - .clone()) + .map_err(|_| "Failed to read quorum cache")? + .current + .clone() + .ok_or_else(|| "Quorum cache is not populated yet".into()) + } + + pub async fn get_quorums(&self) -> LoadResult { + Ok(self.get_snapshot().await?.quorums) } - /// Read the cached previous quorums along with the height they were taken at. - /// - /// As with [`Self::get_quorums`], this is a pure cache read. - pub async fn get_previous_quorums( - &self, - ) -> Result<(u32, QuorumList), Box> { - let data = self - .previous_quorums + /// Returns the rolling union and the lowest sampled Core height. + pub async fn get_previous_quorums(&self) -> LoadResult<(u32, QuorumList)> { + let state = self + .state .read() - .map_err(|_| "Failed to read previous quorum cache")?; - Ok(data + .map_err(|_| "Failed to read quorum cache")?; + let previous = state + .previous .as_ref() - .ok_or("Previous quorum cache is not populated yet")? - .clone()) + .ok_or("Previous quorum cache is not populated yet")?; + Ok((previous.height, previous.quorums.clone())) } - /// Refresh both quorum caches from Dash Core. - /// - /// Called once at startup and then only by the background refresh task. - pub async fn refresh(&self) -> Result<(), Box> { - self.update_current_cache().await?; - self.update_previous_cache().await?; - Ok(()) + /// The permit remains with the blocking worker even if its async waiter times out. + async fn rpc_work(&self, work: F) -> LoadResult + where + T: Send + 'static, + F: FnOnce() -> LoadResult + Send + 'static, + { + tokio::time::timeout(REFRESH_TIMEOUT, async { + let permit = self.rpc_worker.clone().lock_owned().await; + tokio::task::spawn_blocking(move || { + let _permit = permit; + work() + }) + .await? + }) + .await + .map_err(|_| "Quorum RPC worker timed out")? } - async fn update_current_cache(&self) -> Result<(), Box> { - println!("Updating quorum cache..."); - - let result = tokio::time::timeout( - tokio::time::Duration::from_secs(30), - quorum_loader::load_initial_quorums(&self.config), - ) - .await; - - match result { - Ok(Ok(quorums)) => { - { - let mut data = self - .current_quorums - .write() - .map_err(|_| "Failed to write to quorum cache")?; - *data = Some(quorums); - } - { - let mut last_update = self.current_last_update.lock().await; - *last_update = Some(Instant::now()); - } - println!("Quorum cache updated successfully"); - Ok(()) + /// Both lists are attempted; failures preserve each list's last good snapshot. + pub async fn refresh(&self) -> LoadResult<()> { + let config = self.config.clone(); + let memo = self.memo.clone(); + let result = self + .rpc_work(move || { + let mut memo = memo.lock().map_err(|_| "Failed to lock quorum key memo")?; + quorum_loader::refresh(&config, &mut memo) + }) + .await?; + let now = Instant::now(); + let last_updated = chrono::Utc::now().timestamp(); + let mut state = self + .state + .write() + .map_err(|_| "Failed to write quorum cache")?; + let current_error = match result.current { + Ok(quorums) => { + state.current = Some(QuorumSnapshot { + quorums, + height: result.tip, + last_updated, + refreshed: now, + }); + None } - Ok(Err(e)) => Err(e), - Err(_) => { - eprintln!("Quorum cache update timed out after 30 seconds"); - Err("Quorum cache update timed out after 30 seconds".into()) + Err(error) => Some(error), + }; + let previous_error = match result.previous { + Ok((height, quorums)) => { + state.previous = Some(QuorumSnapshot { + quorums, + height, + last_updated, + refreshed: now, + }); + None + } + Err(error) => Some(error), + }; + match (current_error, previous_error) { + (None, None) => Ok(()), + (Some(error), None) | (None, Some(error)) => Err(error), + (Some(current), Some(previous)) => { + Err(format!("Current quorums: {current}; rolling window: {previous}").into()) } } } - async fn update_previous_cache(&self) -> Result<(), Box> { - println!("Updating previous quorum cache..."); - - let result = tokio::time::timeout( - tokio::time::Duration::from_secs(30), - self.fetch_previous_quorums(), - ) - .await; - - match result { - Ok(Ok((height, quorums))) => { - { - let mut data = self - .previous_quorums - .write() - .map_err(|_| "Failed to write to previous quorum cache")?; - *data = Some((height, quorums)); - } - { - let mut last_update = self.previous_last_update.lock().await; - *last_update = Some(Instant::now()); - } - println!("Previous quorum cache updated successfully (height: {})", height); - Ok(()) - } - Ok(Err(e)) => Err(e), - Err(_) => { - eprintln!("Previous quorum cache update timed out after 30 seconds"); - Err("Previous quorum cache update timed out after 30 seconds".into()) + pub async fn is_healthy(&self) -> bool { + let stale_after = self + .update_interval + .saturating_mul(self.config.quorum.health_stale_intervals); + let Ok(state) = self.state.read() else { + return false; + }; + match (&state.current, &state.previous) { + (Some(current), Some(previous)) => { + current.refreshed.elapsed() < stale_after + && previous.refreshed.elapsed() < stale_after } + _ => false, } } - async fn fetch_previous_quorums( - &self, - ) -> Result<(u32, QuorumList), Box> { - let current_height = quorum_loader::get_current_block_height(&self.config).await?; - let previous_height = if current_height >= self.config.quorum.previous_blocks_offset { - current_height - self.config.quorum.previous_blocks_offset - } else { - 0 - }; - let quorums = quorum_loader::load_quorums_at_height(&self.config, previous_height).await?; - Ok((previous_height, quorums)) + fn reload_due(&self, tip: u32) -> LoadResult { + let state = self + .state + .read() + .map_err(|_| "Failed to read quorum cache")?; + Ok(match (&state.current, &state.previous) { + (Some(current), Some(previous)) => { + current.height != tip + || current.refreshed.elapsed() >= self.update_interval + || previous.refreshed.elapsed() >= self.update_interval + || previous.height != tip.saturating_sub(self.config.quorum_retention()) + } + _ => true, + }) } pub async fn start_background_refresh(self: Arc) { tokio::spawn(async move { + let poll_interval = Duration::from_secs(self.config.quorum.poll_interval_seconds) + .min(self.update_interval) + .max(Duration::from_millis(1)); + let mut poll = tokio::time::interval(poll_interval); + poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { - tokio::time::sleep(self.update_interval).await; - let now = Local::now(); - println!( - "🔄 [{}] Background refresh: Starting quorum cache update...", - now.format("%Y-%m-%d %H:%M:%S") - ); - - let current_result = self.update_current_cache().await; - let previous_result = self.update_previous_cache().await; - - match (current_result, previous_result) { - (Ok(_), Ok(_)) => println!( - "✅ [{}] Background refresh: Quorum caches updated successfully", - Local::now().format("%Y-%m-%d %H:%M:%S") - ), - (Err(e1), Err(e2)) => eprintln!( - "❌ [{}] Background refresh: Failed to update quorum caches: current={}, previous={}", - Local::now().format("%Y-%m-%d %H:%M:%S"), - e1, - e2 - ), - (Err(e), Ok(_)) => eprintln!( - "⚠️ [{}] Background refresh: Failed to update current quorum cache: {}", - Local::now().format("%Y-%m-%d %H:%M:%S"), - e - ), - (Ok(_), Err(e)) => eprintln!( - "⚠️ [{}] Background refresh: Failed to update previous quorum cache: {}", - Local::now().format("%Y-%m-%d %H:%M:%S"), - e - ), + poll.tick().await; + let config = self.config.clone(); + match self + .rpc_work(move || quorum_loader::get_current_block_height(&config)) + .await + { + Ok(tip) => match self.reload_due(tip) { + Ok(true) => { + if let Err(error) = self.refresh().await { + eprintln!("Quorum refresh failed: {error}"); + } + } + Ok(false) => {} + Err(error) => eprintln!("Cannot read quorum refresh state: {error}"), + }, + Err(error) => eprintln!("Core tip poll failed: {error}"), } } }); diff --git a/src/quorum_cache_tests.rs b/src/quorum_cache_tests.rs new file mode 100644 index 0000000..751b550 --- /dev/null +++ b/src/quorum_cache_tests.rs @@ -0,0 +1,643 @@ +use crate::config::{Config, Network}; +use crate::quorum_cache::QuorumCache; +use serde_json::{json, Value}; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +#[derive(Clone)] +pub struct Quorum { + pub id: u32, + pub created: u32, + pub retired: u32, +} + +pub struct CoreState { + pub tip: u32, + pub quorums: Vec, + pub calls: Vec, + pub fail: bool, + pub fail_current: bool, + pub malformed: bool, + pub empty: bool, + pub fail_height: Option, + pub fail_current_once: bool, + pub delay: Duration, + pub omitted_type_before: Option, +} + +pub struct MockCore { + pub config: Config, + pub state: Arc>, + stop: Arc, + worker: Option>, +} + +impl MockCore { + pub fn new(tip: u32, quorums: Vec) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let mut config = Config { + network: Network::Devnet, + ..Config::default() + }; + config.rpc.url = format!("http://{}", listener.local_addr().unwrap()); + let state = Arc::new(Mutex::new(CoreState { + tip, + quorums, + calls: Vec::new(), + fail: false, + fail_current: false, + malformed: false, + empty: false, + fail_height: None, + fail_current_once: false, + delay: Duration::ZERO, + omitted_type_before: None, + })); + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let worker_state = state.clone(); + let worker_stop = stop.clone(); + let worker = std::thread::spawn(move || { + while !worker_stop.load(std::sync::atomic::Ordering::Relaxed) { + let (mut stream, _) = match listener.accept() { + Ok(connection) => connection, + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(1)); + continue; + } + Err(e) => panic!("mock RPC accept failed: {e}"), + }; + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut bytes = Vec::new(); + let (header_end, length) = loop { + let mut buf = [0; 4096]; + let read = stream.read(&mut buf).unwrap(); + assert!(read > 0); + bytes.extend_from_slice(&buf[..read]); + if let Some(pos) = bytes.windows(4).position(|w| w == b"\r\n\r\n") { + let headers = String::from_utf8_lossy(&bytes[..pos]); + let length = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + .unwrap(); + break (pos + 4, length); + } + }; + while bytes.len() < header_end + length { + let mut buf = [0; 4096]; + let read = stream.read(&mut buf).unwrap(); + assert!(read > 0); + bytes.extend_from_slice(&buf[..read]); + } + let request: Value = + serde_json::from_slice(&bytes[header_end..header_end + length]).unwrap(); + let mut state = worker_state.lock().unwrap(); + state.calls.push(request.clone()); + let height = request["params"][1] + .as_u64() + .map(|h| h as u32) + .unwrap_or(state.tip); + let is_list = request["params"][0] == "listextended"; + let fail_once = state.fail_current_once && is_list && height == state.tip; + if fail_once { + state.fail_current_once = false; + } + let failed = state.fail + || fail_once + || (is_list && state.fail_height == Some(height)) + || (state.fail_current && is_list && height == state.tip); + let result = if request["method"] == "getblockcount" { + json!(state.tip) + } else if request["params"][0] == "listextended" { + if state + .omitted_type_before + .is_some_and(|activation| height < activation) + { + json!({"llmq_other": []}) + } else if state.malformed { + json!({}) + } else { + let entries: Vec = state.quorums.iter() + .filter(|q| !state.empty && q.created <= height && height < q.retired) + .map(|q| json!({hash(q.id): {"creationHeight":q.created, "numValidMembers":12}})) + .collect(); + json!({"llmq_devnet_platform": entries}) + } + } else { + json!({"quorumPublicKey": "11".repeat(48)}) + }; + let body = if failed { + json!({"result":null, "error":{"code":-1,"message":"Core unavailable"}, "id":request["id"]}) + } else { + json!({"result":result, "error":null, "id":request["id"]}) + }.to_string(); + let delay = state.delay; + drop(state); + std::thread::sleep(delay); + let response = format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body); + let _ = stream.write_all(response.as_bytes()); + } + }); + Self { + config, + state, + stop, + worker: Some(worker), + } + } + + pub fn info_calls(&self) -> Vec { + self.state + .lock() + .unwrap() + .calls + .iter() + .filter(|r| r["method"] == "quorum" && r["params"][0] == "info") + .cloned() + .collect() + } +} + +impl Drop for MockCore { + fn drop(&mut self) { + self.stop.store(true, std::sync::atomic::Ordering::Relaxed); + if let Some(worker) = self.worker.take() { + let result = worker.join(); + if !std::thread::panicking() { + result.unwrap(); + } + } + } +} + +pub fn hash(id: u32) -> String { + format!("{id:064x}") +} + +fn jameson() -> MockCore { + MockCore::new( + 13271, + vec![ + Quorum { + id: 1, + created: 13166, + retired: 13262, + }, + Quorum { + id: 2, + created: 13190, + retired: u32::MAX, + }, + Quorum { + id: 3, + created: 13214, + retired: u32::MAX, + }, + Quorum { + id: 4, + created: 13238, + retired: u32::MAX, + }, + Quorum { + id: 5, + created: 13262, + retired: u32::MAX, + }, + ], + ) +} + +#[tokio::test] +async fn should_serve_a_quorum_retired_after_the_previous_snapshot_height() { + let core = jameson(); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + assert!(!cache + .get_quorums() + .await + .unwrap() + .contains_quorum(&hex::decode(hash(1)).unwrap())); + let (height, previous) = cache.get_previous_quorums().await.unwrap(); + assert!( + previous.contains_quorum(&hex::decode(hash(1)).unwrap()), + "signer active at tip-12 must survive a cold cache refresh even though absent at tip-8" + ); + assert_eq!(height, 13223); +} + +#[tokio::test] +async fn should_cover_every_quorum_active_in_the_window() { + for failed_dkg in [None, Some(8), Some(9)] { + let starts: Vec = (0..20) + .filter(|i| Some(*i) != failed_dkg) + .map(|i| i * 24 + 10) + .collect(); + let quorums: Vec = starts + .iter() + .enumerate() + .map(|(i, &created)| Quorum { + id: i as u32 + 1, + created, + retired: starts.get(i + 4).copied().unwrap_or(u32::MAX), + }) + .collect(); + for (tip, retention, step) in [ + (250, 48, 48), + (265, 48, 48), + (299, 48, 48), + (330, 48, 48), + (250, 167, 73), + (265, 167, 73), + (299, 167, 73), + (330, 167, 73), + ] { + let expected: Vec = quorums + .iter() + .filter(|q| q.created <= tip && q.retired > tip - retention) + .map(|q| hash(q.id)) + .collect(); + let mut core = MockCore::new(tip, quorums.clone()); + core.config.quorum.retention_blocks = retention; + core.config.quorum.sample_step_blocks = Some(step); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let (_, previous) = cache.get_previous_quorums().await.unwrap(); + let actual: Vec = previous + .iter() + .map(|q| hex::encode(&q.quorum_hash)) + .collect(); + for quorum in expected { + assert!( + actual.contains(&quorum), + "missing signer {quorum} at tip {tip}, failed DKG {failed_dkg:?}" + ); + } + } + } +} + +#[tokio::test] +async fn should_return_the_same_window_after_restart() { + let core = jameson(); + let first = QuorumCache::new(core.config.clone()); + first.refresh().await.unwrap(); + let second = QuorumCache::new(core.config.clone()); + second.refresh().await.unwrap(); + assert_eq!( + first.get_previous_quorums().await.unwrap(), + second.get_previous_quorums().await.unwrap() + ); +} + +#[tokio::test] +async fn should_not_call_quorum_info_again_for_memoized_hashes() { + let core = jameson(); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let first = core.info_calls().len(); + cache.refresh().await.unwrap(); + assert_eq!( + core.info_calls().len(), + first, + "known immutable public keys should not cost another info RPC" + ); +} + +#[tokio::test] +async fn should_never_request_secret_key_shares() { + let core = jameson(); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let calls = core.info_calls(); + assert!(!calls.is_empty()); + for call in calls { + assert_eq!(call["params"].as_array().unwrap().len(), 4); + assert_eq!( + call["params"][3], false, + "quorum info must explicitly request public information only" + ); + } +} + +pub async fn endpoint(cache: Arc, config: Config, path: &str) -> Value { + use axum::{ + body::{to_bytes, Body}, + http::Request, + }; + use tower::Service; + let masternodes = Arc::new(crate::masternode_cache::MasternodeCache::new( + config.clone(), + )); + let mut router = crate::api::create_router(cache, config, masternodes); + let response = router + .call(Request::builder().uri(path).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), 200); + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()).unwrap() +} + +#[tokio::test] +async fn should_report_the_core_height_on_quorums() { + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let first = endpoint(cache.clone(), core.config.clone(), "/quorums").await; + assert_eq!(first["height"], 13271); + assert!(first["lastUpdated"].as_i64().unwrap() > 0); + assert!(first["data"].is_array(), "legacy data must remain an array"); + assert_eq!( + first, + endpoint(cache, core.config.clone(), "/quorums").await + ); +} + +#[tokio::test] +async fn should_keep_last_good_window_and_report_degraded_health_on_core_failure() { + let mut core = jameson(); + core.config.quorum.cache_ttl_seconds = 1; + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let before = endpoint(cache.clone(), core.config.clone(), "/previous").await; + core.state.lock().unwrap().fail = true; + assert!(cache.refresh().await.is_err()); + assert_eq!( + before, + endpoint(cache.clone(), core.config.clone(), "/previous").await + ); + tokio::time::sleep(Duration::from_millis(3100)).await; + let health = endpoint(cache, core.config.clone(), "/health").await; + assert_eq!( + health["success"], false, + "stale trusted keys must be observable to health monitors" + ); + assert!(health["message"].as_str().unwrap().contains("degraded")); +} + +#[tokio::test] +async fn should_reload_when_the_tip_changes_before_the_ttl() { + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + cache.clone().start_background_refresh().await; + { + let mut state = core.state.lock().unwrap(); + state.tip += 24; + let created = state.tip; + state.quorums.push(Quorum { + id: 6, + created, + retired: u32::MAX, + }); + } + let deadline = tokio::time::Instant::now() + Duration::from_secs(3); + loop { + if cache + .get_quorums() + .await + .unwrap() + .contains_quorum(&hex::decode(hash(6)).unwrap()) + { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "tip changes must not wait for the 60-second TTL" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +#[tokio::test] +async fn should_order_oldest_to_newest_and_trim_retired_first_at_the_cap() { + let quorums = (0..12) + .map(|id| Quorum { + id, + created: id * 24 + 10, + retired: (id + 4) * 24 + 10, + }) + .collect(); + let mut core = MockCore::new(275, quorums); + core.config.quorum.retention_blocks = 192; + for (cap, expected) in [(6, vec![6, 7, 8, 9, 10, 11]), (2, vec![8, 9, 10, 11])] { + core.config.quorum.max_entries = cap; + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let (_, previous) = cache.get_previous_quorums().await.unwrap(); + let actual: Vec = previous + .iter() + .map(|q| hex::encode(&q.quorum_hash)) + .collect(); + assert_eq!(actual, expected.into_iter().map(hash).collect::>(), + "oldest retired keys must go first and active keys must survive even when active count exceeds cap"); + } +} + +#[tokio::test] +async fn should_sample_the_lower_boundary_and_preserve_legacy_offset_coverage() { + let mut core = jameson(); + core.config.quorum.retention_blocks = 7; + core.config.quorum.previous_blocks_offset = 101; + core.config.quorum.sample_step_blocks = Some(47); + core.state.lock().unwrap().quorums.push(Quorum { + id: 0, + created: 13076, + retired: 13172, + }); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let (floor, previous) = cache.get_previous_quorums().await.unwrap(); + assert_eq!(floor, 13170); + assert!( + previous.contains_quorum(&hex::decode(hash(0)).unwrap()), + "quorum retired two blocks after the floor must be covered" + ); + let sampled: Vec = core + .state + .lock() + .unwrap() + .calls + .iter() + .filter(|r| r["params"][0] == "listextended") + .map(|r| r["params"][1].as_u64().unwrap()) + .collect(); + assert_eq!(sampled, [13271, 13224, 13177, 13170]); +} + +#[tokio::test] +async fn should_keep_the_window_when_a_historical_sample_fails() { + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let before = endpoint(cache.clone(), core.config.clone(), "/previous").await; + core.state.lock().unwrap().fail_height = Some(13223); + assert!(cache.refresh().await.is_err()); + assert_eq!( + before, + endpoint(cache, core.config.clone(), "/previous").await + ); +} + +#[tokio::test] +async fn should_attempt_the_window_even_when_the_current_list_fails() { + let core = jameson(); + core.state.lock().unwrap().fail_current_once = true; + let cache = QuorumCache::new(core.config.clone()); + assert!(cache.refresh().await.is_err()); + assert!(cache.get_quorums().await.is_err()); + let (_, previous) = cache.get_previous_quorums().await.unwrap(); + assert!(previous.contains_quorum(&hex::decode(hash(1)).unwrap())); +} + +#[tokio::test] +async fn should_never_publish_an_empty_or_malformed_success() { + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let before = endpoint(cache.clone(), core.config.clone(), "/quorums").await; + for malformed in [false, true] { + { + let mut state = core.state.lock().unwrap(); + state.empty = !malformed; + state.malformed = malformed; + } + assert!(cache.refresh().await.is_err()); + assert_eq!( + before, + endpoint(cache.clone(), core.config.clone(), "/quorums").await + ); + } + let fresh = Arc::new(QuorumCache::new(core.config.clone())); + assert!(fresh.refresh().await.is_err()); + assert_eq!( + endpoint(fresh, core.config.clone(), "/quorums").await["success"], + false + ); +} + +#[tokio::test] +async fn should_drop_orphaned_quorums_after_a_reorg_without_clearing_the_memo() { + let core = jameson(); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + { + let mut state = core.state.lock().unwrap(); + state.tip -= 1; + state.quorums.retain(|q| q.id != 5); + state.quorums.push(Quorum { + id: 6, + created: 13262, + retired: u32::MAX, + }); + } + cache.refresh().await.unwrap(); + let (_, previous) = cache.get_previous_quorums().await.unwrap(); + assert!(!previous.contains_quorum(&hex::decode(hash(5)).unwrap())); + assert!(previous.contains_quorum(&hex::decode(hash(6)).unwrap())); +} + +#[tokio::test] +async fn should_reload_unchanged_height_at_the_ttl() { + let mut core = jameson(); + core.config.quorum.cache_ttl_seconds = 1; + core.config.quorum.poll_interval_seconds = 60; + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let count_lists = || { + core.state + .lock() + .unwrap() + .calls + .iter() + .filter(|r| r["params"][0] == "listextended") + .count() + }; + let initial = count_lists(); + cache.clone().start_background_refresh().await; + let deadline = tokio::time::Instant::now() + Duration::from_secs(3); + while count_lists() == initial { + assert!( + tokio::time::Instant::now() < deadline, + "TTL must bound refreshes even if configured poll interval is longer" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +#[tokio::test] +async fn should_reject_steps_that_miss_quorums_with_variable_mining_offsets() { + let starts: Vec = (0..16) + .map(|cycle| cycle * 24 + if cycle == 0 { 18 } else { 10 }) + .collect(); + let quorums: Vec = starts + .iter() + .enumerate() + .map(|(i, &created)| Quorum { + id: i as u32, + created, + retired: starts.get(i + 4).copied().unwrap_or(u32::MAX), + }) + .collect(); + assert_eq!(quorums[0].retired - quorums[0].created, 88); + let mut core = MockCore::new(299, quorums); + core.config.quorum.retention_blocks = 288; + core.config.quorum.sample_step_blocks = Some(96); + // The naive grid 299, 203, 107, 11 skips a quorum active on [18, 106). + assert!( + core.config.quorum_sample_step().is_err(), + "A*D is unsafe when commitment mining offsets vary" + ); + core.config.quorum.sample_step_blocks = Some(73); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + assert!(cache + .get_previous_quorums() + .await + .unwrap() + .1 + .contains_quorum(&hex::decode(hash(0)).unwrap())); +} + +#[tokio::test(flavor = "current_thread")] +async fn should_keep_the_async_runtime_responsive_during_blocking_core_calls() { + let core = jameson(); + core.state.lock().unwrap().delay = Duration::from_millis(30); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + let start = std::time::Instant::now(); + let refresh = tokio::spawn(async move { cache.refresh().await }); + tokio::time::sleep(Duration::from_millis(10)).await; + let elapsed = start.elapsed(); + refresh.await.unwrap().unwrap(); + assert!( + elapsed < Duration::from_millis(150), + "async timers stalled for {elapsed:?} during blocking RPC" + ); +} + +#[tokio::test] +async fn should_serve_the_window_when_the_floor_precedes_platform_quorum_activation() { + let core = MockCore::new( + 30, + vec![Quorum { + id: 1, + created: 10, + retired: u32::MAX, + }], + ); + core.state.lock().unwrap().omitted_type_before = Some(10); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let (floor, window) = cache.get_previous_quorums().await.unwrap(); + assert_eq!(floor, 0); + assert_eq!(window.len(), 1); + assert!(window.contains_quorum(&hex::decode(hash(1)).unwrap())); +} diff --git a/src/quorum_loader.rs b/src/quorum_loader.rs index ad71a4c..d80078d 100644 --- a/src/quorum_loader.rs +++ b/src/quorum_loader.rs @@ -1,177 +1,261 @@ use crate::config::Config; use crate::quorum_list::{QuorumList, QuorumListEntry}; -use dashcore_rpc::{Client, RpcApi, Auth}; -use serde::Deserialize; +use dashcore_rpc::{jsonrpc, Client, RpcApi}; +use serde_json::{json, Value}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::error::Error; +use std::time::{Duration, Instant}; -#[derive(Debug, Deserialize)] -pub struct QuorumListResult { - pub quorums: serde_json::Value, +pub type LoadResult = Result>; +pub const REFRESH_TIMEOUT: Duration = Duration::from_secs(30); + +/// Only immutable public keys are memoized; membership always comes from Core. +#[derive(Default)] +pub struct KeyMemo { + keys: HashMap>, + order: VecDeque, +} + +impl KeyMemo { + fn insert(&mut self, hash: String, key: Vec, capacity: usize) { + if self.keys.contains_key(&hash) { + return; + } + while self.keys.len() >= capacity { + if let Some(oldest) = self.order.pop_front() { + self.keys.remove(&oldest); + } else { + break; + } + } + self.order.push_back(hash.clone()); + self.keys.insert(hash, key); + } +} + +pub struct RefreshResult { + pub tip: u32, + pub current: LoadResult, + pub previous: LoadResult<(u32, QuorumList)>, } -#[derive(Debug, Deserialize)] -pub struct QuorumInfo { - pub quorum_hash: String, - pub quorum_public_key: String, +fn client(config: &Config) -> LoadResult { + let url = if config.rpc.url.contains("://") { + config.rpc.url.clone() + } else { + format!("http://{}", config.rpc.url) + }; + let transport = jsonrpc::simple_http::SimpleHttpTransport::builder() + .url(&url)? + .auth(&config.rpc.username, Some(&config.rpc.password)) + .timeout(Duration::from_secs(5)) + .build(); + Ok(Client::from_jsonrpc(jsonrpc::Client::with_transport( + transport, + ))) } -pub async fn load_initial_quorums( +fn block_height(client: &Client) -> LoadResult { + let result: u64 = client.call("getblockcount", &[])?; + Ok(u32::try_from(result)?) +} + +pub fn get_current_block_height(config: &Config) -> LoadResult { + block_height(&client(config)?) +} + +fn load_at_height( + client: &Client, config: &Config, -) -> Result> { - // Create RPC client - let auth = Auth::UserPass(config.rpc.username.clone(), config.rpc.password.clone()); - let client = Client::new(&config.rpc.url, auth)?; - - // Get the extended quorum list (returns detailed info for all quorums) - let result: serde_json::Value = client.call("quorum", &[serde_json::json!("listextended")])?; - - let mut quorum_list = QuorumList::new(); - - // Parse the extended quorum list - extract the appropriate LLMQ type based on network - let llmq_type = config.get_llmq_type(); - if let Some(quorum_obj) = result.as_object() { - if let Some(llmq_quorums) = quorum_obj.get(llmq_type) { - if let Some(quorums_arr) = llmq_quorums.as_array() { - for quorum_item in quorums_arr { - if let Some(quorum_obj) = quorum_item.as_object() { - for (quorum_hash_str, quorum_info) in quorum_obj { - if let Some(info_obj) = quorum_info.as_object() { - let quorum_hash_bytes = hex::decode(quorum_hash_str)?; - if quorum_hash_bytes.len() == 32 { - // Get the actual quorum public key via quorum info - let rpc_params = [ - serde_json::json!("info"), - serde_json::json!(config.get_llmq_type_id()), - serde_json::json!(quorum_hash_str) - ]; - println!("DEBUG: Calling quorum with params: {:?}", rpc_params); - let info_result: serde_json::Value = client.call("quorum", &rpc_params)?; - - if let Some(detailed_info) = info_result.as_object() { - if let Some(pubkey_str) = detailed_info.get("quorumPublicKey").and_then(|v| v.as_str()) { - let public_key = hex::decode(pubkey_str)?; - - if public_key.len() == 48 { - let creation_height = info_obj.get("creationHeight") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as u32; - let valid_members_count = info_obj.get("numValidMembers") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as u32; - - let entry = QuorumListEntry::new_extended( - quorum_hash_bytes, - public_key, - creation_height, - Vec::new(), // No members data needed - String::new(), // No threshold signature needed - 0, // No mining members count needed - valid_members_count, - ); - quorum_list.add_entry(entry); - println!("Loaded quorum: {} at height {} with {} members", - quorum_hash_str, creation_height, valid_members_count); - } - } - } - } - } - } + height: u32, + memo: &mut KeyMemo, + deadline: Instant, + allow_inactive_type: bool, +) -> LoadResult { + check_deadline(deadline)?; + let result: Value = client.call("quorum", &[json!("listextended"), json!(height)])?; + let result = result.as_object().ok_or("Invalid extended quorum list")?; + let quorums = match result.get(config.get_llmq_type()) { + Some(value) => value.as_array().ok_or("Invalid Platform quorum list")?, + // Core omits types that were not enabled at a historical height. + None if allow_inactive_type => return Ok(QuorumList::new()), + None => return Err("Missing Platform quorum list at the tip".into()), + }; + let mut list = QuorumList::new(); + for item in quorums { + let entries = item.as_object().ok_or("Invalid quorum entry")?; + if entries.is_empty() { + return Err("Empty quorum entry".into()); + } + for (hash, details) in entries { + let hash_bytes = hex::decode(hash)?; + if hash_bytes.len() != 32 { + return Err("Invalid quorum hash length".into()); + } + let creation_height = details + .get("creationHeight") + .and_then(Value::as_u64) + .ok_or("Missing quorum creation height")?; + let valid_members = details + .get("numValidMembers") + .and_then(Value::as_u64) + .ok_or("Missing quorum valid member count")?; + let creation_height = u32::try_from(creation_height)?; + let valid_members = u32::try_from(valid_members)?; + let public_key = match memo.keys.get(hash) { + Some(key) => key.clone(), + None => { + check_deadline(deadline)?; + let info: Value = client.call( + "quorum", + &[ + json!("info"), + json!(config.get_llmq_type_id()), + json!(hash), + json!(false), + ], + )?; + let key = hex::decode( + info.get("quorumPublicKey") + .and_then(Value::as_str) + .ok_or("Missing quorum public key")?, + )?; + if key.len() != 48 { + return Err("Invalid quorum public key length".into()); } + memo.insert( + hash.clone(), + key.clone(), + config.quorum.key_memo_max_entries, + ); + key } - } + }; + list.add_entry(QuorumListEntry::new_extended( + hash_bytes, + public_key, + creation_height, + Vec::new(), + String::new(), + 0, + valid_members, + )); } } - - println!("Loaded {} quorums from Dash Core", quorum_list.len()); - Ok(quorum_list) + sort_oldest_first(&mut list); + Ok(list) } -pub async fn get_current_block_height( - config: &Config, -) -> Result> { - let auth = Auth::UserPass(config.rpc.username.clone(), config.rpc.password.clone()); - let client = Client::new(&config.rpc.url, auth)?; - - let result: serde_json::Value = client.call("getblockcount", &[])?; - let height = result.as_u64().ok_or("Invalid block count response")? as u32; - - Ok(height) +fn check_deadline(deadline: Instant) -> LoadResult<()> { + if Instant::now() >= deadline { + return Err("Quorum refresh deadline exceeded".into()); + } + Ok(()) +} + +fn sort_oldest_first(list: &mut QuorumList) { + list.list.sort_by(|a, b| { + a.height + .cmp(&b.height) + .then_with(|| a.quorum_hash.cmp(&b.quorum_hash)) + }); } -pub async fn load_quorums_at_height( +fn load_window( + client: &Client, config: &Config, - height: u32, -) -> Result> { - let auth = Auth::UserPass(config.rpc.username.clone(), config.rpc.password.clone()); - let client = Client::new(&config.rpc.url, auth)?; - - // Get the extended quorum list at specific height - let result: serde_json::Value = client.call("quorum", &[ - serde_json::json!("listextended"), - serde_json::json!(height) - ])?; - - let mut quorum_list = QuorumList::new(); - - // Parse the extended quorum list - extract the appropriate LLMQ type based on network - let llmq_type = config.get_llmq_type(); - if let Some(quorum_obj) = result.as_object() { - if let Some(llmq_quorums) = quorum_obj.get(llmq_type) { - if let Some(quorums_arr) = llmq_quorums.as_array() { - for quorum_item in quorums_arr { - if let Some(quorum_obj) = quorum_item.as_object() { - for (quorum_hash_str, quorum_info) in quorum_obj { - if let Some(info_obj) = quorum_info.as_object() { - let quorum_hash_bytes = hex::decode(quorum_hash_str)?; - if quorum_hash_bytes.len() == 32 { - // Get the actual quorum public key via quorum info - let rpc_params = [ - serde_json::json!("info"), - serde_json::json!(config.get_llmq_type_id()), - serde_json::json!(quorum_hash_str), - serde_json::json!(true) // includeSkShare - ]; - println!("DEBUG: Calling quorum with params: {:?}", rpc_params); - let info_result: serde_json::Value = client.call("quorum", &rpc_params)?; - - if let Some(detailed_info) = info_result.as_object() { - if let Some(pubkey_str) = detailed_info.get("quorumPublicKey").and_then(|v| v.as_str()) { - let public_key = hex::decode(pubkey_str)?; - - if public_key.len() == 48 { - let creation_height = info_obj.get("creationHeight") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as u32; - let valid_members_count = info_obj.get("numValidMembers") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as u32; - - let entry = QuorumListEntry::new_extended( - quorum_hash_bytes, - public_key, - creation_height, - Vec::new(), // No members data needed - String::new(), // No threshold signature needed - 0, // No mining members count needed - valid_members_count, - ); - quorum_list.add_entry(entry); - println!("Loaded quorum at height {}: {} with {} members", - height, quorum_hash_str, valid_members_count); - } - } - } - } - } - } - } + tip: u32, + current: &LoadResult, + memo: &mut KeyMemo, + deadline: Instant, +) -> LoadResult<(u32, QuorumList)> { + let step = config.quorum_sample_step()?; + let floor = tip.saturating_sub(config.quorum_retention()); + // Retry the tip for the window even if the independent active-list fetch failed. + let active = match current { + Ok(list) => Ok(list.clone()), + Err(_) => load_at_height(client, config, tip, memo, deadline, false), + }; + let mut window = QuorumList::new(); + let mut height = tip; + let mut history_error = None; + while height > floor { + // Include the lower boundary when the regular grid would overshoot it. + height = height.saturating_sub(step).max(floor); + match load_at_height(client, config, height, memo, deadline, true) { + Ok(list) => { + for entry in list.list { + window.add_entry(entry); } } + Err(error) => { + history_error = Some(error); + break; + } + } + } + let active = active?; + if active.is_empty() { + return Err("Core returned no active Platform quorums".into()); + } + if let Some(error) = history_error { + return Err(error); + } + let active_hashes: HashSet> = active.iter().map(|q| q.quorum_hash.clone()).collect(); + for entry in active.list { + window.add_entry(entry); + } + sort_oldest_first(&mut window); + let mut excess = window.len().saturating_sub(config.quorum.max_entries); + window.list.retain(|entry| { + if excess > 0 && !active_hashes.contains(&entry.quorum_hash) { + excess -= 1; + false + } else { + true + } + }); + Ok((floor, window)) +} + +/// Blocking RPC work; callers must run this on a blocking worker. +pub fn refresh(config: &Config, memo: &mut KeyMemo) -> LoadResult { + config.quorum_sample_step()?; + let deadline = Instant::now() + REFRESH_TIMEOUT; + let client = client(config)?; + let tip = block_height(&client)?; + let current = load_at_height(&client, config, tip, memo, deadline, false).and_then(|list| { + if list.is_empty() { + Err("Core returned no active Platform quorums".into()) + } else { + Ok(list) } + }); + let previous = load_window(&client, config, tip, ¤t, memo, deadline); + check_deadline(deadline)?; + if block_height(&client)? != tip { + return Err("Core tip changed during quorum refresh".into()); } - - println!("Loaded {} quorums from Dash Core at height {}", quorum_list.len(), height); - Ok(quorum_list) + Ok(RefreshResult { + tip, + current, + previous, + }) } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn key_memo_is_bounded_and_evicts_oldest() { + let mut memo = KeyMemo::default(); + for id in 0..3 { + memo.insert(id.to_string(), vec![id], 2); + } + assert_eq!(memo.keys.len(), 2); + assert!(!memo.keys.contains_key("0")); + assert!(memo.keys.contains_key("2")); + assert_eq!(memo.order.len(), 2); + } +} From 844be981f78b647e3b172c259281971b6817ec2a Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 9 Oct 2026 20:36:27 +0700 Subject: [PATCH 02/14] fix: harden quorum refreshes and readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Memoize public keys by quorum and mined commitment, reject incomplete historical samples, add readiness status without changing health status, and let environment settings override TOML. Preserve the sampled current list when the tip advances and retry the window once. Validate quorum limits once at startup, back off repeated failures while prioritizing new tips, and recover the key memo after a poisoned lock. Test would have caught this in CI: ✖ before fix, ✔ after. Commitment replacement, missing historical entries, readiness status and environment precedence all failed before the fixes; moving-tip and configuration-coherence regressions also failed before correction. All 51 tests now pass. Co-Authored-By: GPT-6.1 Sol --- README.md | 40 ++++++---- src/api.rs | 9 +++ src/config.rs | 149 ++++++++++++++++++++++++-------------- src/main.rs | 2 +- src/quorum_cache.rs | 49 ++++++++++--- src/quorum_cache_tests.rs | 107 ++++++++++++++++++++++++++- src/quorum_loader.rs | 55 +++++++++----- 7 files changed, 310 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index 22b11eb..d233111 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A Rust-based HTTP API server providing the configured network's Platform quorum - RPC-based Platform quorum loading for mainnet, testnet, devnet and regtest - RESTful API for quorum list management -- TOML-based configuration with environment variable fallbacks +- TOML-based configuration with environment variable overrides - Stateless rolling window of active and recently retired quorums - Thread-safe shared state - Cross-platform support (Linux/Windows) @@ -15,7 +15,8 @@ A Rust-based HTTP API server providing the configured network's Platform quorum ## API Endpoints ### Health Check -- `GET /health` - Quorum cache freshness status +- `GET /health` - Quorum cache freshness status (HTTP 200, inspect the JSON body) +- `GET /ready` - Readiness status (HTTP 503 while degraded, 200 when fresh) ### Quorum Management - `GET /quorums` - Get all current quorums @@ -54,7 +55,9 @@ blocks. Sampling every `A*D` blocks could miss a quorum. The conservative maximu step is `(A-1)*D + 1`, allowing any mining offset within a cycle. The default is `max(1, min(A*D/2, (A-1)*D+1))`. Failed DKGs lengthen active runs. Parameter overrides must match the connected Core; unsafe steps, zero intervals and zero -cache limits are rejected at startup with an error. See Core's +cache limits are rejected at startup with an error. The output cap must exceed A, +the key memo must hold at least the output cap, and a window may use at most +256 samples (including both boundaries) to limit work within the refresh budget. See Core's [parameter table](https://github.com/dashpay/dash/blob/v24.0.0-rc.3/src/llmq/params.h) and [mined commitment scan](https://github.com/dashpay/dash/blob/v24.0.0-rc.3/src/llmq/quorumsman.cpp#L243-L244). @@ -67,7 +70,9 @@ accelerated 10-second devnets typically need 360–720 blocks to cover Core/Plat lag, a Platform block interval, the client's 31-minute metadata tolerance and clock skew. -A bounded public-key memo avoids repeat `quorum info` calls for known hashes. +A bounded public-key memo keyed by `(quorumHash, minedBlockHash)` avoids repeat +`quorum info` calls for unchanged commitments. A reorg can replace a commitment +and its public key for the same quorum hash; the new mined block forces a refetch. All such calls explicitly use `includeSkShare=false`. Membership is recomputed, not taken from the memo. All quorum RPC runs on a serialized blocking worker; requests only read cached data. RPC sockets have a five-second timeout, refreshes @@ -79,10 +84,19 @@ changes. The cache TTL bounds reloads even at an unchanged height (including same-height reorgs), and a poll interval longer than the TTL is reduced to the TTL. Both lists are attempted independently and keep their last good data on failure. An empty active list or invalid quorum/key data is a failed refresh, -never an empty success. `/health` still returns HTTP 200 and the same JSON shape; +never an empty success. Historical samples must contain every tip-active quorum +whose creation height plus the mining-window end (18 blocks) precedes the sample; +a short or empty result missing such a quorum fails the window refresh. A moving +tip preserves the current list tied to its sampled height and retries the window +once at the new tip. Repeated failed window reloads back off up to the smaller of +30 seconds and the TTL; a newly observed tip bypasses this backoff. + +`/health` still returns HTTP 200 and the same JSON shape; it returns `success:false`, `data:null` and a degraded message when either list has never loaded or its last successful refresh is older than -`cache_ttl_seconds * health_stale_intervals` (three intervals by default). +`cache_ttl_seconds * health_stale_intervals` (three intervals by default). `/ready` +returns the same body with HTTP 503 while degraded and HTTP 200 when fresh, so +load balancers can use it without changing existing `/health` matchers. ### Masternode bootstrap metadata @@ -152,7 +166,7 @@ health_stale_intervals = 3 # dkg_interval = 24 ``` -### Environment Variables (fallbacks) +### Environment Variables (overrides) - `API_HOST` - Server host (default: 0.0.0.0) - `API_PORT` - HTTP server port (default: 3000) - `DASH_RPC_URL` - RPC endpoint (default: http://127.0.0.1:19998) @@ -160,7 +174,7 @@ health_stale_intervals = 3 - `DASH_RPC_PASSWORD` - RPC password (default: password) - `DASH_NETWORK` - mainnet, testnet, devnet or regtest (default: testnet) -| TOML `[quorum]` key | Environment fallback | Default | +| TOML `[quorum]` key | Environment override | Default | | --- | --- | --- | | `previous_blocks_offset` | `QUORUM_PREVIOUS_BLOCKS_OFFSET` | 8; lower bound on retention | | `retention_blocks` | `QUORUM_RETENTION_BLOCKS` | 48 | @@ -173,8 +187,8 @@ health_stale_intervals = 3 | `poll_interval_seconds` | `QUORUM_POLL_INTERVAL_SECONDS` | 1 | | `health_stale_intervals` | `QUORUM_HEALTH_STALE_INTERVALS` | 3 | -A valid TOML file takes precedence over all environment variables. When the -file cannot be loaded, environment variables supply fallbacks. Older TOML files +Environment variables override TOML values. When the file cannot be loaded, +environment variables override the built-in defaults. Older TOML files with only `previous_blocks_offset` continue to work with the new defaults. ## Usage @@ -183,7 +197,7 @@ with only `previous_blocks_offset` continue to work with the new defaults. # Start the server (uses config.toml) cargo run -# With environment variables (when config.toml is absent) +# With environment variables (overrides config.toml) DASH_RPC_URL="http://192.168.1.100:19998" DASH_RPC_USER="myuser" cargo run # Different port @@ -213,7 +227,7 @@ docker run --rm -p 8080:8080 \ quorum-list-server ``` -The server reads `/app/config.toml` first; environment variables are only used when that file is absent. +The server reads `/app/config.toml` first, then applies environment overrides. ## API Examples @@ -270,7 +284,7 @@ Example quorum response: - **RPC Integration**: Uses `dashcore-rpc` to communicate with Dash Core - **LLMQ Type**: Processes the configured network's Platform quorum type -- **Configuration**: TOML-first with environment variable fallbacks +- **Configuration**: TOML values with environment variable overrides - **State Management**: Cached lists and freshness metadata published under one lock - **API Framework**: Built with Axum for async HTTP handling diff --git a/src/api.rs b/src/api.rs index e1c54e0..477c233 100644 --- a/src/api.rs +++ b/src/api.rs @@ -114,6 +114,7 @@ pub fn create_router(quorum_cache: SharedQuorumCache, config: Config, masternode let shared_config = Arc::new(config); Router::new() .route("/health", get(health_check)) + .route("/ready", get(readiness_check)) .route("/quorums", get(get_all_quorums)) .route("/quorums/stats", get(get_quorum_stats)) .route("/previous", get(get_previous_quorums)) @@ -136,6 +137,14 @@ async fn health_check( } } +async fn readiness_check( + state: State<(SharedQuorumCache, SharedConfig, SharedMasternodeCache)>, +) -> (StatusCode, Json>) { + let response = health_check(state).await; + let status = if response.0.success { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE }; + (status, response) +} + async fn get_all_quorums( State((quorum_cache, _, _)): State<(SharedQuorumCache, SharedConfig, SharedMasternodeCache)>, ) -> Result>>, StatusCode> { diff --git a/src/config.rs b/src/config.rs index 8ed92d2..1d34556 100644 --- a/src/config.rs +++ b/src/config.rs @@ -179,43 +179,41 @@ impl Config { } pub fn load_from_env_or_file>(path: P) -> Self { - // Try to load from file first - if let Ok(config) = Self::load_from_file(path) { - return config; - } + Self::load_with_env(path, |name| std::env::var(name).ok()) + } - // Fall back to environment variables or defaults - let mut config = Config::default(); + fn load_with_env>(path: P, env: impl Fn(&str) -> Option) -> Self { + let mut config = Self::load_from_file(path).unwrap_or_default(); - if let Ok(port) = std::env::var("API_PORT") { + if let Some(port) = env("API_PORT") { if let Ok(port_num) = port.parse::() { config.server.port = port_num; } } - if let Ok(host) = std::env::var("API_HOST") { + if let Some(host) = env("API_HOST") { config.server.host = host; } - if let Ok(url) = std::env::var("DASH_RPC_URL") { + if let Some(url) = env("DASH_RPC_URL") { config.rpc.url = url; } - if let Ok(username) = std::env::var("DASH_RPC_USER") { + if let Some(username) = env("DASH_RPC_USER") { config.rpc.username = username; } - if let Ok(password) = std::env::var("DASH_RPC_PASSWORD") { + if let Some(password) = env("DASH_RPC_PASSWORD") { config.rpc.password = password; } - if let Ok(offset) = std::env::var("QUORUM_PREVIOUS_BLOCKS_OFFSET") { + if let Some(offset) = env("QUORUM_PREVIOUS_BLOCKS_OFFSET") { if let Ok(offset_num) = offset.parse::() { config.quorum.previous_blocks_offset = offset_num; } } - if let Ok(ttl) = std::env::var("QUORUM_CACHE_TTL_SECONDS") { + if let Some(ttl) = env("QUORUM_CACHE_TTL_SECONDS") { if let Ok(ttl_num) = ttl.parse::() { config.quorum.cache_ttl_seconds = ttl_num; } @@ -223,14 +221,14 @@ impl Config { macro_rules! quorum_env { ($name:literal, $field:ident) => { - if let Ok(value) = std::env::var($name) { + if let Some(value) = env($name) { config.quorum.$field = value .parse() .unwrap_or_else(|_| panic!("Invalid {}", $name)); } }; ($name:literal, $field:ident, optional) => { - if let Ok(value) = std::env::var($name) { + if let Some(value) = env($name) { config.quorum.$field = Some( value .parse() @@ -248,16 +246,16 @@ impl Config { quorum_env!("QUORUM_POLL_INTERVAL_SECONDS", poll_interval_seconds); quorum_env!("QUORUM_HEALTH_STALE_INTERVALS", health_stale_intervals); - if let Ok(network_str) = std::env::var("DASH_NETWORK") { + if let Some(network_str) = env("DASH_NETWORK") { config.network = Network::try_from(network_str.as_str()).unwrap_or_else(|e| panic!("{}", e)); } - if let Ok(version_check_host) = std::env::var("VERSION_CHECK_HOST") { + if let Some(version_check_host) = env("VERSION_CHECK_HOST") { config.docker.version_check_host = Some(version_check_host); } - if let Ok(address_host_override) = std::env::var("ADDRESS_HOST_OVERRIDE") { + if let Some(address_host_override) = env("ADDRESS_HOST_OVERRIDE") { config.docker.address_host_override = Some(address_host_override); } @@ -281,10 +279,29 @@ impl Config { self.network.llmq_type_id() } - pub fn quorum_sample_step(&self) -> Result> { + fn quorum_parameters(&self) -> (u32, u32) { let (active, interval) = self.network.platform_quorum_parameters(); - let active = self.quorum.active_quorum_count.unwrap_or(active); - let interval = self.quorum.dkg_interval.unwrap_or(interval); + ( + self.quorum.active_quorum_count.unwrap_or(active), + self.quorum.dkg_interval.unwrap_or(interval), + ) + } + + pub fn quorum_sample_step(&self) -> u32 { + let (active, interval) = self.quorum_parameters(); + let lifetime = active.saturating_mul(interval); + let max_step = active + .saturating_sub(1) + .saturating_mul(interval) + .saturating_add(1); + self.quorum + .sample_step_blocks + .unwrap_or((lifetime / 2).min(max_step).max(1)) + } + + /// Checks operator settings once at startup, before any RPC work is admitted. + pub fn validate_quorum(&self) -> Result<(), Box> { + let (active, interval) = self.quorum_parameters(); let lifetime = active .checked_mul(interval) .filter(|value| *value > 0) @@ -292,23 +309,29 @@ impl Config { "Quorum active count and DKG interval must be positive and their product fit u32", )?; // Commitment mining offsets can differ within a cycle, shortening an active run. - let max_step = (active - 1) * interval + 1; - let step = self - .quorum - .sample_step_blocks - .unwrap_or((lifetime / 2).min(max_step).max(1)); + let max_step = lifetime - interval + 1; + let step = self.quorum_sample_step(); if step == 0 || step > max_step { return Err(format!("Quorum sample step {step} must be between 1 and {max_step}: commitment mining offsets can shorten active runs").into()); } if self.quorum.cache_ttl_seconds == 0 || self.quorum.poll_interval_seconds == 0 || self.quorum.health_stale_intervals == 0 - || self.quorum.max_entries == 0 - || self.quorum.key_memo_max_entries == 0 { - return Err("Quorum intervals and cache limits must be positive".into()); + return Err("Quorum intervals must be positive".into()); + } + if self.quorum.max_entries <= active as usize { + return Err("Quorum max_entries must exceed the active quorum count".into()); } - Ok(step) + if self.quorum.key_memo_max_entries < self.quorum.max_entries { + return Err("Quorum key memo capacity must be at least max_entries".into()); + } + if self.quorum_retention().div_ceil(step).saturating_add(1) > 256 { + return Err( + "Quorum retention and sample step require more than 256 samples per refresh".into(), + ); + } + Ok(()) } pub fn quorum_retention(&self) -> u32 { @@ -408,9 +431,9 @@ previous_blocks_offset = 8 network, ..Config::default() }; - assert_eq!(config.quorum_sample_step().unwrap(), expected); + assert_eq!(config.quorum_sample_step(), expected); config.quorum.active_quorum_count = Some(1); - assert_eq!(config.quorum_sample_step().unwrap(), 1); + assert_eq!(config.quorum_sample_step(), 1); } } @@ -418,15 +441,15 @@ previous_blocks_offset = 8 fn should_reject_invalid_quorum_intervals_and_limits() { let mut config = Config::default(); config.quorum.sample_step_blocks = Some(0); - assert!(config.quorum_sample_step().is_err()); + assert!(config.validate_quorum().is_err()); config.quorum.sample_step_blocks = None; config.quorum.active_quorum_count = Some(0); - assert!(config.quorum_sample_step().is_err()); + assert!(config.validate_quorum().is_err()); config.quorum.active_quorum_count = Some(u32::MAX); - assert!(config.quorum_sample_step().is_err()); + assert!(config.validate_quorum().is_err()); config.quorum.active_quorum_count = None; config.quorum.cache_ttl_seconds = 0; - assert!(config.quorum_sample_step().is_err()); + assert!(config.validate_quorum().is_err()); } #[test] @@ -443,35 +466,55 @@ previous_blocks_offset = 8 ("QUORUM_POLL_INTERVAL_SECONDS", "2"), ("QUORUM_HEALTH_STALE_INTERVALS", "5"), ]; - let original: Vec<_> = variables - .iter() - .map(|(name, _)| (*name, std::env::var_os(name))) - .collect(); - for (name, value) in variables { - std::env::set_var(name, value); - } - let config = Config::load_from_env_or_file("/nonexistent/qls16-config.toml"); - for (name, value) in original { - if let Some(value) = value { - std::env::set_var(name, value); - } else { - std::env::remove_var(name); - } - } + let config = Config::load_with_env("/nonexistent/qls16-config.toml", |name| { + variables + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()) + }); assert_eq!(config.quorum.previous_blocks_offset, 96); assert_eq!(config.quorum.cache_ttl_seconds, 1); assert_eq!(config.quorum_retention(), 720); - assert_eq!(config.quorum_sample_step().unwrap(), 12); + assert_eq!(config.quorum_sample_step(), 12); assert_eq!(config.quorum.max_entries, 80); assert_eq!(config.quorum.key_memo_max_entries, 160); assert_eq!(config.quorum.poll_interval_seconds, 2); assert_eq!(config.quorum.health_stale_intervals, 5); } + /// Environment settings must win even when the shipped TOML file is valid. + #[test] + fn should_override_file_settings_with_environment_variables() { + let config = Config::load_with_env("config.toml", |name| match name { + "QUORUM_CACHE_TTL_SECONDS" => Some("1".into()), + "DASH_NETWORK" => Some("devnet".into()), + "DASH_RPC_URL" => Some("http://core.example:9998".into()), + _ => None, + }); + assert_eq!(config.quorum.cache_ttl_seconds, 1); + assert_eq!(config.network, Network::Devnet); + assert_eq!(config.rpc.url, "http://core.example:9998"); + } + #[test] fn should_load_the_shipped_config_file() { let config: Config = toml::from_str(include_str!("../config.toml")).unwrap(); assert_eq!(config.network, Network::Testnet); - assert_eq!(config.quorum_sample_step().unwrap(), 288); + assert_eq!(config.quorum_sample_step(), 288); + } + + /// Startup must reject settings that defeat retention, memoization or the RPC budget. + #[test] + fn should_reject_incoherent_quorum_capacity_and_excessive_sampling() { + let mut config = Config::default(); + config.quorum.max_entries = 24; + assert!(config.validate_quorum().is_err()); + config.quorum.max_entries = 90; + config.quorum.key_memo_max_entries = 89; + assert!(config.validate_quorum().is_err()); + config.quorum.key_memo_max_entries = 180; + config.quorum.retention_blocks = 10000; + config.quorum.sample_step_blocks = Some(1); + assert!(config.validate_quorum().is_err()); } } diff --git a/src/main.rs b/src/main.rs index 86ed095..2e8d9b1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,7 +24,7 @@ async fn main() -> Result<(), Box> { // Load configuration let config = Config::load_from_env_or_file("config.toml"); - if let Err(error) = config.quorum_sample_step() { + if let Err(error) = config.validate_quorum() { eprintln!("Invalid quorum configuration: {error}"); return Err(std::io::Error::other(error.to_string()).into()); } diff --git a/src/quorum_cache.rs b/src/quorum_cache.rs index dede7e8..b154b82 100644 --- a/src/quorum_cache.rs +++ b/src/quorum_cache.rs @@ -71,8 +71,8 @@ impl QuorumCache { T: Send + 'static, F: FnOnce() -> LoadResult + Send + 'static, { + let permit = self.rpc_worker.clone().lock_owned().await; tokio::time::timeout(REFRESH_TIMEOUT, async { - let permit = self.rpc_worker.clone().lock_owned().await; tokio::task::spawn_blocking(move || { let _permit = permit; work() @@ -89,7 +89,12 @@ impl QuorumCache { let memo = self.memo.clone(); let result = self .rpc_work(move || { - let mut memo = memo.lock().map_err(|_| "Failed to lock quorum key memo")?; + let mut memo = memo.lock().unwrap_or_else(|poisoned| { + memo.clear_poison(); + let mut memo = poisoned.into_inner(); + *memo = KeyMemo::default(); + memo + }); quorum_loader::refresh(&config, &mut memo) }) .await?; @@ -148,19 +153,23 @@ impl QuorumCache { } } - fn reload_due(&self, tip: u32) -> LoadResult { + fn reload_due(&self, tip: u32) -> LoadResult<(bool, bool)> { let state = self .state .read() .map_err(|_| "Failed to read quorum cache")?; Ok(match (&state.current, &state.previous) { (Some(current), Some(previous)) => { - current.height != tip - || current.refreshed.elapsed() >= self.update_interval - || previous.refreshed.elapsed() >= self.update_interval - || previous.height != tip.saturating_sub(self.config.quorum_retention()) + let changed = current.height != tip; + ( + changed, + changed + || current.refreshed.elapsed() >= self.update_interval + || previous.refreshed.elapsed() >= self.update_interval + || previous.height != tip.saturating_sub(self.config.quorum_retention()), + ) } - _ => true, + _ => (true, true), }) } @@ -171,6 +180,8 @@ impl QuorumCache { .max(Duration::from_millis(1)); let mut poll = tokio::time::interval(poll_interval); poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut failures = 0u32; + let mut retry_after = tokio::time::Instant::now(); loop { poll.tick().await; let config = self.config.clone(); @@ -179,12 +190,26 @@ impl QuorumCache { .await { Ok(tip) => match self.reload_due(tip) { - Ok(true) => { - if let Err(error) = self.refresh().await { - eprintln!("Quorum refresh failed: {error}"); + Ok((changed, true)) + if changed || tokio::time::Instant::now() >= retry_after => + { + match self.refresh().await { + Ok(()) => { + failures = 0; + } + Err(error) => { + eprintln!("Quorum refresh failed: {error}"); + failures = failures.saturating_add(1); + let delay = poll_interval + .saturating_mul(1 << failures.saturating_sub(1).min(5)) + .min(self.update_interval) + .min(Duration::from_secs(30)); + retry_after = tokio::time::Instant::now() + delay; + } } } - Ok(false) => {} + Ok((_, true)) => {} + Ok((_, false)) => {} Err(error) => eprintln!("Cannot read quorum refresh state: {error}"), }, Err(error) => eprintln!("Core tip poll failed: {error}"), diff --git a/src/quorum_cache_tests.rs b/src/quorum_cache_tests.rs index 751b550..cd567c5 100644 --- a/src/quorum_cache_tests.rs +++ b/src/quorum_cache_tests.rs @@ -25,6 +25,9 @@ pub struct CoreState { pub fail_current_once: bool, pub delay: Duration, pub omitted_type_before: Option, + pub commitments: std::collections::HashMap, + pub omit_at_height: Option, + pub tip_script: std::collections::VecDeque, } pub struct MockCore { @@ -55,6 +58,9 @@ impl MockCore { fail_current_once: false, delay: Duration::ZERO, omitted_type_before: None, + commitments: std::collections::HashMap::new(), + omit_at_height: None, + tip_script: std::collections::VecDeque::new(), })); let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); let worker_state = state.clone(); @@ -102,6 +108,11 @@ impl MockCore { serde_json::from_slice(&bytes[header_end..header_end + length]).unwrap(); let mut state = worker_state.lock().unwrap(); state.calls.push(request.clone()); + if request["method"] == "getblockcount" { + if let Some(tip) = state.tip_script.pop_front() { + state.tip = tip; + } + } let height = request["params"][1] .as_u64() .map(|h| h as u32) @@ -127,13 +138,16 @@ impl MockCore { json!({}) } else { let entries: Vec = state.quorums.iter() - .filter(|q| !state.empty && q.created <= height && height < q.retired) - .map(|q| json!({hash(q.id): {"creationHeight":q.created, "numValidMembers":12}})) + .filter(|q| !state.empty && state.omit_at_height != Some(height) && q.created <= height && height < q.retired) + .map(|q| json!({hash(q.id): {"creationHeight":q.created, "numValidMembers":12, "minedBlockHash":hash(q.id + 100000 + state.commitments.get(&q.id).copied().unwrap_or(0))}})) .collect(); json!({"llmq_devnet_platform": entries}) } } else { - json!({"quorumPublicKey": "11".repeat(48)}) + let id = u32::from_str_radix(&request["params"][2].as_str().unwrap()[56..], 16) + .unwrap(); + let commitment = state.commitments.get(&id).copied().unwrap_or(0); + json!({"quorumPublicKey": format!("{:02x}", 17 + commitment).repeat(48)}) }; let body = if failed { json!({"result":null, "error":{"code":-1,"message":"Core unavailable"}, "id":request["id"]}) @@ -593,7 +607,7 @@ async fn should_reject_steps_that_miss_quorums_with_variable_mining_offsets() { core.config.quorum.sample_step_blocks = Some(96); // The naive grid 299, 203, 107, 11 skips a quorum active on [18, 106). assert!( - core.config.quorum_sample_step().is_err(), + core.config.validate_quorum().is_err(), "A*D is unsafe when commitment mining offsets vary" ); core.config.quorum.sample_step_blocks = Some(73); @@ -641,3 +655,88 @@ async fn should_serve_the_window_when_the_floor_precedes_platform_quorum_activat assert_eq!(window.len(), 1); assert!(window.contains_quorum(&hex::decode(hash(1)).unwrap())); } + +/// A replacement final commitment for the same quorum hash must replace its key. +#[tokio::test] +async fn should_refetch_a_key_when_the_mined_commitment_changes() { + let core = jameson(); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let before = core.info_calls().len(); + core.state.lock().unwrap().commitments.insert(5, 1); + cache.refresh().await.unwrap(); + assert_eq!(core.info_calls().len(), before + 1); + let list = cache.get_quorums().await.unwrap(); + assert_eq!( + list.get_entry(&hex::decode(hash(5)).unwrap()).unwrap().key, + vec![18; 48] + ); +} + +/// Core's empty historical result must not erase a quorum that must exist there. +#[tokio::test] +async fn should_reject_a_historical_sample_missing_an_older_active_quorum() { + let core = jameson(); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let before = cache.get_previous_quorums().await.unwrap(); + core.state.lock().unwrap().omit_at_height = Some(13223); + assert!(cache.refresh().await.is_err()); + assert_eq!(cache.get_previous_quorums().await.unwrap(), before); +} + +/// Readiness gives load balancers a failing status without changing health's HTTP status. +#[tokio::test] +async fn should_return_service_unavailable_when_not_ready() { + use axum::{body::Body, http::Request}; + use tower::Service; + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + let masternodes = Arc::new(crate::masternode_cache::MasternodeCache::new( + core.config.clone(), + )); + let mut router = crate::api::create_router(cache.clone(), core.config.clone(), masternodes); + let response = router + .call( + Request::builder() + .uri("/ready") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), 503); + cache.refresh().await.unwrap(); + let response = router + .call( + Request::builder() + .uri("/ready") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), 200); +} + +/// A moving chain must not discard an active list tied to an explicit sampled height. +#[tokio::test] +async fn should_publish_current_and_retry_the_window_when_the_tip_moves() { + let core = jameson(); + { + let mut state = core.state.lock().unwrap(); + state.tip_script = [13271, 13295, 13295].into(); + state.quorums.push(Quorum { + id: 6, + created: 13286, + retired: u32::MAX, + }); + } + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + assert_eq!(cache.get_snapshot().await.unwrap().height, 13271); + let (floor, previous) = cache.get_previous_quorums().await.unwrap(); + assert_eq!(floor, 13247); + assert!(previous.contains_quorum(&hex::decode(hash(6)).unwrap())); + assert!(cache.is_healthy().await); +} diff --git a/src/quorum_loader.rs b/src/quorum_loader.rs index d80078d..0dc60a2 100644 --- a/src/quorum_loader.rs +++ b/src/quorum_loader.rs @@ -12,12 +12,12 @@ pub const REFRESH_TIMEOUT: Duration = Duration::from_secs(30); /// Only immutable public keys are memoized; membership always comes from Core. #[derive(Default)] pub struct KeyMemo { - keys: HashMap>, - order: VecDeque, + keys: HashMap<(String, String), Vec>, + order: VecDeque<(String, String)>, } impl KeyMemo { - fn insert(&mut self, hash: String, key: Vec, capacity: usize) { + fn insert(&mut self, hash: (String, String), key: Vec, capacity: usize) { if self.keys.contains_key(&hash) { return; } @@ -92,6 +92,14 @@ fn load_at_height( if hash_bytes.len() != 32 { return Err("Invalid quorum hash length".into()); } + let mined_hash = details + .get("minedBlockHash") + .and_then(Value::as_str) + .ok_or("Missing quorum mined block hash")?; + if hex::decode(mined_hash)?.len() != 32 { + return Err("Invalid mined block hash length".into()); + } + let memo_key = (hash.clone(), mined_hash.to_string()); let creation_height = details .get("creationHeight") .and_then(Value::as_u64) @@ -102,7 +110,7 @@ fn load_at_height( .ok_or("Missing quorum valid member count")?; let creation_height = u32::try_from(creation_height)?; let valid_members = u32::try_from(valid_members)?; - let public_key = match memo.keys.get(hash) { + let public_key = match memo.keys.get(&memo_key) { Some(key) => key.clone(), None => { check_deadline(deadline)?; @@ -123,11 +131,7 @@ fn load_at_height( if key.len() != 48 { return Err("Invalid quorum public key length".into()); } - memo.insert( - hash.clone(), - key.clone(), - config.quorum.key_memo_max_entries, - ); + memo.insert(memo_key, key.clone(), config.quorum.key_memo_max_entries); key } }; @@ -169,7 +173,7 @@ fn load_window( memo: &mut KeyMemo, deadline: Instant, ) -> LoadResult<(u32, QuorumList)> { - let step = config.quorum_sample_step()?; + let step = config.quorum_sample_step(); let floor = tip.saturating_sub(config.quorum_retention()); // Retry the tip for the window even if the independent active-list fetch failed. let active = match current { @@ -184,6 +188,18 @@ fn load_window( height = height.saturating_sub(step).max(floor); match load_at_height(client, config, height, memo, deadline, true) { Ok(list) => { + if let Ok(tip_active) = &active { + for entry in tip_active.iter() { + if u64::from(entry.height) + 18 <= u64::from(height) + && !list.contains_quorum(&entry.quorum_hash) + { + return Err( + "Historical sample is missing a quorum that must still be active" + .into(), + ); + } + } + } for entry in list.list { window.add_entry(entry); } @@ -220,7 +236,6 @@ fn load_window( /// Blocking RPC work; callers must run this on a blocking worker. pub fn refresh(config: &Config, memo: &mut KeyMemo) -> LoadResult { - config.quorum_sample_step()?; let deadline = Instant::now() + REFRESH_TIMEOUT; let client = client(config)?; let tip = block_height(&client)?; @@ -231,10 +246,14 @@ pub fn refresh(config: &Config, memo: &mut KeyMemo) -> LoadResult Ok(list) } }); - let previous = load_window(&client, config, tip, ¤t, memo, deadline); - check_deadline(deadline)?; - if block_height(&client)? != tip { - return Err("Core tip changed during quorum refresh".into()); + let mut previous = load_window(&client, config, tip, ¤t, memo, deadline); + let next_tip = block_height(&client)?; + if next_tip != tip { + let next_active = load_at_height(&client, config, next_tip, memo, deadline, false); + previous = load_window(&client, config, next_tip, &next_active, memo, deadline); + if block_height(&client)? != next_tip { + previous = Err("Core tip changed again while retrying the rolling window".into()); + } } Ok(RefreshResult { tip, @@ -251,11 +270,11 @@ mod tests { fn key_memo_is_bounded_and_evicts_oldest() { let mut memo = KeyMemo::default(); for id in 0..3 { - memo.insert(id.to_string(), vec![id], 2); + memo.insert((id.to_string(), "mined".into()), vec![id], 2); } assert_eq!(memo.keys.len(), 2); - assert!(!memo.keys.contains_key("0")); - assert!(memo.keys.contains_key("2")); + assert!(!memo.keys.contains_key(&("0".into(), "mined".into()))); + assert!(memo.keys.contains_key(&("2".into(), "mined".into()))); assert_eq!(memo.order.len(), 2); } } From b1c66470f8322504812e69f7e0b4711fe60489d4 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 9 Oct 2026 20:41:37 +0700 Subject: [PATCH 03/14] test: strengthen quorum window coverage and compatibility Sweep all 10..18 mining-offset pairs at the maximum safe step, compare warmed and cold caches, verify reorgs retain unchanged key memo entries, and deserialize endpoint output with legacy SDK struct mirrors. Use a controllable monotonic clock for freshness and a gated RPC for runtime responsiveness instead of tight timing thresholds; document each guarantee. These strengthen verification without changing the public behavior. Restart, reorg and unchanged-height TTL tests were already green on the original runtime; the report distinguishes those from red-to-green regressions. All 53 tests pass, and allowing only the existing lint categories leaves clippy clean. Co-Authored-By: GPT-6.1 Sol --- Cargo.toml | 3 + src/api.rs | 22 +++-- src/config.rs | 4 + src/quorum_cache.rs | 3 +- src/quorum_cache_tests.rs | 182 ++++++++++++++++++++++++++++++++++---- src/quorum_loader.rs | 1 + 6 files changed, 193 insertions(+), 22 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9d8b2be..4ea9bb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,3 +46,6 @@ tokio = { version = "1.0", features = ["full"] } [build-dependencies] tonic-build = "0.12" + +[dev-dependencies] +tokio = { version = "1.0", features = ["test-util"] } diff --git a/src/api.rs b/src/api.rs index 477c233..9140a93 100644 --- a/src/api.rs +++ b/src/api.rs @@ -131,9 +131,13 @@ async fn health_check( State((quorum_cache, _, _)): State<(SharedQuorumCache, SharedConfig, SharedMasternodeCache)>, ) -> Json> { if quorum_cache.is_healthy().await { - Json(ApiResponse::success("Quorum List Server is running".to_string())) + Json(ApiResponse::success( + "Quorum List Server is running".to_string(), + )) } else { - Json(ApiResponse::error("Quorum List Server is degraded: quorum refresh is stale or unavailable".to_string())) + Json(ApiResponse::error( + "Quorum List Server is degraded: quorum refresh is stale or unavailable".to_string(), + )) } } @@ -141,7 +145,11 @@ async fn readiness_check( state: State<(SharedQuorumCache, SharedConfig, SharedMasternodeCache)>, ) -> (StatusCode, Json>) { let response = health_check(state).await; - let status = if response.0.success { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE }; + let status = if response.0.success { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; (status, response) } @@ -150,13 +158,17 @@ async fn get_all_quorums( ) -> Result>>, StatusCode> { match quorum_cache.get_snapshot().await { Ok(snapshot) => { - let quorums: Vec = snapshot.quorums.iter().map(|entry| entry.into()).collect(); + let quorums: Vec = + snapshot.quorums.iter().map(|entry| entry.into()).collect(); let mut response = ApiResponse::success(quorums); response.height = Some(snapshot.height); response.last_updated = Some(snapshot.last_updated); Ok(Json(response)) } - Err(e) => Ok(Json(ApiResponse::error(format!("Failed to load quorums: {}", e)))) + Err(e) => Ok(Json(ApiResponse::error(format!( + "Failed to load quorums: {}", + e + )))), } } diff --git a/src/config.rs b/src/config.rs index 1d34556..ec099fc 100644 --- a/src/config.rs +++ b/src/config.rs @@ -415,6 +415,7 @@ previous_blocks_offset = 8 assert_eq!(config.get_llmq_type_id(), 107); } + /// Legacy TOML must gain retention defaults and safe per-network sampling. #[test] fn should_default_legacy_configs_to_a_safe_rolling_window() { let quorum: QuorumConfig = toml::from_str("previous_blocks_offset = 8").unwrap(); @@ -437,6 +438,7 @@ previous_blocks_offset = 8 } } + /// Invalid intervals and arithmetic must fail before background work starts. #[test] fn should_reject_invalid_quorum_intervals_and_limits() { let mut config = Config::default(); @@ -452,6 +454,7 @@ previous_blocks_offset = 8 assert!(config.validate_quorum().is_err()); } + /// Injected environment settings must preserve all quorum knobs without global test races. #[test] fn should_preserve_quorum_env_fallbacks() { let variables = [ @@ -496,6 +499,7 @@ previous_blocks_offset = 8 assert_eq!(config.rpc.url, "http://core.example:9998"); } + /// The shipped TOML must use the same schema as the configuration loader. #[test] fn should_load_the_shipped_config_file() { let config: Config = toml::from_str(include_str!("../config.toml")).unwrap(); diff --git a/src/quorum_cache.rs b/src/quorum_cache.rs index b154b82..7112562 100644 --- a/src/quorum_cache.rs +++ b/src/quorum_cache.rs @@ -2,7 +2,8 @@ use crate::config::Config; use crate::quorum_list::QuorumList; use crate::quorum_loader::{self, KeyMemo, LoadResult, REFRESH_TIMEOUT}; use std::sync::{Arc, Mutex, RwLock}; -use std::time::{Duration, Instant}; +use std::time::Duration; +use tokio::time::Instant; #[derive(Clone)] pub struct QuorumSnapshot { diff --git a/src/quorum_cache_tests.rs b/src/quorum_cache_tests.rs index cd567c5..363a117 100644 --- a/src/quorum_cache_tests.rs +++ b/src/quorum_cache_tests.rs @@ -28,6 +28,8 @@ pub struct CoreState { pub commitments: std::collections::HashMap, pub omit_at_height: Option, pub tip_script: std::collections::VecDeque, + pub first_response_gate: Option>, + pub omit_only: Option, } pub struct MockCore { @@ -61,6 +63,8 @@ impl MockCore { commitments: std::collections::HashMap::new(), omit_at_height: None, tip_script: std::collections::VecDeque::new(), + first_response_gate: None, + omit_only: None, })); let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); let worker_state = state.clone(); @@ -138,7 +142,7 @@ impl MockCore { json!({}) } else { let entries: Vec = state.quorums.iter() - .filter(|q| !state.empty && state.omit_at_height != Some(height) && q.created <= height && height < q.retired) + .filter(|q| !state.empty && (state.omit_at_height != Some(height) || state.omit_only.is_some_and(|id| id != q.id)) && q.created <= height && height < q.retired) .map(|q| json!({hash(q.id): {"creationHeight":q.created, "numValidMembers":12, "minedBlockHash":hash(q.id + 100000 + state.commitments.get(&q.id).copied().unwrap_or(0))}})) .collect(); json!({"llmq_devnet_platform": entries}) @@ -155,7 +159,11 @@ impl MockCore { json!({"result":result, "error":null, "id":request["id"]}) }.to_string(); let delay = state.delay; + let gate = state.first_response_gate.take(); drop(state); + if let Some(gate) = gate { + let _ = gate.recv_timeout(Duration::from_secs(5)); + } std::thread::sleep(delay); let response = format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body); let _ = stream.write_all(response.as_bytes()); @@ -230,6 +238,7 @@ fn jameson() -> MockCore { ) } +/// A cold client must retain a signer retired beyond the legacy snapshot. #[tokio::test] async fn should_serve_a_quorum_retired_after_the_previous_snapshot_height() { let core = jameson(); @@ -248,6 +257,7 @@ async fn should_serve_a_quorum_retired_after_the_previous_snapshot_height() { assert_eq!(height, 13223); } +/// Successful and failed DKG schedules must leave no signer missing from the window. #[tokio::test] async fn should_cover_every_quorum_active_in_the_window() { for failed_dkg in [None, Some(8), Some(9)] { @@ -299,11 +309,15 @@ async fn should_cover_every_quorum_active_in_the_window() { } } +/// A cold restart must reconstruct the same window as a cache warmed through rotations. #[tokio::test] async fn should_return_the_same_window_after_restart() { let core = jameson(); let first = QuorumCache::new(core.config.clone()); - first.refresh().await.unwrap(); + for tip in [13223, 13247, 13271] { + core.state.lock().unwrap().tip = tip; + first.refresh().await.unwrap(); + } let second = QuorumCache::new(core.config.clone()); second.refresh().await.unwrap(); assert_eq!( @@ -312,6 +326,7 @@ async fn should_return_the_same_window_after_restart() { ); } +/// Unchanged commitments must not repeatedly fetch their public keys. #[tokio::test] async fn should_not_call_quorum_info_again_for_memoized_hashes() { let core = jameson(); @@ -326,6 +341,7 @@ async fn should_not_call_quorum_info_again_for_memoized_hashes() { ); } +/// Every key lookup must explicitly exclude secret shares. #[tokio::test] async fn should_never_request_secret_key_shares() { let core = jameson(); @@ -356,10 +372,21 @@ pub async fn endpoint(cache: Arc, config: Config, path: &str) -> Va .call(Request::builder().uri(path).body(Body::empty()).unwrap()) .await .unwrap(); - assert_eq!(response.status(), 200); - serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()).unwrap() + let status = response.status(); + let value: Value = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()).unwrap(); + assert_eq!( + status, + if path == "/ready" && value["success"] == false { + 503 + } else { + 200 + } + ); + value } +/// Height and refresh metadata must describe the cached list while keeping its array shape. #[tokio::test] async fn should_report_the_core_height_on_quorums() { let core = jameson(); @@ -375,6 +402,7 @@ async fn should_report_the_core_height_on_quorums() { ); } +/// Core failures must preserve data and become visible through health and readiness. #[tokio::test] async fn should_keep_last_good_window_and_report_degraded_health_on_core_failure() { let mut core = jameson(); @@ -388,7 +416,10 @@ async fn should_keep_last_good_window_and_report_degraded_health_on_core_failure before, endpoint(cache.clone(), core.config.clone(), "/previous").await ); - tokio::time::sleep(Duration::from_millis(3100)).await; + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(3)).await; + let ready = endpoint(cache.clone(), core.config.clone(), "/ready").await; + assert_eq!(ready["success"], false); let health = endpoint(cache, core.config.clone(), "/health").await; assert_eq!( health["success"], false, @@ -397,6 +428,7 @@ async fn should_keep_last_good_window_and_report_degraded_health_on_core_failure assert!(health["message"].as_str().unwrap().contains("degraded")); } +/// New signing keys must become available without waiting for the cache TTL. #[tokio::test] async fn should_reload_when_the_tip_changes_before_the_ttl() { let core = jameson(); @@ -431,6 +463,7 @@ async fn should_reload_when_the_tip_changes_before_the_ttl() { } } +/// Trimming must preserve active keys and response order must protect newest client cache entries. #[tokio::test] async fn should_order_oldest_to_newest_and_trim_retired_first_at_the_cap() { let quorums = (0..12) @@ -456,6 +489,7 @@ async fn should_order_oldest_to_newest_and_trim_retired_first_at_the_cap() { } } +/// The retention floor and legacy offset must leave no unsampled boundary gap. #[tokio::test] async fn should_sample_the_lower_boundary_and_preserve_legacy_offset_coverage() { let mut core = jameson(); @@ -487,6 +521,7 @@ async fn should_sample_the_lower_boundary_and_preserve_legacy_offset_coverage() assert_eq!(sampled, [13271, 13224, 13177, 13170]); } +/// A failed historical query must never replace a complete window with a partial one. #[tokio::test] async fn should_keep_the_window_when_a_historical_sample_fails() { let core = jameson(); @@ -501,6 +536,7 @@ async fn should_keep_the_window_when_a_historical_sample_fails() { ); } +/// An active-list failure must not skip the independently recoverable rolling window. #[tokio::test] async fn should_attempt_the_window_even_when_the_current_list_fails() { let core = jameson(); @@ -512,6 +548,7 @@ async fn should_attempt_the_window_even_when_the_current_list_fails() { assert!(previous.contains_quorum(&hex::decode(hash(1)).unwrap())); } +/// Malformed or empty upstream data must not become a successful trusted-key response. #[tokio::test] async fn should_never_publish_an_empty_or_malformed_success() { let core = jameson(); @@ -538,11 +575,13 @@ async fn should_never_publish_an_empty_or_malformed_success() { ); } +/// Reorgs must remove orphaned membership without refetching unchanged commitments. #[tokio::test] async fn should_drop_orphaned_quorums_after_a_reorg_without_clearing_the_memo() { let core = jameson(); let cache = QuorumCache::new(core.config.clone()); cache.refresh().await.unwrap(); + let initial_info_calls = core.info_calls().len(); { let mut state = core.state.lock().unwrap(); state.tip -= 1; @@ -554,11 +593,17 @@ async fn should_drop_orphaned_quorums_after_a_reorg_without_clearing_the_memo() }); } cache.refresh().await.unwrap(); + assert_eq!( + core.info_calls().len(), + initial_info_calls + 1, + "unchanged commitments must remain memoized across a reorg" + ); let (_, previous) = cache.get_previous_quorums().await.unwrap(); assert!(!previous.contains_quorum(&hex::decode(hash(5)).unwrap())); assert!(previous.contains_quorum(&hex::decode(hash(6)).unwrap())); } +/// The TTL must force reloads even when Core height does not advance. #[tokio::test] async fn should_reload_unchanged_height_at_the_ttl() { let mut core = jameson(); @@ -587,6 +632,7 @@ async fn should_reload_unchanged_height_at_the_ttl() { } } +/// Unsafe sampling must be rejected when commitment mining offsets shorten active runs. #[tokio::test] async fn should_reject_steps_that_miss_quorums_with_variable_mining_offsets() { let starts: Vec = (0..16) @@ -621,22 +667,27 @@ async fn should_reject_steps_that_miss_quorums_with_variable_mining_offsets() { .contains_quorum(&hex::decode(hash(0)).unwrap())); } +/// A blocked Core RPC must leave the async runtime able to run other tasks. #[tokio::test(flavor = "current_thread")] async fn should_keep_the_async_runtime_responsive_during_blocking_core_calls() { let core = jameson(); - core.state.lock().unwrap().delay = Duration::from_millis(30); + let (release, gate) = std::sync::mpsc::channel(); + core.state.lock().unwrap().first_response_gate = Some(gate); let cache = Arc::new(QuorumCache::new(core.config.clone())); - let start = std::time::Instant::now(); let refresh = tokio::spawn(async move { cache.refresh().await }); - tokio::time::sleep(Duration::from_millis(10)).await; - let elapsed = start.elapsed(); + tokio::time::timeout(Duration::from_secs(10), async { + while core.state.lock().unwrap().calls.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + // This runtime task must be able to release a Core RPC that is still blocked. + release.send(()).unwrap(); refresh.await.unwrap().unwrap(); - assert!( - elapsed < Duration::from_millis(150), - "async timers stalled for {elapsed:?} during blocking RPC" - ); } +/// Pre-activation historical samples must not prevent serving the first active keys. #[tokio::test] async fn should_serve_the_window_when_the_floor_precedes_platform_quorum_activation() { let core = MockCore::new( @@ -680,9 +731,15 @@ async fn should_reject_a_historical_sample_missing_an_older_active_quorum() { let cache = QuorumCache::new(core.config.clone()); cache.refresh().await.unwrap(); let before = cache.get_previous_quorums().await.unwrap(); - core.state.lock().unwrap().omit_at_height = Some(13223); - assert!(cache.refresh().await.is_err()); - assert_eq!(cache.get_previous_quorums().await.unwrap(), before); + for omit_only in [None, Some(2)] { + { + let mut state = core.state.lock().unwrap(); + state.omit_at_height = Some(13223); + state.omit_only = omit_only; + } + assert!(cache.refresh().await.is_err()); + assert_eq!(cache.get_previous_quorums().await.unwrap(), before); + } } /// Readiness gives load balancers a failing status without changing health's HTTP status. @@ -740,3 +797,96 @@ async fn should_publish_current_and_retry_the_window_when_the_tip_moves() { assert!(previous.contains_quorum(&hex::decode(hash(6)).unwrap())); assert!(cache.is_healthy().await); } + +/// Every legal mining-offset pair must leave no hole at the maximum allowed step. +#[tokio::test] +async fn should_cover_all_mining_offset_pairs_at_the_maximum_step() { + for first in 10..=18 { + for replacement in 10..=18 { + let starts: Vec = (0..20) + .map(|cycle| { + cycle * 24 + + match cycle { + 0 => first, + 4 => replacement, + _ => 10, + } + }) + .collect(); + let quorums: Vec = starts + .iter() + .enumerate() + .map(|(i, &created)| Quorum { + id: i as u32, + created, + retired: starts.get(i + 4).copied().unwrap_or(u32::MAX), + }) + .collect(); + let mut core = MockCore::new(250, quorums.clone()); + core.config.quorum.retention_blocks = 288; + core.config.quorum.sample_step_blocks = Some(73); + core.config.validate_quorum().unwrap(); + let cache = QuorumCache::new(core.config.clone()); + for tip in [250u32, 299, 345] { + core.state.lock().unwrap().tip = tip; + cache.refresh().await.unwrap(); + let (floor, window) = cache.get_previous_quorums().await.unwrap(); + let expected: Vec = quorums + .iter() + .filter(|q| q.created <= tip && q.retired > floor) + .map(|q| hash(q.id)) + .collect(); + let actual: Vec = + window.iter().map(|q| hex::encode(&q.quorum_hash)).collect(); + assert_eq!( + actual, expected, + "mining offsets {first}/{replacement}, tip {tip}" + ); + } + } + } +} + +/// The legacy SDK JSON structs must accept actual endpoint responses unchanged. +#[tokio::test] +async fn should_deserialize_responses_into_legacy_sdk_types() { + #[derive(serde::Deserialize)] + struct QuorumData { + quorum_hash: String, + key: String, + height: u64, + valid_members_count: u32, + } + #[derive(serde::Deserialize)] + struct QuorumsResponse { + success: bool, + data: Vec, + } + #[derive(serde::Deserialize)] + struct PreviousQuorumsData { + height: u64, + quorums: Vec, + } + #[derive(serde::Deserialize)] + struct PreviousQuorumsResponse { + success: bool, + data: PreviousQuorumsData, + } + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let current: QuorumsResponse = + serde_json::from_value(endpoint(cache.clone(), core.config.clone(), "/quorums").await) + .unwrap(); + assert!(current.success); + assert_eq!(current.data.len(), 4); + assert_eq!(current.data[0].quorum_hash, hash(2)); + assert_eq!(current.data[0].key.len(), 96); + assert_eq!(current.data[0].height, 13190); + assert_eq!(current.data[0].valid_members_count, 12); + let previous: PreviousQuorumsResponse = + serde_json::from_value(endpoint(cache, core.config.clone(), "/previous").await).unwrap(); + assert!(previous.success); + assert_eq!(previous.data.height, 13223); + assert_eq!(previous.data.quorums.len(), 5); +} diff --git a/src/quorum_loader.rs b/src/quorum_loader.rs index 0dc60a2..dabdfe5 100644 --- a/src/quorum_loader.rs +++ b/src/quorum_loader.rs @@ -266,6 +266,7 @@ pub fn refresh(config: &Config, memo: &mut KeyMemo) -> LoadResult mod tests { use super::*; + /// Key caching must remain bounded even when historical commitments accumulate. #[test] fn key_memo_is_bounded_and_evicts_oldest() { let mut memo = KeyMemo::default(); From 2d0ab1214a75ba96720535e74ea9c625509bbb00 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 9 Oct 2026 21:00:01 +0700 Subject: [PATCH 04/14] fix: preserve mounted server configuration in Docker Remove the image API_HOST and API_PORT defaults so only explicit deployment environment values override config.toml. Dashmate already supplies both values; the binary retains its built-in defaults. Configuration-only change validated with the 8 config tests; Docker build omitted because Docker is not installed. Co-Authored-By: GPT-6.1 Sol --- Dockerfile | 3 --- README.md | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8e490dd..e0df9b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,4 @@ USER appuser EXPOSE 3000 -ENV API_HOST=0.0.0.0 -ENV API_PORT=3000 - CMD ["quorum-list-server"] diff --git a/README.md b/README.md index d233111..e9cb549 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,8 @@ docker run --rm -p 8080:8080 \ ``` The server reads `/app/config.toml` first, then applies environment overrides. +The image does not set `API_HOST` or `API_PORT`; the binary defaults to `0.0.0.0:3000` +when neither the file nor explicitly supplied environment variables configure them. ## API Examples From 46d9fc1bf8c7be8fc795bdb8f2236bf182b9ab94 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 9 Oct 2026 21:07:20 +0700 Subject: [PATCH 05/14] fix: back off quorum failures at an unchanged polled tip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track the last observed Core tip separately from the last successful active snapshot. Repeated polls of a failing tip respect retry_after while newly polled tips still trigger an immediate refresh. Test would have caught this in CI: ✖ before fix, ✔ after. Both paused-time tests fail on b1c6647 with six active RPC attempts where four are expected; both pass with the fix, covering an empty cache and an older successful snapshot plus new-tip bypass. Co-Authored-By: GPT-6.1 Sol --- src/quorum_cache.rs | 148 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 128 insertions(+), 20 deletions(-) diff --git a/src/quorum_cache.rs b/src/quorum_cache.rs index 7112562..712629f 100644 --- a/src/quorum_cache.rs +++ b/src/quorum_cache.rs @@ -25,6 +25,8 @@ pub struct QuorumCache { rpc_worker: Arc>, config: Arc, update_interval: Duration, + #[cfg(test)] + polls_completed: std::sync::atomic::AtomicUsize, } impl QuorumCache { @@ -36,6 +38,8 @@ impl QuorumCache { rpc_worker: Arc::new(tokio::sync::Mutex::new(())), config: Arc::new(config), update_interval: Duration::from_secs(ttl), + #[cfg(test)] + polls_completed: std::sync::atomic::AtomicUsize::new(0), } } @@ -183,6 +187,7 @@ impl QuorumCache { poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let mut failures = 0u32; let mut retry_after = tokio::time::Instant::now(); + let mut last_polled_tip = None; loop { poll.tick().await; let config = self.config.clone(); @@ -190,32 +195,135 @@ impl QuorumCache { .rpc_work(move || quorum_loader::get_current_block_height(&config)) .await { - Ok(tip) => match self.reload_due(tip) { - Ok((changed, true)) - if changed || tokio::time::Instant::now() >= retry_after => - { - match self.refresh().await { - Ok(()) => { - failures = 0; - } - Err(error) => { - eprintln!("Quorum refresh failed: {error}"); - failures = failures.saturating_add(1); - let delay = poll_interval - .saturating_mul(1 << failures.saturating_sub(1).min(5)) - .min(self.update_interval) - .min(Duration::from_secs(30)); - retry_after = tokio::time::Instant::now() + delay; + Ok(tip) => { + let new_tip = last_polled_tip.replace(tip) != Some(tip); + match self.reload_due(tip) { + Ok((_, true)) + if new_tip || tokio::time::Instant::now() >= retry_after => + { + match self.refresh().await { + Ok(()) => { + failures = 0; + } + Err(error) => { + eprintln!("Quorum refresh failed: {error}"); + failures = failures.saturating_add(1); + let delay = poll_interval + .saturating_mul(1 << failures.saturating_sub(1).min(5)) + .min(self.update_interval) + .min(Duration::from_secs(30)); + retry_after = tokio::time::Instant::now() + delay; + } } } + Ok((_, true)) => {} + Ok((_, false)) => {} + Err(error) => eprintln!("Cannot read quorum refresh state: {error}"), } - Ok((_, true)) => {} - Ok((_, false)) => {} - Err(error) => eprintln!("Cannot read quorum refresh state: {error}"), - }, + } Err(error) => eprintln!("Core tip poll failed: {error}"), } + #[cfg(test)] + self.polls_completed + .fetch_add(1, std::sync::atomic::Ordering::Release); } }); } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::quorum_cache_tests::{MockCore, Quorum}; + + async fn finish_poll(cache: &QuorumCache, expected_polls: usize) { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while cache + .polls_completed + .load(std::sync::atomic::Ordering::Acquire) + < expected_polls + { + assert!( + std::time::Instant::now() < deadline, + "Core poll did not finish" + ); + // Keep the paused runtime busy until blocking RPC and retry scheduling finish. + tokio::task::yield_now().await; + } + } + + /// Repeated failures at one tip must back off, even without a successful active list. + #[tokio::test] + async fn should_back_off_active_list_failures_at_an_unchanged_tip() { + check_active_list_backoff(false).await; + } + + /// A failed refresh at a new tip must not keep bypassing backoff using the old snapshot. + #[tokio::test] + async fn should_back_off_active_list_failures_after_a_successful_refresh() { + check_active_list_backoff(true).await; + } + + async fn check_active_list_backoff(populated: bool) { + let core = MockCore::new( + 100, + vec![Quorum { + id: 1, + created: 10, + retired: u32::MAX, + }], + ); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + if populated { + cache.refresh().await.unwrap(); + } + { + let mut state = core.state.lock().unwrap(); + state.tip = 101; + state.fail_current = true; + state.calls.clear(); + } + tokio::time::pause(); + cache.clone().start_background_refresh().await; + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_millis(1)).await; + let active_attempts = || { + core.state + .lock() + .unwrap() + .calls + .iter() + .filter(|call| call["params"][0] == "listextended" && call["params"][1] == 101) + .count() + }; + finish_poll(&cache, 1).await; + assert_eq!(active_attempts(), 2); + tokio::time::advance(Duration::from_secs(1)).await; + finish_poll(&cache, 2).await; + assert_eq!(active_attempts(), 4); + tokio::time::advance(Duration::from_secs(1)).await; + finish_poll(&cache, 3).await; + assert_eq!( + active_attempts(), + 4, + "the second failure must delay the next refresh for two poll intervals" + ); + tokio::time::advance(Duration::from_secs(1)).await; + finish_poll(&cache, 4).await; + assert_eq!(active_attempts(), 6); + core.state.lock().unwrap().tip = 102; + tokio::time::advance(Duration::from_secs(1)).await; + finish_poll(&cache, 5).await; + assert_eq!( + core.state + .lock() + .unwrap() + .calls + .iter() + .filter(|call| call["params"][0] == "listextended" && call["params"][1] == 102) + .count(), + 2, + "a newly polled tip must bypass the four-second retry delay" + ); + } +} From d485c837311481090004aab693f681c1ae9528d3 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 9 Oct 2026 21:08:40 +0700 Subject: [PATCH 06/14] fix: stop quorum history sampling when the tip retry fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve and validate the active list before loading historical samples. An unavailable or empty tip retry now returns immediately while a successful retry still builds the independently recoverable rolling window. Test would have caught this in CI: ✖ before fix, ✔ after. Both mock RPC regressions fail before the fix with one historical listextended call and pass after the fix with zero; both retain the two active-tip attempts and publish no incomplete lists. Co-Authored-By: GPT-6.1 Sol --- src/quorum_cache_tests.rs | 45 +++++++++++++++++++++++++++++++++++++++ src/quorum_loader.rs | 29 +++++++++++-------------- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/src/quorum_cache_tests.rs b/src/quorum_cache_tests.rs index 363a117..db47cfe 100644 --- a/src/quorum_cache_tests.rs +++ b/src/quorum_cache_tests.rs @@ -548,6 +548,51 @@ async fn should_attempt_the_window_even_when_the_current_list_fails() { assert!(previous.contains_quorum(&hex::decode(hash(1)).unwrap())); } +/// An unavailable active list cannot produce a window and must not trigger historical RPCs. +#[tokio::test] +async fn should_skip_historical_queries_when_the_active_tip_retry_fails() { + check_unavailable_active_list_skips_history(false).await; +} + +/// An empty tip retry must stop before spending RPCs on an unusable historical window. +#[tokio::test] +async fn should_skip_historical_queries_when_the_active_tip_retry_is_empty() { + check_unavailable_active_list_skips_history(true).await; +} + +async fn check_unavailable_active_list_skips_history(empty: bool) { + let core = jameson(); + { + let mut state = core.state.lock().unwrap(); + state.fail_current = !empty; + state.empty = empty; + } + let cache = QuorumCache::new(core.config.clone()); + assert!(cache.refresh().await.is_err()); + assert!(cache.get_quorums().await.is_err()); + assert!(cache.get_previous_quorums().await.is_err()); + let sampled: Vec = core + .state + .lock() + .unwrap() + .calls + .iter() + .filter(|call| call["params"][0] == "listextended") + .map(|call| call["params"][1].as_u64().unwrap()) + .collect(); + assert_eq!( + sampled.iter().filter(|&&height| height != 13271).count(), + 0, + "history cannot recover a window without a populated active list at the tip" + ); + assert_eq!( + sampled, + [13271, 13271], + "the active tip must still be retried" + ); + assert!(core.info_calls().is_empty()); +} + /// Malformed or empty upstream data must not become a successful trusted-key response. #[tokio::test] async fn should_never_publish_an_empty_or_malformed_success() { diff --git a/src/quorum_loader.rs b/src/quorum_loader.rs index dabdfe5..ab6b87a 100644 --- a/src/quorum_loader.rs +++ b/src/quorum_loader.rs @@ -177,9 +177,12 @@ fn load_window( let floor = tip.saturating_sub(config.quorum_retention()); // Retry the tip for the window even if the independent active-list fetch failed. let active = match current { - Ok(list) => Ok(list.clone()), - Err(_) => load_at_height(client, config, tip, memo, deadline, false), + Ok(list) => list.clone(), + Err(_) => load_at_height(client, config, tip, memo, deadline, false)?, }; + if active.is_empty() { + return Err("Core returned no active Platform quorums".into()); + } let mut window = QuorumList::new(); let mut height = tip; let mut history_error = None; @@ -188,16 +191,14 @@ fn load_window( height = height.saturating_sub(step).max(floor); match load_at_height(client, config, height, memo, deadline, true) { Ok(list) => { - if let Ok(tip_active) = &active { - for entry in tip_active.iter() { - if u64::from(entry.height) + 18 <= u64::from(height) - && !list.contains_quorum(&entry.quorum_hash) - { - return Err( - "Historical sample is missing a quorum that must still be active" - .into(), - ); - } + for entry in active.iter() { + if u64::from(entry.height) + 18 <= u64::from(height) + && !list.contains_quorum(&entry.quorum_hash) + { + return Err( + "Historical sample is missing a quorum that must still be active" + .into(), + ); } } for entry in list.list { @@ -210,10 +211,6 @@ fn load_window( } } } - let active = active?; - if active.is_empty() { - return Err("Core returned no active Platform quorums".into()); - } if let Some(error) = history_error { return Err(error); } From 656f0e5a47b6eae4ec316fab5c2c3eff73e2a191 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 9 Oct 2026 21:15:08 +0700 Subject: [PATCH 07/14] refactor: clarify quorum refresh eligibility Return only refresh eligibility from reload_due and merge the no-refresh branches. Explain why the paused-time regression begins with a one-millisecond offset. No behavior change; new red-to-green tests are unnecessary. Existing coverage passes: cargo test 57 passed with none ignored, clippy unchanged at 10 existing warnings, and changed-file rustfmt passes. Co-Authored-By: GPT-6.1 Sol --- src/quorum_cache.rs | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/quorum_cache.rs b/src/quorum_cache.rs index 712629f..a39dd62 100644 --- a/src/quorum_cache.rs +++ b/src/quorum_cache.rs @@ -158,7 +158,7 @@ impl QuorumCache { } } - fn reload_due(&self, tip: u32) -> LoadResult<(bool, bool)> { + fn reload_due(&self, tip: u32) -> LoadResult { let state = self .state .read() @@ -166,15 +166,12 @@ impl QuorumCache { Ok(match (&state.current, &state.previous) { (Some(current), Some(previous)) => { let changed = current.height != tip; - ( - changed, - changed - || current.refreshed.elapsed() >= self.update_interval - || previous.refreshed.elapsed() >= self.update_interval - || previous.height != tip.saturating_sub(self.config.quorum_retention()), - ) + changed + || current.refreshed.elapsed() >= self.update_interval + || previous.refreshed.elapsed() >= self.update_interval + || previous.height != tip.saturating_sub(self.config.quorum_retention()) } - _ => (true, true), + _ => true, }) } @@ -198,9 +195,7 @@ impl QuorumCache { Ok(tip) => { let new_tip = last_polled_tip.replace(tip) != Some(tip); match self.reload_due(tip) { - Ok((_, true)) - if new_tip || tokio::time::Instant::now() >= retry_after => - { + Ok(true) if new_tip || tokio::time::Instant::now() >= retry_after => { match self.refresh().await { Ok(()) => { failures = 0; @@ -216,8 +211,7 @@ impl QuorumCache { } } } - Ok((_, true)) => {} - Ok((_, false)) => {} + Ok(_) => {} Err(error) => eprintln!("Cannot read quorum refresh state: {error}"), } } @@ -286,6 +280,7 @@ mod tests { tokio::time::pause(); cache.clone().start_background_refresh().await; tokio::task::yield_now().await; + // Wake the first interval tick; later polls retry exactly when now == retry_after. tokio::time::advance(Duration::from_millis(1)).await; let active_attempts = || { core.state From 79d63ebca779f0c846f2aafbe9e4ec01cf02dcc2 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sun, 11 Oct 2026 17:11:33 +0700 Subject: [PATCH 08/14] fix: publish active quorums before refreshing history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give active and window work separate 30-second budgets, preserve the last good window on verification errors, and report failed windows as degraded until recovery. Test would have caught this in CI: ✖ before fix (2 failed), ✔ after (2 passed); full suite 59 passed, 0 ignored. Co-Authored-By: GPT-6.1 Sol --- README.md | 5 +- src/quorum_cache.rs | 83 +++++++++++++++---------- src/quorum_cache_tests.rs | 123 +++++++++++++++++++++++++++++++++++++- src/quorum_loader.rs | 29 ++++++--- 4 files changed, 197 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index e9cb549..69f5320 100644 --- a/README.md +++ b/README.md @@ -92,8 +92,9 @@ once at the new tip. Repeated failed window reloads back off up to the smaller o 30 seconds and the TTL; a newly observed tip bypasses this backoff. `/health` still returns HTTP 200 and the same JSON shape; -it returns `success:false`, `data:null` and a degraded message when either list -has never loaded or its last successful refresh is older than +it returns `success:false`, `data:null` and a degraded message when the latest +window refresh failed, either list has never loaded, or its last successful +refresh is older than `cache_ttl_seconds * health_stale_intervals` (three intervals by default). `/ready` returns the same body with HTTP 503 while degraded and HTTP 200 when fresh, so load balancers can use it without changing existing `/health` matchers. diff --git a/src/quorum_cache.rs b/src/quorum_cache.rs index a39dd62..b38cf71 100644 --- a/src/quorum_cache.rs +++ b/src/quorum_cache.rs @@ -17,6 +17,7 @@ pub struct QuorumSnapshot { struct CacheState { current: Option, previous: Option, + previous_failed: bool, } pub struct QuorumCache { @@ -88,46 +89,62 @@ impl QuorumCache { .map_err(|_| "Quorum RPC worker timed out")? } - /// Both lists are attempted; failures preserve each list's last good snapshot. - pub async fn refresh(&self) -> LoadResult<()> { + async fn quorum_work(&self, work: F) -> LoadResult + where + T: Send + 'static, + F: FnOnce(&Config, &mut KeyMemo) -> LoadResult + Send + 'static, + { let config = self.config.clone(); let memo = self.memo.clone(); - let result = self - .rpc_work(move || { - let mut memo = memo.lock().unwrap_or_else(|poisoned| { - memo.clear_poison(); - let mut memo = poisoned.into_inner(); - *memo = KeyMemo::default(); - memo - }); - quorum_loader::refresh(&config, &mut memo) - }) - .await?; - let now = Instant::now(); - let last_updated = chrono::Utc::now().timestamp(); + self.rpc_work(move || { + let mut memo = memo.lock().unwrap_or_else(|poisoned| { + memo.clear_poison(); + let mut memo = poisoned.into_inner(); + *memo = KeyMemo::default(); + memo + }); + work(&config, &mut memo) + }) + .await + } + + /// Publish active keys before spending the window's separate RPC budget on history. + pub async fn refresh(&self) -> LoadResult<()> { + let current = self.quorum_work(quorum_loader::refresh_current).await; + let current_error = match ¤t { + Ok(result) => match &result.current { + Ok(quorums) => { + let mut state = self + .state + .write() + .map_err(|_| "Failed to write quorum cache")?; + state.current = Some(QuorumSnapshot { + quorums: quorums.clone(), + height: result.tip, + last_updated: chrono::Utc::now().timestamp(), + refreshed: Instant::now(), + }); + None + } + Err(error) => Some(error.to_string()), + }, + Err(error) => Some(error.to_string()), + }; + let previous = self + .quorum_work(move |config, memo| quorum_loader::refresh_previous(config, memo, current)) + .await; let mut state = self .state .write() .map_err(|_| "Failed to write quorum cache")?; - let current_error = match result.current { - Ok(quorums) => { - state.current = Some(QuorumSnapshot { - quorums, - height: result.tip, - last_updated, - refreshed: now, - }); - None - } - Err(error) => Some(error), - }; - let previous_error = match result.previous { + state.previous_failed = previous.is_err(); + let previous_error = match previous { Ok((height, quorums)) => { state.previous = Some(QuorumSnapshot { quorums, height, - last_updated, - refreshed: now, + last_updated: chrono::Utc::now().timestamp(), + refreshed: Instant::now(), }); None } @@ -135,7 +152,8 @@ impl QuorumCache { }; match (current_error, previous_error) { (None, None) => Ok(()), - (Some(error), None) | (None, Some(error)) => Err(error), + (Some(error), None) => Err(error.into()), + (None, Some(error)) => Err(error), (Some(current), Some(previous)) => { Err(format!("Current quorums: {current}; rolling window: {previous}").into()) } @@ -151,7 +169,8 @@ impl QuorumCache { }; match (&state.current, &state.previous) { (Some(current), Some(previous)) => { - current.refreshed.elapsed() < stale_after + !state.previous_failed + && current.refreshed.elapsed() < stale_after && previous.refreshed.elapsed() < stale_after } _ => false, diff --git a/src/quorum_cache_tests.rs b/src/quorum_cache_tests.rs index db47cfe..f863f2f 100644 --- a/src/quorum_cache_tests.rs +++ b/src/quorum_cache_tests.rs @@ -30,6 +30,8 @@ pub struct CoreState { pub tip_script: std::collections::VecDeque, pub first_response_gate: Option>, pub omit_only: Option, + pub historical_response_gate: Option<(u32, std::sync::mpsc::Receiver<()>)>, + pub fail_block_count_at: Option, } pub struct MockCore { @@ -65,6 +67,8 @@ impl MockCore { tip_script: std::collections::VecDeque::new(), first_response_gate: None, omit_only: None, + historical_response_gate: None, + fail_block_count_at: None, })); let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); let worker_state = state.clone(); @@ -128,6 +132,15 @@ impl MockCore { } let failed = state.fail || fail_once + || (request["method"] == "getblockcount" + && state.fail_block_count_at + == Some( + state + .calls + .iter() + .filter(|call| call["method"] == "getblockcount") + .count(), + )) || (is_list && state.fail_height == Some(height)) || (state.fail_current && is_list && height == state.tip); let result = if request["method"] == "getblockcount" { @@ -159,7 +172,16 @@ impl MockCore { json!({"result":result, "error":null, "id":request["id"]}) }.to_string(); let delay = state.delay; - let gate = state.first_response_gate.take(); + let gate = if is_list + && state + .historical_response_gate + .as_ref() + .is_some_and(|(gated_height, _)| *gated_height == height) + { + state.historical_response_gate.take().map(|(_, gate)| gate) + } else { + state.first_response_gate.take() + }; drop(state); if let Some(gate) = gate { let _ = gate.recv_timeout(Duration::from_secs(5)); @@ -536,6 +558,105 @@ async fn should_keep_the_window_when_a_historical_sample_fails() { ); } +/// New active signing keys must survive a timed-out historical refresh. +#[tokio::test] +async fn should_publish_current_when_the_window_times_out() { + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let previous = cache.get_previous_quorums().await.unwrap(); + let (release, gate) = std::sync::mpsc::channel(); + { + let mut state = core.state.lock().unwrap(); + state.tip += 1; + let created = state.tip; + state.quorums.push(Quorum { + id: 6, + created, + retired: u32::MAX, + }); + state.historical_response_gate = Some((13224, gate)); + state.calls.clear(); + } + let refreshing = cache.clone(); + let refresh = tokio::spawn(async move { refreshing.refresh().await }); + let deadline = std::time::Instant::now() + Duration::from_secs(3); + while !core + .state + .lock() + .unwrap() + .calls + .iter() + .any(|call| call["params"][0] == "listextended" && call["params"][1] == 13224) + { + assert!( + std::time::Instant::now() < deadline, + "historical request did not start" + ); + tokio::task::yield_now().await; + } + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(31)).await; + let result = refresh.await.unwrap(); + release.send(()).unwrap(); + assert!(result.unwrap_err().to_string().contains("timed out")); + let current = endpoint(cache.clone(), core.config.clone(), "/quorums").await; + assert_eq!(current["success"], true); + assert!( + current["data"] + .as_array() + .unwrap() + .iter() + .any(|entry| entry["quorum_hash"] == hash(6)), + "successful active keys must survive the window timeout" + ); + assert_eq!(cache.get_snapshot().await.unwrap().height, 13272); + assert_eq!(cache.get_previous_quorums().await.unwrap(), previous); + assert!(!cache.is_healthy().await); +} + +/// Late verification errors must preserve history without discarding new signing keys. +#[tokio::test] +async fn should_publish_current_when_the_final_tip_verification_fails() { + for failing_call in [2, 3] { + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let previous = cache.get_previous_quorums().await.unwrap(); + { + let mut state = core.state.lock().unwrap(); + state.tip += 1; + let created = state.tip; + state.quorums.push(Quorum { + id: 6, + created, + retired: u32::MAX, + }); + state.calls.clear(); + state.fail_block_count_at = Some(failing_call); + if failing_call == 3 { + state.tip_script = [13272, 13273, 13273].into(); + } + } + assert!(cache.refresh().await.is_err()); + let current = endpoint(cache.clone(), core.config.clone(), "/quorums").await; + assert!( + current["data"] + .as_array() + .unwrap() + .iter() + .any(|entry| entry["quorum_hash"] == hash(6)), + "successful active keys must survive a late getblockcount failure" + ); + assert_eq!(cache.get_snapshot().await.unwrap().height, 13272); + assert_eq!(cache.get_previous_quorums().await.unwrap(), previous); + assert!(!cache.is_healthy().await); + core.state.lock().unwrap().fail_block_count_at = None; + cache.refresh().await.unwrap(); + assert!(cache.is_healthy().await); + } +} + /// An active-list failure must not skip the independently recoverable rolling window. #[tokio::test] async fn should_attempt_the_window_even_when_the_current_list_fails() { diff --git a/src/quorum_loader.rs b/src/quorum_loader.rs index ab6b87a..ea445dc 100644 --- a/src/quorum_loader.rs +++ b/src/quorum_loader.rs @@ -33,10 +33,9 @@ impl KeyMemo { } } -pub struct RefreshResult { +pub struct CurrentResult { pub tip: u32, pub current: LoadResult, - pub previous: LoadResult<(u32, QuorumList)>, } fn client(config: &Config) -> LoadResult { @@ -232,7 +231,7 @@ fn load_window( } /// Blocking RPC work; callers must run this on a blocking worker. -pub fn refresh(config: &Config, memo: &mut KeyMemo) -> LoadResult { +pub fn refresh_current(config: &Config, memo: &mut KeyMemo) -> LoadResult { let deadline = Instant::now() + REFRESH_TIMEOUT; let client = client(config)?; let tip = block_height(&client)?; @@ -243,6 +242,24 @@ pub fn refresh(config: &Config, memo: &mut KeyMemo) -> LoadResult Ok(list) } }); + Ok(CurrentResult { tip, current }) +} + +/// Window failures, including tip verification, are independent of active publication. +pub fn refresh_previous( + config: &Config, + memo: &mut KeyMemo, + current: LoadResult, +) -> LoadResult<(u32, QuorumList)> { + let deadline = Instant::now() + REFRESH_TIMEOUT; + let client = client(config)?; + let CurrentResult { tip, current } = match current { + Ok(result) => result, + Err(error) => CurrentResult { + tip: block_height(&client)?, + current: Err(error), + }, + }; let mut previous = load_window(&client, config, tip, ¤t, memo, deadline); let next_tip = block_height(&client)?; if next_tip != tip { @@ -252,11 +269,7 @@ pub fn refresh(config: &Config, memo: &mut KeyMemo) -> LoadResult previous = Err("Core tip changed again while retrying the rolling window".into()); } } - Ok(RefreshResult { - tip, - current, - previous, - }) + previous } #[cfg(test)] From 7d14b364a955be5472a49f809e629b4d7a12a8dc Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sun, 11 Oct 2026 17:12:41 +0700 Subject: [PATCH 09/14] fix: validate deeper samples against observed quorum history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use both tip-active keys and newer historical samples to reject missing members after their latest possible mining height. Test would have caught this in CI: ✖ before fix (1 failed), ✔ after (1 passed); the 904 sample must retain the quorum observed at 952. Co-Authored-By: GPT-6.1 Sol --- src/quorum_cache_tests.rs | 37 ++++++++++++++++++++++++++++++++++++- src/quorum_loader.rs | 3 ++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/quorum_cache_tests.rs b/src/quorum_cache_tests.rs index f863f2f..6d96c14 100644 --- a/src/quorum_cache_tests.rs +++ b/src/quorum_cache_tests.rs @@ -32,6 +32,7 @@ pub struct CoreState { pub omit_only: Option, pub historical_response_gate: Option<(u32, std::sync::mpsc::Receiver<()>)>, pub fail_block_count_at: Option, + pub creation_heights: std::collections::HashMap, } pub struct MockCore { @@ -69,6 +70,7 @@ impl MockCore { omit_only: None, historical_response_gate: None, fail_block_count_at: None, + creation_heights: std::collections::HashMap::new(), })); let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); let worker_state = state.clone(); @@ -156,7 +158,7 @@ impl MockCore { } else { let entries: Vec = state.quorums.iter() .filter(|q| !state.empty && (state.omit_at_height != Some(height) || state.omit_only.is_some_and(|id| id != q.id)) && q.created <= height && height < q.retired) - .map(|q| json!({hash(q.id): {"creationHeight":q.created, "numValidMembers":12, "minedBlockHash":hash(q.id + 100000 + state.commitments.get(&q.id).copied().unwrap_or(0))}})) + .map(|q| json!({hash(q.id): {"creationHeight":state.creation_heights.get(&q.id).copied().unwrap_or(q.created), "numValidMembers":12, "minedBlockHash":hash(q.id + 100000 + state.commitments.get(&q.id).copied().unwrap_or(0))}})) .collect(); json!({"llmq_devnet_platform": entries}) } @@ -908,6 +910,39 @@ async fn should_reject_a_historical_sample_missing_an_older_active_quorum() { } } +/// Newer history must prove completeness after tip-active quorums cease to constrain samples. +#[tokio::test] +async fn should_reject_a_missing_deep_sample_proven_by_newer_history() { + let mining_heights = [858, 882, 898, 922, 946, 970, 994]; + let quorums = mining_heights + .iter() + .enumerate() + .map(|(id, &created)| Quorum { + id: id as u32, + created, + retired: mining_heights.get(id + 4).copied().unwrap_or(u32::MAX), + }) + .collect(); + let mut core = MockCore::new(1000, quorums); + core.config.quorum.retention_blocks = 144; + core.config.quorum.sample_step_blocks = Some(48); + core.state.lock().unwrap().creation_heights = (0..7).map(|id| (id, 840 + id * 24)).collect(); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let before = cache.get_previous_quorums().await.unwrap(); + assert!( + before.1.contains_quorum(&hex::decode(hash(0)).unwrap()), + "the quorum mined at 858 and retired at 946 appears only at the 904 sample" + ); + core.state.lock().unwrap().omit_at_height = Some(904); + let error = cache + .refresh() + .await + .expect_err("a quorum created at 864 and observed at 952 must exist at 904"); + assert!(error.to_string().contains("must still be active")); + assert_eq!(cache.get_previous_quorums().await.unwrap(), before); +} + /// Readiness gives load balancers a failing status without changing health's HTTP status. #[tokio::test] async fn should_return_service_unavailable_when_not_ready() { diff --git a/src/quorum_loader.rs b/src/quorum_loader.rs index ea445dc..6779e64 100644 --- a/src/quorum_loader.rs +++ b/src/quorum_loader.rs @@ -190,7 +190,8 @@ fn load_window( height = height.saturating_sub(step).max(floor); match load_at_height(client, config, height, memo, deadline, true) { Ok(list) => { - for entry in active.iter() { + // A quorum observed later was already active after its latest mining height. + for entry in active.iter().chain(window.iter()) { if u64::from(entry.height) + 18 <= u64::from(height) && !list.contains_quorum(&entry.quorum_hash) { From ff0ea827a26d8bfceba81e0b4f8be7f987b0244e Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sun, 11 Oct 2026 17:15:02 +0700 Subject: [PATCH 10/14] fix: verify mined commitments before memoizing quorum keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match quorum info minedBlock to listextended minedBlockHash before caching a public key. The mock models a commitment switch between the two RPCs and a return to the original fork. Test would have caught this in CI: ✖ before fix (1 failed), ✔ after (1 passed); mismatches propagate without inserting into the key memo. Co-Authored-By: GPT-6.1 Sol --- src/quorum_cache_tests.rs | 11 +++++++++- src/quorum_loader.rs | 44 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/quorum_cache_tests.rs b/src/quorum_cache_tests.rs index 6d96c14..07bc6e8 100644 --- a/src/quorum_cache_tests.rs +++ b/src/quorum_cache_tests.rs @@ -33,6 +33,7 @@ pub struct CoreState { pub historical_response_gate: Option<(u32, std::sync::mpsc::Receiver<()>)>, pub fail_block_count_at: Option, pub creation_heights: std::collections::HashMap, + pub commitment_change_on_info: Option<(u32, u32)>, } pub struct MockCore { @@ -71,6 +72,7 @@ impl MockCore { historical_response_gate: None, fail_block_count_at: None, creation_heights: std::collections::HashMap::new(), + commitment_change_on_info: None, })); let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); let worker_state = state.clone(); @@ -165,8 +167,15 @@ impl MockCore { } else { let id = u32::from_str_radix(&request["params"][2].as_str().unwrap()[56..], 16) .unwrap(); + if state + .commitment_change_on_info + .is_some_and(|(quorum, _)| quorum == id) + { + let (_, commitment) = state.commitment_change_on_info.take().unwrap(); + state.commitments.insert(id, commitment); + } let commitment = state.commitments.get(&id).copied().unwrap_or(0); - json!({"quorumPublicKey": format!("{:02x}", 17 + commitment).repeat(48)}) + json!({"quorumPublicKey": format!("{:02x}", 17 + commitment).repeat(48), "minedBlock":hash(id + 100000 + commitment)}) }; let body = if failed { json!({"result":null, "error":{"code":-1,"message":"Core unavailable"}, "id":request["id"]}) diff --git a/src/quorum_loader.rs b/src/quorum_loader.rs index 6779e64..c5d86ea 100644 --- a/src/quorum_loader.rs +++ b/src/quorum_loader.rs @@ -130,6 +130,9 @@ fn load_at_height( if key.len() != 48 { return Err("Invalid quorum public key length".into()); } + if info.get("minedBlock").and_then(Value::as_str) != Some(mined_hash) { + return Err("Quorum commitment changed during key lookup".into()); + } memo.insert(memo_key, key.clone(), config.quorum.key_memo_max_entries); key } @@ -276,6 +279,47 @@ pub fn refresh_previous( #[cfg(test)] mod tests { use super::*; + use crate::quorum_cache_tests::{hash, MockCore, Quorum}; + + /// A fork switch during key lookup must not memoize the replacement under the old commitment. + #[test] + fn should_reject_a_commitment_change_between_list_and_info_without_memoizing() { + let core = MockCore::new( + 100, + vec![Quorum { + id: 1, + created: 10, + retired: u32::MAX, + }], + ); + core.state.lock().unwrap().commitment_change_on_info = Some((1, 1)); + let mut memo = KeyMemo::default(); + let result = refresh_current(&core.config, &mut memo).unwrap(); + let error = result.current.expect_err( + "the info key belongs to a different mined commitment than the listed quorum", + ); + assert!(error.to_string().contains("commitment changed")); + assert!( + memo.keys.is_empty(), + "the mismatched commitment must not populate the memo" + ); + assert!(memo.order.is_empty()); + core.state.lock().unwrap().commitments.clear(); + let restored = refresh_current(&core.config, &mut memo) + .unwrap() + .current + .unwrap(); + assert_eq!( + restored + .get_entry(&hex::decode(hash(1)).unwrap()) + .unwrap() + .key, + vec![17; 48], + "returning to the original fork must fetch its original key" + ); + assert_eq!(core.info_calls().len(), 2); + assert_eq!(memo.keys.len(), 1); + } /// Key caching must remain bounded even when historical commitments accumulate. #[test] From 4ce2cd29a0c39dc241b68000775c7294061a8057 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sun, 11 Oct 2026 17:15:14 +0700 Subject: [PATCH 11/14] test: cover RPC permit ownership after waiter timeout Gate an active worker through its paused-time timeout, start a second refresh, and verify the owned permit and RPC exclusion until release. Hold the second active response to prove the successfully completed detached first result never publishes. Coverage only: the new regression was already green (1 passed) with the existing permit ownership implementation; no production behavior changed. Co-Authored-By: GPT-6.1 Sol --- src/quorum_cache.rs | 94 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/src/quorum_cache.rs b/src/quorum_cache.rs index b38cf71..95c2866 100644 --- a/src/quorum_cache.rs +++ b/src/quorum_cache.rs @@ -249,6 +249,100 @@ mod tests { use super::*; use crate::quorum_cache_tests::{MockCore, Quorum}; + /// A timed-out waiter cannot release its worker permit or publish the detached result. + #[tokio::test] + async fn should_hold_the_rpc_permit_after_the_active_waiter_times_out() { + let core = MockCore::new( + 100, + vec![Quorum { + id: 1, + created: 10, + retired: u32::MAX, + }], + ); + let (release_first, first_gate) = std::sync::mpsc::channel(); + core.state.lock().unwrap().first_response_gate = Some(first_gate); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + let first_cache = cache.clone(); + let first = tokio::spawn(async move { + first_cache + .quorum_work(quorum_loader::refresh_current) + .await + }); + let deadline = std::time::Instant::now() + Duration::from_secs(3); + while core.state.lock().unwrap().calls.is_empty() { + assert!( + std::time::Instant::now() < deadline, + "first worker did not start" + ); + tokio::task::yield_now().await; + } + tokio::time::pause(); + tokio::time::advance(REFRESH_TIMEOUT + Duration::from_secs(1)).await; + let error = first + .await + .unwrap() + .err() + .expect("the first RPC waiter must time out"); + assert!(error.to_string().contains("timed out")); + assert!( + cache.rpc_worker.try_lock().is_err(), + "the detached blocking closure must still own the permit" + ); + + let (release_second, second_gate) = std::sync::mpsc::channel(); + { + let mut state = core.state.lock().unwrap(); + state.tip = 101; + state.historical_response_gate = Some((101, second_gate)); + } + let second_cache = cache.clone(); + let (started, starting) = tokio::sync::oneshot::channel(); + let second = tokio::spawn(async move { + started.send(()).unwrap(); + second_cache.refresh().await + }); + starting.await.unwrap(); + assert!(!second.is_finished()); + assert_eq!(core.state.lock().unwrap().calls.len(), 1); + assert!(cache.get_snapshot().await.is_err()); + + release_first.send(()).unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(3); + while !core + .state + .lock() + .unwrap() + .calls + .iter() + .any(|call| call["params"][0] == "listextended" && call["params"][1] == 101) + { + assert!( + std::time::Instant::now() < deadline, + "the second worker did not start after the first was released" + ); + tokio::task::yield_now().await; + } + // The first closure completed successfully; the second result is still gated. + assert_eq!(core.info_calls().len(), 1); + assert!( + cache.get_snapshot().await.is_err(), + "the detached first active result must never publish a snapshot" + ); + assert!(cache.get_previous_quorums().await.is_err()); + release_second.send(()).unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(3); + while !second.is_finished() { + assert!( + std::time::Instant::now() < deadline, + "the second refresh did not finish" + ); + tokio::task::yield_now().await; + } + second.await.unwrap().unwrap(); + assert_eq!(cache.get_snapshot().await.unwrap().height, 101); + } + async fn finish_poll(cache: &QuorumCache, expected_polls: usize) { let deadline = std::time::Instant::now() + Duration::from_secs(5); while cache From decf10455e425b8c5da3aa1ef2d51b203a7764c6 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sun, 11 Oct 2026 17:26:19 +0700 Subject: [PATCH 12/14] fix: retain staleness-based quorum readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preserve the established readiness rule after independent review: transient window failures keep fresh trusted snapshots ready, while persistent history failures become degraded at the staleness boundary even if active signing keys continue refreshing. Test would have caught this in CI: ✖ before correction (1 failed), ✔ after (1 passed), including ready responses before and after the history staleness threshold. Co-Authored-By: GPT-6.1 Sol --- README.md | 5 ++--- src/quorum_cache.rs | 5 +---- src/quorum_cache_tests.rs | 46 +++++++++++++++++++++++++++++++++++---- 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 69f5320..e9cb549 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,8 @@ once at the new tip. Repeated failed window reloads back off up to the smaller o 30 seconds and the TTL; a newly observed tip bypasses this backoff. `/health` still returns HTTP 200 and the same JSON shape; -it returns `success:false`, `data:null` and a degraded message when the latest -window refresh failed, either list has never loaded, or its last successful -refresh is older than +it returns `success:false`, `data:null` and a degraded message when either list +has never loaded or its last successful refresh is older than `cache_ttl_seconds * health_stale_intervals` (three intervals by default). `/ready` returns the same body with HTTP 503 while degraded and HTTP 200 when fresh, so load balancers can use it without changing existing `/health` matchers. diff --git a/src/quorum_cache.rs b/src/quorum_cache.rs index 95c2866..37efa17 100644 --- a/src/quorum_cache.rs +++ b/src/quorum_cache.rs @@ -17,7 +17,6 @@ pub struct QuorumSnapshot { struct CacheState { current: Option, previous: Option, - previous_failed: bool, } pub struct QuorumCache { @@ -137,7 +136,6 @@ impl QuorumCache { .state .write() .map_err(|_| "Failed to write quorum cache")?; - state.previous_failed = previous.is_err(); let previous_error = match previous { Ok((height, quorums)) => { state.previous = Some(QuorumSnapshot { @@ -169,8 +167,7 @@ impl QuorumCache { }; match (&state.current, &state.previous) { (Some(current), Some(previous)) => { - !state.previous_failed - && current.refreshed.elapsed() < stale_after + current.refreshed.elapsed() < stale_after && previous.refreshed.elapsed() < stale_after } _ => false, diff --git a/src/quorum_cache_tests.rs b/src/quorum_cache_tests.rs index 07bc6e8..bd38e71 100644 --- a/src/quorum_cache_tests.rs +++ b/src/quorum_cache_tests.rs @@ -435,7 +435,7 @@ async fn should_report_the_core_height_on_quorums() { ); } -/// Core failures must preserve data and become visible through health and readiness. +/// Window failures preserve trusted data and degrade readiness only once history is stale. #[tokio::test] async fn should_keep_last_good_window_and_report_degraded_health_on_core_failure() { let mut core = jameson(); @@ -443,14 +443,52 @@ async fn should_keep_last_good_window_and_report_degraded_health_on_core_failure let cache = Arc::new(QuorumCache::new(core.config.clone())); cache.refresh().await.unwrap(); let before = endpoint(cache.clone(), core.config.clone(), "/previous").await; - core.state.lock().unwrap().fail = true; + core.state.lock().unwrap().fail_height = Some(13223); assert!(cache.refresh().await.is_err()); assert_eq!( before, endpoint(cache.clone(), core.config.clone(), "/previous").await ); + assert_eq!( + endpoint(cache.clone(), core.config.clone(), "/ready").await["success"], + true, + "one failed window must not reject still-fresh trusted data" + ); tokio::time::pause(); tokio::time::advance(Duration::from_secs(3)).await; + { + let mut state = core.state.lock().unwrap(); + state.tip = 13272; + state.fail_height = Some(13224); + state.quorums.push(Quorum { + id: 6, + created: 13272, + retired: u32::MAX, + }); + } + let refreshing = cache.clone(); + let refresh = tokio::spawn(async move { refreshing.refresh().await }); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while !refresh.is_finished() { + assert!( + std::time::Instant::now() < deadline, + "refresh did not finish" + ); + tokio::task::yield_now().await; + } + assert!(refresh.await.unwrap().is_err()); + assert!( + cache + .get_quorums() + .await + .unwrap() + .contains_quorum(&hex::decode(hash(6)).unwrap()), + "current keys must refresh even while history remains stale" + ); + assert_eq!( + before, + endpoint(cache.clone(), core.config.clone(), "/previous").await + ); let ready = endpoint(cache.clone(), core.config.clone(), "/ready").await; assert_eq!(ready["success"], false); let health = endpoint(cache, core.config.clone(), "/health").await; @@ -623,7 +661,7 @@ async fn should_publish_current_when_the_window_times_out() { ); assert_eq!(cache.get_snapshot().await.unwrap().height, 13272); assert_eq!(cache.get_previous_quorums().await.unwrap(), previous); - assert!(!cache.is_healthy().await); + assert!(cache.is_healthy().await); } /// Late verification errors must preserve history without discarding new signing keys. @@ -661,7 +699,7 @@ async fn should_publish_current_when_the_final_tip_verification_fails() { ); assert_eq!(cache.get_snapshot().await.unwrap().height, 13272); assert_eq!(cache.get_previous_quorums().await.unwrap(), previous); - assert!(!cache.is_healthy().await); + assert!(cache.is_healthy().await); core.state.lock().unwrap().fail_block_count_at = None; cache.refresh().await.unwrap(); assert!(cache.is_healthy().await); From c4f21aab33e280b5c28fda97a1890e4ad78debe2 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sun, 11 Oct 2026 17:28:51 +0700 Subject: [PATCH 13/14] fix: clarify quorum commitment errors and timeout coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Distinguish missing quorum info minedBlock from a commitment mismatch, name the latest Platform mining offset, and use generous wall-clock gates and polling deadlines with descriptive release errors. Exercise a full refresh timeout during its window stage, gate the next active response, and prove the detached first window cannot publish previous data or alter the successful active snapshot. Test would have caught this in CI: ✖ missing minedBlock diagnostic before correction (1 failed), ✔ after (1 passed). Stronger permit/publication coverage is green; full suite 63 passed, 0 ignored. Co-Authored-By: GPT-6.1 Sol --- src/quorum_cache.rs | 73 +++++++++++++++++++++++---------------- src/quorum_cache_tests.rs | 21 +++++++---- src/quorum_loader.rs | 33 ++++++++++++++++-- 3 files changed, 89 insertions(+), 38 deletions(-) diff --git a/src/quorum_cache.rs b/src/quorum_cache.rs index 37efa17..723806e 100644 --- a/src/quorum_cache.rs +++ b/src/quorum_cache.rs @@ -246,9 +246,9 @@ mod tests { use super::*; use crate::quorum_cache_tests::{MockCore, Quorum}; - /// A timed-out waiter cannot release its worker permit or publish the detached result. + /// A timed-out window waiter cannot release its worker permit or publish detached history. #[tokio::test] - async fn should_hold_the_rpc_permit_after_the_active_waiter_times_out() { + async fn should_hold_the_rpc_permit_after_the_window_waiter_times_out() { let core = MockCore::new( 100, vec![Quorum { @@ -258,29 +258,30 @@ mod tests { }], ); let (release_first, first_gate) = std::sync::mpsc::channel(); - core.state.lock().unwrap().first_response_gate = Some(first_gate); + core.state.lock().unwrap().historical_response_gate = Some((52, first_gate)); let cache = Arc::new(QuorumCache::new(core.config.clone())); let first_cache = cache.clone(); - let first = tokio::spawn(async move { - first_cache - .quorum_work(quorum_loader::refresh_current) - .await - }); - let deadline = std::time::Instant::now() + Duration::from_secs(3); - while core.state.lock().unwrap().calls.is_empty() { + let first = tokio::spawn(async move { first_cache.refresh().await }); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while !core + .state + .lock() + .unwrap() + .calls + .iter() + .any(|call| call["params"][0] == "listextended" && call["params"][1] == 52) + { assert!( std::time::Instant::now() < deadline, - "first worker did not start" + "first window worker did not start" ); tokio::task::yield_now().await; } + let published_current = cache.get_snapshot().await.unwrap(); + let gated_call_count = core.state.lock().unwrap().calls.len(); tokio::time::pause(); tokio::time::advance(REFRESH_TIMEOUT + Duration::from_secs(1)).await; - let error = first - .await - .unwrap() - .err() - .expect("the first RPC waiter must time out"); + let error = first.await.unwrap().unwrap_err(); assert!(error.to_string().contains("timed out")); assert!( cache.rpc_worker.try_lock().is_err(), @@ -290,8 +291,7 @@ mod tests { let (release_second, second_gate) = std::sync::mpsc::channel(); { let mut state = core.state.lock().unwrap(); - state.tip = 101; - state.historical_response_gate = Some((101, second_gate)); + state.historical_response_gate = Some((100, second_gate)); } let second_cache = cache.clone(); let (started, starting) = tokio::sync::oneshot::channel(); @@ -301,18 +301,26 @@ mod tests { }); starting.await.unwrap(); assert!(!second.is_finished()); - assert_eq!(core.state.lock().unwrap().calls.len(), 1); - assert!(cache.get_snapshot().await.is_err()); + assert_eq!(core.state.lock().unwrap().calls.len(), gated_call_count); + assert_eq!( + cache.get_snapshot().await.unwrap().refreshed, + published_current.refreshed + ); + assert!(cache.get_previous_quorums().await.is_err()); - release_first.send(()).unwrap(); - let deadline = std::time::Instant::now() + Duration::from_secs(3); - while !core + release_first + .send(()) + .expect("the first window RPC must still be gated after its waiter timed out"); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while core .state .lock() .unwrap() .calls .iter() - .any(|call| call["params"][0] == "listextended" && call["params"][1] == 101) + .filter(|call| call["params"][0] == "listextended" && call["params"][1] == 100) + .count() + < 2 { assert!( std::time::Instant::now() < deadline, @@ -322,13 +330,17 @@ mod tests { } // The first closure completed successfully; the second result is still gated. assert_eq!(core.info_calls().len(), 1); + let current = cache.get_snapshot().await.unwrap(); + assert_eq!(current.refreshed, published_current.refreshed); + assert_eq!(current.quorums, published_current.quorums); assert!( - cache.get_snapshot().await.is_err(), - "the detached first active result must never publish a snapshot" + cache.get_previous_quorums().await.is_err(), + "the detached first window must never publish a previous snapshot" ); - assert!(cache.get_previous_quorums().await.is_err()); - release_second.send(()).unwrap(); - let deadline = std::time::Instant::now() + Duration::from_secs(3); + release_second + .send(()) + .expect("the second active response must still be gated"); + let deadline = std::time::Instant::now() + Duration::from_secs(30); while !second.is_finished() { assert!( std::time::Instant::now() < deadline, @@ -337,7 +349,8 @@ mod tests { tokio::task::yield_now().await; } second.await.unwrap().unwrap(); - assert_eq!(cache.get_snapshot().await.unwrap().height, 101); + assert!(cache.get_snapshot().await.unwrap().refreshed > published_current.refreshed); + assert_eq!(cache.get_previous_quorums().await.unwrap().0, 52); } async fn finish_poll(cache: &QuorumCache, expected_polls: usize) { diff --git a/src/quorum_cache_tests.rs b/src/quorum_cache_tests.rs index bd38e71..0df6455 100644 --- a/src/quorum_cache_tests.rs +++ b/src/quorum_cache_tests.rs @@ -1,5 +1,6 @@ use crate::config::{Config, Network}; use crate::quorum_cache::QuorumCache; +use crate::quorum_loader::LATEST_MINING_OFFSET; use serde_json::{json, Value}; use std::io::{Read, Write}; use std::net::TcpListener; @@ -34,6 +35,7 @@ pub struct CoreState { pub fail_block_count_at: Option, pub creation_heights: std::collections::HashMap, pub commitment_change_on_info: Option<(u32, u32)>, + pub omit_info_mined_block: bool, } pub struct MockCore { @@ -73,6 +75,7 @@ impl MockCore { fail_block_count_at: None, creation_heights: std::collections::HashMap::new(), commitment_change_on_info: None, + omit_info_mined_block: false, })); let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); let worker_state = state.clone(); @@ -175,7 +178,11 @@ impl MockCore { state.commitments.insert(id, commitment); } let commitment = state.commitments.get(&id).copied().unwrap_or(0); - json!({"quorumPublicKey": format!("{:02x}", 17 + commitment).repeat(48), "minedBlock":hash(id + 100000 + commitment)}) + let mut info = json!({"quorumPublicKey": format!("{:02x}", 17 + commitment).repeat(48), "minedBlock":hash(id + 100000 + commitment)}); + if state.omit_info_mined_block { + info.as_object_mut().unwrap().remove("minedBlock"); + } + info }; let body = if failed { json!({"result":null, "error":{"code":-1,"message":"Core unavailable"}, "id":request["id"]}) @@ -195,7 +202,7 @@ impl MockCore { }; drop(state); if let Some(gate) = gate { - let _ = gate.recv_timeout(Duration::from_secs(5)); + let _ = gate.recv_timeout(Duration::from_secs(30)); } std::thread::sleep(delay); let response = format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body); @@ -629,7 +636,7 @@ async fn should_publish_current_when_the_window_times_out() { } let refreshing = cache.clone(); let refresh = tokio::spawn(async move { refreshing.refresh().await }); - let deadline = std::time::Instant::now() + Duration::from_secs(3); + let deadline = std::time::Instant::now() + Duration::from_secs(30); while !core .state .lock() @@ -647,7 +654,9 @@ async fn should_publish_current_when_the_window_times_out() { tokio::time::pause(); tokio::time::advance(Duration::from_secs(31)).await; let result = refresh.await.unwrap(); - release.send(()).unwrap(); + release + .send(()) + .expect("the gated historical RPC must still be waiting after the async timeout"); assert!(result.unwrap_err().to_string().contains("timed out")); let current = endpoint(cache.clone(), core.config.clone(), "/quorums").await; assert_eq!(current["success"], true); @@ -1049,8 +1058,8 @@ async fn should_publish_current_and_retry_the_window_when_the_tip_moves() { /// Every legal mining-offset pair must leave no hole at the maximum allowed step. #[tokio::test] async fn should_cover_all_mining_offset_pairs_at_the_maximum_step() { - for first in 10..=18 { - for replacement in 10..=18 { + for first in 10..=LATEST_MINING_OFFSET { + for replacement in 10..=LATEST_MINING_OFFSET { let starts: Vec = (0..20) .map(|cycle| { cycle * 24 diff --git a/src/quorum_loader.rs b/src/quorum_loader.rs index c5d86ea..6000751 100644 --- a/src/quorum_loader.rs +++ b/src/quorum_loader.rs @@ -8,6 +8,8 @@ use std::time::{Duration, Instant}; pub type LoadResult = Result>; pub const REFRESH_TIMEOUT: Duration = Duration::from_secs(30); +// Platform quorum commitments are mined at offsets 10..18 after the DKG base height. +pub const LATEST_MINING_OFFSET: u32 = 18; /// Only immutable public keys are memoized; membership always comes from Core. #[derive(Default)] @@ -130,7 +132,11 @@ fn load_at_height( if key.len() != 48 { return Err("Invalid quorum public key length".into()); } - if info.get("minedBlock").and_then(Value::as_str) != Some(mined_hash) { + let info_mined_hash = info + .get("minedBlock") + .and_then(Value::as_str) + .ok_or("Missing quorum minedBlock")?; + if info_mined_hash != mined_hash { return Err("Quorum commitment changed during key lookup".into()); } memo.insert(memo_key, key.clone(), config.quorum.key_memo_max_entries); @@ -195,7 +201,8 @@ fn load_window( Ok(list) => { // A quorum observed later was already active after its latest mining height. for entry in active.iter().chain(window.iter()) { - if u64::from(entry.height) + 18 <= u64::from(height) + if u64::from(entry.height) + u64::from(LATEST_MINING_OFFSET) + <= u64::from(height) && !list.contains_quorum(&entry.quorum_hash) { return Err( @@ -281,6 +288,28 @@ mod tests { use super::*; use crate::quorum_cache_tests::{hash, MockCore, Quorum}; + /// Missing commitment identity must be reported separately from a detected fork switch. + #[test] + fn should_report_a_missing_info_mined_block_without_memoizing() { + let core = MockCore::new( + 100, + vec![Quorum { + id: 1, + created: 10, + retired: u32::MAX, + }], + ); + core.state.lock().unwrap().omit_info_mined_block = true; + let mut memo = KeyMemo::default(); + let error = refresh_current(&core.config, &mut memo) + .unwrap() + .current + .unwrap_err(); + assert_eq!(error.to_string(), "Missing quorum minedBlock"); + assert!(memo.keys.is_empty()); + assert!(memo.order.is_empty()); + } + /// A fork switch during key lookup must not memoize the replacement under the old commitment. #[test] fn should_reject_a_commitment_change_between_list_and_info_without_memoizing() { From 5ecf1db06f0d966182b6d754a61ac11217983a74 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sun, 11 Oct 2026 19:17:08 +0700 Subject: [PATCH 14/14] fix: bound quorum RPC permit admission by the refresh timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Include the owned RPC permit wait in each stage timeout while retaining ownership in the blocking closure. A timed-out active stage can no longer hang refresh indefinitely while history waits for its permit. The gated, paused-time regression keeps the active worker stuck through both stage budgets, bounds its own wait, and checks that late completion preserves both snapshots and staleness-based readiness. Test would have caught this in CI: ✖ before fix (1 failed on the admission wait deadline), ✔ after fix (the same regression passed). Validation: cargo test 64 passed, 0 failed, 0 ignored; clippy retains the 10 existing warnings; changed-file rustfmt and diff checks pass. Co-Authored-By: GPT-6.1 Sol --- src/quorum_cache.rs | 115 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 2 deletions(-) diff --git a/src/quorum_cache.rs b/src/quorum_cache.rs index 723806e..73f57c1 100644 --- a/src/quorum_cache.rs +++ b/src/quorum_cache.rs @@ -70,14 +70,14 @@ impl QuorumCache { Ok((previous.height, previous.quorums.clone())) } - /// The permit remains with the blocking worker even if its async waiter times out. + /// Admission shares the timeout, while detached blocking work retains its permit. async fn rpc_work(&self, work: F) -> LoadResult where T: Send + 'static, F: FnOnce() -> LoadResult + Send + 'static, { - let permit = self.rpc_worker.clone().lock_owned().await; tokio::time::timeout(REFRESH_TIMEOUT, async { + let permit = self.rpc_worker.clone().lock_owned().await; tokio::task::spawn_blocking(move || { let _permit = permit; work() @@ -246,6 +246,117 @@ mod tests { use super::*; use crate::quorum_cache_tests::{MockCore, Quorum}; + /// A stuck active worker must bound history admission and never publish its detached result. + #[tokio::test] + async fn should_return_when_a_timed_out_active_worker_blocks_window_admission() { + let core = MockCore::new( + 100, + vec![Quorum { + id: 1, + created: 10, + retired: u32::MAX, + }], + ); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let current = cache.get_snapshot().await.unwrap(); + let previous = cache.state.read().unwrap().previous.clone().unwrap(); + let (release, gate) = std::sync::mpsc::channel(); + { + let mut state = core.state.lock().unwrap(); + state.tip = 101; + state.quorums.push(Quorum { + id: 2, + created: 101, + retired: u32::MAX, + }); + state.historical_response_gate = Some((101, gate)); + state.calls.clear(); + } + let mut refresh = Box::pin(cache.refresh()); + assert!(futures::poll!(&mut refresh).is_pending()); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while !core + .state + .lock() + .unwrap() + .calls + .iter() + .any(|call| call["params"][0] == "listextended" && call["params"][1] == 101) + { + assert!( + std::time::Instant::now() < deadline, + "the active worker did not reach its gated response" + ); + tokio::task::yield_now().await; + } + tokio::time::pause(); + let active_budget = tokio::time::sleep(REFRESH_TIMEOUT); + tokio::time::advance(REFRESH_TIMEOUT + Duration::from_secs(1)).await; + active_budget.await; + // Keep the active response gated through the window's separate admission budget. + let result = { + let result = + tokio::time::timeout(REFRESH_TIMEOUT + Duration::from_secs(1), &mut refresh); + tokio::pin!(result); + // Poll the expired active stage to start the window's admission timeout. + assert!(futures::poll!(&mut result).is_pending()); + tokio::time::advance(REFRESH_TIMEOUT + Duration::from_secs(2)).await; + result.await + }; + drop(refresh); + assert!( + cache.rpc_worker.try_lock().is_err(), + "the active worker must still own the permit through both stage timeouts" + ); + assert_eq!(core.state.lock().unwrap().calls.len(), 2); + release + .send(()) + .expect("the active response must stay gated through the admission timeout"); + let error = result + .expect("refresh hung waiting for a timed-out active worker's permit") + .unwrap_err(); + assert!(error + .to_string() + .contains("Current quorums: Quorum RPC worker timed out; rolling window: Quorum RPC worker timed out")); + + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while cache.rpc_worker.try_lock().is_err() { + assert!( + std::time::Instant::now() < deadline, + "the detached active worker did not finish after release" + ); + tokio::task::yield_now().await; + } + assert_eq!(core.info_calls().len(), 1); + let snapshot = cache.get_snapshot().await.unwrap(); + assert_eq!(snapshot.height, current.height); + assert_eq!(snapshot.quorums, current.quorums); + assert_eq!(snapshot.refreshed, current.refreshed); + assert_eq!( + cache.get_previous_quorums().await.unwrap(), + (previous.height, previous.quorums) + ); + assert_eq!( + cache + .state + .read() + .unwrap() + .previous + .as_ref() + .unwrap() + .refreshed, + previous.refreshed + ); + assert!(cache.is_healthy().await); + tokio::time::advance( + Duration::from_secs(core.config.quorum.cache_ttl_seconds) + .saturating_mul(core.config.quorum.health_stale_intervals), + ) + .await; + assert!(!cache.is_healthy().await); + } + /// A timed-out window waiter cannot release its worker permit or publish detached history. #[tokio::test] async fn should_hold_the_rpc_permit_after_the_window_waiter_times_out() {