diff --git a/Cargo.toml b/Cargo.toml index 9d8b2be..4ea9bb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,3 +46,6 @@ tokio = { version = "1.0", features = ["full"] } [build-dependencies] tonic-build = "0.12" + +[dev-dependencies] +tokio = { version = "1.0", features = ["test-util"] } diff --git a/Dockerfile b/Dockerfile index 8e490dd..e0df9b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,4 @@ USER appuser EXPOSE 3000 -ENV API_HOST=0.0.0.0 -ENV API_PORT=3000 - CMD ["quorum-list-server"] diff --git a/README.md b/README.md index 6a65996..e9cb549 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # Quorum List Server -A Rust-based HTTP API server that provides RESTful endpoints for managing Dash LLMQ_25_67 quorum lists via RPC calls to Dash Core. +A Rust-based HTTP API server providing the configured network's Platform quorum lists via RPC calls to Dash Core. ## Features -- RPC-based quorum loading from Dash Core (LLMQ_25_67 quorums for testnet) +- RPC-based Platform quorum loading for mainnet, testnet, devnet and regtest - RESTful API for quorum list management -- TOML-based configuration with environment variable fallbacks -- Previous quorum state lookup (configurable block offset) +- TOML-based configuration with environment variable overrides +- Stateless rolling window of active and recently retired quorums - Thread-safe shared state - Cross-platform support (Linux/Windows) - Graceful shutdown handling @@ -15,13 +15,88 @@ A Rust-based HTTP API server that provides RESTful endpoints for managing Dash L ## API Endpoints ### Health Check -- `GET /health` - Server health status +- `GET /health` - Quorum cache freshness status (HTTP 200, inspect the JSON body) +- `GET /ready` - Readiness status (HTTP 503 while degraded, 200 when fresh) ### Quorum Management - `GET /quorums` - Get all current quorums - `GET /quorums/stats` - Get quorum statistics - `GET /quorums/{hash}` - Get specific quorum by hash -- `GET /previous` - Get quorums from previous blocks (configurable offset) +- `GET /previous` - Get the union of Platform quorums active in the retention window + +### Quorum retention and freshness + +`/quorums` keeps its existing `data` array and adds top-level `height` and +`lastUpdated`. These describe the Core height sampled and the Unix timestamp in +seconds of the successful refresh. Repeated reads and failed active-list refreshes +do not advance them. Entries are ordered oldest to newest. + +`/previous` keeps `{ "success": true, "data": { "height": ..., "quorums": [...] } }`. +Its list is rebuilt from Core on every refresh by taking the union of +`quorum listextended` at the tip, then descending by the configured sample step, +including the lower boundary `max(0, tip - R)`. Here +`R = max(retention_blocks, previous_blocks_offset)`: existing offset settings +retain at least their old coverage. `data.height` is now the lowest sampled +height. No membership history is persisted, so a restart reconstructs the same +window and a refresh removes orphaned quorums after a reorg. + +The Core 24 rc.3 Platform parameters are: + +| Network | LLMQ type | Active count A | DKG interval D | Default step | +| --- | --- | --- | --- | --- | +| mainnet | 4 (`llmq_100_67`) | 24 | 24 | 288 | +| testnet | 6 (`llmq_25_67`) | 24 | 24 | 288 | +| devnet | 107 (`llmq_devnet_platform`) | 4 | 24 | 48 | +| regtest | 106 (`llmq_test_platform`) | 2 | 24 | 24 | + +Quorums become active when their commitments are mined. Core allows mining +offsets 10 through 18 within a DKG cycle, so a run can be as short as `A*D - 8` +blocks. Sampling every `A*D` blocks could miss a quorum. The conservative maximum +step is `(A-1)*D + 1`, allowing any mining offset within a cycle. The default is +`max(1, min(A*D/2, (A-1)*D+1))`. Failed DKGs lengthen active runs. Parameter +overrides must match the connected Core; unsafe steps, zero intervals and zero +cache limits are rejected at startup with an error. The output cap must exceed A, +the key memo must hold at least the output cap, and a window may use at most +256 samples (including both boundaries) to limit work within the refresh budget. See Core's +[parameter table](https://github.com/dashpay/dash/blob/v24.0.0-rc.3/src/llmq/params.h) +and [mined commitment scan](https://github.com/dashpay/dash/blob/v24.0.0-rc.3/src/llmq/quorumsman.cpp#L243-L244). + +The rolling list is ordered oldest to newest, with a default cap of 90 to fit +clients' 100-entry caches. Overflow removes the oldest retired quorums first; +active quorums are always retained, even if their count alone exceeds the cap. +Trimming sacrifices coverage at the old end of the window. Size the retention +and cap together. The 48-block default is suitable for 150-second blocks; +accelerated 10-second devnets typically need 360–720 blocks to cover Core/Platform +lag, a Platform block interval, the client's 31-minute metadata tolerance and +clock skew. + +A bounded public-key memo keyed by `(quorumHash, minedBlockHash)` avoids repeat +`quorum info` calls for unchanged commitments. A reorg can replace a commitment +and its public key for the same quorum hash; the new mined block forces a refetch. +All such calls explicitly use `includeSkShare=false`. Membership is recomputed, +not taken from the memo. All quorum RPC runs on a serialized blocking worker; +requests only read cached data. RPC sockets have a five-second timeout, refreshes +have a 30-second deadline, and a timed-out worker retains its permit until it +finishes, preventing accumulating background workers. + +The service polls `getblockcount` once per second and reloads when the height +changes. The cache TTL bounds reloads even at an unchanged height (including +same-height reorgs), and a poll interval longer than the TTL is reduced to the +TTL. Both lists are attempted independently and keep their last good data on +failure. An empty active list or invalid quorum/key data is a failed refresh, +never an empty success. Historical samples must contain every tip-active quorum +whose creation height plus the mining-window end (18 blocks) precedes the sample; +a short or empty result missing such a quorum fails the window refresh. A moving +tip preserves the current list tied to its sampled height and retries the window +once at the new tip. Repeated failed window reloads back off up to the smaller of +30 seconds and the TTL; a newly observed tip bypasses this backoff. + +`/health` still returns HTTP 200 and the same JSON shape; +it returns `success:false`, `data:null` and a degraded message when either list +has never loaded or its last successful refresh is older than +`cache_ttl_seconds * health_stale_intervals` (three intervals by default). `/ready` +returns the same body with HTTP 503 while degraded and HTTP 200 when fresh, so +load balancers can use it without changing existing `/health` matchers. ### Masternode bootstrap metadata @@ -67,6 +142,8 @@ validate the node ID and endpoint fields. Existing clients can continue reading ### config.toml ```toml +network = "testnet" + [server] port = 3000 host = "0.0.0.0" @@ -78,15 +155,41 @@ password = "password" [quorum] previous_blocks_offset = 8 +retention_blocks = 48 +cache_ttl_seconds = 60 +poll_interval_seconds = 1 +max_entries = 90 +key_memo_max_entries = 180 +health_stale_intervals = 3 +# sample_step_blocks = 288 +# active_quorum_count = 24 +# dkg_interval = 24 ``` -### Environment Variables (fallbacks) +### Environment Variables (overrides) - `API_HOST` - Server host (default: 0.0.0.0) - `API_PORT` - HTTP server port (default: 3000) - `DASH_RPC_URL` - RPC endpoint (default: http://127.0.0.1:19998) - `DASH_RPC_USER` - RPC username (default: dashrpc) - `DASH_RPC_PASSWORD` - RPC password (default: password) -- `QUORUM_PREVIOUS_BLOCKS_OFFSET` - Previous blocks offset (default: 8) +- `DASH_NETWORK` - mainnet, testnet, devnet or regtest (default: testnet) + +| TOML `[quorum]` key | Environment override | Default | +| --- | --- | --- | +| `previous_blocks_offset` | `QUORUM_PREVIOUS_BLOCKS_OFFSET` | 8; lower bound on retention | +| `retention_blocks` | `QUORUM_RETENTION_BLOCKS` | 48 | +| `cache_ttl_seconds` | `QUORUM_CACHE_TTL_SECONDS` | 60; local dashmate networks can use 1 | +| `sample_step_blocks` | `QUORUM_SAMPLE_STEP_BLOCKS` | Derived from A and D as above | +| `active_quorum_count` | `QUORUM_ACTIVE_QUORUM_COUNT` | Network table above | +| `dkg_interval` | `QUORUM_DKG_INTERVAL` | 24 blocks | +| `max_entries` | `QUORUM_MAX_ENTRIES` | 90; active entries are never trimmed | +| `key_memo_max_entries` | `QUORUM_KEY_MEMO_MAX_ENTRIES` | 180 public keys | +| `poll_interval_seconds` | `QUORUM_POLL_INTERVAL_SECONDS` | 1 | +| `health_stale_intervals` | `QUORUM_HEALTH_STALE_INTERVALS` | 3 | + +Environment variables override TOML values. When the file cannot be loaded, +environment variables override the built-in defaults. Older TOML files +with only `previous_blocks_offset` continue to work with the new defaults. ## Usage @@ -124,7 +227,9 @@ docker run --rm -p 8080:8080 \ quorum-list-server ``` -The server reads `/app/config.toml` first; environment variables are only used when that file is absent. +The server reads `/app/config.toml` first, then applies environment overrides. +The image does not set `API_HOST` or `API_PORT`; the binary defaults to `0.0.0.0:3000` +when neither the file nor explicitly supplied environment variables configure them. ## API Examples @@ -138,7 +243,7 @@ curl http://localhost:3000/quorums # Get quorum stats curl http://localhost:3000/quorums/stats -# Get previous quorums (8 blocks ago by default) +# Get active and recently retired quorums (48-block window by default) curl http://localhost:3000/previous # Get specific quorum by hash @@ -180,9 +285,9 @@ Example quorum response: ## Architecture - **RPC Integration**: Uses `dashcore-rpc` to communicate with Dash Core -- **LLMQ Type**: Only processes LLMQ_25_67 (type 6) quorums for testnet -- **Configuration**: TOML-first with environment variable fallbacks -- **State Management**: Thread-safe Arc> for shared state +- **LLMQ Type**: Processes the configured network's Platform quorum type +- **Configuration**: TOML values with environment variable overrides +- **State Management**: Cached lists and freshness metadata published under one lock - **API Framework**: Built with Axum for async HTTP handling ## Development diff --git a/config.toml b/config.toml index 1c4a8de..4696f6d 100644 --- a/config.toml +++ b/config.toml @@ -1,3 +1,5 @@ +network = "testnet" + [server] port = 8080 host = "0.0.0.0" @@ -9,6 +11,9 @@ password = "password" [quorum] previous_blocks_offset = 8 - -[network] -network = "testnet" \ No newline at end of file +retention_blocks = 48 +cache_ttl_seconds = 60 +poll_interval_seconds = 1 +max_entries = 90 +key_memo_max_entries = 180 +health_stale_intervals = 3 diff --git a/src/api.rs b/src/api.rs index aa67fd2..9140a93 100644 --- a/src/api.rs +++ b/src/api.rs @@ -26,6 +26,8 @@ pub struct ApiResponse { pub message: Option, #[serde(rename = "lastUpdated", skip_serializing_if = "Option::is_none")] pub last_updated: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub height: Option, } impl ApiResponse { @@ -35,6 +37,7 @@ impl ApiResponse { data: Some(data), message: None, last_updated: None, + height: None, } } @@ -44,6 +47,7 @@ impl ApiResponse { data: None, message: Some(message), last_updated: None, + height: None, } } } @@ -110,6 +114,7 @@ pub fn create_router(quorum_cache: SharedQuorumCache, config: Config, masternode let shared_config = Arc::new(config); Router::new() .route("/health", get(health_check)) + .route("/ready", get(readiness_check)) .route("/quorums", get(get_all_quorums)) .route("/quorums/stats", get(get_quorum_stats)) .route("/previous", get(get_previous_quorums)) @@ -122,19 +127,48 @@ pub fn create_router(quorum_cache: SharedQuorumCache, config: Config, masternode .layer(CompressionLayer::new()) } -async fn health_check() -> Json> { - Json(ApiResponse::success("Quorum List Server is running".to_string())) +async fn health_check( + State((quorum_cache, _, _)): State<(SharedQuorumCache, SharedConfig, SharedMasternodeCache)>, +) -> Json> { + if quorum_cache.is_healthy().await { + Json(ApiResponse::success( + "Quorum List Server is running".to_string(), + )) + } else { + Json(ApiResponse::error( + "Quorum List Server is degraded: quorum refresh is stale or unavailable".to_string(), + )) + } +} + +async fn readiness_check( + state: State<(SharedQuorumCache, SharedConfig, SharedMasternodeCache)>, +) -> (StatusCode, Json>) { + let response = health_check(state).await; + let status = if response.0.success { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + (status, response) } async fn get_all_quorums( State((quorum_cache, _, _)): State<(SharedQuorumCache, SharedConfig, SharedMasternodeCache)>, ) -> Result>>, StatusCode> { - match quorum_cache.get_quorums().await { - Ok(quorums) => { - let quorums: Vec = quorums.iter().map(|entry| entry.into()).collect(); - Ok(Json(ApiResponse::success(quorums))) + match quorum_cache.get_snapshot().await { + Ok(snapshot) => { + let quorums: Vec = + snapshot.quorums.iter().map(|entry| entry.into()).collect(); + let mut response = ApiResponse::success(quorums); + response.height = Some(snapshot.height); + response.last_updated = Some(snapshot.last_updated); + Ok(Json(response)) } - Err(e) => Ok(Json(ApiResponse::error(format!("Failed to load quorums: {}", e)))) + Err(e) => Ok(Json(ApiResponse::error(format!( + "Failed to load quorums: {}", + e + )))), } } diff --git a/src/config.rs b/src/config.rs index 661f493..ec099fc 100644 --- a/src/config.rs +++ b/src/config.rs @@ -68,6 +68,15 @@ impl Network { Network::Regtest => 2443, } } + + /// Platform LLMQ defaults from Dash Core's llmq/params.h. + pub fn platform_quorum_parameters(&self) -> (u32, u32) { + match self { + Network::Mainnet | Network::Testnet => (24, 24), + Network::Devnet => (4, 24), + Network::Regtest => (2, 24), + } + } } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -110,14 +119,35 @@ pub struct RpcConfig { } #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] pub struct QuorumConfig { pub previous_blocks_offset: u32, - #[serde(default = "default_cache_ttl")] pub cache_ttl_seconds: u64, + pub retention_blocks: u32, + pub sample_step_blocks: Option, + pub active_quorum_count: Option, + pub dkg_interval: Option, + pub max_entries: usize, + pub key_memo_max_entries: usize, + pub poll_interval_seconds: u64, + pub health_stale_intervals: u32, } -fn default_cache_ttl() -> u64 { - 60 +impl Default for QuorumConfig { + fn default() -> Self { + Self { + previous_blocks_offset: 8, + cache_ttl_seconds: 60, + retention_blocks: 48, + sample_step_blocks: None, + active_quorum_count: None, + dkg_interval: None, + max_entries: 90, + key_memo_max_entries: 180, + poll_interval_seconds: 1, + health_stale_intervals: 3, + } + } } impl Default for Config { @@ -132,10 +162,7 @@ impl Default for Config { username: "dashrpc".to_string(), password: "password".to_string(), }, - quorum: QuorumConfig { - previous_blocks_offset: 8, - cache_ttl_seconds: default_cache_ttl(), - }, + quorum: QuorumConfig::default(), network: Network::default(), docker: DockerConfig::default(), } @@ -152,58 +179,83 @@ impl Config { } pub fn load_from_env_or_file>(path: P) -> Self { - // Try to load from file first - if let Ok(config) = Self::load_from_file(path) { - return config; - } + Self::load_with_env(path, |name| std::env::var(name).ok()) + } - // Fall back to environment variables or defaults - let mut config = Config::default(); + fn load_with_env>(path: P, env: impl Fn(&str) -> Option) -> Self { + let mut config = Self::load_from_file(path).unwrap_or_default(); - if let Ok(port) = std::env::var("API_PORT") { + if let Some(port) = env("API_PORT") { if let Ok(port_num) = port.parse::() { config.server.port = port_num; } } - if let Ok(host) = std::env::var("API_HOST") { + if let Some(host) = env("API_HOST") { config.server.host = host; } - if let Ok(url) = std::env::var("DASH_RPC_URL") { + if let Some(url) = env("DASH_RPC_URL") { config.rpc.url = url; } - if let Ok(username) = std::env::var("DASH_RPC_USER") { + if let Some(username) = env("DASH_RPC_USER") { config.rpc.username = username; } - if let Ok(password) = std::env::var("DASH_RPC_PASSWORD") { + if let Some(password) = env("DASH_RPC_PASSWORD") { config.rpc.password = password; } - if let Ok(offset) = std::env::var("QUORUM_PREVIOUS_BLOCKS_OFFSET") { + if let Some(offset) = env("QUORUM_PREVIOUS_BLOCKS_OFFSET") { if let Ok(offset_num) = offset.parse::() { config.quorum.previous_blocks_offset = offset_num; } } - if let Ok(ttl) = std::env::var("QUORUM_CACHE_TTL_SECONDS") { + if let Some(ttl) = env("QUORUM_CACHE_TTL_SECONDS") { if let Ok(ttl_num) = ttl.parse::() { config.quorum.cache_ttl_seconds = ttl_num; } } - if let Ok(network_str) = std::env::var("DASH_NETWORK") { + macro_rules! quorum_env { + ($name:literal, $field:ident) => { + if let Some(value) = env($name) { + config.quorum.$field = value + .parse() + .unwrap_or_else(|_| panic!("Invalid {}", $name)); + } + }; + ($name:literal, $field:ident, optional) => { + if let Some(value) = env($name) { + config.quorum.$field = Some( + value + .parse() + .unwrap_or_else(|_| panic!("Invalid {}", $name)), + ); + } + }; + } + quorum_env!("QUORUM_RETENTION_BLOCKS", retention_blocks); + quorum_env!("QUORUM_SAMPLE_STEP_BLOCKS", sample_step_blocks, optional); + quorum_env!("QUORUM_ACTIVE_QUORUM_COUNT", active_quorum_count, optional); + quorum_env!("QUORUM_DKG_INTERVAL", dkg_interval, optional); + quorum_env!("QUORUM_MAX_ENTRIES", max_entries); + quorum_env!("QUORUM_KEY_MEMO_MAX_ENTRIES", key_memo_max_entries); + quorum_env!("QUORUM_POLL_INTERVAL_SECONDS", poll_interval_seconds); + quorum_env!("QUORUM_HEALTH_STALE_INTERVALS", health_stale_intervals); + + if let Some(network_str) = env("DASH_NETWORK") { config.network = Network::try_from(network_str.as_str()).unwrap_or_else(|e| panic!("{}", e)); } - if let Ok(version_check_host) = std::env::var("VERSION_CHECK_HOST") { + if let Some(version_check_host) = env("VERSION_CHECK_HOST") { config.docker.version_check_host = Some(version_check_host); } - if let Ok(address_host_override) = std::env::var("ADDRESS_HOST_OVERRIDE") { + if let Some(address_host_override) = env("ADDRESS_HOST_OVERRIDE") { config.docker.address_host_override = Some(address_host_override); } @@ -227,6 +279,67 @@ impl Config { self.network.llmq_type_id() } + fn quorum_parameters(&self) -> (u32, u32) { + let (active, interval) = self.network.platform_quorum_parameters(); + ( + self.quorum.active_quorum_count.unwrap_or(active), + self.quorum.dkg_interval.unwrap_or(interval), + ) + } + + pub fn quorum_sample_step(&self) -> u32 { + let (active, interval) = self.quorum_parameters(); + let lifetime = active.saturating_mul(interval); + let max_step = active + .saturating_sub(1) + .saturating_mul(interval) + .saturating_add(1); + self.quorum + .sample_step_blocks + .unwrap_or((lifetime / 2).min(max_step).max(1)) + } + + /// Checks operator settings once at startup, before any RPC work is admitted. + pub fn validate_quorum(&self) -> Result<(), Box> { + let (active, interval) = self.quorum_parameters(); + let lifetime = active + .checked_mul(interval) + .filter(|value| *value > 0) + .ok_or( + "Quorum active count and DKG interval must be positive and their product fit u32", + )?; + // Commitment mining offsets can differ within a cycle, shortening an active run. + let max_step = lifetime - interval + 1; + let step = self.quorum_sample_step(); + if step == 0 || step > max_step { + return Err(format!("Quorum sample step {step} must be between 1 and {max_step}: commitment mining offsets can shorten active runs").into()); + } + if self.quorum.cache_ttl_seconds == 0 + || self.quorum.poll_interval_seconds == 0 + || self.quorum.health_stale_intervals == 0 + { + return Err("Quorum intervals must be positive".into()); + } + if self.quorum.max_entries <= active as usize { + return Err("Quorum max_entries must exceed the active quorum count".into()); + } + if self.quorum.key_memo_max_entries < self.quorum.max_entries { + return Err("Quorum key memo capacity must be at least max_entries".into()); + } + if self.quorum_retention().div_ceil(step).saturating_add(1) > 256 { + return Err( + "Quorum retention and sample step require more than 256 samples per refresh".into(), + ); + } + Ok(()) + } + + pub fn quorum_retention(&self) -> u32 { + self.quorum + .retention_blocks + .max(self.quorum.previous_blocks_offset) + } + pub fn get_dapi_port(&self) -> u16 { self.network.dapi_port() } @@ -301,4 +414,111 @@ previous_blocks_offset = 8 assert_eq!(config.get_llmq_type(), "llmq_devnet_platform"); assert_eq!(config.get_llmq_type_id(), 107); } + + /// Legacy TOML must gain retention defaults and safe per-network sampling. + #[test] + fn should_default_legacy_configs_to_a_safe_rolling_window() { + let quorum: QuorumConfig = toml::from_str("previous_blocks_offset = 8").unwrap(); + assert_eq!(quorum.retention_blocks, 48); + assert_eq!(quorum.cache_ttl_seconds, 60); + assert_eq!(quorum.max_entries, 90); + for (network, expected) in [ + (Network::Mainnet, 288), + (Network::Testnet, 288), + (Network::Devnet, 48), + (Network::Regtest, 24), + ] { + let mut config = Config { + network, + ..Config::default() + }; + assert_eq!(config.quorum_sample_step(), expected); + config.quorum.active_quorum_count = Some(1); + assert_eq!(config.quorum_sample_step(), 1); + } + } + + /// Invalid intervals and arithmetic must fail before background work starts. + #[test] + fn should_reject_invalid_quorum_intervals_and_limits() { + let mut config = Config::default(); + config.quorum.sample_step_blocks = Some(0); + assert!(config.validate_quorum().is_err()); + config.quorum.sample_step_blocks = None; + config.quorum.active_quorum_count = Some(0); + assert!(config.validate_quorum().is_err()); + config.quorum.active_quorum_count = Some(u32::MAX); + assert!(config.validate_quorum().is_err()); + config.quorum.active_quorum_count = None; + config.quorum.cache_ttl_seconds = 0; + assert!(config.validate_quorum().is_err()); + } + + /// Injected environment settings must preserve all quorum knobs without global test races. + #[test] + fn should_preserve_quorum_env_fallbacks() { + let variables = [ + ("QUORUM_PREVIOUS_BLOCKS_OFFSET", "96"), + ("QUORUM_CACHE_TTL_SECONDS", "1"), + ("QUORUM_RETENTION_BLOCKS", "720"), + ("QUORUM_SAMPLE_STEP_BLOCKS", "12"), + ("QUORUM_ACTIVE_QUORUM_COUNT", "4"), + ("QUORUM_DKG_INTERVAL", "24"), + ("QUORUM_MAX_ENTRIES", "80"), + ("QUORUM_KEY_MEMO_MAX_ENTRIES", "160"), + ("QUORUM_POLL_INTERVAL_SECONDS", "2"), + ("QUORUM_HEALTH_STALE_INTERVALS", "5"), + ]; + let config = Config::load_with_env("/nonexistent/qls16-config.toml", |name| { + variables + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()) + }); + assert_eq!(config.quorum.previous_blocks_offset, 96); + assert_eq!(config.quorum.cache_ttl_seconds, 1); + assert_eq!(config.quorum_retention(), 720); + assert_eq!(config.quorum_sample_step(), 12); + assert_eq!(config.quorum.max_entries, 80); + assert_eq!(config.quorum.key_memo_max_entries, 160); + assert_eq!(config.quorum.poll_interval_seconds, 2); + assert_eq!(config.quorum.health_stale_intervals, 5); + } + + /// Environment settings must win even when the shipped TOML file is valid. + #[test] + fn should_override_file_settings_with_environment_variables() { + let config = Config::load_with_env("config.toml", |name| match name { + "QUORUM_CACHE_TTL_SECONDS" => Some("1".into()), + "DASH_NETWORK" => Some("devnet".into()), + "DASH_RPC_URL" => Some("http://core.example:9998".into()), + _ => None, + }); + assert_eq!(config.quorum.cache_ttl_seconds, 1); + assert_eq!(config.network, Network::Devnet); + assert_eq!(config.rpc.url, "http://core.example:9998"); + } + + /// The shipped TOML must use the same schema as the configuration loader. + #[test] + fn should_load_the_shipped_config_file() { + let config: Config = toml::from_str(include_str!("../config.toml")).unwrap(); + assert_eq!(config.network, Network::Testnet); + assert_eq!(config.quorum_sample_step(), 288); + } + + /// Startup must reject settings that defeat retention, memoization or the RPC budget. + #[test] + fn should_reject_incoherent_quorum_capacity_and_excessive_sampling() { + let mut config = Config::default(); + config.quorum.max_entries = 24; + assert!(config.validate_quorum().is_err()); + config.quorum.max_entries = 90; + config.quorum.key_memo_max_entries = 89; + assert!(config.validate_quorum().is_err()); + config.quorum.key_memo_max_entries = 180; + config.quorum.retention_blocks = 10000; + config.quorum.sample_step_blocks = Some(1); + assert!(config.validate_quorum().is_err()); + } } diff --git a/src/main.rs b/src/main.rs index ab1cf9f..2e8d9b1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,9 @@ mod masternode_loader; mod masternode_cache; mod grpc_client; +#[cfg(test)] +mod quorum_cache_tests; + use config::Config; use masternode_cache::MasternodeCache; use quorum_cache::QuorumCache; @@ -21,6 +24,10 @@ async fn main() -> Result<(), Box> { // Load configuration let config = Config::load_from_env_or_file("config.toml"); + if let Err(error) = config.validate_quorum() { + eprintln!("Invalid quorum configuration: {error}"); + return Err(std::io::Error::other(error.to_string()).into()); + } println!("Configuration loaded:"); println!(" Server: {}:{}", config.server.host, config.server.port); println!(" RPC: {} (user: {})", config.rpc.url, config.rpc.username); diff --git a/src/masternode_cache.rs b/src/masternode_cache.rs index 70f89bb..3f17ec6 100644 --- a/src/masternode_cache.rs +++ b/src/masternode_cache.rs @@ -323,7 +323,7 @@ mod tests { assert_eq!(result["success"], false); assert!(result.get("lastUpdated").is_none()); let health = response(config, cache, "/health").await; - assert_eq!(health["success"], true); + assert_eq!(health["success"], false); assert!(health.get("lastUpdated").is_none()); } } diff --git a/src/quorum_cache.rs b/src/quorum_cache.rs index fd4940b..73f57c1 100644 --- a/src/quorum_cache.rs +++ b/src/quorum_cache.rs @@ -1,190 +1,558 @@ use crate::config::Config; use crate::quorum_list::QuorumList; -use crate::quorum_loader; -use chrono::Local; -use std::sync::{Arc, RwLock}; -use std::time::{Duration, Instant}; -use tokio::sync::Mutex; +use crate::quorum_loader::{self, KeyMemo, LoadResult, REFRESH_TIMEOUT}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::Duration; +use tokio::time::Instant; + +#[derive(Clone)] +pub struct QuorumSnapshot { + pub quorums: QuorumList, + pub height: u32, + pub last_updated: i64, + refreshed: Instant, +} + +#[derive(Default)] +struct CacheState { + current: Option, + previous: Option, +} pub struct QuorumCache { - current_quorums: Arc>>, - previous_quorums: Arc>>, - current_last_update: Arc>>, - previous_last_update: Arc>>, + state: RwLock, + memo: Arc>, + rpc_worker: Arc>, config: Arc, update_interval: Duration, + #[cfg(test)] + polls_completed: std::sync::atomic::AtomicUsize, } impl QuorumCache { pub fn new(config: Config) -> Self { let ttl = config.quorum.cache_ttl_seconds; Self { - current_quorums: Arc::new(RwLock::new(None)), - previous_quorums: Arc::new(RwLock::new(None)), - current_last_update: Arc::new(Mutex::new(None)), - previous_last_update: Arc::new(Mutex::new(None)), + state: RwLock::new(CacheState::default()), + memo: Arc::new(Mutex::new(KeyMemo::default())), + rpc_worker: Arc::new(tokio::sync::Mutex::new(())), config: Arc::new(config), update_interval: Duration::from_secs(ttl), + #[cfg(test)] + polls_completed: std::sync::atomic::AtomicUsize::new(0), } } - /// Read the cached current quorums. - /// - /// This never contacts Dash Core: the background refresh task is the only - /// writer. Serving a request must not depend on RPC latency. - pub async fn get_quorums( - &self, - ) -> Result> { - let data = self - .current_quorums + /// Reads the list and its metadata together without contacting Core. + pub async fn get_snapshot(&self) -> LoadResult { + self.state .read() - .map_err(|_| "Failed to read quorum cache")?; - Ok(data - .as_ref() - .ok_or("Quorum cache is not populated yet")? - .clone()) - } - - /// Read the cached previous quorums along with the height they were taken at. - /// - /// As with [`Self::get_quorums`], this is a pure cache read. - pub async fn get_previous_quorums( - &self, - ) -> Result<(u32, QuorumList), Box> { - let data = self - .previous_quorums + .map_err(|_| "Failed to read quorum cache")? + .current + .clone() + .ok_or_else(|| "Quorum cache is not populated yet".into()) + } + + pub async fn get_quorums(&self) -> LoadResult { + Ok(self.get_snapshot().await?.quorums) + } + + /// Returns the rolling union and the lowest sampled Core height. + pub async fn get_previous_quorums(&self) -> LoadResult<(u32, QuorumList)> { + let state = self + .state .read() - .map_err(|_| "Failed to read previous quorum cache")?; - Ok(data + .map_err(|_| "Failed to read quorum cache")?; + let previous = state + .previous .as_ref() - .ok_or("Previous quorum cache is not populated yet")? - .clone()) + .ok_or("Previous quorum cache is not populated yet")?; + Ok((previous.height, previous.quorums.clone())) } - /// Refresh both quorum caches from Dash Core. - /// - /// Called once at startup and then only by the background refresh task. - pub async fn refresh(&self) -> Result<(), Box> { - self.update_current_cache().await?; - self.update_previous_cache().await?; - Ok(()) + /// Admission shares the timeout, while detached blocking work retains its permit. + async fn rpc_work(&self, work: F) -> LoadResult + where + T: Send + 'static, + F: FnOnce() -> LoadResult + Send + 'static, + { + tokio::time::timeout(REFRESH_TIMEOUT, async { + let permit = self.rpc_worker.clone().lock_owned().await; + tokio::task::spawn_blocking(move || { + let _permit = permit; + work() + }) + .await? + }) + .await + .map_err(|_| "Quorum RPC worker timed out")? } - async fn update_current_cache(&self) -> Result<(), Box> { - println!("Updating quorum cache..."); - - let result = tokio::time::timeout( - tokio::time::Duration::from_secs(30), - quorum_loader::load_initial_quorums(&self.config), - ) - .await; + async fn quorum_work(&self, work: F) -> LoadResult + where + T: Send + 'static, + F: FnOnce(&Config, &mut KeyMemo) -> LoadResult + Send + 'static, + { + let config = self.config.clone(); + let memo = self.memo.clone(); + self.rpc_work(move || { + let mut memo = memo.lock().unwrap_or_else(|poisoned| { + memo.clear_poison(); + let mut memo = poisoned.into_inner(); + *memo = KeyMemo::default(); + memo + }); + work(&config, &mut memo) + }) + .await + } - match result { - Ok(Ok(quorums)) => { - { - let mut data = self - .current_quorums + /// Publish active keys before spending the window's separate RPC budget on history. + pub async fn refresh(&self) -> LoadResult<()> { + let current = self.quorum_work(quorum_loader::refresh_current).await; + let current_error = match ¤t { + Ok(result) => match &result.current { + Ok(quorums) => { + let mut state = self + .state .write() - .map_err(|_| "Failed to write to quorum cache")?; - *data = Some(quorums); - } - { - let mut last_update = self.current_last_update.lock().await; - *last_update = Some(Instant::now()); + .map_err(|_| "Failed to write quorum cache")?; + state.current = Some(QuorumSnapshot { + quorums: quorums.clone(), + height: result.tip, + last_updated: chrono::Utc::now().timestamp(), + refreshed: Instant::now(), + }); + None } - println!("Quorum cache updated successfully"); - Ok(()) + Err(error) => Some(error.to_string()), + }, + Err(error) => Some(error.to_string()), + }; + let previous = self + .quorum_work(move |config, memo| quorum_loader::refresh_previous(config, memo, current)) + .await; + let mut state = self + .state + .write() + .map_err(|_| "Failed to write quorum cache")?; + let previous_error = match previous { + Ok((height, quorums)) => { + state.previous = Some(QuorumSnapshot { + quorums, + height, + last_updated: chrono::Utc::now().timestamp(), + refreshed: Instant::now(), + }); + None } - Ok(Err(e)) => Err(e), - Err(_) => { - eprintln!("Quorum cache update timed out after 30 seconds"); - Err("Quorum cache update timed out after 30 seconds".into()) + Err(error) => Some(error), + }; + match (current_error, previous_error) { + (None, None) => Ok(()), + (Some(error), None) => Err(error.into()), + (None, Some(error)) => Err(error), + (Some(current), Some(previous)) => { + Err(format!("Current quorums: {current}; rolling window: {previous}").into()) } } } - async fn update_previous_cache(&self) -> Result<(), Box> { - println!("Updating previous quorum cache..."); - - let result = tokio::time::timeout( - tokio::time::Duration::from_secs(30), - self.fetch_previous_quorums(), - ) - .await; - - match result { - Ok(Ok((height, quorums))) => { - { - let mut data = self - .previous_quorums - .write() - .map_err(|_| "Failed to write to previous quorum cache")?; - *data = Some((height, quorums)); - } - { - let mut last_update = self.previous_last_update.lock().await; - *last_update = Some(Instant::now()); - } - println!("Previous quorum cache updated successfully (height: {})", height); - Ok(()) - } - Ok(Err(e)) => Err(e), - Err(_) => { - eprintln!("Previous quorum cache update timed out after 30 seconds"); - Err("Previous quorum cache update timed out after 30 seconds".into()) + pub async fn is_healthy(&self) -> bool { + let stale_after = self + .update_interval + .saturating_mul(self.config.quorum.health_stale_intervals); + let Ok(state) = self.state.read() else { + return false; + }; + match (&state.current, &state.previous) { + (Some(current), Some(previous)) => { + current.refreshed.elapsed() < stale_after + && previous.refreshed.elapsed() < stale_after } + _ => false, } } - async fn fetch_previous_quorums( - &self, - ) -> Result<(u32, QuorumList), Box> { - let current_height = quorum_loader::get_current_block_height(&self.config).await?; - let previous_height = if current_height >= self.config.quorum.previous_blocks_offset { - current_height - self.config.quorum.previous_blocks_offset - } else { - 0 - }; - let quorums = quorum_loader::load_quorums_at_height(&self.config, previous_height).await?; - Ok((previous_height, quorums)) + fn reload_due(&self, tip: u32) -> LoadResult { + let state = self + .state + .read() + .map_err(|_| "Failed to read quorum cache")?; + Ok(match (&state.current, &state.previous) { + (Some(current), Some(previous)) => { + let changed = current.height != tip; + changed + || current.refreshed.elapsed() >= self.update_interval + || previous.refreshed.elapsed() >= self.update_interval + || previous.height != tip.saturating_sub(self.config.quorum_retention()) + } + _ => true, + }) } pub async fn start_background_refresh(self: Arc) { tokio::spawn(async move { + let poll_interval = Duration::from_secs(self.config.quorum.poll_interval_seconds) + .min(self.update_interval) + .max(Duration::from_millis(1)); + let mut poll = tokio::time::interval(poll_interval); + poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut failures = 0u32; + let mut retry_after = tokio::time::Instant::now(); + let mut last_polled_tip = None; loop { - tokio::time::sleep(self.update_interval).await; - let now = Local::now(); - println!( - "🔄 [{}] Background refresh: Starting quorum cache update...", - now.format("%Y-%m-%d %H:%M:%S") - ); - - let current_result = self.update_current_cache().await; - let previous_result = self.update_previous_cache().await; - - match (current_result, previous_result) { - (Ok(_), Ok(_)) => println!( - "✅ [{}] Background refresh: Quorum caches updated successfully", - Local::now().format("%Y-%m-%d %H:%M:%S") - ), - (Err(e1), Err(e2)) => eprintln!( - "❌ [{}] Background refresh: Failed to update quorum caches: current={}, previous={}", - Local::now().format("%Y-%m-%d %H:%M:%S"), - e1, - e2 - ), - (Err(e), Ok(_)) => eprintln!( - "⚠️ [{}] Background refresh: Failed to update current quorum cache: {}", - Local::now().format("%Y-%m-%d %H:%M:%S"), - e - ), - (Ok(_), Err(e)) => eprintln!( - "⚠️ [{}] Background refresh: Failed to update previous quorum cache: {}", - Local::now().format("%Y-%m-%d %H:%M:%S"), - e - ), + poll.tick().await; + let config = self.config.clone(); + match self + .rpc_work(move || quorum_loader::get_current_block_height(&config)) + .await + { + Ok(tip) => { + let new_tip = last_polled_tip.replace(tip) != Some(tip); + match self.reload_due(tip) { + Ok(true) if new_tip || tokio::time::Instant::now() >= retry_after => { + match self.refresh().await { + Ok(()) => { + failures = 0; + } + Err(error) => { + eprintln!("Quorum refresh failed: {error}"); + failures = failures.saturating_add(1); + let delay = poll_interval + .saturating_mul(1 << failures.saturating_sub(1).min(5)) + .min(self.update_interval) + .min(Duration::from_secs(30)); + retry_after = tokio::time::Instant::now() + delay; + } + } + } + Ok(_) => {} + Err(error) => eprintln!("Cannot read quorum refresh state: {error}"), + } + } + Err(error) => eprintln!("Core tip poll failed: {error}"), } + #[cfg(test)] + self.polls_completed + .fetch_add(1, std::sync::atomic::Ordering::Release); } }); } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::quorum_cache_tests::{MockCore, Quorum}; + + /// A stuck active worker must bound history admission and never publish its detached result. + #[tokio::test] + async fn should_return_when_a_timed_out_active_worker_blocks_window_admission() { + let core = MockCore::new( + 100, + vec![Quorum { + id: 1, + created: 10, + retired: u32::MAX, + }], + ); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let current = cache.get_snapshot().await.unwrap(); + let previous = cache.state.read().unwrap().previous.clone().unwrap(); + let (release, gate) = std::sync::mpsc::channel(); + { + let mut state = core.state.lock().unwrap(); + state.tip = 101; + state.quorums.push(Quorum { + id: 2, + created: 101, + retired: u32::MAX, + }); + state.historical_response_gate = Some((101, gate)); + state.calls.clear(); + } + let mut refresh = Box::pin(cache.refresh()); + assert!(futures::poll!(&mut refresh).is_pending()); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while !core + .state + .lock() + .unwrap() + .calls + .iter() + .any(|call| call["params"][0] == "listextended" && call["params"][1] == 101) + { + assert!( + std::time::Instant::now() < deadline, + "the active worker did not reach its gated response" + ); + tokio::task::yield_now().await; + } + tokio::time::pause(); + let active_budget = tokio::time::sleep(REFRESH_TIMEOUT); + tokio::time::advance(REFRESH_TIMEOUT + Duration::from_secs(1)).await; + active_budget.await; + // Keep the active response gated through the window's separate admission budget. + let result = { + let result = + tokio::time::timeout(REFRESH_TIMEOUT + Duration::from_secs(1), &mut refresh); + tokio::pin!(result); + // Poll the expired active stage to start the window's admission timeout. + assert!(futures::poll!(&mut result).is_pending()); + tokio::time::advance(REFRESH_TIMEOUT + Duration::from_secs(2)).await; + result.await + }; + drop(refresh); + assert!( + cache.rpc_worker.try_lock().is_err(), + "the active worker must still own the permit through both stage timeouts" + ); + assert_eq!(core.state.lock().unwrap().calls.len(), 2); + release + .send(()) + .expect("the active response must stay gated through the admission timeout"); + let error = result + .expect("refresh hung waiting for a timed-out active worker's permit") + .unwrap_err(); + assert!(error + .to_string() + .contains("Current quorums: Quorum RPC worker timed out; rolling window: Quorum RPC worker timed out")); + + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while cache.rpc_worker.try_lock().is_err() { + assert!( + std::time::Instant::now() < deadline, + "the detached active worker did not finish after release" + ); + tokio::task::yield_now().await; + } + assert_eq!(core.info_calls().len(), 1); + let snapshot = cache.get_snapshot().await.unwrap(); + assert_eq!(snapshot.height, current.height); + assert_eq!(snapshot.quorums, current.quorums); + assert_eq!(snapshot.refreshed, current.refreshed); + assert_eq!( + cache.get_previous_quorums().await.unwrap(), + (previous.height, previous.quorums) + ); + assert_eq!( + cache + .state + .read() + .unwrap() + .previous + .as_ref() + .unwrap() + .refreshed, + previous.refreshed + ); + assert!(cache.is_healthy().await); + tokio::time::advance( + Duration::from_secs(core.config.quorum.cache_ttl_seconds) + .saturating_mul(core.config.quorum.health_stale_intervals), + ) + .await; + assert!(!cache.is_healthy().await); + } + + /// A timed-out window waiter cannot release its worker permit or publish detached history. + #[tokio::test] + async fn should_hold_the_rpc_permit_after_the_window_waiter_times_out() { + let core = MockCore::new( + 100, + vec![Quorum { + id: 1, + created: 10, + retired: u32::MAX, + }], + ); + let (release_first, first_gate) = std::sync::mpsc::channel(); + core.state.lock().unwrap().historical_response_gate = Some((52, first_gate)); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + let first_cache = cache.clone(); + let first = tokio::spawn(async move { first_cache.refresh().await }); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while !core + .state + .lock() + .unwrap() + .calls + .iter() + .any(|call| call["params"][0] == "listextended" && call["params"][1] == 52) + { + assert!( + std::time::Instant::now() < deadline, + "first window worker did not start" + ); + tokio::task::yield_now().await; + } + let published_current = cache.get_snapshot().await.unwrap(); + let gated_call_count = core.state.lock().unwrap().calls.len(); + tokio::time::pause(); + tokio::time::advance(REFRESH_TIMEOUT + Duration::from_secs(1)).await; + let error = first.await.unwrap().unwrap_err(); + assert!(error.to_string().contains("timed out")); + assert!( + cache.rpc_worker.try_lock().is_err(), + "the detached blocking closure must still own the permit" + ); + + let (release_second, second_gate) = std::sync::mpsc::channel(); + { + let mut state = core.state.lock().unwrap(); + state.historical_response_gate = Some((100, second_gate)); + } + let second_cache = cache.clone(); + let (started, starting) = tokio::sync::oneshot::channel(); + let second = tokio::spawn(async move { + started.send(()).unwrap(); + second_cache.refresh().await + }); + starting.await.unwrap(); + assert!(!second.is_finished()); + assert_eq!(core.state.lock().unwrap().calls.len(), gated_call_count); + assert_eq!( + cache.get_snapshot().await.unwrap().refreshed, + published_current.refreshed + ); + assert!(cache.get_previous_quorums().await.is_err()); + + release_first + .send(()) + .expect("the first window RPC must still be gated after its waiter timed out"); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while core + .state + .lock() + .unwrap() + .calls + .iter() + .filter(|call| call["params"][0] == "listextended" && call["params"][1] == 100) + .count() + < 2 + { + assert!( + std::time::Instant::now() < deadline, + "the second worker did not start after the first was released" + ); + tokio::task::yield_now().await; + } + // The first closure completed successfully; the second result is still gated. + assert_eq!(core.info_calls().len(), 1); + let current = cache.get_snapshot().await.unwrap(); + assert_eq!(current.refreshed, published_current.refreshed); + assert_eq!(current.quorums, published_current.quorums); + assert!( + cache.get_previous_quorums().await.is_err(), + "the detached first window must never publish a previous snapshot" + ); + release_second + .send(()) + .expect("the second active response must still be gated"); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while !second.is_finished() { + assert!( + std::time::Instant::now() < deadline, + "the second refresh did not finish" + ); + tokio::task::yield_now().await; + } + second.await.unwrap().unwrap(); + assert!(cache.get_snapshot().await.unwrap().refreshed > published_current.refreshed); + assert_eq!(cache.get_previous_quorums().await.unwrap().0, 52); + } + + async fn finish_poll(cache: &QuorumCache, expected_polls: usize) { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while cache + .polls_completed + .load(std::sync::atomic::Ordering::Acquire) + < expected_polls + { + assert!( + std::time::Instant::now() < deadline, + "Core poll did not finish" + ); + // Keep the paused runtime busy until blocking RPC and retry scheduling finish. + tokio::task::yield_now().await; + } + } + + /// Repeated failures at one tip must back off, even without a successful active list. + #[tokio::test] + async fn should_back_off_active_list_failures_at_an_unchanged_tip() { + check_active_list_backoff(false).await; + } + + /// A failed refresh at a new tip must not keep bypassing backoff using the old snapshot. + #[tokio::test] + async fn should_back_off_active_list_failures_after_a_successful_refresh() { + check_active_list_backoff(true).await; + } + + async fn check_active_list_backoff(populated: bool) { + let core = MockCore::new( + 100, + vec![Quorum { + id: 1, + created: 10, + retired: u32::MAX, + }], + ); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + if populated { + cache.refresh().await.unwrap(); + } + { + let mut state = core.state.lock().unwrap(); + state.tip = 101; + state.fail_current = true; + state.calls.clear(); + } + tokio::time::pause(); + cache.clone().start_background_refresh().await; + tokio::task::yield_now().await; + // Wake the first interval tick; later polls retry exactly when now == retry_after. + tokio::time::advance(Duration::from_millis(1)).await; + let active_attempts = || { + core.state + .lock() + .unwrap() + .calls + .iter() + .filter(|call| call["params"][0] == "listextended" && call["params"][1] == 101) + .count() + }; + finish_poll(&cache, 1).await; + assert_eq!(active_attempts(), 2); + tokio::time::advance(Duration::from_secs(1)).await; + finish_poll(&cache, 2).await; + assert_eq!(active_attempts(), 4); + tokio::time::advance(Duration::from_secs(1)).await; + finish_poll(&cache, 3).await; + assert_eq!( + active_attempts(), + 4, + "the second failure must delay the next refresh for two poll intervals" + ); + tokio::time::advance(Duration::from_secs(1)).await; + finish_poll(&cache, 4).await; + assert_eq!(active_attempts(), 6); + core.state.lock().unwrap().tip = 102; + tokio::time::advance(Duration::from_secs(1)).await; + finish_poll(&cache, 5).await; + assert_eq!( + core.state + .lock() + .unwrap() + .calls + .iter() + .filter(|call| call["params"][0] == "listextended" && call["params"][1] == 102) + .count(), + 2, + "a newly polled tip must bypass the four-second retry delay" + ); + } +} diff --git a/src/quorum_cache_tests.rs b/src/quorum_cache_tests.rs new file mode 100644 index 0000000..0df6455 --- /dev/null +++ b/src/quorum_cache_tests.rs @@ -0,0 +1,1149 @@ +use crate::config::{Config, Network}; +use crate::quorum_cache::QuorumCache; +use crate::quorum_loader::LATEST_MINING_OFFSET; +use serde_json::{json, Value}; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +#[derive(Clone)] +pub struct Quorum { + pub id: u32, + pub created: u32, + pub retired: u32, +} + +pub struct CoreState { + pub tip: u32, + pub quorums: Vec, + pub calls: Vec, + pub fail: bool, + pub fail_current: bool, + pub malformed: bool, + pub empty: bool, + pub fail_height: Option, + pub fail_current_once: bool, + pub delay: Duration, + pub omitted_type_before: Option, + pub commitments: std::collections::HashMap, + pub omit_at_height: Option, + pub tip_script: std::collections::VecDeque, + pub first_response_gate: Option>, + pub omit_only: Option, + pub historical_response_gate: Option<(u32, std::sync::mpsc::Receiver<()>)>, + pub fail_block_count_at: Option, + pub creation_heights: std::collections::HashMap, + pub commitment_change_on_info: Option<(u32, u32)>, + pub omit_info_mined_block: bool, +} + +pub struct MockCore { + pub config: Config, + pub state: Arc>, + stop: Arc, + worker: Option>, +} + +impl MockCore { + pub fn new(tip: u32, quorums: Vec) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let mut config = Config { + network: Network::Devnet, + ..Config::default() + }; + config.rpc.url = format!("http://{}", listener.local_addr().unwrap()); + let state = Arc::new(Mutex::new(CoreState { + tip, + quorums, + calls: Vec::new(), + fail: false, + fail_current: false, + malformed: false, + empty: false, + fail_height: None, + fail_current_once: false, + delay: Duration::ZERO, + omitted_type_before: None, + commitments: std::collections::HashMap::new(), + omit_at_height: None, + tip_script: std::collections::VecDeque::new(), + first_response_gate: None, + omit_only: None, + historical_response_gate: None, + fail_block_count_at: None, + creation_heights: std::collections::HashMap::new(), + commitment_change_on_info: None, + omit_info_mined_block: false, + })); + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let worker_state = state.clone(); + let worker_stop = stop.clone(); + let worker = std::thread::spawn(move || { + while !worker_stop.load(std::sync::atomic::Ordering::Relaxed) { + let (mut stream, _) = match listener.accept() { + Ok(connection) => connection, + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(1)); + continue; + } + Err(e) => panic!("mock RPC accept failed: {e}"), + }; + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut bytes = Vec::new(); + let (header_end, length) = loop { + let mut buf = [0; 4096]; + let read = stream.read(&mut buf).unwrap(); + assert!(read > 0); + bytes.extend_from_slice(&buf[..read]); + if let Some(pos) = bytes.windows(4).position(|w| w == b"\r\n\r\n") { + let headers = String::from_utf8_lossy(&bytes[..pos]); + let length = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + .unwrap(); + break (pos + 4, length); + } + }; + while bytes.len() < header_end + length { + let mut buf = [0; 4096]; + let read = stream.read(&mut buf).unwrap(); + assert!(read > 0); + bytes.extend_from_slice(&buf[..read]); + } + let request: Value = + serde_json::from_slice(&bytes[header_end..header_end + length]).unwrap(); + let mut state = worker_state.lock().unwrap(); + state.calls.push(request.clone()); + if request["method"] == "getblockcount" { + if let Some(tip) = state.tip_script.pop_front() { + state.tip = tip; + } + } + let height = request["params"][1] + .as_u64() + .map(|h| h as u32) + .unwrap_or(state.tip); + let is_list = request["params"][0] == "listextended"; + let fail_once = state.fail_current_once && is_list && height == state.tip; + if fail_once { + state.fail_current_once = false; + } + let failed = state.fail + || fail_once + || (request["method"] == "getblockcount" + && state.fail_block_count_at + == Some( + state + .calls + .iter() + .filter(|call| call["method"] == "getblockcount") + .count(), + )) + || (is_list && state.fail_height == Some(height)) + || (state.fail_current && is_list && height == state.tip); + let result = if request["method"] == "getblockcount" { + json!(state.tip) + } else if request["params"][0] == "listextended" { + if state + .omitted_type_before + .is_some_and(|activation| height < activation) + { + json!({"llmq_other": []}) + } else if state.malformed { + json!({}) + } else { + let entries: Vec = state.quorums.iter() + .filter(|q| !state.empty && (state.omit_at_height != Some(height) || state.omit_only.is_some_and(|id| id != q.id)) && q.created <= height && height < q.retired) + .map(|q| json!({hash(q.id): {"creationHeight":state.creation_heights.get(&q.id).copied().unwrap_or(q.created), "numValidMembers":12, "minedBlockHash":hash(q.id + 100000 + state.commitments.get(&q.id).copied().unwrap_or(0))}})) + .collect(); + json!({"llmq_devnet_platform": entries}) + } + } else { + let id = u32::from_str_radix(&request["params"][2].as_str().unwrap()[56..], 16) + .unwrap(); + if state + .commitment_change_on_info + .is_some_and(|(quorum, _)| quorum == id) + { + let (_, commitment) = state.commitment_change_on_info.take().unwrap(); + state.commitments.insert(id, commitment); + } + let commitment = state.commitments.get(&id).copied().unwrap_or(0); + let mut info = json!({"quorumPublicKey": format!("{:02x}", 17 + commitment).repeat(48), "minedBlock":hash(id + 100000 + commitment)}); + if state.omit_info_mined_block { + info.as_object_mut().unwrap().remove("minedBlock"); + } + info + }; + let body = if failed { + json!({"result":null, "error":{"code":-1,"message":"Core unavailable"}, "id":request["id"]}) + } else { + json!({"result":result, "error":null, "id":request["id"]}) + }.to_string(); + let delay = state.delay; + let gate = if is_list + && state + .historical_response_gate + .as_ref() + .is_some_and(|(gated_height, _)| *gated_height == height) + { + state.historical_response_gate.take().map(|(_, gate)| gate) + } else { + state.first_response_gate.take() + }; + drop(state); + if let Some(gate) = gate { + let _ = gate.recv_timeout(Duration::from_secs(30)); + } + std::thread::sleep(delay); + let response = format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body); + let _ = stream.write_all(response.as_bytes()); + } + }); + Self { + config, + state, + stop, + worker: Some(worker), + } + } + + pub fn info_calls(&self) -> Vec { + self.state + .lock() + .unwrap() + .calls + .iter() + .filter(|r| r["method"] == "quorum" && r["params"][0] == "info") + .cloned() + .collect() + } +} + +impl Drop for MockCore { + fn drop(&mut self) { + self.stop.store(true, std::sync::atomic::Ordering::Relaxed); + if let Some(worker) = self.worker.take() { + let result = worker.join(); + if !std::thread::panicking() { + result.unwrap(); + } + } + } +} + +pub fn hash(id: u32) -> String { + format!("{id:064x}") +} + +fn jameson() -> MockCore { + MockCore::new( + 13271, + vec![ + Quorum { + id: 1, + created: 13166, + retired: 13262, + }, + Quorum { + id: 2, + created: 13190, + retired: u32::MAX, + }, + Quorum { + id: 3, + created: 13214, + retired: u32::MAX, + }, + Quorum { + id: 4, + created: 13238, + retired: u32::MAX, + }, + Quorum { + id: 5, + created: 13262, + retired: u32::MAX, + }, + ], + ) +} + +/// A cold client must retain a signer retired beyond the legacy snapshot. +#[tokio::test] +async fn should_serve_a_quorum_retired_after_the_previous_snapshot_height() { + let core = jameson(); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + assert!(!cache + .get_quorums() + .await + .unwrap() + .contains_quorum(&hex::decode(hash(1)).unwrap())); + let (height, previous) = cache.get_previous_quorums().await.unwrap(); + assert!( + previous.contains_quorum(&hex::decode(hash(1)).unwrap()), + "signer active at tip-12 must survive a cold cache refresh even though absent at tip-8" + ); + assert_eq!(height, 13223); +} + +/// Successful and failed DKG schedules must leave no signer missing from the window. +#[tokio::test] +async fn should_cover_every_quorum_active_in_the_window() { + for failed_dkg in [None, Some(8), Some(9)] { + let starts: Vec = (0..20) + .filter(|i| Some(*i) != failed_dkg) + .map(|i| i * 24 + 10) + .collect(); + let quorums: Vec = starts + .iter() + .enumerate() + .map(|(i, &created)| Quorum { + id: i as u32 + 1, + created, + retired: starts.get(i + 4).copied().unwrap_or(u32::MAX), + }) + .collect(); + for (tip, retention, step) in [ + (250, 48, 48), + (265, 48, 48), + (299, 48, 48), + (330, 48, 48), + (250, 167, 73), + (265, 167, 73), + (299, 167, 73), + (330, 167, 73), + ] { + let expected: Vec = quorums + .iter() + .filter(|q| q.created <= tip && q.retired > tip - retention) + .map(|q| hash(q.id)) + .collect(); + let mut core = MockCore::new(tip, quorums.clone()); + core.config.quorum.retention_blocks = retention; + core.config.quorum.sample_step_blocks = Some(step); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let (_, previous) = cache.get_previous_quorums().await.unwrap(); + let actual: Vec = previous + .iter() + .map(|q| hex::encode(&q.quorum_hash)) + .collect(); + for quorum in expected { + assert!( + actual.contains(&quorum), + "missing signer {quorum} at tip {tip}, failed DKG {failed_dkg:?}" + ); + } + } + } +} + +/// A cold restart must reconstruct the same window as a cache warmed through rotations. +#[tokio::test] +async fn should_return_the_same_window_after_restart() { + let core = jameson(); + let first = QuorumCache::new(core.config.clone()); + for tip in [13223, 13247, 13271] { + core.state.lock().unwrap().tip = tip; + first.refresh().await.unwrap(); + } + let second = QuorumCache::new(core.config.clone()); + second.refresh().await.unwrap(); + assert_eq!( + first.get_previous_quorums().await.unwrap(), + second.get_previous_quorums().await.unwrap() + ); +} + +/// Unchanged commitments must not repeatedly fetch their public keys. +#[tokio::test] +async fn should_not_call_quorum_info_again_for_memoized_hashes() { + let core = jameson(); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let first = core.info_calls().len(); + cache.refresh().await.unwrap(); + assert_eq!( + core.info_calls().len(), + first, + "known immutable public keys should not cost another info RPC" + ); +} + +/// Every key lookup must explicitly exclude secret shares. +#[tokio::test] +async fn should_never_request_secret_key_shares() { + let core = jameson(); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let calls = core.info_calls(); + assert!(!calls.is_empty()); + for call in calls { + assert_eq!(call["params"].as_array().unwrap().len(), 4); + assert_eq!( + call["params"][3], false, + "quorum info must explicitly request public information only" + ); + } +} + +pub async fn endpoint(cache: Arc, config: Config, path: &str) -> Value { + use axum::{ + body::{to_bytes, Body}, + http::Request, + }; + use tower::Service; + let masternodes = Arc::new(crate::masternode_cache::MasternodeCache::new( + config.clone(), + )); + let mut router = crate::api::create_router(cache, config, masternodes); + let response = router + .call(Request::builder().uri(path).body(Body::empty()).unwrap()) + .await + .unwrap(); + let status = response.status(); + let value: Value = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()).unwrap(); + assert_eq!( + status, + if path == "/ready" && value["success"] == false { + 503 + } else { + 200 + } + ); + value +} + +/// Height and refresh metadata must describe the cached list while keeping its array shape. +#[tokio::test] +async fn should_report_the_core_height_on_quorums() { + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let first = endpoint(cache.clone(), core.config.clone(), "/quorums").await; + assert_eq!(first["height"], 13271); + assert!(first["lastUpdated"].as_i64().unwrap() > 0); + assert!(first["data"].is_array(), "legacy data must remain an array"); + assert_eq!( + first, + endpoint(cache, core.config.clone(), "/quorums").await + ); +} + +/// Window failures preserve trusted data and degrade readiness only once history is stale. +#[tokio::test] +async fn should_keep_last_good_window_and_report_degraded_health_on_core_failure() { + let mut core = jameson(); + core.config.quorum.cache_ttl_seconds = 1; + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let before = endpoint(cache.clone(), core.config.clone(), "/previous").await; + core.state.lock().unwrap().fail_height = Some(13223); + assert!(cache.refresh().await.is_err()); + assert_eq!( + before, + endpoint(cache.clone(), core.config.clone(), "/previous").await + ); + assert_eq!( + endpoint(cache.clone(), core.config.clone(), "/ready").await["success"], + true, + "one failed window must not reject still-fresh trusted data" + ); + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(3)).await; + { + let mut state = core.state.lock().unwrap(); + state.tip = 13272; + state.fail_height = Some(13224); + state.quorums.push(Quorum { + id: 6, + created: 13272, + retired: u32::MAX, + }); + } + let refreshing = cache.clone(); + let refresh = tokio::spawn(async move { refreshing.refresh().await }); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while !refresh.is_finished() { + assert!( + std::time::Instant::now() < deadline, + "refresh did not finish" + ); + tokio::task::yield_now().await; + } + assert!(refresh.await.unwrap().is_err()); + assert!( + cache + .get_quorums() + .await + .unwrap() + .contains_quorum(&hex::decode(hash(6)).unwrap()), + "current keys must refresh even while history remains stale" + ); + assert_eq!( + before, + endpoint(cache.clone(), core.config.clone(), "/previous").await + ); + let ready = endpoint(cache.clone(), core.config.clone(), "/ready").await; + assert_eq!(ready["success"], false); + let health = endpoint(cache, core.config.clone(), "/health").await; + assert_eq!( + health["success"], false, + "stale trusted keys must be observable to health monitors" + ); + assert!(health["message"].as_str().unwrap().contains("degraded")); +} + +/// New signing keys must become available without waiting for the cache TTL. +#[tokio::test] +async fn should_reload_when_the_tip_changes_before_the_ttl() { + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + cache.clone().start_background_refresh().await; + { + let mut state = core.state.lock().unwrap(); + state.tip += 24; + let created = state.tip; + state.quorums.push(Quorum { + id: 6, + created, + retired: u32::MAX, + }); + } + let deadline = tokio::time::Instant::now() + Duration::from_secs(3); + loop { + if cache + .get_quorums() + .await + .unwrap() + .contains_quorum(&hex::decode(hash(6)).unwrap()) + { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "tip changes must not wait for the 60-second TTL" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +/// Trimming must preserve active keys and response order must protect newest client cache entries. +#[tokio::test] +async fn should_order_oldest_to_newest_and_trim_retired_first_at_the_cap() { + let quorums = (0..12) + .map(|id| Quorum { + id, + created: id * 24 + 10, + retired: (id + 4) * 24 + 10, + }) + .collect(); + let mut core = MockCore::new(275, quorums); + core.config.quorum.retention_blocks = 192; + for (cap, expected) in [(6, vec![6, 7, 8, 9, 10, 11]), (2, vec![8, 9, 10, 11])] { + core.config.quorum.max_entries = cap; + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let (_, previous) = cache.get_previous_quorums().await.unwrap(); + let actual: Vec = previous + .iter() + .map(|q| hex::encode(&q.quorum_hash)) + .collect(); + assert_eq!(actual, expected.into_iter().map(hash).collect::>(), + "oldest retired keys must go first and active keys must survive even when active count exceeds cap"); + } +} + +/// The retention floor and legacy offset must leave no unsampled boundary gap. +#[tokio::test] +async fn should_sample_the_lower_boundary_and_preserve_legacy_offset_coverage() { + let mut core = jameson(); + core.config.quorum.retention_blocks = 7; + core.config.quorum.previous_blocks_offset = 101; + core.config.quorum.sample_step_blocks = Some(47); + core.state.lock().unwrap().quorums.push(Quorum { + id: 0, + created: 13076, + retired: 13172, + }); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let (floor, previous) = cache.get_previous_quorums().await.unwrap(); + assert_eq!(floor, 13170); + assert!( + previous.contains_quorum(&hex::decode(hash(0)).unwrap()), + "quorum retired two blocks after the floor must be covered" + ); + let sampled: Vec = core + .state + .lock() + .unwrap() + .calls + .iter() + .filter(|r| r["params"][0] == "listextended") + .map(|r| r["params"][1].as_u64().unwrap()) + .collect(); + assert_eq!(sampled, [13271, 13224, 13177, 13170]); +} + +/// A failed historical query must never replace a complete window with a partial one. +#[tokio::test] +async fn should_keep_the_window_when_a_historical_sample_fails() { + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let before = endpoint(cache.clone(), core.config.clone(), "/previous").await; + core.state.lock().unwrap().fail_height = Some(13223); + assert!(cache.refresh().await.is_err()); + assert_eq!( + before, + endpoint(cache, core.config.clone(), "/previous").await + ); +} + +/// New active signing keys must survive a timed-out historical refresh. +#[tokio::test] +async fn should_publish_current_when_the_window_times_out() { + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let previous = cache.get_previous_quorums().await.unwrap(); + let (release, gate) = std::sync::mpsc::channel(); + { + let mut state = core.state.lock().unwrap(); + state.tip += 1; + let created = state.tip; + state.quorums.push(Quorum { + id: 6, + created, + retired: u32::MAX, + }); + state.historical_response_gate = Some((13224, gate)); + state.calls.clear(); + } + let refreshing = cache.clone(); + let refresh = tokio::spawn(async move { refreshing.refresh().await }); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while !core + .state + .lock() + .unwrap() + .calls + .iter() + .any(|call| call["params"][0] == "listextended" && call["params"][1] == 13224) + { + assert!( + std::time::Instant::now() < deadline, + "historical request did not start" + ); + tokio::task::yield_now().await; + } + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(31)).await; + let result = refresh.await.unwrap(); + release + .send(()) + .expect("the gated historical RPC must still be waiting after the async timeout"); + assert!(result.unwrap_err().to_string().contains("timed out")); + let current = endpoint(cache.clone(), core.config.clone(), "/quorums").await; + assert_eq!(current["success"], true); + assert!( + current["data"] + .as_array() + .unwrap() + .iter() + .any(|entry| entry["quorum_hash"] == hash(6)), + "successful active keys must survive the window timeout" + ); + assert_eq!(cache.get_snapshot().await.unwrap().height, 13272); + assert_eq!(cache.get_previous_quorums().await.unwrap(), previous); + assert!(cache.is_healthy().await); +} + +/// Late verification errors must preserve history without discarding new signing keys. +#[tokio::test] +async fn should_publish_current_when_the_final_tip_verification_fails() { + for failing_call in [2, 3] { + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let previous = cache.get_previous_quorums().await.unwrap(); + { + let mut state = core.state.lock().unwrap(); + state.tip += 1; + let created = state.tip; + state.quorums.push(Quorum { + id: 6, + created, + retired: u32::MAX, + }); + state.calls.clear(); + state.fail_block_count_at = Some(failing_call); + if failing_call == 3 { + state.tip_script = [13272, 13273, 13273].into(); + } + } + assert!(cache.refresh().await.is_err()); + let current = endpoint(cache.clone(), core.config.clone(), "/quorums").await; + assert!( + current["data"] + .as_array() + .unwrap() + .iter() + .any(|entry| entry["quorum_hash"] == hash(6)), + "successful active keys must survive a late getblockcount failure" + ); + assert_eq!(cache.get_snapshot().await.unwrap().height, 13272); + assert_eq!(cache.get_previous_quorums().await.unwrap(), previous); + assert!(cache.is_healthy().await); + core.state.lock().unwrap().fail_block_count_at = None; + cache.refresh().await.unwrap(); + assert!(cache.is_healthy().await); + } +} + +/// An active-list failure must not skip the independently recoverable rolling window. +#[tokio::test] +async fn should_attempt_the_window_even_when_the_current_list_fails() { + let core = jameson(); + core.state.lock().unwrap().fail_current_once = true; + let cache = QuorumCache::new(core.config.clone()); + assert!(cache.refresh().await.is_err()); + assert!(cache.get_quorums().await.is_err()); + let (_, previous) = cache.get_previous_quorums().await.unwrap(); + assert!(previous.contains_quorum(&hex::decode(hash(1)).unwrap())); +} + +/// An unavailable active list cannot produce a window and must not trigger historical RPCs. +#[tokio::test] +async fn should_skip_historical_queries_when_the_active_tip_retry_fails() { + check_unavailable_active_list_skips_history(false).await; +} + +/// An empty tip retry must stop before spending RPCs on an unusable historical window. +#[tokio::test] +async fn should_skip_historical_queries_when_the_active_tip_retry_is_empty() { + check_unavailable_active_list_skips_history(true).await; +} + +async fn check_unavailable_active_list_skips_history(empty: bool) { + let core = jameson(); + { + let mut state = core.state.lock().unwrap(); + state.fail_current = !empty; + state.empty = empty; + } + let cache = QuorumCache::new(core.config.clone()); + assert!(cache.refresh().await.is_err()); + assert!(cache.get_quorums().await.is_err()); + assert!(cache.get_previous_quorums().await.is_err()); + let sampled: Vec = core + .state + .lock() + .unwrap() + .calls + .iter() + .filter(|call| call["params"][0] == "listextended") + .map(|call| call["params"][1].as_u64().unwrap()) + .collect(); + assert_eq!( + sampled.iter().filter(|&&height| height != 13271).count(), + 0, + "history cannot recover a window without a populated active list at the tip" + ); + assert_eq!( + sampled, + [13271, 13271], + "the active tip must still be retried" + ); + assert!(core.info_calls().is_empty()); +} + +/// Malformed or empty upstream data must not become a successful trusted-key response. +#[tokio::test] +async fn should_never_publish_an_empty_or_malformed_success() { + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let before = endpoint(cache.clone(), core.config.clone(), "/quorums").await; + for malformed in [false, true] { + { + let mut state = core.state.lock().unwrap(); + state.empty = !malformed; + state.malformed = malformed; + } + assert!(cache.refresh().await.is_err()); + assert_eq!( + before, + endpoint(cache.clone(), core.config.clone(), "/quorums").await + ); + } + let fresh = Arc::new(QuorumCache::new(core.config.clone())); + assert!(fresh.refresh().await.is_err()); + assert_eq!( + endpoint(fresh, core.config.clone(), "/quorums").await["success"], + false + ); +} + +/// Reorgs must remove orphaned membership without refetching unchanged commitments. +#[tokio::test] +async fn should_drop_orphaned_quorums_after_a_reorg_without_clearing_the_memo() { + let core = jameson(); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let initial_info_calls = core.info_calls().len(); + { + let mut state = core.state.lock().unwrap(); + state.tip -= 1; + state.quorums.retain(|q| q.id != 5); + state.quorums.push(Quorum { + id: 6, + created: 13262, + retired: u32::MAX, + }); + } + cache.refresh().await.unwrap(); + assert_eq!( + core.info_calls().len(), + initial_info_calls + 1, + "unchanged commitments must remain memoized across a reorg" + ); + let (_, previous) = cache.get_previous_quorums().await.unwrap(); + assert!(!previous.contains_quorum(&hex::decode(hash(5)).unwrap())); + assert!(previous.contains_quorum(&hex::decode(hash(6)).unwrap())); +} + +/// The TTL must force reloads even when Core height does not advance. +#[tokio::test] +async fn should_reload_unchanged_height_at_the_ttl() { + let mut core = jameson(); + core.config.quorum.cache_ttl_seconds = 1; + core.config.quorum.poll_interval_seconds = 60; + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let count_lists = || { + core.state + .lock() + .unwrap() + .calls + .iter() + .filter(|r| r["params"][0] == "listextended") + .count() + }; + let initial = count_lists(); + cache.clone().start_background_refresh().await; + let deadline = tokio::time::Instant::now() + Duration::from_secs(3); + while count_lists() == initial { + assert!( + tokio::time::Instant::now() < deadline, + "TTL must bound refreshes even if configured poll interval is longer" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +/// Unsafe sampling must be rejected when commitment mining offsets shorten active runs. +#[tokio::test] +async fn should_reject_steps_that_miss_quorums_with_variable_mining_offsets() { + let starts: Vec = (0..16) + .map(|cycle| cycle * 24 + if cycle == 0 { 18 } else { 10 }) + .collect(); + let quorums: Vec = starts + .iter() + .enumerate() + .map(|(i, &created)| Quorum { + id: i as u32, + created, + retired: starts.get(i + 4).copied().unwrap_or(u32::MAX), + }) + .collect(); + assert_eq!(quorums[0].retired - quorums[0].created, 88); + let mut core = MockCore::new(299, quorums); + core.config.quorum.retention_blocks = 288; + core.config.quorum.sample_step_blocks = Some(96); + // The naive grid 299, 203, 107, 11 skips a quorum active on [18, 106). + assert!( + core.config.validate_quorum().is_err(), + "A*D is unsafe when commitment mining offsets vary" + ); + core.config.quorum.sample_step_blocks = Some(73); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + assert!(cache + .get_previous_quorums() + .await + .unwrap() + .1 + .contains_quorum(&hex::decode(hash(0)).unwrap())); +} + +/// A blocked Core RPC must leave the async runtime able to run other tasks. +#[tokio::test(flavor = "current_thread")] +async fn should_keep_the_async_runtime_responsive_during_blocking_core_calls() { + let core = jameson(); + let (release, gate) = std::sync::mpsc::channel(); + core.state.lock().unwrap().first_response_gate = Some(gate); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + let refresh = tokio::spawn(async move { cache.refresh().await }); + tokio::time::timeout(Duration::from_secs(10), async { + while core.state.lock().unwrap().calls.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + // This runtime task must be able to release a Core RPC that is still blocked. + release.send(()).unwrap(); + refresh.await.unwrap().unwrap(); +} + +/// Pre-activation historical samples must not prevent serving the first active keys. +#[tokio::test] +async fn should_serve_the_window_when_the_floor_precedes_platform_quorum_activation() { + let core = MockCore::new( + 30, + vec![Quorum { + id: 1, + created: 10, + retired: u32::MAX, + }], + ); + core.state.lock().unwrap().omitted_type_before = Some(10); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let (floor, window) = cache.get_previous_quorums().await.unwrap(); + assert_eq!(floor, 0); + assert_eq!(window.len(), 1); + assert!(window.contains_quorum(&hex::decode(hash(1)).unwrap())); +} + +/// A replacement final commitment for the same quorum hash must replace its key. +#[tokio::test] +async fn should_refetch_a_key_when_the_mined_commitment_changes() { + let core = jameson(); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let before = core.info_calls().len(); + core.state.lock().unwrap().commitments.insert(5, 1); + cache.refresh().await.unwrap(); + assert_eq!(core.info_calls().len(), before + 1); + let list = cache.get_quorums().await.unwrap(); + assert_eq!( + list.get_entry(&hex::decode(hash(5)).unwrap()).unwrap().key, + vec![18; 48] + ); +} + +/// Core's empty historical result must not erase a quorum that must exist there. +#[tokio::test] +async fn should_reject_a_historical_sample_missing_an_older_active_quorum() { + let core = jameson(); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let before = cache.get_previous_quorums().await.unwrap(); + for omit_only in [None, Some(2)] { + { + let mut state = core.state.lock().unwrap(); + state.omit_at_height = Some(13223); + state.omit_only = omit_only; + } + assert!(cache.refresh().await.is_err()); + assert_eq!(cache.get_previous_quorums().await.unwrap(), before); + } +} + +/// Newer history must prove completeness after tip-active quorums cease to constrain samples. +#[tokio::test] +async fn should_reject_a_missing_deep_sample_proven_by_newer_history() { + let mining_heights = [858, 882, 898, 922, 946, 970, 994]; + let quorums = mining_heights + .iter() + .enumerate() + .map(|(id, &created)| Quorum { + id: id as u32, + created, + retired: mining_heights.get(id + 4).copied().unwrap_or(u32::MAX), + }) + .collect(); + let mut core = MockCore::new(1000, quorums); + core.config.quorum.retention_blocks = 144; + core.config.quorum.sample_step_blocks = Some(48); + core.state.lock().unwrap().creation_heights = (0..7).map(|id| (id, 840 + id * 24)).collect(); + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + let before = cache.get_previous_quorums().await.unwrap(); + assert!( + before.1.contains_quorum(&hex::decode(hash(0)).unwrap()), + "the quorum mined at 858 and retired at 946 appears only at the 904 sample" + ); + core.state.lock().unwrap().omit_at_height = Some(904); + let error = cache + .refresh() + .await + .expect_err("a quorum created at 864 and observed at 952 must exist at 904"); + assert!(error.to_string().contains("must still be active")); + assert_eq!(cache.get_previous_quorums().await.unwrap(), before); +} + +/// Readiness gives load balancers a failing status without changing health's HTTP status. +#[tokio::test] +async fn should_return_service_unavailable_when_not_ready() { + use axum::{body::Body, http::Request}; + use tower::Service; + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + let masternodes = Arc::new(crate::masternode_cache::MasternodeCache::new( + core.config.clone(), + )); + let mut router = crate::api::create_router(cache.clone(), core.config.clone(), masternodes); + let response = router + .call( + Request::builder() + .uri("/ready") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), 503); + cache.refresh().await.unwrap(); + let response = router + .call( + Request::builder() + .uri("/ready") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), 200); +} + +/// A moving chain must not discard an active list tied to an explicit sampled height. +#[tokio::test] +async fn should_publish_current_and_retry_the_window_when_the_tip_moves() { + let core = jameson(); + { + let mut state = core.state.lock().unwrap(); + state.tip_script = [13271, 13295, 13295].into(); + state.quorums.push(Quorum { + id: 6, + created: 13286, + retired: u32::MAX, + }); + } + let cache = QuorumCache::new(core.config.clone()); + cache.refresh().await.unwrap(); + assert_eq!(cache.get_snapshot().await.unwrap().height, 13271); + let (floor, previous) = cache.get_previous_quorums().await.unwrap(); + assert_eq!(floor, 13247); + assert!(previous.contains_quorum(&hex::decode(hash(6)).unwrap())); + assert!(cache.is_healthy().await); +} + +/// Every legal mining-offset pair must leave no hole at the maximum allowed step. +#[tokio::test] +async fn should_cover_all_mining_offset_pairs_at_the_maximum_step() { + for first in 10..=LATEST_MINING_OFFSET { + for replacement in 10..=LATEST_MINING_OFFSET { + let starts: Vec = (0..20) + .map(|cycle| { + cycle * 24 + + match cycle { + 0 => first, + 4 => replacement, + _ => 10, + } + }) + .collect(); + let quorums: Vec = starts + .iter() + .enumerate() + .map(|(i, &created)| Quorum { + id: i as u32, + created, + retired: starts.get(i + 4).copied().unwrap_or(u32::MAX), + }) + .collect(); + let mut core = MockCore::new(250, quorums.clone()); + core.config.quorum.retention_blocks = 288; + core.config.quorum.sample_step_blocks = Some(73); + core.config.validate_quorum().unwrap(); + let cache = QuorumCache::new(core.config.clone()); + for tip in [250u32, 299, 345] { + core.state.lock().unwrap().tip = tip; + cache.refresh().await.unwrap(); + let (floor, window) = cache.get_previous_quorums().await.unwrap(); + let expected: Vec = quorums + .iter() + .filter(|q| q.created <= tip && q.retired > floor) + .map(|q| hash(q.id)) + .collect(); + let actual: Vec = + window.iter().map(|q| hex::encode(&q.quorum_hash)).collect(); + assert_eq!( + actual, expected, + "mining offsets {first}/{replacement}, tip {tip}" + ); + } + } + } +} + +/// The legacy SDK JSON structs must accept actual endpoint responses unchanged. +#[tokio::test] +async fn should_deserialize_responses_into_legacy_sdk_types() { + #[derive(serde::Deserialize)] + struct QuorumData { + quorum_hash: String, + key: String, + height: u64, + valid_members_count: u32, + } + #[derive(serde::Deserialize)] + struct QuorumsResponse { + success: bool, + data: Vec, + } + #[derive(serde::Deserialize)] + struct PreviousQuorumsData { + height: u64, + quorums: Vec, + } + #[derive(serde::Deserialize)] + struct PreviousQuorumsResponse { + success: bool, + data: PreviousQuorumsData, + } + let core = jameson(); + let cache = Arc::new(QuorumCache::new(core.config.clone())); + cache.refresh().await.unwrap(); + let current: QuorumsResponse = + serde_json::from_value(endpoint(cache.clone(), core.config.clone(), "/quorums").await) + .unwrap(); + assert!(current.success); + assert_eq!(current.data.len(), 4); + assert_eq!(current.data[0].quorum_hash, hash(2)); + assert_eq!(current.data[0].key.len(), 96); + assert_eq!(current.data[0].height, 13190); + assert_eq!(current.data[0].valid_members_count, 12); + let previous: PreviousQuorumsResponse = + serde_json::from_value(endpoint(cache, core.config.clone(), "/previous").await).unwrap(); + assert!(previous.success); + assert_eq!(previous.data.height, 13223); + assert_eq!(previous.data.quorums.len(), 5); +} diff --git a/src/quorum_loader.rs b/src/quorum_loader.rs index ad71a4c..6000751 100644 --- a/src/quorum_loader.rs +++ b/src/quorum_loader.rs @@ -1,177 +1,365 @@ use crate::config::Config; use crate::quorum_list::{QuorumList, QuorumListEntry}; -use dashcore_rpc::{Client, RpcApi, Auth}; -use serde::Deserialize; +use dashcore_rpc::{jsonrpc, Client, RpcApi}; +use serde_json::{json, Value}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::error::Error; +use std::time::{Duration, Instant}; -#[derive(Debug, Deserialize)] -pub struct QuorumListResult { - pub quorums: serde_json::Value, +pub type LoadResult = Result>; +pub const REFRESH_TIMEOUT: Duration = Duration::from_secs(30); +// Platform quorum commitments are mined at offsets 10..18 after the DKG base height. +pub const LATEST_MINING_OFFSET: u32 = 18; + +/// Only immutable public keys are memoized; membership always comes from Core. +#[derive(Default)] +pub struct KeyMemo { + keys: HashMap<(String, String), Vec>, + order: VecDeque<(String, String)>, +} + +impl KeyMemo { + fn insert(&mut self, hash: (String, String), key: Vec, capacity: usize) { + if self.keys.contains_key(&hash) { + return; + } + while self.keys.len() >= capacity { + if let Some(oldest) = self.order.pop_front() { + self.keys.remove(&oldest); + } else { + break; + } + } + self.order.push_back(hash.clone()); + self.keys.insert(hash, key); + } +} + +pub struct CurrentResult { + pub tip: u32, + pub current: LoadResult, } -#[derive(Debug, Deserialize)] -pub struct QuorumInfo { - pub quorum_hash: String, - pub quorum_public_key: String, +fn client(config: &Config) -> LoadResult { + let url = if config.rpc.url.contains("://") { + config.rpc.url.clone() + } else { + format!("http://{}", config.rpc.url) + }; + let transport = jsonrpc::simple_http::SimpleHttpTransport::builder() + .url(&url)? + .auth(&config.rpc.username, Some(&config.rpc.password)) + .timeout(Duration::from_secs(5)) + .build(); + Ok(Client::from_jsonrpc(jsonrpc::Client::with_transport( + transport, + ))) } -pub async fn load_initial_quorums( +fn block_height(client: &Client) -> LoadResult { + let result: u64 = client.call("getblockcount", &[])?; + Ok(u32::try_from(result)?) +} + +pub fn get_current_block_height(config: &Config) -> LoadResult { + block_height(&client(config)?) +} + +fn load_at_height( + client: &Client, config: &Config, -) -> Result> { - // Create RPC client - let auth = Auth::UserPass(config.rpc.username.clone(), config.rpc.password.clone()); - let client = Client::new(&config.rpc.url, auth)?; - - // Get the extended quorum list (returns detailed info for all quorums) - let result: serde_json::Value = client.call("quorum", &[serde_json::json!("listextended")])?; - - let mut quorum_list = QuorumList::new(); - - // Parse the extended quorum list - extract the appropriate LLMQ type based on network - let llmq_type = config.get_llmq_type(); - if let Some(quorum_obj) = result.as_object() { - if let Some(llmq_quorums) = quorum_obj.get(llmq_type) { - if let Some(quorums_arr) = llmq_quorums.as_array() { - for quorum_item in quorums_arr { - if let Some(quorum_obj) = quorum_item.as_object() { - for (quorum_hash_str, quorum_info) in quorum_obj { - if let Some(info_obj) = quorum_info.as_object() { - let quorum_hash_bytes = hex::decode(quorum_hash_str)?; - if quorum_hash_bytes.len() == 32 { - // Get the actual quorum public key via quorum info - let rpc_params = [ - serde_json::json!("info"), - serde_json::json!(config.get_llmq_type_id()), - serde_json::json!(quorum_hash_str) - ]; - println!("DEBUG: Calling quorum with params: {:?}", rpc_params); - let info_result: serde_json::Value = client.call("quorum", &rpc_params)?; - - if let Some(detailed_info) = info_result.as_object() { - if let Some(pubkey_str) = detailed_info.get("quorumPublicKey").and_then(|v| v.as_str()) { - let public_key = hex::decode(pubkey_str)?; - - if public_key.len() == 48 { - let creation_height = info_obj.get("creationHeight") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as u32; - let valid_members_count = info_obj.get("numValidMembers") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as u32; - - let entry = QuorumListEntry::new_extended( - quorum_hash_bytes, - public_key, - creation_height, - Vec::new(), // No members data needed - String::new(), // No threshold signature needed - 0, // No mining members count needed - valid_members_count, - ); - quorum_list.add_entry(entry); - println!("Loaded quorum: {} at height {} with {} members", - quorum_hash_str, creation_height, valid_members_count); - } - } - } - } - } - } + height: u32, + memo: &mut KeyMemo, + deadline: Instant, + allow_inactive_type: bool, +) -> LoadResult { + check_deadline(deadline)?; + let result: Value = client.call("quorum", &[json!("listextended"), json!(height)])?; + let result = result.as_object().ok_or("Invalid extended quorum list")?; + let quorums = match result.get(config.get_llmq_type()) { + Some(value) => value.as_array().ok_or("Invalid Platform quorum list")?, + // Core omits types that were not enabled at a historical height. + None if allow_inactive_type => return Ok(QuorumList::new()), + None => return Err("Missing Platform quorum list at the tip".into()), + }; + let mut list = QuorumList::new(); + for item in quorums { + let entries = item.as_object().ok_or("Invalid quorum entry")?; + if entries.is_empty() { + return Err("Empty quorum entry".into()); + } + for (hash, details) in entries { + let hash_bytes = hex::decode(hash)?; + if hash_bytes.len() != 32 { + return Err("Invalid quorum hash length".into()); + } + let mined_hash = details + .get("minedBlockHash") + .and_then(Value::as_str) + .ok_or("Missing quorum mined block hash")?; + if hex::decode(mined_hash)?.len() != 32 { + return Err("Invalid mined block hash length".into()); + } + let memo_key = (hash.clone(), mined_hash.to_string()); + let creation_height = details + .get("creationHeight") + .and_then(Value::as_u64) + .ok_or("Missing quorum creation height")?; + let valid_members = details + .get("numValidMembers") + .and_then(Value::as_u64) + .ok_or("Missing quorum valid member count")?; + let creation_height = u32::try_from(creation_height)?; + let valid_members = u32::try_from(valid_members)?; + let public_key = match memo.keys.get(&memo_key) { + Some(key) => key.clone(), + None => { + check_deadline(deadline)?; + let info: Value = client.call( + "quorum", + &[ + json!("info"), + json!(config.get_llmq_type_id()), + json!(hash), + json!(false), + ], + )?; + let key = hex::decode( + info.get("quorumPublicKey") + .and_then(Value::as_str) + .ok_or("Missing quorum public key")?, + )?; + if key.len() != 48 { + return Err("Invalid quorum public key length".into()); + } + let info_mined_hash = info + .get("minedBlock") + .and_then(Value::as_str) + .ok_or("Missing quorum minedBlock")?; + if info_mined_hash != mined_hash { + return Err("Quorum commitment changed during key lookup".into()); } + memo.insert(memo_key, key.clone(), config.quorum.key_memo_max_entries); + key } - } + }; + list.add_entry(QuorumListEntry::new_extended( + hash_bytes, + public_key, + creation_height, + Vec::new(), + String::new(), + 0, + valid_members, + )); } } - - println!("Loaded {} quorums from Dash Core", quorum_list.len()); - Ok(quorum_list) + sort_oldest_first(&mut list); + Ok(list) } -pub async fn get_current_block_height( - config: &Config, -) -> Result> { - let auth = Auth::UserPass(config.rpc.username.clone(), config.rpc.password.clone()); - let client = Client::new(&config.rpc.url, auth)?; - - let result: serde_json::Value = client.call("getblockcount", &[])?; - let height = result.as_u64().ok_or("Invalid block count response")? as u32; - - Ok(height) +fn check_deadline(deadline: Instant) -> LoadResult<()> { + if Instant::now() >= deadline { + return Err("Quorum refresh deadline exceeded".into()); + } + Ok(()) +} + +fn sort_oldest_first(list: &mut QuorumList) { + list.list.sort_by(|a, b| { + a.height + .cmp(&b.height) + .then_with(|| a.quorum_hash.cmp(&b.quorum_hash)) + }); } -pub async fn load_quorums_at_height( +fn load_window( + client: &Client, config: &Config, - height: u32, -) -> Result> { - let auth = Auth::UserPass(config.rpc.username.clone(), config.rpc.password.clone()); - let client = Client::new(&config.rpc.url, auth)?; - - // Get the extended quorum list at specific height - let result: serde_json::Value = client.call("quorum", &[ - serde_json::json!("listextended"), - serde_json::json!(height) - ])?; - - let mut quorum_list = QuorumList::new(); - - // Parse the extended quorum list - extract the appropriate LLMQ type based on network - let llmq_type = config.get_llmq_type(); - if let Some(quorum_obj) = result.as_object() { - if let Some(llmq_quorums) = quorum_obj.get(llmq_type) { - if let Some(quorums_arr) = llmq_quorums.as_array() { - for quorum_item in quorums_arr { - if let Some(quorum_obj) = quorum_item.as_object() { - for (quorum_hash_str, quorum_info) in quorum_obj { - if let Some(info_obj) = quorum_info.as_object() { - let quorum_hash_bytes = hex::decode(quorum_hash_str)?; - if quorum_hash_bytes.len() == 32 { - // Get the actual quorum public key via quorum info - let rpc_params = [ - serde_json::json!("info"), - serde_json::json!(config.get_llmq_type_id()), - serde_json::json!(quorum_hash_str), - serde_json::json!(true) // includeSkShare - ]; - println!("DEBUG: Calling quorum with params: {:?}", rpc_params); - let info_result: serde_json::Value = client.call("quorum", &rpc_params)?; - - if let Some(detailed_info) = info_result.as_object() { - if let Some(pubkey_str) = detailed_info.get("quorumPublicKey").and_then(|v| v.as_str()) { - let public_key = hex::decode(pubkey_str)?; - - if public_key.len() == 48 { - let creation_height = info_obj.get("creationHeight") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as u32; - let valid_members_count = info_obj.get("numValidMembers") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as u32; - - let entry = QuorumListEntry::new_extended( - quorum_hash_bytes, - public_key, - creation_height, - Vec::new(), // No members data needed - String::new(), // No threshold signature needed - 0, // No mining members count needed - valid_members_count, - ); - quorum_list.add_entry(entry); - println!("Loaded quorum at height {}: {} with {} members", - height, quorum_hash_str, valid_members_count); - } - } - } - } - } - } + tip: u32, + current: &LoadResult, + memo: &mut KeyMemo, + deadline: Instant, +) -> LoadResult<(u32, QuorumList)> { + let step = config.quorum_sample_step(); + let floor = tip.saturating_sub(config.quorum_retention()); + // Retry the tip for the window even if the independent active-list fetch failed. + let active = match current { + Ok(list) => list.clone(), + Err(_) => load_at_height(client, config, tip, memo, deadline, false)?, + }; + if active.is_empty() { + return Err("Core returned no active Platform quorums".into()); + } + let mut window = QuorumList::new(); + let mut height = tip; + let mut history_error = None; + while height > floor { + // Include the lower boundary when the regular grid would overshoot it. + height = height.saturating_sub(step).max(floor); + match load_at_height(client, config, height, memo, deadline, true) { + Ok(list) => { + // A quorum observed later was already active after its latest mining height. + for entry in active.iter().chain(window.iter()) { + if u64::from(entry.height) + u64::from(LATEST_MINING_OFFSET) + <= u64::from(height) + && !list.contains_quorum(&entry.quorum_hash) + { + return Err( + "Historical sample is missing a quorum that must still be active" + .into(), + ); } } + for entry in list.list { + window.add_entry(entry); + } + } + Err(error) => { + history_error = Some(error); + break; } } } - - println!("Loaded {} quorums from Dash Core at height {}", quorum_list.len(), height); - Ok(quorum_list) + if let Some(error) = history_error { + return Err(error); + } + let active_hashes: HashSet> = active.iter().map(|q| q.quorum_hash.clone()).collect(); + for entry in active.list { + window.add_entry(entry); + } + sort_oldest_first(&mut window); + let mut excess = window.len().saturating_sub(config.quorum.max_entries); + window.list.retain(|entry| { + if excess > 0 && !active_hashes.contains(&entry.quorum_hash) { + excess -= 1; + false + } else { + true + } + }); + Ok((floor, window)) +} + +/// Blocking RPC work; callers must run this on a blocking worker. +pub fn refresh_current(config: &Config, memo: &mut KeyMemo) -> LoadResult { + let deadline = Instant::now() + REFRESH_TIMEOUT; + let client = client(config)?; + let tip = block_height(&client)?; + let current = load_at_height(&client, config, tip, memo, deadline, false).and_then(|list| { + if list.is_empty() { + Err("Core returned no active Platform quorums".into()) + } else { + Ok(list) + } + }); + Ok(CurrentResult { tip, current }) } +/// Window failures, including tip verification, are independent of active publication. +pub fn refresh_previous( + config: &Config, + memo: &mut KeyMemo, + current: LoadResult, +) -> LoadResult<(u32, QuorumList)> { + let deadline = Instant::now() + REFRESH_TIMEOUT; + let client = client(config)?; + let CurrentResult { tip, current } = match current { + Ok(result) => result, + Err(error) => CurrentResult { + tip: block_height(&client)?, + current: Err(error), + }, + }; + let mut previous = load_window(&client, config, tip, ¤t, memo, deadline); + let next_tip = block_height(&client)?; + if next_tip != tip { + let next_active = load_at_height(&client, config, next_tip, memo, deadline, false); + previous = load_window(&client, config, next_tip, &next_active, memo, deadline); + if block_height(&client)? != next_tip { + previous = Err("Core tip changed again while retrying the rolling window".into()); + } + } + previous +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::quorum_cache_tests::{hash, MockCore, Quorum}; + + /// Missing commitment identity must be reported separately from a detected fork switch. + #[test] + fn should_report_a_missing_info_mined_block_without_memoizing() { + let core = MockCore::new( + 100, + vec![Quorum { + id: 1, + created: 10, + retired: u32::MAX, + }], + ); + core.state.lock().unwrap().omit_info_mined_block = true; + let mut memo = KeyMemo::default(); + let error = refresh_current(&core.config, &mut memo) + .unwrap() + .current + .unwrap_err(); + assert_eq!(error.to_string(), "Missing quorum minedBlock"); + assert!(memo.keys.is_empty()); + assert!(memo.order.is_empty()); + } + + /// A fork switch during key lookup must not memoize the replacement under the old commitment. + #[test] + fn should_reject_a_commitment_change_between_list_and_info_without_memoizing() { + let core = MockCore::new( + 100, + vec![Quorum { + id: 1, + created: 10, + retired: u32::MAX, + }], + ); + core.state.lock().unwrap().commitment_change_on_info = Some((1, 1)); + let mut memo = KeyMemo::default(); + let result = refresh_current(&core.config, &mut memo).unwrap(); + let error = result.current.expect_err( + "the info key belongs to a different mined commitment than the listed quorum", + ); + assert!(error.to_string().contains("commitment changed")); + assert!( + memo.keys.is_empty(), + "the mismatched commitment must not populate the memo" + ); + assert!(memo.order.is_empty()); + core.state.lock().unwrap().commitments.clear(); + let restored = refresh_current(&core.config, &mut memo) + .unwrap() + .current + .unwrap(); + assert_eq!( + restored + .get_entry(&hex::decode(hash(1)).unwrap()) + .unwrap() + .key, + vec![17; 48], + "returning to the original fork must fetch its original key" + ); + assert_eq!(core.info_calls().len(), 2); + assert_eq!(memo.keys.len(), 1); + } + + /// Key caching must remain bounded even when historical commitments accumulate. + #[test] + fn key_memo_is_bounded_and_evicts_oldest() { + let mut memo = KeyMemo::default(); + for id in 0..3 { + memo.insert((id.to_string(), "mined".into()), vec![id], 2); + } + assert_eq!(memo.keys.len(), 2); + assert!(!memo.keys.contains_key(&("0".into(), "mined".into()))); + assert!(memo.keys.contains_key(&("2".into(), "mined".into()))); + assert_eq!(memo.order.len(), 2); + } +}