diff --git a/src/metrics_tracker.rs b/src/metrics_tracker.rs index 27075eb..fdd4cf1 100644 --- a/src/metrics_tracker.rs +++ b/src/metrics_tracker.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use std::time::Instant; use parking_lot::RwLock; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, watch}; use tracing::{debug, info}; use crate::events::Event; @@ -29,6 +29,7 @@ pub struct MetricsTracker { cores_status_snapshot: RwLock>, core_processing: RwLock>>, live_counters: Arc, + events_recorded: watch::Sender, } impl Default for MetricsTracker { @@ -44,6 +45,7 @@ impl MetricsTracker { cores_status_snapshot: RwLock::new(Arc::new(serde_json::json!(null))), core_processing: RwLock::new(HashMap::new()), live_counters: Arc::new(LiveCounters::new()), + events_recorded: watch::Sender::new(0), } } @@ -62,6 +64,15 @@ impl MetricsTracker { pub fn live_counters(&self) -> &Arc { &self.live_counters } + + /// Wait until the tracker task has recorded at least `expected` events. + pub async fn wait_for_events(&self, expected: u64) { + let _ = self + .events_recorded + .subscribe() + .wait_for(|count| *count >= expected) + .await; + } } // === Internal state, owned exclusively by the tracker task === @@ -563,13 +574,20 @@ pub async fn run(shared: Arc, mut rx: mpsc::Receiver 0 + }); state.maybe_rebuild_snapshots(); state.maybe_log_stats(); state.maybe_cleanup(); diff --git a/tests/api_tests.rs b/tests/api_tests.rs index b716660..b7bcc40 100644 --- a/tests/api_tests.rs +++ b/tests/api_tests.rs @@ -2,13 +2,19 @@ mod common; use axum::http::StatusCode; use axum_test::TestServer; -use serde_json::Value; +use serde_json::{json, Value}; use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tart_backend::api::{create_api_router, ApiState}; use tart_backend::events::Event; +use tart_backend::feature_flags::FeatureFlags; +use tart_backend::metrics_tracker::{self, MetricsTracker}; use tart_backend::{EventStore, TelemetryServer}; -async fn setup_test_api() -> (TestServer, Arc, u16, Arc) { +const SLOT_MICROS: u64 = 6_000_000; +const MICROS_PER_MILLI: u64 = 1_000; + +async fn setup_test_api() -> (TestServer, Arc, u16, Arc) { let database_url = common::test_database_url(); let store = Arc::new( @@ -22,10 +28,24 @@ async fn setup_test_api() -> (TestServer, Arc, u16, Arc (TestServer, Arc, u16, Arc Value { + let response = server.get(path).await; + assert_eq!(response.status_code(), StatusCode::OK, "GET {path}"); + response.json() +} + +/// Best-block changes for consecutive slots, one slot period apart. +fn best_blocks(first_slot: u32, first_at: u64, count: u32) -> Vec { + (0..count) + .map(|offset| { + let slot = first_slot + offset; + common::best_block_event_with_hash( + first_at + u64::from(offset) * SLOT_MICROS, + slot, + common::block_hash(slot), + ) + }) + .collect() +} + +fn status_with_peers(ts: u64, num_peers: u32, num_val_peers: u32, num_sync_peers: u32) -> Event { + Event::Status { + timestamp: ts, + num_peers, + num_val_peers, + num_sync_peers, + num_guarantees: vec![0; common::TEST_CORE_COUNT], + num_shards: 100, + shards_size: 50_000, + num_preimages: 10, + preimages_size: 4096, + } } #[tokio::test] @@ -74,30 +141,21 @@ async fn test_health_endpoint() { assert!(json.get("version").is_some()); } +/// A node is identified by its peer id, and stays without a disconnect time +/// while its connection is open. #[tokio::test] async fn test_node_details_endpoint() { let (server, telemetry_server, telemetry_port, _) = setup_test_api().await; + let _node = common::connect_test_node(telemetry_port, 3, &telemetry_server).await; + let node_id = common::node_id_hex(3); - // Connect a node (includes flush) - let _stream = common::connect_test_node(telemetry_port, 3, &telemetry_server).await; - - // Get node ID (hex encoded peer_id) - let node_id = hex::encode([3u8; 32]); - - let response = server.get(&format!("/api/nodes/{}", node_id)).await; - - assert_eq!(response.status_code(), StatusCode::OK); - - let json: Value = response.json(); + let json = get_json(&server, &format!("/api/nodes/{node_id}")).await; assert_eq!(json["node_id"], node_id); + assert_eq!(json["peer_id"], node_id); assert_eq!(json["implementation_name"], "test-node-3"); assert_eq!(json["is_connected"], true); - assert!(json["connection_info"].is_object()); - - let conn_info = &json["connection_info"]; - assert!(conn_info["address"].is_string()); - assert_eq!(conn_info["event_count"], 0); - assert!(conn_info["connected_duration_secs"].is_number()); + assert_eq!(json["disconnected_at"], Value::Null); + assert_eq!(json["connection_info"]["event_count"], 0); } /// A burst of events reaches the server over many socket reads and writer @@ -107,7 +165,6 @@ async fn test_node_event_count_after_burst() { let (server, telemetry_server, telemetry_port, _) = setup_test_api().await; let mut node = common::connect_test_node(telemetry_port, 4, &telemetry_server).await; - const SLOT_MICROS: u64 = 6_000_000; let first_slot_at = common::now_jce_micros() - 1000 * SLOT_MICROS; let events: Vec = (0..1000u32) .map(|slot| common::best_block_event(first_slot_at + u64::from(slot) * SLOT_MICROS, slot)) @@ -131,7 +188,6 @@ async fn test_send_events_returns_server_event_ids() { let (server, telemetry_server, telemetry_port, _) = setup_test_api().await; let mut node = common::connect_test_node(telemetry_port, 5, &telemetry_server).await; - const SLOT_MICROS: u64 = 6_000_000; let slot_100_at = common::now_jce_micros() - 10 * SLOT_MICROS; let event_ids = common::send_events( &mut node, @@ -179,33 +235,104 @@ async fn test_node_details_not_found() { // --- Metrics endpoints --- +/// The realtime window counts every event of every node, best-block changes +/// as blocks and finalizations separately, and reports the highest best slot +/// seen on the network. #[tokio::test] async fn test_realtime_metrics() { - let (server, _, _, _) = setup_test_api().await; - let response = server.get("/api/metrics/realtime?seconds=60").await; - assert_eq!(response.status_code(), StatusCode::OK); - let json: Value = response.json(); - assert!(json.is_object()); + let (server, telemetry_server, telemetry_port, metrics_tracker) = setup_test_api().await; + let mut author = common::connect_test_node(telemetry_port, 1, &telemetry_server).await; + let mut importer = common::connect_test_node(telemetry_port, 2, &telemetry_server).await; + + let slot_100_at = common::now_jce_micros() - 3 * SLOT_MICROS; + let mut author_events = best_blocks(100, slot_100_at, 3); + author_events.push(common::finalized_block_event_with_hash( + slot_100_at + 2 * SLOT_MICROS + 50 * MICROS_PER_MILLI, + 100, + common::block_hash(100), + )); + common::send_events(&mut author, &author_events).await; + common::send_events( + &mut importer, + &best_blocks(100, slot_100_at + 400 * MICROS_PER_MILLI, 3), + ) + .await; + metrics_tracker.wait_for_events(7).await; + wait_for_current_second_to_end().await; + + let json = get_json(&server, "/api/metrics/realtime?seconds=60").await; + assert_eq!(json["window_seconds"], 60); + assert_eq!(json["data"].as_array().unwrap().len(), 60); + assert_eq!(json["totals"]["events"], 7); + assert_eq!(json["totals"]["best_blocks"], 6); + assert_eq!(json["totals"]["finalized_blocks"], 1); + assert_eq!(json["totals"]["latest_slot"], 102); + assert_eq!(json["totals"]["active_nodes"], 2); } +/// Live counters keep the events of a node that has since disconnected, while +/// active_nodes counts only the nodes connected now. #[tokio::test] async fn test_live_counters() { - let (server, _, _, _) = setup_test_api().await; - let response = server.get("/api/metrics/live").await; - assert_eq!(response.status_code(), StatusCode::OK); - let json: Value = response.json(); - assert!(json.is_object()); + let (server, telemetry_server, telemetry_port, metrics_tracker) = setup_test_api().await; + let mut staying = common::connect_test_node(telemetry_port, 1, &telemetry_server).await; + let mut leaving = common::connect_test_node(telemetry_port, 2, &telemetry_server).await; + + let slot_200_at = common::now_jce_micros() - 4 * SLOT_MICROS; + common::send_events(&mut staying, &best_blocks(200, slot_200_at, 4)).await; + common::send_events( + &mut leaving, + &[ + status_with_peers(slot_200_at, 12, 9, 3), + common::best_block_event_with_hash( + slot_200_at + 300 * MICROS_PER_MILLI, + 200, + common::block_hash(200), + ), + ], + ) + .await; + drop(leaving); + telemetry_server.wait_for_connections(1).await; + metrics_tracker.wait_for_events(6).await; + wait_for_current_second_to_end().await; + + let json = get_json(&server, "/api/metrics/live").await; + assert_eq!(json["last_10s"]["events"], 6); + assert_eq!(json["last_10s"]["blocks"], 5); + assert_eq!(json["last_1m"]["events"], 6); + assert_eq!(json["last_1m"]["blocks"], 5); + assert_eq!(json["latest_slot"], 203); + assert_eq!(json["active_nodes"], 1); } // --- Other endpoints --- +/// Network parameters come from the connected nodes' NodeInformation, and the +/// implementation list names only nodes that are still connected. #[tokio::test] async fn test_network_info() { - let (server, _, _, _) = setup_test_api().await; - let response = server.get("/api/network").await; - assert_eq!(response.status_code(), StatusCode::OK); - let json: Value = response.json(); - assert!(json.is_object()); + let (server, telemetry_server, telemetry_port, _) = setup_test_api().await; + let _first = common::connect_test_node(telemetry_port, 1, &telemetry_server).await; + let _second = common::connect_test_node(telemetry_port, 2, &telemetry_server).await; + let gone = common::connect_test_node(telemetry_port, 3, &telemetry_server).await; + drop(gone); + telemetry_server.wait_for_connections(2).await; + common::flush_and_wait(&telemetry_server).await; + + let json = get_json(&server, "/api/network").await; + assert_eq!(json["network"]["connected_nodes"], 2); + assert_eq!(json["network"]["total_nodes"], 3); + assert_eq!(json["topology"]["core_count"], 16); + assert_eq!(json["topology"]["validator_count"], 48); + assert_eq!(json["topology"]["epoch_period"], 600); + assert_eq!( + json["implementations"], + json!([ + {"name": "test-node-1", "version": "1.0.0"}, + {"name": "test-node-2", "version": "1.0.0"}, + ]) + ); } #[tokio::test] @@ -220,76 +347,302 @@ async fn test_detailed_health() { assert!(json.get("uptime_seconds").is_some()); } +/// A topology link is a peer that announced blocks to a node more than once; +/// a peer heard from only once is not a link, but its announcement still counts. #[tokio::test] async fn test_peer_topology() { - let (server, _, _, _) = setup_test_api().await; - let response = server.get("/api/network/topology").await; - assert_eq!(response.status_code(), StatusCode::OK); - let json: Value = response.json(); - assert!(json.is_object()); + let (server, telemetry_server, telemetry_port, _) = setup_test_api().await; + let mut node = common::connect_test_node(telemetry_port, 1, &telemetry_server).await; + let regular_peer = [2u8; 32]; + let occasional_peer = [3u8; 32]; + + let slot_100_at = common::now_jce_micros() - 4 * SLOT_MICROS; + let announced_at = |slot: u32| slot_100_at + u64::from(slot - 100) * SLOT_MICROS; + common::send_events( + &mut node, + &[ + common::block_announced_event_from( + announced_at(100), + regular_peer, + 100, + common::block_hash(100), + ), + common::block_announced_event_from( + announced_at(101), + regular_peer, + 101, + common::block_hash(101), + ), + common::block_announced_event_from( + announced_at(102), + regular_peer, + 102, + common::block_hash(102), + ), + common::block_announced_event_from( + announced_at(102) + 120 * MICROS_PER_MILLI, + occasional_peer, + 102, + common::block_hash(102), + ), + ], + ) + .await; + common::flush_all(&telemetry_server).await; + + let json = get_json(&server, "/api/network/topology").await; + let connections = json["connections"].as_array().unwrap(); + assert_eq!(connections.len(), 1, "{connections:?}"); + assert_eq!(connections[0]["from_node"], common::node_id_hex(1)); + assert_eq!(connections[0]["to_node"], hex::encode(regular_peer)); + assert_eq!(connections[0]["message_count"], 3); + assert_eq!(json["network_stats"]["total_connections"], 1); + assert_eq!(json["network_stats"]["block_announcements_last_hour"], 4); + assert_eq!(json["network_stats"]["active_nodes"], 1); } +/// A block a node authored belongs to the slot of the Authoring event its +/// Authored event refers to. #[tokio::test] +#[ignore = "bug: /api/slots links Authored to Authoring by created_at, not authoring_id"] async fn test_slot_events() { - let (server, _, _, _) = setup_test_api().await; - let response = server.get("/api/slots/1").await; - assert_eq!(response.status_code(), StatusCode::OK); - let json: Value = response.json(); - assert!(json.is_object() || json.is_array()); + let (server, telemetry_server, telemetry_port, _) = setup_test_api().await; + let mut node = common::connect_test_node(telemetry_port, 1, &telemetry_server).await; + + let authoring_at = common::now_jce_micros() - SLOT_MICROS; + let authoring_id = + common::send_events(&mut node, &[common::authoring_event(authoring_at, 500)]).await[0]; + common::send_events( + &mut node, + &[common::authored_event( + authoring_at + 300 * MICROS_PER_MILLI, + authoring_id, + )], + ) + .await; + common::flush_all(&telemetry_server).await; + + let json = get_json(&server, "/api/slots/500").await; + assert_eq!(json["slot"], 500); + assert_eq!(json["authoring_attempts"], 1); + assert_eq!(json["authoring_failures"], 0); + assert_eq!(json["blocks_authored"], 1); + assert_eq!(json["total_events"], 2); + assert_eq!(json["nodes_involved"], 1); + + let empty = get_json(&server, "/api/slots/501").await; + assert_eq!(empty["slot"], 501); + assert_eq!(empty["total_events"], 0); } // --- Node-specific endpoints (require connected node) --- +/// Chain status reports the node's highest best and finalized slots, the hash +/// of its latest best block, and per-type event counts. #[tokio::test] async fn test_node_status() { let (server, telemetry_server, telemetry_port, _) = setup_test_api().await; - let _stream = common::connect_test_node(telemetry_port, 10, &telemetry_server).await; - let node_id = hex::encode([10u8; 32]); + let mut node = common::connect_test_node(telemetry_port, 10, &telemetry_server).await; + + let slot_776_at = common::now_jce_micros() - 2 * SLOT_MICROS; + let mut events = vec![status_with_peers(slot_776_at - MICROS_PER_MILLI, 12, 9, 3)]; + events.extend(best_blocks(776, slot_776_at, 2)); + events.push(common::finalized_block_event_with_hash( + slot_776_at + SLOT_MICROS + 80 * MICROS_PER_MILLI, + 775, + common::block_hash(775), + )); + common::send_events(&mut node, &events).await; + common::flush_all(&telemetry_server).await; - let response = server.get(&format!("/api/nodes/{}/status", node_id)).await; - assert_eq!(response.status_code(), StatusCode::OK); - let json: Value = response.json(); - assert!(json.is_object()); + let json = get_json( + &server, + &format!("/api/nodes/{}/status", common::node_id_hex(10)), + ) + .await; + let chain = &json["chain_status"]; + assert_eq!(chain["best_slot"], 777); + assert_eq!(chain["finalized_slot"], 775); + assert_eq!(chain["best_hash"], hex::encode(common::block_hash(777))); + assert_eq!( + chain["finalized_hash"], + hex::encode(common::block_hash(775)) + ); + assert_eq!(chain["best_block_events"], 2); + assert_eq!(chain["finalized_block_events"], 1); + assert_eq!(json["node"]["is_connected"], true); + assert_eq!(json["node"]["event_count"], 4); + + let mut breakdown: Vec<(u64, u64)> = json["event_breakdown"] + .as_array() + .unwrap() + .iter() + .map(|entry| { + ( + entry["event_type"].as_u64().unwrap(), + entry["count"].as_u64().unwrap(), + ) + }) + .collect(); + breakdown.sort(); + assert_eq!(breakdown, [(10, 1), (11, 2), (12, 1)]); +} + +/// Sends a work package's journey through a guarantor: submitted by a builder, +/// received for `core`, then either guaranteed or failed during authorization. +async fn send_work_package( + node: &mut common::TestNode, + submitted_at: u64, + core: u16, + guaranteed: bool, +) { + let builder = [0x50; 32]; + let submission_id = + common::send_events(node, &[common::wp_submission_event(submitted_at, builder)]).await[0]; + let step = |millis: u64| submitted_at + millis * MICROS_PER_MILLI; + let mut events = vec![common::wp_received_event(step(20), submission_id, core)]; + if guaranteed { + events.extend([ + common::authorized_event(step(60), submission_id), + common::refined_event(step(900), submission_id), + common::work_report_built_event(step(950), submission_id), + common::guarantee_built_event(step(1000), submission_id), + ]); + } else { + events.push(common::wp_failed_event_with_reason( + step(60), + submission_id, + "authorization failed", + )); + } + common::send_events(node, &events).await; } +/// A guarantor's assigned core is the core it received most work packages for; +/// every core it worked on is listed, busiest first. #[tokio::test] async fn test_node_status_enhanced() { let (server, telemetry_server, telemetry_port, _) = setup_test_api().await; - let _stream = common::connect_test_node(telemetry_port, 11, &telemetry_server).await; - let node_id = hex::encode([11u8; 32]); + let mut node = common::connect_test_node(telemetry_port, 11, &telemetry_server).await; - let response = server - .get(&format!("/api/nodes/{}/status/enhanced", node_id)) - .await; - assert_eq!(response.status_code(), StatusCode::OK); - let json: Value = response.json(); - assert!(json.is_object()); + let first_at = common::now_jce_micros() - 60 * SLOT_MICROS; + send_work_package(&mut node, first_at, 5, false).await; + send_work_package(&mut node, first_at + 30 * SLOT_MICROS, 3, true).await; + send_work_package(&mut node, first_at + 31 * SLOT_MICROS, 3, true).await; + common::flush_all(&telemetry_server).await; + + let json = get_json( + &server, + &format!("/api/nodes/{}/status/enhanced", common::node_id_hex(11)), + ) + .await; + assert_eq!(json["assigned_core"], 3); + assert_eq!(json["cores_active"], json!([3, 5])); + assert_eq!(json["guarantee_activity"]["guarantees_built"], 2); + assert_eq!(json["guarantee_activity"]["guarantees_signed"], 2); + assert_eq!(json["guarantee_activity"]["primary_core"], 3); + assert_eq!(json["ticket_activity"], Value::Null); + assert_eq!(json["is_connected"], true); } +/// Peer counts come from the node's latest Status event; non-validator peers +/// are all peers minus validator peers; history lists every Status, newest first. #[tokio::test] async fn test_node_peers() { let (server, telemetry_server, telemetry_port, _) = setup_test_api().await; - let _stream = common::connect_test_node(telemetry_port, 12, &telemetry_server).await; - let node_id = hex::encode([12u8; 32]); + let mut node = common::connect_test_node(telemetry_port, 12, &telemetry_server).await; - let response = server.get(&format!("/api/nodes/{}/peers", node_id)).await; - assert_eq!(response.status_code(), StatusCode::OK); - let json: Value = response.json(); - assert!(json.is_object()); + let earlier_at = common::now_jce_micros() - SLOT_MICROS; + common::send_events( + &mut node, + &[ + status_with_peers(earlier_at, 28, 19, 6), + status_with_peers(earlier_at + 2_000 * MICROS_PER_MILLI, 30, 20, 4), + ], + ) + .await; + common::flush_all(&telemetry_server).await; + + let json = get_json( + &server, + &format!("/api/nodes/{}/peers", common::node_id_hex(12)), + ) + .await; + assert_eq!(json["total_peers"], 30); + assert_eq!(json["validator_peers"], 20); + assert_eq!(json["non_validator_peers"], 10); + assert_eq!(json["sync_peers"], 4); + assert_eq!(json["block_announcement_peers"], 0); + let history: Vec<(u64, u64, u64)> = json["history"] + .as_array() + .unwrap() + .iter() + .map(|entry| { + ( + entry["total_peers"].as_u64().unwrap(), + entry["validator_peers"].as_u64().unwrap(), + entry["sync_peers"].as_u64().unwrap(), + ) + }) + .collect(); + assert_eq!(history, [(30, 20, 4), (28, 19, 6)]); } +fn event_ids(timeline: &Value) -> Vec { + timeline["events"] + .as_array() + .unwrap() + .iter() + .map(|event| event["event_id"].as_u64().unwrap()) + .collect() +} + +/// The timeline lists a node's events newest first, counts them per category, +/// and a category filter keeps only that category's events. #[tokio::test] async fn test_node_timeline() { let (server, telemetry_server, telemetry_port, _) = setup_test_api().await; - let _stream = common::connect_test_node(telemetry_port, 13, &telemetry_server).await; - let node_id = hex::encode([13u8; 32]); + let mut node = common::connect_test_node(telemetry_port, 13, &telemetry_server).await; - let response = server - .get(&format!("/api/nodes/{}/timeline", node_id)) - .await; - assert_eq!(response.status_code(), StatusCode::OK); - let json: Value = response.json(); - assert!(json.is_object() || json.is_array()); + let slot_900_at = common::now_jce_micros() - 2 * SLOT_MICROS; + let step = |millis: u64| slot_900_at + millis * MICROS_PER_MILLI; + common::send_events( + &mut node, + &[ + common::best_block_event_with_hash(step(0), 900, common::block_hash(900)), + status_with_peers(step(1_500), 12, 9, 3), + ], + ) + .await; + let submission_id = common::send_events( + &mut node, + &[common::wp_submission_event(step(2_000), [0x50; 32])], + ) + .await[0]; + common::send_events( + &mut node, + &[ + common::wp_received_event(step(2_020), submission_id, 2), + common::best_block_event_with_hash(step(6_000), 901, common::block_hash(901)), + ], + ) + .await; + common::flush_all(&telemetry_server).await; + + let timeline_path = format!("/api/nodes/{}/timeline", common::node_id_hex(13)); + let json = get_json(&server, &timeline_path).await; + assert_eq!(event_ids(&json), [4, 3, 2, 1, 0]); + assert_eq!(json["event_count"], 5); + assert_eq!( + json["category_counts"], + json!([ + {"category": "status", "count": 3}, + {"category": "workPackage", "count": 2}, + ]) + ); + + let work_packages = get_json(&server, &format!("{timeline_path}?categories=workPackage")).await; + assert_eq!(event_ids(&work_packages), [3, 2]); } // --- JAM RPC endpoints (no RPC configured → 503) --- @@ -314,3 +667,30 @@ async fn test_jam_cores_no_rpc() { let response = server.get("/api/jam/cores").await; assert_eq!(response.status_code(), StatusCode::SERVICE_UNAVAILABLE); } + +/// A node's event count survives a disconnect and keeps growing after it +/// reconnects, counting each event once. +#[tokio::test] +#[ignore = "bug: node event_count counted twice after disconnect"] +async fn test_node_event_count_reconnect_cycle() { + let (server, telemetry_server, telemetry_port, _) = setup_test_api().await; + let node_path = format!("/api/nodes/{}", common::node_id_hex(6)); + + let slot_1_at = common::now_jce_micros() - 10 * SLOT_MICROS; + let mut node = common::connect_test_node(telemetry_port, 6, &telemetry_server).await; + common::send_events(&mut node, &best_blocks(1, slot_1_at, 4)).await; + common::flush_all(&telemetry_server).await; + assert_eq!(get_json(&server, &node_path).await["event_count"], 4); + + drop(node); + telemetry_server.wait_for_connections(0).await; + common::flush_and_wait(&telemetry_server).await; + let disconnected = get_json(&server, &node_path).await; + assert_eq!(disconnected["is_connected"], false); + assert_eq!(disconnected["event_count"], 4); + + let mut node = common::connect_test_node(telemetry_port, 6, &telemetry_server).await; + common::send_events(&mut node, &best_blocks(5, slot_1_at + 4 * SLOT_MICROS, 2)).await; + common::flush_all(&telemetry_server).await; + assert_eq!(get_json(&server, &node_path).await["event_count"], 6); +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 975b2a6..5253b3e 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -291,6 +291,34 @@ pub fn best_block_event(ts: u64, slot: u32) -> Event { } } +/// A block hash distinct for every slot, for events that name a block. +#[allow(dead_code)] +pub fn block_hash(slot: u32) -> [u8; 32] { + let mut hash = [0xB0; 32]; + hash[..4].copy_from_slice(&slot.to_le_bytes()); + hash +} + +/// Construct a BestBlockChanged event (event_type=11) for the block with the given hash. +#[allow(dead_code)] +pub fn best_block_event_with_hash(ts: u64, slot: u32, hash: [u8; 32]) -> Event { + Event::BestBlockChanged { + timestamp: ts, + slot, + hash, + } +} + +/// Construct a FinalizedBlockChanged event (event_type=12) for the block with the given hash. +#[allow(dead_code)] +pub fn finalized_block_event_with_hash(ts: u64, slot: u32, hash: [u8; 32]) -> Event { + Event::FinalizedBlockChanged { + timestamp: ts, + slot, + hash, + } +} + /// Construct a FinalizedBlockChanged event (event_type=12). #[allow(dead_code)] pub fn finalized_block_event(ts: u64, slot: u32) -> Event { @@ -351,6 +379,16 @@ pub fn importing_event(ts: u64, slot: u32, block_hash: [u8; 32]) -> Event { } } +/// Construct a WorkPackageSubmission event (event_type=90): `builder` submitted a work package. +#[allow(dead_code)] +pub fn wp_submission_event(ts: u64, builder: [u8; 32]) -> Event { + Event::WorkPackageSubmission { + timestamp: ts, + builder, + bundle: false, + } +} + /// Construct a WorkPackageReceived event (event_type=94). #[allow(dead_code)] pub fn wp_received_event(ts: u64, submission_id: u64, core: u16) -> Event { @@ -535,6 +573,18 @@ pub fn block_announced_event(ts: u64, slot: u32) -> Event { } } +/// Construct a BlockAnnounced event (event_type=62): `peer` announced the block to this node. +#[allow(dead_code)] +pub fn block_announced_event_from(ts: u64, peer: [u8; 32], slot: u32, hash: [u8; 32]) -> Event { + Event::BlockAnnounced { + timestamp: ts, + peer, + announcer: ConnectionSide::Remote, + slot, + hash, + } +} + /// Construct an AssuranceSent event (event_type=128). #[allow(dead_code)] pub fn assurance_sent_event(ts: u64) -> Event { diff --git a/tests/ws_tests.rs b/tests/ws_tests.rs index 330d96a..0c19922 100644 --- a/tests/ws_tests.rs +++ b/tests/ws_tests.rs @@ -5,10 +5,109 @@ use serde_json::Value; use std::sync::Arc; use std::time::Duration; use tart_backend::api::{create_api_router, ApiState}; +use tart_backend::events::Event; use tart_backend::{EventStore, TelemetryServer}; -use tokio::time::sleep; use tokio_tungstenite::connect_async; use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream}; + +type WsStream = WebSocketStream>; + +const SLOT_MICROS: u64 = 6_000_000; +const MICROS_PER_MILLI: u64 = 1_000; + +/// Read the next text frame as JSON. +async fn next_json(ws: &mut WsStream) -> Value { + let msg = tokio::time::timeout(Duration::from_secs(8), ws.next()) + .await + .expect("Timeout waiting for WS frame") + .expect("Stream ended") + .expect("WS error"); + match msg { + Message::Text(text) => serde_json::from_str(&text).unwrap(), + other => panic!("Expected text frame, got {:?}", other), + } +} + +/// Read frames until one satisfies `predicate`, skipping the others. +async fn read_until(ws: &mut WsStream, predicate: impl Fn(&Value) -> bool) -> Value { + for _ in 0..50 { + let json = next_json(ws).await; + if predicate(&json) { + return json; + } + } + panic!("No matching frame received"); +} + +/// The per-event envelopes (`id`, `node_id`, `event_type`, `event`) carried by +/// an `event` or `batch` frame; none for any other frame. +fn event_envelopes(frame: &Value) -> Vec { + match frame["type"].as_str() { + Some("event") => vec![frame["data"].clone()], + Some("batch") => frame["data"] + .as_array() + .unwrap() + .iter() + .map(|event| event["data"].clone()) + .collect(), + _ => Vec::new(), + } +} + +/// Collect every event envelope delivered until one matches `is_marker`, +/// so a test sees everything that arrived before its marker event. +async fn read_events_until(ws: &mut WsStream, is_marker: impl Fn(&Value) -> bool) -> Vec { + let mut envelopes = Vec::new(); + for _ in 0..50 { + let frame = next_json(ws).await; + for envelope in event_envelopes(&frame) { + let found = is_marker(&envelope); + envelopes.push(envelope); + if found { + return envelopes; + } + } + } + panic!("Marker event never arrived, got {envelopes:?}"); +} + +async fn open_ws(server_url: &str) -> WsStream { + let ws_url = server_url.replace("http://", "ws://") + "/api/ws"; + let (mut ws, _) = connect_async(&ws_url).await.expect("Failed to connect"); + assert_eq!(next_json(&mut ws).await["type"], "connected"); + ws +} + +async fn subscribe(ws: &mut WsStream, filter: Value) { + let request = serde_json::json!({"type": "Subscribe", "filter": filter}); + ws.send(Message::Text(request.to_string())).await.unwrap(); + read_until(ws, |json| json["type"] == "subscribed").await; +} + +fn best_block(ts: u64, slot: u32) -> Event { + common::best_block_event_with_hash(ts, slot, common::block_hash(slot)) +} + +fn is_best_block(slot: u32) -> impl Fn(&Value) -> bool { + move |envelope| { + envelope["event_type"] == 11 && envelope["event"]["BestBlockChanged"]["slot"] == slot + } +} + +fn event_types(envelopes: &[Value]) -> Vec { + envelopes + .iter() + .map(|envelope| envelope["event_type"].as_u64().unwrap()) + .collect() +} + +fn node_ids(envelopes: &[Value]) -> Vec<&str> { + envelopes + .iter() + .map(|envelope| envelope["node_id"].as_str().unwrap()) + .collect() +} /// Starts a real HTTP server (not axum-test) so WebSocket upgrades work. /// Returns (server_url, telemetry_server, telemetry_port). @@ -60,8 +159,6 @@ async fn setup_ws_test() -> (String, Arc, u16) { axum::serve(listener, app).await.unwrap(); }); - sleep(Duration::from_millis(100)).await; - let server_url = format!("http://127.0.0.1:{}", addr.port()); (server_url, telemetry_server, telemetry_port) } @@ -127,39 +224,31 @@ async fn test_ws_ping_pong() { ws.close(None).await.ok(); } +/// A subscriber to all events receives each event as it arrives, in order, +/// with the payload the node sent. #[tokio::test] async fn test_ws_subscribe_all() { - let (server_url, _, _) = setup_ws_test().await; - let ws_url = server_url.replace("http://", "ws://") + "/api/ws"; - - let (mut ws, _) = connect_async(&ws_url).await.expect("Failed to connect"); - - // Consume the initial "connected" message - let _ = tokio::time::timeout(Duration::from_secs(5), ws.next()).await; - - // Send Subscribe{All} - let sub_msg = serde_json::json!({"type": "Subscribe", "filter": {"type": "All"}}); - ws.send(Message::Text(sub_msg.to_string())).await.unwrap(); - - // Read messages until we find the "subscribed" confirmation - // (stats updates may arrive first due to async timing) - let mut found_subscribed = false; - for _ in 0..5 { - let msg = tokio::time::timeout(Duration::from_secs(6), ws.next()) - .await - .expect("Timeout waiting for subscribed") - .expect("Stream ended") - .expect("WS error"); - - if let Message::Text(text) = msg { - let json: Value = serde_json::from_str(&text).unwrap(); - if json["type"] == "subscribed" { - found_subscribed = true; - break; - } - } - } - assert!(found_subscribed, "Never received 'subscribed' confirmation"); + let (server_url, telemetry_server, telemetry_port) = setup_ws_test().await; + let mut ws = open_ws(&server_url).await; + let mut node = common::connect_test_node(telemetry_port, 1, &telemetry_server).await; + subscribe(&mut ws, serde_json::json!({"type": "All"})).await; + + let slot_1_at = common::now_jce_micros() - SLOT_MICROS; + let sent = [ + best_block(slot_1_at, 1), + best_block(slot_1_at + SLOT_MICROS, 2), + ]; + common::send_events(&mut node, &sent).await; + + let received = read_events_until(&mut ws, is_best_block(2)).await; + let node_id = common::node_id_hex(1); + assert_eq!(node_ids(&received), [node_id.as_str(), node_id.as_str()]); + let payloads: Vec<&Value> = received.iter().map(|envelope| &envelope["event"]).collect(); + let expected: Vec = sent + .iter() + .map(|event| serde_json::to_value(event).unwrap()) + .collect(); + assert_eq!(payloads, expected.iter().collect::>()); ws.close(None).await.ok(); } @@ -228,3 +317,155 @@ async fn test_ws_stats_updates() { ws.close(None).await.ok(); } + +/// An event-type subscription delivers only events of that type. +#[tokio::test] +#[ignore = "bug: WebSocket EventType filter delivers every event"] +async fn test_ws_subscribe_eventtype_filters() { + let (server_url, telemetry_server, telemetry_port) = setup_ws_test().await; + let mut ws = open_ws(&server_url).await; + let mut node = common::connect_test_node(telemetry_port, 1, &telemetry_server).await; + subscribe( + &mut ws, + serde_json::json!({"type": "EventType", "event_type": 11}), + ) + .await; + + let status_at = common::now_jce_micros() - SLOT_MICROS; + common::send_events( + &mut node, + &[ + common::status_event(status_at), + best_block(status_at + 500 * MICROS_PER_MILLI, 1), + ], + ) + .await; + + let received = read_events_until(&mut ws, is_best_block(1)).await; + assert_eq!(event_types(&received), [11]); + ws.close(None).await.ok(); +} + +/// An event-type range subscription delivers only events whose type is in the range. +#[tokio::test] +#[ignore = "bug: WebSocket EventTypeRange filter delivers every event"] +async fn test_ws_subscribe_eventtype_range_filters() { + let (server_url, telemetry_server, telemetry_port) = setup_ws_test().await; + let mut ws = open_ws(&server_url).await; + let mut node = common::connect_test_node(telemetry_port, 1, &telemetry_server).await; + subscribe( + &mut ws, + serde_json::json!({"type": "EventTypeRange", "start": 90, "end": 113}), + ) + .await; + + let slot_1_at = common::now_jce_micros() - SLOT_MICROS; + common::send_events( + &mut node, + &[ + best_block(slot_1_at, 1), + common::status_event(slot_1_at + 900 * MICROS_PER_MILLI), + common::wp_submission_event(slot_1_at + 1_500 * MICROS_PER_MILLI, [0x50; 32]), + ], + ) + .await; + + let received = read_events_until(&mut ws, |envelope| envelope["event_type"] == 90).await; + assert_eq!(event_types(&received), [90]); + ws.close(None).await.ok(); +} + +/// The metrics channel reports the last 10 seconds of events by event time: +/// older blocks fall out of the window, healthy work-package events are not +/// failures, and the latest slot is the highest best block of the last minute. +#[tokio::test] +async fn test_ws_metrics_payload_values() { + let (server_url, telemetry_server, telemetry_port) = setup_ws_test().await; + let mut ws = open_ws(&server_url).await; + let mut author = common::connect_test_node(telemetry_port, 1, &telemetry_server).await; + let mut importer = common::connect_test_node(telemetry_port, 2, &telemetry_server).await; + + let slot_102_at = common::now_jce_micros() - 1_000 * MICROS_PER_MILLI; + common::send_events( + &mut author, + &[ + best_block(slot_102_at - 2 * SLOT_MICROS, 100), + best_block(slot_102_at - SLOT_MICROS, 101), + best_block(slot_102_at, 102), + ], + ) + .await; + common::send_events( + &mut importer, + &[best_block(slot_102_at + 300 * MICROS_PER_MILLI, 102)], + ) + .await; + let submitted_at = slot_102_at - 2_000 * MICROS_PER_MILLI; + let submission_id = common::send_events( + &mut author, + &[common::wp_submission_event(submitted_at, [0x50; 32])], + ) + .await[0]; + common::send_events( + &mut author, + &[common::wp_received_event( + submitted_at + 20 * MICROS_PER_MILLI, + submission_id, + 3, + )], + ) + .await; + common::flush_and_wait(&telemetry_server).await; + + let request = serde_json::json!({"type": "SubscribeMetrics", "interval_ms": 500}); + ws.send(Message::Text(request.to_string())).await.unwrap(); + let metrics = read_until(&mut ws, |json| json["type"] == "metrics").await; + let data = &metrics["data"]; + assert_eq!(data["throughput"]["events_10s"], 5); + assert_eq!(data["throughput"]["blocks_10s"], 3); + assert_eq!(data["active_nodes"], 2); + assert_eq!(data["active_work_packages"], 2); + assert_eq!(data["failure_rate"], 0.0); + assert_eq!(data["latest_slot"], 102); + ws.close(None).await.ok(); +} + +/// A subscriber to one node receives none of the events other nodes send. +#[tokio::test] +async fn test_ws_nodes_filter_only_receives_subscribed_node() { + let (server_url, telemetry_server, telemetry_port) = setup_ws_test().await; + let mut ws = open_ws(&server_url).await; + let mut watched = common::connect_test_node(telemetry_port, 1, &telemetry_server).await; + let mut other = common::connect_test_node(telemetry_port, 2, &telemetry_server).await; + let watched_id = common::node_id_hex(1); + + // Both nodes need a per-node channel, made on a node's first event, so a + // filter that wrongly includes the other node would have it to subscribe to. + let slot_1_at = common::now_jce_micros() - 2 * SLOT_MICROS; + common::send_events(&mut watched, &[best_block(slot_1_at, 1)]).await; + common::send_events( + &mut other, + &[best_block(slot_1_at + 200 * MICROS_PER_MILLI, 1)], + ) + .await; + subscribe( + &mut ws, + serde_json::json!({"type": "Nodes", "node_ids": [watched_id]}), + ) + .await; + + let slot_2_at = slot_1_at + SLOT_MICROS; + common::send_events(&mut other, &[best_block(slot_2_at, 2)]).await; + common::send_events( + &mut watched, + &[best_block(slot_2_at + 250 * MICROS_PER_MILLI, 2)], + ) + .await; + + let received = read_events_until(&mut ws, |envelope| { + envelope["node_id"] == watched_id.as_str() && is_best_block(2)(envelope) + }) + .await; + assert_eq!(node_ids(&received), [watched_id.as_str()]); + ws.close(None).await.ok(); +}