diff --git a/src/server.rs b/src/server.rs index 6b63c7b..434888b 100644 --- a/src/server.rs +++ b/src/server.rs @@ -105,6 +105,8 @@ pub struct NodeConnection { pub connected_at: chrono::DateTime, pub last_event_at: chrono::DateTime, pub event_count: u64, + /// Events handed to the batch writer so far; lets tests wait for ingestion. + pub events_received: Arc>, } /// High-performance TCP telemetry server for JAM/Polkadot blockchain nodes. @@ -675,6 +677,20 @@ impl TelemetryServer { rx.changed().await.expect("connection watch closed"); } } + + /// Wait until the server has handed at least `expected` events from + /// `node_id` to the batch writer. Panics if the node is not connected. + pub async fn wait_for_events(&self, node_id: &str, expected: u64) { + let events_received = self + .connections + .get(node_id) + .map(|connection| Arc::clone(&connection.events_received)) + .unwrap_or_else(|| panic!("node {node_id} is not connected")); + let _ = events_received + .subscribe() + .wait_for(|count| *count >= expected) + .await; + } } /// Shared server state passed to each connection handler. @@ -781,6 +797,8 @@ async fn handle_connection_optimized( node_info.additional_info.as_str().unwrap_or("") ); + let events_received = Arc::new(tokio::sync::watch::Sender::new(0u64)); + // Store connection info (DashMap key is String; one-time alloc per connection) let connection = NodeConnection { id: node_id_str.to_string(), @@ -789,9 +807,9 @@ async fn handle_connection_optimized( connected_at: chrono::Utc::now(), last_event_at: chrono::Utc::now(), event_count: 0, + events_received: Arc::clone(&events_received), }; connections.insert(node_id_str.to_string(), connection); - let _ = connection_watch.send(connections.len()); // Queue node connection event info!("Queueing node connection for {}", node_id_str); @@ -805,6 +823,8 @@ async fn handle_connection_optimized( return Err(e.into()); } } + // Announced only after queueing, so a flush after wait_for_connections includes it. + let _ = connection_watch.send(connections.len()); // Track dropped events let mut dropped_events = 0u64; @@ -2061,6 +2081,10 @@ async fn handle_connection_optimized( } } + if batch_received > 0 { + events_received.send_modify(|count| *count += batch_received); + } + // Read more data from stream let n = stream.read_buf(&mut buffer).await?; if n == 0 { @@ -2084,8 +2108,10 @@ async fn handle_connection_optimized( // Clean up connections.remove(&*node_id_str); broadcaster.remove_node_channel(&node_id_str).await; + // As on connect, announce only after queueing the change. + let disconnected = batch_writer.node_disconnected(node_id_str).await; let _ = connection_watch.send(connections.len()); - batch_writer.node_disconnected(node_id_str).await?; + disconnected?; Ok(()) } diff --git a/tests/api_tests.rs b/tests/api_tests.rs index cf89c7f..4f126de 100644 --- a/tests/api_tests.rs +++ b/tests/api_tests.rs @@ -5,6 +5,7 @@ use axum_test::TestServer; use serde_json::Value; use std::sync::Arc; use tart_backend::api::{create_api_router, ApiState}; +use tart_backend::events::Event; use tart_backend::{EventStore, TelemetryServer}; async fn setup_test_api() -> (TestServer, Arc, u16, Arc) { @@ -99,6 +100,29 @@ async fn test_node_details_endpoint() { assert!(conn_info["connected_duration_secs"].is_number()); } +/// A burst of events reaches the server over many socket reads and writer +/// batches; once flushed, the node's event count must include every one. +#[tokio::test] +async fn test_node_event_count_after_burst() { + let (server, telemetry_server, telemetry_port, _) = setup_test_api().await; + let mut node = common::connect_test_node(telemetry_port, 4, &telemetry_server).await; + + const SLOT_MICROS: u64 = 6_000_000; + let first_slot_at = common::now_jce_micros() - 1000 * SLOT_MICROS; + let events: Vec = (0..1000u32) + .map(|slot| common::best_block_event(first_slot_at + u64::from(slot) * SLOT_MICROS, slot)) + .collect(); + common::send_events(&mut node, &events).await; + common::flush_all(&telemetry_server).await; + + let response = server + .get(&format!("/api/nodes/{}", common::node_id_hex(4))) + .await; + assert_eq!(response.status_code(), StatusCode::OK); + let json: Value = response.json(); + assert_eq!(json["event_count"], 1000); +} + #[tokio::test] async fn test_node_details_not_found() { let (server, _, _, _) = setup_test_api().await; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 5b4f2a1..fdd9959 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -7,7 +7,6 @@ use tart_backend::types::*; use tart_backend::TelemetryServer; use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; -use tokio::time::sleep; pub mod mock_jam_rpc; @@ -44,13 +43,21 @@ pub fn test_database_url() -> String { /// Flush all pending batch writes and wait for database visibility. #[allow(dead_code)] pub async fn flush_and_wait(server: &Arc) { - sleep(Duration::from_millis(100)).await; server.flush_writes().await.expect("Flush failed"); } +/// A fake node connected to the telemetry server. +#[allow(dead_code)] +pub struct TestNode { + stream: TcpStream, + server: Arc, + node_id: String, + events_sent: u64, +} + /// Connect a fake node with the given id and wait until the server sees it. #[allow(dead_code)] -pub async fn connect_test_node(port: u16, node_id: u8, server: &Arc) -> TcpStream { +pub async fn connect_test_node(port: u16, node_id: u8, server: &Arc) -> TestNode { let expected = server.connection_count() + 1; let mut stream = TcpStream::connect(format!("127.0.0.1:{}", port)) @@ -66,16 +73,33 @@ pub async fn connect_test_node(port: u16, node_id: u8, server: &Arc) { - sleep(Duration::from_millis(100)).await; server.flush_writes().await.expect("Flush writes failed"); server.flush_trackers_for_test().await; } diff --git a/tests/grafana_tests.rs b/tests/grafana_tests.rs index 6fc354c..5d2260c 100644 --- a/tests/grafana_tests.rs +++ b/tests/grafana_tests.rs @@ -2,15 +2,13 @@ mod common; use axum::http::StatusCode; use axum_test::TestServer; +use common::{connect_test_node, send_events}; use serde_json::Value; use std::sync::Arc; use tart_backend::api::{create_api_router, ApiState}; -use tart_backend::encoding::encode_message; use tart_backend::events::Event; use tart_backend::types::*; use tart_backend::{EventStore, TelemetryServer}; -use tokio::io::AsyncWriteExt; -use tokio::net::TcpStream; async fn setup_test_api() -> (TestServer, Arc, u16, Arc) { let database_url = common::test_database_url(); @@ -59,32 +57,6 @@ async fn setup_test_api() -> (TestServer, Arc, u16, Arc) -> TcpStream { - let expected = server.connection_count() + 1; - - let mut stream = TcpStream::connect(format!("127.0.0.1:{}", port)) - .await - .unwrap(); - - let mut node_info = common::test_node_info([node_id; 32]); - node_info.implementation_name = BoundedString::new(&format!("test-node-{}", node_id)).unwrap(); - - let encoded = encode_message(&node_info).unwrap(); - stream.write_all(&encoded).await.unwrap(); - - server.wait_for_connections(expected).await; - common::flush_and_wait(server).await; - - stream -} - -async fn send_events(stream: &mut TcpStream, events: &[Event]) { - for event in events { - let encoded = encode_message(event).unwrap(); - stream.write_all(&encoded).await.unwrap(); - } -} - fn time_range_params() -> String { let now = chrono::Utc::now(); let start = now - chrono::Duration::hours(1); @@ -1135,7 +1107,8 @@ async fn test_grafana_node_disconnect() { // Drop node 2's stream drop(stream2); - tokio::time::sleep(std::time::Duration::from_millis(300)).await; + telemetry.wait_for_connections(1).await; + common::flush_and_wait(&telemetry).await; // Verify node 2 disconnected let response = server.get("/api/grafana/nodes").await; @@ -2584,8 +2557,6 @@ async fn test_grafana_da_stats() { ], ) .await; - // Small delay to ensure ordering - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream_a, &[ @@ -2638,7 +2609,6 @@ async fn test_grafana_shard_latency_histogram() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream_a, &[ @@ -2655,7 +2625,6 @@ async fn test_grafana_shard_latency_histogram() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream_a, &[ @@ -2672,7 +2641,6 @@ async fn test_grafana_shard_latency_histogram() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream_a, &[ @@ -2745,7 +2713,6 @@ async fn test_grafana_shard_latency_with_failure() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream_a, &[ @@ -2762,7 +2729,6 @@ async fn test_grafana_shard_latency_with_failure() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream_a, &[ @@ -4210,7 +4176,6 @@ async fn test_grafana_bundle_latency() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream, &[ @@ -4227,7 +4192,6 @@ async fn test_grafana_bundle_latency() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream, &[ @@ -4248,7 +4212,6 @@ async fn test_grafana_bundle_latency() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream, &[ @@ -4338,7 +4301,6 @@ async fn test_grafana_segment_latency() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream, &[ @@ -4395,7 +4357,6 @@ async fn test_grafana_preimage_latency() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream, &[