From a96d1c92ca5cf39bb006b2d76a79d8b60adc94f1 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:20:33 +0000 Subject: [PATCH] test: wait for the server to receive events instead of sleeping Each connection counts the events it hands to the batch writer; send_events waits for that count, so a following flush covers every event sent. Connection changes are announced only after they are queued. Drops the sleeps from the flush helpers and the grafana tests, and the grafana copies of connect_test_node/send_events. --- src/server.rs | 30 ++++++++++++++++++++++++++-- tests/api_tests.rs | 24 ++++++++++++++++++++++ tests/common/mod.rs | 39 ++++++++++++++++++++++++++++-------- tests/grafana_tests.rs | 45 +++--------------------------------------- 4 files changed, 86 insertions(+), 52 deletions(-) diff --git a/src/server.rs b/src/server.rs index 6b63c7b..434888b 100644 --- a/src/server.rs +++ b/src/server.rs @@ -105,6 +105,8 @@ pub struct NodeConnection { pub connected_at: chrono::DateTime, pub last_event_at: chrono::DateTime, pub event_count: u64, + /// Events handed to the batch writer so far; lets tests wait for ingestion. + pub events_received: Arc>, } /// High-performance TCP telemetry server for JAM/Polkadot blockchain nodes. @@ -675,6 +677,20 @@ impl TelemetryServer { rx.changed().await.expect("connection watch closed"); } } + + /// Wait until the server has handed at least `expected` events from + /// `node_id` to the batch writer. Panics if the node is not connected. + pub async fn wait_for_events(&self, node_id: &str, expected: u64) { + let events_received = self + .connections + .get(node_id) + .map(|connection| Arc::clone(&connection.events_received)) + .unwrap_or_else(|| panic!("node {node_id} is not connected")); + let _ = events_received + .subscribe() + .wait_for(|count| *count >= expected) + .await; + } } /// Shared server state passed to each connection handler. @@ -781,6 +797,8 @@ async fn handle_connection_optimized( node_info.additional_info.as_str().unwrap_or("") ); + let events_received = Arc::new(tokio::sync::watch::Sender::new(0u64)); + // Store connection info (DashMap key is String; one-time alloc per connection) let connection = NodeConnection { id: node_id_str.to_string(), @@ -789,9 +807,9 @@ async fn handle_connection_optimized( connected_at: chrono::Utc::now(), last_event_at: chrono::Utc::now(), event_count: 0, + events_received: Arc::clone(&events_received), }; connections.insert(node_id_str.to_string(), connection); - let _ = connection_watch.send(connections.len()); // Queue node connection event info!("Queueing node connection for {}", node_id_str); @@ -805,6 +823,8 @@ async fn handle_connection_optimized( return Err(e.into()); } } + // Announced only after queueing, so a flush after wait_for_connections includes it. + let _ = connection_watch.send(connections.len()); // Track dropped events let mut dropped_events = 0u64; @@ -2061,6 +2081,10 @@ async fn handle_connection_optimized( } } + if batch_received > 0 { + events_received.send_modify(|count| *count += batch_received); + } + // Read more data from stream let n = stream.read_buf(&mut buffer).await?; if n == 0 { @@ -2084,8 +2108,10 @@ async fn handle_connection_optimized( // Clean up connections.remove(&*node_id_str); broadcaster.remove_node_channel(&node_id_str).await; + // As on connect, announce only after queueing the change. + let disconnected = batch_writer.node_disconnected(node_id_str).await; let _ = connection_watch.send(connections.len()); - batch_writer.node_disconnected(node_id_str).await?; + disconnected?; Ok(()) } diff --git a/tests/api_tests.rs b/tests/api_tests.rs index cf89c7f..4f126de 100644 --- a/tests/api_tests.rs +++ b/tests/api_tests.rs @@ -5,6 +5,7 @@ use axum_test::TestServer; use serde_json::Value; use std::sync::Arc; use tart_backend::api::{create_api_router, ApiState}; +use tart_backend::events::Event; use tart_backend::{EventStore, TelemetryServer}; async fn setup_test_api() -> (TestServer, Arc, u16, Arc) { @@ -99,6 +100,29 @@ async fn test_node_details_endpoint() { assert!(conn_info["connected_duration_secs"].is_number()); } +/// A burst of events reaches the server over many socket reads and writer +/// batches; once flushed, the node's event count must include every one. +#[tokio::test] +async fn test_node_event_count_after_burst() { + let (server, telemetry_server, telemetry_port, _) = setup_test_api().await; + let mut node = common::connect_test_node(telemetry_port, 4, &telemetry_server).await; + + const SLOT_MICROS: u64 = 6_000_000; + let first_slot_at = common::now_jce_micros() - 1000 * SLOT_MICROS; + let events: Vec = (0..1000u32) + .map(|slot| common::best_block_event(first_slot_at + u64::from(slot) * SLOT_MICROS, slot)) + .collect(); + common::send_events(&mut node, &events).await; + common::flush_all(&telemetry_server).await; + + let response = server + .get(&format!("/api/nodes/{}", common::node_id_hex(4))) + .await; + assert_eq!(response.status_code(), StatusCode::OK); + let json: Value = response.json(); + assert_eq!(json["event_count"], 1000); +} + #[tokio::test] async fn test_node_details_not_found() { let (server, _, _, _) = setup_test_api().await; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 5b4f2a1..fdd9959 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -7,7 +7,6 @@ use tart_backend::types::*; use tart_backend::TelemetryServer; use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; -use tokio::time::sleep; pub mod mock_jam_rpc; @@ -44,13 +43,21 @@ pub fn test_database_url() -> String { /// Flush all pending batch writes and wait for database visibility. #[allow(dead_code)] pub async fn flush_and_wait(server: &Arc) { - sleep(Duration::from_millis(100)).await; server.flush_writes().await.expect("Flush failed"); } +/// A fake node connected to the telemetry server. +#[allow(dead_code)] +pub struct TestNode { + stream: TcpStream, + server: Arc, + node_id: String, + events_sent: u64, +} + /// Connect a fake node with the given id and wait until the server sees it. #[allow(dead_code)] -pub async fn connect_test_node(port: u16, node_id: u8, server: &Arc) -> TcpStream { +pub async fn connect_test_node(port: u16, node_id: u8, server: &Arc) -> TestNode { let expected = server.connection_count() + 1; let mut stream = TcpStream::connect(format!("127.0.0.1:{}", port)) @@ -66,16 +73,33 @@ pub async fn connect_test_node(port: u16, node_id: u8, server: &Arc) { - sleep(Duration::from_millis(100)).await; server.flush_writes().await.expect("Flush writes failed"); server.flush_trackers_for_test().await; } diff --git a/tests/grafana_tests.rs b/tests/grafana_tests.rs index 6fc354c..5d2260c 100644 --- a/tests/grafana_tests.rs +++ b/tests/grafana_tests.rs @@ -2,15 +2,13 @@ mod common; use axum::http::StatusCode; use axum_test::TestServer; +use common::{connect_test_node, send_events}; use serde_json::Value; use std::sync::Arc; use tart_backend::api::{create_api_router, ApiState}; -use tart_backend::encoding::encode_message; use tart_backend::events::Event; use tart_backend::types::*; use tart_backend::{EventStore, TelemetryServer}; -use tokio::io::AsyncWriteExt; -use tokio::net::TcpStream; async fn setup_test_api() -> (TestServer, Arc, u16, Arc) { let database_url = common::test_database_url(); @@ -59,32 +57,6 @@ async fn setup_test_api() -> (TestServer, Arc, u16, Arc) -> TcpStream { - let expected = server.connection_count() + 1; - - let mut stream = TcpStream::connect(format!("127.0.0.1:{}", port)) - .await - .unwrap(); - - let mut node_info = common::test_node_info([node_id; 32]); - node_info.implementation_name = BoundedString::new(&format!("test-node-{}", node_id)).unwrap(); - - let encoded = encode_message(&node_info).unwrap(); - stream.write_all(&encoded).await.unwrap(); - - server.wait_for_connections(expected).await; - common::flush_and_wait(server).await; - - stream -} - -async fn send_events(stream: &mut TcpStream, events: &[Event]) { - for event in events { - let encoded = encode_message(event).unwrap(); - stream.write_all(&encoded).await.unwrap(); - } -} - fn time_range_params() -> String { let now = chrono::Utc::now(); let start = now - chrono::Duration::hours(1); @@ -1135,7 +1107,8 @@ async fn test_grafana_node_disconnect() { // Drop node 2's stream drop(stream2); - tokio::time::sleep(std::time::Duration::from_millis(300)).await; + telemetry.wait_for_connections(1).await; + common::flush_and_wait(&telemetry).await; // Verify node 2 disconnected let response = server.get("/api/grafana/nodes").await; @@ -2584,8 +2557,6 @@ async fn test_grafana_da_stats() { ], ) .await; - // Small delay to ensure ordering - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream_a, &[ @@ -2638,7 +2609,6 @@ async fn test_grafana_shard_latency_histogram() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream_a, &[ @@ -2655,7 +2625,6 @@ async fn test_grafana_shard_latency_histogram() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream_a, &[ @@ -2672,7 +2641,6 @@ async fn test_grafana_shard_latency_histogram() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream_a, &[ @@ -2745,7 +2713,6 @@ async fn test_grafana_shard_latency_with_failure() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream_a, &[ @@ -2762,7 +2729,6 @@ async fn test_grafana_shard_latency_with_failure() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream_a, &[ @@ -4210,7 +4176,6 @@ async fn test_grafana_bundle_latency() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream, &[ @@ -4227,7 +4192,6 @@ async fn test_grafana_bundle_latency() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream, &[ @@ -4248,7 +4212,6 @@ async fn test_grafana_bundle_latency() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream, &[ @@ -4338,7 +4301,6 @@ async fn test_grafana_segment_latency() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream, &[ @@ -4395,7 +4357,6 @@ async fn test_grafana_preimage_latency() { ], ) .await; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; send_events( &mut stream, &[