Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 28 additions & 2 deletions src/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,8 @@ pub struct NodeConnection {
pub connected_at: chrono::DateTime<chrono::Utc>,
pub last_event_at: chrono::DateTime<chrono::Utc>,
pub event_count: u64,
/// Events handed to the batch writer so far; lets tests wait for ingestion.
pub events_received: Arc<tokio::sync::watch::Sender<u64>>,
}

/// High-performance TCP telemetry server for JAM/Polkadot blockchain nodes.
Expand Down Expand Up @@ -675,6 +677,20 @@ impl TelemetryServer {
rx.changed().await.expect("connection watch closed");
}
}

/// Wait until the server has handed at least `expected` events from
/// `node_id` to the batch writer. Panics if the node is not connected.
pub async fn wait_for_events(&self, node_id: &str, expected: u64) {
let events_received = self
.connections
.get(node_id)
.map(|connection| Arc::clone(&connection.events_received))
.unwrap_or_else(|| panic!("node {node_id} is not connected"));
let _ = events_received
.subscribe()
.wait_for(|count| *count >= expected)
.await;
}
}

/// Shared server state passed to each connection handler.
Expand Down Expand Up @@ -781,6 +797,8 @@ async fn handle_connection_optimized(
node_info.additional_info.as_str().unwrap_or("")
);

let events_received = Arc::new(tokio::sync::watch::Sender::new(0u64));

// Store connection info (DashMap key is String; one-time alloc per connection)
let connection = NodeConnection {
id: node_id_str.to_string(),
Expand All @@ -789,9 +807,9 @@ async fn handle_connection_optimized(
connected_at: chrono::Utc::now(),
last_event_at: chrono::Utc::now(),
event_count: 0,
events_received: Arc::clone(&events_received),
};
connections.insert(node_id_str.to_string(), connection);
let _ = connection_watch.send(connections.len());

// Queue node connection event
info!("Queueing node connection for {}", node_id_str);
Expand All @@ -805,6 +823,8 @@ async fn handle_connection_optimized(
return Err(e.into());
}
}
// Announced only after queueing, so a flush after wait_for_connections includes it.
let _ = connection_watch.send(connections.len());

// Track dropped events
let mut dropped_events = 0u64;
Expand Down Expand Up @@ -2061,6 +2081,10 @@ async fn handle_connection_optimized(
}
}

if batch_received > 0 {
events_received.send_modify(|count| *count += batch_received);
}

// Read more data from stream
let n = stream.read_buf(&mut buffer).await?;
if n == 0 {
Expand All @@ -2084,8 +2108,10 @@ async fn handle_connection_optimized(
// Clean up
connections.remove(&*node_id_str);
broadcaster.remove_node_channel(&node_id_str).await;
// As on connect, announce only after queueing the change.
let disconnected = batch_writer.node_disconnected(node_id_str).await;
let _ = connection_watch.send(connections.len());
batch_writer.node_disconnected(node_id_str).await?;
disconnected?;

Ok(())
}
Expand Down
24 changes: 24 additions & 0 deletions tests/api_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use axum_test::TestServer;
use serde_json::Value;
use std::sync::Arc;
use tart_backend::api::{create_api_router, ApiState};
use tart_backend::events::Event;
use tart_backend::{EventStore, TelemetryServer};

async fn setup_test_api() -> (TestServer, Arc<TelemetryServer>, u16, Arc<EventStore>) {
Expand Down Expand Up @@ -99,6 +100,29 @@ async fn test_node_details_endpoint() {
assert!(conn_info["connected_duration_secs"].is_number());
}

/// A burst of events reaches the server over many socket reads and writer
/// batches; once flushed, the node's event count must include every one.
#[tokio::test]
async fn test_node_event_count_after_burst() {
let (server, telemetry_server, telemetry_port, _) = setup_test_api().await;
let mut node = common::connect_test_node(telemetry_port, 4, &telemetry_server).await;

const SLOT_MICROS: u64 = 6_000_000;
let first_slot_at = common::now_jce_micros() - 1000 * SLOT_MICROS;
let events: Vec<Event> = (0..1000u32)
.map(|slot| common::best_block_event(first_slot_at + u64::from(slot) * SLOT_MICROS, slot))
.collect();
common::send_events(&mut node, &events).await;
common::flush_all(&telemetry_server).await;

let response = server
.get(&format!("/api/nodes/{}", common::node_id_hex(4)))
.await;
assert_eq!(response.status_code(), StatusCode::OK);
let json: Value = response.json();
assert_eq!(json["event_count"], 1000);
}

#[tokio::test]
async fn test_node_details_not_found() {
let (server, _, _, _) = setup_test_api().await;
Expand Down
39 changes: 31 additions & 8 deletions tests/common/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ use tart_backend::types::*;
use tart_backend::TelemetryServer;
use tokio::io::AsyncWriteExt;
use tokio::net::TcpStream;
use tokio::time::sleep;

pub mod mock_jam_rpc;

Expand Down Expand Up @@ -44,13 +43,21 @@ pub fn test_database_url() -> String {
/// Flush all pending batch writes and wait for database visibility.
#[allow(dead_code)]
pub async fn flush_and_wait(server: &Arc<TelemetryServer>) {
sleep(Duration::from_millis(100)).await;
server.flush_writes().await.expect("Flush failed");
}

/// A fake node connected to the telemetry server.
#[allow(dead_code)]
pub struct TestNode {
stream: TcpStream,
server: Arc<TelemetryServer>,
node_id: String,
events_sent: u64,
}

/// Connect a fake node with the given id and wait until the server sees it.
#[allow(dead_code)]
pub async fn connect_test_node(port: u16, node_id: u8, server: &Arc<TelemetryServer>) -> TcpStream {
pub async fn connect_test_node(port: u16, node_id: u8, server: &Arc<TelemetryServer>) -> TestNode {
let expected = server.connection_count() + 1;

let mut stream = TcpStream::connect(format!("127.0.0.1:{}", port))
Expand All @@ -66,16 +73,33 @@ pub async fn connect_test_node(port: u16, node_id: u8, server: &Arc<TelemetrySer
server.wait_for_connections(expected).await;
flush_and_wait(server).await;

stream
TestNode {
stream,
server: Arc::clone(server),
node_id: node_id_hex(node_id),
events_sent: 0,
}
}

/// Write a batch of encoded events to a connected node stream.
/// Send events from a fake node and wait until the server has received them all,
/// so a following flush covers them.
#[allow(dead_code)]
pub async fn send_events(stream: &mut TcpStream, events: &[Event]) {
pub async fn send_events(node: &mut TestNode, events: &[Event]) {
for event in events {
let encoded = encode_message(event).unwrap();
stream.write_all(&encoded).await.unwrap();
node.stream.write_all(&encoded).await.unwrap();
}
node.events_sent += events.len() as u64;

let received = node.server.wait_for_events(&node.node_id, node.events_sent);
tokio::time::timeout(Duration::from_secs(10), received)
.await
.unwrap_or_else(|_| {
panic!(
"server did not receive {} events from node {}",
node.events_sent, node.node_id
)
});
}

/// Creates a test BlockSummary with reasonable default values
Expand Down Expand Up @@ -223,7 +247,6 @@ pub async fn refresh_aggregates(pool: &sqlx::PgPool) {
/// Flush batch_writer + trackers. Combined helper for grafana tests.
#[allow(dead_code)]
pub async fn flush_all(server: &Arc<TelemetryServer>) {
sleep(Duration::from_millis(100)).await;
server.flush_writes().await.expect("Flush writes failed");
server.flush_trackers_for_test().await;
}
Expand Down
45 changes: 3 additions & 42 deletions tests/grafana_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,13 @@ mod common;

use axum::http::StatusCode;
use axum_test::TestServer;
use common::{connect_test_node, send_events};
use serde_json::Value;
use std::sync::Arc;
use tart_backend::api::{create_api_router, ApiState};
use tart_backend::encoding::encode_message;
use tart_backend::events::Event;
use tart_backend::types::*;
use tart_backend::{EventStore, TelemetryServer};
use tokio::io::AsyncWriteExt;
use tokio::net::TcpStream;

async fn setup_test_api() -> (TestServer, Arc<TelemetryServer>, u16, Arc<EventStore>) {
let database_url = common::test_database_url();
Expand Down Expand Up @@ -59,32 +57,6 @@ async fn setup_test_api() -> (TestServer, Arc<TelemetryServer>, u16, Arc<EventSt
(test_server, telemetry_server, telemetry_port, store)
}

async fn connect_test_node(port: u16, node_id: u8, server: &Arc<TelemetryServer>) -> TcpStream {
let expected = server.connection_count() + 1;

let mut stream = TcpStream::connect(format!("127.0.0.1:{}", port))
.await
.unwrap();

let mut node_info = common::test_node_info([node_id; 32]);
node_info.implementation_name = BoundedString::new(&format!("test-node-{}", node_id)).unwrap();

let encoded = encode_message(&node_info).unwrap();
stream.write_all(&encoded).await.unwrap();

server.wait_for_connections(expected).await;
common::flush_and_wait(server).await;

stream
}

async fn send_events(stream: &mut TcpStream, events: &[Event]) {
for event in events {
let encoded = encode_message(event).unwrap();
stream.write_all(&encoded).await.unwrap();
}
}

fn time_range_params() -> String {
let now = chrono::Utc::now();
let start = now - chrono::Duration::hours(1);
Expand Down Expand Up @@ -1135,7 +1107,8 @@ async fn test_grafana_node_disconnect() {

// Drop node 2's stream
drop(stream2);
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
telemetry.wait_for_connections(1).await;
common::flush_and_wait(&telemetry).await;

// Verify node 2 disconnected
let response = server.get("/api/grafana/nodes").await;
Expand Down Expand Up @@ -2584,8 +2557,6 @@ async fn test_grafana_da_stats() {
],
)
.await;
// Small delay to ensure ordering
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
send_events(
&mut stream_a,
&[
Expand Down Expand Up @@ -2638,7 +2609,6 @@ async fn test_grafana_shard_latency_histogram() {
],
)
.await;
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
send_events(
&mut stream_a,
&[
Expand All @@ -2655,7 +2625,6 @@ async fn test_grafana_shard_latency_histogram() {
],
)
.await;
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
send_events(
&mut stream_a,
&[
Expand All @@ -2672,7 +2641,6 @@ async fn test_grafana_shard_latency_histogram() {
],
)
.await;
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
send_events(
&mut stream_a,
&[
Expand Down Expand Up @@ -2745,7 +2713,6 @@ async fn test_grafana_shard_latency_with_failure() {
],
)
.await;
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
send_events(
&mut stream_a,
&[
Expand All @@ -2762,7 +2729,6 @@ async fn test_grafana_shard_latency_with_failure() {
],
)
.await;
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
send_events(
&mut stream_a,
&[
Expand Down Expand Up @@ -4210,7 +4176,6 @@ async fn test_grafana_bundle_latency() {
],
)
.await;
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
send_events(
&mut stream,
&[
Expand All @@ -4227,7 +4192,6 @@ async fn test_grafana_bundle_latency() {
],
)
.await;
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
send_events(
&mut stream,
&[
Expand All @@ -4248,7 +4212,6 @@ async fn test_grafana_bundle_latency() {
],
)
.await;
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
send_events(
&mut stream,
&[
Expand Down Expand Up @@ -4338,7 +4301,6 @@ async fn test_grafana_segment_latency() {
],
)
.await;
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
send_events(
&mut stream,
&[
Expand Down Expand Up @@ -4395,7 +4357,6 @@ async fn test_grafana_preimage_latency() {
],
)
.await;
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
send_events(
&mut stream,
&[
Expand Down
Loading