From d3dfc7fd802333ba44fe5422b27991a7fa55422e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Hodul=C3=A1k?= <338111167+roman-parity@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:19:46 +0100 Subject: [PATCH 01/21] test: Add functional test for bottlenecks endpoint * Adds surrounding setup and teardown process * Includes new steps in CI The tests are run using ``` cargo test --test functional ``` Assuming these ENV variables are set: * `TEST_SPAMMER_CMD` containing path to jip-3-spammer binary * `TEST_DATABASE_URL` containing postgres URL to Timescale DB --- .github/workflows/ci.yml | 15 +++ Cargo.lock | 16 +++ Cargo.toml | 2 + tests/functional.rs | 1 + tests/functional_tests/bottlenecks.rs | 49 ++++++++ tests/functional_tests/mod.rs | 155 ++++++++++++++++++++++++++ 6 files changed, 238 insertions(+) create mode 100644 tests/functional.rs create mode 100644 tests/functional_tests/bottlenecks.rs create mode 100644 tests/functional_tests/mod.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11fe347..a5c9c2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,6 +56,12 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Checkout jip-3-spammer for functional tests + uses: actions/checkout@v4 + with: + repository: 'paritytech/jip-3-load-test' + path: 'jip-3-spammer' + - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -72,6 +78,15 @@ jobs: RUST_LOG: info run: cargo test --test api_tests --test ws_tests --test integration_tests --test optimized_server_tests -- --test-threads=1 + - name: Build jip-3-spammer for functional tests + run: cargo build --manifest-path jip-3-spammer/Cargo.toml -p jip-3-spammer --bin jip-3-spammer + + - name: Run functional tests + env: + TEST_DATABASE_URL: postgres://tart:tart_password@localhost:5432/tart_test + TEST_SPAMMER_CMD: jip-3-spammer/target/debug/jip-3-spammer + run: cargo test --test functional + build-and-bench: name: Build & Benchmark runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index be8ed27..a96888c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4288,6 +4288,7 @@ dependencies = [ "tracing-subscriber", "url", "utoipa", + "xshell", ] [[package]] @@ -5495,6 +5496,21 @@ dependencies = [ "tap", ] +[[package]] +name = "xshell" +version = "0.3.0-pre.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e6192cff94a7552e329df8c2c6f00371f107eb6e8c970f9948879216d7c46fe" +dependencies = [ + "xshell-macros", +] + +[[package]] +name = "xshell-macros" +version = "0.3.0-pre.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259fc1522c6f4915b1cc3b52dff32a88069ba979b2513fe8ee6a058670dd5dc" + [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index e99a92b..75ef9db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,6 +109,8 @@ profiling = ["dhat"] axum-test = "15.0" tokio-tungstenite = "0.24" futures-util = "0.3" +reqwest = { version = "0.11", features = ["json", "blocking"] } +xshell = "0.3.0-pre.2" [profile.dev] opt-level = 1 diff --git a/tests/functional.rs b/tests/functional.rs new file mode 100644 index 0000000..9dff7bc --- /dev/null +++ b/tests/functional.rs @@ -0,0 +1 @@ +mod functional_tests; diff --git a/tests/functional_tests/bottlenecks.rs b/tests/functional_tests/bottlenecks.rs new file mode 100644 index 0000000..5cb3fc7 --- /dev/null +++ b/tests/functional_tests/bottlenecks.rs @@ -0,0 +1,49 @@ +use crate::functional_tests::TestContext; +use chrono::{SecondsFormat, Utc}; +use xshell::cmd; + +#[test] +fn test_bottlenecks_detect_work_packages() { + super::run_test("test_bottlenecks_detect_work_packages", |ctx| { + let TestContext { + sh, + api_bind, + telemetry_bind, + spammer_cmd, + } = ctx; + + let start = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); + + cmd!( + sh, + "{spammer_cmd} sim --params tiny --duration 120 --server {telemetry_bind}" + ) + .run() + .unwrap(); + + let end = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); + + let actual_response: serde_json::Value = reqwest::blocking::get(format!( + "http://{api_bind}/api/grafana/bottlenecks?start={start}&end={end}" + )) + .unwrap() + .json() + .unwrap(); + + let actual_total_wps = actual_response + .as_array() + .unwrap() + .first() + .unwrap() + .as_object() + .unwrap() + .get("total_wps") + .unwrap() + .as_i64() + .unwrap(); + + assert!(actual_total_wps > 0); + + println!("actual response: {actual_response}"); + }); +} diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs new file mode 100644 index 0000000..dd0f62f --- /dev/null +++ b/tests/functional_tests/mod.rs @@ -0,0 +1,155 @@ +use sqlx::postgres::PgConnectOptions; +use sqlx::{Connection, PgConnection}; +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::panic::UnwindSafe; +use std::process::{Command, Stdio}; +use xshell::{cmd, Shell}; + +mod bottlenecks; + +/// A collection of parameters intended for the test scenario to use to locate TART services and +/// related tools. +pub struct TestContext { + /// A [`Shell`] to use to run commands with. + /// + /// The shell may come with workdir and environment variables set. + pub sh: Shell, + /// A string of `addr:port` where TART telemetry server is listening. + /// + /// This should be the server to write JIP-3 events to. + pub telemetry_bind: String, + /// A string of `addr:port` where TART API server is listening. + /// + /// This should be the server to read statistics from. + pub api_bind: String, + /// A filename of the CLI executable to generate input with. + pub spammer_cmd: String, +} + +/// Runs the test scenario provided in `f`. +/// +/// Before running `f`, this function prepares and launches TART. The address of the TART instance +/// is then passed to `f` using [`TestContext`]. +/// +/// The `unique_name` is used to derive parameters. It should be different for every test, otherwise +/// it raises a conflict when running them in parallel. It is intentionally not generated as a +/// straight-forward way to remain deterministic. +/// +/// After running `f`, a cleanup of resources and a graceful shutdown of TART is performed. Because +/// `f` may panic e.g. due to a failed assertion, it has an additional constraint on [`UnwindSafe`]. +/// The panic is only caught if it triggers unwind. +/// +/// # Environment requirements +/// +/// There are a few assumptions that need to pass in order for the test to run. +/// * ENV var `TEST_DATABASE_URL` is defined and contains a postgres URL. +/// * ENV var `TEST_SPAMMER_CMD` is defined and contains a path to executable `jip-3-spammer` binary +pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { + let work_dir = env!("CARGO_MANIFEST_DIR"); + let database_url = + std::env::var("TEST_DATABASE_URL").expect("ENV var TEST_DATABASE_URL should be set"); + let options: PgConnectOptions = database_url + .parse() + .expect("TEST_DATABASE_URL should be a postgres URL"); + let default_db_name = options + .get_database() + .expect("TEST_DATABASE_URL should contain a default database name"); + + let test_db_name = unique_name; + let spammer_cmd = + std::env::var("TEST_SPAMMER_CMD").expect("ENV var TEST_SPAMMER_CMD should be set"); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_io() + .build() + .unwrap(); + + runtime + .block_on(async { + let mut connection = PgConnection::connect_with(&options).await?; + + sqlx::query(format!("DROP DATABASE IF EXISTS {test_db_name}").as_str()) + .execute(&mut connection) + .await?; + + sqlx::query(format!("CREATE DATABASE {test_db_name}").as_str()) + .execute(&mut connection) + .await?; + + Ok::<_, sqlx::Error>(()) + }) + .unwrap(); + + let test_database_url = database_url.replace(default_db_name, test_db_name); + let sh = Shell::new() + .unwrap() + .with_var("DATABASE_URL", test_database_url.clone()) + .with_current_dir(work_dir); + + cmd!( + sh, + "cargo build --release -p tart-backend --bin tart-backend" + ) + .run_echo() + .expect("cargo build should succeed"); + + // Allocate port numbers uniquely for the test + let hash = { + let mut hasher = DefaultHasher::new(); + unique_name.hash(&mut hasher); + hasher.finish() + }; + let telemetry_port = (9000 + (hash % (u16::MAX as u64 - 9000))) as u16; + let api_port = (8080 + (hash % (u16::MAX as u64 - 8080))) as u16; + + assert_ne!( + telemetry_port, api_port, + "Telemetry port and API port should not match" + ); + + let telemetry_bind = format!("127.0.0.1:{telemetry_port}"); + let api_bind = format!("127.0.0.1:{api_port}"); + let telemetry_process = cmd!( + sh, + "./target/release/tart-backend --telemetry-bind {telemetry_bind} --api-bind {api_bind}" + ) + .to_command() + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + + let telemetry_pid = telemetry_process.id(); + + let result = std::panic::catch_unwind(|| { + f(TestContext { + sh, + telemetry_bind, + api_bind, + spammer_cmd, + }); + }); + + Command::new("kill") + .arg("-2") + .arg(telemetry_pid.to_string()) + .status() + .unwrap(); + + let output = telemetry_process.wait_with_output().unwrap(); + + println!("{}", String::from_utf8_lossy(&output.stdout)); + + runtime + .block_on(async { + let mut connection = PgConnection::connect(database_url.as_str()).await?; + + sqlx::query(format!("DROP DATABASE {test_db_name}").as_str()) + .execute(&mut connection) + .await?; + + Ok::<_, sqlx::Error>(()) + }) + .unwrap(); + + result.expect("test should succeed"); +} From 7abb39ef0fa417a5cd7f096c1c962ebc6cc4f7de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Hodul=C3=A1k?= <338111167+roman-parity@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:30:50 +0100 Subject: [PATCH 02/21] test: Move responsibility for building the tart backend outside of the test setup --- .github/workflows/ci.yml | 4 ++++ tests/functional_tests/mod.rs | 10 ++-------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5c9c2a..09c3fb6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,10 +81,14 @@ jobs: - name: Build jip-3-spammer for functional tests run: cargo build --manifest-path jip-3-spammer/Cargo.toml -p jip-3-spammer --bin jip-3-spammer + - name: Build TART for functional tests + run: cargo build --release -p tart-backend --bin tart-backend + - name: Run functional tests env: TEST_DATABASE_URL: postgres://tart:tart_password@localhost:5432/tart_test TEST_SPAMMER_CMD: jip-3-spammer/target/debug/jip-3-spammer + TEST_TART_CMD: target/release/tart-backend run: cargo test --test functional build-and-bench: diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index dd0f62f..5a3b59d 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -58,6 +58,7 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { let test_db_name = unique_name; let spammer_cmd = std::env::var("TEST_SPAMMER_CMD").expect("ENV var TEST_SPAMMER_CMD should be set"); + let tart_cmd = std::env::var("TEST_TART_CMD").expect("ENV var TEST_TART_CMD should be set"); let runtime = tokio::runtime::Builder::new_current_thread() .enable_io() @@ -86,13 +87,6 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { .with_var("DATABASE_URL", test_database_url.clone()) .with_current_dir(work_dir); - cmd!( - sh, - "cargo build --release -p tart-backend --bin tart-backend" - ) - .run_echo() - .expect("cargo build should succeed"); - // Allocate port numbers uniquely for the test let hash = { let mut hasher = DefaultHasher::new(); @@ -111,7 +105,7 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { let api_bind = format!("127.0.0.1:{api_port}"); let telemetry_process = cmd!( sh, - "./target/release/tart-backend --telemetry-bind {telemetry_bind} --api-bind {api_bind}" + "{tart_cmd} --telemetry-bind {telemetry_bind} --api-bind {api_bind}" ) .to_command() .stdout(Stdio::piped()) From 66234eade9af8693b62ea05109121d10f612998b Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:43:15 +0000 Subject: [PATCH 03/21] test: Replace xshell with std::process::Command --- Cargo.lock | 16 ----------- Cargo.toml | 1 - tests/functional_tests/bottlenecks.rs | 17 ++--------- tests/functional_tests/mod.rs | 41 ++++++++++++++------------- 4 files changed, 23 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a96888c..be8ed27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4288,7 +4288,6 @@ dependencies = [ "tracing-subscriber", "url", "utoipa", - "xshell", ] [[package]] @@ -5496,21 +5495,6 @@ dependencies = [ "tap", ] -[[package]] -name = "xshell" -version = "0.3.0-pre.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e6192cff94a7552e329df8c2c6f00371f107eb6e8c970f9948879216d7c46fe" -dependencies = [ - "xshell-macros", -] - -[[package]] -name = "xshell-macros" -version = "0.3.0-pre.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3259fc1522c6f4915b1cc3b52dff32a88069ba979b2513fe8ee6a058670dd5dc" - [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 75ef9db..d452d73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,7 +110,6 @@ axum-test = "15.0" tokio-tungstenite = "0.24" futures-util = "0.3" reqwest = { version = "0.11", features = ["json", "blocking"] } -xshell = "0.3.0-pre.2" [profile.dev] opt-level = 1 diff --git a/tests/functional_tests/bottlenecks.rs b/tests/functional_tests/bottlenecks.rs index 5cb3fc7..7dd2fd5 100644 --- a/tests/functional_tests/bottlenecks.rs +++ b/tests/functional_tests/bottlenecks.rs @@ -1,28 +1,15 @@ -use crate::functional_tests::TestContext; use chrono::{SecondsFormat, Utc}; -use xshell::cmd; #[test] fn test_bottlenecks_detect_work_packages() { super::run_test("test_bottlenecks_detect_work_packages", |ctx| { - let TestContext { - sh, - api_bind, - telemetry_bind, - spammer_cmd, - } = ctx; - let start = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); - cmd!( - sh, - "{spammer_cmd} sim --params tiny --duration 120 --server {telemetry_bind}" - ) - .run() - .unwrap(); + ctx.run_spammer(&["sim", "--params", "tiny", "--duration", "120"]); let end = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); + let api_bind = &ctx.api_bind; let actual_response: serde_json::Value = reqwest::blocking::get(format!( "http://{api_bind}/api/grafana/bottlenecks?start={start}&end={end}" )) diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index 5a3b59d..0d18f17 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -3,27 +3,34 @@ use sqlx::{Connection, PgConnection}; use std::hash::{DefaultHasher, Hash, Hasher}; use std::panic::UnwindSafe; use std::process::{Command, Stdio}; -use xshell::{cmd, Shell}; mod bottlenecks; /// A collection of parameters intended for the test scenario to use to locate TART services and /// related tools. pub struct TestContext { - /// A [`Shell`] to use to run commands with. - /// - /// The shell may come with workdir and environment variables set. - pub sh: Shell, /// A string of `addr:port` where TART telemetry server is listening. /// /// This should be the server to write JIP-3 events to. - pub telemetry_bind: String, + telemetry_bind: String, /// A string of `addr:port` where TART API server is listening. /// /// This should be the server to read statistics from. pub api_bind: String, /// A filename of the CLI executable to generate input with. - pub spammer_cmd: String, + spammer_cmd: String, +} + +impl TestContext { + /// Runs `jip-3-spammer` with `args`, sending its events to the TART under test. + pub fn run_spammer(&self, args: &[&str]) { + let status = Command::new(&self.spammer_cmd) + .args(args) + .args(["--server", &self.telemetry_bind]) + .status() + .expect("jip-3-spammer should start"); + assert!(status.success(), "jip-3-spammer failed: {status}"); + } } /// Runs the test scenario provided in `f`. @@ -82,10 +89,6 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { .unwrap(); let test_database_url = database_url.replace(default_db_name, test_db_name); - let sh = Shell::new() - .unwrap() - .with_var("DATABASE_URL", test_database_url.clone()) - .with_current_dir(work_dir); // Allocate port numbers uniquely for the test let hash = { @@ -103,20 +106,18 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { let telemetry_bind = format!("127.0.0.1:{telemetry_port}"); let api_bind = format!("127.0.0.1:{api_port}"); - let telemetry_process = cmd!( - sh, - "{tart_cmd} --telemetry-bind {telemetry_bind} --api-bind {api_bind}" - ) - .to_command() - .stdout(Stdio::piped()) - .spawn() - .unwrap(); + let telemetry_process = Command::new(&tart_cmd) + .args(["--telemetry-bind", &telemetry_bind, "--api-bind", &api_bind]) + .env("DATABASE_URL", &test_database_url) + .current_dir(work_dir) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); let telemetry_pid = telemetry_process.id(); let result = std::panic::catch_unwind(|| { f(TestContext { - sh, telemetry_bind, api_bind, spammer_cmd, From e7c6d1686261f079ae0640f07061ea0b682c89ec Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:43:37 +0000 Subject: [PATCH 04/21] test: Build TART database URL by replacing only its path --- tests/functional_tests/mod.rs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index 0d18f17..213643e 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -3,6 +3,7 @@ use sqlx::{Connection, PgConnection}; use std::hash::{DefaultHasher, Hash, Hasher}; use std::panic::UnwindSafe; use std::process::{Command, Stdio}; +use url::Url; mod bottlenecks; @@ -58,9 +59,6 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { let options: PgConnectOptions = database_url .parse() .expect("TEST_DATABASE_URL should be a postgres URL"); - let default_db_name = options - .get_database() - .expect("TEST_DATABASE_URL should contain a default database name"); let test_db_name = unique_name; let spammer_cmd = @@ -88,7 +86,7 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { }) .unwrap(); - let test_database_url = database_url.replace(default_db_name, test_db_name); + let test_database_url = database_url_for(&database_url, test_db_name); // Allocate port numbers uniquely for the test let hash = { @@ -148,3 +146,20 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { result.expect("test should succeed"); } + +fn database_url_for(server_url: &str, database_name: &str) -> String { + let mut url = Url::parse(server_url).expect("TEST_DATABASE_URL should be a valid URL"); + url.set_path(database_name); + url.to_string() +} + +#[test] +fn database_url_for_keeps_user_and_options_when_they_contain_the_database_name() { + assert_eq!( + database_url_for( + "postgres://postgres:pw@localhost/postgres?sslmode=disable", + "functional" + ), + "postgres://postgres:pw@localhost/functional?sslmode=disable" + ); +} From 0f2ddd3e9b2d9a47b2b8907c8a3b3d03d8225ef7 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:43:50 +0000 Subject: [PATCH 05/21] test: Ask the OS for free TART ports instead of hashing the test name --- tests/functional_tests/mod.rs | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index 213643e..dbfde25 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -1,6 +1,6 @@ use sqlx::postgres::PgConnectOptions; use sqlx::{Connection, PgConnection}; -use std::hash::{DefaultHasher, Hash, Hasher}; +use std::net::TcpListener; use std::panic::UnwindSafe; use std::process::{Command, Stdio}; use url::Url; @@ -39,9 +39,9 @@ impl TestContext { /// Before running `f`, this function prepares and launches TART. The address of the TART instance /// is then passed to `f` using [`TestContext`]. /// -/// The `unique_name` is used to derive parameters. It should be different for every test, otherwise -/// it raises a conflict when running them in parallel. It is intentionally not generated as a -/// straight-forward way to remain deterministic. +/// The `unique_name` is used as the name of the test database. It should be different for every +/// test, otherwise it raises a conflict when running them in parallel. It is intentionally not +/// generated as a straight-forward way to remain deterministic. /// /// After running `f`, a cleanup of resources and a graceful shutdown of TART is performed. Because /// `f` may panic e.g. due to a failed assertion, it has an additional constraint on [`UnwindSafe`]. @@ -88,22 +88,8 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { let test_database_url = database_url_for(&database_url, test_db_name); - // Allocate port numbers uniquely for the test - let hash = { - let mut hasher = DefaultHasher::new(); - unique_name.hash(&mut hasher); - hasher.finish() - }; - let telemetry_port = (9000 + (hash % (u16::MAX as u64 - 9000))) as u16; - let api_port = (8080 + (hash % (u16::MAX as u64 - 8080))) as u16; - - assert_ne!( - telemetry_port, api_port, - "Telemetry port and API port should not match" - ); - - let telemetry_bind = format!("127.0.0.1:{telemetry_port}"); - let api_bind = format!("127.0.0.1:{api_port}"); + let telemetry_bind = format!("127.0.0.1:{}", free_local_port()); + let api_bind = format!("127.0.0.1:{}", free_local_port()); let telemetry_process = Command::new(&tart_cmd) .args(["--telemetry-bind", &telemetry_bind, "--api-bind", &api_bind]) .env("DATABASE_URL", &test_database_url) @@ -147,6 +133,11 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { result.expect("test should succeed"); } +fn free_local_port() -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").expect("should find a free port"); + listener.local_addr().unwrap().port() +} + fn database_url_for(server_url: &str, database_name: &str) -> String { let mut url = Url::parse(server_url).expect("TEST_DATABASE_URL should be a valid URL"); url.set_path(database_name); From 6c3efd553d982c15fce6f93538d55899e182b353 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:44:01 +0000 Subject: [PATCH 06/21] test: Run functional tests one at a time --- tests/functional_tests/mod.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index dbfde25..35d69d3 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -3,10 +3,13 @@ use sqlx::{Connection, PgConnection}; use std::net::TcpListener; use std::panic::UnwindSafe; use std::process::{Command, Stdio}; +use std::sync::{Mutex, PoisonError}; use url::Url; mod bottlenecks; +static ONE_TEST_AT_A_TIME: Mutex<()> = Mutex::new(()); + /// A collection of parameters intended for the test scenario to use to locate TART services and /// related tools. pub struct TestContext { @@ -39,9 +42,11 @@ impl TestContext { /// Before running `f`, this function prepares and launches TART. The address of the TART instance /// is then passed to `f` using [`TestContext`]. /// -/// The `unique_name` is used as the name of the test database. It should be different for every -/// test, otherwise it raises a conflict when running them in parallel. It is intentionally not -/// generated as a straight-forward way to remain deterministic. +/// The `unique_name` is used as the name of the test database, so every test starts from an empty +/// one. It is intentionally not generated as a straight-forward way to remain deterministic. +/// +/// Only one test runs at a time, even when the test harness runs them on multiple threads. The +/// tests are meant to catch performance regressions, so they must not compete for resources. /// /// After running `f`, a cleanup of resources and a graceful shutdown of TART is performed. Because /// `f` may panic e.g. due to a failed assertion, it has an additional constraint on [`UnwindSafe`]. @@ -53,6 +58,9 @@ impl TestContext { /// * ENV var `TEST_DATABASE_URL` is defined and contains a postgres URL. /// * ENV var `TEST_SPAMMER_CMD` is defined and contains a path to executable `jip-3-spammer` binary pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { + let _one_test_at_a_time = ONE_TEST_AT_A_TIME + .lock() + .unwrap_or_else(PoisonError::into_inner); let work_dir = env!("CARGO_MANIFEST_DIR"); let database_url = std::env::var("TEST_DATABASE_URL").expect("ENV var TEST_DATABASE_URL should be set"); From 6b40901210015c9a90c574f471fb1b8e751703ca Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:45:06 +0000 Subject: [PATCH 07/21] test: Write TART output to a log file, print it when the test fails --- tests/functional_tests/mod.rs | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index 35d69d3..90ab721 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -1,8 +1,10 @@ use sqlx::postgres::PgConnectOptions; use sqlx::{Connection, PgConnection}; +use std::fs::File; use std::net::TcpListener; use std::panic::UnwindSafe; -use std::process::{Command, Stdio}; +use std::path::Path; +use std::process::Command; use std::sync::{Mutex, PoisonError}; use url::Url; @@ -98,11 +100,15 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { let telemetry_bind = format!("127.0.0.1:{}", free_local_port()); let api_bind = format!("127.0.0.1:{}", free_local_port()); - let telemetry_process = Command::new(&tart_cmd) + let tart_log_path = + Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("{unique_name}-tart.log")); + let tart_log = File::create(&tart_log_path).expect("TART log file should be writable"); + let mut telemetry_process = Command::new(&tart_cmd) .args(["--telemetry-bind", &telemetry_bind, "--api-bind", &api_bind]) .env("DATABASE_URL", &test_database_url) .current_dir(work_dir) - .stdout(Stdio::piped()) + .stdout(tart_log.try_clone().unwrap()) + .stderr(tart_log) .spawn() .unwrap(); @@ -122,9 +128,11 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { .status() .unwrap(); - let output = telemetry_process.wait_with_output().unwrap(); + telemetry_process.wait().unwrap(); - println!("{}", String::from_utf8_lossy(&output.stdout)); + if result.is_err() { + print_tart_log(&tart_log_path); + } runtime .block_on(async { @@ -141,6 +149,13 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { result.expect("test should succeed"); } +fn print_tart_log(path: &Path) { + match std::fs::read_to_string(path) { + Ok(log) => println!("TART log ({}):\n{log}", path.display()), + Err(error) => println!("Could not read TART log {}: {error}", path.display()), + } +} + fn free_local_port() -> u16 { let listener = TcpListener::bind("127.0.0.1:0").expect("should find a free port"); listener.local_addr().unwrap().port() From aa4d99d0bfedd1d71be4cacb507c6ddedbcc2d35 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:46:03 +0000 Subject: [PATCH 08/21] test: Stop TART with SIGTERM and fail if it does not exit in time --- Cargo.lock | 19 ++++++++++++++ Cargo.toml | 1 + tests/functional_tests/mod.rs | 47 +++++++++++++++++++++++++++-------- 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be8ed27..96804fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -702,6 +702,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "chrono" version = "0.4.42" @@ -2683,6 +2689,18 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags 2.9.4", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -4268,6 +4286,7 @@ dependencies = [ "libc", "metrics", "metrics-exporter-prometheus", + "nix", "parking_lot", "rand 0.8.5", "ratatui", diff --git a/Cargo.toml b/Cargo.toml index d452d73..01604e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,6 +110,7 @@ axum-test = "15.0" tokio-tungstenite = "0.24" futures-util = "0.3" reqwest = { version = "0.11", features = ["json", "blocking"] } +nix = { version = "0.30", features = ["signal"] } [profile.dev] opt-level = 1 diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index 90ab721..eb70d25 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -1,17 +1,24 @@ +use nix::sys::signal::{kill, Signal}; +use nix::unistd::Pid; use sqlx::postgres::PgConnectOptions; use sqlx::{Connection, PgConnection}; use std::fs::File; use std::net::TcpListener; use std::panic::UnwindSafe; use std::path::Path; -use std::process::Command; +use std::process::{Child, Command, ExitStatus}; use std::sync::{Mutex, PoisonError}; +use std::thread; +use std::time::{Duration, Instant}; use url::Url; mod bottlenecks; static ONE_TEST_AT_A_TIME: Mutex<()> = Mutex::new(()); +const TART_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30); +const POLL_INTERVAL: Duration = Duration::from_millis(200); + /// A collection of parameters intended for the test scenario to use to locate TART services and /// related tools. pub struct TestContext { @@ -112,8 +119,6 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { .spawn() .unwrap(); - let telemetry_pid = telemetry_process.id(); - let result = std::panic::catch_unwind(|| { f(TestContext { telemetry_bind, @@ -122,15 +127,18 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { }); }); - Command::new("kill") - .arg("-2") - .arg(telemetry_pid.to_string()) - .status() - .unwrap(); - - telemetry_process.wait().unwrap(); + kill( + Pid::from_raw(telemetry_process.id() as i32), + Signal::SIGTERM, + ) + .expect("TART should accept SIGTERM"); + let stopped_in_time = wait_for_exit(&mut telemetry_process, TART_SHUTDOWN_TIMEOUT).is_some(); + if !stopped_in_time { + telemetry_process.kill().unwrap(); + telemetry_process.wait().unwrap(); + } - if result.is_err() { + if result.is_err() || !stopped_in_time { print_tart_log(&tart_log_path); } @@ -147,6 +155,23 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { .unwrap(); result.expect("test should succeed"); + assert!( + stopped_in_time, + "TART should shut down within {TART_SHUTDOWN_TIMEOUT:?}" + ); +} + +fn wait_for_exit(process: &mut Child, timeout: Duration) -> Option { + let deadline = Instant::now() + timeout; + loop { + if let Some(status) = process.try_wait().unwrap() { + return Some(status); + } + if Instant::now() >= deadline { + return None; + } + thread::sleep(POLL_INTERVAL); + } } fn print_tart_log(path: &Path) { From e5d00e9eb8642c65b35153ae944bedb50603b819 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:46:36 +0000 Subject: [PATCH 09/21] test: Clean up TART and the test database with drop guards Cleanup now also runs when setup panics, e.g. when TART fails to spawn. --- tests/functional_tests/mod.rs | 208 +++++++++++++++++++--------------- 1 file changed, 114 insertions(+), 94 deletions(-) diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index eb70d25..9f7348e 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -1,11 +1,9 @@ use nix::sys::signal::{kill, Signal}; use nix::unistd::Pid; -use sqlx::postgres::PgConnectOptions; use sqlx::{Connection, PgConnection}; use std::fs::File; use std::net::TcpListener; -use std::panic::UnwindSafe; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{Child, Command, ExitStatus}; use std::sync::{Mutex, PoisonError}; use std::thread; @@ -57,114 +55,143 @@ impl TestContext { /// Only one test runs at a time, even when the test harness runs them on multiple threads. The /// tests are meant to catch performance regressions, so they must not compete for resources. /// -/// After running `f`, a cleanup of resources and a graceful shutdown of TART is performed. Because -/// `f` may panic e.g. due to a failed assertion, it has an additional constraint on [`UnwindSafe`]. -/// The panic is only caught if it triggers unwind. +/// TART is shut down gracefully and the test database is dropped afterwards, also when `f` panics +/// e.g. due to a failed assertion. TART's output is written to a log file in +/// `CARGO_TARGET_TMPDIR`, and printed when the test fails. /// /// # Environment requirements /// /// There are a few assumptions that need to pass in order for the test to run. /// * ENV var `TEST_DATABASE_URL` is defined and contains a postgres URL. /// * ENV var `TEST_SPAMMER_CMD` is defined and contains a path to executable `jip-3-spammer` binary -pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext) + UnwindSafe) { +pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext)) { let _one_test_at_a_time = ONE_TEST_AT_A_TIME .lock() .unwrap_or_else(PoisonError::into_inner); - let work_dir = env!("CARGO_MANIFEST_DIR"); - let database_url = - std::env::var("TEST_DATABASE_URL").expect("ENV var TEST_DATABASE_URL should be set"); - let options: PgConnectOptions = database_url - .parse() - .expect("TEST_DATABASE_URL should be a postgres URL"); - - let test_db_name = unique_name; - let spammer_cmd = - std::env::var("TEST_SPAMMER_CMD").expect("ENV var TEST_SPAMMER_CMD should be set"); - let tart_cmd = std::env::var("TEST_TART_CMD").expect("ENV var TEST_TART_CMD should be set"); + let database = TestDatabase::create(&required_env("TEST_DATABASE_URL"), unique_name); + let tart = Tart::start(&required_env("TEST_TART_CMD"), &database.url, unique_name); - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_io() - .build() - .unwrap(); - - runtime - .block_on(async { - let mut connection = PgConnection::connect_with(&options).await?; - - sqlx::query(format!("DROP DATABASE IF EXISTS {test_db_name}").as_str()) - .execute(&mut connection) - .await?; - - sqlx::query(format!("CREATE DATABASE {test_db_name}").as_str()) - .execute(&mut connection) - .await?; - - Ok::<_, sqlx::Error>(()) - }) - .unwrap(); - - let test_database_url = database_url_for(&database_url, test_db_name); - - let telemetry_bind = format!("127.0.0.1:{}", free_local_port()); - let api_bind = format!("127.0.0.1:{}", free_local_port()); - let tart_log_path = - Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("{unique_name}-tart.log")); - let tart_log = File::create(&tart_log_path).expect("TART log file should be writable"); - let mut telemetry_process = Command::new(&tart_cmd) - .args(["--telemetry-bind", &telemetry_bind, "--api-bind", &api_bind]) - .env("DATABASE_URL", &test_database_url) - .current_dir(work_dir) - .stdout(tart_log.try_clone().unwrap()) - .stderr(tart_log) - .spawn() - .unwrap(); - - let result = std::panic::catch_unwind(|| { - f(TestContext { - telemetry_bind, - api_bind, - spammer_cmd, - }); + f(TestContext { + telemetry_bind: tart.telemetry_bind.clone(), + api_bind: tart.api_bind.clone(), + spammer_cmd: required_env("TEST_SPAMMER_CMD"), }); +} + +fn required_env(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| panic!("ENV var {name} should be set")) +} + +struct TestDatabase { + server_url: String, + name: String, + url: String, +} - kill( - Pid::from_raw(telemetry_process.id() as i32), - Signal::SIGTERM, - ) - .expect("TART should accept SIGTERM"); - let stopped_in_time = wait_for_exit(&mut telemetry_process, TART_SHUTDOWN_TIMEOUT).is_some(); - if !stopped_in_time { - telemetry_process.kill().unwrap(); - telemetry_process.wait().unwrap(); +impl TestDatabase { + fn create(server_url: &str, name: &str) -> Self { + execute_on_server( + server_url, + &[ + format!("DROP DATABASE IF EXISTS {name} WITH (FORCE)"), + format!("CREATE DATABASE {name}"), + ], + ) + .expect("test database should be created"); + Self { + server_url: server_url.to_string(), + name: name.to_string(), + url: database_url_for(server_url, name), + } } +} - if result.is_err() || !stopped_in_time { - print_tart_log(&tart_log_path); +impl Drop for TestDatabase { + fn drop(&mut self) { + let drop_database = format!("DROP DATABASE {} WITH (FORCE)", self.name); + if let Err(error) = execute_on_server(&self.server_url, &[drop_database]) { + println!("Could not drop test database {}: {error}", self.name); + } } +} - runtime - .block_on(async { - let mut connection = PgConnection::connect(database_url.as_str()).await?; +fn execute_on_server(server_url: &str, statements: &[String]) -> Result<(), sqlx::Error> { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("tokio runtime should start"); + runtime.block_on(async { + let mut connection = PgConnection::connect(server_url).await?; + for statement in statements { + sqlx::query(statement).execute(&mut connection).await?; + } + Ok(()) + }) +} - sqlx::query(format!("DROP DATABASE {test_db_name}").as_str()) - .execute(&mut connection) - .await?; +struct Tart { + process: Child, + telemetry_bind: String, + api_bind: String, + log_path: PathBuf, +} - Ok::<_, sqlx::Error>(()) - }) - .unwrap(); +impl Tart { + fn start(tart_cmd: &str, database_url: &str, name: &str) -> Self { + let telemetry_bind = format!("127.0.0.1:{}", free_local_port()); + let api_bind = format!("127.0.0.1:{}", free_local_port()); + let log_path = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("{name}-tart.log")); + let log = File::create(&log_path).expect("TART log file should be writable"); + let process = Command::new(tart_cmd) + .args(["--telemetry-bind", &telemetry_bind, "--api-bind", &api_bind]) + .env("DATABASE_URL", database_url) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .stdout(log.try_clone().unwrap()) + .stderr(log) + .spawn() + .expect("TART should start"); + Self { + process, + telemetry_bind, + api_bind, + log_path, + } + } - result.expect("test should succeed"); - assert!( - stopped_in_time, - "TART should shut down within {TART_SHUTDOWN_TIMEOUT:?}" - ); + fn print_log(&self) { + match std::fs::read_to_string(&self.log_path) { + Ok(log) => println!("TART log ({}):\n{log}", self.log_path.display()), + Err(error) => println!( + "Could not read TART log {}: {error}", + self.log_path.display() + ), + } + } +} + +impl Drop for Tart { + fn drop(&mut self) { + let _ = kill(Pid::from_raw(self.process.id() as i32), Signal::SIGTERM); + let stopped_in_time = wait_for_exit(&mut self.process, TART_SHUTDOWN_TIMEOUT).is_some(); + if !stopped_in_time { + let _ = self.process.kill(); + let _ = self.process.wait(); + } + + if thread::panicking() || !stopped_in_time { + self.print_log(); + } + assert!( + stopped_in_time || thread::panicking(), + "TART should shut down within {TART_SHUTDOWN_TIMEOUT:?}" + ); + } } fn wait_for_exit(process: &mut Child, timeout: Duration) -> Option { let deadline = Instant::now() + timeout; loop { - if let Some(status) = process.try_wait().unwrap() { + if let Ok(Some(status)) = process.try_wait() { return Some(status); } if Instant::now() >= deadline { @@ -174,13 +201,6 @@ fn wait_for_exit(process: &mut Child, timeout: Duration) -> Option { } } -fn print_tart_log(path: &Path) { - match std::fs::read_to_string(path) { - Ok(log) => println!("TART log ({}):\n{log}", path.display()), - Err(error) => println!("Could not read TART log {}: {error}", path.display()), - } -} - fn free_local_port() -> u16 { let listener = TcpListener::bind("127.0.0.1:0").expect("should find a free port"); listener.local_addr().unwrap().port() From 8142fc821b1e35cac417066c54b6e9dadf412c0d Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:46:50 +0000 Subject: [PATCH 10/21] test: Wait until TART is ready before running the scenario --- tests/functional_tests/mod.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index 9f7348e..2f7ee19 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -14,6 +14,7 @@ mod bottlenecks; static ONE_TEST_AT_A_TIME: Mutex<()> = Mutex::new(()); +const TART_STARTUP_TIMEOUT: Duration = Duration::from_secs(60); const TART_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30); const POLL_INTERVAL: Duration = Duration::from_millis(200); @@ -150,11 +151,30 @@ impl Tart { .stderr(log) .spawn() .expect("TART should start"); - Self { + let mut tart = Self { process, telemetry_bind, api_bind, log_path, + }; + tart.wait_until_ready(); + tart + } + + fn wait_until_ready(&mut self) { + let health_url = format!("http://{}/api/health", self.api_bind); + let deadline = Instant::now() + TART_STARTUP_TIMEOUT; + while !reqwest::blocking::get(&health_url) + .is_ok_and(|response| response.status().is_success()) + { + if let Ok(Some(status)) = self.process.try_wait() { + panic!("TART exited during startup: {status}"); + } + assert!( + Instant::now() < deadline, + "TART should be ready within {TART_STARTUP_TIMEOUT:?}" + ); + thread::sleep(POLL_INTERVAL); } } From 35ddcced0733a6925ccb041cb67ccadbf3f05b24 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:47:10 +0000 Subject: [PATCH 11/21] test: Time out the spammer and check API response status --- tests/functional_tests/bottlenecks.rs | 27 +++++---------------------- tests/functional_tests/mod.rs | 23 ++++++++++++++++++++--- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/functional_tests/bottlenecks.rs b/tests/functional_tests/bottlenecks.rs index 7dd2fd5..f36e68a 100644 --- a/tests/functional_tests/bottlenecks.rs +++ b/tests/functional_tests/bottlenecks.rs @@ -9,28 +9,11 @@ fn test_bottlenecks_detect_work_packages() { let end = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); - let api_bind = &ctx.api_bind; - let actual_response: serde_json::Value = reqwest::blocking::get(format!( - "http://{api_bind}/api/grafana/bottlenecks?start={start}&end={end}" - )) - .unwrap() - .json() - .unwrap(); + let response = ctx.get_json(&format!("/api/grafana/bottlenecks?start={start}&end={end}")); - let actual_total_wps = actual_response - .as_array() - .unwrap() - .first() - .unwrap() - .as_object() - .unwrap() - .get("total_wps") - .unwrap() - .as_i64() - .unwrap(); - - assert!(actual_total_wps > 0); - - println!("actual response: {actual_response}"); + assert!( + response[0]["total_wps"].as_i64() > Some(0), + "expected work packages in the response: {response}" + ); }); } diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index 2f7ee19..9c575f6 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -15,6 +15,7 @@ mod bottlenecks; static ONE_TEST_AT_A_TIME: Mutex<()> = Mutex::new(()); const TART_STARTUP_TIMEOUT: Duration = Duration::from_secs(60); +const SPAMMER_TIMEOUT: Duration = Duration::from_secs(600); const TART_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30); const POLL_INTERVAL: Duration = Duration::from_millis(200); @@ -28,7 +29,7 @@ pub struct TestContext { /// A string of `addr:port` where TART API server is listening. /// /// This should be the server to read statistics from. - pub api_bind: String, + api_bind: String, /// A filename of the CLI executable to generate input with. spammer_cmd: String, } @@ -36,13 +37,29 @@ pub struct TestContext { impl TestContext { /// Runs `jip-3-spammer` with `args`, sending its events to the TART under test. pub fn run_spammer(&self, args: &[&str]) { - let status = Command::new(&self.spammer_cmd) + let mut spammer = Command::new(&self.spammer_cmd) .args(args) .args(["--server", &self.telemetry_bind]) - .status() + .spawn() .expect("jip-3-spammer should start"); + let Some(status) = wait_for_exit(&mut spammer, SPAMMER_TIMEOUT) else { + let _ = spammer.kill(); + panic!("jip-3-spammer should finish within {SPAMMER_TIMEOUT:?}"); + }; assert!(status.success(), "jip-3-spammer failed: {status}"); } + + /// Sends a GET request to the TART API, e.g. `get_json("/api/health")`. + pub fn get_json(&self, path_and_query: &str) -> serde_json::Value { + let url = format!("http://{}{path_and_query}", self.api_bind); + let response = + reqwest::blocking::get(&url).unwrap_or_else(|error| panic!("GET {url}: {error}")); + let status = response.status(); + let body = response.text().unwrap_or_default(); + assert!(status.is_success(), "GET {url} returned {status}: {body}"); + serde_json::from_str(&body) + .unwrap_or_else(|error| panic!("GET {url} returned invalid JSON ({error}): {body}")) + } } /// Runs the test scenario provided in `f`. From 5e34451afd1222081ababbbd07db6f643c0bd6da Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:57:37 +0000 Subject: [PATCH 12/21] test: Check exact bottleneck stats from a reproducible simulation Fixed seed and a start in the past make the spammer generate the same events on every run, so the full response can be compared. --- tests/functional_tests/bottlenecks.rs | 45 ++++++++++++++++++++------- tests/functional_tests/mod.rs | 7 +++-- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/tests/functional_tests/bottlenecks.rs b/tests/functional_tests/bottlenecks.rs index f36e68a..6f65ef8 100644 --- a/tests/functional_tests/bottlenecks.rs +++ b/tests/functional_tests/bottlenecks.rs @@ -1,19 +1,42 @@ -use chrono::{SecondsFormat, Utc}; +use serde_json::json; +use std::thread; +use std::time::{Duration, Instant}; + +const SIM_START: &str = "2026-01-01T00:00:00Z"; +const SIM_END: &str = "2026-01-01T00:02:00Z"; +const RESULTS_TIMEOUT: Duration = Duration::from_secs(30); #[test] fn test_bottlenecks_detect_work_packages() { super::run_test("test_bottlenecks_detect_work_packages", |ctx| { - let start = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); - - ctx.run_spammer(&["sim", "--params", "tiny", "--duration", "120"]); - - let end = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); + // Faster timelines make TART drop late pipeline events, which changes the results. + ctx.run_spammer(&format!( + "sim --params tiny --seed 42 --start {SIM_START} --duration 120 --speed 1 --quiet" + )); - let response = ctx.get_json(&format!("/api/grafana/bottlenecks?start={start}&end={end}")); + let expected = json!([{ + "total_wps": 7, + "failed_wps": 0, + "failure_rate": 0.0, + "avg_pipeline_ms": 3129.095428571429, + "stage_timing": { + "authorize": { "p50_ms": 49.0, "p95_ms": 71.6 }, + "refine": { "p50_ms": 219.329, "p95_ms": 255.6086 }, + "report": { "p50_ms": 8.0, "p95_ms": 11.7 }, + "guarantee": { "p50_ms": 2697.0074999999997, "p95_ms": 3854.51675 }, + "distribute": { "p50_ms": 335.5, "p95_ms": 410.0 }, + "pipeline_total": { "p50_ms": 3230.064, "p95_ms": 4509.0547 }, + }, + }]); - assert!( - response[0]["total_wps"].as_i64() > Some(0), - "expected work packages in the response: {response}" - ); + let path = format!("/api/grafana/bottlenecks?start={SIM_START}&end={SIM_END}"); + // TART writes work-package tracking to the database periodically, not on every event. + let deadline = Instant::now() + RESULTS_TIMEOUT; + let mut actual = ctx.get_json(&path); + while actual != expected && Instant::now() < deadline { + thread::sleep(Duration::from_secs(1)); + actual = ctx.get_json(&path); + } + assert_eq!(actual, expected); }); } diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index 9c575f6..5dc582d 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -35,10 +35,11 @@ pub struct TestContext { } impl TestContext { - /// Runs `jip-3-spammer` with `args`, sending its events to the TART under test. - pub fn run_spammer(&self, args: &[&str]) { + /// Runs `jip-3-spammer` with whitespace-separated `args`, sending its events to the TART under + /// test. + pub fn run_spammer(&self, args: &str) { let mut spammer = Command::new(&self.spammer_cmd) - .args(args) + .args(args.split_whitespace()) .args(["--server", &self.telemetry_bind]) .spawn() .expect("jip-3-spammer should start"); From ca573540085f119cd23cf2f1d39004fb7bd8eda2 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:57:45 +0000 Subject: [PATCH 13/21] ci: Pin jip-3-spammer used by the functional tests --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09c3fb6..73f12ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,8 @@ jobs: uses: actions/checkout@v4 with: repository: 'paritytech/jip-3-load-test' + # The functional tests expect exact results from this spammer version. + ref: 'ba9d87cd0448e2e570da0c4169fcca2f441e5af2' path: 'jip-3-spammer' - name: Install Rust toolchain From 30c8debf80f80a047a562734a17c95686de5eff2 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:57:49 +0000 Subject: [PATCH 14/21] test: Document all functional test environment requirements --- tests/functional_tests/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index 5dc582d..0e086cb 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -81,8 +81,10 @@ impl TestContext { /// # Environment requirements /// /// There are a few assumptions that need to pass in order for the test to run. -/// * ENV var `TEST_DATABASE_URL` is defined and contains a postgres URL. +/// * ENV var `TEST_DATABASE_URL` is defined and contains a postgres URL of a TimescaleDB server. +/// Its user must be allowed to create databases. /// * ENV var `TEST_SPAMMER_CMD` is defined and contains a path to executable `jip-3-spammer` binary +/// * ENV var `TEST_TART_CMD` is defined and contains a path to executable `tart-backend` binary pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext)) { let _one_test_at_a_time = ONE_TEST_AT_A_TIME .lock() From 2a1363eacbf14138146957ef198efbe4893e0f03 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:58:15 +0000 Subject: [PATCH 15/21] test: Keep ports reserved until both TART ports are picked --- tests/functional_tests/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index 0e086cb..44ceced 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -159,8 +159,7 @@ struct Tart { impl Tart { fn start(tart_cmd: &str, database_url: &str, name: &str) -> Self { - let telemetry_bind = format!("127.0.0.1:{}", free_local_port()); - let api_bind = format!("127.0.0.1:{}", free_local_port()); + let [telemetry_bind, api_bind] = free_local_addresses(); let log_path = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("{name}-tart.log")); let log = File::create(&log_path).expect("TART log file should be writable"); let process = Command::new(tart_cmd) @@ -241,9 +240,10 @@ fn wait_for_exit(process: &mut Child, timeout: Duration) -> Option { } } -fn free_local_port() -> u16 { - let listener = TcpListener::bind("127.0.0.1:0").expect("should find a free port"); - listener.local_addr().unwrap().port() +fn free_local_addresses() -> [String; COUNT] { + let listeners = + [(); COUNT].map(|()| TcpListener::bind("127.0.0.1:0").expect("should find a free port")); + listeners.map(|listener| listener.local_addr().unwrap().to_string()) } fn database_url_for(server_url: &str, database_name: &str) -> String { From 60b90b2669d8863227dae72905b2e50c630e86b3 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:58:15 +0000 Subject: [PATCH 16/21] test: Check all required ENV vars before starting TART --- tests/functional_tests/mod.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index 44ceced..13ef538 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -89,13 +89,17 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext)) { let _one_test_at_a_time = ONE_TEST_AT_A_TIME .lock() .unwrap_or_else(PoisonError::into_inner); - let database = TestDatabase::create(&required_env("TEST_DATABASE_URL"), unique_name); - let tart = Tart::start(&required_env("TEST_TART_CMD"), &database.url, unique_name); + let database_url = required_env("TEST_DATABASE_URL"); + let tart_cmd = required_env("TEST_TART_CMD"); + let spammer_cmd = required_env("TEST_SPAMMER_CMD"); + + let database = TestDatabase::create(&database_url, unique_name); + let tart = Tart::start(&tart_cmd, &database.url, unique_name); f(TestContext { telemetry_bind: tart.telemetry_bind.clone(), api_bind: tart.api_bind.clone(), - spammer_cmd: required_env("TEST_SPAMMER_CMD"), + spammer_cmd, }); } From 84a4ebdd9395b6f85bffeb5ff0d400ab11131258 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:24:41 +0000 Subject: [PATCH 17/21] test: Write spammer output to a log file, print it when the test fails --- tests/functional_tests/mod.rs | 47 +++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index 13ef538..75cd7ee 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -32,15 +32,23 @@ pub struct TestContext { api_bind: String, /// A filename of the CLI executable to generate input with. spammer_cmd: String, + /// Output of all spammer runs in the test, printed when the test fails. + spammer_log_path: PathBuf, } impl TestContext { /// Runs `jip-3-spammer` with whitespace-separated `args`, sending its events to the TART under /// test. pub fn run_spammer(&self, args: &str) { + let log = File::options() + .append(true) + .open(&self.spammer_log_path) + .expect("jip-3-spammer log file should be writable"); let mut spammer = Command::new(&self.spammer_cmd) .args(args.split_whitespace()) .args(["--server", &self.telemetry_bind]) + .stdout(log.try_clone().unwrap()) + .stderr(log) .spawn() .expect("jip-3-spammer should start"); let Some(status) = wait_for_exit(&mut spammer, SPAMMER_TIMEOUT) else { @@ -63,6 +71,14 @@ impl TestContext { } } +impl Drop for TestContext { + fn drop(&mut self) { + if thread::panicking() { + print_log(&self.spammer_log_path); + } + } +} + /// Runs the test scenario provided in `f`. /// /// Before running `f`, this function prepares and launches TART. The address of the TART instance @@ -75,7 +91,7 @@ impl TestContext { /// tests are meant to catch performance regressions, so they must not compete for resources. /// /// TART is shut down gracefully and the test database is dropped afterwards, also when `f` panics -/// e.g. due to a failed assertion. TART's output is written to a log file in +/// e.g. due to a failed assertion. The output of TART and of the spammer is written to log files in /// `CARGO_TARGET_TMPDIR`, and printed when the test fails. /// /// # Environment requirements @@ -96,13 +112,28 @@ pub fn run_test(unique_name: &str, f: impl FnOnce(TestContext)) { let database = TestDatabase::create(&database_url, unique_name); let tart = Tart::start(&tart_cmd, &database.url, unique_name); + let spammer_log_path = log_path(unique_name, "jip-3-spammer"); + File::create(&spammer_log_path).expect("jip-3-spammer log file should be writable"); + f(TestContext { telemetry_bind: tart.telemetry_bind.clone(), api_bind: tart.api_bind.clone(), spammer_cmd, + spammer_log_path, }); } +fn log_path(test_name: &str, program: &str) -> PathBuf { + Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("{test_name}-{program}.log")) +} + +fn print_log(path: &Path) { + match std::fs::read_to_string(path) { + Ok(log) => println!("Log {}:\n{log}", path.display()), + Err(error) => println!("Could not read log {}: {error}", path.display()), + } +} + fn required_env(name: &str) -> String { std::env::var(name).unwrap_or_else(|_| panic!("ENV var {name} should be set")) } @@ -164,7 +195,7 @@ struct Tart { impl Tart { fn start(tart_cmd: &str, database_url: &str, name: &str) -> Self { let [telemetry_bind, api_bind] = free_local_addresses(); - let log_path = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("{name}-tart.log")); + let log_path = log_path(name, "tart"); let log = File::create(&log_path).expect("TART log file should be writable"); let process = Command::new(tart_cmd) .args(["--telemetry-bind", &telemetry_bind, "--api-bind", &api_bind]) @@ -200,16 +231,6 @@ impl Tart { thread::sleep(POLL_INTERVAL); } } - - fn print_log(&self) { - match std::fs::read_to_string(&self.log_path) { - Ok(log) => println!("TART log ({}):\n{log}", self.log_path.display()), - Err(error) => println!( - "Could not read TART log {}: {error}", - self.log_path.display() - ), - } - } } impl Drop for Tart { @@ -222,7 +243,7 @@ impl Drop for Tart { } if thread::panicking() || !stopped_in_time { - self.print_log(); + print_log(&self.log_path); } assert!( stopped_in_time || thread::panicking(), From d80de29e37b493ef5cdc67f80c97ee0f58f3fb31 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:32:17 +0000 Subject: [PATCH 18/21] test: Run TART without rate limiter and simulate at 10x speed --- tests/functional_tests/bottlenecks.rs | 5 +++-- tests/functional_tests/mod.rs | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/functional_tests/bottlenecks.rs b/tests/functional_tests/bottlenecks.rs index 6f65ef8..a209749 100644 --- a/tests/functional_tests/bottlenecks.rs +++ b/tests/functional_tests/bottlenecks.rs @@ -9,9 +9,10 @@ const RESULTS_TIMEOUT: Duration = Duration::from_secs(30); #[test] fn test_bottlenecks_detect_work_packages() { super::run_test("test_bottlenecks_detect_work_packages", |ctx| { - // Faster timelines make TART drop late pipeline events, which changes the results. + // Above 10x, events from all nodes arrive together and their order decides which + // timestamps TART records, so the results would differ between runs. ctx.run_spammer(&format!( - "sim --params tiny --seed 42 --start {SIM_START} --duration 120 --speed 1 --quiet" + "sim --params tiny --seed 42 --start {SIM_START} --duration 120 --speed 10 --quiet" )); let expected = json!([{ diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index 75cd7ee..b6a9262 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -199,6 +199,8 @@ impl Tart { let log = File::create(&log_path).expect("TART log file should be writable"); let process = Command::new(tart_cmd) .args(["--telemetry-bind", &telemetry_bind, "--api-bind", &api_bind]) + // Production runs without the per-node rate limiter too. + .arg("--no-rate-limit") .env("DATABASE_URL", database_url) .current_dir(env!("CARGO_MANIFEST_DIR")) .stdout(log.try_clone().unwrap()) From daca2ef5ff4660858c1b4c22663d956cf83b0b9f Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:44:29 +0000 Subject: [PATCH 19/21] test: Lower spammer timeout to 60 s --- tests/functional_tests/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional_tests/mod.rs b/tests/functional_tests/mod.rs index b6a9262..47bbcfc 100644 --- a/tests/functional_tests/mod.rs +++ b/tests/functional_tests/mod.rs @@ -15,7 +15,7 @@ mod bottlenecks; static ONE_TEST_AT_A_TIME: Mutex<()> = Mutex::new(()); const TART_STARTUP_TIMEOUT: Duration = Duration::from_secs(60); -const SPAMMER_TIMEOUT: Duration = Duration::from_secs(600); +const SPAMMER_TIMEOUT: Duration = Duration::from_secs(60); const TART_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30); const POLL_INTERVAL: Duration = Duration::from_millis(200); From ce5a72e81816c5cbba9efa6617e43a60893f2622 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:54:27 +0000 Subject: [PATCH 20/21] ci: Cache jip-3-spammer build --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73f12ec..bf94286 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,10 @@ jobs: - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 + with: + workspaces: | + . -> target + jip-3-spammer -> target - name: Run unit tests run: cargo test --lib --test types_tests --test events_tests --test error_tests --test encoding_tests From ec9751b353c9aae2081cd5995b8dbc15a97fbdc1 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:17:25 +0000 Subject: [PATCH 21/21] ci: Run functional tests in the common test step Since #50 a single cargo test runs every test target, including the functional tests, so it needs the spammer, TART and their ENV vars. --- .github/workflows/ci.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3b123d..70d57b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,25 +74,20 @@ jobs: . -> target jip-3-spammer -> target - - name: Run tests - env: - DATABASE_URL: postgres://tart:tart_password@localhost:5432/tart_test - TEST_DATABASE_URL: postgres://tart:tart_password@localhost:5432/tart_test - RUST_LOG: info - run: cargo test --no-fail-fast -- --test-threads=1 - - name: Build jip-3-spammer for functional tests run: cargo build --manifest-path jip-3-spammer/Cargo.toml -p jip-3-spammer --bin jip-3-spammer - name: Build TART for functional tests run: cargo build --release -p tart-backend --bin tart-backend - - name: Run functional tests + - name: Run tests env: + DATABASE_URL: postgres://tart:tart_password@localhost:5432/tart_test TEST_DATABASE_URL: postgres://tart:tart_password@localhost:5432/tart_test TEST_SPAMMER_CMD: jip-3-spammer/target/debug/jip-3-spammer TEST_TART_CMD: target/release/tart-backend - run: cargo test --test functional + RUST_LOG: info + run: cargo test --no-fail-fast -- --test-threads=1 build-and-bench: name: Build & Benchmark