Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
d3dfc7f
test: Add functional test for bottlenecks endpoint
roman-parity Oct 29, 2026
7abb39e
test: Move responsibility for building the tart backend outside of th…
roman-parity Oct 29, 2026
66234ea
test: Replace xshell with std::process::Command
michalkucharczyk Nov 10, 2026
e7c6d16
test: Build TART database URL by replacing only its path
michalkucharczyk Nov 10, 2026
0f2ddd3
test: Ask the OS for free TART ports instead of hashing the test name
michalkucharczyk Nov 10, 2026
6c3efd5
test: Run functional tests one at a time
michalkucharczyk Nov 10, 2026
6b40901
test: Write TART output to a log file, print it when the test fails
michalkucharczyk Nov 10, 2026
aa4d99d
test: Stop TART with SIGTERM and fail if it does not exit in time
michalkucharczyk Nov 10, 2026
e5d00e9
test: Clean up TART and the test database with drop guards
michalkucharczyk Nov 10, 2026
8142fc8
test: Wait until TART is ready before running the scenario
michalkucharczyk Nov 10, 2026
35ddcce
test: Time out the spammer and check API response status
michalkucharczyk Nov 10, 2026
5e34451
test: Check exact bottleneck stats from a reproducible simulation
michalkucharczyk Nov 10, 2026
ca57354
ci: Pin jip-3-spammer used by the functional tests
michalkucharczyk Nov 10, 2026
30c8deb
test: Document all functional test environment requirements
michalkucharczyk Nov 10, 2026
2a1363e
test: Keep ports reserved until both TART ports are picked
michalkucharczyk Nov 10, 2026
60b90b2
test: Check all required ENV vars before starting TART
michalkucharczyk Nov 10, 2026
84a4ebd
test: Write spammer output to a log file, print it when the test fails
michalkucharczyk Nov 10, 2026
d80de29
test: Run TART without rate limiter and simulate at 10x speed
michalkucharczyk Nov 10, 2026
daca2ef
test: Lower spammer timeout to 60 s
michalkucharczyk Nov 11, 2026
ce5a72e
ci: Cache jip-3-spammer build
michalkucharczyk Nov 11, 2026
f7e04ee
Merge remote-tracking branch 'origin/main' into functional-tests
michalkucharczyk Nov 11, 2026
ec9751b
ci: Run functional tests in the common test step
michalkucharczyk Nov 11, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,16 +56,36 @@ jobs:
- name: Checkout code
uses: actions/checkout@v4

- name: Checkout jip-3-spammer for functional tests
uses: actions/checkout@v4
with:
repository: 'paritytech/jip-3-load-test'
# The functional tests expect exact results from this spammer version.
ref: 'ba9d87cd0448e2e570da0c4169fcca2f441e5af2'
path: 'jip-3-spammer'

- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable

- name: Cache Rust dependencies
uses: Swatinem/rust-cache@v2
with:
workspaces: |
. -> target
jip-3-spammer -> target

- name: Build jip-3-spammer for functional tests
run: cargo build --manifest-path jip-3-spammer/Cargo.toml -p jip-3-spammer --bin jip-3-spammer

- name: Build TART for functional tests
run: cargo build --release -p tart-backend --bin tart-backend

- name: Run tests
env:
DATABASE_URL: postgres://tart:tart_password@localhost:5432/tart_test
TEST_DATABASE_URL: postgres://tart:tart_password@localhost:5432/tart_test
TEST_SPAMMER_CMD: jip-3-spammer/target/debug/jip-3-spammer
TEST_TART_CMD: target/release/tart-backend
RUST_LOG: info
run: cargo test --no-fail-fast -- --test-threads=1

Expand Down
19 changes: 19 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ profiling = ["dhat"]
axum-test = "15.0"
tokio-tungstenite = "0.24"
futures-util = "0.3"
reqwest = { version = "0.11", features = ["json", "blocking"] }
nix = { version = "0.30", features = ["signal"] }

[profile.dev]
opt-level = 1
Expand Down
1 change: 1 addition & 0 deletions tests/functional.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
mod functional_tests;
43 changes: 43 additions & 0 deletions tests/functional_tests/bottlenecks.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
use serde_json::json;
use std::thread;
use std::time::{Duration, Instant};

const SIM_START: &str = "2026-01-01T00:00:00Z";
const SIM_END: &str = "2026-01-01T00:02:00Z";
const RESULTS_TIMEOUT: Duration = Duration::from_secs(30);

#[test]
fn test_bottlenecks_detect_work_packages() {
super::run_test("test_bottlenecks_detect_work_packages", |ctx| {
// Above 10x, events from all nodes arrive together and their order decides which
// timestamps TART records, so the results would differ between runs.
ctx.run_spammer(&format!(
"sim --params tiny --seed 42 --start {SIM_START} --duration 120 --speed 10 --quiet"
));

let expected = json!([{
"total_wps": 7,
"failed_wps": 0,
"failure_rate": 0.0,
"avg_pipeline_ms": 3129.095428571429,
"stage_timing": {
"authorize": { "p50_ms": 49.0, "p95_ms": 71.6 },
"refine": { "p50_ms": 219.329, "p95_ms": 255.6086 },
"report": { "p50_ms": 8.0, "p95_ms": 11.7 },
"guarantee": { "p50_ms": 2697.0074999999997, "p95_ms": 3854.51675 },
"distribute": { "p50_ms": 335.5, "p95_ms": 410.0 },
"pipeline_total": { "p50_ms": 3230.064, "p95_ms": 4509.0547 },
},
}]);

let path = format!("/api/grafana/bottlenecks?start={SIM_START}&end={SIM_END}");
// TART writes work-package tracking to the database periodically, not on every event.
let deadline = Instant::now() + RESULTS_TIMEOUT;
let mut actual = ctx.get_json(&path);
while actual != expected && Instant::now() < deadline {
thread::sleep(Duration::from_secs(1));
actual = ctx.get_json(&path);
}
assert_eq!(actual, expected);
});
}
Loading
Loading