Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,4 @@ clap = { version = "4", features = ["derive"] }
colored = "2"
serde = { version = "1", features = ["derive"] }
toml = "0.8"
futures = "0.3"
6 changes: 5 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ Built on top of [`integration_tests_sv2`](https://crates.io/crates/integration_t
cargo run --bin sv2-compliance -- --config targets.toml
```

`--config` defaults to `targets.toml` in the current directory. Exits `0` when no scenario fails, `1` otherwise; a scenario the endpoint legitimately did not let the client exercise (e.g. a standard-channel scenario against a server that set `REQUIRES_EXTENDED_CHANNELS`) is reported but does not fail the run. See `targets.example.toml` for the config format and `--help` for all options.
`--config` defaults to `targets.toml` in the current directory. Addresses accept either an IP or a hostname followed by a port (e.g. `stratum.ckpool.org:3336`); hostnames are resolved before the run starts, and an unresolvable one is reported as a failed target without stopping the remaining sites. Exits `0` when no scenario fails and every target resolves, `1` otherwise; a scenario the endpoint legitimately did not let the client exercise (e.g. a standard-channel scenario against a server that set `REQUIRES_EXTENDED_CHANNELS`) is reported but does not fail the run. See `targets.example.toml` for the config format and `--help` for all options.

Use `--verbose` to also print the underlying resolver message when a hostname fails to resolve.

## Library

Expand All @@ -29,6 +31,8 @@ async fn sv2_compliance() {
}
```

For TOML batches, `interoperability_tests_sv2::config::Config::from_path` parses the file and `resolve_targets().await` returns `Targets`, whose `resolved` and `unresolved` fields separate reachable targets from failed lookups. `Targets::is_empty()` reports a config with no addresses at all.

## Running the tests

`cargo test` spins up a local SRI Template Provider and Pool (binaries under `template-provider/` are downloaded on first run) and runs the suite against them.
Expand Down
178 changes: 133 additions & 45 deletions src/bin/sv2-compliance.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ use clap::Parser;
use colored::Colorize;
use integration_tests_sv2::start_tracing;
use interoperability_tests_sv2::{
config::Config,
app_type::AppType,
config::{Config, Target, TargetAddress, UnresolvedTarget},
runner::run_suite,
scenarios::{ScenarioError, ScenarioReport},
};
Expand All @@ -19,20 +20,95 @@ struct Cli {
#[arg(short = 'c', long, default_value = "targets.toml")]
config: PathBuf,

/// Verbose logging
/// Verbose logging and resolver error details
#[arg(short, long)]
verbose: bool,
}

#[derive(Default)]
struct RunSummary {
passed: u32,
failed: u32,
not_exercised: u32,
}
impl RunSummary {
fn record(&mut self, report: &ScenarioReport) {
match &report.result {
Ok(_) => self.passed += 1,
Err(ScenarioError::NotExercised(_)) => self.not_exercised += 1,
Err(ScenarioError::Failed(_)) => self.failed += 1,
}
}

fn succeeded(&self, unresolved: usize) -> bool {
self.failed == 0 && unresolved == 0
}

fn summary_line(&self, resolved: usize, unresolved: usize) -> String {
if resolved == 0 && unresolved > 0 {
return format!(
"{} no targets resolved ({} unresolved)",
"✗".red().bold(),
unresolved
);
}

let total = self.passed + self.failed + self.not_exercised;
let failed_note = if self.failed > 0 {
format!(", {} failed", self.failed)
} else {
String::new()
};
let not_exercised_note = if self.not_exercised > 0 {
format!(", {} not exercised", self.not_exercised)
} else {
String::new()
};
let unresolved_note = if unresolved > 0 {
format!("; {unresolved} target(s) unresolved")
} else {
String::new()
};

if self.succeeded(unresolved) {
format!(
"{} {}/{} scenarios passed{} across {} resolved target(s)",
"✓".green().bold(),
self.passed,
total,
not_exercised_note,
resolved
)
} else {
format!(
"{} {}/{} scenarios passed{}{}{} across {} resolved target(s)",
"✗".red().bold(),
self.passed,
total,
failed_note,
not_exercised_note,
unresolved_note,
resolved
)
}
}

fn print(&self, resolved: usize, unresolved: usize) {
println!("{}", self.summary_line(resolved, unresolved));
}
}

#[tokio::main]
async fn main() {
let mut summary = RunSummary::default();

let cli = Cli::parse();
if cli.verbose {
start_tracing();
}

let targets = match Config::from_path(&cli.config) {
Ok(config) => config.targets(),
Ok(config) => config.resolve_targets().await,
Err(e) => {
eprintln!("{} {e}", "error:".red().bold());
std::process::exit(1);
Expand All @@ -44,60 +120,72 @@ async fn main() {
std::process::exit(1);
}

let mut passed = 0u32;
let mut failed = 0u32;
let mut not_exercised = 0u32;
for target in &targets {
println!(
"── {} ──",
format!(
"{}: {:?} at {}",
target.site_name, target.endpoint.app_type, target.endpoint.addr
)
.blue()
.bold()
for target in &targets.resolved {
print_target_header(
&target.site_name,
target.endpoint.app_type,
format!("{}{}", target.address, resolved_note(target)),
);
let reports = run_suite(&target.endpoint).await;
for report in &reports {
summary.record(report);
print_report(report);
match &report.result {
Ok(_) => passed += 1,
Err(ScenarioError::NotExercised(_)) => not_exercised += 1,
Err(ScenarioError::Failed(_)) => failed += 1,
}
}
println!();
}

let total = passed + failed + not_exercised;
let not_exercised_note = if not_exercised > 0 {
format!(", {not_exercised} not exercised")
} else {
String::new()
};
if failed == 0 {
println!(
"{} {}/{} scenarios passed{} across {} target(s)",
"✓".green().bold(),
passed,
total,
not_exercised_note,
targets.len()
);
} else {
println!(
"{} {}/{} scenarios passed, {} failed{} across {} target(s)",
"✗".red().bold(),
passed,
total,
failed,
not_exercised_note,
targets.len()
);
for target in &targets.unresolved {
print_unresolved(target, cli.verbose);
}

summary.print(targets.resolved.len(), targets.unresolved.len());

if !summary.succeeded(targets.unresolved.len()) {
std::process::exit(1);
}
}

fn print_target_header(site_name: &str, app_type: AppType, address: String) {
println!(
"── {} ──",
format!("{site_name}: {app_type:?} at {address}")
.blue()
.bold()
);
}

fn print_unresolved(target: &UnresolvedTarget, verbose: bool) {
print_target_header(
&target.site_name,
target.app_type,
target.address.to_string(),
);

println!(
" {} target unresolved: {}",
"✗".red(),
target.error.to_string().red()
);

if verbose {
if let Some(detail) = target.error.detail() {
println!(" {}", detail.dimmed());
}
}

println!();
}

/// Shows the resolved IP when the configured address was a hostname.
fn resolved_note(target: &Target) -> String {
match &target.address {
TargetAddress::Socket(_) => String::new(),
TargetAddress::Hostname { .. } => {
format!(" (resolved to {})", target.endpoint.addr)
}
}
}

fn print_report(report: &ScenarioReport) {
match &report.result {
Ok(detail) => {
Expand Down
Loading
Loading