|
| 1 | +//! Reproducible, per-metric comparison of two complete cost-report pairs. |
| 2 | +
|
| 3 | +use anyhow::{ensure, Context, Result}; |
| 4 | +use serde_json::{json, Value}; |
| 5 | +use std::{collections::BTreeMap, fs, path::Path}; |
| 6 | + |
| 7 | +fn read(path: &Path) -> Result<Value> { |
| 8 | + serde_json::from_slice(&fs::read(path)?).with_context(|| format!("parse {}", path.display())) |
| 9 | +} |
| 10 | + |
| 11 | +fn named<'a>(value: &'a Value, field: &str) -> Result<BTreeMap<String, &'a Value>> { |
| 12 | + let mut rows = BTreeMap::new(); |
| 13 | + for row in value[field].as_array().with_context(|| format!("missing {field}"))? { |
| 14 | + let name = row["name"].as_str().context("missing row name")?; |
| 15 | + ensure!(rows.insert(name.to_owned(), row).is_none(), "duplicate {field}/{name}"); |
| 16 | + } |
| 17 | + Ok(rows) |
| 18 | +} |
| 19 | + |
| 20 | +#[derive(Default)] |
| 21 | +struct Comparison { |
| 22 | + count: usize, |
| 23 | + regressions: Vec<Value>, |
| 24 | + scalar: Vec<Value>, |
| 25 | +} |
| 26 | + |
| 27 | +impl Comparison { |
| 28 | + fn metric(&mut self, name: &str, field: &str, before: &Value, after: &Value, exact: bool) -> Result<()> { |
| 29 | + let left = before.as_u64().with_context(|| format!("missing baseline {name}/{field}"))?; |
| 30 | + let right = after.as_u64().with_context(|| format!("missing candidate {name}/{field}"))?; |
| 31 | + self.count += 1; |
| 32 | + if if exact { left != right } else { right > left } { |
| 33 | + self.regressions.push(json!({"name":name,"metric":field,"before":left,"after":right,"exact":exact})); |
| 34 | + } |
| 35 | + Ok(()) |
| 36 | + } |
| 37 | + |
| 38 | + fn observations(&mut self, name: &str, before: &Value, after: &Value) -> Result<()> { |
| 39 | + ensure!(before["scope"] == after["scope"], "{name}: measurement scope changed"); |
| 40 | + ensure!(before["exit_category"] == after["exit_category"], "{name}: exit category changed"); |
| 41 | + let left = &before["observation"]; |
| 42 | + let right = &after["observation"]; |
| 43 | + ensure!(left["status"] == right["status"] && left["exit_code"] == right["exit_code"], "{name}: outcome changed"); |
| 44 | + if left["status"] == "measured" { |
| 45 | + self.metric(name, "cycles", &left["cycles"], &right["cycles"], false)?; |
| 46 | + } |
| 47 | + Ok(()) |
| 48 | + } |
| 49 | + |
| 50 | + fn corpus(&mut self, before: &Value, after: &Value) -> Result<()> { |
| 51 | + for family in ["matched", "growth", "expanded", "scalar", "executions"] { |
| 52 | + let left = named(before, family)?; |
| 53 | + let right = named(after, family)?; |
| 54 | + ensure!(left.keys().eq(right.keys()), "{family}: fixture set changed"); |
| 55 | + for (name, row) in left { |
| 56 | + let candidate = right[&name]; |
| 57 | + ensure!(row["source_sha256"] == candidate["source_sha256"], "{name}: fixture source changed"); |
| 58 | + for field in [ |
| 59 | + "elf_bytes", |
| 60 | + "positive_cycles", |
| 61 | + "group_positive_cycles", |
| 62 | + "group_rejection_cycles", |
| 63 | + "max_stack_frame_bytes", |
| 64 | + "static_call_chain_stack_bound_bytes", |
| 65 | + "witness_bytes", |
| 66 | + "transaction_bytes", |
| 67 | + ] { |
| 68 | + if !row[field].is_null() || !candidate[field].is_null() { |
| 69 | + self.metric( |
| 70 | + &name, |
| 71 | + field, |
| 72 | + &row[field], |
| 73 | + &candidate[field], |
| 74 | + matches!(field, "witness_bytes" | "transaction_bytes"), |
| 75 | + )?; |
| 76 | + } |
| 77 | + } |
| 78 | + if family == "matched" { |
| 79 | + for field in ["rust_elf_bytes", "rust_positive_cycles", "elf_budget", "cycle_budget"] { |
| 80 | + ensure!(row[field] == candidate[field], "{name}: Rust reference or budget changed: {field}"); |
| 81 | + } |
| 82 | + } |
| 83 | + if family == "growth" { |
| 84 | + self.metric( |
| 85 | + &name, |
| 86 | + "static_stack", |
| 87 | + &row["static_stack"]["static_call_chain_stack_bound_bytes"], |
| 88 | + &candidate["static_stack"]["static_call_chain_stack_bound_bytes"], |
| 89 | + false, |
| 90 | + )?; |
| 91 | + } |
| 92 | + if family == "scalar" { |
| 93 | + super::cost_evidence::static_memory(&row["static_memory"])?; |
| 94 | + super::cost_evidence::static_memory(&candidate["static_memory"])?; |
| 95 | + self.scalar.push(json!({"name":name,"source_sha256":row["source_sha256"], |
| 96 | + "before_elf_sha256":row["elf_sha256"],"after_elf_sha256":candidate["elf_sha256"], |
| 97 | + "before":row["static_memory"],"after":candidate["static_memory"]})); |
| 98 | + } |
| 99 | + if family == "executions" { |
| 100 | + if name.split('/').nth(1) == Some("rust") { |
| 101 | + ensure!(row["elf_sha256"] == candidate["elf_sha256"], "{name}: Rust reference ELF changed"); |
| 102 | + } |
| 103 | + ensure!(row["oracle_exit_code"] == candidate["oracle_exit_code"], "{name}: oracle changed"); |
| 104 | + for scope in ["transaction", "group"] { |
| 105 | + self.observations(&format!("{name}/{scope}"), &row[scope], &candidate[scope])?; |
| 106 | + } |
| 107 | + for field in ["role", "input_indices", "output_indices"] { |
| 108 | + ensure!(row["group"]["group"][field] == candidate["group"]["group"][field], "{name}: group selection changed"); |
| 109 | + } |
| 110 | + } |
| 111 | + } |
| 112 | + } |
| 113 | + Ok(()) |
| 114 | + } |
| 115 | + |
| 116 | + fn multi(&mut self, before: &Value, after: &Value) -> Result<()> { |
| 117 | + let left = named(before, "rows")?; |
| 118 | + let right = named(after, "rows")?; |
| 119 | + ensure!(left.keys().eq(right.keys()), "multi-Script fixture set changed"); |
| 120 | + for (name, row) in left { |
| 121 | + let candidate = right[&name]; |
| 122 | + for field in ["elf_bytes", "max_stack_frame_bytes", "transaction_bytes", "witness_bytes"] { |
| 123 | + self.metric(&name, field, &row[field], &candidate[field], matches!(field, "transaction_bytes" | "witness_bytes"))?; |
| 124 | + } |
| 125 | + self.observations(&name, &row["transaction"], &candidate["transaction"])?; |
| 126 | + let group_key = |group: &Value| { |
| 127 | + json!([group["group"]["role"], group["group"]["input_indices"], group["group"]["output_indices"]]).to_string() |
| 128 | + }; |
| 129 | + let groups = |value: &Value| -> Result<BTreeMap<String, Value>> { |
| 130 | + let mut groups = BTreeMap::new(); |
| 131 | + for group in value["groups"].as_array().context("missing groups")? { |
| 132 | + ensure!(groups.insert(group_key(group), group.clone()).is_none(), "duplicate group"); |
| 133 | + } |
| 134 | + Ok(groups) |
| 135 | + }; |
| 136 | + let left_groups = groups(row)?; |
| 137 | + let right_groups = groups(candidate)?; |
| 138 | + ensure!(left_groups.keys().eq(right_groups.keys()), "{name}: group membership changed"); |
| 139 | + for (key, group) in left_groups { |
| 140 | + self.observations(&format!("{name}/{key}"), &group, &right_groups[&key])?; |
| 141 | + } |
| 142 | + let stacks = named(row, "static_stacks")?; |
| 143 | + let candidate_stacks = named(candidate, "static_stacks")?; |
| 144 | + ensure!(stacks.keys().eq(candidate_stacks.keys()), "{name}: static stack set changed"); |
| 145 | + for (key, stack) in stacks { |
| 146 | + self.metric( |
| 147 | + &format!("{name}/{key}"), |
| 148 | + "static_stack", |
| 149 | + &stack["bound"]["static_call_chain_stack_bound_bytes"], |
| 150 | + &candidate_stacks[&key]["bound"]["static_call_chain_stack_bound_bytes"], |
| 151 | + false, |
| 152 | + )?; |
| 153 | + } |
| 154 | + } |
| 155 | + Ok(()) |
| 156 | + } |
| 157 | +} |
| 158 | + |
| 159 | +pub fn run(root: &Path, before: &Path, before_multi: &Path, after: &Path, after_multi: &Path, output: &Path) -> Result<()> { |
| 160 | + // A failed comparison must not leave an earlier passing result in place. |
| 161 | + fs::write(output, b"{\"status\":\"not-generated\"}\n")?; |
| 162 | + super::cost_evidence::check(root, before, before_multi)?; |
| 163 | + super::cost_evidence::check(root, after, after_multi)?; |
| 164 | + let baseline = read(before)?; |
| 165 | + let candidate = read(after)?; |
| 166 | + for field in [ |
| 167 | + "cargo_lock_sha256", |
| 168 | + "rust_reference_lock_sha256", |
| 169 | + "rust_toolchain", |
| 170 | + "strip_sha256", |
| 171 | + "strip_version", |
| 172 | + "target", |
| 173 | + "target_profile", |
| 174 | + "edition", |
| 175 | + "opt_level", |
| 176 | + "rust_reference_profile", |
| 177 | + "vm_configuration", |
| 178 | + "measurement_fixture_sha256", |
| 179 | + ] { |
| 180 | + ensure!(baseline["provenance"][field] == candidate["provenance"][field], "comparison configuration changed: {field}"); |
| 181 | + } |
| 182 | + let mut comparison = Comparison::default(); |
| 183 | + comparison.corpus(&baseline, &candidate)?; |
| 184 | + comparison.multi(&read(before_multi)?, &read(after_multi)?)?; |
| 185 | + let report = json!({"schema":"cellscript-cost-comparison-v2","baseline":baseline["provenance"], |
| 186 | + "candidate":candidate["provenance"],"comparisons":comparison.count,"regressions":comparison.regressions, |
| 187 | + "scalar_static_memory":comparison.scalar,"status":if comparison.regressions.is_empty(){"passed"}else{"failed"}}); |
| 188 | + fs::write(output, serde_json::to_vec_pretty(&report)?)?; |
| 189 | + ensure!(comparison.regressions.is_empty(), "cost regressions: see {}", output.display()); |
| 190 | + println!("{} comparable metrics passed; static memory counts reported separately", comparison.count); |
| 191 | + Ok(()) |
| 192 | +} |
| 193 | + |
| 194 | +#[cfg(test)] |
| 195 | +mod tests { |
| 196 | + use super::*; |
| 197 | + |
| 198 | + #[test] |
| 199 | + fn comparison_rejects_missing_duplicate_or_changed_measurements() { |
| 200 | + assert!(named(&json!({"rows":[{"name":"a"},{"name":"a"}]}), "rows").is_err()); |
| 201 | + let mut comparison = Comparison::default(); |
| 202 | + assert!(comparison.metric("a", "cycles", &json!(8), &Value::Null, false).is_err()); |
| 203 | + comparison.metric("a", "cycles", &json!(8), &json!(9), false).unwrap(); |
| 204 | + comparison.metric("a", "witness", &json!(8), &json!(7), true).unwrap(); |
| 205 | + assert_eq!(comparison.regressions.len(), 2); |
| 206 | + let measured = json!({"scope":"transaction_scripts","exit_category":"success","observation":{"status":"measured","exit_code":0,"cycles":8}}); |
| 207 | + let mut changed = measured.clone(); |
| 208 | + changed["observation"]["status"] = json!("unavailable"); |
| 209 | + assert!(comparison.observations("a", &measured, &changed).is_err()); |
| 210 | + } |
| 211 | +} |
0 commit comments