perf 1/5: benchmark harness, TOASTTY_STATS counters and baseline

bench/README.md

@@ -0,0 +1,65 @@
+# Benchmarks
+
+Tools for measuring toastty's throughput, frame rate, CPU, memory and
+latency. Everything runs off-screen on a private Xvfb display. The
+safety rules in `AGENTS.md` ("Running toastty on a virtual screen")
+apply, and the scripts follow them.
+
+## Full suite
+
+```sh
+cargo build --release
+bench/run.py --bin target/release/toastty --out after.json --work /tmp/toastty-bench
+bench/compare.py before.json after.json      # Markdown table
+bench/run.py ... --only dense_cells,idle_blink --reps 5
+```
+
+The workloads are generated once (seeded, so they're identical between runs)
+and cached in `--work/data`. Every scenario reads toastty's own report,
+which the binary writes when `TOASTTY_STATS=<path>` is set
+(`crates/toastty/src/perf_stats.rs`). The idle scenarios also sample `/proc`.
+
+| Scenario | What it does | Key metrics |
+|---|---|---|
+| `startup` | `toastty -- sleep 1.5` | launch → first frame, idle RSS |
+| `idle_blink` / `idle_steady` | bash prompt for 10 s, cursor blinking / steady | CPU %, RSS |
+| `throughput_ascii` / `_sgr` / `_unicode` / `_scroll` | `cat` 64 / 32 / 16 / 23 MB | MB/s, frames and FPS while streaming, CPU s, peak RSS |
+| `dense_cells` / `_big` | every cell recoloured each frame, one synchronized-output frame at a time (128×42, and 256×73 at 2560×1400) | FPS, frame ms |
+| `scrollback_memory` | 30k full-width coloured lines (fills the 10k scrollback) | RSS after |
+| `key_latency` | types into `cat` at 25 keys/s | key → frame ms |
+| `scroll_wheel` / `_big` | wheel-scrolls 20k lines of history | frame ms p50/p99 |
+
+"Frame ms" is the CPU time of `Renderer::render_term`, including the
+present. Output → frame is measured from the first PTY batch after a frame
+to the end of the next frame.
+
+## Headless parse throughput
+
+No window or GPU, so it gives quick, low-noise numbers for the parser and the
+terminal state:
+
+```sh
+cargo run --release -p toastty --example parse_bench -- /tmp/toastty-bench/data/*.txt
+```
+
+## CPU profiles
+
+`perf`/`samply` need `perf_event_paranoid <= 1`. Without it, use the
+in-process SIGPROF sampler behind the `pprof` feature:
+
+```sh
+cargo build --profile profiling --features pprof
+bench/profile.sh /tmp/p.folded -- cat /tmp/toastty-bench/data/ascii.txt
+bench/folded.py /tmp/p.folded --top 40 [--thread toastty] [--focus render_term]
+# headless:
+cargo build --profile profiling -p toastty --example parse_bench --features pprof
+TOASTTY_PPROF=/tmp/p.folded target/profiling/examples/parse_bench FILE
+```
+
+## Caveats
+
+- Only the X11 path with software presentation (`MESA_VK_WSI_DEBUG=sw`) is
+  exercised. Absolute FPS and frame times differ from a real compositor, so
+  compare runs against each other.
+- Run comparisons one after another on a quiet machine. A parallel cargo
+  build skews every number.

bench/compare.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Compare two bench/run.py result files as a Markdown table.
+
+    bench/compare.py base.json new.json [more.json ...]
+
+The change column is relative to the first file. Metrics where lower is
+better (time, CPU, memory, latency) and higher is better (throughput,
+FPS) are marked so a regression stands out.
+"""
+
+import json
+import math
+import sys
+
+# metric -> (label, higher_is_better, format)
+METRICS = {
+    "mb_per_s": ("throughput MB/s", True, "{:.1f}"),
+    "fps": ("FPS during run", True, "{:.1f}"),
+    "frames": ("frames", True, "{:.0f}"),
+    "cpu_s": ("CPU s", False, "{:.2f}"),
+    "active_s": ("wall s", False, "{:.2f}"),
+    "peak_rss_mb": ("peak RSS MB", False, "{:.1f}"),
+    "final_rss_mb": ("final RSS MB", False, "{:.1f}"),
+    "frame_ms_p50": ("frame ms p50", False, "{:.2f}"),
+    "frame_ms_p99": ("frame ms p99", False, "{:.2f}"),
+    "frame_ms_mean": ("frame ms mean", False, "{:.2f}"),
+    "output_latency_ms_p50": ("output→frame ms p50", False, "{:.2f}"),
+    "output_latency_ms_p99": ("output→frame ms p99", False, "{:.2f}"),
+    "key_latency_ms_p50": ("key→frame ms p50", False, "{:.2f}"),
+    "key_latency_ms_p99": ("key→frame ms p99", False, "{:.2f}"),
+    "first_frame_ms": ("first frame ms", False, "{:.1f}"),
+    "launch_to_frame_ms": ("launch→first frame ms", False, "{:.1f}"),
+    "idle_rss_mb": ("idle RSS MB", False, "{:.1f}"),
+    "cpu_pct": ("CPU %", False, "{:.2f}"),
+    "rss_mb": ("RSS MB", False, "{:.1f}"),
+}
+
+# Per-scenario metric selection (others are noise for that scenario).
+PICK = {
+    "startup": ["launch_to_frame_ms", "first_frame_ms", "idle_rss_mb"],
+    "idle_blink": ["cpu_pct", "rss_mb"],
+    "idle_steady": ["cpu_pct", "rss_mb"],
+    "key_latency": ["key_latency_ms_p50", "key_latency_ms_p99"],
+    "scroll_wheel": ["frame_ms_p50", "frame_ms_p99", "cpu_s"],
+    "scroll_wheel_big": ["frame_ms_p50", "frame_ms_p99", "cpu_s"],
+    "scrollback_memory": ["final_rss_mb", "peak_rss_mb", "mb_per_s"],
+}
+DEFAULT = ["mb_per_s", "fps", "frame_ms_p50", "frame_ms_p99", "cpu_s", "peak_rss_mb",
+           "output_latency_ms_p50", "output_latency_ms_p99"]
+
+
+def fmt(fmt_s, v):
+    if v is None or (isinstance(v, float) and math.isnan(v)):
+        return "–"
+    return fmt_s.format(v)
+
+
+def main():
+    files = sys.argv[1:]
+    if len(files) < 2:
+        sys.exit(__doc__)
+    data = [json.load(open(f)) for f in files]
+    names = [f.rsplit("/", 1)[-1].removesuffix(".json") for f in files]
+    head = "| scenario | metric | " + " | ".join(names) + " | change |"
+    print(head)
+    print("|" + "---|" * (len(names) + 3))
+    print(
+        "| binary | size MB | "
+        + " | ".join(f"{d['binary_size_mb']:.1f}" for d in data)
+        + " | |"
+    )
+    for scen in data[0]["scenarios"]:
+        for m in PICK.get(scen, DEFAULT):
+            vals = [d["scenarios"].get(scen, {}).get(m) for d in data]
+            if all(v is None for v in vals):
+                continue
+            label, hib, f = METRICS[m]
+            base, last = vals[0], vals[-1]
+            change = ""
+            if (
+                isinstance(base, (int, float)) and isinstance(last, (int, float))
+                and not math.isnan(base) and not math.isnan(last) and base != 0
+            ):
+                r = last / base
+                better = (r > 1) == hib
+                arrow = "✅" if better and abs(r - 1) > 0.05 else ("⚠️" if abs(r - 1) > 0.05 else "")
+                change = f"{r:.2f}× {arrow}"
+            print(f"| {scen} | {label} | " + " | ".join(fmt(f, v) for v in vals) + f" | {change} |")
+
+
+if __name__ == "__main__":
+    main()