From 04d390e70fa0bafdf8d6adb51222a50790a92d70 Mon Sep 17 00:00:00 2001 From: EDToaster Date: Thu, 22 Oct 2026 01:26:04 -0700 Subject: [PATCH 1/4] bench: TOASTTY_STATS counters, Xvfb harness, headless parse bench, pprof feature --- Cargo.lock | 134 +++++- bench/folded.py | 48 ++ bench/profile.sh | 19 + bench/run.py | 587 +++++++++++++++++++++++++ crates/toastty/Cargo.toml | 6 + crates/toastty/examples/parse_bench.rs | 52 +++ crates/toastty/src/lib.rs | 1 + crates/toastty/src/main.rs | 15 + crates/toastty/src/perf_stats.rs | 234 ++++++++++ 9 files changed, 1094 insertions(+), 2 deletions(-) create mode 100755 bench/folded.py create mode 100755 bench/profile.sh create mode 100755 bench/run.py create mode 100644 crates/toastty/examples/parse_bench.rs create mode 100644 crates/toastty/src/perf_stats.rs diff --git a/Cargo.lock b/Cargo.lock index cbbc64b4..f2cd3044 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,6 +18,15 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "366ffbaa4442f4684d91e2cd7c5ea7c4ed8add41959a31447066e279e432b618" +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -260,6 +269,21 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + [[package]] name = "base64" version = "0.22.1" @@ -938,6 +962,15 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376" +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + [[package]] name = "deltae" version = "0.3.2" @@ -1207,6 +1240,18 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + [[package]] name = "finl_unicode" version = "1.4.0" @@ -1432,6 +1477,12 @@ dependencies = [ "weezl", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "gl_generator" version = "0.14.0" @@ -2233,7 +2284,7 @@ version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0aeb26bf5e836cc1c341c8106051b573f1766dfa05aa87f0b98be5e51b02303" dependencies = [ - "nix", + "nix 0.29.0", "winapi", ] @@ -2426,6 +2477,17 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + [[package]] name = "nix" version = "0.29.0" @@ -2892,6 +2954,15 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -3301,6 +3372,27 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "pprof" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187" +dependencies = [ + "aligned-vec", + "backtrace", + "cfg-if", + "findshlibs", + "libc", + "log", + "nix 0.26.4", + "once_cell", + "smallvec", + "spin", + "symbolic-demangle", + "tempfile", + "thiserror 2.0.18", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -4031,6 +4123,12 @@ dependencies = [ "walkdir", ] +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -4453,6 +4551,15 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" +dependencies = [ + "lock_api", +] + [[package]] name = "spirv" version = "0.4.0+sdk-1.4.341.0" @@ -4537,6 +4644,28 @@ dependencies = [ "zeno", ] +[[package]] +name = "symbolic-common" +version = "12.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332615d90111d8eeaf86a84dc9bbe9f65d0d8c5cf11b4caccedc37754eb0dcfd" +dependencies = [ + "debugid", + "memmap2", + "stable_deref_trait", + "uuid", +] + +[[package]] +name = "symbolic-demangle" +version = "12.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "912017718eb4d21930546245af9a3475c9dccf15675a5c215664e76621afc471" +dependencies = [ + "rustc-demangle", + "symbolic-common", +] + [[package]] name = "syn" version = "1.0.109" @@ -4678,7 +4807,7 @@ dependencies = [ "libc", "log", "memmem", - "nix", + "nix 0.29.0", "num-derive", "num-traits", "ordered-float 4.6.0", @@ -4870,6 +4999,7 @@ dependencies = [ "image", "notify", "pollster", + "pprof", "thiserror 2.0.18", "toastty-config", "toastty-graphics", diff --git a/bench/folded.py b/bench/folded.py new file mode 100755 index 00000000..1a6a7ce3 --- /dev/null +++ b/bench/folded.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Summarize a folded-stack profile (bench/profile.sh output). + + bench/folded.py prof.folded [--top 40] [--thread main] [--focus substr] + +Prints the top functions by self and inclusive sample share. +""" +import argparse +import re +from collections import Counter + +ap = argparse.ArgumentParser() +ap.add_argument("path") +ap.add_argument("--top", type=int, default=40) +ap.add_argument("--thread", default=None) +ap.add_argument("--focus", default=None, help="only stacks containing this substring") +ap.add_argument("--width", type=int, default=110) +a = ap.parse_args() + +def short(name: str) -> str: + name = re.sub(r"::h[0-9a-f]{16}$", "", name) + name = re.sub(r"<(.*) as (.*)>", r"<\2>", name) + return name + +selfc, incl, threads = Counter(), Counter(), Counter() +total = 0 +for line in open(a.path): + stack, _, n = line.rstrip("\n").rpartition(" ") + n = int(n) + frames = stack.split(";") + thread, frames = frames[0], [short(f) for f in frames[1:]] + threads[thread] += n + if a.thread and a.thread not in thread: + continue + if a.focus and not any(a.focus in f for f in frames): + continue + total += n + if frames: + selfc[frames[-1]] += n + for f in set(frames): + incl[f] += n + +print("threads:", ", ".join(f"{t}={c}" for t, c in threads.most_common())) +print(f"samples in selection: {total}") +for title, c in (("SELF", selfc), ("INCLUSIVE", incl)): + print(f"\n== {title} ==") + for f, n in c.most_common(a.top): + print(f"{100*n/total:6.2f}% {f[:a.width]}") diff --git a/bench/profile.sh b/bench/profile.sh new file mode 100755 index 00000000..72c61ac3 --- /dev/null +++ b/bench/profile.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Profile toastty with the in-process pprof sampler on a private Xvfb +# display. Build first: cargo build --profile profiling --features pprof +# bench/profile.sh -- +# BIN overrides the binary; CFG names a config.toml to use. +set -euo pipefail +out=$1; shift; [ "$1" = "--" ] && shift +bin=${BIN:-target/profiling/toastty} +work=$(mktemp -d) +N=99; while [ -e /tmp/.X$N-lock ] || [ -e /tmp/.X11-unix/X$N ]; do N=$((N+1)); done +Xvfb :$N -screen 0 2560x1440x24 -nolisten tcp >"$work/xvfb.log" 2>&1 & +xvfb=$! +trap 'kill $xvfb 2>/dev/null; rm -rf "$work"' EXIT +for i in $(seq 50); do [ -e /tmp/.X11-unix/X$N ] && break; sleep 0.1; done +mkdir -p "$work/cfg/toastty" +if [ -n "${CFG:-}" ]; then cp "$CFG" "$work/cfg/toastty/config.toml"; else "$bin" --print-default-config >"$work/cfg/toastty/config.toml"; fi +env -u WAYLAND_DISPLAY DISPLAY=":$N" MESA_VK_WSI_DEBUG=sw XDG_CONFIG_HOME="$work/cfg" \ + HISTFILE=/dev/null RUST_LOG=warn TOASTTY_STATS="$out.stats.json" TOASTTY_PPROF="$out" \ + "$bin" -- "$@" diff --git a/bench/run.py b/bench/run.py new file mode 100755 index 00000000..1083f077 --- /dev/null +++ b/bench/run.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +"""toastty benchmark harness. + +Runs the release binary on a private Xvfb display (see AGENTS.md, +"Running toastty on a virtual screen") and records, per scenario: + +- throughput (MB/s), frames rendered and FPS while catting a fixed file; +- CPU time (user + sys) of the toastty process; +- peak and final RSS; +- frame time (CPU time spent in `render_term`, which includes present); +- output→frame and key→frame latency; +- idle CPU % with and without a blinking cursor; +- startup time to the first frame. + +The numbers come from toastty's own `TOASTTY_STATS=` report +(`crates/toastty/src/perf_stats.rs`) plus /proc sampling for the idle +scenarios. + + bench/run.py --bin target/release/toastty --out results.json + bench/run.py --bin ... --only throughput_ascii,idle_blink + bench/compare.py base.json new.json + +Only the X11 path with software presentation (MESA_VK_WSI_DEBUG=sw) is +exercised, so absolute FPS differs from a real compositor; compare runs +against each other, not against other terminals. +""" + +from __future__ import annotations + +import argparse +import json +import os +import random +import shutil +import signal +import statistics +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +CLK_TCK = os.sysconf("SC_CLK_TCK") +SCREEN = (2560, 1440) + +# -------------------------------------------------------------------------- +# Workloads. Deterministic (seeded) and cached under the work dir. +# -------------------------------------------------------------------------- + +WORDS = ( + "lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod " + "tempor incididunt ut labore et dolore magna aliqua enim ad minim veniam " + "quis nostrud exercitation ullamco laboris nisi aliquip ex ea commodo " + "consequat duis aute irure in reprehenderit voluptate velit esse cillum " + "fugiat nulla pariatur excepteur sint occaecat cupidatat non proident" +).split() + + +def _repeat_to(chunk: bytes, size: int) -> bytes: + n = size // len(chunk) + 1 + return (chunk * n)[:size] + + +def gen_ascii(rng: random.Random, size: int) -> bytes: + lines = [] + total = 0 + while total < 4 << 20: + width = rng.randint(20, 160) + line = [] + w = 0 + while w < width: + word = rng.choice(WORDS) + line.append(word) + w += len(word) + 1 + s = " ".join(line) + "\n" + lines.append(s) + total += len(s) + return _repeat_to("".join(lines).encode(), size) + + +def gen_sgr(rng: random.Random, size: int) -> bytes: + out = [] + total = 0 + while total < 4 << 20: + parts = [] + for _ in range(rng.randint(4, 20)): + word = rng.choice(WORDS) + k = rng.random() + if k < 0.4: + parts.append(f"\x1b[38;5;{rng.randrange(256)}m{word}") + elif k < 0.7: + r, g, b = (rng.randrange(256) for _ in range(3)) + parts.append(f"\x1b[38;2;{r};{g};{b}m{word}") + elif k < 0.85: + parts.append(f"\x1b[1;4;48;5;{rng.randrange(256)}m{word}\x1b[22;24;49m") + else: + parts.append(word) + s = " ".join(parts) + "\x1b[0m\n" + out.append(s) + total += len(s) + return _repeat_to("".join(out).encode(), size) + + +def gen_unicode(rng: random.Random, size: int) -> bytes: + cjk = [chr(c) for c in range(0x4E00, 0x4E00 + 400)] + greek = [chr(c) for c in range(0x3B1, 0x3C9)] + emoji = ["😀", "🚀", "🎉", "👍", "🔥", "✨", "🍕", "🐍", "🦀", "👨‍👩‍👧"] + combining = ["é", "ä", "õ", "ñ"] + out = [] + total = 0 + while total < 2 << 20: + parts = [] + for _ in range(rng.randint(5, 25)): + k = rng.random() + if k < 0.35: + parts.append("".join(rng.choice(cjk) for _ in range(rng.randint(1, 6)))) + elif k < 0.5: + parts.append("".join(rng.choice(greek) for _ in range(rng.randint(2, 8)))) + elif k < 0.6: + parts.append(rng.choice(emoji)) + elif k < 0.7: + parts.append(rng.choice(combining)) + else: + parts.append(rng.choice(WORDS)) + s = " ".join(parts) + "\n" + out.append(s) + total += len(s.encode()) + return _repeat_to("".join(out).encode(), size) + + +def gen_dense(rng: random.Random, cols: int, rows: int, frames: int) -> bytes: + """vtebench `dense_cells`-like: every cell gets a new fg/bg and glyph, + one synchronized-output frame at a time.""" + chars = [chr(c) for c in range(0x21, 0x7F)] + out = [] + for f in range(frames): + buf = ["\x1b[?2026h\x1b[H"] + for r in range(rows): + buf.append(f"\x1b[{r + 1};1H") + for c in range(cols): + fg = (r * 7 + c + f) % 256 + bg = (r + c * 3 + f * 5) % 256 + buf.append(f"\x1b[38;5;{fg};48;5;{bg}m{chars[(r + c + f) % len(chars)]}") + buf.append("\x1b[0m\x1b[?2026l") + out.append("".join(buf)) + return "".join(out).encode() + + +def gen_scroll(lines: int) -> bytes: + return "".join(f"{i}\n" for i in range(1, lines + 1)).encode() + + +def gen_wide_lines(rng: random.Random, cols: int, lines: int) -> bytes: + """Full-width coloured lines, to fill the scrollback with real cells.""" + out = [] + for _ in range(lines): + parts = [] + w = 0 + while w < cols - 12: + word = rng.choice(WORDS) + parts.append(f"\x1b[38;5;{rng.randrange(256)}m{word}") + w += len(word) + 1 + out.append(" ".join(parts) + "\x1b[0m\n") + return "".join(out).encode() + + +def workloads(work: Path) -> dict[str, Path]: + d = work / "data" + d.mkdir(parents=True, exist_ok=True) + specs = { + "ascii.txt": lambda: gen_ascii(random.Random(1), 64 << 20), + "sgr.txt": lambda: gen_sgr(random.Random(2), 32 << 20), + "unicode.txt": lambda: gen_unicode(random.Random(3), 16 << 20), + "dense.txt": lambda: gen_dense(random.Random(4), 128, 42, 400), + "dense_big.txt": lambda: gen_dense(random.Random(5), 256, 73, 150), + "scroll.txt": lambda: gen_scroll(3_000_000), + "scrollback.txt": lambda: gen_wide_lines(random.Random(6), 128, 30_000), + } + paths = {} + for name, make in specs.items(): + p = d / name + if not p.exists(): + print(f" generating {name}", file=sys.stderr) + p.write_bytes(make()) + paths[name] = p + return paths + + +# -------------------------------------------------------------------------- +# Virtual display + toastty process control. +# -------------------------------------------------------------------------- + + +class Display: + """A private Xvfb on the first free display number >= 99.""" + + def __init__(self, work: Path): + n = 99 + while Path(f"/tmp/.X{n}-lock").exists() or Path(f"/tmp/.X11-unix/X{n}").exists(): + n += 1 + self.name = f":{n}" + self.log = open(work / "xvfb.log", "wb") + self.proc = subprocess.Popen( + ["Xvfb", self.name, "-screen", "0", f"{SCREEN[0]}x{SCREEN[1]}x24", "-nolisten", "tcp"], + stdout=self.log, + stderr=subprocess.STDOUT, + ) + for _ in range(100): + if Path(f"/tmp/.X11-unix/X{n}").exists(): + break + time.sleep(0.05) + else: + raise RuntimeError("Xvfb did not start") + + def xdotool(self, *args: str) -> str: + env = dict(os.environ, DISPLAY=self.name) + return subprocess.run( + ["xdotool", *args], env=env, capture_output=True, text=True, check=False + ).stdout.strip() + + def close(self) -> None: + if self.proc.poll() is None: + self.proc.send_signal(signal.SIGTERM) + try: + self.proc.wait(5) + except subprocess.TimeoutExpired: + self.proc.kill() + self.proc.wait() + self.log.close() + + +def write_config(work: Path, binary: Path, *, width: int, height: int, blink: bool) -> Path: + cfg_home = work / f"cfg_{width}x{height}_{'blink' if blink else 'steady'}" + (cfg_home / "toastty").mkdir(parents=True, exist_ok=True) + text = subprocess.run( + [str(binary), "--print-default-config"], capture_output=True, text=True, check=True + ).stdout + out = [] + section = None + for line in text.splitlines(): + s = line.strip() + if s.startswith("["): + section = s + if section == "[window]" and s.startswith("width ="): + line = f"width = {width}" + elif section == "[window]" and s.startswith("height ="): + line = f"height = {height}" + elif section == "[cursor]" and s.startswith("blink ="): + line = f"blink = {'true' if blink else 'false'}" + out.append(line) + (cfg_home / "toastty" / "config.toml").write_text("\n".join(out) + "\n") + return cfg_home + + +class Toastty: + def __init__(self, display: Display, binary: Path, cfg_home: Path, work: Path, cmd: list[str]): + self.stats_path = work / f"stats_{time.monotonic_ns()}.json" + env = { + k: v + for k, v in os.environ.items() + if k not in ("WAYLAND_DISPLAY", "TOASTTY_DEBUG", "TOASTTY_PTY_LOG") + } + env.update( + DISPLAY=display.name, + MESA_VK_WSI_DEBUG="sw", + XDG_CONFIG_HOME=str(cfg_home), + HISTFILE="/dev/null", + TOASTTY_STATS=str(self.stats_path), + RUST_LOG="warn", + ) + self.log = open(work / "toastty.log", "ab") + self.launch_ns = time.time_ns() + self.proc = subprocess.Popen( + [str(binary), "--", *cmd], env=env, stdout=self.log, stderr=subprocess.STDOUT + ) + self.display = display + + def window(self, timeout: float = 10.0) -> str: + end = time.monotonic() + timeout + while time.monotonic() < end: + w = self.display.xdotool("search", "--class", "toastty") + if w: + return w.splitlines()[0] + time.sleep(0.05) + raise RuntimeError("toastty window did not appear") + + def wait(self, timeout: float) -> dict: + try: + self.proc.wait(timeout) + except subprocess.TimeoutExpired: + self.kill() + raise RuntimeError(f"toastty did not exit within {timeout}s") + self.log.close() + if not self.stats_path.exists(): + raise RuntimeError("toastty exited without writing TOASTTY_STATS") + stats = json.loads(self.stats_path.read_text()) + self.stats_path.unlink() + stats["launch_ns"] = self.launch_ns + return stats + + def proc_sample(self) -> tuple[int, int]: + """(utime+stime ticks, VmRSS kB) of the toastty process.""" + pid = self.proc.pid + stat = Path(f"/proc/{pid}/stat").read_text() + f = stat.rsplit(")", 1)[1].split() + ticks = int(f[11]) + int(f[12]) + rss = 0 + for line in Path(f"/proc/{pid}/status").read_text().splitlines(): + if line.startswith("VmRSS:"): + rss = int(line.split()[1]) + return ticks, rss + + def kill(self) -> None: + if self.proc.poll() is None: + self.proc.send_signal(signal.SIGTERM) + try: + self.proc.wait(5) + except subprocess.TimeoutExpired: + self.proc.kill() + self.proc.wait() + if not self.log.closed: + self.log.close() + + +# -------------------------------------------------------------------------- +# Metrics helpers. +# -------------------------------------------------------------------------- + + +def pct(v: list[float], p: float) -> float: + if not v: + return float("nan") + s = sorted(v) + k = min(len(s) - 1, max(0, round(p / 100 * (len(s) - 1)))) + return s[k] + + +def summarize_run(st: dict) -> dict: + active_us = max(1, st["elapsed_us"] - max(0, st["first_bytes_us"])) + frame_ms = [x / 1000 for x in st["frame_us"]] + return { + "active_s": active_us / 1e6, + "mb_per_s": st["pty_bytes"] / active_us, # bytes/us == MB/s + "frames": st["frames"], + "fps": st["frames"] / (active_us / 1e6), + "cpu_s": (st["utime_ticks"] + st["stime_ticks"]) / CLK_TCK, + "parse_s": st["parse_us"] / 1e6, + "render_s": sum(st["frame_us"]) / 1e6, + "peak_rss_mb": st["vm_hwm_kb"] / 1024, + "final_rss_mb": st["vm_rss_kb"] / 1024, + "frame_ms_p50": pct(frame_ms, 50), + "frame_ms_p99": pct(frame_ms, 99), + "pty_events": st["pty_events"], + "first_frame_ms": st["first_frame_us"] / 1000, + "launch_to_frame_ms": ( + st["start_unix_ns"] + st["first_frame_us"] * 1000 - st["launch_ns"] + ) + / 1e6, + "output_latency_ms_p50": pct([x / 1000 for x in st["output_latency_us"]], 50), + "output_latency_ms_p99": pct([x / 1000 for x in st["output_latency_us"]], 99), + "key_latency_ms_p50": pct([x / 1000 for x in st["key_latency_us"]], 50), + "key_latency_ms_p99": pct([x / 1000 for x in st["key_latency_us"]], 99), + "key_samples": len(st["key_latency_us"]), + } + + +def median_runs(runs: list[dict]) -> dict: + out = {} + for k in runs[0]: + vals = [r[k] for r in runs] + out[k] = statistics.median(vals) + out["runs"] = len(runs) + return out + + +# -------------------------------------------------------------------------- +# Scenarios. +# -------------------------------------------------------------------------- + + +class Bench: + def __init__(self, binary: Path, work: Path, reps: int): + self.binary = binary + self.work = work + self.reps = reps + self.data = workloads(work) + self.display = Display(work) + self.cfg = {} + + def config(self, width=1280, height=800, blink=True) -> Path: + key = (width, height, blink) + if key not in self.cfg: + self.cfg[key] = write_config(self.work, self.binary, width=width, height=height, blink=blink) + return self.cfg[key] + + def launch(self, cmd: list[str], **cfg) -> Toastty: + return Toastty(self.display, self.binary, self.config(**cfg), self.work, cmd) + + def cat(self, name: str, **cfg) -> dict: + runs = [] + for _ in range(self.reps): + t = self.launch(["cat", str(self.data[name])], **cfg) + runs.append(summarize_run(t.wait(300))) + return median_runs(runs) + + # Each scenario returns a flat dict of metrics. + + def throughput_ascii(self): + return self.cat("ascii.txt") + + def throughput_sgr(self): + return self.cat("sgr.txt") + + def throughput_unicode(self): + return self.cat("unicode.txt") + + def throughput_scroll(self): + return self.cat("scroll.txt") + + def dense_cells(self): + return self.cat("dense.txt") + + def dense_cells_big(self): + return self.cat("dense_big.txt", width=2560, height=1400) + + def scrollback_memory(self): + return self.cat("scrollback.txt") + + def startup(self): + runs = [] + for _ in range(max(3, self.reps)): + t = self.launch(["sleep", "1.5"]) + runs.append(summarize_run(t.wait(30))) + r = median_runs(runs) + return { + "first_frame_ms": r["first_frame_ms"], + "launch_to_frame_ms": r["launch_to_frame_ms"], + "idle_rss_mb": r["final_rss_mb"], + "peak_rss_mb": r["peak_rss_mb"], + "cpu_s": r["cpu_s"], + "runs": r["runs"], + } + + def _idle(self, blink: bool, seconds: float = 10.0): + t = self.launch(["bash", "--norc", "--noprofile"], blink=blink) + try: + t.window() + time.sleep(3.0) + t0, _ = t.proc_sample() + m0 = time.monotonic() + rss = [] + while time.monotonic() - m0 < seconds: + time.sleep(0.25) + rss.append(t.proc_sample()[1]) + t1, _ = t.proc_sample() + dt = time.monotonic() - m0 + finally: + t.kill() + return { + "cpu_pct": 100 * (t1 - t0) / CLK_TCK / dt, + "rss_mb": statistics.median(rss) / 1024, + } + + def idle_blink(self): + return self._idle(True) + + def idle_steady(self): + return self._idle(False) + + def key_latency(self): + t = self.launch(["cat"], blink=False) + try: + w = t.window() + self.display.xdotool("windowfocus", "--sync", w) + time.sleep(1.0) + text = "the quick brown fox jumps over the lazy dog " * 3 + for ch in text: + key = "space" if ch == " " else ch + self.display.xdotool("key", key) + time.sleep(0.04) + self.display.xdotool("key", "Return") + time.sleep(0.2) + self.display.xdotool("key", "ctrl+d") + r = summarize_run(t.wait(30)) + finally: + t.kill() + return { + k: r[k] for k in ("key_latency_ms_p50", "key_latency_ms_p99", "key_samples") + } + + def _scroll(self, **cfg): + lines = 20000 + t = self.launch(["sh", "-c", f"seq 1 {lines}; sleep 9"], blink=False, **cfg) + try: + w = t.window() + self.display.xdotool("windowfocus", "--sync", w) + time.sleep(2.0) + self.display.xdotool("mousemove", "--window", w, "300", "300") + # ~3 s of continuous wheel up, then down. + for button in ("4", "5"): + self.display.xdotool("click", "--repeat", "50", "--delay", "30", button) + st = t.wait(30) + finally: + t.kill() + # Skip the frames that rendered the `seq` output itself: keep only + # the last N frames, those during the wheel phase. + frame_ms = [x / 1000 for x in st["frame_us"]] + tail = frame_ms[-300:] if len(frame_ms) > 300 else frame_ms + return { + "frames": st["frames"], + "frame_ms_p50": pct(tail, 50), + "frame_ms_p99": pct(tail, 99), + "frame_ms_mean": statistics.fmean(tail) if tail else float("nan"), + "cpu_s": (st["utime_ticks"] + st["stime_ticks"]) / CLK_TCK, + } + + def scroll_wheel(self): + return self._scroll() + + def scroll_wheel_big(self): + return self._scroll(width=2560, height=1400) + + def close(self): + self.display.close() + + +SCENARIOS = [ + "startup", + "idle_blink", + "idle_steady", + "throughput_ascii", + "throughput_sgr", + "throughput_unicode", + "throughput_scroll", + "dense_cells", + "dense_cells_big", + "scrollback_memory", + "key_latency", + "scroll_wheel", + "scroll_wheel_big", +] + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--bin", required=True, type=Path) + ap.add_argument("--out", required=True, type=Path) + ap.add_argument("--work", type=Path, default=None, help="scratch dir (workloads are cached here)") + ap.add_argument("--reps", type=int, default=3) + ap.add_argument("--only", default="", help="comma-separated scenario names") + args = ap.parse_args() + + binary = args.bin.resolve() + work = args.work or Path(tempfile.gettempdir()) / "toastty-bench" + work.mkdir(parents=True, exist_ok=True) + only = [s for s in args.only.split(",") if s] or SCENARIOS + for s in only: + if s not in SCENARIOS: + ap.error(f"unknown scenario {s}") + + results = { + "binary": str(binary), + "binary_size_mb": binary.stat().st_size / (1 << 20), + "scenarios": {}, + } + bench = Bench(binary, work, args.reps) + try: + for name in only: + print(f"running {name} ...", file=sys.stderr, flush=True) + t0 = time.monotonic() + try: + results["scenarios"][name] = getattr(bench, name)() + except Exception as e: # keep going; record the failure + results["scenarios"][name] = {"error": str(e)} + print( + f" {name}: {json.dumps(results['scenarios'][name])} ({time.monotonic() - t0:.1f}s)", + file=sys.stderr, + ) + finally: + bench.close() + args.out.write_text(json.dumps(results, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/crates/toastty/Cargo.toml b/crates/toastty/Cargo.toml index c2fd67c5..564573b8 100644 --- a/crates/toastty/Cargo.toml +++ b/crates/toastty/Cargo.toml @@ -31,6 +31,9 @@ tracing.workspace = true thiserror.workspace = true tracing-subscriber.workspace = true webbrowser = "1" +# In-process sampling profiler for `bench/` (SIGPROF; needs no perf +# permissions). Off by default: `--features pprof`, then `TOASTTY_PPROF=`. +pprof = { version = "0.15", optional = true, default-features = false, features = ["large-depth"] } toastty-config.workspace = true toastty-io.workspace = true @@ -42,6 +45,9 @@ toastty-render.workspace = true toastty-term.workspace = true toastty-window.workspace = true +[features] +pprof = ["dep:pprof"] + [dev-dependencies] toastty-graphics.workspace = true diff --git a/crates/toastty/examples/parse_bench.rs b/crates/toastty/examples/parse_bench.rs new file mode 100644 index 00000000..0b548a62 --- /dev/null +++ b/crates/toastty/examples/parse_bench.rs @@ -0,0 +1,52 @@ +//! Headless parser + terminal-state throughput, no window or GPU. +//! +//! cargo run --release -p toastty --example parse_bench -- FILE... [--cols 128 --rows 42 --reps 3] +//! +//! Feeds each file through `Parser::advance` into a `Term` in 64 KiB +//! chunks (the shape of real PTY reads) and prints MB/s. With +//! `--features pprof` and `TOASTTY_PPROF=`, also writes a folded +//! CPU profile (see `bench/folded.py`). + +use std::time::Instant; + +use toastty_parser::Parser; +use toastty_term::Term; + +fn main() { + let mut files = Vec::new(); + let (mut cols, mut rows, mut reps, mut scrollback) = (128u16, 42u16, 3u32, 10_000u16); + let mut args = std::env::args().skip(1); + while let Some(a) = args.next() { + let mut num = || args.next().and_then(|v| v.parse().ok()).expect("number"); + match a.as_str() { + "--cols" => cols = num(), + "--rows" => rows = num(), + "--reps" => reps = u32::from(num()), + "--scrollback" => scrollback = num(), + _ => files.push(a), + } + } + toastty::perf_stats::start_profiler(); + for f in &files { + let data = std::fs::read(f).expect("read workload"); + let mut best = f64::MAX; + for _ in 0..reps { + let mut term = Term::new(rows, cols, scrollback); + let mut parser = Parser::new(); + let t = Instant::now(); + for chunk in data.chunks(64 << 10) { + parser.advance(&mut term, chunk); + term.drain_pty_replies(); + } + best = best.min(t.elapsed().as_secs_f64()); + } + let name = std::path::Path::new(f).file_name().unwrap().to_string_lossy(); + println!( + "{name:<16} {:>8.1} MB/s ({:.3} s for {:.1} MB, best of {reps})", + data.len() as f64 / best / 1e6, + best, + data.len() as f64 / 1e6 + ); + } + toastty::perf_stats::finish_profiler(); +} diff --git a/crates/toastty/src/lib.rs b/crates/toastty/src/lib.rs index f5c7f2b1..29cff98c 100644 --- a/crates/toastty/src/lib.rs +++ b/crates/toastty/src/lib.rs @@ -23,6 +23,7 @@ pub mod markdown; pub mod md_cmd; pub mod mouse; pub mod paste; +pub mod perf_stats; pub mod pty_log; pub mod shell; pub mod theme_bridge; diff --git a/crates/toastty/src/main.rs b/crates/toastty/src/main.rs index 0f99505d..774c4ceb 100644 --- a/crates/toastty/src/main.rs +++ b/crates/toastty/src/main.rs @@ -48,6 +48,7 @@ use toastty::mouse::{ protocol_wants_event, }; use toastty::paste::wrap_for_paste; +use toastty::perf_stats::PerfStats; use toastty::pty_log::{Direction, PtyLogger}; use toastty::shell::resolve_command; use toastty::theme_bridge::{ @@ -229,6 +230,7 @@ fn main() -> Result<()> { .init(); info!("toastty {} starting", env!("CARGO_PKG_VERSION")); + toastty::perf_stats::start_profiler(); let (config, source, load_err) = Config::load_default_with_error(); info!("config source: {source}"); @@ -364,6 +366,8 @@ struct Toastty { /// Optional bidirectional PTY-byte logger, enabled by /// `TOASTTY_PTY_LOG=`. No-op when the env var is unset. pty_log: PtyLogger, + /// Benchmark counters, enabled by `TOASTTY_STATS=`. + stats: PerfStats, /// Sub-row pixel accumulator for the alt-screen-arrow translation /// path. macOS trackpad inertial frames arrive as small per-frame /// pixel deltas; we accumulate them here and emit one ↑/↓ arrow @@ -511,6 +515,7 @@ impl Toastty { held_modifier_keys: Vec::new(), middle_paste_consumed: false, pty_log: PtyLogger::from_env(), + stats: PerfStats::from_env(), alt_scroll_pixel_accum: 0.0, mouse_scroll_accum_y: 0.0, mouse_scroll_accum_x: 0.0, @@ -1513,6 +1518,9 @@ impl Toastty { } return self.handle_close_prompt_key(logical, text); } + if state == KeyState::Pressed { + self.stats.key_press(Instant::now()); + } // KBD-2: report a release as the key that was pressed, and keep a // modifier's bit while its other side is held. Before any early // return, so synthetic focus-loss releases are tracked too. @@ -2763,8 +2771,10 @@ impl App for Toastty { self.refresh_rich_chip(); let mut next_blink: Option = None; if let Some(r) = self.renderer.as_mut() { + let frame_start = Instant::now(); match r.render_term(&mut self.term) { Ok(RenderOutcome::Rendered) => { + self.stats.frame(frame_start, Instant::now()); // Consume the per-cell damage signal so the // next render only re-emits cells that // changed. Clear the BSU timeout-flush @@ -2777,6 +2787,7 @@ impl App for Toastty { } } Ok(RenderOutcome::Skipped) => { + self.stats.skipped(); // Frame skipped (pause-gated or surface // hiccup): leave the damage signal and the // BSU force-flushed flag alone so the next @@ -2927,7 +2938,9 @@ impl App for Toastty { } } Event::PtyBytes(bytes) => { + let at = Instant::now(); let fresh_bsu = self.handle_pty_bytes(&bytes); + self.stats.pty_batch(at, bytes.len(), at.elapsed()); // New output may have rewritten or moved the hovered // link; re-resolve it (clears when it's gone). if self.term.link_hover().is_some() { @@ -2954,6 +2967,8 @@ impl App for Toastty { } Event::PtyClosed => { debug!("pty closed; exiting"); + self.stats.write(); + toastty::perf_stats::finish_profiler(); if let Some(pty) = self.pty.as_mut() { let _ = pty.wait(); } diff --git a/crates/toastty/src/perf_stats.rs b/crates/toastty/src/perf_stats.rs new file mode 100644 index 00000000..68969739 --- /dev/null +++ b/crates/toastty/src/perf_stats.rs @@ -0,0 +1,234 @@ +//! Performance counters for benchmarking, enabled by +//! `TOASTTY_STATS=`. +//! +//! When the variable is unset every hook is a branch on `None` and +//! nothing is recorded. When it is set, the binary counts frames, PTY +//! bytes and the time spent parsing and rendering, plus two latencies: +//! +//! - **output → frame**: from the first PTY batch after a frame to the +//! end of the frame that shows it; +//! - **key → frame**: from a key press to the end of the first frame +//! that follows the app's reply (the echo). +//! +//! [`PerfStats::write`] dumps everything as one JSON object when the +//! PTY closes. `bench/run.py` reads it. + +use std::fmt::Write as _; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +#[derive(Debug)] +struct Inner { + path: std::path::PathBuf, + start: Instant, + start_unix: Duration, + first_frame: Option, + first_bytes: Option, + frames: u64, + skipped: u64, + frame_us: Vec, + pty_bytes: u64, + pty_events: u64, + parse: Duration, + output_pending: Option, + output_latency_us: Vec, + key_pending: Option, + key_echoed: bool, + key_latency_us: Vec, +} + +/// See the module docs. `Default` is the disabled state. +#[derive(Debug, Default)] +pub struct PerfStats(Option>); + +impl PerfStats { + /// Enabled when `TOASTTY_STATS` names an output path. + pub fn from_env() -> Self { + let Some(path) = std::env::var_os("TOASTTY_STATS") else { + return Self(None); + }; + Self(Some(Box::new(Inner { + path: path.into(), + start: Instant::now(), + start_unix: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default(), + first_frame: None, + first_bytes: None, + frames: 0, + skipped: 0, + frame_us: Vec::new(), + pty_bytes: 0, + pty_events: 0, + parse: Duration::ZERO, + output_pending: None, + output_latency_us: Vec::new(), + key_pending: None, + key_echoed: false, + key_latency_us: Vec::new(), + }))) + } + + pub fn enabled(&self) -> bool { + self.0.is_some() + } + + /// A PTY batch of `len` bytes arrived at `at` and took `parse` to + /// handle. + pub fn pty_batch(&mut self, at: Instant, len: usize, parse: Duration) { + let Some(s) = self.0.as_mut() else { return }; + s.first_bytes.get_or_insert(at - s.start); + s.pty_bytes += len as u64; + s.pty_events += 1; + s.parse += parse; + s.output_pending.get_or_insert(at); + if s.key_pending.is_some() { + s.key_echoed = true; + } + } + + pub fn key_press(&mut self, at: Instant) { + let Some(s) = self.0.as_mut() else { return }; + if s.key_pending.is_none() { + s.key_pending = Some(at); + s.key_echoed = false; + } + } + + /// A frame was rendered between `start` and `end`. + pub fn frame(&mut self, start: Instant, end: Instant) { + let Some(s) = self.0.as_mut() else { return }; + s.first_frame.get_or_insert(end - s.start); + s.frames += 1; + s.frame_us.push(micros(end - start)); + if let Some(at) = s.output_pending.take() { + s.output_latency_us.push(micros(end - at)); + } + if s.key_echoed + && let Some(at) = s.key_pending.take() + { + s.key_latency_us.push(micros(end - at)); + s.key_echoed = false; + } + } + + pub fn skipped(&mut self) { + if let Some(s) = self.0.as_mut() { + s.skipped += 1; + } + } + + /// Write the JSON report. Called once, when the PTY closes. + pub fn write(&mut self) { + let Some(s) = self.0.as_mut() else { return }; + let end = s.start.elapsed(); + let (utime, stime) = self_cpu_ticks(); + let mut o = String::new(); + let _ = write!( + o, + "{{\"start_unix_ns\":{},\"elapsed_us\":{},\"first_frame_us\":{},\"first_bytes_us\":{},\ + \"frames\":{},\"skipped\":{},\"pty_bytes\":{},\"pty_events\":{},\"parse_us\":{},\ + \"utime_ticks\":{},\"stime_ticks\":{},\"vm_hwm_kb\":{},\"vm_rss_kb\":{},", + s.start_unix.as_nanos(), + micros(end), + s.first_frame.map_or(-1, |d| i64::from(micros(d))), + s.first_bytes.map_or(-1, |d| i64::from(micros(d))), + s.frames, + s.skipped, + s.pty_bytes, + s.pty_events, + s.parse.as_micros(), + utime, + stime, + status_kb("VmHWM:"), + status_kb("VmRSS:"), + ); + push_list(&mut o, "frame_us", &s.frame_us); + o.push(','); + push_list(&mut o, "output_latency_us", &s.output_latency_us); + o.push(','); + push_list(&mut o, "key_latency_us", &s.key_latency_us); + o.push('}'); + if let Err(e) = std::fs::write(&s.path, o) { + tracing::warn!("TOASTTY_STATS: write {}: {e}", s.path.display()); + } + } +} + +fn micros(d: Duration) -> u32 { + u32::try_from(d.as_micros()).unwrap_or(u32::MAX) +} + +fn push_list(o: &mut String, name: &str, v: &[u32]) { + let _ = write!(o, "\"{name}\":["); + for (i, x) in v.iter().enumerate() { + if i > 0 { + o.push(','); + } + let _ = write!(o, "{x}"); + } + o.push(']'); +} + +/// `utime` and `stime` of this process (all threads), in clock ticks. +fn self_cpu_ticks() -> (u64, u64) { + let Ok(stat) = std::fs::read_to_string("/proc/self/stat") else { + return (0, 0); + }; + // Fields after the `)` that closes `comm`: state is field 3, utime 14. + let rest = stat.rsplit_once(')').map_or("", |(_, r)| r); + let f: Vec<&str> = rest.split_whitespace().collect(); + let get = |i: usize| f.get(i).and_then(|x| x.parse().ok()).unwrap_or(0); + (get(11), get(12)) +} + +fn status_kb(key: &str) -> u64 { + std::fs::read_to_string("/proc/self/status") + .ok() + .and_then(|s| { + s.lines() + .find(|l| l.starts_with(key)) + .and_then(|l| l.split_whitespace().nth(1)?.parse().ok()) + }) + .unwrap_or(0) +} + +#[cfg(feature = "pprof")] +thread_local! { + static PROFILER: std::cell::RefCell)>> = + const { std::cell::RefCell::new(None) }; +} + +/// Start the in-process sampling profiler when built with +/// `--features pprof` and `TOASTTY_PPROF=` is set. The folded +/// stacks (`thread;root;...;leaf count`) are written by +/// [`finish_profiler`]. +pub fn start_profiler() { + #[cfg(feature = "pprof")] + if let Some(path) = std::env::var_os("TOASTTY_PPROF") { + match pprof::ProfilerGuardBuilder::default().frequency(1999).build() { + Ok(g) => PROFILER.with(|p| *p.borrow_mut() = Some((path.into(), g))), + Err(e) => tracing::warn!("TOASTTY_PPROF: {e}"), + } + } +} + +/// Write the profile started by [`start_profiler`], if any. +pub fn finish_profiler() { + #[cfg(feature = "pprof")] + PROFILER.with(|p| { + let Some((path, guard)) = p.borrow_mut().take() else { + return; + }; + let Ok(report) = guard.report().build() else { return }; + let mut o = String::new(); + for (frames, count) in &report.data { + o.push_str(&frames.thread_name); + for frame in frames.frames.iter().rev() { + for sym in frame.iter().rev() { + o.push(';'); + o.push_str(&sym.name().replace(';', ":")); + } + } + let _ = writeln!(o, " {count}"); + } + let _ = std::fs::write(path, o); + }); +} From bf62262e0ce5eca030d7fe9c79c8bd08d44dc0ca Mon Sep 17 00:00:00 2001 From: EDToaster Date: Thu, 22 Oct 2026 01:27:12 -0700 Subject: [PATCH 2/4] bench: compare.py --- bench/compare.py | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100755 bench/compare.py diff --git a/bench/compare.py b/bench/compare.py new file mode 100755 index 00000000..016988c4 --- /dev/null +++ b/bench/compare.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Compare two bench/run.py result files as a Markdown table. + + bench/compare.py base.json new.json [more.json ...] + +The change column is relative to the first file. Metrics where lower is +better (time, CPU, memory, latency) and higher is better (throughput, +FPS) are marked so a regression stands out. +""" + +import json +import math +import sys + +# metric -> (label, higher_is_better, format) +METRICS = { + "mb_per_s": ("throughput MB/s", True, "{:.1f}"), + "fps": ("FPS during run", True, "{:.1f}"), + "frames": ("frames", True, "{:.0f}"), + "cpu_s": ("CPU s", False, "{:.2f}"), + "active_s": ("wall s", False, "{:.2f}"), + "peak_rss_mb": ("peak RSS MB", False, "{:.1f}"), + "final_rss_mb": ("final RSS MB", False, "{:.1f}"), + "frame_ms_p50": ("frame ms p50", False, "{:.2f}"), + "frame_ms_p99": ("frame ms p99", False, "{:.2f}"), + "frame_ms_mean": ("frame ms mean", False, "{:.2f}"), + "output_latency_ms_p50": ("output→frame ms p50", False, "{:.2f}"), + "output_latency_ms_p99": ("output→frame ms p99", False, "{:.2f}"), + "key_latency_ms_p50": ("key→frame ms p50", False, "{:.2f}"), + "key_latency_ms_p99": ("key→frame ms p99", False, "{:.2f}"), + "first_frame_ms": ("first frame ms", False, "{:.1f}"), + "launch_to_frame_ms": ("launch→first frame ms", False, "{:.1f}"), + "idle_rss_mb": ("idle RSS MB", False, "{:.1f}"), + "cpu_pct": ("CPU %", False, "{:.2f}"), + "rss_mb": ("RSS MB", False, "{:.1f}"), +} + +# Per-scenario metric selection (others are noise for that scenario). +PICK = { + "startup": ["launch_to_frame_ms", "first_frame_ms", "idle_rss_mb"], + "idle_blink": ["cpu_pct", "rss_mb"], + "idle_steady": ["cpu_pct", "rss_mb"], + "key_latency": ["key_latency_ms_p50", "key_latency_ms_p99"], + "scroll_wheel": ["frame_ms_p50", "frame_ms_p99", "cpu_s"], + "scroll_wheel_big": ["frame_ms_p50", "frame_ms_p99", "cpu_s"], + "scrollback_memory": ["final_rss_mb", "peak_rss_mb", "mb_per_s"], +} +DEFAULT = ["mb_per_s", "fps", "frame_ms_p50", "frame_ms_p99", "cpu_s", "peak_rss_mb", + "output_latency_ms_p50", "output_latency_ms_p99"] + + +def fmt(fmt_s, v): + if v is None or (isinstance(v, float) and math.isnan(v)): + return "–" + return fmt_s.format(v) + + +def main(): + files = sys.argv[1:] + if len(files) < 2: + sys.exit(__doc__) + data = [json.load(open(f)) for f in files] + names = [f.rsplit("/", 1)[-1].removesuffix(".json") for f in files] + head = "| scenario | metric | " + " | ".join(names) + " | change |" + print(head) + print("|" + "---|" * (len(names) + 3)) + print( + "| binary | size MB | " + + " | ".join(f"{d['binary_size_mb']:.1f}" for d in data) + + " | |" + ) + for scen in data[0]["scenarios"]: + for m in PICK.get(scen, DEFAULT): + vals = [d["scenarios"].get(scen, {}).get(m) for d in data] + if all(v is None for v in vals): + continue + label, hib, f = METRICS[m] + base, last = vals[0], vals[-1] + change = "" + if ( + isinstance(base, (int, float)) and isinstance(last, (int, float)) + and not math.isnan(base) and not math.isnan(last) and base != 0 + ): + r = last / base + better = (r > 1) == hib + arrow = "✅" if better and abs(r - 1) > 0.05 else ("⚠️" if abs(r - 1) > 0.05 else "") + change = f"{r:.2f}× {arrow}" + print(f"| {scen} | {label} | " + " | ".join(fmt(f, v) for v in vals) + f" | {change} |") + + +if __name__ == "__main__": + main() From 3a9cecddc850a15548ddbe7f085aeba9891e7edf Mon Sep 17 00:00:00 2001 From: EDToaster Date: Thu, 22 Oct 2026 01:27:51 -0700 Subject: [PATCH 3/4] bench: README --- bench/README.md | 65 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 bench/README.md diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 00000000..845ae891 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,65 @@ +# Benchmarks + +Tools for measuring toastty's throughput, frame rate, CPU, memory and +latency. Everything runs off-screen on a private Xvfb display. The +safety rules in `AGENTS.md` ("Running toastty on a virtual screen") +apply, and the scripts follow them. + +## Full suite + +```sh +cargo build --release +bench/run.py --bin target/release/toastty --out after.json --work /tmp/toastty-bench +bench/compare.py before.json after.json # Markdown table +bench/run.py ... --only dense_cells,idle_blink --reps 5 +``` + +The workloads are generated once (seeded, so they're identical between runs) +and cached in `--work/data`. Every scenario reads toastty's own report, +which the binary writes when `TOASTTY_STATS=` is set +(`crates/toastty/src/perf_stats.rs`). The idle scenarios also sample `/proc`. + +| Scenario | What it does | Key metrics | +|---|---|---| +| `startup` | `toastty -- sleep 1.5` | launch → first frame, idle RSS | +| `idle_blink` / `idle_steady` | bash prompt for 10 s, cursor blinking / steady | CPU %, RSS | +| `throughput_ascii` / `_sgr` / `_unicode` / `_scroll` | `cat` 64 / 32 / 16 / 23 MB | MB/s, frames and FPS while streaming, CPU s, peak RSS | +| `dense_cells` / `_big` | every cell recoloured each frame, one synchronized-output frame at a time (128×42, and 256×73 at 2560×1400) | FPS, frame ms | +| `scrollback_memory` | 30k full-width coloured lines (fills the 10k scrollback) | RSS after | +| `key_latency` | types into `cat` at 25 keys/s | key → frame ms | +| `scroll_wheel` / `_big` | wheel-scrolls 20k lines of history | frame ms p50/p99 | + +"Frame ms" is the CPU time of `Renderer::render_term`, including the +present. Output → frame is measured from the first PTY batch after a frame +to the end of the next frame. + +## Headless parse throughput + +No window or GPU, so it gives quick, low-noise numbers for the parser and the +terminal state: + +```sh +cargo run --release -p toastty --example parse_bench -- /tmp/toastty-bench/data/*.txt +``` + +## CPU profiles + +`perf`/`samply` need `perf_event_paranoid <= 1`. Without it, use the +in-process SIGPROF sampler behind the `pprof` feature: + +```sh +cargo build --profile profiling --features pprof +bench/profile.sh /tmp/p.folded -- cat /tmp/toastty-bench/data/ascii.txt +bench/folded.py /tmp/p.folded --top 40 [--thread toastty] [--focus render_term] +# headless: +cargo build --profile profiling -p toastty --example parse_bench --features pprof +TOASTTY_PPROF=/tmp/p.folded target/profiling/examples/parse_bench FILE +``` + +## Caveats + +- Only the X11 path with software presentation (`MESA_VK_WSI_DEBUG=sw`) is + exercised. Absolute FPS and frame times differ from a real compositor, so + compare runs against each other. +- Run comparisons one after another on a quiet machine. A parallel cargo + build skews every number. From 813cc76a4e3e4425fc24f9370ce36e10a7fbd129 Mon Sep 17 00:00:00 2001 From: EDToaster Date: Thu, 22 Oct 2026 02:13:45 -0700 Subject: [PATCH 4/4] bench: rustfmt + clippy for the stats module and parse_bench --- crates/toastty/examples/parse_bench.rs | 6 +++++- crates/toastty/src/perf_stats.rs | 13 ++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/crates/toastty/examples/parse_bench.rs b/crates/toastty/examples/parse_bench.rs index 0b548a62..726f96b9 100644 --- a/crates/toastty/examples/parse_bench.rs +++ b/crates/toastty/examples/parse_bench.rs @@ -12,6 +12,7 @@ use std::time::Instant; use toastty_parser::Parser; use toastty_term::Term; +#[allow(clippy::cast_precision_loss)] // MB/s for a report fn main() { let mut files = Vec::new(); let (mut cols, mut rows, mut reps, mut scrollback) = (128u16, 42u16, 3u32, 10_000u16); @@ -40,7 +41,10 @@ fn main() { } best = best.min(t.elapsed().as_secs_f64()); } - let name = std::path::Path::new(f).file_name().unwrap().to_string_lossy(); + let name = std::path::Path::new(f) + .file_name() + .unwrap() + .to_string_lossy(); println!( "{name:<16} {:>8.1} MB/s ({:.3} s for {:.1} MB, best of {reps})", data.len() as f64 / best / 1e6, diff --git a/crates/toastty/src/perf_stats.rs b/crates/toastty/src/perf_stats.rs index 68969739..c6ddda1c 100644 --- a/crates/toastty/src/perf_stats.rs +++ b/crates/toastty/src/perf_stats.rs @@ -49,7 +49,9 @@ impl PerfStats { Self(Some(Box::new(Inner { path: path.into(), start: Instant::now(), - start_unix: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default(), + start_unix: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(), first_frame: None, first_bytes: None, frames: 0, @@ -203,7 +205,10 @@ thread_local! { pub fn start_profiler() { #[cfg(feature = "pprof")] if let Some(path) = std::env::var_os("TOASTTY_PPROF") { - match pprof::ProfilerGuardBuilder::default().frequency(1999).build() { + match pprof::ProfilerGuardBuilder::default() + .frequency(1999) + .build() + { Ok(g) => PROFILER.with(|p| *p.borrow_mut() = Some((path.into(), g))), Err(e) => tracing::warn!("TOASTTY_PPROF: {e}"), } @@ -217,7 +222,9 @@ pub fn finish_profiler() { let Some((path, guard)) = p.borrow_mut().take() else { return; }; - let Ok(report) = guard.report().build() else { return }; + let Ok(report) = guard.report().build() else { + return; + }; let mut o = String::new(); for (frames, count) in &report.data { o.push_str(&frames.thread_name);