diff --git a/benchmark/README.md b/benchmark/README.md index 1d8ac6d..3c188b3 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,32 +1,39 @@ -# Daft audio stability benchmark +# Daft 音频稳定性基准测试 -For complete local and multi-node Ray instructions, sizing guidance, metrics, -and extension points, see the [audio benchmark guide](../docs/audio-benchmark.md). +完整的本地与多节点 Ray 操作说明、容量规划、指标口径和扩展点,见 +[音频基准测试指南](../docs/audio-benchmark.md)。 -This package exercises the real audio path without calling a real LLM: +本 package 在不调用真实 LLM 的前提下跑通完整的音频链路: ``` -MinIO -> Daft/Ray download -> SenseVoice ASR -> HTTP mock LLM -> Lance blob v2 +MinIO -> Daft/Ray 下载 -> SenseVoice ASR -> HTTP mock LLM -> Lance blob v2 ``` -It always uses the MinIO credentials and endpoint from the repository `.env`. -It never starts, stops, or deletes the configured MinIO service. Each run writes -under its own `s3://///` prefix. +它始终使用仓库 `.env` 中的 MinIO 凭据与 endpoint,绝不会启动、停止或删除已配置的 +MinIO 服务。每次运行写入各自独立的 `s3://///` 前缀下。 -## Local smoke test +## 本地冒烟测试 -Prerequisites: the configured MinIO is running, the SenseVoice and FSMN VAD -models are cached or downloadable, and `data/audio/` contains seed audio. +前置条件:配置的 MinIO 已在运行,SenseVoice 与 FSMN VAD 模型已缓存或可下载, +且 `data/audio/` 中有种子音频。 ```bash uv run python -m benchmark.audio local-smoke --count 4 ``` -Results are written to `.benchmarks//`, including `metadata.json`, -`summary.json`, `resources.csv`, `report.json`, `report.md`, the Daft physical -plan, and Daft event logs. +如果需要可重复的本地性能 baseline —— 生成 50 条固定 60 秒音频,预热一次模型/缓存, +再用同一份 manifest 跑两轮: -## Separate commands +```bash +uv run python -m benchmark.audio local-baseline +``` + +比较结果写入 `.benchmarks//baseline-summary.{json,md}`。 + +运行产物写入 `.benchmarks//`,包含 `metadata.json`、`summary.json`、 +`resources.csv`、`report.json`、`report.md`、Daft 物理执行计划以及 Daft 事件日志。 + +## 分步命令 ```bash uv run python -m benchmark.audio generate --profile smoke --count 4 @@ -47,6 +54,5 @@ uv run python -m benchmark.audio submit \ --ray-address auto --max-minutes 60 --wait ``` -Use `generate --profile fixed` for controlled fixed-duration data and -`--profile mixed` for a 70% short / 25% medium / 5% long distribution. Data -generation happens before the timed benchmark. +`generate --profile fixed` 用于生成受控的固定时长数据,`--profile mixed` 生成 +70% 短 / 25% 中 / 5% 长 的分布。数据生成发生在计时的基准测试之前。 diff --git a/benchmark/audio/cli.py b/benchmark/audio/cli.py index bdd5764..5df74e5 100644 --- a/benchmark/audio/cli.py +++ b/benchmark/audio/cli.py @@ -8,7 +8,9 @@ from .config import BenchmarkConfig, DEFAULT_RUN_ROOT, new_run_id -def _add_run_arguments(parser: argparse.ArgumentParser, *, ray_default: str = "local") -> None: +def _add_run_arguments( + parser: argparse.ArgumentParser, *, ray_default: str = "local" +) -> None: parser.add_argument("--run-id", default=None) parser.add_argument("--manifest", required=True) parser.add_argument("--lance-uri", required=True) @@ -71,7 +73,9 @@ def _cmd_generate(args) -> None: source_dir=Path(args.source_dir) if args.source_dir else None, seed=args.seed, ) - print(json.dumps({"run_id": run_id, **asdict(result)}, ensure_ascii=False, indent=2)) + print( + json.dumps({"run_id": run_id, **asdict(result)}, ensure_ascii=False, indent=2) + ) def _cmd_run(args) -> None: @@ -87,7 +91,11 @@ def _cmd_run(args) -> None: raise else: report = build_report(cfg.output_dir) - print(json.dumps({"summary": summary, "report": report}, ensure_ascii=False, indent=2)) + print( + json.dumps( + {"summary": summary, "report": report}, ensure_ascii=False, indent=2 + ) + ) def _cmd_submit(args) -> None: @@ -154,7 +162,9 @@ def _cmd_local_smoke(args) -> None: raise report = build_report(cfg.output_dir) if summary.get("rows") != dataset.rows: - raise RuntimeError(f"row count mismatch: manifest={dataset.rows}, lance={summary.get('rows')}") + raise RuntimeError( + f"row count mismatch: manifest={dataset.rows}, lance={summary.get('rows')}" + ) if summary.get("unique_doc_ids") != dataset.rows: raise RuntimeError("output doc_id values are not unique") print(json.dumps(report, ensure_ascii=False, indent=2)) @@ -167,6 +177,178 @@ def _cmd_local_smoke(args) -> None: ray.shutdown() +def _cmd_local_baseline(args) -> None: + import ray + + from .data import generate_dataset + from .mock_llm import start_in_thread + from .pipeline import run_pipeline + from .report import build_baseline_report, build_report + from .storage import preflight + + if args.repeats < 2: + raise ValueError("--repeats must be at least 2") + standard_baseline = ( + 40 <= args.count <= 60 + and args.duration_s == 60.0 + and 0 < args.max_minutes <= 45.0 + ) + if not standard_baseline: + print( + "[warning] non-standard diagnostic baseline: the comparable baseline uses " + "40-60 files, 60s duration, and a per-repeat limit no greater than 45 minutes" + ) + base_run_id = args.run_id or f"baseline-{new_run_id()}" + base_dir = DEFAULT_RUN_ROOT / base_run_id + print(f"[preflight] checking configured MinIO bucket {args.bucket}") + print(json.dumps(preflight(args.bucket), ensure_ascii=False)) + print( + f"[generate] creating {args.count} fixed-duration objects " + f"({args.duration_s:.0f}s each)" + ) + dataset = generate_dataset( + run_id=base_run_id, + bucket=args.bucket, + prefix=args.prefix, + profile="fixed", + count=args.count, + fixed_duration_s=args.duration_s, + source_dir=Path(args.source_dir) if args.source_dir else None, + seed=args.seed, + ) + warmup_dataset = None + if not args.skip_warmup: + print("[generate] creating one warm-up object") + warmup_dataset = generate_dataset( + run_id=f"{base_run_id}-warmup", + bucket=args.bucket, + prefix=args.prefix, + profile="smoke", + count=1, + source_dir=Path(args.source_dir) if args.source_dir else None, + seed=args.seed, + ) + server, thread = start_in_thread(port=0, profile=args.mock_profile) + port = server.server_address[1] + reports = [] + repeat_error: Exception | None = None + try: + if warmup_dataset is not None: + warmup_cfg = BenchmarkConfig( + run_id=f"{base_run_id}-warmup", + manifest_uri=warmup_dataset.manifest_uri, + lance_uri=( + f"s3://{args.bucket}/{args.prefix.strip('/')}/{base_run_id}/warmup.lance" + ), + mock_url=f"http://127.0.0.1:{port}", + ray_address="local", + max_minutes=min(args.max_minutes, 15.0), + asr_actor_cpus=args.asr_actor_cpus, + asr_actor_concurrency=args.asr_actor_concurrency, + asr_batch_size=args.asr_batch_size, + llm_concurrency=args.llm_concurrency, + llm_timeout_s=15.0, + sample_interval_s=args.sample_interval_s, + num_partitions=1, + run_dir=str(base_dir / "warmup"), + ) + print(f"[warmup] {warmup_dataset.manifest_uri} -> {warmup_cfg.lance_uri}") + warmup_summary = run_pipeline(warmup_cfg) + build_report(warmup_cfg.output_dir) + if warmup_summary.get("rows") != 1 or warmup_summary.get( + "status_counts" + ) != {"ok": 1}: + raise RuntimeError(f"warm-up validation failed: {warmup_summary}") + for repeat in range(1, args.repeats + 1): + repeat_name = f"repeat-{repeat:02d}" + repeat_run_id = f"{base_run_id}-{repeat_name}" + lance_uri = ( + f"s3://{args.bucket}/{args.prefix.strip('/')}/{base_run_id}/" + f"output-{repeat_name}.lance" + ) + cfg = BenchmarkConfig( + run_id=repeat_run_id, + manifest_uri=dataset.manifest_uri, + lance_uri=lance_uri, + mock_url=f"http://127.0.0.1:{port}", + ray_address="local", + max_minutes=args.max_minutes, + asr_actor_cpus=args.asr_actor_cpus, + asr_actor_concurrency=args.asr_actor_concurrency, + asr_batch_size=args.asr_batch_size, + llm_concurrency=args.llm_concurrency, + llm_timeout_s=15.0, + sample_interval_s=args.sample_interval_s, + num_partitions=args.num_partitions, + run_dir=str(base_dir / repeat_name), + ) + print( + f"[run {repeat}/{args.repeats}] {dataset.manifest_uri} -> {lance_uri}" + ) + try: + summary = run_pipeline(cfg) + except Exception as exc: + if (cfg.output_dir / "summary.json").exists(): + reports.append(build_report(cfg.output_dir)) + else: + reports.append( + { + "run_id": repeat_run_id, + "state": "failed", + "summary": {"elapsed_s": 0.0, "error": repr(exc)}, + } + ) + repeat_error = exc + break + report = build_report(cfg.output_dir) + if ( + summary.get("rows") != dataset.rows + or summary.get("unique_doc_ids") != dataset.rows + ): + repeat_error = RuntimeError( + f"repeat {repeat} row validation failed: expected={dataset.rows}, " + f"rows={summary.get('rows')}, unique={summary.get('unique_doc_ids')}" + ) + reports.append(report) + if repeat_error is not None: + break + comparison = build_baseline_report( + base_dir, + reports, + { + "run_id": base_run_id, + "manifest_uri": dataset.manifest_uri, + "rows": dataset.rows, + "duration_s": args.duration_s, + "total_bytes": dataset.total_bytes, + "total_audio_seconds": dataset.total_audio_seconds, + "planned_repeats": args.repeats, + "seed": args.seed, + "source_dir": args.source_dir, + "mock_profile": args.mock_profile, + "warmup_enabled": not args.skip_warmup, + "standard_baseline": standard_baseline, + "max_minutes_per_repeat": args.max_minutes, + "asr_actor_cpus": args.asr_actor_cpus, + "asr_actor_concurrency": args.asr_actor_concurrency, + "asr_batch_size": args.asr_batch_size, + "llm_concurrency": args.llm_concurrency, + "sample_interval_s": args.sample_interval_s, + "num_partitions": args.num_partitions, + }, + ) + print(json.dumps(comparison, ensure_ascii=False, indent=2)) + print(f"[summary] comparison: {base_dir / 'baseline-summary.md'}") + if repeat_error is not None: + raise repeat_error + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + if ray.is_initialized(): + ray.shutdown() + + def _cmd_serve_mock(args) -> None: from .mock_llm import serve @@ -181,7 +363,9 @@ def build_parser() -> argparse.ArgumentParser: generate.add_argument("--run-id", default=None) generate.add_argument("--bucket", default="benchmark") generate.add_argument("--prefix", default="audio") - generate.add_argument("--profile", choices=("smoke", "fixed", "mixed"), default="smoke") + generate.add_argument( + "--profile", choices=("smoke", "fixed", "mixed"), default="smoke" + ) generate.add_argument("--count", type=int, default=4) generate.add_argument("--duration-s", type=float, default=30.0) generate.add_argument("--source-dir", default=None) @@ -208,7 +392,9 @@ def build_parser() -> argparse.ArgumentParser: report.add_argument("--run-dir", required=True) report.set_defaults(func=_cmd_report) - smoke = commands.add_parser("local-smoke", help="run a real local end-to-end smoke test") + smoke = commands.add_parser( + "local-smoke", help="run a real local end-to-end smoke test" + ) smoke.add_argument("--run-id", default=None) smoke.add_argument("--bucket", default="benchmark") smoke.add_argument("--prefix", default="audio") @@ -217,6 +403,35 @@ def build_parser() -> argparse.ArgumentParser: smoke.add_argument("--max-minutes", type=float, default=15.0) smoke.add_argument("--asr-actor-cpus", type=float, default=4.0) smoke.set_defaults(func=_cmd_local_smoke) + + baseline = commands.add_parser( + "local-baseline", + help="run repeated fixed-duration local performance baselines", + ) + baseline.add_argument("--run-id", default=None) + baseline.add_argument("--bucket", default="benchmark") + baseline.add_argument("--prefix", default="audio") + baseline.add_argument("--count", type=int, default=50) + baseline.add_argument("--duration-s", type=float, default=60.0) + baseline.add_argument("--repeats", type=int, default=2) + baseline.add_argument("--source-dir", default=None) + baseline.add_argument("--seed", type=int, default=20260715) + baseline.add_argument( + "--mock-profile", choices=("fast", "standard"), default="fast" + ) + baseline.add_argument( + "--skip-warmup", + action="store_true", + help="skip the one-row model/cache warm-up before measured repeats", + ) + baseline.add_argument("--max-minutes", type=float, default=45.0) + baseline.add_argument("--asr-actor-cpus", type=float, default=4.0) + baseline.add_argument("--asr-actor-concurrency", type=int, default=1) + baseline.add_argument("--asr-batch-size", type=int, default=1) + baseline.add_argument("--llm-concurrency", type=int, default=8) + baseline.add_argument("--sample-interval-s", type=float, default=5.0) + baseline.add_argument("--num-partitions", type=int, default=None) + baseline.set_defaults(func=_cmd_local_baseline) return parser diff --git a/benchmark/audio/report.py b/benchmark/audio/report.py index 49e54dc..bfd1881 100644 --- a/benchmark/audio/report.py +++ b/benchmark/audio/report.py @@ -3,6 +3,7 @@ import csv import json import math +import statistics from collections import defaultdict from pathlib import Path from typing import Iterable @@ -35,19 +36,39 @@ def build_report(run_dir: Path) -> dict: summary = json.loads((run_dir / "summary.json").read_text()) metadata = json.loads((run_dir / "metadata.json").read_text()) - report = {"run_id": summary["run_id"], "state": summary["state"], "summary": summary} + report = { + "run_id": summary["run_id"], + "state": summary["state"], + "summary": summary, + } if summary["state"] == "success": - ds = lance.dataset(summary["lance_uri"], storage_options=lance_storage_options(summary["lance_uri"])) + ds = lance.dataset( + summary["lance_uri"], + storage_options=lance_storage_options(summary["lance_uri"]), + ) table = ds.to_table( - columns=["asr_ms", "llm_ms", "input_bytes", "duration_s", "llm_attempts", "status"] + columns=[ + "asr_ms", + "llm_ms", + "input_bytes", + "duration_s", + "llm_attempts", + "status", + ] ) elapsed = summary["elapsed_s"] report["throughput"] = { "files_per_s": summary["rows"] / elapsed if elapsed else None, - "mb_per_s": summary["input_bytes"] / elapsed / 1_000_000 if elapsed else None, - "audio_seconds_per_wall_second": summary["audio_seconds"] / elapsed if elapsed else None, + "mb_per_s": summary["input_bytes"] / elapsed / 1_000_000 + if elapsed + else None, + "audio_seconds_per_wall_second": summary["audio_seconds"] / elapsed + if elapsed + else None, } - asr_values = [x for x in table.column("asr_ms").to_pylist() if x is not None and x > 0] + asr_values = [ + x for x in table.column("asr_ms").to_pylist() if x is not None and x > 0 + ] attempts = table.column("llm_attempts").to_pylist() llm_values = [ value @@ -55,8 +76,16 @@ def build_report(run_dir: Path) -> dict: if value is not None and attempt is not None and attempt > 0 ] report["latency_ms"] = { - "asr_ms": {"p50": _percentile(asr_values, 50), "p95": _percentile(asr_values, 95), "p99": _percentile(asr_values, 99)}, - "llm_ms": {"p50": _percentile(llm_values, 50), "p95": _percentile(llm_values, 95), "p99": _percentile(llm_values, 99)}, + "asr_ms": { + "p50": _percentile(asr_values, 50), + "p95": _percentile(asr_values, 95), + "p99": _percentile(asr_values, 99), + }, + "llm_ms": { + "p50": _percentile(llm_values, 50), + "p95": _percentile(llm_values, 95), + "p99": _percentile(llm_values, 99), + }, } report["llm_retry_rows"] = sum(x > 1 for x in attempts if x) @@ -75,10 +104,19 @@ def build_report(run_dir: Path) -> dict: "ray_process_rss_peak_bytes": max(rss), "ray_process_rss_p95_bytes": _percentile(rss, 95), "cpu_percent_p95": _percentile( - [float(row["cpu_percent"]) for row in resource_rows if row.get("cpu_percent")], 95 + [ + float(row["cpu_percent"]) + for row in resource_rows + if row.get("cpu_percent") + ], + 95, ), "object_store_used_peak_bytes": max( - [float(row["object_store_used_bytes"]) for row in resource_rows if row.get("object_store_used_bytes")] + [ + float(row["object_store_used_bytes"]) + for row in resource_rows + if row.get("object_store_used_bytes") + ] or [0.0] ), } @@ -115,6 +153,110 @@ def build_report(run_dir: Path) -> dict: return report +def build_baseline_report(run_dir: Path, reports: list[dict], dataset: dict) -> dict: + """Compare repeated runs over the same fixed-duration manifest.""" + planned_repeats = int(dataset.get("planned_repeats", len(reports))) + successful = [report for report in reports if report.get("state") == "success"] + audio_rates = [ + report["throughput"]["audio_seconds_per_wall_second"] for report in successful + ] + file_rates = [report["throughput"]["files_per_s"] for report in successful] + mean_audio_rate = statistics.fmean(audio_rates) if audio_rates else None + throughput_cv = ( + statistics.pstdev(audio_rates) / mean_audio_rate + if len(audio_rates) >= 2 and mean_audio_rate + else None + ) + all_succeeded = ( + len(reports) == planned_repeats and len(successful) == planned_repeats + ) + repeat_results = [] + for report in reports: + throughput = report.get("throughput", {}) + resources = report.get("resources", {}) + repeat_results.append( + { + "run_id": report["run_id"], + "state": report.get("state", "unknown"), + "elapsed_s": report.get("summary", {}).get("elapsed_s"), + "error": report.get("summary", {}).get("error"), + "audio_seconds_per_wall_second": throughput.get( + "audio_seconds_per_wall_second" + ), + "files_per_s": throughput.get("files_per_s"), + "ray_process_rss_peak_bytes": resources.get( + "ray_process_rss_peak_bytes" + ), + "node_memory_used_peak_bytes": resources.get( + "node_memory_used_peak_bytes" + ), + "passes_memory_slo": resources.get("passes_memory_slo"), + } + ) + comparison = { + "run_id": dataset["run_id"], + "dataset": dataset, + "repeats": len(reports), + "planned_repeats": planned_repeats, + "completed_repeats": len(reports), + "all_succeeded": all_succeeded, + "elapsed_s": [result["elapsed_s"] for result in repeat_results], + "audio_seconds_per_wall_second": audio_rates, + "files_per_s": file_rates, + "mean_audio_seconds_per_wall_second": mean_audio_rate, + "throughput_cv": throughput_cv, + "repeat_delta_percent": ( + (audio_rates[-1] - audio_rates[0]) / audio_rates[0] * 100 + if audio_rates[0] + else None + ) + if len(audio_rates) >= 2 + else None, + "passes_repeatability_slo": bool( + all_succeeded and throughput_cv is not None and throughput_cv <= 0.10 + ), + "reports": [report["run_id"] for report in reports], + "repeat_results": repeat_results, + } + write_json(run_dir / "baseline-summary.json", comparison) + lines = [ + f"# Local baseline {dataset['run_id']}", + "", + f"- Dataset: {dataset['rows']} files × {dataset['duration_s']:.0f}s target duration", + f"- Repeats: {len(reports)} completed / {planned_repeats} planned", + f"- All succeeded: `{'YES' if all_succeeded else 'NO'}`", + f"- Mean audio seconds / wall second: {mean_audio_rate:.4f}" + if mean_audio_rate is not None + else "- Mean audio seconds / wall second: N/A", + f"- Throughput CV: {throughput_cv * 100:.2f}%" + if throughput_cv is not None + else "- Throughput CV: N/A", + f"- First-to-last delta: {comparison['repeat_delta_percent']:.2f}%" + if comparison["repeat_delta_percent"] is not None + else "- First-to-last delta: N/A", + f"- Repeatability SLO (CV ≤ 10%): `{'PASS' if comparison['passes_repeatability_slo'] else 'FAIL'}`", + "", + "| Repeat | State | Elapsed (s) | Audio sec / wall sec | Files/s |", + "|---:|:---|---:|---:|---:|", + ] + for index, result in enumerate(repeat_results, start=1): + elapsed_s = result["elapsed_s"] + audio_rate = result["audio_seconds_per_wall_second"] + file_rate = result["files_per_s"] + lines.append( + f"| {index} | {result['state']} | {elapsed_s:.2f} | " + if elapsed_s is not None + else f"| {index} | {result['state']} | N/A | " + ) + lines[-1] += ( + f"{audio_rate:.4f} | {file_rate:.4f} |" + if audio_rate is not None and file_rate is not None + else "N/A | N/A |" + ) + (run_dir / "baseline-summary.md").write_text("\n".join(lines) + "\n") + return comparison + + def _write_markdown(path: Path, report: dict) -> None: summary = report["summary"] lines = [ @@ -142,7 +284,9 @@ def _write_markdown(path: Path, report: dict) -> None: lines.extend(["", "## Latency", ""]) for stage, values in latency.items(): rendered = ", ".join( - f"{name.upper()}={value:.2f}ms" for name, value in values.items() if value is not None + f"{name.upper()}={value:.2f}ms" + for name, value in values.items() + if value is not None ) lines.append(f"- {stage}: {rendered or 'no executed rows'}") resources = report.get("resources") diff --git a/docs/audio-benchmark.md b/docs/audio-benchmark.md index e33f99f..b53b0e0 100644 --- a/docs/audio-benchmark.md +++ b/docs/audio-benchmark.md @@ -24,6 +24,7 @@ uv run python -m benchmark.audio --help | `submit` | 通过 Ray Jobs API 提交同一个 pipeline | | `report` | 从已有运行产物重建报告 | | `local-smoke` | 一键完成 MinIO 预检、造数、本地 Ray 执行和校验 | +| `local-baseline` | 构造固定时长数据并重复运行,比较吞吐稳定性 | 每次运行在 `.benchmarks//` 或 `--run-dir` 指定目录生成: @@ -75,6 +76,51 @@ uv run python -m benchmark.audio local-smoke \ --count 8 ``` +### 本地性能 Baseline + +`local-smoke` 只有少量文件,仅用于验证功能。要得到有意义的本地吞吐和内存曲线,使用: + +```bash +uv run python -m benchmark.audio local-baseline +``` + +默认行为: + +- 生成 50 条固定 60 秒音频; +- 先用一条音频预热 Ray、模型和本地缓存,warm-up 不计入两轮比较; +- 两轮复用完全相同的 manifest; +- 每轮写入独立的 Lance 表,避免覆盖; +- 使用 fast Mock LLM,减少外部延迟随机性; +- 单轮最多 45 分钟; +- 输出两轮吞吐均值、首尾差异和变异系数;CV ≤ 10% 判为重复性通过。 + +标准 Baseline 定义为 40–60 条、每条 60 秒且单轮时限不超过 45 分钟;默认取 50 条。CLI 仍允许缩小 `--count`/`--duration-s` 来快速验证编排,但会标记为 non-standard diagnostic baseline,不应与标准结果横向比较。任一轮失败或超时也会写出 `baseline-summary.{json,md}`,并将重复性 SLO 标为失败。 + +运行目录结构: + +```text +.benchmarks// +├── repeat-01/ +├── repeat-02/ +├── warmup/ +├── baseline-summary.json +└── baseline-summary.md +``` + +常用调整: + +```bash +uv run python -m benchmark.audio local-baseline \ + --count 60 \ + --duration-s 60 \ + --repeats 2 \ + --max-minutes 45 +``` + +如果当前 Ray/模型缓存已经由其他任务预热,可传 `--skip-warmup`。常规可重复测试不建议跳过,否则首次下载/解压模型会显著抬高第一轮耗时并造成虚假的高 CV。 + +默认数据量约 50 音频分钟。根据此前本地约 2 audio seconds / wall second 的结果,单轮预计 20–30 分钟;实际时间仍取决于 CPU、actor 初始化和种子编码。两轮运行共享同一个 local Ray 进程,但每轮重新建立 Daft query/ASR actor,因此报告仍包含每轮 actor 初始化成本,不包含首次模型下载成本。 + ## 3. 构造测试数据 三种 profile 的定位如下: diff --git a/tests/benchmark/test_config.py b/tests/benchmark/test_config.py index 65498ac..4e9e4cc 100644 --- a/tests/benchmark/test_config.py +++ b/tests/benchmark/test_config.py @@ -32,3 +32,13 @@ def test_cli_parses_run_controls(): assert args.max_minutes == 60 assert args.asr_batch_size == 2 assert args.num_partitions == 64 + + +def test_local_baseline_defaults_to_two_fixed_sixty_second_runs(): + args = build_parser().parse_args(["local-baseline"]) + assert args.count == 50 + assert args.duration_s == 60 + assert args.repeats == 2 + assert args.mock_profile == "fast" + assert args.max_minutes == 45 + assert args.skip_warmup is False diff --git a/tests/benchmark/test_report.py b/tests/benchmark/test_report.py index fa1f6eb..628420c 100644 --- a/tests/benchmark/test_report.py +++ b/tests/benchmark/test_report.py @@ -3,35 +3,50 @@ import csv import json -from benchmark.audio.report import build_report +from benchmark.audio.report import build_baseline_report, build_report def test_failed_run_report_does_not_require_lance(tmp_path): (tmp_path / "summary.json").write_text( - json.dumps({"run_id": "run-1", "state": "failed", "elapsed_s": 2.5, "error": "boom"}) + json.dumps( + {"run_id": "run-1", "state": "failed", "elapsed_s": 2.5, "error": "boom"} + ) ) (tmp_path / "metadata.json").write_text(json.dumps({"versions": {"daft": "test"}})) with (tmp_path / "resources.csv").open("w", newline="") as out: writer = csv.DictWriter( out, fieldnames=[ - "timestamp", "hostname", "memory_total_bytes", "memory_used_bytes", - "ray_process_rss_bytes", "cpu_percent", "object_store_used_bytes", + "timestamp", + "hostname", + "memory_total_bytes", + "memory_used_bytes", + "ray_process_rss_bytes", + "cpu_percent", + "object_store_used_bytes", ], ) writer.writeheader() writer.writerow( { - "timestamp": 1, "hostname": "node", "memory_total_bytes": 1000, - "memory_used_bytes": 100, "ray_process_rss_bytes": 50, - "cpu_percent": 10, "object_store_used_bytes": 20, + "timestamp": 1, + "hostname": "node", + "memory_total_bytes": 1000, + "memory_used_bytes": 100, + "ray_process_rss_bytes": 50, + "cpu_percent": 10, + "object_store_used_bytes": 20, } ) writer.writerow( { - "timestamp": 2, "hostname": "node", "memory_total_bytes": 1000, - "memory_used_bytes": 200, "ray_process_rss_bytes": 75, - "cpu_percent": 20, "object_store_used_bytes": 30, + "timestamp": 2, + "hostname": "node", + "memory_total_bytes": 1000, + "memory_used_bytes": 200, + "ray_process_rss_bytes": 75, + "cpu_percent": 20, + "object_store_used_bytes": 30, } ) @@ -41,3 +56,58 @@ def test_failed_run_report_does_not_require_lance(tmp_path): assert report["resources"]["nodes"]["node"]["memory_used_p95_bytes"] == 195 assert (tmp_path / "report.json").exists() assert (tmp_path / "report.md").exists() + + +def test_baseline_report_compares_repeat_throughput(tmp_path): + reports = [ + { + "run_id": "base-repeat-01", + "state": "success", + "summary": {"elapsed_s": 100.0}, + "throughput": {"audio_seconds_per_wall_second": 2.0, "files_per_s": 0.5}, + }, + { + "run_id": "base-repeat-02", + "state": "success", + "summary": {"elapsed_s": 102.0}, + "throughput": {"audio_seconds_per_wall_second": 1.96, "files_per_s": 0.49}, + }, + ] + result = build_baseline_report( + tmp_path, + reports, + {"run_id": "base", "rows": 50, "duration_s": 60.0}, + ) + assert result["repeats"] == 2 + assert result["passes_repeatability_slo"] is True + assert result["throughput_cv"] < 0.02 + assert (tmp_path / "baseline-summary.json").exists() + assert (tmp_path / "baseline-summary.md").exists() + + +def test_baseline_report_preserves_failed_repeat(tmp_path): + reports = [ + { + "run_id": "base-repeat-01", + "state": "success", + "summary": {"elapsed_s": 100.0}, + "throughput": {"audio_seconds_per_wall_second": 2.0, "files_per_s": 0.5}, + }, + { + "run_id": "base-repeat-02", + "state": "failed", + "summary": {"elapsed_s": 45.0, "error": "timeout"}, + "resources": {"ray_process_rss_peak_bytes": 123}, + }, + ] + result = build_baseline_report( + tmp_path, + reports, + {"run_id": "base", "rows": 50, "duration_s": 60.0, "planned_repeats": 2}, + ) + assert result["all_succeeded"] is False + assert result["passes_repeatability_slo"] is False + assert result["throughput_cv"] is None + assert result["repeat_results"][1]["error"] == "timeout" + assert result["repeat_results"][1]["ray_process_rss_peak_bytes"] == 123 + assert "All succeeded: `NO`" in (tmp_path / "baseline-summary.md").read_text()