From 7a81f3a7090178902d87af649e0537d7902dd49e Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 05:27:13 +0900 Subject: [PATCH 1/6] musebench: add report builder, eval-only manifest; audit script gains --device report.py computes human FPR, FMA-OOD accuracy, per-generator / co-created / per-transform FNR and AUROC vs Jamendo at the run threshold, for all / seen / unseen rows, with Wilson intervals and failure accounting. select.py eval-manifest writes the eval rows (with runtime_path) because the runner's --protocol-split has no 'eval' choice. audit_artifactnet_nonfinite.py: optional --device (default cpu, unchanged), needed because 20% of tracks hit non-finite chunks and a CPU re-run of ~6.6k tracks would take ~8 h. --- scripts/audit_artifactnet_nonfinite.py | 4 +- scripts/musebench/report.py | 273 +++++++++++++++++++++++++ scripts/musebench/select.py | 26 ++- tests/test_musebench_report.py | 111 ++++++++++ 4 files changed, 412 insertions(+), 2 deletions(-) create mode 100644 scripts/musebench/report.py create mode 100644 tests/test_musebench_report.py diff --git a/scripts/audit_artifactnet_nonfinite.py b/scripts/audit_artifactnet_nonfinite.py index 663df20..8f48ba9 100644 --- a/scripts/audit_artifactnet_nonfinite.py +++ b/scripts/audit_artifactnet_nonfinite.py @@ -27,6 +27,8 @@ def main(): parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--failures", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--device", default="cpu", choices=["cpu", "cuda"], + help="onnxruntime device for the chunk re-run (default: cpu)") args = parser.parse_args() manifest = json.loads(args.manifest.read_text()) @@ -35,7 +37,7 @@ def main(): target_ids = [f["track_id"] for f in failures if f.get("stage") == "inference"] model = ArtifactNetModel(n_chunks=N_CHUNKS_DEFAULT) - model.load(device="cpu") + model.load(device=args.device) audits = [] for index, track_id in enumerate(target_ids, 1): row = rows[track_id] diff --git a/scripts/musebench/report.py b/scripts/musebench/report.py new file mode 100644 index 0000000..c348659 --- /dev/null +++ b/scripts/musebench/report.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +# Created: 2026-11-04 +# Purpose: Turn an ArtifactBench run on the MuseBench eval cohort into a MuseBench report: +# human FPR, FMA-OOD accuracy, per-generator / co-created / per-transform FNR and +# AUROC at a fixed threshold, each for all, seen and unseen rows. +# Dependencies: standard library only. +# Test Status: tests/test_musebench_report.py +"""Build ``musebench_report.json`` and ``musebench_report.md`` from a bench run. + +Inputs: the eval manifest (with ``seen_in_training`` from leak_map.py) and the run directory +written by ``python -m artifactbench.bench`` (``run_manifest.json``, ``/track_probs.json``, +``/inference_failures.json``). + +Conventions: a track is called AI when ``prob >= threshold`` (the runner's rule). FPR is the +share of human tracks called AI, FNR the share of AI tracks called human, FMA-OOD accuracy the +share of FMA tracks called human. AUROC compares an AI group with the Jamendo human negatives of +the same seen/unseen split. Rates carry a Wilson 95 % interval. Nothing is tuned on MuseBench. +""" +from __future__ import annotations + +import os as _os +import sys as _sys + +# This directory holds select.py, which shadows the stdlib ``select`` module when the +# directory is on sys.path (``python scripts/musebench/x.py`` without -I). Drop it. +if _sys.path and _os.path.abspath(_sys.path[0] or ".") == _os.path.dirname(_os.path.abspath(__file__)): + _sys.path.pop(0) + +import argparse # noqa: E402 +import json # noqa: E402 +import math # noqa: E402 +import os # noqa: E402 +import sys # noqa: E402 +import time # noqa: E402 +from collections import Counter, defaultdict # noqa: E402 +from pathlib import Path # noqa: E402 + +DEFAULT_ROOT = Path(os.environ.get("MUSEBENCH_ROOT", "~/data/musebench")).expanduser() +EVAL_MANIFEST_REL = Path("manifests") / "musebench_eval_manifest.json" +SPLITS = ("all", "seen", "unseen") +HUMAN_NEG = "musebench_human_jamendo" +FMA = "musebench_human_fma_ood" +GENERATOR_ORDER = ["musicgen", "audioldm2", "riffusion", "magnet", "jasco", "stable_audio_open", + "mustango", "lyria_rt", "suno", "udio"] +TRANSFORM_ORDER = ["mp3", "ogg", "aac", "eq", "pitch_time", "loudness", "gaussian", "pgd"] +# arXiv:2610.20310v1, Table 10 (FMA OOD accuracy) and Table 1 (FNR on co-created / transforms), +# in %. These detectors were trained on MuseBench Default; quoted for context only. +PAPER_TABLE10 = {"Q-SVM": 55.9, "Deezer": 94.4, "M5": 75.8, "RawNet2": 39.6, "ConvNeXt": 43.5, + "ResNet18+Spec": 28.4, "AST": 79.8, "ViT": 40.7, "SpecTTTra": 39.7} +PAPER_TABLE1_COLS = ["co_created", "mp3", "ogg", "aac", "eq", "pitch_time", "loudness", + "gaussian", "pgd"] +PAPER_TABLE1 = { + "Deezer": [36.8, 12.3, 9.5, 7.7, 6.4, 93.6, 5.2, 19.6, 26.4], + "M5": [20.1, 15.2, 1.9, 1.9, 2.0, 26.2, 1.3, 2.1, 92.4], + "ConvNeXt": [12.8, 0.2, 0.3, 0.6, 0.2, 17.0, 0.2, 0.4, 11.7], + "ResNet18+Spec": [0.0, 0.9, 3.7, 0.1, 3.1, 11.3, 2.0, 5.2, 11.1], + "AST": [4.8, 24.9, 7.3, 3.5, 0.0, 4.2, 0.0, 0.0, 9.2], + "ViT": [1.7, 0.4, 0.5, 0.5, 0.2, 19.9, 0.2, 0.8, 10.7], + "SpecTTTra": [33.6, 0.5, 1.0, 1.0, 0.5, 16.7, 0.5, 0.9, 10.7], +} + + +# --------------------------------------------------------------------------- statistics + +def wilson(k: int, n: int, z: float = 1.959964) -> tuple[float, float] | None: + if n == 0: + return None + p = k / n + den = 1 + z * z / n + centre = (p + z * z / (2 * n)) / den + half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / den + return (max(0.0, centre - half), min(1.0, centre + half)) + + +def rate(k: int, n: int) -> dict: + return {"n": n, "k": k, "rate": (k / n) if n else None, "ci95": wilson(k, n)} + + +def auroc(pos: list[float], neg: list[float]) -> float | None: + """Mann-Whitney AUROC (ties count one half).""" + if not pos or not neg: + return None + scored = sorted([(p, 1) for p in pos] + [(q, 0) for q in neg]) + rank_sum, i = 0.0, 0 + while i < len(scored): + j = i + while j < len(scored) and scored[j][0] == scored[i][0]: + j += 1 + mid = (i + 1 + j) / 2 # average 1-based rank of the tie block + rank_sum += mid * sum(1 for _, y in scored[i:j] if y == 1) + i = j + n_pos, n_neg = len(pos), len(neg) + return (rank_sum - n_pos * (n_pos + 1) / 2) / (n_pos * n_neg) + + +# --------------------------------------------------------------------------- report + +def split_of(row: dict) -> str: + return "seen" if row["seen_in_training"] else "unseen" + + +def build_report(eval_rows: list[dict], probs: dict[str, float], failures: list[dict], + threshold: float = 0.5) -> dict: + """Pure function: eval manifest rows + {track_id: prob} + failures -> report dict.""" + failed = {f["track_id"] for f in failures} + scored = [dict(r, prob=probs[r["track_id"]]) for r in eval_rows if r["track_id"] in probs] + missing = [r["track_id"] for r in eval_rows + if r["track_id"] not in probs and r["track_id"] not in failed] + + def rows(split: str, pred) -> list[dict]: + return [r for r in scored if pred(r) and (split == "all" or split_of(r) == split)] + + def called_ai(r): + return r["prob"] >= threshold + + def neg_probs(split): + return [r["prob"] for r in rows(split, lambda r: r["source"] == HUMAN_NEG)] + + def ai_block(split, pred): + rs = rows(split, pred) + out = rate(sum(not called_ai(r) for r in rs), len(rs)) # FNR + out["auroc_vs_jamendo"] = auroc([r["prob"] for r in rs], neg_probs(split)) + out["prob_median"] = sorted(r["prob"] for r in rs)[len(rs) // 2] if rs else None + return out + + report: dict = {"threshold": threshold, "splits": {}} + for split in SPLITS: + hum = rows(split, lambda r: r["source"] == HUMAN_NEG) + fma = rows(split, lambda r: r["source"] == FMA) + clean_ai = rows(split, lambda r: r["label"] == "ai" and r["variant"] == "clean") + fpr = rate(sum(called_ai(r) for r in hum), len(hum)) + fnr_clean = rate(sum(not called_ai(r) for r in clean_ai), len(clean_ai)) + s = { + "human_jamendo_fpr": fpr, + "fma_ood_accuracy": rate(sum(not called_ai(r) for r in fma), len(fma)), + "clean_ai_fnr": fnr_clean, + "balanced_accuracy_clean": ( + 1 - (fpr["rate"] + fnr_clean["rate"]) / 2 + if fpr["n"] and fnr_clean["n"] else None), + "auroc_clean_ai_vs_jamendo": auroc([r["prob"] for r in clean_ai], neg_probs(split)), + "per_generator_clean_fnr": { + g: ai_block(split, lambda r, g=g: r["generator"] == g and r["variant"] == "clean") + for g in GENERATOR_ORDER}, + "co_created_fnr": ai_block(split, lambda r: r["variant"] == "co_created"), + "per_transform_fnr": { + v: ai_block(split, lambda r, v=v: r["variant"] == v) for v in TRANSFORM_ORDER}, + "per_transform_generator_fnr": { + f"{g}__{v}": ai_block(split, lambda r, g=g, v=v: r["generator"] == g and r["variant"] == v) + for v in TRANSFORM_ORDER for g in GENERATOR_ORDER}, + } + report["splits"][split] = s + + by_src_fail = Counter(f["source"] for f in failures) + report["coverage"] = { + "eval_rows": len(eval_rows), "scored": len(scored), "failures": len(failures), + "missing": len(missing), "failure_rate": len(failures) / len(eval_rows) if eval_rows else None, + "failures_by_source": dict(sorted(by_src_fail.items())), + "failures_by_stage": dict(Counter(f.get("stage", "?") for f in failures)), + "seen_rows": sum(1 for r in eval_rows if r["seen_in_training"]), + } + report["paper_context"] = {"table10_fma_ood_accuracy": PAPER_TABLE10, + "table1_columns": PAPER_TABLE1_COLS, "table1_fnr": PAPER_TABLE1, + "note": "paper detectors are trained on MuseBench Default; " + "this detector is zero-shot. Not a head-to-head."} + return report + + +def pct(block) -> str: + if block is None: + return "–" + if isinstance(block, (int, float)): + return f"{100 * block:.1f}" + if not block.get("n"): + return "–" + lo, hi = block["ci95"] + return f"{100 * block['rate']:.1f} [{100 * lo:.1f}, {100 * hi:.1f}] (n={block['n']})" + + +def auc_s(x) -> str: + return "–" if x is None else f"{x:.3f}" + + +def to_markdown(report: dict, title: str) -> str: + sp = report["splits"] + L = [f"# {title}", "", + f"Threshold {report['threshold']} (fixed, not tuned on MuseBench). Rates in % with Wilson " + "95 % CI. `seen` = audio (or its direct ancestor) in the detector's training candidate " + "pool; `unseen` = the zero-shot part.", ""] + cov = report["coverage"] + L += [f"Coverage: {cov['scored']} of {cov['eval_rows']} eval rows scored, {cov['failures']} " + f"failures, {cov['missing']} missing; {cov['seen_rows']} rows are `seen`.", ""] + L += ["## Headline", "", "| Metric | all | seen | unseen |", "|---|---|---|---|"] + for key, name in [("human_jamendo_fpr", "Human (Jamendo) FPR"), + ("fma_ood_accuracy", "FMA-OOD accuracy"), + ("clean_ai_fnr", "Clean AI FNR (all generators)")]: + L.append(f"| {name} | " + " | ".join(pct(sp[s][key]) for s in SPLITS) + " |") + L.append("| Balanced accuracy, clean | " + " | ".join(pct(sp[s]["balanced_accuracy_clean"]) for s in SPLITS) + " |") + L.append("| AUROC clean AI vs Jamendo | " + " | ".join(auc_s(sp[s]["auroc_clean_ai_vs_jamendo"]) for s in SPLITS) + " |") + L += ["", "## Clean AI FNR per generator", "", + "| Generator | FNR all | FNR unseen | FNR seen | AUROC all | AUROC unseen |", + "|---|---|---|---|---|---|"] + for g in GENERATOR_ORDER: + a, u, s_ = (sp[x]["per_generator_clean_fnr"][g] for x in ("all", "unseen", "seen")) + L.append(f"| {g} | {pct(a)} | {pct(u)} | {pct(s_)} | {auc_s(a['auroc_vs_jamendo'])} | " + f"{auc_s(u['auroc_vs_jamendo'])} |") + a, u, s_ = (sp[x]["co_created_fnr"] for x in ("all", "unseen", "seen")) + L.append(f"| co-created | {pct(a)} | {pct(u)} | {pct(s_)} | {auc_s(a['auroc_vs_jamendo'])} | " + f"{auc_s(u['auroc_vs_jamendo'])} |") + L += ["", "## FNR per transform (all generators)", "", + "| Transform | FNR all | FNR unseen | FNR seen | AUROC all |", "|---|---|---|---|---|"] + for v in TRANSFORM_ORDER: + a, u, s_ = (sp[x]["per_transform_fnr"][v] for x in ("all", "unseen", "seen")) + L.append(f"| {v} | {pct(a)} | {pct(u)} | {pct(s_)} | {auc_s(a['auroc_vs_jamendo'])} |") + L += ["", "## FNR per transform × generator (all rows, %)", "", + "| Generator | " + " | ".join(TRANSFORM_ORDER) + " |", + "|---|" + "---|" * len(TRANSFORM_ORDER)] + for g in GENERATOR_ORDER: + cells = [] + for v in TRANSFORM_ORDER: + b = sp["all"]["per_transform_generator_fnr"][f"{g}__{v}"] + cells.append("–" if not b["n"] else f"{100 * b['rate']:.1f}") + L.append(f"| {g} | " + " | ".join(cells) + " |") + ctx = report["paper_context"] + L += ["", "## Paper context (arXiv:2610.20310v1; detectors trained on MuseBench)", "", + "| Detector | FMA-OOD acc. (T10) | " + " | ".join(ctx["table1_columns"]) + " |", + "|---|---|" + "---|" * len(ctx["table1_columns"])] + for det, row in ctx["table1_fnr"].items(): + L.append(f"| {det} | {ctx['table10_fma_ood_accuracy'].get(det, '–')} | " + + " | ".join(f"{x:.1f}" for x in row) + " |") + L += ["", "Failures by source: " + (json.dumps(cov["failures_by_source"]) or "{}"), ""] + return "\n".join(L) + + +def load_run(run_dir: Path, model_dir: str = "artifactnet") -> tuple[dict[str, float], list[dict], dict]: + probs = {t["track_id"]: float(t["prob"]) for t in + json.loads((run_dir / model_dir / "track_probs.json").read_text(encoding="utf-8"))} + failures = json.loads((run_dir / model_dir / "inference_failures.json").read_text(encoding="utf-8")) + run_manifest = json.loads((run_dir / "run_manifest.json").read_text(encoding="utf-8")) + return probs, failures, run_manifest + + +def atomic_write(path: Path, text: str) -> None: + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(text, encoding="utf-8") + os.replace(tmp, path) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Build the MuseBench report from a bench run.") + ap.add_argument("--root", type=Path, default=DEFAULT_ROOT) + ap.add_argument("--run-dir", type=Path, default=None, + help="bench output directory (default: /results/artifactnet_v94)") + ap.add_argument("--model-dir", default="artifactnet") + ap.add_argument("--title", default="ArtifactNet v9.4 on MuseBench (zero-shot)") + args = ap.parse_args(argv) + run_dir = args.run_dir or args.root / "results" / "artifactnet_v94" + eval_rows = json.loads((args.root / EVAL_MANIFEST_REL).read_text(encoding="utf-8"))["tracks"] + probs, failures, run_manifest = load_run(run_dir, args.model_dir) + report = build_report(eval_rows, probs, failures, threshold=run_manifest.get("threshold", 0.5)) + report["run"] = {"run_dir": str(run_dir), "model_dir": args.model_dir, + "input_manifest_sha256": run_manifest.get("input_manifest_sha256"), + "runner_git_revision": run_manifest.get("runner_git_revision"), + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} + atomic_write(run_dir / "musebench_report.json", json.dumps(report, indent=1) + "\n") + atomic_write(run_dir / "musebench_report.md", to_markdown(report, args.title) + "\n") + cov = report["coverage"] + print(f"report: {cov['scored']}/{cov['eval_rows']} scored, {cov['failures']} failures, " + f"{cov['missing']} missing -> {run_dir / 'musebench_report.md'}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/musebench/select.py b/scripts/musebench/select.py index 2a576e2..6089d63 100644 --- a/scripts/musebench/select.py +++ b/scripts/musebench/select.py @@ -420,6 +420,23 @@ def build_manifest(selection: dict, ledger: dict[str, dict], root: Path) -> dict }, missing +EVAL_MANIFEST_REL = Path("manifests") / "musebench_eval_manifest.json" + + +def eval_manifest(runtime: dict) -> dict: + """Eval-only copy of the runtime manifest (the runner's --protocol-split has no 'eval').""" + # runtime_path: the field the non-finite audit and finite-chunk builder read. + tracks = [dict(t, runtime_path=t["path"]) for t in runtime["tracks"] + if t["musebench_role"] == "eval"] + meta = dict(runtime.get("metadata", {})) + meta.update({"n_tracks": len(tracks), "derived_from": str(MANIFEST_REL_DEFAULT), + "roles": {"eval": len(tracks)}}) + return {"tracks": tracks, "metadata": meta} + + +MANIFEST_REL_DEFAULT = Path("manifests") / "musebench_runtime_manifest.json" + + def atomic_write_json(path: Path, obj: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(path.suffix + ".tmp") @@ -429,7 +446,7 @@ def atomic_write_json(path: Path, obj: object) -> None: def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description="Select MuseBench members / build the manifest.") - ap.add_argument("command", choices=["plan", "manifest"]) + ap.add_argument("command", choices=["plan", "manifest", "eval-manifest"]) ap.add_argument("--root", type=Path, default=DEFAULT_ROOT) ap.add_argument("--budget-gb", type=float, default=170.0, help="cap on all kept audio (eval + train pool), in GB") @@ -460,6 +477,13 @@ def main(argv: list[str] | None = None) -> int: print(f" {role:10s} {group:28s} {n}") return 1 if selection["unclassified"] else 0 + if args.command == "eval-manifest": + runtime = json.loads((args.root / MANIFEST_REL_DEFAULT).read_text(encoding="utf-8")) + out = eval_manifest(runtime) + atomic_write_json(args.root / EVAL_MANIFEST_REL, out) + print(f"eval manifest: {out['metadata']['n_tracks']} tracks") + return 0 + selection = json.loads(sel_path.read_text(encoding="utf-8")) manifest, missing = build_manifest(selection, load_ledger(args.root), args.root) atomic_write_json(args.root / "manifests" / "musebench_runtime_manifest.json", manifest) diff --git a/tests/test_musebench_report.py b/tests/test_musebench_report.py new file mode 100644 index 0000000..bf47889 --- /dev/null +++ b/tests/test_musebench_report.py @@ -0,0 +1,111 @@ +# Created: 2026-11-04 +# Purpose: Cover scripts/musebench/report.py: rates, Wilson CI, AUROC, seen/unseen splits, +# failure accounting and Markdown rendering. +# Dependencies: pytest +# Test Status: passing +import json + +import pytest + +from scripts.musebench import report as rp + + +def test_wilson_interval_bounds(): + lo, hi = rp.wilson(5, 10) + assert 0.23 < lo < 0.24 and 0.76 < hi < 0.77 + assert rp.wilson(0, 0) is None + assert rp.wilson(0, 20)[0] == 0.0 and rp.wilson(20, 20)[1] == 1.0 + + +def test_auroc_matches_definition_with_ties(): + assert rp.auroc([0.9, 0.8], [0.1, 0.2]) == 1.0 + assert rp.auroc([0.1], [0.9]) == 0.0 + assert rp.auroc([0.5, 0.5], [0.5]) == 0.5 + pos, neg = [0.3, 0.7, 0.7, 0.9], [0.1, 0.7, 0.4] + pairs = [(p > n) + 0.5 * (p == n) for p in pos for n in neg] + assert rp.auroc(pos, neg) == pytest.approx(sum(pairs) / len(pairs)) + assert rp.auroc([], [0.1]) is None + + +def row(tid, source, label, generator, variant, seen=False): + return {"track_id": tid, "source": source, "label": label, "generator": generator, + "variant": variant, "seen_in_training": seen} + + +def cohort(): + rows, probs = [], {} + + def add(tid, prob, *a, **k): + rows.append(row(tid, *a, **k)) + if prob is not None: + probs[tid] = prob + + # Jamendo humans: 4 unseen (1 false positive), 1 seen (false positive) + for i, p in enumerate([0.1, 0.2, 0.3, 0.9]): + add(f"j{i}", p, "musebench_human_jamendo", "real", "human", "clean") + add("js", 0.95, "musebench_human_jamendo", "real", "human", "clean", seen=True) + # FMA: 2 called human, 1 called AI + for i, p in enumerate([0.1, 0.4, 0.6]): + add(f"f{i}", p, "musebench_human_fma_ood", "real", "human", "clean") + # Clean MAGNeT (unseen): 1 of 4 missed; Suno (seen): 0 of 2 missed + for i, p in enumerate([0.99, 0.8, 0.7, 0.2]): + add(f"m{i}", p, "musebench_magnet", "ai", "magnet", "clean") + for i, p in enumerate([0.9, 0.97]): + add(f"s{i}", p, "musebench_suno", "ai", "suno", "clean", seen=True) + # Transforms and co-created + add("ma", 0.4, "musebench_magnet__aac", "ai", "magnet", "aac") + add("sp", 0.6, "musebench_suno__pgd", "ai", "suno", "pgd", seen=True) + add("cc", 0.5, "musebench_co_created", "ai", "musicgen_style", "co_created") + # One failure, one silently missing row + add("bad", None, "musebench_magnet", "ai", "magnet", "clean") + add("gone", None, "musebench_jasco", "ai", "jasco", "clean") + failures = [{"track_id": "bad", "source": "musebench_magnet", "stage": "decode"}] + return rows, probs, failures + + +def test_build_report_rates_and_splits(): + rows, probs, failures = cohort() + r = rp.build_report(rows, probs, failures) + allx, uns, seen = (r["splits"][s] for s in ("all", "unseen", "seen")) + assert allx["human_jamendo_fpr"]["k"] == 2 and allx["human_jamendo_fpr"]["n"] == 5 + assert uns["human_jamendo_fpr"]["rate"] == 0.25 + assert seen["human_jamendo_fpr"]["rate"] == 1.0 + assert allx["fma_ood_accuracy"]["rate"] == pytest.approx(2 / 3) + mg = uns["per_generator_clean_fnr"]["magnet"] + assert (mg["k"], mg["n"]) == (1, 4) + assert mg["auroc_vs_jamendo"] == pytest.approx(rp.auroc([0.99, 0.8, 0.7, 0.2], [0.1, 0.2, 0.3, 0.9])) + assert seen["per_generator_clean_fnr"]["suno"]["rate"] == 0.0 + assert uns["per_generator_clean_fnr"]["suno"]["n"] == 0 + assert allx["per_transform_fnr"]["aac"]["rate"] == 1.0 + assert allx["per_transform_fnr"]["pgd"]["rate"] == 0.0 + assert allx["per_transform_generator_fnr"]["suno__pgd"]["n"] == 1 + assert allx["co_created_fnr"]["rate"] == 0.0 # 0.5 >= threshold counts as AI + assert allx["balanced_accuracy_clean"] == pytest.approx(1 - (2 / 5 + 1 / 6) / 2) + cov = r["coverage"] + assert (cov["eval_rows"], cov["scored"], cov["failures"], cov["missing"]) == (19, 17, 1, 1) + assert cov["failures_by_source"] == {"musebench_magnet": 1} + + +def test_markdown_renders_all_sections(): + rows, probs, failures = cohort() + md = rp.to_markdown(rp.build_report(rows, probs, failures), "T") + for heading in ("## Headline", "## Clean AI FNR per generator", "## FNR per transform", + "## Paper context"): + assert heading in md + assert "| magnet | 25.0 [" in md or "| magnet | 20.0 [" in md + + +def test_main_reads_run_dir_and_writes_outputs(tmp_path): + rows, probs, failures = cohort() + (tmp_path / "manifests").mkdir() + (tmp_path / rp.EVAL_MANIFEST_REL).write_text(json.dumps({"tracks": rows})) + run = tmp_path / "results" / "artifactnet_v94" + (run / "artifactnet").mkdir(parents=True) + (run / "run_manifest.json").write_text(json.dumps({"threshold": 0.5, "input_manifest_sha256": "x"})) + (run / "artifactnet" / "track_probs.json").write_text(json.dumps( + [{"track_id": k, "prob": v} for k, v in probs.items()])) + (run / "artifactnet" / "inference_failures.json").write_text(json.dumps(failures)) + assert rp.main(["--root", str(tmp_path)]) == 0 + rep = json.loads((run / "musebench_report.json").read_text()) + assert rep["coverage"]["scored"] == 17 and rep["run"]["input_manifest_sha256"] == "x" + assert (run / "musebench_report.md").read_text().startswith("# ArtifactNet v9.4 on MuseBench") From 82cb6366ec18d508904a7a939298d9a3001f5b94 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 05:30:59 +0900 Subject: [PATCH 2/6] musebench: report primary finite-chunk result with strict sensitivity; add run verifier The declared ArtifactBench v2 ArtifactNet policy (median of finite scores, at least 4 of 7 chunks) is the primary result when finite_chunk/ exists; the strict any-non-finite result stays as a sensitivity table. verify.py run checks the full-cohort selection, pinned ONNX revision, that every eval track is scored or a recorded failure in both results, and that the saved report equals a rebuild from raw scores. --- scripts/musebench/report.py | 65 +++++++++++++++++++++++++---- scripts/musebench/verify.py | 66 ++++++++++++++++++++++++++++- tests/test_musebench_report.py | 35 +++++++++++++--- tests/test_musebench_verify.py | 2 +- tests/test_musebench_verify_run.py | 67 ++++++++++++++++++++++++++++++ 5 files changed, 218 insertions(+), 17 deletions(-) create mode 100644 tests/test_musebench_verify_run.py diff --git a/scripts/musebench/report.py b/scripts/musebench/report.py index c348659..0de47d0 100644 --- a/scripts/musebench/report.py +++ b/scripts/musebench/report.py @@ -245,6 +245,57 @@ def atomic_write(path: Path, text: str) -> None: os.replace(tmp, path) +def build_full_report(eval_rows: list[dict], run_dir: Path, model_dir: str = "artifactnet", + finite_subdir: str = "finite_chunk") -> dict: + """Primary = declared finite-chunk policy when present, else strict; strict kept as sensitivity.""" + probs, failures, run_manifest = load_run(run_dir, model_dir) + threshold = run_manifest.get("threshold", 0.5) + strict = build_report(eval_rows, probs, failures, threshold=threshold) + out = {"strict": strict} + finite_dir = run_dir / finite_subdir + if (finite_dir / model_dir / "track_probs.json").exists(): + f_probs, f_failures, _ = load_run_scores(finite_dir, model_dir) + out["primary"] = build_report(eval_rows, f_probs, f_failures, threshold=threshold) + policy_path = finite_dir / model_dir / "finite_chunk_policy.json" + policy = json.loads(policy_path.read_text(encoding="utf-8")) if policy_path.exists() else {} + out["primary_policy"] = {k: v for k, v in policy.items() if k != "recovered"} + out["primary_policy"]["result_dir"] = str(finite_dir) + else: + out["primary"] = strict + out["primary_policy"] = {"track_aggregation": "strict: any non-finite chunk fails the track"} + out["run"] = {"run_dir": str(run_dir), "model_dir": model_dir, + "threshold": threshold, + "input_manifest_sha256": run_manifest.get("input_manifest_sha256"), + "runner_git_revision": run_manifest.get("runner_git_revision")} + return out + + +def load_run_scores(result_dir: Path, model_dir: str) -> tuple[dict[str, float], list[dict], None]: + probs = {t["track_id"]: float(t["prob"]) for t in + json.loads((result_dir / model_dir / "track_probs.json").read_text(encoding="utf-8"))} + failures = json.loads((result_dir / model_dir / "inference_failures.json").read_text(encoding="utf-8")) + return probs, failures, None + + +def full_markdown(full: dict, title: str) -> str: + md = to_markdown(full["primary"], title) + pol = full["primary_policy"] + head = [f"Primary track policy: {pol.get('track_aggregation')}" + + (f" (at least {pol['minimum_valid_chunks']} finite chunks)" if "minimum_valid_chunks" in pol else ""), + ""] + st = full["strict"]["splits"] + sens = ["", "## Strict sensitivity (any non-finite chunk fails the track)", "", + f"Coverage: {full['strict']['coverage']['scored']} of {full['strict']['coverage']['eval_rows']} " + f"scored, {full['strict']['coverage']['failures']} failures.", "", + "| Metric | all | seen | unseen |", "|---|---|---|---|"] + for key, name in [("human_jamendo_fpr", "Human (Jamendo) FPR"), + ("fma_ood_accuracy", "FMA-OOD accuracy"), + ("clean_ai_fnr", "Clean AI FNR")]: + sens.append(f"| {name} | " + " | ".join(pct(st[x][key]) for x in SPLITS) + " |") + lines = md.splitlines() + return "\n".join(lines[:2] + head + lines[2:] + sens) + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description="Build the MuseBench report from a bench run.") ap.add_argument("--root", type=Path, default=DEFAULT_ROOT) @@ -255,15 +306,11 @@ def main(argv: list[str] | None = None) -> int: args = ap.parse_args(argv) run_dir = args.run_dir or args.root / "results" / "artifactnet_v94" eval_rows = json.loads((args.root / EVAL_MANIFEST_REL).read_text(encoding="utf-8"))["tracks"] - probs, failures, run_manifest = load_run(run_dir, args.model_dir) - report = build_report(eval_rows, probs, failures, threshold=run_manifest.get("threshold", 0.5)) - report["run"] = {"run_dir": str(run_dir), "model_dir": args.model_dir, - "input_manifest_sha256": run_manifest.get("input_manifest_sha256"), - "runner_git_revision": run_manifest.get("runner_git_revision"), - "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} - atomic_write(run_dir / "musebench_report.json", json.dumps(report, indent=1) + "\n") - atomic_write(run_dir / "musebench_report.md", to_markdown(report, args.title) + "\n") - cov = report["coverage"] + full = build_full_report(eval_rows, run_dir, args.model_dir) + full["run"]["created_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + atomic_write(run_dir / "musebench_report.json", json.dumps(full, indent=1) + "\n") + atomic_write(run_dir / "musebench_report.md", full_markdown(full, args.title) + "\n") + cov = full["primary"]["coverage"] print(f"report: {cov['scored']}/{cov['eval_rows']} scored, {cov['failures']} failures, " f"{cov['missing']} missing -> {run_dir / 'musebench_report.md'}") return 0 diff --git a/scripts/musebench/verify.py b/scripts/musebench/verify.py index 1fdb4e4..e93905f 100644 --- a/scripts/musebench/verify.py +++ b/scripts/musebench/verify.py @@ -289,6 +289,70 @@ def verify_leak(root: Path) -> tuple[list[str], dict]: return problems, stats +RUN_REL = Path("results") / "artifactnet_v94" + + +def verify_run(root: Path, run_rel: Path = RUN_REL) -> tuple[list[str], dict]: + """Check the bench run on the whole eval cohort and the MuseBench report built from it.""" + from artifactbench.models.artifactnet import DEFAULT_HF_REVISION + from scripts.musebench import report as rp + + problems: list[str] = [] + run_dir = root / run_rel + eval_path = root / rp.EVAL_MANIFEST_REL + needed = [eval_path, run_dir / "run_manifest.json", run_dir / "summary.json", + run_dir / "artifactnet" / "track_probs.json", + run_dir / "artifactnet" / "inference_failures.json", + run_dir / "finite_chunk" / "artifactnet" / "track_probs.json", + run_dir / "finite_chunk" / "artifactnet" / "inference_failures.json", + run_dir / "finite_chunk" / "artifactnet" / "finite_chunk_policy.json", + run_dir / "musebench_report.json", run_dir / "musebench_report.md"] + problems += [f"missing {p}" for p in needed if not p.exists()] + if problems: + return problems, {} + eval_rows = json.loads(eval_path.read_text(encoding="utf-8"))["tracks"] + eval_ids = {r["track_id"] for r in eval_rows} + run_manifest = json.loads((run_dir / "run_manifest.json").read_text(encoding="utf-8")) + summary = json.loads((run_dir / "summary.json").read_text(encoding="utf-8")) + + # The run covered exactly the eval cohort, with the pinned public ONNX build. + if set(run_manifest.get("selected_track_ids", [])) != eval_ids: + problems.append("run did not select exactly the eval cohort") + if run_manifest.get("models") != ["artifactnet"] or run_manifest.get("n_per_source") != 0: + problems.append("run is not a full-cohort artifactnet run (models / n_per_source)") + if run_manifest.get("threshold") != 0.5: + problems.append(f"threshold {run_manifest.get('threshold')} != 0.5") + prov = (summary.get("models") or [{}])[0].get("provenance", {}) + if prov.get("hf_revision") != DEFAULT_HF_REVISION: + problems.append(f"model revision {prov.get('hf_revision')} is not the pinned {DEFAULT_HF_REVISION}") + + # Every eval track is either scored or a recorded failure, in strict and primary results. + for label, sub in (("strict", run_dir), ("finite_chunk", run_dir / "finite_chunk")): + probs, failures, _ = rp.load_run_scores(sub, "artifactnet") + failed = {f["track_id"] for f in failures} + if set(probs) & failed: + problems.append(f"{label}: tracks both scored and failed") + if (set(probs) | failed) != eval_ids: + problems.append(f"{label}: scored + failed ({len(set(probs) | failed)}) != eval cohort ({len(eval_ids)})") + if any(not (0.0 <= p <= 1.0) for p in probs.values()): + problems.append(f"{label}: probability outside [0, 1]") + + # The saved report equals a fresh rebuild from the raw scores. + saved = json.loads((run_dir / "musebench_report.json").read_text(encoding="utf-8")) + fresh = json.loads(json.dumps(rp.build_full_report(eval_rows, run_dir))) + for key in ("primary", "strict", "primary_policy"): + if saved.get(key) != fresh.get(key): + problems.append(f"musebench_report.json '{key}' differs from a rebuild from raw scores") + for split in ("all", "seen", "unseen"): + if split not in saved.get("primary", {}).get("splits", {}): + problems.append(f"report lacks the '{split}' split") + cov = saved.get("primary", {}).get("coverage", {}) + stats = {"eval_rows": len(eval_rows), "scored_primary": cov.get("scored"), + "failures_primary": cov.get("failures"), + "failures_strict": saved.get("strict", {}).get("coverage", {}).get("failures")} + return problems, stats + + def not_implemented(stage: str): def check(root: Path) -> tuple[list[str], dict]: return [f"stage '{stage}' has no verifier yet (later milestone)"], {} @@ -299,7 +363,7 @@ def check(root: Path) -> tuple[list[str], dict]: "index": verify_index, "acquire": verify_acquire, "leak": verify_leak, - "run": not_implemented("run"), + "run": verify_run, } diff --git a/tests/test_musebench_report.py b/tests/test_musebench_report.py index bf47889..61f3bad 100644 --- a/tests/test_musebench_report.py +++ b/tests/test_musebench_report.py @@ -95,17 +95,40 @@ def test_markdown_renders_all_sections(): assert "| magnet | 25.0 [" in md or "| magnet | 20.0 [" in md -def test_main_reads_run_dir_and_writes_outputs(tmp_path): - rows, probs, failures = cohort() - (tmp_path / "manifests").mkdir() - (tmp_path / rp.EVAL_MANIFEST_REL).write_text(json.dumps({"tracks": rows})) - run = tmp_path / "results" / "artifactnet_v94" +def write_run(run, probs, failures): (run / "artifactnet").mkdir(parents=True) (run / "run_manifest.json").write_text(json.dumps({"threshold": 0.5, "input_manifest_sha256": "x"})) (run / "artifactnet" / "track_probs.json").write_text(json.dumps( [{"track_id": k, "prob": v} for k, v in probs.items()])) (run / "artifactnet" / "inference_failures.json").write_text(json.dumps(failures)) + + +def test_main_reads_run_dir_and_writes_outputs(tmp_path): + rows, probs, failures = cohort() + (tmp_path / "manifests").mkdir() + (tmp_path / rp.EVAL_MANIFEST_REL).write_text(json.dumps({"tracks": rows})) + run = tmp_path / "results" / "artifactnet_v94" + write_run(run, probs, failures) assert rp.main(["--root", str(tmp_path)]) == 0 rep = json.loads((run / "musebench_report.json").read_text()) - assert rep["coverage"]["scored"] == 17 and rep["run"]["input_manifest_sha256"] == "x" + assert rep["primary"]["coverage"]["scored"] == 17 and rep["run"]["input_manifest_sha256"] == "x" + assert rep["primary"] == rep["strict"] # no finite-chunk result: primary falls back to strict assert (run / "musebench_report.md").read_text().startswith("# ArtifactNet v9.4 on MuseBench") + + +def test_finite_chunk_result_becomes_primary(tmp_path): + rows, probs, failures = cohort() + (tmp_path / "manifests").mkdir() + (tmp_path / rp.EVAL_MANIFEST_REL).write_text(json.dumps({"tracks": rows})) + run = tmp_path / "results" / "artifactnet_v94" + write_run(run, probs, failures) + recovered = dict(probs, bad=0.9) + write_run(run / "finite_chunk", recovered, []) + (run / "finite_chunk" / "artifactnet" / "finite_chunk_policy.json").write_text(json.dumps( + {"track_aggregation": "median of finite", "minimum_valid_chunks": 4, "recovered": [{"track_id": "bad"}]})) + assert rp.main(["--root", str(tmp_path)]) == 0 + rep = json.loads((run / "musebench_report.json").read_text()) + assert rep["primary"]["coverage"]["scored"] == 18 and rep["strict"]["coverage"]["scored"] == 17 + assert rep["primary_policy"]["minimum_valid_chunks"] == 4 and "recovered" not in rep["primary_policy"] + md = (run / "musebench_report.md").read_text() + assert "at least 4 finite chunks" in md and "## Strict sensitivity" in md diff --git a/tests/test_musebench_verify.py b/tests/test_musebench_verify.py index d517a89..04e7ce8 100644 --- a/tests/test_musebench_verify.py +++ b/tests/test_musebench_verify.py @@ -92,4 +92,4 @@ def test_cli_runs_isolated_and_reports_failure(tmp_path): assert out.returncode == 1 and "[index] FAIL" in out.stdout out = subprocess.run([sys.executable, "-I", str(script), "--root", str(tmp_path), "run"], capture_output=True, text=True) - assert out.returncode == 1 and "no verifier yet" in out.stdout + assert out.returncode == 1 and "[run] FAIL" in out.stdout and "missing" in out.stdout diff --git a/tests/test_musebench_verify_run.py b/tests/test_musebench_verify_run.py new file mode 100644 index 0000000..25c5eee --- /dev/null +++ b/tests/test_musebench_verify_run.py @@ -0,0 +1,67 @@ +# Created: 2026-11-04 +# Purpose: Cover verify.py's run stage: full-cohort selection, pinned model revision, every +# track scored or failed, and a report that matches a rebuild from raw scores. +# Dependencies: pytest +# Test Status: passing +import json + +import pytest + +from artifactbench.models.artifactnet import DEFAULT_HF_REVISION +from scripts.musebench import report as rp +from scripts.musebench import verify as vf +from test_musebench_report import cohort # tests/ is on sys.path (rootdir mode) + + +def write_result(d, probs, failures, extra=None): + (d / "artifactnet").mkdir(parents=True) + (d / "artifactnet" / "track_probs.json").write_text(json.dumps( + [{"track_id": k, "prob": v} for k, v in probs.items()])) + (d / "artifactnet" / "inference_failures.json").write_text(json.dumps(failures)) + for name, obj in (extra or {}).items(): + (d / "artifactnet" / name).write_text(json.dumps(obj)) + + +@pytest.fixture +def run_root(tmp_path): + rows, probs, failures = cohort() + failures = failures + [{"track_id": "gone", "source": "musebench_jasco", "stage": "inference"}] + (tmp_path / "manifests").mkdir() + (tmp_path / rp.EVAL_MANIFEST_REL).write_text(json.dumps({"tracks": rows})) + run = tmp_path / vf.RUN_REL + write_result(run, probs, failures) + (run / "run_manifest.json").write_text(json.dumps({ + "selected_track_ids": [r["track_id"] for r in rows], "models": ["artifactnet"], + "n_per_source": 0, "threshold": 0.5})) + (run / "summary.json").write_text(json.dumps( + {"models": [{"name": "ArtifactNet v9.4", "provenance": {"hf_revision": DEFAULT_HF_REVISION}}]})) + write_result(run / "finite_chunk", dict(probs, gone=0.8), failures[:1], + {"finite_chunk_policy.json": {"track_aggregation": "median", "minimum_valid_chunks": 4}}) + assert rp.main(["--root", str(tmp_path)]) == 0 + return tmp_path + + +def test_verify_run_passes(run_root): + problems, stats = vf.verify_run(run_root) + assert problems == [] + assert stats["scored_primary"] == 18 and stats["failures_strict"] == 2 + + +def test_verify_run_rejects_partial_cohort_and_wrong_revision(run_root): + run = run_root / vf.RUN_REL + rm = json.loads((run / "run_manifest.json").read_text()) + rm["selected_track_ids"] = rm["selected_track_ids"][:-1] + rm["n_per_source"] = 2 + (run / "run_manifest.json").write_text(json.dumps(rm)) + (run / "summary.json").write_text(json.dumps({"models": [{"provenance": {"hf_revision": "main"}}]})) + text = "\n".join(vf.verify_run(run_root)[0]) + assert "exactly the eval cohort" in text and "n_per_source" in text and "pinned" in text + + +def test_verify_run_detects_unaccounted_track_and_stale_report(run_root): + run = run_root / vf.RUN_REL + probs = json.loads((run / "artifactnet" / "track_probs.json").read_text()) + (run / "artifactnet" / "track_probs.json").write_text(json.dumps(probs[1:])) + text = "\n".join(vf.verify_run(run_root)[0]) + assert "strict: scored + failed" in text + assert "differs from a rebuild" in text From e940b98ff3b9849a4cae3161118842393f78d07e Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 05:32:17 +0900 Subject: [PATCH 3/6] docs: document the MuseBench detector run; audit script runs from a checkout audit_artifactnet_nonfinite.py adds the repo root to sys.path so the documented command works without pip install -e (check_docs verifies it). --- docs/MUSEBENCH.md | 37 ++++++++++++++++++++++++++ scripts/audit_artifactnet_nonfinite.py | 10 +++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/docs/MUSEBENCH.md b/docs/MUSEBENCH.md index 340ba8e..dd72d99 100644 --- a/docs/MUSEBENCH.md +++ b/docs/MUSEBENCH.md @@ -281,6 +281,43 @@ any detector trained on SONICS has seen them. JamendoMaxCaps names match MTG-Jam for only a minority of tracks (266 of 3,877 acquired Jamendo rows), so most MuseBench Jamendo audio is outside MTG-Jamendo. +## Running a detector + +The runner's `--protocol-split` has no `eval` choice, so the eval rows are written to their own +manifest first. For ArtifactNet the declared primary policy of the frozen v2 protocol applies: +median of the finite scores of seven 4 s chunks, at least four finite; the strict result (any +non-finite chunk fails the track) is kept as a sensitivity analysis. + +```bash +python scripts/musebench/select.py eval-manifest --root ~/data/musebench +python -m artifactbench.bench --model artifactnet \ + --manifest ~/data/musebench/manifests/musebench_eval_manifest.json \ + --split all --n-per-source 0 --n-codec-pair 0 --device cuda \ + --output ~/data/musebench/results/artifactnet_v94 +python scripts/audit_artifactnet_nonfinite.py --device cuda \ + --manifest ~/data/musebench/manifests/musebench_eval_manifest.json \ + --failures ~/data/musebench/results/artifactnet_v94/artifactnet/inference_failures.json \ + --output ~/data/musebench/results/artifactnet_v94/nonfinite_audit.json +python scripts/build_artifactnet_finite_chunk_result.py --min-valid-chunks 4 \ + --manifest ~/data/musebench/manifests/musebench_eval_manifest.json \ + --strict-result ~/data/musebench/results/artifactnet_v94 \ + --chunk-audit ~/data/musebench/results/artifactnet_v94/nonfinite_audit.json \ + --output ~/data/musebench/results/artifactnet_v94/finite_chunk +python scripts/musebench/report.py --root ~/data/musebench +python scripts/musebench/verify.py --root ~/data/musebench all +``` + +`report.py` writes `musebench_report.json` and `.md` next to the run: human (Jamendo) FPR, +FMA-OOD accuracy, clean-AI FNR per generator, co-created FNR, FNR per transform and per +transform × generator, AUROC against the Jamendo negatives, Wilson 95 % intervals, all for +`all`, `seen` and `unseen` rows, plus failure counts. The paper's Table 1 and Table 10 values are +included for context only. Reports stay in the data root; they are not committed here. + +With onnxruntime-gpu installed from pip, the CUDA provider needs the `nvidia/*/lib` folders of +that environment on `LD_LIBRARY_PATH`; otherwise onnxruntime silently falls back to the CPU +(about 7× slower for ArtifactNet). CUDA and CPU scores differed by at most 0.016 on a 30-track +probe, with no decision changes at 0.5. + ## Licenses - Collection: CC BY-SA 3.0 (inherited from JamendoMaxCaps). diff --git a/scripts/audit_artifactnet_nonfinite.py b/scripts/audit_artifactnet_nonfinite.py index 8f48ba9..0339c15 100644 --- a/scripts/audit_artifactnet_nonfinite.py +++ b/scripts/audit_artifactnet_nonfinite.py @@ -5,12 +5,18 @@ import argparse import json +import sys from pathlib import Path import numpy as np -from artifactbench.data.manifest import load_audio_mono -from artifactbench.models.artifactnet import ( +# Run from a checkout without `pip install -e .`: make the repository importable. +_REPO_ROOT = Path(__file__).resolve().parents[1] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from artifactbench.data.manifest import load_audio_mono # noqa: E402 +from artifactbench.models.artifactnet import ( # noqa: E402 CHUNK_SAMPLES, N_CHUNKS_DEFAULT, ArtifactNetModel, From 2db99bfae6a9bc54c06cbeea360312b27657f894 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 11:25:35 +0900 Subject: [PATCH 4/6] musebench: shard and merge the ArtifactNet non-finite audit 9,738 of 33,000 strict-run tracks hit a non-finite chunk; the single audit process ran at ~1 track/s (decode-bound, GPU 65%). merge_audits.py splits the failure list round-robin and merges shard audits back into the audit script's schema, rejecting missing, duplicate or extra tracks. --- docs/MUSEBENCH.md | 13 ++++ scripts/musebench/merge_audits.py | 95 ++++++++++++++++++++++++++++ tests/test_musebench_merge_audits.py | 47 ++++++++++++++ 3 files changed, 155 insertions(+) create mode 100644 scripts/musebench/merge_audits.py create mode 100644 tests/test_musebench_merge_audits.py diff --git a/docs/MUSEBENCH.md b/docs/MUSEBENCH.md index dd72d99..4ac28ab 100644 --- a/docs/MUSEBENCH.md +++ b/docs/MUSEBENCH.md @@ -307,6 +307,19 @@ python scripts/musebench/report.py --root ~/data/musebench python scripts/musebench/verify.py --root ~/data/musebench all ``` +The audit re-runs each failed track chunk by chunk and is decode-bound; it can be run in +parallel on shards of the failure list and merged back with the same schema: + +```bash +python scripts/musebench/merge_audits.py split --shards 4 \ + --failures ~/data/musebench/results/artifactnet_v94/artifactnet/inference_failures.json \ + --out-dir ~/data/musebench/results/artifactnet_v94/audit_shards +python scripts/musebench/merge_audits.py merge \ + --failures ~/data/musebench/results/artifactnet_v94/artifactnet/inference_failures.json \ + --out ~/data/musebench/results/artifactnet_v94/nonfinite_audit.json \ + ~/data/musebench/results/artifactnet_v94/audit_shards/audit_0.json +``` + `report.py` writes `musebench_report.json` and `.md` next to the run: human (Jamendo) FPR, FMA-OOD accuracy, clean-AI FNR per generator, co-created FNR, FNR per transform and per transform × generator, AUROC against the Jamendo negatives, Wilson 95 % intervals, all for diff --git a/scripts/musebench/merge_audits.py b/scripts/musebench/merge_audits.py new file mode 100644 index 0000000..8ab5a22 --- /dev/null +++ b/scripts/musebench/merge_audits.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# Created: 2026-11-04 +# Purpose: Split an inference_failures.json into N shards for parallel runs of +# scripts/audit_artifactnet_nonfinite.py, and merge the shard audits back into one +# report with the same schema (summary fields recomputed, original failure order). +# Dependencies: standard library only. +# Test Status: tests/test_musebench_merge_audits.py +"""Shard / merge helper for the ArtifactNet non-finite chunk audit. + + python scripts/musebench/merge_audits.py split --failures F.json --shards 4 --out-dir D + python scripts/musebench/merge_audits.py merge --failures F.json --out A.json D/audit_*.json +""" +from __future__ import annotations + +import os as _os +import sys as _sys + +# This directory holds select.py, which shadows the stdlib ``select`` module when the +# directory is on sys.path (``python scripts/musebench/x.py`` without -I). Drop it. +if _sys.path and _os.path.abspath(_sys.path[0] or ".") == _os.path.dirname(_os.path.abspath(__file__)): + _sys.path.pop(0) + +import argparse # noqa: E402 +import json # noqa: E402 +import statistics # noqa: E402 +import sys # noqa: E402 +from pathlib import Path # noqa: E402 + + +def split(failures: list[dict], n: int) -> list[list[dict]]: + """Round-robin over the inference failures only (the audit ignores other stages).""" + inf = [f for f in failures if f.get("stage") == "inference"] + return [inf[i::n] for i in range(n)] + + +def merge(failures: list[dict], audits: list[dict]) -> dict: + by_id = {} + for a in audits: + for t in a["tracks"]: + if t["track_id"] in by_id: + raise ValueError(f"track audited twice: {t['track_id']}") + by_id[t["track_id"]] = t + wanted = [f["track_id"] for f in failures if f.get("stage") == "inference"] + missing = [t for t in wanted if t not in by_id] + if missing: + raise ValueError(f"{len(missing)} inference failures were not audited (e.g. {missing[0]})") + extra = set(by_id) - set(wanted) + if extra: + raise ValueError(f"{len(extra)} audited tracks are not inference failures") + tracks = [by_id[t] for t in wanted] + chunks = [c for t in tracks for c in t["chunks"]] + nonfinite = [c for c in chunks if not c["finite"]] + dbfs = [c["rms_dbfs"] for c in nonfinite] + return { + "tracks_audited": len(tracks), + "chunks_audited": len(chunks), + "nonfinite_chunks": len(nonfinite), + "tracks_with_at_least_four_finite_chunks": sum(t["valid_chunks"] >= 4 for t in tracks), + "nonfinite_rms_dbfs": { + "min": min(dbfs, default=None), + "median": float(statistics.median(dbfs)) if dbfs else None, + "max": max(dbfs, default=None), + }, + "tracks": tracks, + "merged_from_shards": len(audits), + } + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Shard / merge the ArtifactNet non-finite audit.") + sub = ap.add_subparsers(dest="cmd", required=True) + s = sub.add_parser("split") + s.add_argument("--failures", type=Path, required=True) + s.add_argument("--shards", type=int, required=True) + s.add_argument("--out-dir", type=Path, required=True) + m = sub.add_parser("merge") + m.add_argument("--failures", type=Path, required=True) + m.add_argument("--out", type=Path, required=True) + m.add_argument("audits", type=Path, nargs="+") + args = ap.parse_args(argv) + failures = json.loads(args.failures.read_text(encoding="utf-8")) + if args.cmd == "split": + args.out_dir.mkdir(parents=True, exist_ok=True) + for i, part in enumerate(split(failures, args.shards)): + (args.out_dir / f"failures_{i}.json").write_text(json.dumps(part, indent=1) + "\n") + print(f"shard {i}: {len(part)} failures") + return 0 + merged = merge(failures, [json.loads(p.read_text(encoding="utf-8")) for p in args.audits]) + args.out.write_text(json.dumps(merged, indent=2, ensure_ascii=False) + "\n") + print(f"merged {merged['tracks_audited']} tracks from {len(args.audits)} shards") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_musebench_merge_audits.py b/tests/test_musebench_merge_audits.py new file mode 100644 index 0000000..138dbcb --- /dev/null +++ b/tests/test_musebench_merge_audits.py @@ -0,0 +1,47 @@ +# Created: 2026-11-04 +# Purpose: Cover scripts/musebench/merge_audits.py (sharding and lossless merge of audits). +# Dependencies: pytest +# Test Status: passing +import pytest + +from scripts.musebench import merge_audits as ma + + +def failures(n): + return [{"track_id": f"t{i}", "stage": "inference"} for i in range(n)] + \ + [{"track_id": "dec", "stage": "decode"}] + + +def audit_of(ids, finite=5): + return {"tracks": [{"track_id": t, "valid_chunks": finite, "total_chunks": 7, + "chunks": [{"finite": j < finite, "rms_dbfs": -90.0 - j} for j in range(7)]} + for t in ids]} + + +def test_split_round_robin_skips_non_inference(): + parts = ma.split(failures(10), 3) + assert [len(p) for p in parts] == [4, 3, 3] + assert sorted(f["track_id"] for p in parts for f in p) == sorted(f"t{i}" for i in range(10)) + + +def test_merge_restores_order_and_recomputes_summary(): + fl = failures(6) + parts = ma.split(fl, 2) + audits = [audit_of([f["track_id"] for f in parts[0]], finite=5), + audit_of([f["track_id"] for f in parts[1]], finite=2)] + m = ma.merge(fl, audits) + assert [t["track_id"] for t in m["tracks"]] == [f"t{i}" for i in range(6)] + assert m["tracks_audited"] == 6 and m["chunks_audited"] == 42 + assert m["nonfinite_chunks"] == 3 * 2 + 3 * 5 + assert m["tracks_with_at_least_four_finite_chunks"] == 3 + assert m["nonfinite_rms_dbfs"]["max"] == -92.0 and m["merged_from_shards"] == 2 + + +def test_merge_rejects_missing_duplicate_or_extra_tracks(): + fl = failures(3) + with pytest.raises(ValueError, match="not audited"): + ma.merge(fl, [audit_of(["t0", "t1"])]) + with pytest.raises(ValueError, match="twice"): + ma.merge(fl, [audit_of(["t0", "t1", "t2"]), audit_of(["t0"])]) + with pytest.raises(ValueError, match="not inference failures"): + ma.merge(fl, [audit_of(["t0", "t1", "t2", "zz"])]) From deffe108049b615bf97fdef3dcf38a774659b3ca Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 11:40:09 +0900 Subject: [PATCH 5/6] musebench: import external scorer output; report honours explicit decisions import_scores.py maps a JSONL of per-file scores (path, probability, optional decision field) onto the eval cohort and writes the run layout report.py reads; unknown/unreliable verdicts, errors and missing lines are recorded failures. report.py uses a per-track decision_ai when present (pipelines whose verdict includes a rescue gate), AUROC still from the probability. Needed to report a non-ONNX detector on the same cohort. --- scripts/musebench/import_scores.py | 113 ++++++++++++++++++++++++++ scripts/musebench/report.py | 22 ++++- tests/test_musebench_import_scores.py | 64 +++++++++++++++ 3 files changed, 195 insertions(+), 4 deletions(-) create mode 100644 scripts/musebench/import_scores.py create mode 100644 tests/test_musebench_import_scores.py diff --git a/scripts/musebench/import_scores.py b/scripts/musebench/import_scores.py new file mode 100644 index 0000000..1245d3f --- /dev/null +++ b/scripts/musebench/import_scores.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# Created: 2026-11-04 +# Purpose: Turn per-file scores from an external scorer (JSONL: path, probability, verdict, error) +# into the run-directory layout report.py reads, so detectors that do not run inside +# artifactbench.bench can be reported on the same MuseBench eval cohort. +# Dependencies: standard library only. +# Test Status: tests/test_musebench_import_scores.py +"""Import external MuseBench scores. + + python scripts/musebench/import_scores.py --root ~/data/musebench \\ + --scores results.jsonl --run-dir ~/data/musebench/results/ \\ + --model-name "" --threshold 0.2 --decision-field verdict + +Each JSONL line must carry ``path`` (as listed in the eval manifest) and ``probability``. +With ``--decision-field``, a value equal to ``--ai-value`` marks the track as called AI and +any value in ``--fail-values`` (or a missing probability / an ``error``) makes it a recorded +failure. Without it, report.py applies ``probability >= threshold``. +""" +from __future__ import annotations + +import os as _os +import sys as _sys + +# This directory holds select.py, which shadows the stdlib ``select`` module when the +# directory is on sys.path (``python scripts/musebench/x.py`` without -I). Drop it. +if _sys.path and _os.path.abspath(_sys.path[0] or ".") == _os.path.dirname(_os.path.abspath(__file__)): + _sys.path.pop(0) + +import argparse # noqa: E402 +import hashlib # noqa: E402 +import json # noqa: E402 +import os # noqa: E402 +import sys # noqa: E402 +from pathlib import Path # noqa: E402 + +DEFAULT_ROOT = Path(os.environ.get("MUSEBENCH_ROOT", "~/data/musebench")).expanduser() +EVAL_MANIFEST_REL = Path("manifests") / "musebench_eval_manifest.json" + + +def convert(eval_rows: list[dict], score_lines: list[dict], *, decision_field: str | None, + ai_value: str = "ai", fail_values: tuple[str, ...] = ("unknown", "unreliable"), + ) -> tuple[list[dict], list[dict], list[str]]: + """Return (track_probs, failures, unmatched score paths).""" + by_path = {r["path"]: r for r in eval_rows} + seen: dict[str, dict] = {} + unmatched = [] + for line in score_lines: + row = by_path.get(line.get("path")) + if row is None: + unmatched.append(line.get("path")) + continue + if row["track_id"] in seen: + raise ValueError(f"duplicate score for {line['path']!r}") + seen[row["track_id"]] = line + probs, failures = [], [] + for row in eval_rows: + line = seen.get(row["track_id"]) + base = {"track_id": row["track_id"], "source": row["source"]} + if line is None: + failures.append({**base, "stage": "missing", "error": "no score line for this path"}) + continue + p = line.get("probability") + decision = line.get(decision_field) if decision_field else None + if line.get("error") or p is None or (decision_field and decision in fail_values): + failures.append({**base, "stage": "inference", + "error": line.get("error") or f"{decision_field}={decision}, probability={p}"}) + continue + entry = {**base, "prob": float(p), "label": 1 if row["label"] == "ai" else 0} + if decision_field: + entry["decision_ai"] = decision == ai_value + probs.append(entry) + return probs, failures, unmatched + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Import external per-file scores for report.py.") + ap.add_argument("--root", type=Path, default=DEFAULT_ROOT) + ap.add_argument("--scores", type=Path, required=True, help="JSONL with path/probability/...") + ap.add_argument("--run-dir", type=Path, required=True) + ap.add_argument("--model-dir", default="artifactnet") + ap.add_argument("--model-name", required=True) + ap.add_argument("--threshold", type=float, required=True) + ap.add_argument("--decision-field", default=None) + ap.add_argument("--ai-value", default="ai") + ap.add_argument("--fail-values", nargs="*", default=["unknown", "unreliable"]) + args = ap.parse_args(argv) + man_path = args.root / EVAL_MANIFEST_REL + eval_rows = json.loads(man_path.read_text(encoding="utf-8"))["tracks"] + with open(args.scores, encoding="utf-8") as f: + lines = [json.loads(x) for x in f if x.strip()] + probs, failures, unmatched = convert(eval_rows, lines, decision_field=args.decision_field, + ai_value=args.ai_value, fail_values=tuple(args.fail_values)) + out = args.run_dir / args.model_dir + out.mkdir(parents=True, exist_ok=True) + (out / "track_probs.json").write_text(json.dumps(probs) + "\n", encoding="utf-8") + (out / "inference_failures.json").write_text(json.dumps(failures, indent=1) + "\n", encoding="utf-8") + run_manifest = { + "imported_from": str(args.scores), + "scores_sha256": hashlib.sha256(args.scores.read_bytes()).hexdigest(), + "input_manifest": str(man_path), + "input_manifest_sha256": hashlib.sha256(man_path.read_bytes()).hexdigest(), + "models": [args.model_dir], "model_name": args.model_name, + "threshold": args.threshold, "decision_field": args.decision_field, + "n_per_source": 0, "selected_track_ids": [r["track_id"] for r in eval_rows], + } + (args.run_dir / "run_manifest.json").write_text(json.dumps(run_manifest, indent=1) + "\n", + encoding="utf-8") + print(f"imported {len(probs)} scores, {len(failures)} failures, {len(unmatched)} unmatched paths") + return 1 if unmatched else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/musebench/report.py b/scripts/musebench/report.py index 0de47d0..2bb4ffd 100644 --- a/scripts/musebench/report.py +++ b/scripts/musebench/report.py @@ -100,7 +100,7 @@ def split_of(row: dict) -> str: def build_report(eval_rows: list[dict], probs: dict[str, float], failures: list[dict], - threshold: float = 0.5) -> dict: + threshold: float = 0.5, decisions: dict[str, bool] | None = None) -> dict: """Pure function: eval manifest rows + {track_id: prob} + failures -> report dict.""" failed = {f["track_id"] for f in failures} scored = [dict(r, prob=probs[r["track_id"]]) for r in eval_rows if r["track_id"] in probs] @@ -111,6 +111,9 @@ def rows(split: str, pred) -> list[dict]: return [r for r in scored if pred(r) and (split == "all" or split_of(r) == split)] def called_ai(r): + # An explicit per-track decision (e.g. a pipeline with a rescue gate) overrides prob >= threshold. + if decisions is not None and r["track_id"] in decisions: + return decisions[r["track_id"]] return r["prob"] >= threshold def neg_probs(split): @@ -123,7 +126,8 @@ def ai_block(split, pred): out["prob_median"] = sorted(r["prob"] for r in rs)[len(rs) // 2] if rs else None return out - report: dict = {"threshold": threshold, "splits": {}} + report: dict = {"threshold": threshold, "splits": {}, + "decision_rule": "explicit per-track decision" if decisions else f"prob >= {threshold}"} for split in SPLITS: hum = rows(split, lambda r: r["source"] == HUMAN_NEG) fma = rows(split, lambda r: r["source"] == FMA) @@ -239,6 +243,14 @@ def load_run(run_dir: Path, model_dir: str = "artifactnet") -> tuple[dict[str, f return probs, failures, run_manifest +def load_decisions(result_dir: Path, model_dir: str) -> dict[str, bool] | None: + """Explicit per-track decisions (``decision_ai``) if the score file carries them.""" + rows = json.loads((result_dir / model_dir / "track_probs.json").read_text(encoding="utf-8")) + if rows and all("decision_ai" in t for t in rows): + return {t["track_id"]: bool(t["decision_ai"]) for t in rows} + return None + + def atomic_write(path: Path, text: str) -> None: tmp = path.with_suffix(path.suffix + ".tmp") tmp.write_text(text, encoding="utf-8") @@ -250,12 +262,14 @@ def build_full_report(eval_rows: list[dict], run_dir: Path, model_dir: str = "ar """Primary = declared finite-chunk policy when present, else strict; strict kept as sensitivity.""" probs, failures, run_manifest = load_run(run_dir, model_dir) threshold = run_manifest.get("threshold", 0.5) - strict = build_report(eval_rows, probs, failures, threshold=threshold) + strict = build_report(eval_rows, probs, failures, threshold=threshold, + decisions=load_decisions(run_dir, model_dir)) out = {"strict": strict} finite_dir = run_dir / finite_subdir if (finite_dir / model_dir / "track_probs.json").exists(): f_probs, f_failures, _ = load_run_scores(finite_dir, model_dir) - out["primary"] = build_report(eval_rows, f_probs, f_failures, threshold=threshold) + out["primary"] = build_report(eval_rows, f_probs, f_failures, threshold=threshold, + decisions=load_decisions(finite_dir, model_dir)) policy_path = finite_dir / model_dir / "finite_chunk_policy.json" policy = json.loads(policy_path.read_text(encoding="utf-8")) if policy_path.exists() else {} out["primary_policy"] = {k: v for k, v in policy.items() if k != "recovered"} diff --git a/tests/test_musebench_import_scores.py b/tests/test_musebench_import_scores.py new file mode 100644 index 0000000..b57836e --- /dev/null +++ b/tests/test_musebench_import_scores.py @@ -0,0 +1,64 @@ +# Created: 2026-11-04 +# Purpose: Cover scripts/musebench/import_scores.py and report.py's explicit-decision rule. +# Dependencies: pytest +# Test Status: passing +import json + +import pytest + +from scripts.musebench import import_scores as im +from scripts.musebench import report as rp +from test_musebench_report import cohort # tests/ is on sys.path (rootdir mode) + + +def eval_rows(): + rows, _, _ = cohort() + for r in rows: + r["path"] = f"/a/{r['track_id']}.wav" + return rows + + +def test_convert_maps_paths_decisions_and_failures(): + rows = eval_rows() + lines = [{"path": "/a/j0.wav", "probability": 0.9, "verdict": "real"}, # rescued -> human + {"path": "/a/m0.wav", "probability": 0.1, "verdict": "ai"}, + {"path": "/a/m1.wav", "probability": 0.5, "verdict": "unknown"}, + {"path": "/a/m2.wav", "probability": None, "verdict": "ai"}, + {"path": "/a/m3.wav", "probability": 0.7, "verdict": "ai", "error": "boom"}, + {"path": "/elsewhere.wav", "probability": 0.2, "verdict": "ai"}] + probs, failures, unmatched = im.convert(rows, lines, decision_field="verdict") + got = {p["track_id"]: p["decision_ai"] for p in probs} + assert got == {"j0": False, "m0": True} + stages = {f["track_id"]: f["stage"] for f in failures} + assert stages["m1"] == stages["m2"] == stages["m3"] == "inference" + assert stages["js"] == "missing" and len(failures) == len(rows) - 2 + assert unmatched == ["/elsewhere.wav"] + with pytest.raises(ValueError, match="duplicate"): + im.convert(rows, lines[:1] * 2, decision_field="verdict") + + +def test_report_uses_explicit_decisions_over_threshold(): + rows, probs, failures = cohort() + decisions = {tid: False for tid in probs} # call everything human + r = rp.build_report(rows, probs, failures, threshold=0.5, decisions=decisions) + assert r["splits"]["all"]["human_jamendo_fpr"]["k"] == 0 + assert r["splits"]["all"]["clean_ai_fnr"]["rate"] == 1.0 + assert r["decision_rule"] == "explicit per-track decision" + # AUROC still comes from probabilities. + assert r["splits"]["all"]["auroc_clean_ai_vs_jamendo"] == \ + rp.build_report(rows, probs, failures)["splits"]["all"]["auroc_clean_ai_vs_jamendo"] + + +def test_main_writes_run_dir_that_report_reads(tmp_path): + rows = eval_rows() + (tmp_path / "manifests").mkdir() + (tmp_path / im.EVAL_MANIFEST_REL).write_text(json.dumps({"tracks": rows})) + scores = tmp_path / "s.jsonl" + scores.write_text("\n".join(json.dumps({"path": r["path"], "probability": 0.3, "verdict": "real"}) + for r in rows) + "\n") + run = tmp_path / "results" / "x" + assert im.main(["--root", str(tmp_path), "--scores", str(scores), "--run-dir", str(run), + "--model-name", "Det", "--threshold", "0.2", "--decision-field", "verdict"]) == 0 + full = rp.build_full_report(rows, run) + assert full["primary"]["coverage"]["scored"] == len(rows) + assert full["primary"]["splits"]["all"]["human_jamendo_fpr"]["k"] == 0 # verdict, not 0.3 >= 0.2 From 6e9f49843e06c8e205a54d3849c38dbfe1c230f4 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 12:41:14 +0900 Subject: [PATCH 6/6] fix: finite-chunk builder keeps failures the chunk audit does not cover build_artifactnet_finite_chunk_result.py copied only audited (inference) failures into the finite-chunk result, so a strict decode failure vanished from both scores and failures (MuseBench run: 32,999 of 33,000 tracks accounted for). Non-audited strict failures are now carried over. MuseBench v9.4 baseline complete: 30,767 of 33,000 eval rows scored under the declared finite-chunk policy (7,506 recovered), 2,233 failures; verify.py all passes. --- .../build_artifactnet_finite_chunk_result.py | 5 +++ ...est_finite_chunk_builder_keeps_failures.py | 34 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 tests/test_finite_chunk_builder_keeps_failures.py diff --git a/scripts/build_artifactnet_finite_chunk_result.py b/scripts/build_artifactnet_finite_chunk_result.py index 8a59319..5d7022b 100644 --- a/scripts/build_artifactnet_finite_chunk_result.py +++ b/scripts/build_artifactnet_finite_chunk_result.py @@ -53,6 +53,11 @@ def main(): "policy": f"requires at least {args.min_valid_chunks} finite chunks", }) + # Failures the chunk audit does not cover (decode, resolve, ...) stay failures; dropping + # them would make tracks silently disappear from the result. + audited = {track["track_id"] for track in audit["tracks"]} + unresolved.extend(f for f in strict_failures if f["track_id"] not in audited) + args.output.mkdir(parents=True, exist_ok=True) model_dir = args.output / "artifactnet" model_dir.mkdir(exist_ok=True) diff --git a/tests/test_finite_chunk_builder_keeps_failures.py b/tests/test_finite_chunk_builder_keeps_failures.py new file mode 100644 index 0000000..34cddc5 --- /dev/null +++ b/tests/test_finite_chunk_builder_keeps_failures.py @@ -0,0 +1,34 @@ +# Created: 2026-11-04 +# Purpose: build_artifactnet_finite_chunk_result.py must keep strict failures the chunk audit +# does not cover (e.g. decode failures) instead of dropping those tracks. +# Dependencies: pytest +# Test Status: passing +import json +import subprocess +import sys +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "build_artifactnet_finite_chunk_result.py" + + +def test_decode_failures_survive_and_recovery_applies(tmp_path): + rows = [{"track_id": t, "label": "ai", "source": "s"} for t in ("ok", "nan_rec", "nan_bad", "dec")] + (tmp_path / "m.json").write_text(json.dumps({"tracks": rows})) + strict = tmp_path / "strict" / "artifactnet" + strict.mkdir(parents=True) + (strict / "track_probs.json").write_text(json.dumps([{"track_id": "ok", "prob": 0.9, "label": 1, "source": "s"}])) + (strict / "inference_failures.json").write_text(json.dumps([ + {"track_id": "nan_rec", "source": "s", "stage": "inference"}, + {"track_id": "nan_bad", "source": "s", "stage": "inference"}, + {"track_id": "dec", "source": "s", "stage": "decode"}])) + audit = {"tracks": [{"track_id": "nan_rec", "valid_chunks": 5, "total_chunks": 7, "finite_chunk_median": 0.7}, + {"track_id": "nan_bad", "valid_chunks": 2, "total_chunks": 7, "finite_chunk_median": 0.1}]} + (tmp_path / "audit.json").write_text(json.dumps(audit)) + out = tmp_path / "out" + subprocess.run([sys.executable, str(SCRIPT), "--manifest", str(tmp_path / "m.json"), + "--strict-result", str(tmp_path / "strict"), "--chunk-audit", str(tmp_path / "audit.json"), + "--output", str(out)], check=True, capture_output=True) + probs = {p["track_id"] for p in json.loads((out / "artifactnet" / "track_probs.json").read_text())} + fails = {f["track_id"]: f["stage"] for f in json.loads((out / "artifactnet" / "inference_failures.json").read_text())} + assert probs == {"ok", "nan_rec"} + assert fails == {"nan_bad": "inference", "dec": "decode"}