Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions scripts/musebench/cohort.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
# Created: 2026-11-05
# Purpose: Evaluation cohorts for MuseBench reports: a subset of the eval manifest named by track id (for example a
# stratified pilot sample), shared by import_scores.py and report.py so both restrict to the same rows and
# a report records which cohort it describes.
# Dependencies: standard library only.
# Test Status: tests/test_musebench_cohort.py
"""A cohort file is JSON: either a list of track ids, or an object with a ``track_ids`` list.

The cohort fingerprint is the sha256 of the sorted, newline-joined ids, so the same set of tracks has the same
fingerprint whichever file or order it came from; compare.py relies on it to refuse mismatched reports.
"""
from __future__ import annotations

import hashlib
import json
from pathlib import Path


def load(path: Path) -> list[str]:
"""Track ids of a cohort file, in file order. Raises ValueError on a malformed, empty or duplicated list."""
data = json.loads(Path(path).read_text(encoding="utf-8"))
ids = data.get("track_ids") if isinstance(data, dict) else data
if not isinstance(ids, list) or not all(isinstance(t, str) and t for t in ids):
raise ValueError(f"{path}: a cohort is a JSON list of track ids or an object with a 'track_ids' list")
if not ids:
raise ValueError(f"{path}: the cohort is empty")
if len(set(ids)) != len(ids):
raise ValueError(f"{path}: duplicated track ids in the cohort")
return ids


def fingerprint(ids: list[str]) -> str:
return hashlib.sha256("\n".join(sorted(ids)).encode("utf-8")).hexdigest()


def info(path: Path, ids: list[str]) -> dict:
return {"path": str(path), "size": len(ids), "sha256": fingerprint(ids)}


def restrict(eval_rows: list[dict], ids: list[str]) -> list[dict]:
"""Eval rows of the cohort, in eval-manifest order. Every cohort id must be an eval row."""
wanted = set(ids)
rows = [r for r in eval_rows if r["track_id"] in wanted]
if len(rows) != len(wanted):
unknown = sorted(wanted - {r["track_id"] for r in rows})
raise ValueError(f"{len(unknown)} cohort track ids are not in the eval manifest, e.g. {unknown[:5]}")
return rows

26 changes: 24 additions & 2 deletions scripts/musebench/import_scores.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# into the run-directory layout report.py reads, so detectors that do not run inside
# artifactbench.bench can be reported on the same MuseBench eval cohort.
# Dependencies: standard library only.
# Test Status: tests/test_musebench_import_scores.py
# Test Status: tests/test_musebench_import_scores.py, tests/test_musebench_cohort.py
"""Import external MuseBench scores.

python scripts/musebench/import_scores.py --root ~/data/musebench \\
Expand All @@ -31,8 +31,14 @@
import json # noqa: E402
import os # noqa: E402
import sys # noqa: E402
import importlib.util # noqa: E402
from pathlib import Path # noqa: E402

# cohort.py by file path: this directory is not on sys.path when the script runs (see above).
_spec = importlib.util.spec_from_file_location("musebench_cohort", Path(__file__).with_name("cohort.py"))
cohort = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(cohort)

DEFAULT_ROOT = Path(os.environ.get("MUSEBENCH_ROOT", "~/data/musebench")).expanduser()
EVAL_MANIFEST_REL = Path("manifests") / "musebench_eval_manifest.json"

Expand Down Expand Up @@ -83,11 +89,24 @@ def main(argv: list[str] | None = None) -> int:
ap.add_argument("--decision-field", default=None)
ap.add_argument("--ai-value", default="ai")
ap.add_argument("--fail-values", nargs="*", default=["unknown", "unreliable"])
ap.add_argument("--cohort", type=Path, default=None,
help="JSON list of track ids (or {\"track_ids\": [...]}): import only these eval rows; "
"scores of other eval rows are ignored, coverage is relative to the cohort")
args = ap.parse_args(argv)
man_path = args.root / EVAL_MANIFEST_REL
eval_rows = json.loads(man_path.read_text(encoding="utf-8"))["tracks"]
with open(args.scores, encoding="utf-8") as f:
lines = [json.loads(x) for x in f if x.strip()]
cohort_info, outside = None, 0
if args.cohort is not None:
ids = cohort.load(args.cohort)
all_paths = {r["path"] for r in eval_rows}
eval_rows = cohort.restrict(eval_rows, ids)
kept = {r["path"] for r in eval_rows}
# a score for an eval row outside the cohort is expected (the scorer ran the full set), not unmatched
outside = sum(1 for x in lines if x.get("path") in all_paths and x.get("path") not in kept)
lines = [x for x in lines if x.get("path") in kept or x.get("path") not in all_paths]
cohort_info = cohort.info(args.cohort, ids)
probs, failures, unmatched = convert(eval_rows, lines, decision_field=args.decision_field,
ai_value=args.ai_value, fail_values=tuple(args.fail_values))
out = args.run_dir / args.model_dir
Expand All @@ -103,9 +122,12 @@ def main(argv: list[str] | None = None) -> int:
"threshold": args.threshold, "decision_field": args.decision_field,
"n_per_source": 0, "selected_track_ids": [r["track_id"] for r in eval_rows],
}
if cohort_info is not None:
run_manifest["cohort"] = cohort_info
(args.run_dir / "run_manifest.json").write_text(json.dumps(run_manifest, indent=1) + "\n",
encoding="utf-8")
print(f"imported {len(probs)} scores, {len(failures)} failures, {len(unmatched)} unmatched paths")
print(f"imported {len(probs)} scores, {len(failures)} failures, {len(unmatched)} unmatched paths"
+ (f", cohort {cohort_info['size']} rows ({outside} scores outside it ignored)" if cohort_info else ""))
return 1 if unmatched else 0


Expand Down
35 changes: 34 additions & 1 deletion scripts/musebench/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# human FPR, FMA-OOD accuracy, per-generator / co-created / per-transform FNR and
# AUROC at a fixed threshold, each for all, seen and unseen rows.
# Dependencies: standard library only.
# Test Status: tests/test_musebench_report.py
# Test Status: tests/test_musebench_report.py, tests/test_musebench_cohort.py
"""Build ``musebench_report.json`` and ``musebench_report.md`` from a bench run.

Inputs: the eval manifest (with ``seen_in_training`` from leak_map.py) and the run directory
Expand Down Expand Up @@ -33,8 +33,14 @@
import sys # noqa: E402
import time # noqa: E402
from collections import Counter, defaultdict # noqa: E402
import importlib.util # noqa: E402
from pathlib import Path # noqa: E402

# cohort.py by file path: this directory is not on sys.path when the script runs (see above).
_spec = importlib.util.spec_from_file_location("musebench_cohort", Path(__file__).with_name("cohort.py"))
cohort = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(cohort)

DEFAULT_ROOT = Path(os.environ.get("MUSEBENCH_ROOT", "~/data/musebench")).expanduser()
EVAL_MANIFEST_REL = Path("manifests") / "musebench_eval_manifest.json"
SPLITS = ("all", "seen", "unseen")
Expand Down Expand Up @@ -332,6 +338,23 @@ def full_markdown(full: dict, title: str) -> str:
return "\n".join(lines[:2] + head + lines[2:] + sens)


def select_cohort(eval_rows: list[dict], cohort_path: Path | None, run_dir: Path) -> tuple[list[dict], dict | None]:
"""Restrict to --cohort. A run imported for a cohort must be reported on that same cohort (sha256), so a
pilot run cannot be reported as if it covered the whole eval set and vice versa."""
rm_path = run_dir / "run_manifest.json"
run_cohort = json.loads(rm_path.read_text(encoding="utf-8")).get("cohort") if rm_path.exists() else None
if cohort_path is None:
if run_cohort:
raise ValueError(f"the run was imported for cohort {run_cohort['sha256'][:12]} "
f"({run_cohort['size']} rows); pass --cohort")
return eval_rows, None
ids = cohort.load(cohort_path)
info = cohort.info(cohort_path, ids)
if run_cohort and run_cohort["sha256"] != info["sha256"]:
raise ValueError(f"--cohort {info['sha256'][:12]} differs from the run's cohort {run_cohort['sha256'][:12]}")
return cohort.restrict(eval_rows, ids), info


def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(description="Build the MuseBench report from a bench run.")
ap.add_argument("--root", type=Path, default=DEFAULT_ROOT)
Expand All @@ -343,11 +366,21 @@ def main(argv: list[str] | None = None) -> int:
help="row field that splits seen/unseen, e.g. seen_in_training_<key> from "
"leak_map.py --model-key (joined from the runtime manifest when the eval "
"manifest lacks it)")
ap.add_argument("--cohort", type=Path, default=None,
help="JSON list of track ids (or {\"track_ids\": [...]}): report only these eval rows; "
"coverage is relative to the cohort and run.cohort records its size and sha256")
args = ap.parse_args(argv)
run_dir = args.run_dir or args.root / "results" / "artifactnet_v94"
eval_rows = json.loads((args.root / EVAL_MANIFEST_REL).read_text(encoding="utf-8"))["tracks"]
eval_rows = attach_seen_field(eval_rows, args.root, args.seen_field)
try:
eval_rows, cohort_info = select_cohort(eval_rows, args.cohort, run_dir)
except ValueError as exc:
print(f"report: {exc}", file=sys.stderr)
return 2
full = build_full_report(eval_rows, run_dir, args.model_dir, seen_field=args.seen_field)
if cohort_info is not None:
full["run"]["cohort"] = cohort_info
full["run"]["created_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
atomic_write(run_dir / "musebench_report.json", json.dumps(full, indent=1) + "\n")
atomic_write(run_dir / "musebench_report.md", full_markdown(full, args.title) + "\n")
Expand Down
115 changes: 115 additions & 0 deletions tests/test_musebench_cohort.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
# Created: 2026-11-05
# Purpose: Cover --cohort in scripts/musebench/import_scores.py and report.py (scripts/musebench/cohort.py).
# Dependencies: pytest
# Test Status: passing
import json

import pytest

from scripts.musebench import import_scores as im
from scripts.musebench import report as rp
from test_musebench_report import cohort as synthetic # tests/ is on sys.path (rootdir mode)

COHORT = im.cohort


def eval_rows():
rows, _, _ = synthetic()
for r in rows:
r["path"] = f"/a/{r['track_id']}.wav"
return rows


def write_root(tmp_path, rows):
(tmp_path / "manifests").mkdir()
(tmp_path / im.EVAL_MANIFEST_REL).write_text(json.dumps({"tracks": rows}))
scores = tmp_path / "s.jsonl"
# the scorer ran the whole eval set; "gone" has no score line
scores.write_text("".join(json.dumps({"path": r["path"], "probability": 0.3}) + "\n"
for r in rows if r["track_id"] != "gone"))
return scores


def run_import(tmp_path, scores, run, *extra):
return im.main(["--root", str(tmp_path), "--scores", str(scores), "--run-dir", str(run),
"--model-name", "Det", "--threshold", "0.5", *extra])


@pytest.mark.parametrize("form", ["list", "object"])
def test_cohort_file_forms_and_fingerprint(tmp_path, form):
ids = ["m0", "j1", "s0"]
p = tmp_path / "c.json"
p.write_text(json.dumps(ids if form == "list" else {"track_ids": ids, "note": "pilot"}))
assert COHORT.load(p) == ids
info = COHORT.info(p, ids)
assert info["size"] == 3 and info["sha256"] == COHORT.fingerprint(list(reversed(ids))) # order-free


@pytest.mark.parametrize("content", [[], ["a", "a"], {"ids": ["a"]}, [1, 2], "x"])
def test_malformed_cohorts_are_rejected(tmp_path, content):
p = tmp_path / "c.json"
p.write_text(json.dumps(content))
with pytest.raises(ValueError):
COHORT.load(p)


def test_unknown_cohort_id_is_an_error(tmp_path):
rows = eval_rows()
with pytest.raises(ValueError, match="not in the eval manifest"):
COHORT.restrict(rows, ["m0", "nope"])
scores = write_root(tmp_path, rows)
c = tmp_path / "c.json"
c.write_text(json.dumps(["m0", "nope"]))
with pytest.raises(ValueError):
run_import(tmp_path, scores, tmp_path / "run", "--cohort", str(c))


def test_import_and_report_on_a_cohort(tmp_path):
rows = eval_rows()
scores = write_root(tmp_path, rows)
ids = ["j0", "j3", "m0", "m3", "gone"] # "gone" has no score line -> missing failure
c = tmp_path / "pilot.json"
c.write_text(json.dumps({"track_ids": ids}))
run = tmp_path / "run"
assert run_import(tmp_path, scores, run, "--cohort", str(c)) == 0 # scores outside the cohort: not unmatched
rm = json.loads((run / "run_manifest.json").read_text())
assert rm["selected_track_ids"] == [r["track_id"] for r in rows if r["track_id"] in ids]
assert rm["cohort"] == {"path": str(c), "size": 5, "sha256": COHORT.fingerprint(ids)}
probs = json.loads((run / "artifactnet" / "track_probs.json").read_text())
failures = json.loads((run / "artifactnet" / "inference_failures.json").read_text())
assert len(probs) + len(failures) == len(ids) # scored + failed == cohort size
assert rp.main(["--root", str(tmp_path), "--run-dir", str(run), "--cohort", str(c)]) == 0
rep = json.loads((run / "musebench_report.json").read_text())
cov = rep["primary"]["coverage"]
assert (cov["eval_rows"], cov["scored"], cov["failures"], cov["missing"]) == (5, 4, 1, 0)
assert rep["run"]["cohort"]["sha256"] == COHORT.fingerprint(ids)
assert rep["primary"]["splits"]["all"]["human_jamendo_fpr"]["n"] == 2


def test_report_refuses_a_cohort_mismatch_with_the_run(tmp_path, capsys):
rows = eval_rows()
scores = write_root(tmp_path, rows)
a, b = tmp_path / "a.json", tmp_path / "b.json"
a.write_text(json.dumps(["m0", "m1"]))
b.write_text(json.dumps(["m0", "m2"]))
run = tmp_path / "run"
assert run_import(tmp_path, scores, run, "--cohort", str(a)) == 0
assert rp.main(["--root", str(tmp_path), "--run-dir", str(run), "--cohort", str(b)]) == 2
assert rp.main(["--root", str(tmp_path), "--run-dir", str(run)]) == 2 # cohort run needs --cohort
assert "pass --cohort" in capsys.readouterr().err


def test_full_cohort_report_is_unchanged_without_cohort(tmp_path):
rows = eval_rows()
scores = write_root(tmp_path, rows)
run = tmp_path / "run"
assert run_import(tmp_path, scores, run) == 0
rm = json.loads((run / "run_manifest.json").read_text())
assert "cohort" not in rm
assert rp.main(["--root", str(tmp_path), "--run-dir", str(run)]) == 0
rep = json.loads((run / "musebench_report.json").read_text())
assert "cohort" not in rep["run"]
expect = json.loads(json.dumps(rp.build_full_report(rows, run))) # pre-cohort code path, JSON round trip
for key in ("strict", "primary", "primary_policy"):
assert rep[key] == expect[key]
assert rep["primary"]["coverage"]["eval_rows"] == len(rows)
Loading