diff --git a/scripts/musebench/cohort.py b/scripts/musebench/cohort.py new file mode 100644 index 0000000..9714967 --- /dev/null +++ b/scripts/musebench/cohort.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# Created: 2026-11-05 +# Purpose: Evaluation cohorts for MuseBench reports: a subset of the eval manifest named by track id (for example a +# stratified pilot sample), shared by import_scores.py and report.py so both restrict to the same rows and +# a report records which cohort it describes. +# Dependencies: standard library only. +# Test Status: tests/test_musebench_cohort.py +"""A cohort file is JSON: either a list of track ids, or an object with a ``track_ids`` list. + +The cohort fingerprint is the sha256 of the sorted, newline-joined ids, so the same set of tracks has the same +fingerprint whichever file or order it came from; compare.py relies on it to refuse mismatched reports. +""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + + +def load(path: Path) -> list[str]: + """Track ids of a cohort file, in file order. Raises ValueError on a malformed, empty or duplicated list.""" + data = json.loads(Path(path).read_text(encoding="utf-8")) + ids = data.get("track_ids") if isinstance(data, dict) else data + if not isinstance(ids, list) or not all(isinstance(t, str) and t for t in ids): + raise ValueError(f"{path}: a cohort is a JSON list of track ids or an object with a 'track_ids' list") + if not ids: + raise ValueError(f"{path}: the cohort is empty") + if len(set(ids)) != len(ids): + raise ValueError(f"{path}: duplicated track ids in the cohort") + return ids + + +def fingerprint(ids: list[str]) -> str: + return hashlib.sha256("\n".join(sorted(ids)).encode("utf-8")).hexdigest() + + +def info(path: Path, ids: list[str]) -> dict: + return {"path": str(path), "size": len(ids), "sha256": fingerprint(ids)} + + +def restrict(eval_rows: list[dict], ids: list[str]) -> list[dict]: + """Eval rows of the cohort, in eval-manifest order. Every cohort id must be an eval row.""" + wanted = set(ids) + rows = [r for r in eval_rows if r["track_id"] in wanted] + if len(rows) != len(wanted): + unknown = sorted(wanted - {r["track_id"] for r in rows}) + raise ValueError(f"{len(unknown)} cohort track ids are not in the eval manifest, e.g. {unknown[:5]}") + return rows + diff --git a/scripts/musebench/import_scores.py b/scripts/musebench/import_scores.py index 1245d3f..daf3335 100644 --- a/scripts/musebench/import_scores.py +++ b/scripts/musebench/import_scores.py @@ -4,7 +4,7 @@ # into the run-directory layout report.py reads, so detectors that do not run inside # artifactbench.bench can be reported on the same MuseBench eval cohort. # Dependencies: standard library only. -# Test Status: tests/test_musebench_import_scores.py +# Test Status: tests/test_musebench_import_scores.py, tests/test_musebench_cohort.py """Import external MuseBench scores. python scripts/musebench/import_scores.py --root ~/data/musebench \\ @@ -31,8 +31,14 @@ import json # noqa: E402 import os # noqa: E402 import sys # noqa: E402 +import importlib.util # noqa: E402 from pathlib import Path # noqa: E402 +# cohort.py by file path: this directory is not on sys.path when the script runs (see above). +_spec = importlib.util.spec_from_file_location("musebench_cohort", Path(__file__).with_name("cohort.py")) +cohort = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(cohort) + DEFAULT_ROOT = Path(os.environ.get("MUSEBENCH_ROOT", "~/data/musebench")).expanduser() EVAL_MANIFEST_REL = Path("manifests") / "musebench_eval_manifest.json" @@ -83,11 +89,24 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--decision-field", default=None) ap.add_argument("--ai-value", default="ai") ap.add_argument("--fail-values", nargs="*", default=["unknown", "unreliable"]) + ap.add_argument("--cohort", type=Path, default=None, + help="JSON list of track ids (or {\"track_ids\": [...]}): import only these eval rows; " + "scores of other eval rows are ignored, coverage is relative to the cohort") args = ap.parse_args(argv) man_path = args.root / EVAL_MANIFEST_REL eval_rows = json.loads(man_path.read_text(encoding="utf-8"))["tracks"] with open(args.scores, encoding="utf-8") as f: lines = [json.loads(x) for x in f if x.strip()] + cohort_info, outside = None, 0 + if args.cohort is not None: + ids = cohort.load(args.cohort) + all_paths = {r["path"] for r in eval_rows} + eval_rows = cohort.restrict(eval_rows, ids) + kept = {r["path"] for r in eval_rows} + # a score for an eval row outside the cohort is expected (the scorer ran the full set), not unmatched + outside = sum(1 for x in lines if x.get("path") in all_paths and x.get("path") not in kept) + lines = [x for x in lines if x.get("path") in kept or x.get("path") not in all_paths] + cohort_info = cohort.info(args.cohort, ids) probs, failures, unmatched = convert(eval_rows, lines, decision_field=args.decision_field, ai_value=args.ai_value, fail_values=tuple(args.fail_values)) out = args.run_dir / args.model_dir @@ -103,9 +122,12 @@ def main(argv: list[str] | None = None) -> int: "threshold": args.threshold, "decision_field": args.decision_field, "n_per_source": 0, "selected_track_ids": [r["track_id"] for r in eval_rows], } + if cohort_info is not None: + run_manifest["cohort"] = cohort_info (args.run_dir / "run_manifest.json").write_text(json.dumps(run_manifest, indent=1) + "\n", encoding="utf-8") - print(f"imported {len(probs)} scores, {len(failures)} failures, {len(unmatched)} unmatched paths") + print(f"imported {len(probs)} scores, {len(failures)} failures, {len(unmatched)} unmatched paths" + + (f", cohort {cohort_info['size']} rows ({outside} scores outside it ignored)" if cohort_info else "")) return 1 if unmatched else 0 diff --git a/scripts/musebench/report.py b/scripts/musebench/report.py index e988bae..6bcef60 100644 --- a/scripts/musebench/report.py +++ b/scripts/musebench/report.py @@ -4,7 +4,7 @@ # human FPR, FMA-OOD accuracy, per-generator / co-created / per-transform FNR and # AUROC at a fixed threshold, each for all, seen and unseen rows. # Dependencies: standard library only. -# Test Status: tests/test_musebench_report.py +# Test Status: tests/test_musebench_report.py, tests/test_musebench_cohort.py """Build ``musebench_report.json`` and ``musebench_report.md`` from a bench run. Inputs: the eval manifest (with ``seen_in_training`` from leak_map.py) and the run directory @@ -33,8 +33,14 @@ import sys # noqa: E402 import time # noqa: E402 from collections import Counter, defaultdict # noqa: E402 +import importlib.util # noqa: E402 from pathlib import Path # noqa: E402 +# cohort.py by file path: this directory is not on sys.path when the script runs (see above). +_spec = importlib.util.spec_from_file_location("musebench_cohort", Path(__file__).with_name("cohort.py")) +cohort = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(cohort) + DEFAULT_ROOT = Path(os.environ.get("MUSEBENCH_ROOT", "~/data/musebench")).expanduser() EVAL_MANIFEST_REL = Path("manifests") / "musebench_eval_manifest.json" SPLITS = ("all", "seen", "unseen") @@ -332,6 +338,23 @@ def full_markdown(full: dict, title: str) -> str: return "\n".join(lines[:2] + head + lines[2:] + sens) +def select_cohort(eval_rows: list[dict], cohort_path: Path | None, run_dir: Path) -> tuple[list[dict], dict | None]: + """Restrict to --cohort. A run imported for a cohort must be reported on that same cohort (sha256), so a + pilot run cannot be reported as if it covered the whole eval set and vice versa.""" + rm_path = run_dir / "run_manifest.json" + run_cohort = json.loads(rm_path.read_text(encoding="utf-8")).get("cohort") if rm_path.exists() else None + if cohort_path is None: + if run_cohort: + raise ValueError(f"the run was imported for cohort {run_cohort['sha256'][:12]} " + f"({run_cohort['size']} rows); pass --cohort") + return eval_rows, None + ids = cohort.load(cohort_path) + info = cohort.info(cohort_path, ids) + if run_cohort and run_cohort["sha256"] != info["sha256"]: + raise ValueError(f"--cohort {info['sha256'][:12]} differs from the run's cohort {run_cohort['sha256'][:12]}") + return cohort.restrict(eval_rows, ids), info + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description="Build the MuseBench report from a bench run.") ap.add_argument("--root", type=Path, default=DEFAULT_ROOT) @@ -343,11 +366,21 @@ def main(argv: list[str] | None = None) -> int: help="row field that splits seen/unseen, e.g. seen_in_training_ from " "leak_map.py --model-key (joined from the runtime manifest when the eval " "manifest lacks it)") + ap.add_argument("--cohort", type=Path, default=None, + help="JSON list of track ids (or {\"track_ids\": [...]}): report only these eval rows; " + "coverage is relative to the cohort and run.cohort records its size and sha256") args = ap.parse_args(argv) run_dir = args.run_dir or args.root / "results" / "artifactnet_v94" eval_rows = json.loads((args.root / EVAL_MANIFEST_REL).read_text(encoding="utf-8"))["tracks"] eval_rows = attach_seen_field(eval_rows, args.root, args.seen_field) + try: + eval_rows, cohort_info = select_cohort(eval_rows, args.cohort, run_dir) + except ValueError as exc: + print(f"report: {exc}", file=sys.stderr) + return 2 full = build_full_report(eval_rows, run_dir, args.model_dir, seen_field=args.seen_field) + if cohort_info is not None: + full["run"]["cohort"] = cohort_info full["run"]["created_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) atomic_write(run_dir / "musebench_report.json", json.dumps(full, indent=1) + "\n") atomic_write(run_dir / "musebench_report.md", full_markdown(full, args.title) + "\n") diff --git a/tests/test_musebench_cohort.py b/tests/test_musebench_cohort.py new file mode 100644 index 0000000..be4c71b --- /dev/null +++ b/tests/test_musebench_cohort.py @@ -0,0 +1,115 @@ +# Created: 2026-11-05 +# Purpose: Cover --cohort in scripts/musebench/import_scores.py and report.py (scripts/musebench/cohort.py). +# Dependencies: pytest +# Test Status: passing +import json + +import pytest + +from scripts.musebench import import_scores as im +from scripts.musebench import report as rp +from test_musebench_report import cohort as synthetic # tests/ is on sys.path (rootdir mode) + +COHORT = im.cohort + + +def eval_rows(): + rows, _, _ = synthetic() + for r in rows: + r["path"] = f"/a/{r['track_id']}.wav" + return rows + + +def write_root(tmp_path, rows): + (tmp_path / "manifests").mkdir() + (tmp_path / im.EVAL_MANIFEST_REL).write_text(json.dumps({"tracks": rows})) + scores = tmp_path / "s.jsonl" + # the scorer ran the whole eval set; "gone" has no score line + scores.write_text("".join(json.dumps({"path": r["path"], "probability": 0.3}) + "\n" + for r in rows if r["track_id"] != "gone")) + return scores + + +def run_import(tmp_path, scores, run, *extra): + return im.main(["--root", str(tmp_path), "--scores", str(scores), "--run-dir", str(run), + "--model-name", "Det", "--threshold", "0.5", *extra]) + + +@pytest.mark.parametrize("form", ["list", "object"]) +def test_cohort_file_forms_and_fingerprint(tmp_path, form): + ids = ["m0", "j1", "s0"] + p = tmp_path / "c.json" + p.write_text(json.dumps(ids if form == "list" else {"track_ids": ids, "note": "pilot"})) + assert COHORT.load(p) == ids + info = COHORT.info(p, ids) + assert info["size"] == 3 and info["sha256"] == COHORT.fingerprint(list(reversed(ids))) # order-free + + +@pytest.mark.parametrize("content", [[], ["a", "a"], {"ids": ["a"]}, [1, 2], "x"]) +def test_malformed_cohorts_are_rejected(tmp_path, content): + p = tmp_path / "c.json" + p.write_text(json.dumps(content)) + with pytest.raises(ValueError): + COHORT.load(p) + + +def test_unknown_cohort_id_is_an_error(tmp_path): + rows = eval_rows() + with pytest.raises(ValueError, match="not in the eval manifest"): + COHORT.restrict(rows, ["m0", "nope"]) + scores = write_root(tmp_path, rows) + c = tmp_path / "c.json" + c.write_text(json.dumps(["m0", "nope"])) + with pytest.raises(ValueError): + run_import(tmp_path, scores, tmp_path / "run", "--cohort", str(c)) + + +def test_import_and_report_on_a_cohort(tmp_path): + rows = eval_rows() + scores = write_root(tmp_path, rows) + ids = ["j0", "j3", "m0", "m3", "gone"] # "gone" has no score line -> missing failure + c = tmp_path / "pilot.json" + c.write_text(json.dumps({"track_ids": ids})) + run = tmp_path / "run" + assert run_import(tmp_path, scores, run, "--cohort", str(c)) == 0 # scores outside the cohort: not unmatched + rm = json.loads((run / "run_manifest.json").read_text()) + assert rm["selected_track_ids"] == [r["track_id"] for r in rows if r["track_id"] in ids] + assert rm["cohort"] == {"path": str(c), "size": 5, "sha256": COHORT.fingerprint(ids)} + probs = json.loads((run / "artifactnet" / "track_probs.json").read_text()) + failures = json.loads((run / "artifactnet" / "inference_failures.json").read_text()) + assert len(probs) + len(failures) == len(ids) # scored + failed == cohort size + assert rp.main(["--root", str(tmp_path), "--run-dir", str(run), "--cohort", str(c)]) == 0 + rep = json.loads((run / "musebench_report.json").read_text()) + cov = rep["primary"]["coverage"] + assert (cov["eval_rows"], cov["scored"], cov["failures"], cov["missing"]) == (5, 4, 1, 0) + assert rep["run"]["cohort"]["sha256"] == COHORT.fingerprint(ids) + assert rep["primary"]["splits"]["all"]["human_jamendo_fpr"]["n"] == 2 + + +def test_report_refuses_a_cohort_mismatch_with_the_run(tmp_path, capsys): + rows = eval_rows() + scores = write_root(tmp_path, rows) + a, b = tmp_path / "a.json", tmp_path / "b.json" + a.write_text(json.dumps(["m0", "m1"])) + b.write_text(json.dumps(["m0", "m2"])) + run = tmp_path / "run" + assert run_import(tmp_path, scores, run, "--cohort", str(a)) == 0 + assert rp.main(["--root", str(tmp_path), "--run-dir", str(run), "--cohort", str(b)]) == 2 + assert rp.main(["--root", str(tmp_path), "--run-dir", str(run)]) == 2 # cohort run needs --cohort + assert "pass --cohort" in capsys.readouterr().err + + +def test_full_cohort_report_is_unchanged_without_cohort(tmp_path): + rows = eval_rows() + scores = write_root(tmp_path, rows) + run = tmp_path / "run" + assert run_import(tmp_path, scores, run) == 0 + rm = json.loads((run / "run_manifest.json").read_text()) + assert "cohort" not in rm + assert rp.main(["--root", str(tmp_path), "--run-dir", str(run)]) == 0 + rep = json.loads((run / "musebench_report.json").read_text()) + assert "cohort" not in rep["run"] + expect = json.loads(json.dumps(rp.build_full_report(rows, run))) # pre-cohort code path, JSON round trip + for key in ("strict", "primary", "primary_policy"): + assert rep[key] == expect[key] + assert rep["primary"]["coverage"]["eval_rows"] == len(rows)