From 6b90b0580816b1685d422665ec65c712b6b233b6 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 23:13:02 +0900 Subject: [PATCH 1/3] musebench: model-keyed leak map over a lineage of training manifests; report --seen-field; production verifier leak_map.py - references JSON may list `lineage_manifests` (the training manifests of every generation in a model's weight lineage) instead of a candidate-pool directory; rows are read by path, track id and origin_* fields, yielding SONICS stems, FMA ids and MTG-Jamendo ids. - `--model-key KEY` writes seen_in_training_KEY / seen_evidence_KEY / ... and leak/leak_report_KEY.json, leaving the unkeyed annotation of an earlier model untouched. report.py - `--seen-field` splits seen/unseen by any annotation field. The field is joined from the runtime manifest by track id when the eval manifest lacks it, so the eval manifest a run consumed is never rewritten (its sha256 is recorded in that run's run_manifest.json). verify.py - new `production` stage (part of `all`): raw score lines cover the eval cohort once each and are unchanged since import (scores_sha256), scored + failed == cohort, every score carries a boolean decision_ai, the keyed leak annotation is present/coherent and its report current, and the saved report equals a rebuild split by the keyed field. Assumption: which run directory and model key the production stage checks is read from /results/production_run.json, so no internal model name is committed to this public repo. --- scripts/musebench/leak_map.py | 217 ++++++++++++++++++---- scripts/musebench/report.py | 47 ++++- scripts/musebench/verify.py | 108 +++++++++++ tests/test_musebench_leak_map.py | 82 ++++++++ tests/test_musebench_verify_production.py | 113 +++++++++++ 5 files changed, 518 insertions(+), 49 deletions(-) create mode 100644 tests/test_musebench_verify_production.py diff --git a/scripts/musebench/leak_map.py b/scripts/musebench/leak_map.py index d25cc1e..90064ab 100644 --- a/scripts/musebench/leak_map.py +++ b/scripts/musebench/leak_map.py @@ -139,6 +139,81 @@ def scan_sonics(fake_dir: Path) -> dict[str, int]: return {f.name.rsplit(".", 1)[0]: f.stat().st_size for f in fake_dir.iterdir() if f.is_file()} +# --------------------------------------------------------------------------- lineage manifests +# A detector's real training history is a chain of training manifests (one per generation), not a +# candidate-pool directory. Their rows name files by path and sometimes by a derived track id +# (codec cells: "cell:::"), and derived rows point back with origin_* fields. +LINEAGE_SPLITS = ("train", "val", "test", "hardneg", "codec_cells", "rows", "tracks", "items") +LINEAGE_ROW_KEYS = ("path", "track_id", "origin_path", "origin_track_id", "source_path") +SONICS_STEM = re.compile(r"fake_\d+_(suno|udio)") +UNKNOWN_SIZE = -1 # a SONICS name taken from a manifest whose file is not on this host + + +def lineage_rows(data: object) -> list: + """Rows of one training manifest: a list, or a dict of split lists (a dict of lists is flattened).""" + if isinstance(data, list): + return data + if not isinstance(data, dict): + return [] + out: list = [] + for split in LINEAGE_SPLITS: + v = data.get(split) + if isinstance(v, dict): + v = [r for sub in v.values() if isinstance(sub, list) for r in sub] + if isinstance(v, list): + out.extend(v) + return out + + +def lineage_identity(value: object) -> tuple[str, object] | None: + """('sonics', stem) | ('fma', id) | ('mtg', id) | None for one path or track id. + + The identity is the last path component (after the last ':' for cell ids), up to the first '.'. + FMA and MTG-Jamendo are told apart by the corpus name appearing anywhere in the string; both + name their files by a bare numeric track id.""" + if not value: + return None + s = str(value) + stem = s.rsplit("/", 1)[-1].rsplit(":", 1)[-1].split(".")[0] + low = s.lower() + if SONICS_STEM.match(stem): + return "sonics", stem + if "fma" in low and stem.isdigit(): + return "fma", int(stem) + if "jamendo" in low and stem.isdigit(): + return "mtg", int(stem) + return None + + +def load_lineage(paths: list[Path]) -> tuple[dict[str, int], set[int], set[int], dict[str, int]]: + """(SONICS stem -> size, FMA ids, MTG-Jamendo ids, rows per manifest) over a lineage of manifests.""" + sonics: dict[str, int] = {} + fma: set[int] = set() + mtg: set[int] = set() + per_manifest: dict[str, int] = {} + for mp in paths: + rows = lineage_rows(json.loads(Path(mp).read_text(encoding="utf-8"))) + if not rows: + raise ValueError(f"{mp}: no rows (unsupported manifest shape)") + for r in rows: + values = [r] if isinstance(r, str) else [r.get(k) for k in LINEAGE_ROW_KEYS] if isinstance(r, dict) else [] + for v in values: + ident = lineage_identity(v) + if ident is None: + continue + kind, key = ident + if kind == "sonics": + if key not in sonics: + p = Path(str(v)) + sonics[key] = p.stat().st_size if p.is_absolute() and p.is_file() else UNKNOWN_SIZE + elif kind == "fma": + fma.add(key) + else: + mtg.add(key) + per_manifest[str(mp)] = len(rows) + return sonics, fma, mtg, per_manifest + + def ffprobe_duration(path: Path) -> float | None: try: out = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration", @@ -229,13 +304,33 @@ def annotate(tracks: list[dict], *, sonics: dict[str, int], fma_ids: set[int], return dict(stats) -def per_source_counts(tracks: list[dict]) -> dict[str, dict]: +def keyed(field: str, model_key: str | None) -> str: + """Field name for a model-specific annotation: seen_in_training -> seen_in_training_.""" + return f"{field}_{model_key}" if model_key else field + + +def annotate_keyed(tracks: list[dict], model_key: str, **kwargs) -> dict: + """Run annotate() on copies and store its fields under model-specific names, so the rows keep any + earlier (unkeyed) annotation untouched. Idempotent: stale keyed fields are dropped first.""" + if not model_key or not re.fullmatch(r"[a-z0-9_]+", model_key): + raise ValueError(f"model key must be [a-z0-9_]+, got {model_key!r}") + copies = [dict(t) for t in tracks] + stats = annotate(copies, **kwargs) + for t, c in zip(tracks, copies): + for f in LEAK_FIELDS: + t.pop(keyed(f, model_key), None) + if f in c: + t[keyed(f, model_key)] = c[f] + return stats + + +def per_source_counts(tracks: list[dict], seen_field: str = "seen_in_training") -> dict[str, dict]: out: dict[str, Counter] = defaultdict(Counter) for t in tracks: c = out[t["source"]] c["total"] += 1 - c["seen" if t["seen_in_training"] else "unseen"] += 1 - c[f"{t['musebench_role']}_{'seen' if t['seen_in_training'] else 'unseen'}"] += 1 + c["seen" if t[seen_field] else "unseen"] += 1 + c[f"{t['musebench_role']}_{'seen' if t[seen_field] else 'unseen'}"] += 1 return {k: {"total": v["total"], "seen": v["seen"], "unseen": v["unseen"], "eval_seen": v["eval_seen"], "eval_unseen": v["eval_unseen"], "train_pool_seen": v["train_pool_seen"], "train_pool_unseen": v["train_pool_unseen"]} @@ -256,73 +351,117 @@ def atomic_write_json(path: Path, obj: object) -> None: os.replace(tmp, path) +def report_rel(model_key: str | None) -> Path: + """leak/leak_report.json for the unkeyed (first) annotation, leak/leak_report_.json otherwise.""" + return REPORT_REL if not model_key else REPORT_REL.with_name(f"leak_report_{model_key}.json") + + +def load_references(refs: dict) -> dict: + """Reference identities from a references JSON. + + Two shapes: a *candidate pool* (``sonics_fake_dir`` + ``fma_id_lists`` + ``mtg_pool_folders``) + or a *lineage* (``lineage_manifests``: the training manifests of every generation in the model's + weight lineage, read row by row). Both need ``mtg_meta_tsv`` and ``mtg_audio_dir``.""" + mtg_meta = load_mtg_meta(Path(refs["mtg_meta_tsv"])) + mtg_all = scan_mtg_audio(Path(refs["mtg_audio_dir"]), None) + if refs.get("lineage_manifests"): + sonics, fma_ids, mtg_ids, per_manifest = load_lineage([Path(p) for p in refs["lineage_manifests"]]) + return dict(kind="lineage", sonics=sonics, fma_ids=fma_ids, mtg_meta=mtg_meta, mtg_all=mtg_all, + mtg_pool={i: p for i, p in mtg_all.items() if i in mtg_ids}, fma_files={}, + corpora={"lineage_manifests": per_manifest, "sonics_stems": len(sonics), + "fma_ids": len(fma_ids), "mtg_ids": len(mtg_ids), + "mtg_ids_local": sum(1 for i in mtg_ids if i in mtg_all), + "mtg_meta_tsv": refs["mtg_meta_tsv"], "mtg_audio_dir": refs["mtg_audio_dir"]}) + sonics = scan_sonics(Path(refs["sonics_fake_dir"])) + fma_ids: set[int] = set() + for p in refs["fma_id_lists"]: + fma_ids |= load_fma_ids(Path(p)) + fma_files = scan_fma_audio([Path(p) for p in refs.get("fma_audio_dirs", [])]) + mtg_pool = scan_mtg_audio(Path(refs["mtg_audio_dir"]), set(refs["mtg_pool_folders"])) + return dict(kind="pool", sonics=sonics, fma_ids=fma_ids, mtg_meta=mtg_meta, mtg_all=mtg_all, + mtg_pool=mtg_pool, fma_files=fma_files, + corpora={ + "sonics_fake_dir": {"path": refs["sonics_fake_dir"], "files": len(sonics)}, + "fma_id_lists": {"paths": refs["fma_id_lists"], "track_ids": len(fma_ids), + "audio_dirs": refs.get("fma_audio_dirs", []), + "local_files": len(fma_files)}, + "mtg_jamendo": {"meta_tsv": refs["mtg_meta_tsv"], "audio_dir": refs["mtg_audio_dir"], + "pool_folders": refs["mtg_pool_folders"], + "pool_tracks": len(mtg_pool), "local_tracks": len(mtg_all)}}) + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description="Annotate the MuseBench manifest with training overlap.") ap.add_argument("--root", type=Path, default=DEFAULT_ROOT) ap.add_argument("--references", type=Path, default=None, - help="JSON describing the detector and its training pool " + help="JSON describing the detector and its training pool or lineage " "(default: /leak/references.json)") + ap.add_argument("--model-key", default=None, + help="write seen_in_training_ etc. and leak/leak_report_.json, keeping the " + "unkeyed annotation of another model intact") ap.add_argument("--no-duration-check", action="store_true") args = ap.parse_args(argv) ref_path = args.references or args.root / "leak" / "references.json" refs = json.loads(ref_path.read_text(encoding="utf-8")) - - sonics = scan_sonics(Path(refs["sonics_fake_dir"])) - fma_ids: set[int] = set() - for p in refs["fma_id_lists"]: - fma_ids |= load_fma_ids(Path(p)) - fma_files = scan_fma_audio([Path(p) for p in refs.get("fma_audio_dirs", [])]) - mtg_meta = load_mtg_meta(Path(refs["mtg_meta_tsv"])) - mtg_dir = Path(refs["mtg_audio_dir"]) - mtg_pool = scan_mtg_audio(mtg_dir, set(refs["mtg_pool_folders"])) - mtg_all = scan_mtg_audio(mtg_dir, None) + ref = load_references(refs) + key = args.model_key + seen_field = keyed("seen_in_training", key) man_path = args.root / MANIFEST_REL manifest = json.loads(man_path.read_text(encoding="utf-8")) tracks = manifest["tracks"] - stats = annotate(tracks, sonics=sonics, fma_ids=fma_ids, mtg_meta=mtg_meta, - mtg_pool=mtg_pool, mtg_all=mtg_all, - generators_in_training=refs["generators_in_training"], - fma_files=fma_files, duration=None if args.no_duration_check else ffprobe_duration) - manifest["metadata"]["leak_annotation"] = { - "model": refs["model"]["name"], "report": str(args.root / REPORT_REL), - "annotated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} + kwargs = dict(sonics=ref["sonics"], fma_ids=ref["fma_ids"], mtg_meta=ref["mtg_meta"], + mtg_pool=ref["mtg_pool"], mtg_all=ref["mtg_all"], + generators_in_training=refs["generators_in_training"], fma_files=ref["fma_files"], + duration=None if args.no_duration_check else ffprobe_duration) + stats = annotate_keyed(tracks, key, **kwargs) if key else annotate(tracks, **kwargs) + rep_rel = report_rel(key) + note = {"model": refs["model"]["name"], "report": str(args.root / rep_rel), "seen_field": seen_field, + "annotated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} + if key: + manifest["metadata"].setdefault("leak_annotations", {})[key] = note + else: + manifest["metadata"]["leak_annotation"] = {k: v for k, v in note.items() if k != "seen_field"} atomic_write_json(man_path, manifest) + lineage = ref["kind"] == "lineage" report = { "model": refs["model"], + "model_key": key, + "seen_field": seen_field, "training_manifests": refs["training_manifests"], - "reference_corpora": { - "sonics_fake_dir": {"path": refs["sonics_fake_dir"], "files": len(sonics)}, - "fma_id_lists": {"paths": refs["fma_id_lists"], "track_ids": len(fma_ids), - "audio_dirs": refs.get("fma_audio_dirs", []), - "local_files": len(fma_files)}, - "mtg_jamendo": {"meta_tsv": refs["mtg_meta_tsv"], "audio_dir": refs["mtg_audio_dir"], - "pool_folders": refs["mtg_pool_folders"], - "pool_tracks": len(mtg_pool), "local_tracks": len(mtg_all)}, - }, + "reference_corpora": ref["corpora"], "method": { - "suno_udio": "SONICS file-name match (fake___), size compared", - "fma": "FMA track id (leading digits of the file name) in the training FMA id lists; " - "size compared with a local FMA copy where one exists", + "suno_udio": "SONICS file-name match (fake___), size compared" + + (" when the training file is on this host" if lineage else ""), + "fma": ("FMA track id (numeric file stem of a training row whose path or id names FMA)" + if lineage else + "FMA track id (leading digits of the file name) in the training FMA id lists; " + "size compared with a local FMA copy where one exists"), "jamendo": "normalised (title, artist) vs MTG-Jamendo raw.meta.tsv, restricted to the " - f"pool folders; duration within {DURATION_TOLERANCE_S}s" + + ("MTG ids named by training rows" if lineage else "pool folders") + + f"; duration within {DURATION_TOLERANCE_S}s" + (" (not checked)" if args.no_duration_check else ""), "inheritance": "transforms inherit from their clean track; co-created remixes from " "their human source; caption-conditioned generations do not", - "scope": "candidate pool of the training dataset builder: an upper bound on exposure", + "scope": ("union of the train/val/test rows of every manifest in the weight lineage: an upper " + "bound on exposure for the manifests listed, a lower bound if one is missing" + if lineage else + "candidate pool of the training dataset builder: an upper bound on exposure"), }, "generators_in_training": refs["generators_in_training"], "caveats": refs.get("caveats", []), "match_stats": stats, - "per_source": per_source_counts(tracks), - "totals": dict(Counter("seen" if t["seen_in_training"] else "unseen" for t in tracks)), + "per_source": per_source_counts(tracks, seen_field), + "totals": dict(Counter("seen" if t[seen_field] else "unseen" for t in tracks)), + "eval_seen": sum(1 for t in tracks if t[seen_field] and t.get("musebench_role") == "eval"), "manifest": str(man_path), "manifest_fingerprint": manifest_fingerprint(tracks), "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), } - atomic_write_json(args.root / REPORT_REL, report) - print(json.dumps({"match_stats": stats, "totals": report["totals"]}, indent=1)) + atomic_write_json(args.root / rep_rel, report) + print(json.dumps({"match_stats": stats, "totals": report["totals"], "eval_seen": report["eval_seen"], + "report": str(args.root / rep_rel)}, indent=1)) return 0 diff --git a/scripts/musebench/report.py b/scripts/musebench/report.py index 2bb4ffd..e988bae 100644 --- a/scripts/musebench/report.py +++ b/scripts/musebench/report.py @@ -95,12 +95,34 @@ def auroc(pos: list[float], neg: list[float]) -> float | None: # --------------------------------------------------------------------------- report -def split_of(row: dict) -> str: - return "seen" if row["seen_in_training"] else "unseen" +SEEN_FIELD = "seen_in_training" +RUNTIME_MANIFEST_REL = Path("manifests") / "musebench_runtime_manifest.json" + + +def split_of(row: dict, seen_field: str = SEEN_FIELD) -> str: + return "seen" if row[seen_field] else "unseen" + + +def attach_seen_field(eval_rows: list[dict], root: Path, seen_field: str) -> list[dict]: + """Eval rows carrying `seen_field`. A model-specific annotation (leak_map.py --model-key) is written + to the runtime manifest only, so the eval manifest a run consumed is never rewritten; missing values + are joined from there by track id. A row without the field is an error, not "unseen".""" + if all(seen_field in r for r in eval_rows): + return eval_rows + runtime = json.loads((root / RUNTIME_MANIFEST_REL).read_text(encoding="utf-8"))["tracks"] + by_id = {t["track_id"]: t for t in runtime} + out = [] + for r in eval_rows: + src = r if seen_field in r else by_id.get(r["track_id"], {}) + if not isinstance(src.get(seen_field), bool): + raise ValueError(f"eval row {r['track_id']} has no boolean {seen_field}") + out.append(r if src is r else dict(r, **{seen_field: src[seen_field]})) + return out def build_report(eval_rows: list[dict], probs: dict[str, float], failures: list[dict], - threshold: float = 0.5, decisions: dict[str, bool] | None = None) -> dict: + threshold: float = 0.5, decisions: dict[str, bool] | None = None, + seen_field: str = SEEN_FIELD) -> dict: """Pure function: eval manifest rows + {track_id: prob} + failures -> report dict.""" failed = {f["track_id"] for f in failures} scored = [dict(r, prob=probs[r["track_id"]]) for r in eval_rows if r["track_id"] in probs] @@ -108,7 +130,7 @@ def build_report(eval_rows: list[dict], probs: dict[str, float], failures: list[ if r["track_id"] not in probs and r["track_id"] not in failed] def rows(split: str, pred) -> list[dict]: - return [r for r in scored if pred(r) and (split == "all" or split_of(r) == split)] + return [r for r in scored if pred(r) and (split == "all" or split_of(r, seen_field) == split)] def called_ai(r): # An explicit per-track decision (e.g. a pipeline with a rescue gate) overrides prob >= threshold. @@ -160,7 +182,7 @@ def ai_block(split, pred): "missing": len(missing), "failure_rate": len(failures) / len(eval_rows) if eval_rows else None, "failures_by_source": dict(sorted(by_src_fail.items())), "failures_by_stage": dict(Counter(f.get("stage", "?") for f in failures)), - "seen_rows": sum(1 for r in eval_rows if r["seen_in_training"]), + "seen_rows": sum(1 for r in eval_rows if r[seen_field]), } report["paper_context"] = {"table10_fma_ood_accuracy": PAPER_TABLE10, "table1_columns": PAPER_TABLE1_COLS, "table1_fnr": PAPER_TABLE1, @@ -258,18 +280,18 @@ def atomic_write(path: Path, text: str) -> None: def build_full_report(eval_rows: list[dict], run_dir: Path, model_dir: str = "artifactnet", - finite_subdir: str = "finite_chunk") -> dict: + finite_subdir: str = "finite_chunk", seen_field: str = SEEN_FIELD) -> dict: """Primary = declared finite-chunk policy when present, else strict; strict kept as sensitivity.""" probs, failures, run_manifest = load_run(run_dir, model_dir) threshold = run_manifest.get("threshold", 0.5) strict = build_report(eval_rows, probs, failures, threshold=threshold, - decisions=load_decisions(run_dir, model_dir)) + decisions=load_decisions(run_dir, model_dir), seen_field=seen_field) out = {"strict": strict} finite_dir = run_dir / finite_subdir if (finite_dir / model_dir / "track_probs.json").exists(): f_probs, f_failures, _ = load_run_scores(finite_dir, model_dir) out["primary"] = build_report(eval_rows, f_probs, f_failures, threshold=threshold, - decisions=load_decisions(finite_dir, model_dir)) + decisions=load_decisions(finite_dir, model_dir), seen_field=seen_field) policy_path = finite_dir / model_dir / "finite_chunk_policy.json" policy = json.loads(policy_path.read_text(encoding="utf-8")) if policy_path.exists() else {} out["primary_policy"] = {k: v for k, v in policy.items() if k != "recovered"} @@ -278,7 +300,7 @@ def build_full_report(eval_rows: list[dict], run_dir: Path, model_dir: str = "ar out["primary"] = strict out["primary_policy"] = {"track_aggregation": "strict: any non-finite chunk fails the track"} out["run"] = {"run_dir": str(run_dir), "model_dir": model_dir, - "threshold": threshold, + "threshold": threshold, "seen_field": seen_field, "input_manifest_sha256": run_manifest.get("input_manifest_sha256"), "runner_git_revision": run_manifest.get("runner_git_revision")} return out @@ -317,10 +339,15 @@ def main(argv: list[str] | None = None) -> int: help="bench output directory (default: /results/artifactnet_v94)") ap.add_argument("--model-dir", default="artifactnet") ap.add_argument("--title", default="ArtifactNet v9.4 on MuseBench (zero-shot)") + ap.add_argument("--seen-field", default=SEEN_FIELD, + help="row field that splits seen/unseen, e.g. seen_in_training_ from " + "leak_map.py --model-key (joined from the runtime manifest when the eval " + "manifest lacks it)") args = ap.parse_args(argv) run_dir = args.run_dir or args.root / "results" / "artifactnet_v94" eval_rows = json.loads((args.root / EVAL_MANIFEST_REL).read_text(encoding="utf-8"))["tracks"] - full = build_full_report(eval_rows, run_dir, args.model_dir) + eval_rows = attach_seen_field(eval_rows, args.root, args.seen_field) + full = build_full_report(eval_rows, run_dir, args.model_dir, seen_field=args.seen_field) full["run"]["created_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) atomic_write(run_dir / "musebench_report.json", json.dumps(full, indent=1) + "\n") atomic_write(run_dir / "musebench_report.md", full_markdown(full, args.title) + "\n") diff --git a/scripts/musebench/verify.py b/scripts/musebench/verify.py index e93905f..0430d1f 100644 --- a/scripts/musebench/verify.py +++ b/scripts/musebench/verify.py @@ -353,6 +353,113 @@ def verify_run(root: Path, run_rel: Path = RUN_REL) -> tuple[list[str], dict]: return problems, stats +# The production run is private: which run directory and model key it uses is read from the data +# root, so no internal model name lives in this public repository. +PRODUCTION_SPEC_REL = Path("results") / "production_run.json" +PRODUCTION_SCORES = "scores.jsonl" + + +def verify_production(root: Path) -> tuple[list[str], dict]: + """Check an externally scored production run: raw score lines cover the eval cohort, the import + accounts for every track with an explicit decision, the model-specific leak annotation is + current, and the saved report equals a rebuild split by that annotation. + + `/results/production_run.json` names the run: {"run_dir": "results/", + "model_key": "", "model_dir": "artifactnet"}.""" + import hashlib + + from scripts.musebench import leak_map as lm + from scripts.musebench import report as rp + + spec_path = root / PRODUCTION_SPEC_REL + if not spec_path.exists(): + return [f"missing {spec_path}"], {} + spec = json.loads(spec_path.read_text(encoding="utf-8")) + run_rel, model_key, model_dir = Path(spec["run_dir"]), spec["model_key"], spec.get("model_dir", "artifactnet") + problems: list[str] = [] + run_dir = root / run_rel + seen_field = lm.keyed("seen_in_training", model_key) + rep_path = root / lm.report_rel(model_key) + needed = [root / rp.EVAL_MANIFEST_REL, root / MANIFEST_REL, rep_path, run_dir / PRODUCTION_SCORES, + run_dir / "run_manifest.json", run_dir / model_dir / "track_probs.json", + run_dir / model_dir / "inference_failures.json", + run_dir / "musebench_report.json", run_dir / "musebench_report.md"] + problems += [f"missing {p}" for p in needed if not p.exists()] + if problems: + return problems, {} + eval_rows = json.loads((root / rp.EVAL_MANIFEST_REL).read_text(encoding="utf-8"))["tracks"] + eval_ids = {r["track_id"] for r in eval_rows} + runtime = json.loads((root / MANIFEST_REL).read_text(encoding="utf-8"))["tracks"] + run_manifest = json.loads((run_dir / "run_manifest.json").read_text(encoding="utf-8")) + + # Raw score lines: one per eval track, unchanged since the import. + scores_path = run_dir / PRODUCTION_SCORES + lines = [json.loads(x) for x in scores_path.read_text(encoding="utf-8").splitlines() if x.strip()] + paths = [ln.get("path") for ln in lines] + if len(lines) != len(eval_rows): + problems.append(f"{PRODUCTION_SCORES} has {len(lines)} rows, eval cohort has {len(eval_rows)}") + if set(paths) != {r["path"] for r in eval_rows} or len(set(paths)) != len(paths): + problems.append(f"{PRODUCTION_SCORES} paths are not exactly the eval cohort (once each)") + if run_manifest.get("scores_sha256") != hashlib.sha256(scores_path.read_bytes()).hexdigest(): + problems.append(f"{PRODUCTION_SCORES} changed after it was imported (scores_sha256)") + if set(run_manifest.get("selected_track_ids", [])) != eval_ids: + problems.append("run manifest does not select exactly the eval cohort") + if not run_manifest.get("decision_field"): + problems.append("run manifest records no decision field (import without the pipeline's verdict)") + + # Import: every eval track scored (with an explicit decision) or a recorded failure. + track_probs = json.loads((run_dir / model_dir / "track_probs.json").read_text(encoding="utf-8")) + failures = json.loads((run_dir / model_dir / "inference_failures.json").read_text(encoding="utf-8")) + scored, failed = {t["track_id"] for t in track_probs}, {f["track_id"] for f in failures} + if scored & failed: + problems.append("tracks both scored and failed") + if (scored | failed) != eval_ids: + problems.append(f"scored + failed ({len(scored | failed)}) != eval cohort ({len(eval_ids)})") + no_decision = [t["track_id"] for t in track_probs if not isinstance(t.get("decision_ai"), bool)] + if no_decision: + problems.append(f"{len(no_decision)} scores carry no boolean decision_ai (e.g. {no_decision[0]})") + if any(not (0.0 <= float(t["prob"]) <= 1.0) for t in track_probs): + problems.append("probability outside [0, 1]") + + # Model-specific leak annotation: present on every row, coherent, and the report is current. + report = json.loads(rep_path.read_text(encoding="utf-8")) + evid = lm.keyed("seen_evidence", model_key) + bad = [t["track_id"] for t in runtime if not isinstance(t.get(seen_field), bool)] + if bad: + problems.append(f"{len(bad)} manifest rows lack a boolean {seen_field} (e.g. {bad[0]})") + elif any(t[seen_field] != (t.get(evid) is not None) for t in runtime): + problems.append(f"{seen_field} disagrees with {evid}") + if report.get("manifest_fingerprint") != lm.manifest_fingerprint(runtime): + problems.append(f"{rep_path.name} was built from a different manifest") + if report.get("seen_field") != seen_field or not report.get("training_manifests"): + problems.append(f"{rep_path.name} does not name its seen field and training manifests") + if not bad: + if lm.per_source_counts(runtime, seen_field) != report.get("per_source"): + problems.append(f"per-source counts differ from {rep_path.name}") + if not all(isinstance(t.get("seen_in_training"), bool) for t in runtime): + problems.append("the unkeyed seen_in_training annotation was lost") + + # The saved report equals a rebuild split by the model-specific field. + eval_seen = None + if not bad: + rows = rp.attach_seen_field(eval_rows, root, seen_field) + eval_seen = sum(1 for r in rows if r[seen_field]) + if report.get("eval_seen") != eval_seen: + problems.append(f"{rep_path.name} eval_seen {report.get('eval_seen')} != manifest {eval_seen}") + saved = json.loads((run_dir / "musebench_report.json").read_text(encoding="utf-8")) + fresh = json.loads(json.dumps(rp.build_full_report(rows, run_dir, model_dir, seen_field=seen_field))) + for key in ("primary", "strict", "primary_policy"): + if saved.get(key) != fresh.get(key): + problems.append(f"musebench_report.json '{key}' differs from a rebuild split by {seen_field}") + if saved.get("run", {}).get("seen_field") != seen_field: + problems.append(f"musebench_report.json was not split by {seen_field}") + cov = json.loads((run_dir / "musebench_report.json").read_text(encoding="utf-8")).get("primary", {}).get( + "coverage", {}) + stats = {"score_lines": len(lines), "scored": len(scored), "failures": len(failures), + "eval_seen": eval_seen, "report_seen_rows": cov.get("seen_rows"), "seen_field": seen_field} + return problems, stats + + def not_implemented(stage: str): def check(root: Path) -> tuple[list[str], dict]: return [f"stage '{stage}' has no verifier yet (later milestone)"], {} @@ -364,6 +471,7 @@ def check(root: Path) -> tuple[list[str], dict]: "acquire": verify_acquire, "leak": verify_leak, "run": verify_run, + "production": verify_production, } diff --git a/tests/test_musebench_leak_map.py b/tests/test_musebench_leak_map.py index 55c6de8..93e397a 100644 --- a/tests/test_musebench_leak_map.py +++ b/tests/test_musebench_leak_map.py @@ -172,3 +172,85 @@ def test_verify_leak_requires_named_training_manifests(leak_root): (leak_root / lm.REPORT_REL).write_text(json.dumps(rep)) problems, _ = vf.verify_leak(leak_root) assert any("training manifest" in p for p in problems) + + +# ----------------------------------------------------------------------- lineage + model key + +def test_lineage_identity_kinds(): + assert lm.lineage_identity("/d/sonics/fake_songs/fake_21689_suno_1.mp3") == ("sonics", "fake_21689_suno_1") + assert lm.lineage_identity("fake_songs/fake_00002_udio_1") == ("sonics", "fake_00002_udio_1") + assert lm.lineage_identity("/a/fma_full/000/000708.mp3") == ("fma", 708) + assert lm.lineage_identity("cell:mp3_128:fma_full_s1:10000") == ("fma", 10000) + assert lm.lineage_identity("/m/mtg_jamendo/extracted/05/5.mp3") == ("mtg", 5) + assert lm.lineage_identity("jamendo_full/00/6") == ("mtg", 6) + assert lm.lineage_identity("/x/udio/123.mp3") is None # a numeric stem outside FMA/Jamendo + assert lm.lineage_identity("/x/fma_full/abc.mp3") is None + assert lm.lineage_identity(None) is None + + +def test_load_lineage_reads_every_manifest_shape(tmp_path): + sonics_file = tmp_path / "fake_00001_suno_0.mp3" + sonics_file.write_bytes(b"s" * 111) + split_manifest = tmp_path / "m1.json" + split_manifest.write_text(json.dumps({ + "seed": 1, "epoch_caps": {"fma_full": 5}, + "train": [{"path": str(sonics_file), "label": "ai"}, + {"path": "/a/fma_full/010/010000.mp3", "track_id": "fma_full/010/010000"}], + "val": [{"path": "/m/mtg_jamendo/extracted/05/5.mp3"}], + "codec_cells": {"mp3_128": [{"path": "/c/x.wav", "track_id": "cell:mp3_128:fma_full_s1:20000", + "origin_path": "/a/fma_full/030/030000.mp3"}]}})) + list_manifest = tmp_path / "m2.json" + list_manifest.write_text(json.dumps(["/s/fake_00002_udio_1.mp3", {"source_path": "/j/jamendo/7.mp3"}])) + sonics, fma, mtg, per = lm.load_lineage([split_manifest, list_manifest]) + assert sonics == {"fake_00001_suno_0": 111, "fake_00002_udio_1": lm.UNKNOWN_SIZE} + assert fma == {10000, 20000, 30000} + assert mtg == {5, 7} + assert per == {str(split_manifest): 4, str(list_manifest): 2} + empty = tmp_path / "m3.json" + empty.write_text(json.dumps({"seed": 1})) + with pytest.raises(ValueError): + lm.load_lineage([empty]) + + +def test_annotate_keyed_keeps_the_unkeyed_annotation(): + rows = corpus_rows() + annotate(rows) # first model: unkeyed fields + before = [{k: r.get(k) for k in lm.LEAK_FIELDS} for r in rows] + kw = dict(sonics={}, fma_ids={20000}, mtg_meta={("other", "art"): {6}}, mtg_pool={6: Path("/m/6.mp3")}, + mtg_all={6: Path("/m/6.mp3")}, generators_in_training={"magnet": True}) + lm.annotate_keyed(rows, "prod", **kw) + assert [{k: r.get(k) for k in lm.LEAK_FIELDS} for r in rows] == before + seen = {(r["source"], r["base_key"]): r["seen_in_training_prod"] for r in rows} + assert seen[("musebench_suno", "fake_00001_suno_0")] is False + assert seen[("musebench_human_fma_ood", "20000_y")] is True + assert seen[("musebench_human_jamendo", "Other - Art")] is True + gen = {r["generator"]: r["generator_in_training_prod"] for r in rows} + assert gen["magnet"] is True and gen["suno"] is False + lm.annotate_keyed(rows, "prod", **dict(kw, fma_ids=set())) # idempotent: stale keyed fields replaced + assert all(r["seen_in_training_prod"] is False or r["source"] != "musebench_human_fma_ood" for r in rows) + with pytest.raises(ValueError): + lm.annotate_keyed(rows, "Bad-Key", **kw) + + +def test_main_with_model_key_and_lineage_refs(leak_root, tmp_path): + lineage = tmp_path / "gen1.json" + lineage.write_text(json.dumps({"train": [{"path": "/a/fma_full/020/020000.mp3"}, + {"path": "/m/mtg_jamendo/extracted/50/6.mp3"}]})) + base = json.loads((leak_root / "leak" / "references.json").read_text()) + refs = {"model": {"name": "Det v2"}, "training_manifests": [{"name": "gen1", "path": str(lineage)}], + "lineage_manifests": [str(lineage)], "mtg_meta_tsv": base["mtg_meta_tsv"], + "mtg_audio_dir": base["mtg_audio_dir"], "generators_in_training": {}} + ref_path = leak_root / "leak" / "references_prod.json" + ref_path.write_text(json.dumps(refs)) + v1_report = (leak_root / lm.REPORT_REL).read_text() + assert lm.main(["--root", str(leak_root), "--references", str(ref_path), "--model-key", "prod", + "--no-duration-check"]) == 0 + rep = json.loads((leak_root / lm.report_rel("prod")).read_text()) + assert rep["seen_field"] == "seen_in_training_prod" and rep["model"]["name"] == "Det v2" + assert rep["totals"] == {"seen": 2, "unseen": 7} # FMA 20000 and Jamendo "Other - Art" + assert rep["eval_seen"] == 2 + assert rep["reference_corpora"]["fma_ids"] == 1 and rep["reference_corpora"]["mtg_ids_local"] == 1 + assert (leak_root / lm.REPORT_REL).read_text() == v1_report # the first model's report is untouched + man = json.loads((leak_root / vf.MANIFEST_REL).read_text()) + assert man["metadata"]["leak_annotations"]["prod"]["seen_field"] == "seen_in_training_prod" + assert vf.verify_leak(leak_root)[0] == [] # the unkeyed annotation still verifies diff --git a/tests/test_musebench_verify_production.py b/tests/test_musebench_verify_production.py new file mode 100644 index 0000000..846d33f --- /dev/null +++ b/tests/test_musebench_verify_production.py @@ -0,0 +1,113 @@ +# Created: 2026-11-04 +# Purpose: Cover verify.py's production stage and report.py --seen-field on a synthetic data root: +# raw score lines -> import_scores.py -> model-keyed leak annotation -> report split by it. +# Dependencies: pytest +# Test Status: passing +import json + +import pytest + +from scripts.musebench import import_scores as isc +from scripts.musebench import leak_map as lm +from scripts.musebench import report as rp +from scripts.musebench import verify as vf +from test_musebench_report import cohort # tests/ is on sys.path (rootdir mode) + +KEY = "prod" +FIELD = "seen_in_training_" + KEY + + +def build_root(tmp_path): + rows, probs, _ = cohort() + root = tmp_path / "root" + (root / "manifests").mkdir(parents=True) + eval_rows = [] + for r in rows: + r = dict(r, path=f"/audio/{r['track_id']}.mp3", musebench_role="eval", + musebench_path=f"x/{r['track_id']}", seen_evidence="v1" if r["seen_in_training"] else None) + eval_rows.append(r) + (root / rp.EVAL_MANIFEST_REL).write_text(json.dumps({"tracks": eval_rows})) + # Runtime manifest = eval rows + one train_pool row; the second model saw only the FMA tracks. + runtime = [dict(r) for r in eval_rows] + [dict(eval_rows[0], track_id="tp", musebench_role="train_pool", + musebench_path="x/tp", path="/audio/tp.mp3")] + for r in runtime: + r[FIELD] = r["source"] == "musebench_human_fma_ood" + r["seen_evidence_" + KEY] = "FMA id" if r[FIELD] else None + (root / vf.MANIFEST_REL).write_text(json.dumps({"tracks": runtime, "metadata": {}})) + report = {"model": {"name": "Prod"}, "seen_field": FIELD, "training_manifests": [{"name": "gen1"}], + "per_source": lm.per_source_counts(runtime, FIELD), + "eval_seen": sum(1 for r in runtime if r[FIELD] and r["musebench_role"] == "eval"), + "manifest_fingerprint": lm.manifest_fingerprint(runtime)} + (root / "leak").mkdir() + (root / lm.report_rel(KEY)).write_text(json.dumps(report)) + # Raw scores from an external pipeline: one line per eval path, one 'unknown' verdict, the rest scored. + run = root / "results" / "prod_run" + run.mkdir(parents=True) + lines = [] + for r in eval_rows: + p = probs.get(r["track_id"], 0.5) + verdict = "unknown" if r["track_id"] == "bad" else ("ai" if p >= 0.5 else "real") + lines.append({"path": r["path"], "probability": p, "verdict": verdict, "error": None}) + (run / vf.PRODUCTION_SCORES).write_text("".join(json.dumps(x) + "\n" for x in lines)) + assert isc.main(["--root", str(root), "--scores", str(run / vf.PRODUCTION_SCORES), "--run-dir", str(run), + "--model-name", "prod", "--threshold", "0.5", "--decision-field", "verdict"]) == 0 + (root / vf.PRODUCTION_SPEC_REL).write_text(json.dumps({"run_dir": "results/prod_run", "model_key": KEY})) + assert rp.main(["--root", str(root), "--run-dir", str(run), "--seen-field", FIELD]) == 0 + return root, run + + +@pytest.fixture +def prod_root(tmp_path): + return build_root(tmp_path) + + +def test_report_splits_by_the_model_specific_field(prod_root): + _, run = prod_root + saved = json.loads((run / "musebench_report.json").read_text()) + assert saved["run"]["seen_field"] == FIELD + cov = saved["primary"]["coverage"] + assert cov["seen_rows"] == 3 # the three FMA tracks, not the first model's seen rows + assert saved["primary"]["splits"]["seen"]["fma_ood_accuracy"]["n"] == 3 + assert saved["primary"]["splits"]["seen"]["human_jamendo_fpr"]["n"] == 0 + + +def test_attach_seen_field_joins_from_runtime_and_rejects_missing(prod_root): + root, _ = prod_root + eval_rows = json.loads((root / rp.EVAL_MANIFEST_REL).read_text())["tracks"] + assert all(FIELD not in r for r in eval_rows) # the consumed eval manifest is never rewritten + joined = rp.attach_seen_field(eval_rows, root, FIELD) + assert sum(r[FIELD] for r in joined) == 3 + with pytest.raises(ValueError): + rp.attach_seen_field(eval_rows, root, "seen_in_training_other") + + +def test_verify_production_passes(prod_root): + root, _ = prod_root + problems, stats = vf.verify_production(root) + assert problems == [] + assert stats["eval_seen"] == 3 and stats["failures"] == 1 and stats["scored"] == 18 and stats["seen_field"] == FIELD + + +def test_verify_production_detects_tampering(prod_root): + root, run = prod_root + lines = (run / vf.PRODUCTION_SCORES).read_text().splitlines() + (run / vf.PRODUCTION_SCORES).write_text("\n".join(lines[1:]) + "\n") + tp = json.loads((run / "artifactnet" / "track_probs.json").read_text()) + del tp[0]["decision_ai"] + (run / "artifactnet" / "track_probs.json").write_text(json.dumps(tp)) + man = json.loads((root / vf.MANIFEST_REL).read_text()) + for t in man["tracks"]: + if t["source"] == "musebench_human_jamendo": + t[FIELD], t["seen_evidence_" + KEY] = True, "late" + (root / vf.MANIFEST_REL).write_text(json.dumps(man)) + text = "\n".join(vf.verify_production(root)[0]) + assert "has 18 rows, eval cohort has 19" in text + assert "scores_sha256" in text + assert "decision_ai" in text + assert "per-source counts differ" in text + assert "differs from a rebuild" in text + + +def test_verify_production_requires_spec(tmp_path): + problems, _ = vf.verify_production(tmp_path) + assert problems and "production_run.json" in problems[0] From b517cdc57e5b9333234db89f5b70e8eb2fb4d86b Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 23:14:52 +0900 Subject: [PATCH 2/3] musebench: count SONICS names whose training file is not on this host as size_unknown, not size_differs A lineage manifest can name a SONICS file that is on an unmounted drive (here the v10_training symlinks into an offloaded disk). The match is still a name match and still marks the row seen; only the size confirmation is unavailable, which the report now says instead of 'size differs'. --- scripts/musebench/leak_map.py | 11 ++++++++--- tests/test_musebench_leak_map.py | 4 ++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/musebench/leak_map.py b/scripts/musebench/leak_map.py index 90064ab..de10716 100644 --- a/scripts/musebench/leak_map.py +++ b/scripts/musebench/leak_map.py @@ -245,9 +245,14 @@ def annotate(tracks: list[dict], *, sonics: dict[str, int], fma_ids: set[int], size = sonics.get(base) if size is not None: stats["sonics_name_match"] += 1 - same = size == t["size"] - stats["sonics_size_equal" if same else "sonics_size_differs"] += 1 - seen_family[t["family"]] = f"SONICS file {base} ({'same size' if same else 'size differs'})" + if size == UNKNOWN_SIZE: # named by a training manifest, file not on this host + stats["sonics_size_unknown"] += 1 + note = "size not checked: training file not on this host" + else: + same = size == t["size"] + stats["sonics_size_equal" if same else "sonics_size_differs"] += 1 + note = "same size" if same else "size differs" + seen_family[t["family"]] = f"SONICS file {base} ({note})" elif t["source"] == "musebench_human_fma_ood": fid = fma_id(base) if fid is not None and fid in fma_ids: diff --git a/tests/test_musebench_leak_map.py b/tests/test_musebench_leak_map.py index 93e397a..c955784 100644 --- a/tests/test_musebench_leak_map.py +++ b/tests/test_musebench_leak_map.py @@ -225,6 +225,10 @@ def test_annotate_keyed_keeps_the_unkeyed_annotation(): assert seen[("musebench_human_fma_ood", "20000_y")] is True assert seen[("musebench_human_jamendo", "Other - Art")] is True gen = {r["generator"]: r["generator_in_training_prod"] for r in rows} + stats = lm.annotate_keyed(rows, "unk", **dict(kw, sonics={"fake_00001_suno_0": lm.UNKNOWN_SIZE})) + assert stats["sonics_size_unknown"] == 1 and "sonics_size_differs" not in stats + suno = [r for r in rows if r["source"] == "musebench_suno"][0] + assert suno["seen_in_training_unk"] is True and "not checked" in suno["seen_evidence_unk"] assert gen["magnet"] is True and gen["suno"] is False lm.annotate_keyed(rows, "prod", **dict(kw, fma_ids=set())) # idempotent: stale keyed fields replaced assert all(r["seen_in_training_prod"] is False or r["source"] != "musebench_human_fma_ood" for r in rows) From b03c757b17670659f4c225a4f38e09499db9b292 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 23:17:27 +0900 Subject: [PATCH 3/3] docs: lineage manifests and model-keyed leak annotation; externally scored runs and verify.py production --- docs/MUSEBENCH.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/MUSEBENCH.md b/docs/MUSEBENCH.md index 4ac28ab..b103b06 100644 --- a/docs/MUSEBENCH.md +++ b/docs/MUSEBENCH.md @@ -281,6 +281,26 @@ any detector trained on SONICS has seen them. JamendoMaxCaps names match MTG-Jam for only a minority of tracks (266 of 3,877 acquired Jamendo rows), so most MuseBench Jamendo audio is outside MTG-Jamendo. +### A second detector: lineage manifests and a model key + +A detector's real history is the chain of training manifests of every generation in its weight +lineage, not a candidate-pool directory. A references JSON may list them as `lineage_manifests` +(plus `mtg_meta_tsv` and `mtg_audio_dir`); their rows are read by `path`, `track_id` and the +`origin_*` fields of derived rows (codec cells such as `cell:::`), and yield +SONICS stems, FMA ids and MTG-Jamendo ids by the same identity rules. The union over train, val +and test rows is an upper bound on exposure for the manifests listed; a lineage step whose +manifest is missing makes it a lower bound, which the references' `caveats` should say. + +`--model-key KEY` writes `seen_in_training_KEY` (and the other annotation fields with the same +suffix) and `leak/leak_report_KEY.json`, so the first model's unkeyed annotation and report stay +as they are. A SONICS name whose training file is not on this host counts as +`sonics_size_unknown` rather than a size mismatch. + +```bash +python scripts/musebench/leak_map.py --root ~/data/musebench \ + --references ~/data/musebench/leak/references_.json --model-key +``` + ## Running a detector The runner's `--protocol-split` has no `eval` choice, so the eval rows are written to their own @@ -331,6 +351,32 @@ that environment on `LD_LIBRARY_PATH`; otherwise onnxruntime silently falls back (about 7× slower for ArtifactNet). CUDA and CPU scores differed by at most 0.016 on a 30-track probe, with no decision changes at 0.5. +### Externally scored runs + +A pipeline that is not wrapped by `artifactbench.bench` (for example a production stack with its +own batch runner and decision rule) writes one JSON line per file with `path`, `probability`, a +decision field and `error`. `import_scores.py` turns that into the runner's result layout, keeping +the pipeline's own decision per track (`decision_ai`), which `report.py` honours over +`prob >= threshold`. Tracks with no line or a failing decision value become recorded failures. +`report.py --seen-field` splits seen/unseen by a model-keyed annotation; the field is joined from +the runtime manifest by track id, so the eval manifest a run consumed is never rewritten. + +```bash +python scripts/musebench/import_scores.py --root ~/data/musebench \ + --scores ~/data/musebench/results//scores.jsonl --run-dir ~/data/musebench/results/ \ + --model-name --threshold --decision-field verdict +python scripts/musebench/report.py --root ~/data/musebench --run-dir ~/data/musebench/results/ \ + --seen-field seen_in_training_ +python scripts/musebench/verify.py --root ~/data/musebench production +``` + +`verify.py production` reads which run and model key to check from +`$MUSEBENCH_ROOT/results/production_run.json` (`{"run_dir": "results/", "model_key": ""}`), +so private model names stay in the data root. It checks that the score lines cover the eval cohort +once each and are unchanged since the import, that scored plus failed tracks equal the cohort with +an explicit decision on every score, that the keyed leak annotation is present, coherent and current, +and that the saved report equals a rebuild split by the keyed field. It is part of `verify.py all`. + ## Licenses - Collection: CC BY-SA 3.0 (inherited from JamendoMaxCaps).