diff --git a/docs/MUSEBENCH.md b/docs/MUSEBENCH.md index 7885884..340ba8e 100644 --- a/docs/MUSEBENCH.md +++ b/docs/MUSEBENCH.md @@ -248,6 +248,39 @@ Rules for the data root (`$MUSEBENCH_ROOT`, default `~/data/musebench`): - Nothing derived from MuseBench audio (manifests with local paths, scores, reports) is committed to this repository. +## Leak map + +`scripts/musebench/leak_map.py` marks every manifest row `seen_in_training` (boolean) for the +detector under test and writes `$MUSEBENCH_ROOT/leak/leak_report.json`. The detector and its +training pool are described in `$MUSEBENCH_ROOT/leak/references.json` (machine paths stay out of +this repository): the model, the training manifest(s) or dataset builder it came from, a SONICS +`fake_songs` directory, FMA track-id lists, MTG-Jamendo `raw.meta.tsv` with the audio folders +that were in the pool, and generator-level exposure. + +```bash +python scripts/musebench/leak_map.py --root ~/data/musebench --references ~/data/musebench/leak/references.json +python scripts/musebench/verify.py --root ~/data/musebench leak +``` + +Identity rules: + +| MuseBench rows | Key | Confirmation | +|---|---|---| +| Suno, Udio | SONICS file name `fake___` | file size | +| OOD FMA | FMA track id (leading digits of the file name) | file size vs a local FMA copy, where available | +| Jamendo | normalised (title, artist) vs MTG-Jamendo `raw.meta.tsv`, any ` - ` split | duration within 2 s | + +A transform inherits the flag of its clean track and a co-created remix that of its human +source. Generations conditioned on a seen track's caption are different audio and are not +"seen"; exposure to a *generator* is recorded separately in `generator_in_training`. The match +is against the training **candidate pool**, an upper bound on what a checkpoint saw. + +Dataset facts found on the way: all MuseBench Suno and Udio files checked so far (2,000 of +2,000 in the acquired corpus) have the name **and byte size** of a SONICS `fake_songs` file, so +any detector trained on SONICS has seen them. JamendoMaxCaps names match MTG-Jamendo metadata +for only a minority of tracks (266 of 3,877 acquired Jamendo rows), so most MuseBench Jamendo +audio is outside MTG-Jamendo. + ## Licenses - Collection: CC BY-SA 3.0 (inherited from JamendoMaxCaps). diff --git a/scripts/musebench/leak_map.py b/scripts/musebench/leak_map.py new file mode 100644 index 0000000..d25cc1e --- /dev/null +++ b/scripts/musebench/leak_map.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +# Created: 2026-11-04 +# Purpose: Annotate every MuseBench runtime-manifest row with whether its audio (or its direct +# audio ancestor) is in the training candidate pool of the detector under test, and +# write a leak report with per-source seen/unseen counts and the evidence used. +# Dependencies: standard library; ffprobe (optional, duration confirmation of Jamendo matches). +# Test Status: tests/test_musebench_leak_map.py +"""Leak map for MuseBench against a detector's training history. + +MuseBench overlaps three corpora that detectors are commonly trained on: + +* **Suno / Udio** tracks are SONICS tracks with SONICS file names + (``fake___.mp3``) -> matched by name, confirmed by file size. +* **OOD FMA** files are named ``_.mp3`` -> matched by FMA track id + against the FMA track lists used for training. +* **Jamendo** files are named ``<title> - <artist>.mp3`` (JamendoMaxCaps) -> matched by + normalised (title, artist) against MTG-Jamendo ``raw.meta.tsv``, restricted to the local + MTG-Jamendo folders that were part of the training pool, confirmed by duration. + +A row is ``seen_in_training`` when its own audio, or its direct audio ancestor, is in that pool: +a transform inherits from its clean track, a co-created remix from its human source track. +Generations conditioned on a seen track's caption are different audio and are not "seen"; +``generator_in_training`` records generator-level exposure separately. + +The match is against the training *candidate pool* (what the dataset builder could draw from), +which is an upper bound on what the checkpoint actually saw. +""" +from __future__ import annotations + +import os as _os +import sys as _sys + +# This directory holds select.py, which shadows the stdlib ``select`` module when the +# directory is on sys.path (``python scripts/musebench/x.py`` without -I). Drop it. +if _sys.path and _os.path.abspath(_sys.path[0] or ".") == _os.path.dirname(_os.path.abspath(__file__)): + _sys.path.pop(0) + +import argparse # noqa: E402 +import csv # noqa: E402 +import hashlib # noqa: E402 +import json # noqa: E402 +import os # noqa: E402 +import re # noqa: E402 +import subprocess # noqa: E402 +import sys # noqa: E402 +import time # noqa: E402 +import unicodedata # noqa: E402 +from collections import Counter, defaultdict # noqa: E402 +from pathlib import Path # noqa: E402 +from typing import Callable # noqa: E402 + +DEFAULT_ROOT = Path(os.environ.get("MUSEBENCH_ROOT", "~/data/musebench")).expanduser() +MANIFEST_REL = Path("manifests") / "musebench_runtime_manifest.json" +REPORT_REL = Path("leak") / "leak_report.json" +DURATION_TOLERANCE_S = 2.0 +LEAK_FIELDS = ("seen_in_training", "seen_evidence", "generator_in_training", + "in_local_mtg_jamendo", "mtg_jamendo_ids", "leak_note") + + +# --------------------------------------------------------------------------- identity keys + +def norm(text: str) -> str: + text = unicodedata.normalize("NFKC", text).replace("&", "&").casefold() + return re.sub(r"\s+", " ", text).strip() + + +def load_mtg_meta(tsv: Path) -> dict[tuple[str, str], set[int]]: + """(normalised title, normalised artist) -> MTG-Jamendo track ids.""" + index: dict[tuple[str, str], set[int]] = defaultdict(set) + with open(tsv, encoding="utf-8", newline="") as f: + reader = csv.reader(f, delimiter="\t") + header = next(reader) + col = {name: i for i, name in enumerate(header)} + for row in reader: + if len(row) <= max(col["TRACK_ID"], col["TRACK_NAME"], col["ARTIST_NAME"]): + continue + tid = int(row[col["TRACK_ID"]].split("_")[-1]) + index[(norm(row[col["TRACK_NAME"]]), norm(row[col["ARTIST_NAME"]]))].add(tid) + return index + + +def jamendo_candidates(base: str, meta: dict[tuple[str, str], set[int]]) -> set[int]: + """MTG ids whose (title, artist) matches ``"<title> - <artist>"`` at any ' - ' split.""" + parts = base.split(" - ") + out: set[int] = set() + for i in range(1, len(parts)): + out |= meta.get((norm(" - ".join(parts[:i])), norm(" - ".join(parts[i:]))), set()) + return out + + +def scan_mtg_audio(audio_dir: Path, folders: set[str] | None) -> dict[int, Path]: + """MTG track id -> local file, for the given two-digit folders (all if None).""" + out: dict[int, Path] = {} + for sub in sorted(audio_dir.iterdir()): + if not sub.is_dir() or (folders is not None and sub.name not in folders): + continue + for f in sub.iterdir(): + stem = f.name.split(".")[0] + if stem.isdigit(): + out[int(stem)] = f + return out + + +FMA_ID = re.compile(r"^(\d+)") + + +def fma_id(name: str) -> int | None: + m = FMA_ID.match(Path(name).name) + return int(m.group(1)) if m else None + + +def load_fma_ids(path: Path) -> set[int]: + """FMA track ids from a text list of paths or a JSON list of {filename|path} records.""" + text = path.read_text(encoding="utf-8") + names: list[str] = [] + if path.suffix == ".json": + data = json.loads(text) + for item in data: + names.append(item if isinstance(item, str) else + item.get("filename") or item.get("filepath") or item.get("path", "")) + else: + names = [line.strip() for line in text.splitlines() if line.strip()] + return {i for i in (fma_id(n) for n in names) if i is not None} + + +def scan_fma_audio(dirs: list[Path]) -> dict[int, Path]: + """FMA track id -> local file (flat or fma_full/<000>/ layout).""" + out: dict[int, Path] = {} + for d in dirs: + for f in d.rglob("*.mp3"): + fid = fma_id(f.name) + if fid is not None: + out.setdefault(fid, f) + return out + + +def scan_sonics(fake_dir: Path) -> dict[str, int]: + """SONICS fake file stem -> size in bytes.""" + return {f.name.rsplit(".", 1)[0]: f.stat().st_size for f in fake_dir.iterdir() if f.is_file()} + + +def ffprobe_duration(path: Path) -> float | None: + try: + out = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration", + "-of", "default=nw=1:nk=1", str(path)], + capture_output=True, text=True, timeout=60) + return float(out.stdout.strip()) + except (OSError, ValueError, subprocess.SubprocessError): + return None + + +# --------------------------------------------------------------------------- annotation + +def annotate(tracks: list[dict], *, sonics: dict[str, int], fma_ids: set[int], + mtg_meta: dict[tuple[str, str], set[int]], mtg_pool: dict[int, Path], + mtg_all: dict[int, Path], generators_in_training: dict[str, bool], + fma_files: dict[int, Path] | None = None, + duration: Callable[[Path], float | None] | None = None, + tolerance_s: float = DURATION_TOLERANCE_S) -> dict: + """Add ``seen_in_training``/``seen_evidence``/... to each row; return match statistics.""" + stats = Counter() + for t in tracks: # idempotent: drop fields from an earlier annotation + for key in LEAK_FIELDS: + t.pop(key, None) + # 1. Ancestor identities: clean Suno/Udio, FMA and Jamendo human rows. + seen_family: dict[str, str] = {} + for t in tracks: + gen, var, base = t["generator"], t["variant"], t["base_key"] + if gen in ("suno", "udio") and var == "clean": + size = sonics.get(base) + if size is not None: + stats["sonics_name_match"] += 1 + same = size == t["size"] + stats["sonics_size_equal" if same else "sonics_size_differs"] += 1 + seen_family[t["family"]] = f"SONICS file {base} ({'same size' if same else 'size differs'})" + elif t["source"] == "musebench_human_fma_ood": + fid = fma_id(base) + if fid is not None and fid in fma_ids: + stats["fma_id_match"] += 1 + note = f"FMA track {fid} in training FMA list" + ref = (fma_files or {}).get(fid) + if ref is not None: + same = ref.stat().st_size == t["size"] + stats["fma_size_equal" if same else "fma_size_differs"] += 1 + note += f" ({'same size as' if same else 'size differs from'} local copy)" + seen_family[t["family"]] = note + elif t["source"] == "musebench_human_jamendo": + cands = jamendo_candidates(base, mtg_meta) + if not cands: + continue + stats["jamendo_meta_match"] += 1 + local = [c for c in sorted(cands) if c in mtg_all] + if local: + t["mtg_jamendo_ids"] = local + pooled = [c for c in local if c in mtg_pool] + if not pooled: + continue + note = f"MTG-Jamendo track {pooled[0]} (title/artist match)" + if duration is not None: + d_ref = duration(mtg_pool[pooled[0]]) + d_mb = duration(Path(t["path"])) + if d_ref is not None and d_mb is not None: + if abs(d_ref - d_mb) > tolerance_s: + stats["jamendo_duration_mismatch"] += 1 + t["leak_note"] = f"{note}; duration {d_mb:.1f}s vs {d_ref:.1f}s: not counted" + continue + note += f", duration {d_mb:.1f}s vs {d_ref:.1f}s" + stats["jamendo_duration_confirmed"] += 1 + stats["jamendo_pool_match"] += 1 + seen_family[t["family"]] = note + + # 2. Every row: own identity or direct ancestor (same family, and the row derives from it). + for t in tracks: + evidence = None + fam = t["family"] + if fam in seen_family: + if t["variant"] == "clean" and t["generator"] in ("human", "suno", "udio"): + evidence = seen_family[fam] + elif t["generator"] in ("suno", "udio"): + evidence = "transform of " + seen_family[fam] # transforms of a seen SONICS track + elif t["variant"] == "co_created": + evidence = "co-created remix of " + seen_family[fam] + t["seen_in_training"] = evidence is not None + t["seen_evidence"] = evidence + g = t["generator"] + t["generator_in_training"] = None if g == "human" else bool(generators_in_training.get(g, False)) + if t["source"] == "musebench_human_jamendo": + t["in_local_mtg_jamendo"] = bool(t.get("mtg_jamendo_ids")) + return dict(stats) + + +def per_source_counts(tracks: list[dict]) -> dict[str, dict]: + out: dict[str, Counter] = defaultdict(Counter) + for t in tracks: + c = out[t["source"]] + c["total"] += 1 + c["seen" if t["seen_in_training"] else "unseen"] += 1 + c[f"{t['musebench_role']}_{'seen' if t['seen_in_training'] else 'unseen'}"] += 1 + return {k: {"total": v["total"], "seen": v["seen"], "unseen": v["unseen"], + "eval_seen": v["eval_seen"], "eval_unseen": v["eval_unseen"], + "train_pool_seen": v["train_pool_seen"], "train_pool_unseen": v["train_pool_unseen"]} + for k, v in sorted(out.items())} + + +def manifest_fingerprint(tracks: list[dict]) -> str: + h = hashlib.sha256() + for t in sorted(tracks, key=lambda t: t["track_id"]): + h.update(f"{t['track_id']}\t{t['musebench_path']}\n".encode("utf-8", "surrogateescape")) + return h.hexdigest() + + +def atomic_write_json(path: Path, obj: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(obj, ensure_ascii=True, indent=1) + "\n", encoding="utf-8") + os.replace(tmp, path) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Annotate the MuseBench manifest with training overlap.") + ap.add_argument("--root", type=Path, default=DEFAULT_ROOT) + ap.add_argument("--references", type=Path, default=None, + help="JSON describing the detector and its training pool " + "(default: <root>/leak/references.json)") + ap.add_argument("--no-duration-check", action="store_true") + args = ap.parse_args(argv) + ref_path = args.references or args.root / "leak" / "references.json" + refs = json.loads(ref_path.read_text(encoding="utf-8")) + + sonics = scan_sonics(Path(refs["sonics_fake_dir"])) + fma_ids: set[int] = set() + for p in refs["fma_id_lists"]: + fma_ids |= load_fma_ids(Path(p)) + fma_files = scan_fma_audio([Path(p) for p in refs.get("fma_audio_dirs", [])]) + mtg_meta = load_mtg_meta(Path(refs["mtg_meta_tsv"])) + mtg_dir = Path(refs["mtg_audio_dir"]) + mtg_pool = scan_mtg_audio(mtg_dir, set(refs["mtg_pool_folders"])) + mtg_all = scan_mtg_audio(mtg_dir, None) + + man_path = args.root / MANIFEST_REL + manifest = json.loads(man_path.read_text(encoding="utf-8")) + tracks = manifest["tracks"] + stats = annotate(tracks, sonics=sonics, fma_ids=fma_ids, mtg_meta=mtg_meta, + mtg_pool=mtg_pool, mtg_all=mtg_all, + generators_in_training=refs["generators_in_training"], + fma_files=fma_files, duration=None if args.no_duration_check else ffprobe_duration) + manifest["metadata"]["leak_annotation"] = { + "model": refs["model"]["name"], "report": str(args.root / REPORT_REL), + "annotated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} + atomic_write_json(man_path, manifest) + + report = { + "model": refs["model"], + "training_manifests": refs["training_manifests"], + "reference_corpora": { + "sonics_fake_dir": {"path": refs["sonics_fake_dir"], "files": len(sonics)}, + "fma_id_lists": {"paths": refs["fma_id_lists"], "track_ids": len(fma_ids), + "audio_dirs": refs.get("fma_audio_dirs", []), + "local_files": len(fma_files)}, + "mtg_jamendo": {"meta_tsv": refs["mtg_meta_tsv"], "audio_dir": refs["mtg_audio_dir"], + "pool_folders": refs["mtg_pool_folders"], + "pool_tracks": len(mtg_pool), "local_tracks": len(mtg_all)}, + }, + "method": { + "suno_udio": "SONICS file-name match (fake_<id>_<gen>_<seed>), size compared", + "fma": "FMA track id (leading digits of the file name) in the training FMA id lists; " + "size compared with a local FMA copy where one exists", + "jamendo": "normalised (title, artist) vs MTG-Jamendo raw.meta.tsv, restricted to the " + f"pool folders; duration within {DURATION_TOLERANCE_S}s" + + (" (not checked)" if args.no_duration_check else ""), + "inheritance": "transforms inherit from their clean track; co-created remixes from " + "their human source; caption-conditioned generations do not", + "scope": "candidate pool of the training dataset builder: an upper bound on exposure", + }, + "generators_in_training": refs["generators_in_training"], + "caveats": refs.get("caveats", []), + "match_stats": stats, + "per_source": per_source_counts(tracks), + "totals": dict(Counter("seen" if t["seen_in_training"] else "unseen" for t in tracks)), + "manifest": str(man_path), + "manifest_fingerprint": manifest_fingerprint(tracks), + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + atomic_write_json(args.root / REPORT_REL, report) + print(json.dumps({"match_stats": stats, "totals": report["totals"]}, indent=1)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/musebench/verify.py b/scripts/musebench/verify.py index f2e7f00..1fdb4e4 100644 --- a/scripts/musebench/verify.py +++ b/scripts/musebench/verify.py @@ -243,6 +243,52 @@ def verify_acquire(root: Path, sha_sample: int = 300, full_sha: bool = False, return problems, stats +def verify_leak(root: Path) -> tuple[list[str], dict]: + """Check the leak report and that every manifest row carries a boolean seen_in_training.""" + from scripts.musebench import leak_map as lm + + problems: list[str] = [] + rep_path, man_path = root / lm.REPORT_REL, root / MANIFEST_REL + for p in (rep_path, man_path): + if not p.exists(): + problems.append(f"missing {p}") + if problems: + return problems, {} + report = json.loads(rep_path.read_text(encoding="utf-8")) + tracks = json.loads(man_path.read_text(encoding="utf-8"))["tracks"] + + if not report.get("model", {}).get("name"): + problems.append("leak report does not name the model") + manifests = report.get("training_manifests") or [] + if not manifests or not all(m.get("name") for m in manifests): + problems.append("leak report does not name the training manifest(s) it compared against") + for key in ("suno_udio", "fma", "jamendo", "inheritance", "scope"): + if not report.get("method", {}).get(key): + problems.append(f"leak report method lacks '{key}'") + + bad = [t["track_id"] for t in tracks if not isinstance(t.get("seen_in_training"), bool)] + if bad: + problems.append(f"{len(bad)} manifest rows lack a boolean seen_in_training (e.g. {bad[0]})") + incoherent = [t["track_id"] for t in tracks if isinstance(t.get("seen_in_training"), bool) + and t["seen_in_training"] != (t.get("seen_evidence") is not None)] + if incoherent: + problems.append(f"{len(incoherent)} rows: seen_in_training disagrees with seen_evidence") + if report.get("manifest_fingerprint") != lm.manifest_fingerprint(tracks): + problems.append("leak report was built from a different manifest") + if not bad: + recount = lm.per_source_counts(tracks) + if recount != report.get("per_source"): + problems.append("per-source seen/unseen counts differ from the manifest") + if sum(report.get("totals", {}).values()) != len(tracks): + problems.append("report totals do not add up to the manifest size") + seen = sum(1 for t in tracks if t.get("seen_in_training") is True) + stats = {"tracks": len(tracks), "seen": seen, "unseen": len(tracks) - seen, + "eval_seen": sum(1 for t in tracks if t.get("seen_in_training") is True + and t.get("musebench_role") == "eval"), + "training_manifests": len(manifests)} + return problems, stats + + def not_implemented(stage: str): def check(root: Path) -> tuple[list[str], dict]: return [f"stage '{stage}' has no verifier yet (later milestone)"], {} @@ -252,7 +298,7 @@ def check(root: Path) -> tuple[list[str], dict]: STAGES = { "index": verify_index, "acquire": verify_acquire, - "leak": not_implemented("leak"), + "leak": verify_leak, "run": not_implemented("run"), } diff --git a/tests/test_musebench_leak_map.py b/tests/test_musebench_leak_map.py new file mode 100644 index 0000000..55c6de8 --- /dev/null +++ b/tests/test_musebench_leak_map.py @@ -0,0 +1,174 @@ +# Created: 2026-11-04 +# Purpose: Cover scripts/musebench/leak_map.py (identity keys, inheritance rules, report) and +# verify.py's leak stage on synthetic reference corpora. +# Dependencies: pytest +# Test Status: passing +import json +from pathlib import Path + +import pytest + +from scripts.musebench import leak_map as lm +from scripts.musebench import verify as vf + + +def test_norm_and_jamendo_candidates_try_every_split(): + meta = {(lm.norm("Song - Part 2"), lm.norm("The Artist")): {7}, + (lm.norm("Black Peace & Der Pate"), lm.norm("DJ X")): {9}} + assert lm.jamendo_candidates("Song - Part 2 - The Artist", meta) == {7} + assert lm.jamendo_candidates("BLACK Peace & Der Pate - dj x", meta) == {9} + assert lm.jamendo_candidates("Unknown - Nobody", meta) == set() + + +def test_fma_ids_from_names_and_lists(tmp_path): + assert lm.fma_id("10000_Vena_Cava_50_Foot_Wave") == 10000 + assert lm.fma_id("000002.mp3") == 2 + assert lm.fma_id("abc.mp3") is None + txt = tmp_path / "paths.txt" + txt.write_text("/Volumes/x/fma_full/000/000002.mp3\n/Volumes/x/fma_full/010/010000.mp3\n\n") + js = tmp_path / "files.json" + js.write_text(json.dumps([{"filename": "012827.mp3"}, {"path": "/a/000148.mp3"}, "000005.mp3"])) + assert lm.load_fma_ids(txt) == {2, 10000} + assert lm.load_fma_ids(js) == {12827, 148, 5} + + +def test_load_mtg_meta(tmp_path): + tsv = tmp_path / "raw.meta.tsv" + tsv.write_text("TRACK_ID\tARTIST_ID\tALBUM_ID\tTRACK_NAME\tARTIST_NAME\tALBUM_NAME\n" + "track_0000214\tartist_1\talbum_1\tIntro\tDavid TMX\tPouce\n" + "track_0000215\tartist_1\talbum_1\tIntro\tDavid TMX\tOther\n", encoding="utf-8") + assert lm.load_mtg_meta(tsv)[("intro", "david tmx")] == {214, 215} + + +def row(source, generator, variant, family, base, role="eval", size=10, tid=None, path="/x"): + return {"track_id": tid or f"mb_{source}_{base}_{variant}", "source": source, + "generator": generator, "variant": variant, "family": family, "base_key": base, + "musebench_role": role, "size": size, "path": path, "musebench_path": f"{source}/{base}"} + + +def corpus_rows(): + return [ + row("musebench_suno", "suno", "clean", "suno/fake_00001_suno_0", "fake_00001_suno_0", size=111), + row("musebench_suno__aac", "suno", "aac", "suno/fake_00001_suno_0", "fake_00001_suno_0"), + row("musebench_udio", "udio", "clean", "udio/fake_00002_udio_1", "fake_00002_udio_1"), + row("musebench_human_fma_ood", "human", "clean", "fma/10000_x", "10000_x", size=50), + row("musebench_human_fma_ood", "human", "clean", "fma/20000_y", "20000_y"), + row("musebench_human_jamendo", "human", "clean", "jamendo/Song - Art", "Song - Art"), + row("musebench_co_created", "musicgen_style", "co_created", "jamendo/Song - Art", "Song - Art", + role="train_pool"), + row("musebench_magnet", "magnet", "clean", "jamendo/Song - Art", "Song - Art"), + row("musebench_human_jamendo", "human", "clean", "jamendo/Other - Art", "Other - Art"), + ] + + +def annotate(rows, duration=None): + return lm.annotate( + rows, sonics={"fake_00001_suno_0": 111}, fma_ids={10000}, + mtg_meta={("song", "art"): {5}, ("other", "art"): {6}}, + mtg_pool={5: Path("/mtg/00/5.mp3")}, mtg_all={5: Path("/mtg/00/5.mp3"), 6: Path("/mtg/50/6.mp3")}, + generators_in_training={"suno": True, "udio": True, "magnet": False}, + fma_files={10000: Path(__file__)}, duration=duration) + + +def test_annotate_identity_and_inheritance_rules(): + rows = corpus_rows() + stats = annotate(rows) + seen = {(r["source"], r["base_key"]): r["seen_in_training"] for r in rows} + assert seen[("musebench_suno", "fake_00001_suno_0")] is True + assert seen[("musebench_suno__aac", "fake_00001_suno_0")] is True # transform inherits + assert seen[("musebench_udio", "fake_00002_udio_1")] is False # not in SONICS list + assert seen[("musebench_human_fma_ood", "10000_x")] is True + assert seen[("musebench_human_fma_ood", "20000_y")] is False + assert seen[("musebench_human_jamendo", "Song - Art")] is True + assert seen[("musebench_co_created", "Song - Art")] is True # remix of a seen song + assert seen[("musebench_magnet", "Song - Art")] is False # caption-conditioned: new audio + assert seen[("musebench_human_jamendo", "Other - Art")] is False # MTG track outside pool + other = [r for r in rows if r["base_key"] == "Other - Art"][0] + assert other["in_local_mtg_jamendo"] is True and other["mtg_jamendo_ids"] == [6] + assert stats["sonics_size_equal"] == 1 and stats["fma_id_match"] == 1 + assert stats["fma_size_differs"] == 1 # this test file is not 50 bytes + gen = {r["generator"]: r["generator_in_training"] for r in rows} + assert gen["suno"] is True and gen["magnet"] is False and gen["human"] is None + assert all(r["seen_in_training"] == (r["seen_evidence"] is not None) for r in rows) + + +def test_duration_mismatch_is_not_counted_and_annotation_is_idempotent(): + rows = corpus_rows() + annotate(rows, duration=lambda p: 100.0 if "mtg" in str(p) else 30.0) + jam = [r for r in rows if r["source"] == "musebench_human_jamendo" and r["base_key"] == "Song - Art"][0] + assert jam["seen_in_training"] is False and "not counted" in jam["leak_note"] + annotate(rows, duration=lambda p: 30.0) + assert jam["seen_in_training"] is True and "leak_note" not in jam + + +def test_per_source_counts_split_by_role(): + rows = corpus_rows() + annotate(rows) + c = lm.per_source_counts(rows) + assert c["musebench_human_jamendo"] == {"total": 2, "seen": 1, "unseen": 1, "eval_seen": 1, + "eval_unseen": 1, "train_pool_seen": 0, + "train_pool_unseen": 0} + assert c["musebench_co_created"]["train_pool_seen"] == 1 + + +# ----------------------------------------------------------------------- end to end + verify + +@pytest.fixture +def leak_root(tmp_path): + root = tmp_path / "root" + sonics = tmp_path / "sonics" + sonics.mkdir() + (sonics / "fake_00001_suno_0.mp3").write_bytes(b"s" * 111) + fma_list = tmp_path / "fma.txt" + fma_list.write_text("/v/fma_full/010/010000.mp3\n") + mtg = tmp_path / "mtg" + (mtg / "00").mkdir(parents=True) + (mtg / "00" / "5.mp3").write_bytes(b"m") + (mtg / "50").mkdir() + (mtg / "50" / "6.mp3").write_bytes(b"m") + tsv = tmp_path / "raw.meta.tsv" + tsv.write_text("TRACK_ID\tARTIST_ID\tALBUM_ID\tTRACK_NAME\tARTIST_NAME\n" + "track_0000005\ta\tb\tSong\tArt\ntrack_0000006\ta\tb\tOther\tArt\n") + refs = {"model": {"name": "Det v1"}, + "training_manifests": [{"name": "train set", "definition": "builder.py"}], + "sonics_fake_dir": str(sonics), "fma_id_lists": [str(fma_list)], + "mtg_meta_tsv": str(tsv), "mtg_audio_dir": str(mtg), "mtg_pool_folders": ["00"], + "generators_in_training": {"suno": True}} + (root / "leak").mkdir(parents=True) + (root / "leak" / "references.json").write_text(json.dumps(refs)) + man = {"tracks": corpus_rows(), "metadata": {}} + (root / "manifests").mkdir() + (root / vf.MANIFEST_REL).write_text(json.dumps(man)) + assert lm.main(["--root", str(root), "--no-duration-check"]) == 0 + return root + + +def test_main_writes_report_and_verify_leak_passes(leak_root): + report = json.loads((leak_root / lm.REPORT_REL).read_text()) + assert report["model"]["name"] == "Det v1" + assert report["training_manifests"][0]["name"] == "train set" + assert report["totals"] == {"seen": 5, "unseen": 4} + assert report["reference_corpora"]["mtg_jamendo"]["pool_tracks"] == 1 + problems, stats = vf.verify_leak(leak_root) + assert problems == [] and stats["seen"] == 5 + + +def test_verify_leak_detects_missing_flag_and_stale_report(leak_root): + path = leak_root / vf.MANIFEST_REL + man = json.loads(path.read_text()) + del man["tracks"][0]["seen_in_training"] + path.write_text(json.dumps(man)) + problems, _ = vf.verify_leak(leak_root) + assert any("boolean seen_in_training" in p for p in problems) + man["tracks"] = man["tracks"][1:] + path.write_text(json.dumps(man)) + problems, _ = vf.verify_leak(leak_root) + assert any("different manifest" in p for p in problems) + + +def test_verify_leak_requires_named_training_manifests(leak_root): + rep = json.loads((leak_root / lm.REPORT_REL).read_text()) + rep["training_manifests"] = [] + (leak_root / lm.REPORT_REL).write_text(json.dumps(rep)) + problems, _ = vf.verify_leak(leak_root) + assert any("training manifest" in p for p in problems) diff --git a/tests/test_musebench_verify.py b/tests/test_musebench_verify.py index fd0fb67..d517a89 100644 --- a/tests/test_musebench_verify.py +++ b/tests/test_musebench_verify.py @@ -90,6 +90,6 @@ def test_cli_runs_isolated_and_reports_failure(tmp_path): out = subprocess.run([sys.executable, "-I", str(script), "--root", str(tmp_path), "index"], capture_output=True, text=True) assert out.returncode == 1 and "[index] FAIL" in out.stdout - out = subprocess.run([sys.executable, "-I", str(script), "--root", str(tmp_path), "leak"], + out = subprocess.run([sys.executable, "-I", str(script), "--root", str(tmp_path), "run"], capture_output=True, text=True) assert out.returncode == 1 and "no verifier yet" in out.stdout