Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions docs/MUSEBENCH.md
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,56 @@ any detector trained on SONICS has seen them. JamendoMaxCaps names match MTG-Jam
for only a minority of tracks (266 of 3,877 acquired Jamendo rows), so most MuseBench Jamendo
audio is outside MTG-Jamendo.

## Running a detector

The runner's `--protocol-split` has no `eval` choice, so the eval rows are written to their own
manifest first. For ArtifactNet the declared primary policy of the frozen v2 protocol applies:
median of the finite scores of seven 4 s chunks, at least four finite; the strict result (any
non-finite chunk fails the track) is kept as a sensitivity analysis.

```bash
python scripts/musebench/select.py eval-manifest --root ~/data/musebench
python -m artifactbench.bench --model artifactnet \
--manifest ~/data/musebench/manifests/musebench_eval_manifest.json \
--split all --n-per-source 0 --n-codec-pair 0 --device cuda \
--output ~/data/musebench/results/artifactnet_v94
python scripts/audit_artifactnet_nonfinite.py --device cuda \
--manifest ~/data/musebench/manifests/musebench_eval_manifest.json \
--failures ~/data/musebench/results/artifactnet_v94/artifactnet/inference_failures.json \
--output ~/data/musebench/results/artifactnet_v94/nonfinite_audit.json
python scripts/build_artifactnet_finite_chunk_result.py --min-valid-chunks 4 \
--manifest ~/data/musebench/manifests/musebench_eval_manifest.json \
--strict-result ~/data/musebench/results/artifactnet_v94 \
--chunk-audit ~/data/musebench/results/artifactnet_v94/nonfinite_audit.json \
--output ~/data/musebench/results/artifactnet_v94/finite_chunk
python scripts/musebench/report.py --root ~/data/musebench
python scripts/musebench/verify.py --root ~/data/musebench all
```

The audit re-runs each failed track chunk by chunk and is decode-bound; it can be run in
parallel on shards of the failure list and merged back with the same schema:

```bash
python scripts/musebench/merge_audits.py split --shards 4 \
--failures ~/data/musebench/results/artifactnet_v94/artifactnet/inference_failures.json \
--out-dir ~/data/musebench/results/artifactnet_v94/audit_shards
python scripts/musebench/merge_audits.py merge \
--failures ~/data/musebench/results/artifactnet_v94/artifactnet/inference_failures.json \
--out ~/data/musebench/results/artifactnet_v94/nonfinite_audit.json \
~/data/musebench/results/artifactnet_v94/audit_shards/audit_0.json
```

`report.py` writes `musebench_report.json` and `.md` next to the run: human (Jamendo) FPR,
FMA-OOD accuracy, clean-AI FNR per generator, co-created FNR, FNR per transform and per
transform × generator, AUROC against the Jamendo negatives, Wilson 95 % intervals, all for
`all`, `seen` and `unseen` rows, plus failure counts. The paper's Table 1 and Table 10 values are
included for context only. Reports stay in the data root; they are not committed here.

With onnxruntime-gpu installed from pip, the CUDA provider needs the `nvidia/*/lib` folders of
that environment on `LD_LIBRARY_PATH`; otherwise onnxruntime silently falls back to the CPU
(about 7× slower for ArtifactNet). CUDA and CPU scores differed by at most 0.016 on a 30-track
probe, with no decision changes at 0.5.

## Licenses

- Collection: CC BY-SA 3.0 (inherited from JamendoMaxCaps).
Expand Down
14 changes: 11 additions & 3 deletions scripts/audit_artifactnet_nonfinite.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,18 @@

import argparse
import json
import sys
from pathlib import Path

import numpy as np

from artifactbench.data.manifest import load_audio_mono
from artifactbench.models.artifactnet import (
# Run from a checkout without `pip install -e .`: make the repository importable.
_REPO_ROOT = Path(__file__).resolve().parents[1]
if str(_REPO_ROOT) not in sys.path:
sys.path.insert(0, str(_REPO_ROOT))

from artifactbench.data.manifest import load_audio_mono # noqa: E402
from artifactbench.models.artifactnet import ( # noqa: E402
CHUNK_SAMPLES,
N_CHUNKS_DEFAULT,
ArtifactNetModel,
Expand All @@ -27,6 +33,8 @@ def main():
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--failures", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--device", default="cpu", choices=["cpu", "cuda"],
help="onnxruntime device for the chunk re-run (default: cpu)")
args = parser.parse_args()

manifest = json.loads(args.manifest.read_text())
Expand All @@ -35,7 +43,7 @@ def main():
target_ids = [f["track_id"] for f in failures if f.get("stage") == "inference"]

model = ArtifactNetModel(n_chunks=N_CHUNKS_DEFAULT)
model.load(device="cpu")
model.load(device=args.device)
audits = []
for index, track_id in enumerate(target_ids, 1):
row = rows[track_id]
Expand Down
5 changes: 5 additions & 0 deletions scripts/build_artifactnet_finite_chunk_result.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ def main():
"policy": f"requires at least {args.min_valid_chunks} finite chunks",
})

# Failures the chunk audit does not cover (decode, resolve, ...) stay failures; dropping
# them would make tracks silently disappear from the result.
audited = {track["track_id"] for track in audit["tracks"]}
unresolved.extend(f for f in strict_failures if f["track_id"] not in audited)

args.output.mkdir(parents=True, exist_ok=True)
model_dir = args.output / "artifactnet"
model_dir.mkdir(exist_ok=True)
Expand Down
113 changes: 113 additions & 0 deletions scripts/musebench/import_scores.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
# Created: 2026-11-04
# Purpose: Turn per-file scores from an external scorer (JSONL: path, probability, verdict, error)
# into the run-directory layout report.py reads, so detectors that do not run inside
# artifactbench.bench can be reported on the same MuseBench eval cohort.
# Dependencies: standard library only.
# Test Status: tests/test_musebench_import_scores.py
"""Import external MuseBench scores.

python scripts/musebench/import_scores.py --root ~/data/musebench \\
--scores results.jsonl --run-dir ~/data/musebench/results/<name> \\
--model-name "<detector>" --threshold 0.2 --decision-field verdict

Each JSONL line must carry ``path`` (as listed in the eval manifest) and ``probability``.
With ``--decision-field``, a value equal to ``--ai-value`` marks the track as called AI and
any value in ``--fail-values`` (or a missing probability / an ``error``) makes it a recorded
failure. Without it, report.py applies ``probability >= threshold``.
"""
from __future__ import annotations

import os as _os
import sys as _sys

# This directory holds select.py, which shadows the stdlib ``select`` module when the
# directory is on sys.path (``python scripts/musebench/x.py`` without -I). Drop it.
if _sys.path and _os.path.abspath(_sys.path[0] or ".") == _os.path.dirname(_os.path.abspath(__file__)):
_sys.path.pop(0)

import argparse # noqa: E402
import hashlib # noqa: E402
import json # noqa: E402
import os # noqa: E402
import sys # noqa: E402
from pathlib import Path # noqa: E402

DEFAULT_ROOT = Path(os.environ.get("MUSEBENCH_ROOT", "~/data/musebench")).expanduser()
EVAL_MANIFEST_REL = Path("manifests") / "musebench_eval_manifest.json"


def convert(eval_rows: list[dict], score_lines: list[dict], *, decision_field: str | None,
ai_value: str = "ai", fail_values: tuple[str, ...] = ("unknown", "unreliable"),
) -> tuple[list[dict], list[dict], list[str]]:
"""Return (track_probs, failures, unmatched score paths)."""
by_path = {r["path"]: r for r in eval_rows}
seen: dict[str, dict] = {}
unmatched = []
for line in score_lines:
row = by_path.get(line.get("path"))
if row is None:
unmatched.append(line.get("path"))
continue
if row["track_id"] in seen:
raise ValueError(f"duplicate score for {line['path']!r}")
seen[row["track_id"]] = line
probs, failures = [], []
for row in eval_rows:
line = seen.get(row["track_id"])
base = {"track_id": row["track_id"], "source": row["source"]}
if line is None:
failures.append({**base, "stage": "missing", "error": "no score line for this path"})
continue
p = line.get("probability")
decision = line.get(decision_field) if decision_field else None
if line.get("error") or p is None or (decision_field and decision in fail_values):
failures.append({**base, "stage": "inference",
"error": line.get("error") or f"{decision_field}={decision}, probability={p}"})
continue
entry = {**base, "prob": float(p), "label": 1 if row["label"] == "ai" else 0}
if decision_field:
entry["decision_ai"] = decision == ai_value
probs.append(entry)
return probs, failures, unmatched


def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(description="Import external per-file scores for report.py.")
ap.add_argument("--root", type=Path, default=DEFAULT_ROOT)
ap.add_argument("--scores", type=Path, required=True, help="JSONL with path/probability/...")
ap.add_argument("--run-dir", type=Path, required=True)
ap.add_argument("--model-dir", default="artifactnet")
ap.add_argument("--model-name", required=True)
ap.add_argument("--threshold", type=float, required=True)
ap.add_argument("--decision-field", default=None)
ap.add_argument("--ai-value", default="ai")
ap.add_argument("--fail-values", nargs="*", default=["unknown", "unreliable"])
args = ap.parse_args(argv)
man_path = args.root / EVAL_MANIFEST_REL
eval_rows = json.loads(man_path.read_text(encoding="utf-8"))["tracks"]
with open(args.scores, encoding="utf-8") as f:
lines = [json.loads(x) for x in f if x.strip()]
probs, failures, unmatched = convert(eval_rows, lines, decision_field=args.decision_field,
ai_value=args.ai_value, fail_values=tuple(args.fail_values))
out = args.run_dir / args.model_dir
out.mkdir(parents=True, exist_ok=True)
(out / "track_probs.json").write_text(json.dumps(probs) + "\n", encoding="utf-8")
(out / "inference_failures.json").write_text(json.dumps(failures, indent=1) + "\n", encoding="utf-8")
run_manifest = {
"imported_from": str(args.scores),
"scores_sha256": hashlib.sha256(args.scores.read_bytes()).hexdigest(),
"input_manifest": str(man_path),
"input_manifest_sha256": hashlib.sha256(man_path.read_bytes()).hexdigest(),
"models": [args.model_dir], "model_name": args.model_name,
"threshold": args.threshold, "decision_field": args.decision_field,
"n_per_source": 0, "selected_track_ids": [r["track_id"] for r in eval_rows],
}
(args.run_dir / "run_manifest.json").write_text(json.dumps(run_manifest, indent=1) + "\n",
encoding="utf-8")
print(f"imported {len(probs)} scores, {len(failures)} failures, {len(unmatched)} unmatched paths")
return 1 if unmatched else 0


if __name__ == "__main__":
sys.exit(main())
95 changes: 95 additions & 0 deletions scripts/musebench/merge_audits.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
#!/usr/bin/env python3
# Created: 2026-11-04
# Purpose: Split an inference_failures.json into N shards for parallel runs of
# scripts/audit_artifactnet_nonfinite.py, and merge the shard audits back into one
# report with the same schema (summary fields recomputed, original failure order).
# Dependencies: standard library only.
# Test Status: tests/test_musebench_merge_audits.py
"""Shard / merge helper for the ArtifactNet non-finite chunk audit.

python scripts/musebench/merge_audits.py split --failures F.json --shards 4 --out-dir D
python scripts/musebench/merge_audits.py merge --failures F.json --out A.json D/audit_*.json
"""
from __future__ import annotations

import os as _os
import sys as _sys

# This directory holds select.py, which shadows the stdlib ``select`` module when the
# directory is on sys.path (``python scripts/musebench/x.py`` without -I). Drop it.
if _sys.path and _os.path.abspath(_sys.path[0] or ".") == _os.path.dirname(_os.path.abspath(__file__)):
_sys.path.pop(0)

import argparse # noqa: E402
import json # noqa: E402
import statistics # noqa: E402
import sys # noqa: E402
from pathlib import Path # noqa: E402


def split(failures: list[dict], n: int) -> list[list[dict]]:
"""Round-robin over the inference failures only (the audit ignores other stages)."""
inf = [f for f in failures if f.get("stage") == "inference"]
return [inf[i::n] for i in range(n)]


def merge(failures: list[dict], audits: list[dict]) -> dict:
by_id = {}
for a in audits:
for t in a["tracks"]:
if t["track_id"] in by_id:
raise ValueError(f"track audited twice: {t['track_id']}")
by_id[t["track_id"]] = t
wanted = [f["track_id"] for f in failures if f.get("stage") == "inference"]
missing = [t for t in wanted if t not in by_id]
if missing:
raise ValueError(f"{len(missing)} inference failures were not audited (e.g. {missing[0]})")
extra = set(by_id) - set(wanted)
if extra:
raise ValueError(f"{len(extra)} audited tracks are not inference failures")
tracks = [by_id[t] for t in wanted]
chunks = [c for t in tracks for c in t["chunks"]]
nonfinite = [c for c in chunks if not c["finite"]]
dbfs = [c["rms_dbfs"] for c in nonfinite]
return {
"tracks_audited": len(tracks),
"chunks_audited": len(chunks),
"nonfinite_chunks": len(nonfinite),
"tracks_with_at_least_four_finite_chunks": sum(t["valid_chunks"] >= 4 for t in tracks),
"nonfinite_rms_dbfs": {
"min": min(dbfs, default=None),
"median": float(statistics.median(dbfs)) if dbfs else None,
"max": max(dbfs, default=None),
},
"tracks": tracks,
"merged_from_shards": len(audits),
}


def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(description="Shard / merge the ArtifactNet non-finite audit.")
sub = ap.add_subparsers(dest="cmd", required=True)
s = sub.add_parser("split")
s.add_argument("--failures", type=Path, required=True)
s.add_argument("--shards", type=int, required=True)
s.add_argument("--out-dir", type=Path, required=True)
m = sub.add_parser("merge")
m.add_argument("--failures", type=Path, required=True)
m.add_argument("--out", type=Path, required=True)
m.add_argument("audits", type=Path, nargs="+")
args = ap.parse_args(argv)
failures = json.loads(args.failures.read_text(encoding="utf-8"))
if args.cmd == "split":
args.out_dir.mkdir(parents=True, exist_ok=True)
for i, part in enumerate(split(failures, args.shards)):
(args.out_dir / f"failures_{i}.json").write_text(json.dumps(part, indent=1) + "\n")
print(f"shard {i}: {len(part)} failures")
return 0
merged = merge(failures, [json.loads(p.read_text(encoding="utf-8")) for p in args.audits])
args.out.write_text(json.dumps(merged, indent=2, ensure_ascii=False) + "\n")
print(f"merged {merged['tracks_audited']} tracks from {len(args.audits)} shards")
return 0


if __name__ == "__main__":
sys.exit(main())
Loading
Loading