From 24d8a5f080337b83f8ed9078e705c1e75b663caf Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 01:04:20 +0900 Subject: [PATCH 1/6] musebench: add HTTP range tar-header shard indexer with tests Walks each of the 75 MuseBench tar shards header by header over HTTP range requests (GNU long names, POSIX prefix, pax paths), resumable per shard, writing index/.jsonl and .done under MUSEBENCH_ROOT. Assumption: shards are standalone tars starting at offset 0 (verified on part1_002, part2_001, part3_008, part5_015). --- scripts/musebench/index_shards.py | 414 ++++++++++++++++++++++++++++++ tests/test_musebench_index.py | 208 +++++++++++++++ 2 files changed, 622 insertions(+) create mode 100644 scripts/musebench/index_shards.py create mode 100644 tests/test_musebench_index.py diff --git a/scripts/musebench/index_shards.py b/scripts/musebench/index_shards.py new file mode 100644 index 0000000..83240fc --- /dev/null +++ b/scripts/musebench/index_shards.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python3 +# Created: 2026-11-04 +# Purpose: Build a byte-range index of every MuseBench tar shard on Hugging Face by walking +# tar headers with HTTP range requests (no full download), so any member can later be +# fetched by {shard, data_offset, size}. +# Dependencies: standard library only. +# Test Status: tests/test_musebench_index.py +"""Index the MuseBench tar shards without downloading them. + +MuseBench (arXiv:2610.20310) is published as five Hugging Face dataset repositories +(``Anonymousv22222/MuseBench-part1`` .. ``part5``), each holding 15 standalone GNU tar +shards (~800 GB in total). Every shard starts with a tar header at offset 0, so a shard can +be walked header by header: read 512 bytes, parse name and size, jump to the next header. + +Output, under ``/index/``: + +* ``shards.json`` -- the pinned shard list: repo, revision (commit sha), file, size, LFS sha256. +* ``.jsonl`` -- one record per member: ``offset`` (first header byte, including any + GNU long-name / pax extension headers), ``data_offset``, ``size``, ``path``, ``type``. +* ``.done`` -- written atomically when the end-of-archive marker is reached. + +A shard without ``.done`` is resumed from the end of its last complete record. + +Shard bytes are untrusted input. This script only parses header blocks and never writes +member content; run it with ``python -I``. +""" +from __future__ import annotations + +import argparse +import http.client +import json +import os +import sys +import threading +import time +import urllib.parse +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Callable, Iterator + +BLOCK = 512 +HF = "https://huggingface.co" +REPO_FMT = "Anonymousv22222/MuseBench-part{part}" +PARTS = (1, 2, 3, 4, 5) +DEFAULT_ROOT = Path(os.environ.get("MUSEBENCH_ROOT", "~/data/musebench")).expanduser() +USER_AGENT = "artifactbench-musebench-indexer/1.0" + +Fetch = Callable[[int, int], bytes] + + +class TarFormatError(ValueError): + """Raised when a header block is not a valid tar header.""" + + +# --------------------------------------------------------------------------- tar parsing + +def parse_number(field: bytes) -> int: + """Parse a tar numeric field: NUL/space-terminated octal, or GNU base-256.""" + if field and field[0] & 0x80: + # GNU base-256: the high bit of the first byte flags the encoding. + raw = bytes([field[0] & 0x7F]) + field[1:] + return int.from_bytes(raw, "big") + text = field.split(b"\0", 1)[0].strip(b" ") + if not text: + return 0 + try: + return int(text, 8) + except ValueError as exc: + raise TarFormatError(f"bad numeric field {field!r}") from exc + + +def header_checksum_ok(header: bytes) -> bool: + stored = parse_number(header[148:156]) + computed = sum(header[:148]) + 8 * 0x20 + sum(header[156:BLOCK]) + return stored == computed + + +def padded(size: int) -> int: + return (size + BLOCK - 1) // BLOCK * BLOCK + + +def parse_pax(data: bytes) -> dict[str, bytes]: + """Parse pax extended-header records ``" =\\n"``.""" + out: dict[str, bytes] = {} + pos = 0 + while pos < len(data): + if data[pos:pos + 1] == b"\0": + break + space = data.find(b" ", pos) + digits = data[pos:space] if space > pos else b"" + if not digits.isdigit(): + raise TarFormatError("bad pax record length") + length = int(digits) + if length <= space - pos or pos + length > len(data): + raise TarFormatError("bad pax record length") + record = data[space + 1:pos + length] + if not record.endswith(b"\n") or b"=" not in record: + raise TarFormatError("bad pax record") + key, value = record[:-1].split(b"=", 1) + out[key.decode("utf-8", "replace")] = value + pos += length + return out + + +def decode_name(raw: bytes) -> str: + # surrogateescape keeps non-UTF-8 names round-trippable (json escapes the surrogates). + return raw.decode("utf-8", "surrogateescape") + + +def walk_tar(fetch: Fetch, start: int = 0, prefetch: int = 4 * BLOCK) -> Iterator[dict]: + """Yield member records of a tar archive reachable through ``fetch(offset, length)``. + + ``fetch`` may return fewer bytes than asked only at end of file. The generator's + return value (``StopIteration.value``) is the end-of-archive offset: the first byte of + the zero block that terminates the archive. + """ + off = start + ext_start: int | None = None + long_name: bytes | None = None + pax_path: bytes | None = None + while True: + buf = fetch(off, prefetch) + header = buf[:BLOCK] + if len(header) < BLOCK: + raise TarFormatError(f"truncated archive at offset {off}") + if header == b"\0" * BLOCK: + if ext_start is not None: + raise TarFormatError(f"extension header without member before {off}") + return off + if not header_checksum_ok(header): + raise TarFormatError(f"header checksum mismatch at offset {off}") + typ = header[156:157] + size = parse_number(header[124:136]) + data_offset = off + BLOCK + if ext_start is None: + ext_start = off + + if typ in (b"L", b"K", b"x", b"g"): + have = buf[BLOCK:BLOCK + size] + data = have if len(have) == size else fetch(data_offset, size) + if len(data) < size: + raise TarFormatError(f"truncated extension header at offset {off}") + if typ == b"L": + long_name = data.split(b"\0", 1)[0] + elif typ == b"x": + pax_path = parse_pax(data).get("path", pax_path) + elif typ == b"g": + pass # global pax header: no per-member path + off = data_offset + padded(size) + if typ == b"g": + ext_start = None + continue + + name = header[0:100].split(b"\0", 1)[0] + magic = header[257:263] + if magic == b"ustar\0": + # POSIX ustar: bytes 345..500 are the name prefix. (In GNU "ustar " headers + # the same bytes hold atime/ctime, so they must not be read as a prefix.) + prefix = header[345:500].split(b"\0", 1)[0] + if prefix: + name = prefix + b"/" + name + if long_name is not None: + name = long_name + if pax_path is not None: + name = pax_path + + kind = {b"0": "file", b"\0": "file", b"7": "file", b"5": "dir", + b"1": "hardlink", b"2": "symlink"}.get(typ, "other:" + typ.decode("latin-1")) + # Hard links, symlinks and directories carry no data even if a size is recorded. + if kind in ("dir", "hardlink", "symlink"): + size = 0 + yield { + "offset": ext_start, + "data_offset": data_offset, + "size": size, + "path": decode_name(name), + "type": kind, + "mtime": parse_number(header[136:148]), + } + off = data_offset + padded(size) + ext_start, long_name, pax_path = None, None, None + + +def record_end(rec: dict) -> int: + return rec["data_offset"] + padded(rec["size"]) + + +# --------------------------------------------------------------------------- HTTP fetch + +class RangeFetcher: + """Persistent-connection HTTP range reader for one Hugging Face file. + + The ``resolve`` URL redirects to a signed CDN URL; it is resolved once and re-resolved + when the CDN answers 401/403/404/410 (signature expired) or the connection breaks. + """ + + def __init__(self, url: str, timeout: float = 60.0, max_retries: int = 12): + self.url = url + self.timeout = timeout + self.max_retries = max_retries + self.signed: urllib.parse.SplitResult | None = None + self.conn: http.client.HTTPSConnection | None = None + self.requests = 0 + + def _resolve(self) -> None: + target = self.url + for _ in range(8): + parts = urllib.parse.urlsplit(target) + conn = http.client.HTTPSConnection(parts.netloc, timeout=self.timeout) + path = parts.path + ("?" + parts.query if parts.query else "") + conn.request("GET", path, headers={"Range": "bytes=0-0", "User-Agent": USER_AGENT}) + resp = conn.getresponse() + resp.read() + conn.close() + if resp.status in (301, 302, 303, 307, 308): + target = urllib.parse.urljoin(target, resp.getheader("Location")) + continue + if resp.status in (200, 206): + self.signed = urllib.parse.urlsplit(target) + self._close() + return + raise OSError(f"resolve {self.url}: HTTP {resp.status}") + raise OSError(f"resolve {self.url}: too many redirects") + + def _close(self) -> None: + if self.conn is not None: + try: + self.conn.close() + except OSError: + pass + self.conn = None + + def __call__(self, offset: int, length: int) -> bytes: + delay = 1.0 + for attempt in range(self.max_retries): + try: + if self.signed is None: + self._resolve() + if self.conn is None: + self.conn = http.client.HTTPSConnection(self.signed.netloc, timeout=self.timeout) + path = self.signed.path + ("?" + self.signed.query if self.signed.query else "") + self.conn.request("GET", path, headers={ + "Range": f"bytes={offset}-{offset + length - 1}", "User-Agent": USER_AGENT}) + resp = self.conn.getresponse() + body = resp.read() + self.requests += 1 + if resp.status == 206: + return body + if resp.status == 416: + return b"" + if resp.status in (401, 403, 404, 410): + self.signed = None + self._close() + elif resp.status == 429 or resp.status >= 500: + retry_after = resp.getheader("Retry-After") + if retry_after and retry_after.isdigit(): + delay = max(delay, float(retry_after)) + self._close() + else: + raise OSError(f"range {offset}+{length}: HTTP {resp.status}") + except (OSError, http.client.HTTPException): + self._close() + if attempt == self.max_retries - 1: + raise + time.sleep(min(delay, 120.0)) + delay *= 2 + raise OSError(f"range {offset}+{length}: retries exhausted") + + +# --------------------------------------------------------------------------- shard index + +def resolve_url(repo: str, revision: str, filename: str) -> str: + return f"{HF}/datasets/{repo}/resolve/{revision}/{urllib.parse.quote(filename)}" + + +def _get_json(url: str) -> object: + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(req, timeout=60) as resp: + return json.load(resp) + + +def list_shards(parts=PARTS) -> list[dict]: + """Return the pinned shard list from the Hugging Face API.""" + shards = [] + for part in parts: + repo = REPO_FMT.format(part=part) + revision = _get_json(f"{HF}/api/datasets/{repo}")["sha"] + tree = _get_json(f"{HF}/api/datasets/{repo}/tree/{revision}") + for entry in tree: + if entry.get("type") == "file" and entry["path"].endswith(".tar"): + shards.append({ + "part": part, "repo": repo, "revision": revision, "file": entry["path"], + "size": entry["size"], "lfs_sha256": (entry.get("lfs") or {}).get("oid"), + }) + return sorted(shards, key=lambda s: (s["part"], s["file"])) + + +def shard_stem(shard: dict) -> str: + return shard["file"][:-len(".tar")] + + +def atomic_write_json(path: Path, obj: object) -> None: + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(obj, indent=1, sort_keys=True) + "\n", encoding="utf-8") + os.replace(tmp, path) + + +def resume_offset(jsonl: Path) -> tuple[int, int]: + """Return (next header offset, complete record count); drop a torn trailing line.""" + if not jsonl.exists(): + return 0, 0 + raw = jsonl.read_bytes() + keep = raw.rfind(b"\n") + 1 + if keep != len(raw): + with open(jsonl, "r+b") as f: + f.truncate(keep) + raw = raw[:keep] + lines = raw.splitlines() + if not lines: + return 0, 0 + return record_end(json.loads(lines[-1])), len(lines) + + +def index_shard(shard: dict, out_dir: Path, fetch: Fetch, + progress: Callable[[str, int], None] | None = None) -> dict: + """Walk one shard into ``.jsonl`` and mark it ``.done``. Idempotent.""" + stem = shard_stem(shard) + jsonl, done = out_dir / f"{stem}.jsonl", out_dir / f"{stem}.done" + if done.exists(): + return json.loads(done.read_text(encoding="utf-8")) + start, count = resume_offset(jsonl) + gen = walk_tar(fetch, start) + with open(jsonl, "a", encoding="utf-8") as f: + while True: + try: + rec = next(gen) + except StopIteration as stop: + end_offset = stop.value + break + rec["shard"] = shard["file"] + f.write(json.dumps(rec, sort_keys=True) + "\n") + f.flush() + count += 1 + if progress is not None: + progress(stem, count) + if end_offset > shard["size"]: + raise TarFormatError(f"{stem}: end offset {end_offset} beyond file size {shard['size']}") + info = { + **shard, + "members": count, + "end_offset": end_offset, + "finished_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + atomic_write_json(done, info) + return info + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--root", type=Path, default=DEFAULT_ROOT, + help="data root (default: $MUSEBENCH_ROOT or ~/data/musebench)") + ap.add_argument("--parts", type=int, nargs="+", default=list(PARTS)) + ap.add_argument("--only", nargs="*", default=None, help="restrict to these shard stems") + ap.add_argument("--workers", type=int, default=75, help="shards walked concurrently") + ap.add_argument("--refresh-list", action="store_true", + help="re-query the shard list even if index/shards.json exists") + args = ap.parse_args(argv) + + out_dir = args.root / "index" + out_dir.mkdir(parents=True, exist_ok=True) + list_path = out_dir / "shards.json" + if list_path.exists() and not args.refresh_list: + shards = json.loads(list_path.read_text(encoding="utf-8")) + else: + shards = list_shards(PARTS) + atomic_write_json(list_path, shards) + todo = [s for s in shards if s["part"] in args.parts + and (args.only is None or shard_stem(s) in args.only)] + + counts: dict[str, int] = {} + lock = threading.Lock() + last_report = [0.0] + + def progress(stem: str, n: int) -> None: + with lock: + counts[stem] = n + now = time.time() + if now - last_report[0] >= 60: + last_report[0] = now + print(f"[{time.strftime('%H:%M:%S')}] members indexed: {sum(counts.values())} " + f"across {len(counts)} shards", file=sys.stderr, flush=True) + + def run(shard: dict) -> dict: + url = resolve_url(shard["repo"], shard["revision"], shard["file"]) + return index_shard(shard, out_dir, RangeFetcher(url), progress) + + failures = 0 + with ThreadPoolExecutor(max_workers=max(1, args.workers)) as pool: + futures = {pool.submit(run, s): s for s in todo} + for fut in as_completed(futures): + stem = shard_stem(futures[fut]) + try: + info = fut.result() + print(f"done {stem}: {info['members']} members", file=sys.stderr, flush=True) + except Exception as exc: # noqa: BLE001 - report and continue with other shards + failures += 1 + print(f"FAILED {stem}: {exc!r}", file=sys.stderr, flush=True) + print(f"{len(todo) - failures}/{len(todo)} shards indexed", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_musebench_index.py b/tests/test_musebench_index.py new file mode 100644 index 0000000..f8cf348 --- /dev/null +++ b/tests/test_musebench_index.py @@ -0,0 +1,208 @@ +# Created: 2026-11-04 +# Purpose: Cover scripts/musebench/index_shards.py (tar header walk over a range reader) +# on synthetic GNU, PAX and USTAR archives, plus resume after an interrupted walk. +# Dependencies: pytest +# Test Status: passing +import io +import json +import tarfile + +import pytest + +from scripts.musebench import index_shards as ix + + +def make_tar(members, fmt): + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w", format=fmt) as tf: + for name, data in members: + if data is None: + info = tarfile.TarInfo(name) + info.type = tarfile.DIRTYPE + tf.addfile(info) + continue + info = tarfile.TarInfo(name) + info.size = len(data) + info.mtime = 1_700_000_000 + tf.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +def reader(blob, calls=None): + def fetch(offset, length): + if calls is not None: + calls.append((offset, length)) + return blob[offset:offset + length] + return fetch + + +def walk_all(blob, start=0): + gen = ix.walk_tar(reader(blob), start) + recs = [] + while True: + try: + recs.append(next(gen)) + except StopIteration as stop: + return recs, stop.value + + +def check_against_tarfile(blob, recs): + """Every record must locate exactly the bytes tarfile reads for that member.""" + with tarfile.open(fileobj=io.BytesIO(blob)) as tf: + expected = [(m.name, m.size, m.offset_data) for m in tf.getmembers() if m.isfile()] + got = [(r["path"], r["size"], r["data_offset"]) for r in recs if r["type"] == "file"] + assert got == expected + + +LONG = "MagNet_Generated_Music/Genius-Lyrics/" + "x" * 150 + ".mp3.wav" +UNICODE = "Jamendo/Часть 1_4 - Ilyin_Books & ‘quote’.mp3" +PREFIXED = "ai_human_co-created_music/Jamendo/" + "d" * 60 + "/" + "f" * 90 + ".wav" + + +@pytest.mark.parametrize("fmt", [tarfile.GNU_FORMAT, tarfile.PAX_FORMAT, tarfile.USTAR_FORMAT]) +def test_walk_matches_tarfile(fmt): + members = [ + ("Caption/captions-genius.csv", b"a,b\n" * 300), + ("Jamendo", None), + ("Jamendo/empty.mp3", b""), + ("Jamendo/exact.mp3", b"z" * 1024), + (PREFIXED, b"p" * 700), + ] + if fmt != tarfile.USTAR_FORMAT: # USTAR cannot store these names + members += [(LONG, b"l" * 513), (UNICODE, b"u" * 5)] + blob = make_tar(members, fmt) + recs, end = walk_all(blob) + check_against_tarfile(blob, recs) + names = [r["path"] for r in recs] + assert PREFIXED in names + if fmt != tarfile.USTAR_FORMAT: + assert LONG in names and UNICODE in names + assert recs[0]["offset"] == 0 + for prev, cur in zip(recs, recs[1:]): + assert cur["offset"] == ix.record_end(prev) # contiguous, extensions included + assert end == ix.record_end(recs[-1]) + assert blob[end:end + 1024] == b"\0" * 1024 + + +def test_gnu_long_name_record_starts_at_extension_header(): + blob = make_tar([("a.wav", b"1"), (LONG, b"2")], tarfile.GNU_FORMAT) + recs, _ = walk_all(blob) + long_rec = recs[1] + assert long_rec["path"] == LONG + # The record offset is the ././@LongLink header, one header + one data block earlier. + assert long_rec["data_offset"] - long_rec["offset"] == 3 * ix.BLOCK + + +def test_gnu_header_tail_is_not_read_as_prefix(): + # GNU headers store atime/ctime where POSIX keeps the prefix; it must be ignored. + blob = bytearray(make_tar([("short.wav", b"abc")], tarfile.GNU_FORMAT)) + assert bytes(blob[257:263]) == b"ustar " + blob[345:357] = b"14672366670\0" # fake atime + blob[148:156] = b" " + chk = sum(blob[:512]) + blob[148:156] = b"%06o\0 " % chk + recs, _ = walk_all(bytes(blob)) + assert recs[0]["path"] == "short.wav" + + +def test_checksum_mismatch_raises(): + blob = bytearray(make_tar([("a.wav", b"1")], tarfile.GNU_FORMAT)) + blob[0] = ord("b") + with pytest.raises(ix.TarFormatError, match="checksum"): + walk_all(bytes(blob)) + + +def test_truncated_archive_raises(): + blob = make_tar([("a.wav", b"1" * 2000)], tarfile.GNU_FORMAT) + with pytest.raises(ix.TarFormatError, match="truncated"): + walk_all(blob[:512 + 2048]) + + +def test_parse_number_octal_and_base256(): + assert ix.parse_number(b"0000644\0") == 0o644 + assert ix.parse_number(b" 12 \0") == 0o12 + assert ix.parse_number(b"\0" * 8) == 0 + big = 20 * 1024 ** 3 + field = bytes([0x80]) + big.to_bytes(11, "big") + assert ix.parse_number(field) == big + + +def pax_record(key, value): + body = b" " + key + b"=" + value + b"\n" + n = len(body) + 1 + while len(str(n)) + len(body) != n: + n += 1 + return str(n).encode() + body + + +def test_parse_pax_records(): + data = pax_record(b"mtime", b"1700000000.5") + pax_record(b"path", "Jamendo/Ч.mp3".encode()) + out = ix.parse_pax(data + b"\0" * 7) + assert out["path"] == "Jamendo/Ч.mp3".encode() + assert out["mtime"] == b"1700000000.5" + with pytest.raises(ix.TarFormatError): + ix.parse_pax(b"99 path=x\n") + + +def test_one_fetch_per_plain_member(): + members = [(f"Jamendo/{i}.mp3", b"m" * 5000) for i in range(5)] + blob = make_tar(members, tarfile.GNU_FORMAT) + calls = [] + gen = ix.walk_tar(reader(blob, calls)) + assert len(list(gen)) == 5 + assert len(calls) == 6 # five headers + end marker + + +def shard_for(blob, stem="part9_001"): + return {"part": 9, "repo": "r", "revision": "abc", "file": f"{stem}.tar", + "size": len(blob), "lfs_sha256": None} + + +def test_index_shard_writes_jsonl_and_done(tmp_path): + blob = make_tar([(f"suno_generated_music/fake_{i:05d}_suno_0.mp3", b"s" * 900) + for i in range(4)], tarfile.GNU_FORMAT) + info = ix.index_shard(shard_for(blob), tmp_path, reader(blob)) + lines = (tmp_path / "part9_001.jsonl").read_text().splitlines() + assert info["members"] == len(lines) == 4 + done = json.loads((tmp_path / "part9_001.done").read_text()) + assert done["end_offset"] == ix.record_end(json.loads(lines[-1])) + assert all(json.loads(l)["shard"] == "part9_001.tar" for l in lines) + # Idempotent: a second call does not touch the network. + assert ix.index_shard(shard_for(blob), tmp_path, lambda o, n: 1 / 0) == done + + +def test_resume_after_interruption_drops_torn_line(tmp_path): + members = [(f"udio_generated_music/fake_{i:05d}_udio_1.mp3", b"u" * (300 + i)) for i in range(6)] + blob = make_tar(members, tarfile.GNU_FORMAT) + shard = shard_for(blob) + fetch = reader(blob) + + class Boom(Exception): + pass + + n = [0] + + def flaky(offset, length): + n[0] += 1 + if n[0] == 4: + raise Boom + return fetch(offset, length) + + with pytest.raises(Boom): + ix.index_shard(shard, tmp_path, flaky) + jsonl = tmp_path / "part9_001.jsonl" + with open(jsonl, "a") as f: + f.write('{"torn": ') # simulate a crash mid-write + assert not (tmp_path / "part9_001.done").exists() + ix.index_shard(shard, tmp_path, fetch) + recs = [json.loads(l) for l in jsonl.read_text().splitlines()] + assert [r["path"] for r in recs] == [m[0] for m in members] + check_against_tarfile(blob, recs) + + +def test_end_offset_beyond_listed_size_is_rejected(tmp_path): + blob = make_tar([("a.wav", b"1")], tarfile.GNU_FORMAT) + shard = shard_for(blob) + shard["size"] = 100 + with pytest.raises(ix.TarFormatError, match="beyond"): + ix.index_shard(shard, tmp_path, reader(blob)) From 4dde3d0c55eedce0ee2296f995ecbdf6b91d9dc5 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 01:05:23 +0900 Subject: [PATCH 2/6] musebench: add stage verifier (index stage) with tests verify.py inserts the repo root into sys.path so it runs under python -I. Later stages (acquire, leak, run) report 'no verifier yet' until their milestones land. --- scripts/musebench/verify.py | 155 +++++++++++++++++++++++++++++++++ tests/test_musebench_verify.py | 95 ++++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 scripts/musebench/verify.py create mode 100644 tests/test_musebench_verify.py diff --git a/scripts/musebench/verify.py b/scripts/musebench/verify.py new file mode 100644 index 0000000..9151420 --- /dev/null +++ b/scripts/musebench/verify.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# Created: 2026-11-04 +# Purpose: Assert the on-disk state of each MuseBench pipeline stage (index, acquire, leak, run) +# under MUSEBENCH_ROOT; exit 0 only when the requested stage is complete and consistent. +# Dependencies: standard library only (later stages import artifactbench from this repo). +# Test Status: tests/test_musebench_verify.py +"""Verify MuseBench pipeline stages. + +Usage: ``python -I scripts/musebench/verify.py [--root DIR] {index,acquire,leak,run,all}`` + +``-I`` keeps the script directory and user site-packages off ``sys.path``; this script +adds the repository root itself so it can import project code without trusting anything +under the data root. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from collections import Counter +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from scripts.musebench.index_shards import BLOCK, padded, record_end # noqa: E402 + +DEFAULT_ROOT = Path(os.environ.get("MUSEBENCH_ROOT", "~/data/musebench")).expanduser() +EXPECTED_PARTS = {1: 15, 2: 15, 3: 15, 4: 15, 5: 15} +# Lower bound on audio files in a complete index. The paper reports 361,166 tracks; the +# count actually found in the published shards is recorded in docs/MUSEBENCH.md. +MIN_AUDIO_FILES = 300_000 +AUDIO_EXT = (".wav", ".mp3", ".flac", ".ogg", ".m4a", ".aac", ".opus") + + +def load_jsonl(path: Path) -> list[dict]: + with open(path, encoding="utf-8") as f: + return [json.loads(line) for line in f if line.strip()] + + +def is_audio(path: str) -> bool: + return path.lower().endswith(AUDIO_EXT) + + +def verify_index(root: Path, expected_parts=EXPECTED_PARTS, + min_audio_files: int = MIN_AUDIO_FILES) -> tuple[list[str], dict]: + """Return (problems, stats) for the shard index under ``root/index``.""" + problems: list[str] = [] + index = root / "index" + list_path = index / "shards.json" + if not list_path.exists(): + return [f"missing {list_path}"], {} + shards = json.loads(list_path.read_text(encoding="utf-8")) + per_part = Counter(s["part"] for s in shards) + if dict(per_part) != dict(expected_parts): + problems.append(f"shard list per part {dict(per_part)} != expected {dict(expected_parts)}") + if len({s["file"] for s in shards}) != len(shards): + problems.append("duplicate shard names in shards.json") + for part in per_part: + if len({s["revision"] for s in shards if s["part"] == part}) != 1: + problems.append(f"part {part}: shards not pinned to a single revision") + + stats = {"shards": len(shards), "done": 0, "members": 0, "files": 0, "audio_files": 0, + "bytes": 0, "top_level": Counter()} + seen_paths: set[str] = set() + for shard in shards: + stem = shard["file"][:-len(".tar")] + done_path, jsonl = index / f"{stem}.done", index / f"{stem}.jsonl" + if not done_path.exists(): + problems.append(f"{stem}: not finished (no .done)") + continue + done = json.loads(done_path.read_text(encoding="utf-8")) + recs = load_jsonl(jsonl) if jsonl.exists() else [] + stats["done"] += 1 + if done.get("members") != len(recs): + problems.append(f"{stem}: .done members {done.get('members')} != {len(recs)} records") + if not recs: + problems.append(f"{stem}: empty index") + continue + if recs[0]["offset"] != 0: + problems.append(f"{stem}: first record at {recs[0]['offset']}, not 0") + for prev, cur in zip(recs, recs[1:]): + if cur["offset"] != record_end(prev): + problems.append(f"{stem}: gap/overlap before {cur['path']!r} at {cur['offset']}") + break + end = done.get("end_offset") + if end != record_end(recs[-1]): + problems.append(f"{stem}: end_offset {end} != end of last record {record_end(recs[-1])}") + elif end + 2 * BLOCK > shard["size"]: + problems.append(f"{stem}: end-of-archive marker does not fit in {shard['size']} bytes") + for rec in recs: + if rec["shard"] != shard["file"]: + problems.append(f"{stem}: record names shard {rec['shard']}") + break + if rec["data_offset"] + padded(rec["size"]) > shard["size"]: + problems.append(f"{stem}: {rec['path']!r} extends past end of shard") + break + stats["members"] += 1 + if rec["type"] != "file": + continue + if rec["path"] in seen_paths: + problems.append(f"duplicate member path across shards: {rec['path']!r}") + seen_paths.add(rec["path"]) + stats["files"] += 1 + stats["bytes"] += rec["size"] + if is_audio(rec["path"]): + stats["audio_files"] += 1 + stats["top_level"][rec["path"].split("/", 1)[0]] += 1 + if stats["done"] == len(shards) and stats["audio_files"] < min_audio_files: + problems.append(f"only {stats['audio_files']} audio files indexed (< {min_audio_files})") + stats["top_level"] = dict(sorted(stats["top_level"].items())) + return problems, stats + + +def not_implemented(stage: str): + def check(root: Path) -> tuple[list[str], dict]: + return [f"stage '{stage}' has no verifier yet (later milestone)"], {} + return check + + +STAGES = { + "index": verify_index, + "acquire": not_implemented("acquire"), + "leak": not_implemented("leak"), + "run": not_implemented("run"), +} + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Verify MuseBench pipeline stages.") + ap.add_argument("stage", choices=[*STAGES, "all"]) + ap.add_argument("--root", type=Path, default=DEFAULT_ROOT) + ap.add_argument("--json", action="store_true", help="print stats as JSON") + args = ap.parse_args(argv) + stages = list(STAGES) if args.stage == "all" else [args.stage] + failed = False + for stage in stages: + problems, stats = STAGES[stage](args.root) + if args.json: + print(json.dumps({"stage": stage, "problems": problems, "stats": stats}, indent=1)) + else: + brief = {k: v for k, v in stats.items() if k != "top_level"} + print(f"[{stage}] {'OK' if not problems else 'FAIL'} {brief}") + for p in problems[:50]: + print(f" - {p}") + if len(problems) > 50: + print(f" ... {len(problems) - 50} more") + failed = failed or bool(problems) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_musebench_verify.py b/tests/test_musebench_verify.py new file mode 100644 index 0000000..d6102e5 --- /dev/null +++ b/tests/test_musebench_verify.py @@ -0,0 +1,95 @@ +# Created: 2026-11-04 +# Purpose: Cover scripts/musebench/verify.py index-stage checks on a synthetic shard index. +# Dependencies: pytest +# Test Status: passing +import io +import json +import subprocess +import sys +import tarfile +from pathlib import Path + +from scripts.musebench import index_shards as ix +from scripts.musebench import verify as vf + +PARTS = {1: 2, 2: 1} + + +def make_tar(names): + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w", format=tarfile.GNU_FORMAT) as tf: + for name in names: + data = name.encode() * 3 + info = tarfile.TarInfo(name) + info.size = len(data) + tf.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +def build_index(root: Path, finish=True): + index = root / "index" + index.mkdir(parents=True) + contents = { + "part1_001": ["Caption/captions-genius.csv", "Jamendo/a.mp3", "Jamendo/b.mp3"], + "part1_002": ["Musicgen_Generated_Music/Jamedo/a.mp3.wav"], + "part2_001": ["suno_generated_music/fake_00001_suno_0.mp3", "OOD FMA/000002.mp3"], + } + shards = [] + for stem, names in contents.items(): + blob = make_tar(names) + shard = {"part": int(stem[4]), "repo": f"r{stem[4]}", "revision": f"sha{stem[4]}", + "file": f"{stem}.tar", "size": len(blob), "lfs_sha256": None} + shards.append(shard) + if finish or stem != "part2_001": + ix.index_shard(shard, index, lambda o, n, b=blob: b[o:o + n]) + (index / "shards.json").write_text(json.dumps(shards)) + return index + + +def test_complete_index_passes(tmp_path): + build_index(tmp_path) + problems, stats = vf.verify_index(tmp_path, expected_parts=PARTS, min_audio_files=5) + assert problems == [] + assert stats["done"] == 3 and stats["files"] == 6 and stats["audio_files"] == 5 + assert stats["top_level"]["Jamendo"] == 2 and stats["top_level"]["OOD FMA"] == 1 + + +def test_unfinished_shard_fails(tmp_path): + build_index(tmp_path, finish=False) + problems, _ = vf.verify_index(tmp_path, expected_parts=PARTS, min_audio_files=1) + assert any("part2_001: not finished" in p for p in problems) + + +def test_wrong_shard_count_and_too_few_files_fail(tmp_path): + build_index(tmp_path) + problems, _ = vf.verify_index(tmp_path, expected_parts={1: 15, 2: 15}, min_audio_files=99) + assert any("shard list per part" in p for p in problems) + assert any("audio files indexed" in p for p in problems) + + +def test_gap_and_member_count_mismatch_fail(tmp_path): + index = build_index(tmp_path) + jsonl = index / "part1_001.jsonl" + lines = jsonl.read_text().splitlines() + rec = json.loads(lines[1]) + rec["offset"] += 512 + lines[1] = json.dumps(rec) + jsonl.write_text("\n".join(lines[:2]) + "\n") + problems, _ = vf.verify_index(tmp_path, expected_parts=PARTS, min_audio_files=1) + assert any("gap/overlap" in p for p in problems) + assert any("members 3 != 2" in p for p in problems) + + +def test_missing_index_fails(tmp_path): + problems, _ = vf.verify_index(tmp_path) + assert problems and "missing" in problems[0] + + +def test_cli_runs_isolated_and_reports_failure(tmp_path): + script = Path(vf.__file__) + out = subprocess.run([sys.executable, "-I", str(script), "--root", str(tmp_path), "index"], + capture_output=True, text=True) + assert out.returncode == 1 and "[index] FAIL" in out.stdout + out = subprocess.run([sys.executable, "-I", str(script), "--root", str(tmp_path), "acquire"], + capture_output=True, text=True) + assert out.returncode == 1 and "no verifier yet" in out.stdout From e736f71619e2d23cde52edac5c19adf269822da2 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 01:07:13 +0900 Subject: [PATCH 3/6] docs: add MuseBench research note and zero-shot protocol Summarizes arXiv:2610.20310v1 (Tables 1, 4, 10 checked against the arXiv HTML), the HF dataset card, their discrepancies, pinned shard revisions, the ArtifactBench zero-shot protocol, acquisition rules and licenses. No ArtifactNet scores are included (repo is public). --- docs/MUSEBENCH.md | 162 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 docs/MUSEBENCH.md diff --git a/docs/MUSEBENCH.md b/docs/MUSEBENCH.md new file mode 100644 index 0000000..cd46286 --- /dev/null +++ b/docs/MUSEBENCH.md @@ -0,0 +1,162 @@ +# MuseBench — research note and zero-shot protocol + +MuseBench is a large human-vs-AI music detection benchmark: + +> Yi Chen Liu, Syed Irfan Ali Meerza, Jiawei Yu, Xi Guo, Samuel Bright, Jian Liu. +> *MuseBench: A Benchmark for AI-Generated Music Detection and Artifact Analysis.* +> arXiv:2610.20310v1, 14 Oct 2026. + +Data: Hugging Face `Anonymousv22222/MuseBench-part1` … `MuseBench-part5` (full release) and +`Anonymousv22222/MuseBench-Example` (one file per folder). This page summarizes the paper and the +dataset card, lists where they disagree, and defines how ArtifactBench evaluates a detector that +was **not** trained on MuseBench. It contains no ArtifactNet scores: results stay with the +maintainer's data root. + +## Composition (paper, Table 4) + +| Source | Tracks | Hours | Type | +|---|---:|---:|---| +| Genius-Lyrics (human, **not redistributed**) | 9,983 | 609 | vocal | +| JamendoMaxCaps (human) | 9,920 | 648 | instrumental | +| MusicGen, AudioLDM2, MAGNeT, Stable Audio Open, Lyria RealTime | 19,903 each | 165 each | instr. | +| Riffusion | 19,903 | 169 | vocal | +| JASCO | 19,903 | 55 | instr. | +| Mustango | 19,903 | 56 | vocal | +| Suno (from SONICS: v2, v3, v3.5) | 10,000 | 485 | vocal | +| Udio (from SONICS: Udio-32, Udio-130) | 10,000 | 307 | vocal | +| Human–AI co-created (MusicGen-Style remixes of human tracks) | 9,887 | 82 | mixed | +| Lossy transforms (MP3 / OGG / AAC) | 53,805 | 571 | — | +| Acoustic post-processing (EQ / pitch-time / loudness) | 53,805 | 572 | — | +| Additive (Gaussian / PGD) | 35,870 | 239 | — | +| FMA, pre-2010 (human, out-of-domain) | 8,672 | 678 | mixed | +| **Total** | **361,166** | **5,296** | | + +The paired generators were conditioned on captions of the Genius-Lyrics and Jamendo tracks (one +generation per source track, capped at 30 s; JASCO and Mustango produce ~10 s). Each transform +family has 17,935 tracks per transform (53,805 = 3 × 17,935), about 10 % of the 179,224 AI tracks, +consistent with transforms being applied to the test split only. + +## Distribution on Hugging Face + +Five dataset repositories, each with 15 standalone GNU tar shards (`partN_001.tar` … +`partN_015.tar`): 75 shards, 798.4 GB. Every shard starts with a tar header at offset 0, so a +shard can be indexed by walking its headers with HTTP range requests, without downloading it. +The shard list is pinned to these commits: + +| Repository | Revision | +|---|---| +| `Anonymousv22222/MuseBench-part1` | `7c7131dba082dcee0858327d1beac17b1bb540f8` | +| `Anonymousv22222/MuseBench-part2` | `906fccf9d3f7608fb1078e0d38973ea086b70eb8` | +| `Anonymousv22222/MuseBench-part3` | `f15d3bc904d5d76b883c840c6963e86802528152` | +| `Anonymousv22222/MuseBench-part4` | `251fd5aeab23b9ec5bb6b7c2f1c8847c1d0bda57` | +| `Anonymousv22222/MuseBench-part5` | `95dbba8b8e3b6e917c1119af8046d3ab8ded7189` | + + + +## Dataset card vs paper + +| Topic | Dataset card | Paper | +|---|---|---| +| Tenth generator | `magenta_generated_music/` ("Magenta", Apache 2.0) | Lyria RealTime via API | +| Paired generators | 8 (MusicGen, MAGNeT, AudioLDM, JASCO, Magenta, Mustango, Riffusion, Stable Audio) | 8 open + Lyria; AudioLDM is **AudioLDM2** | +| Jamendo human count | ~9,945 | 9,920 | +| OOD FMA window | "2008–2010" in one place, "pre-2008" in the tree comment | "uploaded before 2010" | +| Genius-Lyrics human audio | not shipped (copyrighted) | used in every split | +| Suno / Udio origin | "closed-source commercial generators" | reused from SONICS | +| Splits | none published | Default 70/20/10, LOMO; no split file, no seeds | +| Transforms | "every AI track" | test-set AI tracks only (Appendix A.3) | + +Consequences: the Default and LOMO human negatives cannot be rebuilt (half of them, the vocal +Genius-Lyrics side, is missing), and the paper's test split is unknown, so paper numbers for +Default/LOMO cannot be reproduced on the released files. Folder spellings are preserved as +shipped: `Jamedo/` under each generator, `Genius-lyrics/` under JASCO, +`gaussion_noised_music/`, `loudness_generation_music/`. + +## Paper protocol + +- **Input:** one random 10 s mono excerpt at 48 kHz per track (30 s in an appendix ablation); + minority class oversampled by drawing fresh excerpts. +- **Settings:** *Default* (human negative vs AI positive, 70/20/10 train/val/test); *LOMO* + (leave one generator out, remaining AI 80/20 train/val); *OOD* (FMA accuracy with Default + checkpoints); *robustness* (co-created and transformed AI tracks, FNR, Default checkpoints). +- **Metrics:** balanced accuracy, precision, recall, FPR (human flagged as AI), FNR (AI passed + as human). +- **Detectors:** Q-SVM, Deezer, M5, RawNet2, ConvNeXt, ResNet18+Spec, ViT, AST, SpecTTTra, and + the Ircam Amplify and Resemble AI APIs (on a 1,990 human / 4,500 AI subset). Except for the + two APIs, every detector is **trained on MuseBench Default**. +- **Transforms (Appendix A.3):** MP3 and AAC at 128 kbps; OGG Vorbis q3; EQ ±6 dB boost/cut in + one of three bands (25–250 Hz, 250 Hz–4 kHz, 4–20 kHz; scaled to the track bandwidth); + pitch shift −2/−1/+1/+2 semitones; speed 0.9×/1.1×; loudness normalized to −14 LUFS; Gaussian + noise σ = 0.025; PGD (ℓ∞, α = 0.001, ε = 0.005, T = 10) with M5 / RawNet2 gradients. + +Paper results that a zero-shot detector can be placed next to: + +| Detector | FMA OOD acc. (Table 10) | Co-created FNR | MP3 | OGG | AAC | EQ | Pitch/time | Loudness | Gaussian | PGD | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| Deezer | 94.4 | 36.8 | 12.3 | 9.5 | 7.7 | 6.4 | 93.6 | 5.2 | 19.6 | 26.4 | +| M5 | 75.8 | 20.1 | 15.2 | 1.9 | 1.9 | 2.0 | 26.2 | 1.3 | 2.1 | 92.4 | +| ConvNeXt | 43.5 | 12.8 | 0.2 | 0.3 | 0.6 | 0.2 | 17.0 | 0.2 | 0.4 | 11.7 | +| ResNet18+Spec | 28.4 | 0.0 | 0.9 | 3.7 | 0.1 | 3.1 | 11.3 | 2.0 | 5.2 | 11.1 | +| AST | 79.8 | 4.8 | 24.9 | 7.3 | 3.5 | 0.0 | 4.2 | 0.0 | 0.0 | 9.2 | +| SpecTTTra | 39.7 | 33.6 | 0.5 | 1.0 | 1.0 | 0.5 | 16.7 | 0.5 | 0.9 | 10.7 | + +(All values in %, from Tables 1 and 10 of arXiv:2610.20310v1; transform columns are FNR.) + +## ArtifactBench zero-shot protocol for MuseBench + +ArtifactBench evaluates detectors that were not trained on MuseBench. The rules: + +1. **No tuning on MuseBench.** The detector's own fixed operating threshold is used (0.5 for + ArtifactNet), and AUROC is reported as the threshold-free view. Nothing (threshold, crop, + calibration) is selected on MuseBench data. +2. **Native input.** Each detector receives the audio through its own adapter (for ArtifactNet: + 44.1 kHz, seven 4 s chunks, centre crop), not the paper's 10 s / 48 kHz excerpt. +3. **Eval cohort** (deterministic, sealed): all Jamendo human tracks, all OOD-FMA tracks, + up to 2,000 clean tracks per generator (including Suno and Udio), up to 2,000 co-created + tracks, and up to 200 tracks per transform × generator. Rows in the cohort are marked + `musebench_role = eval` and must never be used for training. +4. **Leak annotation.** Suno and Udio are SONICS tracks; FMA and Jamendo audio may also appear in + a detector's training history. Every row carries `seen_in_training` for the detector under + test, and every metric is reported for seen and unseen rows separately. +5. **What is reported:** human FPR (Jamendo), OOD-FMA accuracy, per-generator FNR on clean + tracks, co-created FNR, per-transform FNR, AUROC (human vs each AI group), decode/inference + failures. Comparisons are drawn against Tables 1 and 10 only, with the caveat that the + paper's detectors were trained in-distribution; there is no valid comparison with Table 8. + +## Acquisition + +The full release (~800 GB) is larger than the space available, so it is indexed in full and +acquired selectively: + +```bash +# Byte-range index of all 75 shards (no audio is downloaded); resumable. +python scripts/musebench/index_shards.py --root ~/data/musebench --workers 75 +# Assert the index is complete and internally consistent. +python scripts/musebench/verify.py --root ~/data/musebench index +``` + +Rules for the data root (`$MUSEBENCH_ROOT`, default `~/data/musebench`): + +- Shard bytes and extracted audio are untrusted: they live only under the data root, scripts live + in this repository, and Python that parses them runs with `-I`. +- Audio is kept lossless and byte-identical to the shard member (verified by size and SHA-256); + it is never re-encoded, because re-encoding changes the artifacts detectors look for. +- Non-PGD transforms can be regenerated from clean audio with the parameters above, so only the + scored transform sample is kept. +- Nothing derived from MuseBench audio (manifests with local paths, scores, reports) is committed + to this repository. + +## Licenses + +- Collection: CC BY-SA 3.0 (inherited from JamendoMaxCaps). +- Jamendo audio: per-track Creative Commons licenses (CC BY, BY-SA, BY-NC, BY-NC-ND), recorded + in the `license_ccurl` metadata; NC and ND tracks restrict commercial use and derivatives. +- FMA audio: per-track Creative Commons licenses; FMA metadata CC BY 4.0. +- AI audio: bound by each generator's terms; most open weights are non-commercial + (MusicGen / MAGNeT / JASCO weights CC BY-NC 4.0, AudioLDM and Mustango CC BY-NC-SA 4.0, + Stable Audio Open community license). +- Suno / Udio outputs: research use only under each platform's terms. + +Benchmarking is research use. Whether any part of MuseBench may be used to train a commercial +detector is a separate legal decision; per-row license fields are kept so the corpus can be +filtered. From 260c1071026444da7fa96fb1d053adc5de4f4da4 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 01:07:43 +0900 Subject: [PATCH 4/6] musebench: add index inventory summary tool with tests --- scripts/musebench/inventory.py | 81 +++++++++++++++++++++++++++++++ tests/test_musebench_inventory.py | 32 ++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 scripts/musebench/inventory.py create mode 100644 tests/test_musebench_inventory.py diff --git a/scripts/musebench/inventory.py b/scripts/musebench/inventory.py new file mode 100644 index 0000000..91d7092 --- /dev/null +++ b/scripts/musebench/inventory.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# Created: 2026-11-04 +# Purpose: Summarize a MuseBench shard index by folder (file counts, bytes, extensions, shards). +# Dependencies: standard library only. +# Test Status: tests/test_musebench_inventory.py +"""Summarize the MuseBench shard index. + +Groups regular files by their first ``--depth`` path components (default 2, e.g. +``Musicgen_Generated_Music/Jamedo``) and prints a Markdown table, or JSON with ``--json``. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from collections import Counter, defaultdict +from pathlib import Path + +DEFAULT_ROOT = Path(os.environ.get("MUSEBENCH_ROOT", "~/data/musebench")).expanduser() + + +def iter_records(index_dir: Path): + for jsonl in sorted(index_dir.glob("*.jsonl")): + with open(jsonl, encoding="utf-8") as f: + for line in f: + if line.strip(): + yield json.loads(line) + + +def group_key(path: str, depth: int) -> str: + parts = path.split("/") + # Files directly under a folder at a shallower level keep their folder as the key. + return "/".join(parts[:min(depth, len(parts) - 1)]) or "." + + +def summarize(records, depth: int = 2) -> dict[str, dict]: + groups: dict[str, dict] = defaultdict(lambda: {"files": 0, "bytes": 0, "ext": Counter(), + "shards": set()}) + for rec in records: + if rec.get("type") != "file": + continue + g = groups[group_key(rec["path"], depth)] + g["files"] += 1 + g["bytes"] += rec["size"] + name = rec["path"].rsplit("/", 1)[-1] + g["ext"][name.rsplit(".", 1)[-1].lower() if "." in name else ""] += 1 + g["shards"].add(rec["shard"]) + out = {} + for key in sorted(groups): + g = groups[key] + out[key] = {"files": g["files"], "bytes": g["bytes"], "ext": dict(g["ext"].most_common()), + "shards": sorted(g["shards"])} + return out + + +def to_markdown(summary: dict[str, dict]) -> str: + lines = ["| Folder | Files | GB | Extensions | Shards |", "|---|---:|---:|---|---:|"] + total_files = total_bytes = 0 + for key, g in summary.items(): + ext = ", ".join(f"{e or '(none)'} {n}" for e, n in g["ext"].items()) + lines.append(f"| `{key}` | {g['files']:,} | {g['bytes'] / 1e9:.1f} | {ext} | {len(g['shards'])} |") + total_files += g["files"] + total_bytes += g["bytes"] + lines.append(f"| **Total** | **{total_files:,}** | **{total_bytes / 1e9:.1f}** | | |") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Summarize the MuseBench shard index by folder.") + ap.add_argument("--root", type=Path, default=DEFAULT_ROOT) + ap.add_argument("--depth", type=int, default=2) + ap.add_argument("--json", action="store_true") + args = ap.parse_args(argv) + summary = summarize(iter_records(args.root / "index"), args.depth) + print(json.dumps(summary, indent=1) if args.json else to_markdown(summary)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_musebench_inventory.py b/tests/test_musebench_inventory.py new file mode 100644 index 0000000..d5e21fe --- /dev/null +++ b/tests/test_musebench_inventory.py @@ -0,0 +1,32 @@ +# Created: 2026-11-04 +# Purpose: Cover scripts/musebench/inventory.py grouping and Markdown rendering. +# Dependencies: pytest +# Test Status: passing +from scripts.musebench import inventory as inv + + +def rec(path, size, shard="part1_001.tar", typ="file"): + return {"path": path, "size": size, "shard": shard, "type": typ} + + +def test_summarize_groups_by_depth_and_skips_non_files(): + records = [ + rec("Jamendo/a.mp3", 10), + rec("Jamendo/b.mp3", 20, shard="part1_002.tar"), + rec("Musicgen_Generated_Music/Jamedo/a.mp3.wav", 5), + rec("Musicgen_Generated_Music/Genius-Lyrics/x.mp3.wav", 7), + rec("aac_generated_music/udio_generated_music/fake_1_udio_0.m4a", 3), + rec("Jamendo", 0, typ="dir"), + ] + out = inv.summarize(records, depth=2) + assert out["Jamendo"] == {"files": 2, "bytes": 30, "ext": {"mp3": 2}, + "shards": ["part1_001.tar", "part1_002.tar"]} + assert out["Musicgen_Generated_Music/Jamedo"]["ext"] == {"wav": 1} + assert "aac_generated_music/udio_generated_music" in out + assert sum(g["files"] for g in out.values()) == 5 + + +def test_markdown_has_total_row(): + md = inv.to_markdown(inv.summarize([rec("OOD FMA/1.mp3", 2_000_000_000)])) + assert "| `OOD FMA` | 1 | 2.0 | mp3 1 | 1 |" in md + assert md.splitlines()[-1].startswith("| **Total** | **1** | **2.0**") From 2c62e566da6a4eb6801aca9f2924bd591b5899a8 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 01:15:22 +0900 Subject: [PATCH 5/6] musebench: add sequential-stream shard reader (--mode stream) with tests Per-header range requests cost ~0.7 s each; shards with ~18k small members would take ~5.7 h. A single forward GET per shard discards member data locally and resumes with Range: bytes=- after a dropped connection. Assumption: single-stream throughput (~12 MB/s measured) makes this faster than range mode for member-dense shards. --- scripts/musebench/index_shards.py | 94 ++++++++++++++++++++++++- tests/test_musebench_index.py | 110 ++++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+), 1 deletion(-) diff --git a/scripts/musebench/index_shards.py b/scripts/musebench/index_shards.py index 83240fc..3960ca8 100644 --- a/scripts/musebench/index_shards.py +++ b/scripts/musebench/index_shards.py @@ -268,6 +268,93 @@ def __call__(self, offset: int, length: int) -> bytes: raise OSError(f"range {offset}+{length}: retries exhausted") +class StreamFetcher: + """Forward-only reader: one streaming GET from the resume offset, data skipped locally. + + For shards with many small members, one range request per header is latency-bound + (~0.7 s each); reading the shard sequentially and discarding member data is faster. + ``walk_tar`` only moves forward, so a small look-back buffer is enough. A broken stream + is reopened with ``Range: bytes=-``. + """ + + CHUNK = 1 << 20 + + def __init__(self, url: str, opener: Callable[[str, int], object] | None = None, + max_retries: int = 12): + self.url = url + self.opener = opener or self._open_http + self.max_retries = max_retries + self.stream = None + self.buf = b"" + self.buf_start = 0 # file offset of buf[0] + self.bytes_read = 0 + + @staticmethod + def _open_http(url: str, offset: int): + req = urllib.request.Request(url, headers={"Range": f"bytes={offset}-", + "User-Agent": USER_AGENT}) + resp = urllib.request.urlopen(req, timeout=120) + if resp.status != 206 and not (resp.status == 200 and offset == 0): + raise OSError(f"stream {url}: HTTP {resp.status}") + return resp + + def _reopen(self, offset: int) -> None: + if self.stream is not None: + try: + self.stream.close() + except OSError: + pass + self.stream = self.opener(self.url, offset) + self.buf, self.buf_start = b"", offset + + def _read_more(self) -> bool: + delay = 1.0 + for attempt in range(self.max_retries): + try: + if self.stream is None: + self._reopen(self.buf_start + len(self.buf)) + chunk = self.stream.read(self.CHUNK) + if chunk: + self.buf += chunk + self.bytes_read += len(chunk) + return True + return False # EOF + except (OSError, http.client.HTTPException): + if attempt == self.max_retries - 1: + raise + keep_start, keep = self.buf_start, self.buf + self.stream = None + time.sleep(min(delay, 120.0)) + delay *= 2 + try: + self._reopen(keep_start + len(keep)) + except (OSError, http.client.HTTPException): + self.stream = None + self.buf, self.buf_start = keep, keep_start + raise OSError(f"stream {self.url}: retries exhausted") + + def __call__(self, offset: int, length: int) -> bytes: + if self.stream is None and not self.buf: + self.buf_start = offset + if offset < self.buf_start: + raise ValueError(f"stream reader cannot seek back to {offset} (< {self.buf_start})") + end_have = self.buf_start + len(self.buf) + if offset > end_have + 64 * self.CHUNK: + # Far jump (resume or huge member): cheaper to reopen than to read through. + self._reopen(offset) + # Drop bytes before ``offset`` and read until the request is covered or EOF. + while True: + if offset > self.buf_start: + drop = min(offset - self.buf_start, len(self.buf)) + self.buf, self.buf_start = self.buf[drop:], self.buf_start + drop + if offset == self.buf_start and len(self.buf) >= length: + break + if not self._read_more(): + break + rel = offset - self.buf_start + return self.buf[rel:rel + length] if rel >= 0 else b"" + + # --------------------------------------------------------------------------- shard index def resolve_url(repo: str, revision: str, filename: str) -> str: @@ -363,6 +450,10 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--parts", type=int, nargs="+", default=list(PARTS)) ap.add_argument("--only", nargs="*", default=None, help="restrict to these shard stems") ap.add_argument("--workers", type=int, default=75, help="shards walked concurrently") + ap.add_argument("--mode", choices=["range", "stream"], default="range", + help="range: one range request per header (cheap on bandwidth); " + "stream: one sequential GET per shard, member data discarded " + "(faster for shards with many small members)") ap.add_argument("--refresh-list", action="store_true", help="re-query the shard list even if index/shards.json exists") args = ap.parse_args(argv) @@ -393,7 +484,8 @@ def progress(stem: str, n: int) -> None: def run(shard: dict) -> dict: url = resolve_url(shard["repo"], shard["revision"], shard["file"]) - return index_shard(shard, out_dir, RangeFetcher(url), progress) + fetcher = StreamFetcher(url) if args.mode == "stream" else RangeFetcher(url) + return index_shard(shard, out_dir, fetcher, progress) failures = 0 with ThreadPoolExecutor(max_workers=max(1, args.workers)) as pool: diff --git a/tests/test_musebench_index.py b/tests/test_musebench_index.py index f8cf348..05e97ff 100644 --- a/tests/test_musebench_index.py +++ b/tests/test_musebench_index.py @@ -206,3 +206,113 @@ def test_end_offset_beyond_listed_size_is_rejected(tmp_path): shard["size"] = 100 with pytest.raises(ix.TarFormatError, match="beyond"): ix.index_shard(shard, tmp_path, reader(blob)) + + +# ----------------------------------------------------------------------- StreamFetcher + +class FakeStream: + def __init__(self, blob, offset, fail_after=None): + self.data, self.pos, self.fail_after, self.reads = blob, offset, fail_after, 0 + + def read(self, n): + self.reads += 1 + if self.fail_after is not None and self.reads > self.fail_after: + raise ConnectionResetError("boom") + chunk = self.data[self.pos:self.pos + n] + self.pos += len(chunk) + return chunk + + def close(self): + pass + + +def stream_fetcher(blob, fail_first_streams=0, chunk=4096): + opened = [] + + def opener(url, offset): + stream = FakeStream(blob, offset, fail_after=2 if len(opened) < fail_first_streams else None) + opened.append(offset) + return stream + + f = ix.StreamFetcher("https://example.invalid/x.tar", opener=opener) + f.CHUNK = chunk + return f, opened + + +def big_blob(): + members = [(f"Jamendo/{i:03d}.mp3", bytes([i % 251]) * (1000 + 37 * i)) for i in range(40)] + members.insert(5, (LONG, b"L" * 70_000)) + return make_tar(members, tarfile.GNU_FORMAT) + + +def test_stream_fetcher_matches_range_reader(): + blob = big_blob() + fetcher, opened = stream_fetcher(blob) + recs, end = walk_all_with(fetcher) + check_against_tarfile(blob, recs) + assert recs == walk_all(blob)[0] and end == walk_all(blob)[1] + assert opened == [0] # one sequential GET for the whole shard + + +def walk_all_with(fetch, start=0): + gen = ix.walk_tar(fetch, start) + recs = [] + while True: + try: + recs.append(next(gen)) + except StopIteration as stop: + return recs, stop.value + + +def test_stream_fetcher_reopens_after_dropped_connection(monkeypatch): + blob = big_blob() + fetcher, opened = stream_fetcher(blob, fail_first_streams=2) + monkeypatch.setattr(ix.time, "sleep", lambda s: None) # no real backoff in tests + recs, _ = walk_all_with(fetcher) + check_against_tarfile(blob, recs) + assert len(opened) == 3 and opened[0] == 0 and opened[1] > 0 and opened[2] > opened[1] + + +def test_stream_fetcher_resumes_mid_archive_with_range_open(): + blob = big_blob() + recs_full, _ = walk_all(blob) + mid = recs_full[12] + start = ix.record_end(recs_full[11]) + assert mid["offset"] == start + fetcher, opened = stream_fetcher(blob) + recs, _ = walk_all_with(fetcher, start) + assert recs == recs_full[12:] + assert opened == [start] + + +def test_stream_fetcher_refuses_to_seek_back(): + blob = big_blob() + fetcher, _ = stream_fetcher(blob) + fetcher(10_000, 512) + with pytest.raises(ValueError, match="seek back"): + fetcher(0, 512) + + +def test_index_shard_with_stream_fetcher_resumes(tmp_path): + blob = big_blob() + shard = shard_for(blob) + fetcher, _ = stream_fetcher(blob) + n = [0] + real = fetcher.__call__ + + class Boom(Exception): + pass + + def flaky(offset, length): + n[0] += 1 + if n[0] == 9: + raise Boom + return real(offset, length) + + with pytest.raises(Boom): + ix.index_shard(shard, tmp_path, flaky) + fetcher2, opened2 = stream_fetcher(blob) + ix.index_shard(shard, tmp_path, fetcher2) + recs = [json.loads(l) for l in (tmp_path / "part9_001.jsonl").read_text().splitlines()] + check_against_tarfile(blob, recs) + assert opened2[0] > 0 From 1902b9013c73c660a69d534a19c82488a540e1b4 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Wed, 4 Nov 2026 03:47:58 +0900 Subject: [PATCH 6/6] docs: add measured MuseBench shard inventory; tighten index verifier All 75 shards indexed (351,188 files, 798.0 GB, 351,183 audio = paper total minus the 9,983 unshipped Genius-Lyrics tracks). Corrects the generator-count rows of the card-vs-paper table. Assumption: magenta_generated_music is the paper's Lyria RealTime (identical 19,903 count and hours); marked as inferred. --- docs/MUSEBENCH.md | 60 +++++++++++++++++++++++++++++++++---- scripts/musebench/verify.py | 6 ++-- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/docs/MUSEBENCH.md b/docs/MUSEBENCH.md index cd46286..9c8dc7a 100644 --- a/docs/MUSEBENCH.md +++ b/docs/MUSEBENCH.md @@ -51,20 +51,70 @@ The shard list is pinned to these commits: | `Anonymousv22222/MuseBench-part4` | `251fd5aeab23b9ec5bb6b7c2f1c8847c1d0bda57` | | `Anonymousv22222/MuseBench-part5` | `95dbba8b8e3b6e917c1119af8046d3ab8ded7189` | - +## Measured inventory (shard index, 2026-11-04) + +Built with `scripts/musebench/index_shards.py` and summarized with `scripts/musebench/inventory.py` +(`python scripts/musebench/inventory.py --depth 1`). The 75 shards hold 351,188 regular files, +798.0 GB: 351,183 audio files plus 5 CSV files (`Caption/captions-genius.csv`, +`Caption/captions-jamendo.csv`, two files under `Genius-Lyrics/`, and one CSV in `Jamendo/`). +351,183 is exactly the paper's 361,166 tracks minus the 9,983 Genius-Lyrics human tracks that +are not redistributed. + +| Folder | Files | GB | Extensions | Shards | +|---|---:|---:|---|---:| +| `Caption` | 2 | 0.0 | csv 2 | 2 | +| `Genius-Lyrics` | 2 | 0.1 | csv 2 | 2 | +| `Jamendo` | 9,921 | 54.9 | mp3 9920, csv 1 | 10 | +| `MagNet_Generated_Music` | 19,903 | 38.2 | wav 19903 | 5 | +| `Musicgen_Generated_Music` | 19,903 | 38.2 | wav 19903 | 10 | +| `OOD FMA` | 8,672 | 70.9 | mp3 8672 | 10 | +| `aac_generated_music` | 17,935 | 9.1 | m4a 17935 | 6 | +| `ai_human_co-created_music` | 9,887 | 26.2 | wav 9887 | 9 | +| `audioldm_generated_music` | 19,903 | 38.2 | wav 19903 | 10 | +| `eq_generated_music` | 17,935 | 48.0 | wav 17935 | 8 | +| `gaussion_noised_music` | 17,935 | 40.2 | wav 15935, mp3 2000 | 7 | +| `jasco_generated_music` | 19,903 | 12.7 | wav 19903 | 8 | +| `loudness_generation_music` | 17,935 | 48.0 | wav 17935 | 7 | +| `magenta_generated_music` | 19,903 | 114.6 | wav 19903 | 20 | +| `mp3_noised_music` | 17,935 | 11.0 | mp3 17935 | 5 | +| `mustango_generated_music` | 19,903 | 6.5 | wav 19903 | 5 | +| `ogg_noised_music` | 17,935 | 4.7 | ogg 17935 | 5 | +| `pgd_generated_music` | 17,935 | 15.5 | wav 15935, mp3 2000 | 5 | +| `pitch_time_generated_music` | 17,935 | 48.5 | wav 17935 | 10 | +| `riffusion_generated_music` | 19,903 | 53.9 | wav 19903 | 10 | +| `stable_audio_generated_music` | 19,903 | 105.3 | wav 19903 | 15 | +| `suno_generated_music` | 10,000 | 8.1 | mp3 10000 | 5 | +| `udio_generated_music` | 10,000 | 5.1 | mp3 10000 | 5 | +| **Total** | **351,188** | **798.0** | | | + +Observations that matter for acquisition: + +- Every AI generator folder holds 19,903 files (9,983 `Genius-Lyrics/` + 9,920 `Jamedo/`), except + Suno and Udio (10,000 each, `fake___.mp3`). +- The clean AI audio is large: `magenta` 114.6 GB, `stable_audio` 105.3 GB, `riffusion` 53.9 GB, + `MagNet`/`Musicgen`/`audioldm` 38.2 GB each, `jasco` 12.7 GB, `mustango` 6.5 GB, Suno 8.1 GB, + Udio 5.1 GB. Human audio: Jamendo 54.9 GB, OOD FMA 70.9 GB. Co-created: 26.2 GB. +- Each of the eight transform folders holds 17,935 files spread over the ten generator + sub-folders (about 2,000 per open generator, 1,000 for each of Suno and Udio, i.e. a test-split + sample). Their formats follow the transform: `aac_*` is `.m4a`, `ogg_*` is `.ogg`, `mp3_*` is + `.mp3`. In `gaussion_noised_music` and `pgd_generated_music` the Suno and Udio files are `.mp3` + (2,000 each) and the rest `.wav`. +- Member paths are unique across all 75 shards (checked by `verify.py index`); a path therefore + identifies a track without the shard name. + ## Dataset card vs paper | Topic | Dataset card | Paper | |---|---|---| -| Tenth generator | `magenta_generated_music/` ("Magenta", Apache 2.0) | Lyria RealTime via API | -| Paired generators | 8 (MusicGen, MAGNeT, AudioLDM, JASCO, Magenta, Mustango, Riffusion, Stable Audio) | 8 open + Lyria; AudioLDM is **AudioLDM2** | -| Jamendo human count | ~9,945 | 9,920 | +| Eighth paired generator | `magenta_generated_music/` ("Magenta", Apache 2.0) | Lyria RealTime via API; the folder holds 19,903 files, the same count and hours as the paper's Lyria row, so it is inferred to be that generator (neither source says so) | +| Paired generators | 8: MusicGen, MAGNeT, AudioLDM, JASCO, Magenta, Mustango, Riffusion, Stable Audio | 8: seven open-source (MusicGen, AudioLDM**2**, Riffusion, MAGNeT, JASCO, Mustango, Stable Audio Open) plus Lyria RealTime; with Suno and Udio, 10 generators in both | +| Jamendo human count | ~9,945 | 9,920 (the shards hold exactly 9,920 mp3 files) | | OOD FMA window | "2008–2010" in one place, "pre-2008" in the tree comment | "uploaded before 2010" | | Genius-Lyrics human audio | not shipped (copyrighted) | used in every split | | Suno / Udio origin | "closed-source commercial generators" | reused from SONICS | | Splits | none published | Default 70/20/10, LOMO; no split file, no seeds | -| Transforms | "every AI track" | test-set AI tracks only (Appendix A.3) | +| Transforms | "every AI track" | test-set AI tracks only (Appendix A.3); the shards hold 17,935 files per transform | Consequences: the Default and LOMO human negatives cannot be rebuilt (half of them, the vocal Genius-Lyrics side, is missing), and the paper's test split is unknown, so paper numbers for diff --git a/scripts/musebench/verify.py b/scripts/musebench/verify.py index 9151420..767cf52 100644 --- a/scripts/musebench/verify.py +++ b/scripts/musebench/verify.py @@ -29,9 +29,9 @@ DEFAULT_ROOT = Path(os.environ.get("MUSEBENCH_ROOT", "~/data/musebench")).expanduser() EXPECTED_PARTS = {1: 15, 2: 15, 3: 15, 4: 15, 5: 15} -# Lower bound on audio files in a complete index. The paper reports 361,166 tracks; the -# count actually found in the published shards is recorded in docs/MUSEBENCH.md. -MIN_AUDIO_FILES = 300_000 +# Lower bound on audio files in a complete index: the measured count is 351,183 (paper total +# 361,166 minus the 9,983 Genius-Lyrics human tracks that are not shipped); see docs/MUSEBENCH.md. +MIN_AUDIO_FILES = 351_000 AUDIO_EXT = (".wav", ".mp3", ".flac", ".ogg", ".m4a", ".aac", ".opus")