From 83dd161e7a2af6953e97c56548341ab6839def0c Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 28 Sep 2026 01:02:06 +0800 Subject: [PATCH 01/11] fix(unity): treat a MonoBehaviour with a missing script as classless script_class returned an error for a component whose m_Script is null; it now returns None, so prefabs that keep such a component export normally. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/nnnotes/unity.py | 19 ++++++++++++------- tests/test_speed_exact.py | 13 +++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/nnnotes/unity.py b/src/nnnotes/unity.py index 087b046..ca2c7f2 100644 --- a/src/nnnotes/unity.py +++ b/src/nnnotes/unity.py @@ -327,18 +327,23 @@ def _head_node(node: TypeTreeNode) -> TypeTreeNode | None: return head -def script_class(obj) -> str: - """Class name of a MonoBehaviour (needs the monoscript bundle loaded). Reads the header up to m_Script with the - object's own typetree (not its fields); the whole object when that is not possible.""" +def script_class(obj) -> str | None: + """Class name of a MonoBehaviour (needs the monoscript bundle loaded); None for a missing script (null m_Script). + Reads the header up to m_Script with the object's own typetree (not its fields); the whole object when that is + not possible.""" try: head = _head_node(obj._get_typetree_node()) if head is not None: - ms = deref(obj, obj.read_typetree(head, check_read=False)["m_Script"]) - if ms is not None: - return ms.read().m_ClassName + pptr = obj.read_typetree(head, check_read=False)["m_Script"] + if not pptr["m_PathID"]: + return None + return deref(obj, pptr).read().m_ClassName except Exception: pass - return obj.read().m_Script.read().m_ClassName + script = obj.read().m_Script + if not script.m_PathID: + return None + return script.read().m_ClassName def strip_pptrs(tt: dict) -> dict: diff --git a/tests/test_speed_exact.py b/tests/test_speed_exact.py index d42c40f..d96b33e 100644 --- a/tests/test_speed_exact.py +++ b/tests/test_speed_exact.py @@ -137,6 +137,19 @@ def test_head_node_cut_after_script(): assert unity._head_node(TypeTreeNode(0, "X", "Base", -1, 1, m_Children=kids[:2])) is None +def test_script_class_missing_script_is_none(): + kids = [TypeTreeNode(1, t, n, 0, 0) for t, n in (("PPtr", "m_GameObject"), ("UInt8", "m_Enabled"), + ("PPtr", "m_Script"))] + node = TypeTreeNode(0, "MonoBehaviour", "Base", -1, 1, m_Children=kids) + null = {"m_FileID": 0, "m_PathID": 0} + obj = SimpleNamespace(_get_typetree_node=lambda: node, + read_typetree=lambda *a, **k: {"m_GameObject": null, "m_Enabled": 1, "m_Script": null}) + assert unity.script_class(obj) is None + whole = SimpleNamespace(_get_typetree_node=lambda: None, + read=lambda: SimpleNamespace(m_Script=SimpleNamespace(m_PathID=0))) + assert unity.script_class(whole) is None + + # ---------------------------------------------------------------- typetree values def ref_value(ex, owner, v): if unity.is_pptr(v): From 8465dd5d89d3a1857d4dcb80b9f5ff8a36895473 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 28 Sep 2026 23:22:55 +0800 Subject: [PATCH 02/11] fix(room): write a valid glTF for a room without geometry A room with no nodes or buffers produced empty glTF arrays and an empty BIN chunk, which glTF validators reject. Empty arrays are now left out, and a room without geometry is a scene without nodes in a file without the BIN chunk. Rooms with geometry are written byte for byte as before. Co-Authored-By: Claude Opus 5.5 --- src/nnnotes/room.py | 14 ++++++++++---- tests/test_room.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/nnnotes/room.py b/src/nnnotes/room.py index c0f4421..7a4001f 100644 --- a/src/nnnotes/room.py +++ b/src/nnnotes/room.py @@ -258,23 +258,29 @@ def add_mesh(self, name, pos, nrm, uv, prims, active=True) -> None: "extras": {"unityActive": active}}) def write(self, path: Path, extras: dict): + """glTF requires the arrays it has to be non-empty and a buffer to hold bytes: an empty array is left out, + and a room without geometry is a scene without nodes and a file without the BIN chunk.""" + scene = {"nodes": list(range(len(self.nodes)))} if self.nodes else {} gltf = { "asset": {"version": "2.0", "generator": "nnnotes/room", "extras": extras}, - "scene": 0, "scenes": [{"nodes": list(range(len(self.nodes)))}], + "scene": 0, "scenes": [scene], "nodes": self.nodes, "meshes": self.meshes, "accessors": self.accessors, - "bufferViews": self.bufferViews, "buffers": [{"byteLength": len(self.bin)}], + "bufferViews": self.bufferViews, "buffers": [{"byteLength": len(self.bin)}] if self.bin else [], "materials": self.materials, "textures": self.textures, "images": self.images, "samplers": self.samplers, } + gltf = {k: v for k, v in gltf.items() if v != []} if self.ext_used: gltf["extensionsUsed"] = sorted(self.ext_used) js = dumps(gltf, separators=(",", ":")).encode("utf-8") js += b" " * (-len(js) % 4) self.bin.extend(b"\0" * (-len(self.bin) % 4)) + size = 12 + 8 + len(js) + (8 + len(self.bin) if self.bin else 0) with open(path, "wb") as f: - f.write(struct.pack(" Date: Mon, 28 Sep 2026 23:22:55 +0800 Subject: [PATCH 03/11] fix(story): share a models directory between concurrent story runs Several `nnnotes story` processes sharing one `--models` directory could remove a model another process had just installed, or read one while it was being replaced, and fail. A model is now installed in one rename; when another process installs the same model first, its copy is kept (exports are byte-identical) and this one is dropped. `--force` moves the previous directory aside before installing, and a reader that finds the directory gone in between exports the model itself. Co-Authored-By: Claude Opus 5.5 --- docs/commands.md | 4 ++- src/nnnotes/webmodel.py | 60 ++++++++++++++++++++++++++++++----------- tests/test_webmodel.py | 41 ++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 17 deletions(-) diff --git a/docs/commands.md b/docs/commands.md index c489b58..b552eb8 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -177,7 +177,9 @@ OUT/story.json index of the above (a file the episode does not need i story.json `models` maps the key of each model to its id (`` of `Character/Live2D///model/`) and `modelsDir` is the path from OUT to MODELS. A model directory that exists is used as it is, so the stories of one models directory export each model once; `--force` exports the episode's models again. The printed summary lists -the models exported (`modelsBuilt`) and those used as they were (`modelsSkipped`). +the models exported (`modelsBuilt`) and those used as they were (`modelsSkipped`). Several `story` processes may +share a models directory: a model appears in it in one rename, whole, and a model two processes export at once is +kept from the first to finish (the other's export, the same bytes, is dropped and counted in `modelsSkipped`). `--format` (default `flac`) is the audio format and `--flac-level` its FLAC compression level (as for `audio`); `--no-audio` leaves the cue sheets undecoded (no `audio/`, `audio` diff --git a/src/nnnotes/webmodel.py b/src/nnnotes/webmodel.py index 6bd83e6..c2922a4 100644 --- a/src/nnnotes/webmodel.py +++ b/src/nnnotes/webmodel.py @@ -321,7 +321,13 @@ def source_entry(address: str, doc: dict, where: str) -> dict: class ModelDir: """The models of `root`: each model in //, the files of a model manifest (read_files; the shader index reduced to the listed GLES3 programs as web.collect reduces it) in the layout export_model writes. ensure exports - a model whose directory does not exist, or every model once when `force`; `built` / `skipped`: the ids.""" + a model whose directory does not exist, or every model once when `force`; `built` / `skipped`: the ids (a model + another process installed meanwhile is skipped). + + Processes may share `root` (several `nnnotes story` at once): a model directory appears in one rename, whole, and + the first process to install a model wins; the others drop their export (the same bytes: exports are + deterministic) and use it. `force` moves a previous directory aside before installing; a process that finds the + directory gone in between exports the model itself.""" def __init__(self, cat, player, root: Path, force: bool = False): self.cat, self.player, self.root, self.force = cat, player, Path(root), force @@ -333,32 +339,42 @@ def ensure(self, address: str) -> dict: mid = model_id(address) if mid not in self._done: d = self.root / mid - if self.force or not d.exists(): - self._export(address, d) - self.built.append(mid) - else: - self.skipped.append(mid) index = d / MODEL_INDEX - if not index.is_file(): + built = (self.force or not d.exists()) and self._export(address, d) + if not index.is_file() and not d.exists(): # moved aside by a forced export of another process + built = self._export(address, d) + (self.built if built else self.skipped).append(mid) + try: + doc = json.loads(index.read_text(encoding="utf-8")) + except FileNotFoundError: raise RuntimeError(f"{d}: no {MODEL_INDEX} (not a model directory); remove it or export again " - f"(--force)") - self._done[mid] = source_entry(address, json.loads(index.read_text(encoding="utf-8")), str(index)) + f"(--force)") from None + self._done[mid] = source_entry(address, doc, str(index)) return self._done[mid] - def _export(self, address: str, d: Path) -> None: - """The model into a temporary directory next to `d`, then renamed to `d` (a previous `d` removed).""" + def _export(self, address: str, d: Path) -> bool: + """The model into a temporary directory in `root`, then renamed to `d`; False when another process's `d` is + there instead (kept: without `force` any `d`, with `force` one installed after the previous `d` was moved + aside).""" self.root.mkdir(parents=True, exist_ok=True) work = Path(tempfile.mkdtemp(prefix=f".{d.name}-", dir=self.root)) try: summary = export_model(self.cat, self.player, address, work / "export") text, binary = collect(work / "export", summary["files"]) files = {**{p: t.encode("utf-8") for p, t in text.items()}, **binary} + model = work / "model" for rel, data in sorted(files.items()): - (work / "model" / rel).parent.mkdir(parents=True, exist_ok=True) - (work / "model" / rel).write_bytes(data) - if d.exists(): - shutil.rmtree(d) - (work / "model").rename(d) + (model / rel).parent.mkdir(parents=True, exist_ok=True) + (model / rel).write_bytes(data) + if _install(model, d): + return True + if not self.force: + return False + try: + d.rename(work / "previous") + except FileNotFoundError: # moved aside by another process + pass + return _install(model, d) finally: shutil.rmtree(work, ignore_errors=True) @@ -367,6 +383,18 @@ def story_fields(self, story_dir: Path) -> dict: return {"modelsDir": Path(os.path.relpath(os.path.abspath(self.root), os.path.abspath(story_dir))).as_posix()} +def _install(src: Path, d: Path) -> bool: + """Rename the directory `src` to `d`; False when `d` exists (a non-empty directory: POSIX refuses to replace it, + Windows any directory).""" + try: + src.rename(d) + return True + except OSError: + if not d.exists(): + raise + return False + + class SiteModels: """The models of the site at `site`: ensure reads model.json from the model manifest models/.json (the models are built before the stories: storysite.build).""" diff --git a/tests/test_webmodel.py b/tests/test_webmodel.py index 2d2129d..1585a83 100644 --- a/tests/test_webmodel.py +++ b/tests/test_webmodel.py @@ -642,6 +642,47 @@ def failing(*a): webmodel.ModelDir(None, player, root).ensure(KEY_B) +@pytest.mark.parametrize("force", [False, True]) +def test_model_dir_shared_by_processes_keeps_one_whole_model(tmp_path, monkeypatch, force): + """Another process (another ModelDir on the same root) installs the model while this one exports it: without + force the first installed copy is kept and this export dropped, with force this export replaces it; either way + one whole model directory and no temporary directory is left.""" + player = fake_export(monkeypatch, motion_sync=True) + root, real, other = tmp_path / "live2d", webmodel.export_model, [] + + def racing(*a): + if not other: + other.append(webmodel.ModelDir(None, player, root, force=force)) + other[0].ensure(KEY_A) + return real(*a) + monkeypatch.setattr(webmodel, "export_model", racing) + a = webmodel.ModelDir(None, player, root, force=force) + assert a.ensure(KEY_A) == {"id": "adv_model_a", "motionSync": True} + assert (a.built, a.skipped) == ((["adv_model_a"], []) if force else ([], ["adv_model_a"])) + assert other[0].built == ["adv_model_a"] + assert [p.name for p in root.iterdir()] == ["adv_model_a"] + assert dir_files(root) == [f"adv_model_a/{f}" for f in model_files("adv_model_a")] + + +def test_model_dir_exports_a_model_moved_aside_after_the_check(tmp_path, monkeypatch): + """The model directory existed when checked but is gone when read (a forced export of another process moved it + aside): exported here.""" + player = fake_export(monkeypatch, motion_sync=True) + root = tmp_path / "live2d" + d, real, seen = root / "adv_model_a", webmodel.Path.exists, [] + + def exists(p): + if p == d and not seen: + seen.append(p) + return True + return real(p) + monkeypatch.setattr(webmodel.Path, "exists", exists) + a = webmodel.ModelDir(None, player, root) + assert a.ensure(KEY_A) == {"id": "adv_model_a", "motionSync": True} + assert seen and (a.built, a.skipped) == (["adv_model_a"], []) + assert dir_files(root) == [f"adv_model_a/{f}" for f in model_files("adv_model_a")] + + def test_story_command_exports_its_models_next_to_the_story(tmp_path, capsys, monkeypatch): from nnnotes import story player = fake_export(monkeypatch) From c4d67403b434a05f011a247a5c5bf48f52ebb9d1 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 28 Sep 2026 23:22:55 +0800 Subject: [PATCH 04/11] docs(readme): completion after verifying every story, spot and region Every story episode is exported and checked one by one, every spot and room background exports and validates as glTF, and full multi-region site builds are byte-identical across two runs. Co-Authored-By: Claude Opus 5.5 --- README.en.md | 8 ++++---- README.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.en.md b/README.en.md index 951b73b..6c47442 100644 --- a/README.en.md +++ b/README.en.md @@ -74,13 +74,13 @@ Based on all data of the Taiwan server, version 1.0.1 (zh-Hant): | catalog / bundle decryption / dependency closure | working | | master data decoding | working | | stories, `adv` | 946 / 946 episodes | -| stories, `story` (full directory) | 946 / 946 episodes have all their resources in the catalog and of supported kinds (the resource closure equals the game's own per-episode download list); 759 exported one by one, the other 187 (with frames, effects, post effects, stills and the like) not yet one by one | +| stories, `story` (full directory) | 946 / 946 episodes have all their resources in the catalog and of supported kinds (the resource closure equals the game's own per-episode download list); 946 / 946 exported and checked one by one: the files, models and audio the output refers to exist, every JSON parses and every path it refers to resolves; the 235 episodes with frames, effects, post effects, stills, chats, talk windows or videos are byte-identical across two exports except for the models directory path | | Live2D models | 239 / 239 (every model of the catalog; episodes use 185 of them) | | CRI audio | 681 / 681 cue sheets | | charts, `live` | 336 / 336 (music, difficulty) pairs | -| web site, `web` | 336 / 336 charts, 239 / 239 Live2D models, 946 / 946 story episodes (two full builds in English without audio are byte-identical and 946 / 946 pass the data validation; a 27-episode sample in five languages with AAC audio is likewise identical across two builds and valid) | -| spots, `spot` / `room` | one spot verified, the others not individually checked | -| other regions (en / kr) and languages | checked: the regions serve the same catalog for a language and the same bundles, and the keys are shared; the master tables the charts use are the same in the three regions, and the text tables have all five languages; the chart exports checked match the Taiwan server's. A full multi-region site build is not verified yet | +| web site, `web` | 336 / 336 charts, 239 / 239 Live2D models, 946 / 946 story episodes (two full builds in English without audio are byte-identical and 946 / 946 pass the data validation); two full builds of the three regions in five languages with AAC audio (`--all-regions`, with the live master data: 340 charts of 85 musics) are likewise byte-identical and pass the data validation in full | +| spots, `spot` / `room` | 39 / 39 spots, every reference resolves; `room` exports 20 / 20 room backgrounds of the catalog. Every glb passes the Khronos glTF-Validator with 0 errors | +| other regions (en / kr) and languages | checked: the regions serve the same catalog for a language and the same bundles, and the keys are shared; the master tables the charts use are the same in the three regions, and the text tables have all five languages; the chart exports checked match the Taiwan server's. A full multi-region site build is verified (the row above) | ## Requirements diff --git a/README.md b/README.md index 35beea1..afe18e6 100644 --- a/README.md +++ b/README.md @@ -59,13 +59,13 @@ nnnotes config check # 每项设置的来源和格式是否有效， | catalog / 资源包解密 / 依赖闭包 | 可用 | | masterdata 解码 | 可用 | | 剧情 `adv` | 946 / 946 集可导出 | -| 剧情 `story`（完整目录） | 946 / 946 集的资源都在 catalog 中且类型均受支持（资源闭包与游戏自带的每集下载清单一致）；已逐集导出验证 759 集，其余 187 集（用到 Frame / Effect / PostEffect / Still 等资源）未逐一验证 | +| 剧情 `story`（完整目录） | 946 / 946 集的资源都在 catalog 中且类型均受支持（资源闭包与游戏自带的每集下载清单一致）；946 / 946 集已逐集导出验证：输出中引用的文件、模型与音频均存在，JSON 均可解析，引用的路径均可解析；其中 235 集用到 Frame / Effect / PostEffect / Still / Chat / TalkWindow / 视频资源，两次导出除模型目录路径外逐字节一致 | | Live2D 模型 | 239 / 239 个可导出（catalog 中的全部模型，剧情用到其中 185 个） | | CRI 音频 | 681 / 681 个 cue sheet 可解码 | | 谱面 `live` | 336 / 336 个（曲目, 难度）组合可导出 | -| 网页站点 `web` | 336 / 336 张谱面，239 / 239 个 Live2D 模型，946 / 946 集剧情（英语、无音频的全量构建两次结果逐字节一致，946 / 946 集通过数据校验；五种语言含 AAC 音频的 27 集抽样同样一致，并通过数据校验） | -| 据点 `spot` / `room` | 已验证单个据点，其余未逐一验证 | -| 其他区服（en / kr）与其他语言 | 已核对：各区服同一语言的 catalog 相同、资源包相同，密钥通用；谱面用到的 masterdata 表在三个区服间一致，文本表五种语言齐全；抽查的谱面导出与台服一致。多区服整站构建未做全量验证 | +| 网页站点 `web` | 336 / 336 张谱面，239 / 239 个 Live2D 模型，946 / 946 集剧情（英语、无音频的全量构建两次结果逐字节一致，946 / 946 集通过数据校验）；三个区服、五种语言、含 AAC 音频的全量构建（`--all-regions`，用线上 masterdata：85 首曲目的 340 张谱面）两次结果同样逐字节一致，全部通过数据校验 | +| 据点 `spot` / `room` | 39 / 39 个据点可导出，引用均可解析；catalog 中 20 / 20 个房间背景可由 `room` 导出。生成的 glb 均通过 Khronos glTF-Validator，0 个错误 | +| 其他区服（en / kr）与其他语言 | 已核对：各区服同一语言的 catalog 相同、资源包相同，密钥通用；谱面用到的 masterdata 表在三个区服间一致，文本表五种语言齐全；抽查的谱面导出与台服一致。多区服整站已全量构建验证（见上一行） | ## 需要准备 From ae51eaa73e81c8285151b020504020e14443fced Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 28 Sep 2026 23:23:03 +0800 Subject: [PATCH 05/11] chore(release): 0.1.2 --- src/nnnotes/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nnnotes/__init__.py b/src/nnnotes/__init__.py index 60485bb..9bf2a64 100644 --- a/src/nnnotes/__init__.py +++ b/src/nnnotes/__init__.py @@ -4,4 +4,4 @@ models, spots, shaders, CRI audio and live charts as JSON and common file formats. Game files, keys and server addresses are supplied by the user through the configuration; nothing of the game is included. """ -__version__ = "0.1.1" +__version__ = "0.1.2" From f0038c15887b2ad8b61ddde320560837dbab1b5f Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 28 Sep 2026 23:28:57 +0800 Subject: [PATCH 06/11] ci: move the GitHub Actions to their current majors The actions ran on the deprecated Node.js 20 runtime: checkout v7, setup-python v7, setup-node v7, upload-artifact v7, download-artifact v8 and action-gh-release v3 run on Node.js 24. Artifacts are downloaded by name, which the path change of download-artifact v5 leaves as it was. commitlint runs on Node.js 22 (20 is past its end of life). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 8 ++++---- .github/workflows/commitlint.yml | 6 +++--- .github/workflows/release.yml | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad56323..92f4e16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,8 +20,8 @@ jobs: matrix: python-version: ["3.11", "3.12", "3.13"] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} cache: pip @@ -42,8 +42,8 @@ jobs: # versions installed an output change without a stage or converter version bump fails instead of being skipped. runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.13" cache: pip diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index 1bec0c5..bc489b4 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -13,12 +13,12 @@ jobs: commitlint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: - node-version: 20 + node-version: 22 - name: Install commitlint run: npm install --no-save --no-package-lock @commitlint/cli@19 @commitlint/config-conventional@19 - name: Pull request title diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3866318..6379d52 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,8 +11,8 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.13" - name: Check the tag against the package version @@ -27,7 +27,7 @@ jobs: python -m pip install --upgrade pip build twine python -m build python -m twine check --strict dist/* - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: dist path: dist/ @@ -42,7 +42,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: name: dist path: dist/ @@ -57,7 +57,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: Release notes @@ -68,12 +68,12 @@ jobs: env: OUTPUT: RELEASE_NOTES.md GITHUB_REPO: ${{ github.repository }} - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: name: dist path: dist/ - name: GitHub release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: body_path: RELEASE_NOTES.md files: | From 48b6557490f638f2b37b73dabb8fa175f0cce788 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 28 Sep 2026 23:32:47 +0800 Subject: [PATCH 07/11] ci: run commitlint on Node.js 24, the active LTS Co-Authored-By: Claude Opus 5.5 --- .github/workflows/commitlint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index bc489b4..8bdfb96 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -18,7 +18,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-node@v7 with: - node-version: 22 + node-version: 24 - name: Install commitlint run: npm install --no-save --no-package-lock @commitlint/cli@19 @commitlint/config-conventional@19 - name: Pull request title From 3dae671302ca843c2ce573802e1309b4b85893a2 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Tue, 29 Sep 2026 01:29:30 +0800 Subject: [PATCH 08/11] feat(songs): export every song's metadata and chart facts `nnnotes songs` writes one canonical JSON file (nnnotes.songs/1) with every live song of a master data version: titles, readings and credits in the five text languages, bands, vocal characters, category, tags, release time, the BGM cue and its length read from the ACB, and per difficulty the level, note counts by operate type, BPM, note times, music length, skill events and fevers. It joins a deck data file of the same version by score id. `--jackets DIR` also writes every song's jacket as WebP. Co-Authored-By: Claude Opus 5.5 --- README.en.md | 1 + README.md | 1 + docs/commands.md | 20 +++ docs/songs.md | 98 ++++++++++++ src/nnnotes/cli.py | 37 +++++ src/nnnotes/deckdata.py | 8 +- src/nnnotes/songs.py | 338 ++++++++++++++++++++++++++++++++++++++++ tests/test_songs.py | 218 ++++++++++++++++++++++++++ 8 files changed, 717 insertions(+), 4 deletions(-) create mode 100644 docs/songs.md create mode 100644 src/nnnotes/songs.py create mode 100644 tests/test_songs.py diff --git a/README.en.md b/README.en.md index 6c47442..b7cc40a 100644 --- a/README.en.md +++ b/README.en.md @@ -50,6 +50,7 @@ nnnotes config check # each setting's origin and whether it is valid | `live` | music ID + difficulty | a full chart directory: chart and runtime notes, 3D scene, note and effect assets, BGM and sounds, sound routing | | `web` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; the Live2D models of the stories are built first as models, listed in `models.json`, and the story manifests reference them; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`); compressible assets (JSON, shaders, moc3, ...) are stored gzip encoded by default (`--compress br` / `none`) | | `deck-data` | master data files (`--master-files` directory or `--apk-master`) | one JSON file for deck-building tools: every chart's runtime notes, skill events and fever ranges, and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/deck-data.md)) | +| `songs` | master data files (`--master-files` directory or `--apk-master`) | one JSON file with every song's metadata: titles and credits in five languages, bands, vocal characters, category, tags, release time, BGM length, and per difficulty the level, note counts, BPM, chart times, skill events and fever ranges ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/songs.md)) | Export conventions: diff --git a/README.md b/README.md index afe18e6..b78ba09 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ nnnotes config check # 每项设置的来源和格式是否有效， | `live` | 曲目 ID + 难度 | 完整谱面目录：谱面与运行时音符、3D 场景、音符与特效资源、BGM 与音效、声音路由 | | `web` | `--pair 曲目:难度`（可重复）或 `--all`；`--live2d 模型`（可重复）或 `--all-live2d`；`--story 剧情 ID`（可重复）或 `--all-stories`；`--region 区服`（可重复）或 `--all-regions` | ournotes-player 静态站点：共享播放器 + 每谱 / 每模型 / 每集剧情清单 + 内容寻址资源；剧情用到的 Live2D 模型先按模型构建并列入 `models.json`，剧情清单引用它们；一个站点可服务多个区服，列表文本含五种语言；剧情的界面文字按语言分组，字形由开源字体生成 TextMeshPro 字体资源（`--fonts game` 时用游戏字体）；可压缩的资源（JSON、着色器、moc3 等）默认以 gzip 存储，`--compress br` / `none` 可改 | | `deck-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`） | 供组卡工具读取的单个 JSON：每张谱面的运行时音符、技能事件与 fever 区间，以及卡牌、技能、加成、分数与活动相关的 masterdata 表（[格式](docs/deck-data.md)） | +| `songs` | masterdata 文件（`--master-files` 目录或 `--apk-master`） | 全部歌曲元数据的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、BGM 时长，以及每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间（[格式](docs/songs.md)） | 导出约定： diff --git a/docs/commands.md b/docs/commands.md index b552eb8..4860bb8 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -683,3 +683,23 @@ same inputs give the same bytes. Prints `{out, format, region, masterSource, mas notes, bytes, fileBytes, sha256}`. A missing or mismatching input (a master data file, a column, a chart asset) stops the command with exit status 1 before the file is written. The format is described in [deck-data.md](deck-data.md). + +## songs + +``` +nnnotes songs (--master-files DIR | --apk-master) [--no-bgm] [--jackets DIR] -o FILE +``` + +Writes one JSON file with every `MasterLiveMusic` song's metadata for one master data version: titles, readings and +credits in the five text languages, bands (and a song's own band name), vocal characters, category, tags, release +time, jacket, Gekisou missions, the whole `MasterLiveMusic` row, the live BGM's cue and length (read from the cue +sheet's ACB, without decoding audio), and per difficulty the chart facts a song listing shows: level and display +level, full combo count, note counts (judged, total, per operate type), BPM (main, min, max and every change), the +first and last note times, the live's music length (last note + 1000 ms), skill event times and fever ranges. The +master data is read as for `deck-data` (same options, same checks), so the two files of one master data version join +by `scoreId`. `--no-bgm` skips the cue sheets (every `bgm.length` is null). `--jackets DIR` also writes every song's +jacket as `DIR/.webp` (at most 320 px on the longer side). `FILE` ending in `.gz` is written gzip-compressed; +the file is canonical. Prints `{out, format, region, masterSource, masterVersion, songs, charts, bgm, jackets, bytes, +fileBytes, sha256}`. A missing or unreadable input (a master data file, a text id, a chart asset, a cue sheet or cue, a +jacket) stops the command with exit status 1 before the file is written. The format is described in +[songs.md](songs.md). diff --git a/docs/songs.md b/docs/songs.md new file mode 100644 index 0000000..ea2456a --- /dev/null +++ b/docs/songs.md @@ -0,0 +1,98 @@ +# Songs + +`nnnotes songs` writes one JSON file with every live song's metadata for one master data version: what a song +listing shows (titles and credits in every language, bands, vocal characters, category, tags, release time, jacket, +BGM length) and, per difficulty, the chart facts (level, note counts, BPM, chart times, skill events, fevers). The +format is `nnnotes.songs/1`. + +``` +nnnotes songs (--master-files DIR | --apk-master) [--no-bgm] [--jackets DIR] -o FILE +``` + +The master data is read as for [deck data](deck-data.md): the files as served, each checked against the manifest's +SHA-256. A songs file and a deck data file made from the same master data version join by `scoreId` +(`MasterLiveMusicScore._id`). Charts are read from the catalog of `[catalog] language`; the BGM length from the cue +sheet's ACB (its `CueTable` and `WaveformTable`, no audio is decoded). `--no-bgm` does not read the cue sheets. +`--jackets DIR` also writes every song's jacket, the Texture2D `Image/Jacket/`, as `DIR/.webp` +(WebP quality 88, scaled down with Lanczos to at most 320 pixels on the longer side, without alpha when opaque); a +page next to the songs file finds a song's jacket at `/.webp`. + +The command writes the file only when every table, chart and cue sheet was read: a missing master data file, a text +id that `MasterText` does not have, a score id that `MasterLiveMusicScore` does not have, a missing or unreadable +chart asset, a cue sheet without the song's cue, or (with `--jackets`) a missing jacket texture stops it with exit status 1 and a line naming the input. + +## Layout + +```json +{ + "format": "nnnotes.songs/1", + "provenance": {"region": ..., "client": {...}, "catalog": {...}, "master": {...}, "exporter": {...}}, + "languages": ["ja", "en", "zh-Hant", "zh-Hans", "ko"], + "bands": [...], "characters": [...], "tags": [...], "categories": [...], + "songs": [{"id": 100001, ..., "charts": [...]}, ...] +} +``` + +The file is minified UTF-8 with one line feed at the end, keys in the order shown, and canonical in the same way as +deck data (same inputs, same nnnotes version: same bytes; floats are the shortest decimal of their binary32 value). +`provenance` is the deck data provenance, with `master.tables` listing the tables this file reads: `MasterLiveMusic`, +`MasterLiveMusicScore`, `MasterText`, `MasterBand`, `MasterCharacter`, `MasterTag`, `MasterLiveMusicCategory`, +`MasterSound`, `MasterSoundCueSheet`. + +A **text** is an object with one string per language of `languages` (`{"ja": ..., "en": ..., "zh-Hant": ..., +"zh-Hans": ..., "ko": ...}`), the `MasterText` row of the id; a text field is null when the master data gives no id +(an empty string). A language's string may be empty when the game has no text in that language. + +### bands, characters, tags, categories + +| Field | Content | +|---|---| +| `bands[].id`, `.name`, `.mainColor`, `.subColor` | `MasterBand`: id, name text, color codes | +| `characters[].id`, `.bandId`, `.name`, `.shortName`, `.mainColor` | `MasterCharacter` | +| `tags[].id`, `.name` | `MasterTag` (the ids of `bestMusicTagIds`) | +| `categories[].id`, `.musicCategories`, `.name` | `MasterLiveMusicCategory` (the listing's category tabs; `musicCategories` are the song category values it shows) | + +### songs + +One entry per `MasterLiveMusic` row, sorted by `id`. + +| Field | Content | +|---|---| +| `id`, `sortOrder`, `startAt`, `defaultUnlock` | the row's `_id`, `_sortOrder`, `_startAt` (as served, server time), `_defaultUnlock` | +| `title`, `ruby`, `phonetic` | texts of `_titleTextID`, `_rubyTitleTextID`, `_phoneticTextID` | +| `bandIds`, `bandName` | `_bandIDs`; the text of `_bandNameTextID`, a name the song shows instead of its first band's (null: none) | +| `vocalCharacterIds` | `_vocalCharacterIDs` | +| `lyricist`, `composer`, `arranger` | texts of `_lyricistTextID`, `_composerTextID`, `_arrangerTextID` | +| `musicType`, `musicCategories`, `bestMusicTagIds` | `_musicType` (the song type that card type bonuses match), `_musicCategories`, `_bestMusicTagIDs` | +| `jacket` | `_jacketAssetName` (the jacket sprite `Image/Jacket/`) | +| `gekisouMissions` | `[_gekisouMission1, _gekisouMission2, _gekisouMission3]` | +| `bgm.soundId`, `.cueSheet`, `.cue` | `_musicSoundID` and its `MasterSound` / `MasterSoundCueSheet` cue | +| `bgm.length` | `{lengthMs, samples, sampleRate, durationMs}`: the cue's `Length` in the ACB `CueTable`, the sample count and rate of its first waveform, and `samples * 1000 // sampleRate`; null with `--no-bgm` | +| `charts` | one entry per difficulty the song has (`_easyID` .. `_expertID` not 0), in the order easy, normal, hard, expert | +| `master.MasterLiveMusic` | the whole `MasterLiveMusic` row as decoded | + +### charts + +| Field | Content | +|---|---| +| `difficulty` | `easy`, `normal`, `hard` or `expert` | +| `scoreId`, `level`, `displayLevel`, `fullComboCount` | `MasterLiveMusicScore`: `_id`, `_musicScoreLevel`, `_musicScoreDisplayLevel`, `_fullComboCount` | +| `asset.key`, `asset.sha256` | the chart TextAsset `Live/MusicScore/<_musicScoreTextFileName>` and the SHA-256 of its bytes as shipped | +| `notes.judged` | notes that are judged (and count for a full combo) | +| `notes.total` | every runtime note, including hidden notes, guide notes and slide combo ticks | +| `notes.byOperateType` | `{"": count}` over every runtime note, keys in ascending order | +| `bpm.changes` | every BPM change `{timeMs, bpm}` in time order | +| `bpm.main`, `.min`, `.max` | over the played span (first to last judged note): the BPM that holds longest (the earliest on a tie), the lowest and the highest | +| `firstNoteMs`, `lastJudgedNoteMs` | times of the first and the last judged note | +| `lastNoteMs` | the latest time of any runtime note (the time the score code calls the last timing note) | +| `musicLengthMs` | `lastNoteMs + 1000`: the live's music length on the game's score path (skill effects end at it at the latest) | +| `skillEventsMs` | skill event times in chart order; event `i` fires the skill of the member at performance position `i` | +| `fevers` | fever ranges `[startMs, endMs]` sorted by start | + +Chart times are milliseconds of chart time. The two lengths are different facts: `bgm.length` is how long the music +plays, `musicLengthMs` is the length the score code uses; a listing chooses the one it needs. + +## Versions + +As for deck data: within `nnnotes.songs/1` fields are only added, never renamed, removed or given another meaning; +readers ignore keys they do not know and reject a major version they do not know. diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index 4056856..d9bb7db 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -37,6 +37,7 @@ [--font emoji=] [--region [--region ...] | --all-regions] nnnotes deck-data --master-files | --apk-master -o out/deck-data.json[.gz] + nnnotes songs --master-files | --apk-master [--no-bgm] [--jackets DIR] -o out/songs.json[.gz] nnnotes export -o out/assets [--select group: | key: | bundle: ...] [--layout original,cas] nnnotes plan [--select ...] [--json] [--check] [--emit-tasks ] nnnotes run-stage [...] @@ -656,6 +657,28 @@ def cmd_deck_data(args, cfg): _print_json(r) +def cmd_songs(args, cfg): + from . import deckdata, songs + if args.apk_master: + cfg.require_path("paths", "apk") # the master data files ship in the APK + apk = _existing(cfg, "paths", "apk") + try: + if args.apk_master: + src, region = deckdata.apk_master(apk), deckdata.EMBEDDED + else: + src, region = deckdata.master_files(Path(args.master_files)), cfg.region() + key = master_key(cfg) + cat = open_catalog(cfg) + r = songs.export(Path(args.out), src, key, deckdata.catalog_fetch(cat), + None if args.no_bgm else songs.catalog_bgm(cat), region=region, + client=deckdata.apk_client(apk) if apk is not None else {}, + catalog=deckdata.catalog_info(cat, cli_assets.store_root(args, cfg)), + jacket=songs.catalog_jacket(cat) if args.jackets else None, jackets_dir=args.jackets) + except (deckdata.DeckDataError, songs.SongsError) as e: + sys.exit(f"nnnotes: {e}") + _print_json(r) + + # ---------------------------------------------------------------- parser def parse_pair(s: str) -> tuple[int, str]: m = re.fullmatch(r"(\d+)[:_](easy|normal|hard|expert)", s) @@ -930,6 +953,20 @@ def target(m, what): _out(c, "output file (.json, or .json.gz for gzip)") c.set_defaults(func=cmd_deck_data, usage=c.error) + c = sub.add_parser("songs", help="every live song's metadata (titles, credits, bands, BGM length) and chart facts " + "(levels, note counts, BPM, times) -> one JSON file") + g = c.add_mutually_exclusive_group(required=True) + g.add_argument("--master-files", metavar="DIR", + help="master data files as served: MasterManifest.json and the .bin files it lists " + "(`master download`)") + g.add_argument("--apk-master", action="store_true", help="the master data files of base.apk ([paths] apk)") + c.add_argument("--no-bgm", action="store_true", + help="do not read the BGM cue sheets (every song's bgm.length is null)") + c.add_argument("--jackets", metavar="DIR", + help="also write every song's jacket as DIR/.webp (at most 320 px on the longer side)") + _out(c, "output file (.json, or .json.gz for gzip)") + c.set_defaults(func=cmd_songs, usage=c.error) + cli_assets.register(sub, argparse.Namespace(open_catalog=open_catalog, print_json=_print_json)) voices.register(sub, argparse.Namespace(open_catalog=open_catalog, master_dir=master_dir)) return p diff --git a/src/nnnotes/deckdata.py b/src/nnnotes/deckdata.py index 447364b..462b065 100644 --- a/src/nnnotes/deckdata.py +++ b/src/nnnotes/deckdata.py @@ -183,11 +183,11 @@ def read(names): return MasterSource(EMBEDDED, f"{apk} {APK_MASTER}", version, hashes, read) -def read_master(src: MasterSource, key) -> tuple[dict[str, list[dict]], dict[str, str]]: - """The rows (`_allData`) of every table of TABLES and the SHA-256 of each file as served. `key`: a - master.MasterKey.""" +def read_master(src: MasterSource, key, tables=None) -> tuple[dict[str, list[dict]], dict[str, str]]: + """The rows (`_allData`) of every table of `tables` (default: TABLES) and the SHA-256 of each file as served. + `key`: a master.MasterKey.""" from . import master - names = {t: f"{t}.bin" for t, _ in TABLES} + names = {t: f"{t}.bin" for t in (tables if tables is not None else (t for t, _ in TABLES))} unlisted = [t for t, n in names.items() if n not in src.hashes] if unlisted: raise DeckDataError(f"master data {src.where}: {MANIFEST} lists no {', '.join(unlisted)}") diff --git a/src/nnnotes/songs.py b/src/nnnotes/songs.py new file mode 100644 index 0000000..40f373a --- /dev/null +++ b/src/nnnotes/songs.py @@ -0,0 +1,338 @@ +"""Songs: one JSON file with every live song's metadata for one master data version (format `nnnotes.songs/1`, +docs/songs.md): titles and credits in every language, bands, vocal characters, category, tags, release time, the +live BGM's length, and per difficulty the chart facts a song listing shows (level, note counts, BPM, chart times, +skill events, fever ranges). + +Master data is read from the files as served, like deck data (deckdata.master_files / apk_master: SHA-256 checked +against the manifest, decoded with master.decode), so a songs file and a deck data file of the same master data +version join by `scoreId`. Charts are the TextAssets `Live/MusicScore/` converted by score.runtime_score; +the BGM length is read from the cue sheet's ACB (cue `Length` and the stream's sample count), without decoding +audio. With a jackets directory, every song's jacket (the Texture2D `Image/Jacket/`) is written there as +`.webp`, scaled to at most JACKET_SIZE pixels on its longer side. + +The output is canonical in the same way as deck data (deckdata.encode): minified UTF-8 with one trailing LF, keys +in a fixed order, songs sorted by id, floats as the shortest decimal of their binary32 value. The file is written +only when every table, chart and cue sheet was read; any missing or unreadable input is a SongsError naming it. +""" +from __future__ import annotations + +import hashlib +import io +from collections import Counter +from pathlib import Path +from typing import Callable + +from . import deckdata +from .languages import LANGUAGES + +FORMAT = "nnnotes.songs/1" +DIFFICULTIES = ("easy", "normal", "hard", "expert") +MUSIC_LENGTH_TAIL_MS = 1000 # the live's music length: the last note time + 1000 ms (LiveScore skip path) + +TABLES = ("MasterLiveMusic", "MasterLiveMusicScore", "MasterText", "MasterBand", "MasterCharacter", "MasterTag", + "MasterLiveMusicCategory", "MasterSound", "MasterSoundCueSheet") + + +class SongsError(ValueError): + """An input the songs file cannot be made from (the message names it).""" + + +# ---------------------------------------------------------------- texts +class Texts: + """MasterText rows by id -> {language: text} in every language (languages.LANGUAGES).""" + + def __init__(self, rows: list[dict]): + self.rows = {r.get("_id"): r for r in rows} + + def get(self, text_id) -> dict | None: + """{language: text} of a text id; None for an empty id; a SongsError for an id MasterText does not have.""" + if not text_id: + return None + r = self.rows.get(text_id) + if r is None: + raise SongsError(f"MasterText has no text {text_id!r}") + return {code: r.get(col) for code, (_, col) in LANGUAGES.items()} + + +# ---------------------------------------------------------------- charts +def _f(v) -> float: + return float(v) + + +def bpm_facts(bpm_events: list, first_ms: int, last_ms: int) -> dict: + """{main, min, max, changes} of a chart's BPM changes [(bpm, Pos)] (tick order). `changes` lists every change + as {timeMs, bpm}; main / min / max are taken over the played span [first_ms, last_ms] (the first and the last + judged note): main is the BPM that holds longest in the span (the earliest on a tie), min / max the lowest and + the highest that hold in it. A span of one instant takes the BPM at that time.""" + changes = sorted(((int(p.ms), _f(b)) for b, p in bpm_events), key=lambda x: x[0]) + if not changes: + raise SongsError("no BPM change") + held: dict[float, int] = {} + order: list[float] = [] + for i, (t, b) in enumerate(changes): + end = changes[i + 1][0] if i + 1 < len(changes) else None + lo = max(t, first_ms) + hi = last_ms if end is None else min(end, last_ms) + at_span = (end is None or end > first_ms) and t <= last_ms + if not at_span: + continue + if b not in held: + order.append(b) + held[b] = 0 + held[b] += max(0, hi - lo) + if not order: # every change after the span: the first one holds before it + order, held = [changes[0][1]], {changes[0][1]: 0} + main = max(order, key=lambda b: (held[b], -order.index(b))) + return {"main": main, "min": min(order), "max": max(order), + "changes": [{"timeMs": t, "bpm": b} for t, b in changes]} + + +def chart_facts(score_row: dict, key: str, raw: bytes) -> dict: + """The facts of one chart: level, note counts, BPM, chart times, skill events and fevers.""" + from . import score + try: + root = score.load_bytes(raw) + except (ValueError, OSError, EOFError): + raise SongsError(f"chart {key}: not a chart (gzip or JSON cannot be read)") from None + try: + rs = score.runtime_score(root) + except Exception as e: # the converter's own errors and malformed chart fields + raise SongsError(f"chart {key}: cannot be converted ({type(e).__name__}: {e})") from None + notes = rs.notes + if not notes: + raise SongsError(f"chart {key}: no notes") + judged = sorted(int(n.pos.ms) for n in notes if score.is_judgement_note(n.op)) + if not judged: + raise SongsError(f"chart {key}: no judged note") + last = max(int(n.pos.ms) for n in notes) + by_type = Counter(int(n.op) for n in notes) + try: + bpm = bpm_facts(rs.bpm_events, judged[0], judged[-1]) + except SongsError as e: + raise SongsError(f"chart {key}: {e}") from None + return { + "scoreId": score_row["_id"], + "level": score_row["_musicScoreLevel"], + "displayLevel": score_row.get("_musicScoreDisplayLevel"), + "fullComboCount": score_row["_fullComboCount"], + "asset": {"key": key, "sha256": hashlib.sha256(raw).hexdigest()}, + "notes": {"judged": len(judged), "total": len(notes), + "byOperateType": {str(op): by_type[op] for op in sorted(by_type)}}, + "bpm": bpm, + "firstNoteMs": judged[0], + "lastJudgedNoteMs": judged[-1], + "lastNoteMs": last, + "musicLengthMs": last + MUSIC_LENGTH_TAIL_MS, + "skillEventsMs": [int(p.ms) for _, p in rs.skills], + "fevers": [[int(a.ms), int(b.ms)] for _, a, b in rs.fevers], + } + + +# ---------------------------------------------------------------- BGM +def cue_length(acb: bytes, cue: str, where: str) -> dict: + """{lengthMs, samples, sampleRate, durationMs} of a cue of an ACB: the CueTable `Length` and the first stream's + sample count and rate (durationMs = samples * 1000 // sampleRate).""" + from . import acb as acbmod + try: + cues = acbmod.cue_streams(acb) + except Exception as e: + raise SongsError(f"{where}: the ACB cannot be read ({type(e).__name__}: {e})") from None + c = cues.get(cue) + if c is None: + raise SongsError(f"{where}: no cue {cue!r}") + s = (c.get("streams") or [None])[0] or {} + samples, rate = s.get("samples"), s.get("sampleRate") + return {"lengthMs": c.get("lengthMs"), "samples": samples, "sampleRate": rate, + "durationMs": samples * 1000 // rate if isinstance(samples, int) and isinstance(rate, int) and rate + else None} + + +def catalog_bgm(cat) -> Callable[[str, str], dict]: + """bgm(cue sheet, cue) for a catalog: the cue sheet's ACB (cri.acb_data) -> cue_length.""" + from . import cri + + def bgm(sheet: str, cue: str) -> dict: + try: + files, _ = cri.acb_data(cat, sheet) + except KeyError: + raise SongsError(f"cue sheet {sheet}: no such asset") from None + except Exception as e: + raise SongsError(f"cue sheet {sheet}: cannot be read ({type(e).__name__}: {e})") from None + return cue_length(files["acb"], cue, f"cue sheet {sheet}") + return bgm + + +# ---------------------------------------------------------------- document +JACKET_KEY = "Image/Jacket/{jacket}" +JACKET_SIZE = 320 # longer side of a written jacket, pixels +JACKET_QUALITY = 88 # WebP quality + + +def jacket_webp(image, size: int = JACKET_SIZE) -> bytes: + """WebP bytes of a jacket image (a Pillow image), scaled down (Lanczos) to at most `size` pixels on its longer + side; opaque images are written without alpha.""" + from PIL import Image + img = image.convert("RGBA") + if img.getextrema()[3][0] == 255: + img = img.convert("RGB") + w, h = img.size + if max(w, h) > size: + img = img.resize((max(1, round(w * size / max(w, h))), max(1, round(h * size / max(w, h)))), + Image.LANCZOS) + buf = io.BytesIO() + img.save(buf, format="WEBP", quality=JACKET_QUALITY, method=6) + return buf.getvalue() + + +def catalog_jacket(cat, size: int = JACKET_SIZE) -> Callable[[str], bytes]: + """jacket(name) for a catalog: the WebP bytes (jacket_webp) of the Texture2D `Image/Jacket/`.""" + import tempfile + from .export import Exporter + ex = Exporter(cat, Path(tempfile.gettempdir()), textures="deferred") + + def jacket(name: str) -> bytes: + key = JACKET_KEY.format(jacket=name) + try: + o = ex.key_object(key) + except KeyError: + raise SongsError(f"jacket {name}: no asset {key}") from None + except Exception as e: + raise SongsError(f"jacket {name}: {key} cannot be read ({type(e).__name__}: {e})") from None + if o is None or o.type.name != "Texture2D": + raise SongsError(f"jacket {name}: {key} is not a Texture2D") + return jacket_webp(o.read().image, size) + return jacket + + +def _by_id(rows: list[dict], table: str) -> dict: + out = {} + for r in rows: + if r.get("_id") in out: + raise SongsError(f"{table}: _id {r.get('_id')} occurs twice") + out[r.get("_id")] = r + return out + + +def build(tables: dict[str, list[dict]], table_sha: dict[str, str], fetch: Callable[[str], bytes], + bgm: Callable[[str, str], dict] | None, *, region: str, client: dict, catalog: dict, master_source: str, + master_version: str | None) -> dict: + """The songs document. `fetch(file name)`: a chart TextAsset's bytes (KeyError when there is none); `bgm(cue + sheet, cue)`: the BGM length (catalog_bgm), None to leave every song's `bgm.length` null.""" + from . import __version__ + text = Texts(tables["MasterText"]) + scores = _by_id(tables["MasterLiveMusicScore"], "MasterLiveMusicScore") + sounds = _by_id(tables["MasterSound"], "MasterSound") + sheets = _by_id(tables["MasterSoundCueSheet"], "MasterSoundCueSheet") + musics = sorted(_by_id(tables["MasterLiveMusic"], "MasterLiveMusic").values(), key=lambda r: r["_id"]) + + bands = [{"id": b["_id"], "name": text.get(b.get("_nameTextID")), "mainColor": b.get("_mainColorCode"), + "subColor": b.get("_subColorCode")} for b in sorted(tables["MasterBand"], key=lambda r: r["_id"])] + characters = [{"id": c["_id"], "bandId": c.get("_bandID"), "name": text.get(c.get("_nameTextID")), + "shortName": text.get(c.get("_shortNameTextID")), "mainColor": c.get("_mainColorCode")} + for c in sorted(tables["MasterCharacter"], key=lambda r: r["_id"])] + tags = [{"id": t["_id"], "name": text.get(t.get("_nameTextID"))} + for t in sorted(tables["MasterTag"], key=lambda r: r["_id"])] + categories = [{"id": c["_id"], "musicCategories": list(c.get("_musicCategories") or []), + "name": text.get(c.get("_textKey"))} + for c in sorted(tables["MasterLiveMusicCategory"], key=lambda r: r["_id"])] + + songs = [] + for m in musics: + where = f"MasterLiveMusic {m['_id']}" + charts = [] + for d in DIFFICULTIES: + sid = m.get(f"_{d}ID") + if not sid: + continue + row = scores.get(sid) + if row is None: + raise SongsError(f"{where}: {d} score {sid} is not in MasterLiveMusicScore") + key = deckdata.chart_key(row["_musicScoreTextFileName"]) + try: + raw = fetch(row["_musicScoreTextFileName"]) + except KeyError: + raise SongsError(f"chart {key} (MasterLiveMusicScore {sid}): no such asset") from None + charts.append({"difficulty": d, **chart_facts(row, key, raw)}) + snd = sounds.get(m.get("_musicSoundID")) + if snd is None: + raise SongsError(f"{where}: sound {m.get('_musicSoundID')} is not in MasterSound") + sheet = sheets.get(snd.get("_soundCueSheetID")) + if sheet is None: + raise SongsError(f"{where}: cue sheet {snd.get('_soundCueSheetID')} is not in MasterSoundCueSheet") + length = bgm(sheet["_cueSheetName"], snd["_cueName"]) if bgm is not None else None + songs.append({ + "id": m["_id"], + "sortOrder": m.get("_sortOrder"), + "startAt": m.get("_startAt"), + "defaultUnlock": m.get("_defaultUnlock"), + "title": text.get(m.get("_titleTextID")), + "ruby": text.get(m.get("_rubyTitleTextID")), + "phonetic": text.get(m.get("_phoneticTextID")), + "bandIds": list(m.get("_bandIDs") or []), + "bandName": text.get(m.get("_bandNameTextID")), + "vocalCharacterIds": list(m.get("_vocalCharacterIDs") or []), + "lyricist": text.get(m.get("_lyricistTextID")), + "composer": text.get(m.get("_composerTextID")), + "arranger": text.get(m.get("_arrangerTextID")), + "musicType": m.get("_musicType"), + "musicCategories": list(m.get("_musicCategories") or []), + "bestMusicTagIds": list(m.get("_bestMusicTagIDs") or []), + "jacket": m.get("_jacketAssetName"), + "gekisouMissions": [m.get("_gekisouMission1"), m.get("_gekisouMission2"), m.get("_gekisouMission3")], + "bgm": {"soundId": snd["_id"], "cueSheet": sheet["_cueSheetName"], "cue": snd["_cueName"], + "length": length}, + "charts": charts, + "master": {"MasterLiveMusic": m}, + }) + return { + "format": FORMAT, + "provenance": { + "region": region, + "client": {"versionName": client.get("versionName"), "versionCode": client.get("versionCode")}, + "catalog": {"resourceVersion": catalog.get("resourceVersion"), "sha256": catalog.get("sha256")}, + "master": {"source": master_source, "version": master_version, + "tables": {t: {"sha256": table_sha[t]} for t in TABLES}}, + "exporter": {"name": "nnnotes", "version": __version__, "chartFormat": deckdata.CHART_FORMAT}, + }, + "languages": list(LANGUAGES), + "bands": bands, + "characters": characters, + "tags": tags, + "categories": categories, + "songs": songs, + } + + +def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes], + bgm: Callable[[str, str], dict] | None, *, region: str, client: dict, catalog: dict, + jacket: Callable[[str], bytes] | None = None, jackets_dir=None) -> dict: + """Read the master data, every chart and every BGM cue sheet, then write the file `out` (gzip when it ends in + `.gz`) through a temporary file and a rename; with `jacket` and `jackets_dir`, first every song's jacket as + `/.webp`. Returns the summary.""" + from .cache import write_atomic + out = Path(out) + try: + tables, shas = deckdata.read_master(src, key, TABLES) + except deckdata.DeckDataError as e: + raise SongsError(str(e)) from None + doc = build(tables, shas, fetch, bgm, region=region, client=client, catalog=catalog, master_source=src.source, + master_version=src.version) + jackets = sorted({s["jacket"] for s in doc["songs"] if s["jacket"]}) if jacket is not None else [] + images = {name: jacket(name) for name in jackets} + if jackets: + d = Path(jackets_dir) + d.mkdir(parents=True, exist_ok=True) + for name, data in images.items(): + write_atomic(d / f"{name}.webp", data) + try: + data = deckdata.encode(deckdata._value(doc, "songs")) + except deckdata.DeckDataError as e: + raise SongsError(str(e)) from None + written = deckdata.file_bytes(data, out.name.endswith(".gz")) + out.parent.mkdir(parents=True, exist_ok=True) + write_atomic(out, written) + return {"out": str(out), "format": FORMAT, "region": region, "masterSource": src.source, + "masterVersion": src.version, "songs": len(doc["songs"]), + "charts": sum(len(s["charts"]) for s in doc["songs"]), "bgm": bgm is not None, + "jackets": len(jackets), + "bytes": len(data), "fileBytes": len(written), "sha256": hashlib.sha256(written).hexdigest()} diff --git a/tests/test_songs.py b/tests/test_songs.py new file mode 100644 index 0000000..7d806b0 --- /dev/null +++ b/tests/test_songs.py @@ -0,0 +1,218 @@ +"""Songs export (songs): metadata, chart facts and BGM length on synthetic master data, charts and ACBs.""" +import gzip +import hashlib +import io +import json + +import pytest + +import synth +from nnnotes import cli, deckdata, score, songs +from nnnotes.master import MasterKey +from test_deckdata import CHART, SMALL, FakeCatalog, run +from test_voices import simple_acb + +KEY = MasterKey(synth.MASTER_KEY, synth.MASTER_IV) + + +def text(tid, stem): + return {"_id": tid, "_japanese": f"{stem}-ja", "_english": f"{stem}-en", "_traditionalChinese": f"{stem}-tw", + "_simplifiedChinese": f"{stem}-cn", "_korean": f"{stem}-ko"} + + +MUSIC = {"_id": 100002, "_sortOrder": 2, "_startAt": "2026/01/01 0:00:00", "_defaultUnlock": True, + "_titleTextID": "T2", "_rubyTitleTextID": "", "_phoneticTextID": "P2", "_bandIDs": [1], + "_bandNameTextID": "", "_vocalCharacterIDs": [1], "_lyricistTextID": "L2", "_composerTextID": "C2", + "_arrangerTextID": "", "_musicType": 4, "_musicCategories": [1], "_bestMusicTagIDs": [1], + "_jacketAssetName": "jkt_2", "_gekisouMission1": 1, "_gekisouMission2": 3, "_gekisouMission3": 3, + "_musicSoundID": 13, "_easyID": 20, "_normalID": 0, "_hardID": 0, "_expertID": 30, "_extra": [7]} +MUSIC1 = dict(MUSIC, _id=100001, _sortOrder=1, _titleTextID="T1", _bandNameTextID="BN", _easyID=10, _expertID=0) +TABLE_ROWS = { + "MasterLiveMusic": [MUSIC, MUSIC1], + "MasterLiveMusicScore": [ + {"_id": 30, "_musicScoreTextFileName": "c/c_03", "_musicScoreLevel": 20, "_fullComboCount": 2, + "_musicScoreDisplayLevel": 20.5}, + {"_id": 10, "_musicScoreTextFileName": "c/c_01", "_musicScoreLevel": 5, "_fullComboCount": 11, + "_musicScoreDisplayLevel": 5.0}, + {"_id": 20, "_musicScoreTextFileName": "c/c_02", "_musicScoreLevel": 9, "_fullComboCount": 2, + "_musicScoreDisplayLevel": 9.0}], + "MasterText": [text("T1", "one"), text("T2", "two"), text("P2", "pho"), text("L2", "lyr"), text("C2", "com"), + text("BN", "crychic"), text("Band1", "mygo"), text("Ch1", "tomori"), text("Ch1s", "tomo"), + text("Tag1", "tag"), text("Cat1", "original")], + "MasterBand": [{"_id": 1, "_nameTextID": "Band1", "_mainColorCode": "#3388BB", "_subColorCode": "#FFFFFF"}], + "MasterCharacter": [{"_id": 1, "_nameTextID": "Ch1", "_shortNameTextID": "Ch1s", "_bandID": 1, + "_mainColorCode": "#77BBDD"}], + "MasterTag": [{"_id": 1, "_nameTextID": "Tag1"}], + "MasterLiveMusicCategory": [{"_id": 1, "_musicCategories": [1], "_textKey": "Cat1"}], + "MasterSound": [{"_id": 13, "_soundCueSheetID": 5, "_cueName": "song2"}], + "MasterSoundCueSheet": [{"_id": 5, "_cueSheetName": "Bgm2"}], +} +CHARTS = {"c/c_01": gzip.compress(json.dumps(CHART).encode("utf-8"), mtime=0), + "c/c_02": json.dumps(SMALL).encode("utf-8"), + "c/c_03": gzip.compress(json.dumps(SMALL).encode("utf-8"), mtime=0)} +ACB = simple_acb({"song2": [1]}, [1], {1: 48000 * 90 + 24}) # 90.0005 s at 48 kHz, cue Length 100 +PROV = {"region": "xx", "client": {"versionName": "9.9.9", "versionCode": 99}, + "catalog": {"resourceVersion": None, "sha256": "ab" * 32}} + + +def master_dir(tmp_path, rows=TABLE_ROWS): + d = tmp_path / "m" + d.mkdir() + files = [] + for t in songs.TABLES: + data = synth.master_file({"_allData": rows.get(t, [])}) + (d / f"{t}.bin").write_bytes(data) + files.append({"name": f"{t}.bin", "hash": hashlib.sha256(data).hexdigest(), "size": len(data)}) + (d / "MasterManifest.json").write_text(json.dumps({"version": "v-test", "files": files}), encoding="utf-8") + return d + + +def bgm(sheet, cue): + assert sheet == "Bgm2" + return songs.cue_length(ACB, cue, f"cue sheet {sheet}") + + +def export(tmp_path, rows=TABLE_ROWS, charts=CHARTS, bgm=bgm, out="songs.json", **kw): + d = master_dir(tmp_path, rows) + return songs.export(tmp_path / out, deckdata.master_files(d), KEY, charts.__getitem__, bgm, **PROV, **kw) + + +def test_document(tmp_path): + r = export(tmp_path) + raw = (tmp_path / "songs.json").read_bytes() + assert raw.endswith(b"}\n") and raw.count(b"\n") == 1 + assert (r["songs"], r["charts"], r["sha256"]) == (2, 3, hashlib.sha256(raw).hexdigest()) + doc = json.loads(raw) + assert list(doc) == ["format", "provenance", "languages", "bands", "characters", "tags", "categories", "songs"] + assert doc["format"] == "nnnotes.songs/1" and doc["languages"] == ["ja", "en", "zh-Hant", "zh-Hans", "ko"] + assert list(doc["provenance"]["master"]["tables"]) == list(songs.TABLES) + assert doc["bands"] == [{"id": 1, "name": {"ja": "mygo-ja", "en": "mygo-en", "zh-Hant": "mygo-tw", + "zh-Hans": "mygo-cn", "ko": "mygo-ko"}, + "mainColor": "#3388BB", "subColor": "#FFFFFF"}] + assert doc["characters"][0]["name"]["ja"] == "tomori-ja" and doc["characters"][0]["bandId"] == 1 + assert doc["categories"] == [{"id": 1, "musicCategories": [1], "name": dict(doc["categories"][0]["name"])}] + one, two = doc["songs"] + assert (one["id"], two["id"]) == (100001, 100002) # sorted by id + assert one["bandName"]["ja"] == "crychic-ja" and two["bandName"] is None + assert two["title"]["zh-Hans"] == "two-cn" and two["ruby"] is None and two["arranger"] is None + assert two["lyricist"]["en"] == "lyr-en" and two["gekisouMissions"] == [1, 3, 3] + assert two["bgm"] == {"soundId": 13, "cueSheet": "Bgm2", "cue": "song2", + "length": {"lengthMs": 100, "samples": 48000 * 90 + 24, "sampleRate": 48000, + "durationMs": 90000}} + assert two["master"]["MasterLiveMusic"]["_extra"] == [7] # the whole row + assert [c["difficulty"] for c in two["charts"]] == ["easy", "expert"] # difficulties without a score are left out + assert [c["difficulty"] for c in one["charts"]] == ["easy"] + + +def test_chart_facts(tmp_path): + export(tmp_path) + doc = json.loads((tmp_path / "songs.json").read_bytes()) + c = doc["songs"][0]["charts"][0] # CHART at 120 BPM + assert (c["scoreId"], c["level"], c["displayLevel"], c["fullComboCount"]) == (10, 5, 5.0, 11) + assert c["asset"] == {"key": "Live/MusicScore/c/c_01", "sha256": hashlib.sha256(CHARTS["c/c_01"]).hexdigest()} + rs = score.runtime_score(CHART) + judged = sorted(n.pos.ms for n in rs.notes if score.is_judgement_note(n.op)) + assert c["notes"]["judged"] == len(judged) and c["notes"]["total"] == len(rs.notes) == 17 + assert sum(c["notes"]["byOperateType"].values()) == 17 and list(c["notes"]["byOperateType"])[0] == "1" + assert c["bpm"] == {"main": 120, "min": 120, "max": 120, "changes": [{"timeMs": 0, "bpm": 120}]} + assert (c["firstNoteMs"], c["lastJudgedNoteMs"], c["lastNoteMs"], c["musicLengthMs"]) == (0, 5000, 5000, 6000) + assert c["skillEventsMs"] == [2500, 500] and c["fevers"] == [[0, 1000], [2000, 2500]] + + +def test_bpm_facts(): + class P: + def __init__(self, ms): + self.ms = ms + ev = [(100.0, P(0)), (200.0, P(1000)), (150.0, P(1500)), (300.0, P(9000))] + f = songs.bpm_facts(ev, 500, 4000) # 100 for 500 ms, 200 for 500, 150 for 2500; 300 after + assert (f["main"], f["min"], f["max"]) == (150.0, 100.0, 200.0) + assert [x["timeMs"] for x in f["changes"]] == [0, 1000, 1500, 9000] + assert songs.bpm_facts(ev, 1200, 1200)["main"] == 200.0 # one instant: the BPM at that time + tie = [(120.0, P(0)), (180.0, P(1000))] + assert songs.bpm_facts(tie, 0, 2000)["main"] == 120.0 # equal time: the earliest + with pytest.raises(songs.SongsError): + songs.bpm_facts([], 0, 1) + + +def test_deterministic_and_no_bgm(tmp_path): + export(tmp_path, out="a.json") + (tmp_path / "m").rename(tmp_path / "m0") + export(tmp_path, out="b.json") + assert (tmp_path / "a.json").read_bytes() == (tmp_path / "b.json").read_bytes() + (tmp_path / "m").rename(tmp_path / "m1") + r = export(tmp_path, bgm=None, out="c.json.gz") + doc = json.loads(gzip.decompress((tmp_path / "c.json.gz").read_bytes())) + assert r["bgm"] is False and all(s["bgm"]["length"] is None for s in doc["songs"]) + + +def test_jacket_webp(): + from PIL import Image + img = Image.new("RGBA", (1024, 512), (10, 200, 30, 255)) + out = Image.open(io.BytesIO(songs.jacket_webp(img))) + assert (out.format, out.size, out.mode) == ("WEBP", (320, 160), "RGB") + img.putpixel((0, 0), (0, 0, 0, 0)) + out = Image.open(io.BytesIO(songs.jacket_webp(img, size=2000))) + assert (out.size, out.mode) == ((1024, 512), "RGBA") + + +def test_jackets(tmp_path): + from PIL import Image + asked = [] + + def jacket(name): + asked.append(name) + return songs.jacket_webp(Image.new("RGB", (64, 64), (1, 2, 3))) + r = export(tmp_path, jacket=jacket, jackets_dir=tmp_path / "j") + assert r["jackets"] == 1 and asked == ["jkt_2"] # the two songs share one jacket + assert Image.open(tmp_path / "j" / "jkt_2.webp").size == (64, 64) + (tmp_path / "x").mkdir() + assert export(tmp_path / "x", out="b.json")["jackets"] == 0 + + def missing(name): + raise songs.SongsError(f"jacket {name}: no asset") + (tmp_path / "y").mkdir() + with pytest.raises(songs.SongsError, match="jacket jkt_2"): + export(tmp_path / "y", jacket=missing, jackets_dir=tmp_path / "y" / "j") + assert not (tmp_path / "y" / "songs.json").exists() + + +@pytest.mark.parametrize("change, match", [ + (lambda r: r["MasterText"].pop(0), "MasterText has no text 'T1'"), + (lambda r: r["MasterLiveMusicScore"].pop(0), "expert score 30 is not in MasterLiveMusicScore"), + (lambda r: r["MasterSound"].clear(), "sound 13 is not in MasterSound"), + (lambda r: r["MasterLiveMusic"].append(dict(MUSIC)), "MasterLiveMusic: _id 100002 occurs twice"), +]) +def test_master_errors(tmp_path, change, match): + rows = json.loads(json.dumps(TABLE_ROWS)) + change(rows) + with pytest.raises(songs.SongsError, match=match): + export(tmp_path, rows=rows) + assert not (tmp_path / "songs.json").exists() + + +def test_input_errors(tmp_path): + with pytest.raises(songs.SongsError, match="c/c_03 .*no such asset"): + export(tmp_path, charts={k: v for k, v in CHARTS.items() if k != "c/c_03"}) + (tmp_path / "m").rename(tmp_path / "m0") + with pytest.raises(songs.SongsError, match="not a chart"): + export(tmp_path, charts=dict(CHARTS, **{"c/c_02": b"\x1f\x8bnot"})) + (tmp_path / "m").rename(tmp_path / "m1") + with pytest.raises(songs.SongsError, match="no cue 'song2'"): + export(tmp_path, bgm=lambda s, c: songs.cue_length(simple_acb({"x": [1]}, [1], {1: 5}), c, s)) + + +def test_command(tmp_path, capsys, monkeypatch): + d = master_dir(tmp_path) + out = tmp_path / "o" / "songs.json" + code, _, err = run(["songs", "-o", str(out)], capsys) + assert code == 2 and "--master-files" in err + monkeypatch.setenv("NNNOTES_MASTER_KEY", synth.MASTER_KEY.hex()) + monkeypatch.setenv("NNNOTES_MASTER_IV", synth.MASTER_IV.hex()) + monkeypatch.setattr(cli, "open_catalog", lambda cfg, **kw: FakeCatalog(CHARTS)) + monkeypatch.setattr(score, "fetch_chart", lambda cat, name: cat.charts[name]) + code, stdout, err = run(["--region", "xx", "--cache", str(tmp_path / "cache"), "songs", "--master-files", + str(d), "--no-bgm", "-o", str(out)], capsys) + assert code == 0, err + r = json.loads(stdout) + assert (r["songs"], r["charts"], r["bgm"], r["region"]) == (2, 3, False, "xx") + assert synth.MASTER_KEY.hex() not in stdout + err From 197b436dfccbefa5296fd40fcf5167458ab1406e Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Tue, 29 Sep 2026 22:14:50 +0800 Subject: [PATCH 09/11] feat!: one music-data command with the deck model built in `nnnotes music-data` replaces `songs` and `deck-data`: one canonical JSON file (nnnotes.music-data/1) with every song's metadata, every chart's facts and, per chart, the deck statistics of ournotes-deck (chart-stats/2: whole-live simulation, Gekisou on, per seed). The deck model is built into nnnotes as the PyO3 extension nnnotes._deck, pinned to one ournotes-deck commit in rust/Cargo.toml, so one nnnotes version always carries one deck model; its results are checked against the chart facts (notes, times, missions, skill events, fevers) and a mismatch fails the export. `--full` adds the deck input (the master subset and every chart), `--no-deck` leaves the statistics out. The build moves to maturin: abi3 wheels for Python 3.11+ on Linux, Windows and macOS, and an sdist with the Rust sources. The deck workflow opens a pull request when ournotes-deck's main moves past the pinned commit. BREAKING CHANGE: the `songs` and `deck-data` commands and their formats are gone; `music-data` writes both in one file. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 22 +- .github/workflows/deck.yml | 50 ++ .github/workflows/release.yml | 76 +- .gitignore | 3 + README.en.md | 3 +- README.md | 3 +- docs/commands.md | 64 +- docs/configuration.md | 2 +- docs/contracts.md | 6 +- docs/deck-data.md | 160 ----- docs/music-data.md | 323 +++++++++ docs/schema/deck-data.schema.json | 273 -------- docs/schema/music-data.schema.json | 1034 ++++++++++++++++++++++++++++ docs/songs.md | 98 --- pyproject.toml | 20 +- rust/Cargo.lock | 215 ++++++ rust/Cargo.toml | 23 + rust/build.rs | 33 + rust/rustfmt.toml | 2 + rust/src/lib.rs | 98 +++ src/nnnotes/cli.py | 68 +- src/nnnotes/deckdata.py | 66 +- src/nnnotes/musicdata.py | 460 +++++++++++++ src/nnnotes/songs.py | 338 --------- tests/test_deckdata.py | 103 +-- tests/test_musicdata.py | 433 ++++++++++++ tests/test_songs.py | 218 ------ 27 files changed, 2878 insertions(+), 1316 deletions(-) create mode 100644 .github/workflows/deck.yml delete mode 100644 docs/deck-data.md create mode 100644 docs/music-data.md delete mode 100644 docs/schema/deck-data.schema.json create mode 100644 docs/schema/music-data.schema.json delete mode 100644 docs/songs.md create mode 100644 rust/Cargo.lock create mode 100644 rust/Cargo.toml create mode 100644 rust/build.rs create mode 100644 rust/rustfmt.toml create mode 100644 rust/src/lib.rs create mode 100644 src/nnnotes/musicdata.py delete mode 100644 src/nnnotes/songs.py create mode 100644 tests/test_musicdata.py delete mode 100644 tests/test_songs.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92f4e16..4889741 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,10 @@ jobs: python-version: ${{ matrix.python-version }} cache: pip cache-dependency-path: pyproject.toml - - name: Install + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust + - name: Install (builds the extension module nnnotes._deck) run: | python -m pip install --upgrade pip python -m pip install -e ".[test]" @@ -50,6 +53,9 @@ jobs: cache-dependency-path: | pyproject.toml .github/goldens-constraints.txt + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust - name: Install (pinned) run: | python -m pip install --upgrade pip @@ -58,3 +64,17 @@ jobs: env: GOLDENS_STRICT: "1" run: python -m pytest tests/test_goldens.py tests/test_objexport.py + + rust: + # the extension module's own code (rust/); ournotes-deck is tested in its repository + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust + - name: Format and lints + working-directory: rust + run: | + cargo fmt --check + cargo clippy --locked --all-targets -- -D warnings diff --git a/.github/workflows/deck.yml b/.github/workflows/deck.yml new file mode 100644 index 0000000..688eabf --- /dev/null +++ b/.github/workflows/deck.yml @@ -0,0 +1,50 @@ +name: Deck model + +# nnnotes builds the deck model ournotes-deck at the commit rust/Cargo.toml pins, so that one nnnotes version always +# carries one deck model. When ournotes-deck's main moves past that commit, this opens a pull request that moves the +# pin; CI then builds and tests it like any change. +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + update: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Compare the pinned commit with ournotes-deck's main + id: deck + env: + GH_TOKEN: ${{ github.token }} + run: | + pinned=$(sed -nE 's/^ournotes-deck = .*rev = "([0-9a-f]{40})".*/\1/p' rust/Cargo.toml) + head=$(git ls-remote https://github.com/empty-sekai/ournotes-deck refs/heads/main | cut -f1) + # ahead: main contains the pinned commit and more; behind or diverged: leave the pin to a person + status=$(gh api "repos/empty-sekai/ournotes-deck/compare/$pinned...$head" --jq .status) + echo "pinned $pinned, main $head: $status" + echo "pinned=$pinned" >> "$GITHUB_OUTPUT" + echo "head=$head" >> "$GITHUB_OUTPUT" + echo "move=$([ "$status" = ahead ] && echo true || echo false)" >> "$GITHUB_OUTPUT" + - name: Move the pin + if: steps.deck.outputs.move == 'true' + working-directory: rust + run: | + sed -i -E 's/(rev = ")${{ steps.deck.outputs.pinned }}(")/\1${{ steps.deck.outputs.head }}\2/' Cargo.toml + cargo update -p ournotes-deck + - uses: peter-evans/create-pull-request@v7 + if: steps.deck.outputs.move == 'true' + with: + branch: deck/update + commit-message: "build(deck): ournotes-deck ${{ steps.deck.outputs.head }}" + title: "build(deck): ournotes-deck ${{ steps.deck.outputs.head }}" + body: | + Moves the deck model to ournotes-deck's main, + https://github.com/empty-sekai/ournotes-deck/compare/${{ steps.deck.outputs.pinned }}...${{ steps.deck.outputs.head }} + + Music data files made after this change name the new commit in `provenance.deck.commit`; their deck + statistics may differ. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6379d52..9808b4d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,25 +8,79 @@ permissions: contents: read jobs: - build: + check: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: "3.13" - - name: Check the tag against the package version + - name: Check the tag against the package and extension versions run: | - version=$(python -c 'import re, pathlib; print(re.search(r"__version__ = \"([^\"]+)\"", pathlib.Path("src/nnnotes/__init__.py").read_text()).group(1))') - if [ "v$version" != "$GITHUB_REF_NAME" ]; then - echo "tag $GITHUB_REF_NAME does not match the package version $version" >&2 + version=$(python3 -c 'import re, pathlib; print(re.search(r"__version__ = \"([^\"]+)\"", pathlib.Path("src/nnnotes/__init__.py").read_text()).group(1))') + cargo=$(python3 -c 'import re, pathlib; print(re.search(r"^version = \"([^\"]+)\"", pathlib.Path("rust/Cargo.toml").read_text(), re.M).group(1))') + if [ "v$version" != "$GITHUB_REF_NAME" ] || [ "$cargo" != "$version" ]; then + echo "tag $GITHUB_REF_NAME, package $version and rust/Cargo.toml $cargo differ" >&2 exit 1 fi - - name: Build sdist and wheel + + # one abi3 wheel (CPython 3.11 and later) per platform with the extension module nnnotes._deck, and the sdist + wheels: + needs: check + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + include: + - {os: ubuntu-latest, target: x86_64, manylinux: "2_28"} + - {os: ubuntu-24.04-arm, target: aarch64, manylinux: "2_28"} + - {os: windows-latest, target: x64} + - {os: macos-latest, target: aarch64} + - {os: macos-15-intel, target: x86_64} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux || 'auto' }} + args: --release --locked --out dist + - uses: actions/upload-artifact@v7 + with: + name: wheel-${{ matrix.os }}-${{ matrix.target }} + path: dist/ + if-no-files-found: error + + sdist: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + - uses: actions/upload-artifact@v7 + with: + name: sdist + path: dist/ + if-no-files-found: error + + build: + needs: [wheels, sdist] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v8 + with: + path: dist/ + merge-multiple: true + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - name: Check the distributions run: | - python -m pip install --upgrade pip build twine - python -m build + python -m pip install --upgrade pip twine python -m twine check --strict dist/* + ls dist - uses: actions/upload-artifact@v7 with: name: dist diff --git a/.gitignore b/.gitignore index a179109..0c92583 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ nnnotes.toml .env out/ cache/ +rust/target/ +src/nnnotes/_deck*.pyd +src/nnnotes/_deck*.so diff --git a/README.en.md b/README.en.md index b7cc40a..34ca1f0 100644 --- a/README.en.md +++ b/README.en.md @@ -49,8 +49,7 @@ nnnotes config check # each setting's origin and whether it is valid | `player` | APK | render-related global settings (color space, quality levels, renderers) as JSON | | `live` | music ID + difficulty | a full chart directory: chart and runtime notes, 3D scene, note and effect assets, BGM and sounds, sound routing | | `web` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; the Live2D models of the stories are built first as models, listed in `models.json`, and the story manifests reference them; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`); compressible assets (JSON, shaders, moc3, ...) are stored gzip encoded by default (`--compress br` / `none`) | -| `deck-data` | master data files (`--master-files` directory or `--apk-master`) | one JSON file for deck-building tools: every chart's runtime notes, skill events and fever ranges, and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/deck-data.md)) | -| `songs` | master data files (`--master-files` directory or `--apk-master`) | one JSON file with every song's metadata: titles and credits in five languages, bands, vocal characters, category, tags, release time, BGM length, and per difficulty the level, note counts, BPM, chart times, skill events and fever ranges ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/songs.md)) | +| `music-data` | master data files (`--master-files` directory or `--apk-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (built into nnnotes) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) | Export conventions: diff --git a/README.md b/README.md index b78ba09..4d9397b 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,7 @@ nnnotes config check # 每项设置的来源和格式是否有效， | `player` | APK | 渲染相关的全局设置（色彩空间、画质等级、渲染器）JSON | | `live` | 曲目 ID + 难度 | 完整谱面目录：谱面与运行时音符、3D 场景、音符与特效资源、BGM 与音效、声音路由 | | `web` | `--pair 曲目:难度`（可重复）或 `--all`；`--live2d 模型`（可重复）或 `--all-live2d`；`--story 剧情 ID`（可重复）或 `--all-stories`；`--region 区服`（可重复）或 `--all-regions` | ournotes-player 静态站点：共享播放器 + 每谱 / 每模型 / 每集剧情清单 + 内容寻址资源；剧情用到的 Live2D 模型先按模型构建并列入 `models.json`，剧情清单引用它们；一个站点可服务多个区服，列表文本含五种语言；剧情的界面文字按语言分组，字形由开源字体生成 TextMeshPro 字体资源（`--fonts game` 时用游戏字体）；可压缩的资源（JSON、着色器、moc3 等）默认以 gzip 存储，`--compress br` / `none` 可改 | -| `deck-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`） | 供组卡工具读取的单个 JSON：每张谱面的运行时音符、技能事件与 fever 区间，以及卡牌、技能、加成、分数与活动相关的 masterdata 表（[格式](docs/deck-data.md)） | -| `songs` | masterdata 文件（`--master-files` 目录或 `--apk-master`） | 全部歌曲元数据的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、BGM 时长，以及每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间（[格式](docs/songs.md)） | +| `music-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`） | 全部歌曲与谱面的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、评级线、BGM 时长；每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间；以及组卡模型 ournotes-deck（内置于 nnnotes）在整场模拟上实测的谱面统计（无技能得分、每种加分技能在每个演出位的权重）。`--full` 另附组卡模型的输入：每张谱面的运行时音符与卡牌、技能、加成、分数、活动相关的 masterdata 表（[格式](docs/music-data.md)） | 导出约定： diff --git a/docs/commands.md b/docs/commands.md index 4860bb8..8271439 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -667,39 +667,31 @@ byte-identical outputs; the encoded assets also need the same zlib and brotli ve in the printed summary and in `SITE.failures.json`, models that fail in the summary and in `SITE.model-failures.json`, stories in `SITE.story-failures.json`; the exit status is then 1. -## deck-data - -``` -nnnotes deck-data (--master-files DIR | --apk-master) -o FILE -``` - -Writes one JSON file for deck-building tools, for one master data version: every `MasterLiveMusicScore` row's chart -as the client builds it at runtime (notes, skill events, fever ranges) and the master data tables about cards, -skills, bonuses, scores and events, with their provenance. The master data is decoded from the files as served: -`--master-files DIR` reads `DIR/MasterManifest.json` and the `.bin` files it lists (`master download`; the file's -region is `[catalog] region`), `--apk-master` the same files inside `[paths] apk` (region `embedded`); each file is -checked against the manifest's SHA-256. `FILE` ending in `.gz` is written gzip-compressed. The file is canonical: the -same inputs give the same bytes. Prints `{out, format, region, masterSource, masterVersion, tables, rows, charts, -notes, bytes, fileBytes, sha256}`. A missing or mismatching input (a master data file, a column, a chart asset) -stops the command with exit status 1 before the file is written. The format is described in -[deck-data.md](deck-data.md). - -## songs - -``` -nnnotes songs (--master-files DIR | --apk-master) [--no-bgm] [--jackets DIR] -o FILE -``` - -Writes one JSON file with every `MasterLiveMusic` song's metadata for one master data version: titles, readings and -credits in the five text languages, bands (and a song's own band name), vocal characters, category, tags, release -time, jacket, Gekisou missions, the whole `MasterLiveMusic` row, the live BGM's cue and length (read from the cue -sheet's ACB, without decoding audio), and per difficulty the chart facts a song listing shows: level and display -level, full combo count, note counts (judged, total, per operate type), BPM (main, min, max and every change), the -first and last note times, the live's music length (last note + 1000 ms), skill event times and fever ranges. The -master data is read as for `deck-data` (same options, same checks), so the two files of one master data version join -by `scoreId`. `--no-bgm` skips the cue sheets (every `bgm.length` is null). `--jackets DIR` also writes every song's -jacket as `DIR/.webp` (at most 320 px on the longer side). `FILE` ending in `.gz` is written gzip-compressed; -the file is canonical. Prints `{out, format, region, masterSource, masterVersion, songs, charts, bgm, jackets, bytes, -fileBytes, sha256}`. A missing or unreadable input (a master data file, a text id, a chart asset, a cue sheet or cue, a -jacket) stops the command with exit status 1 before the file is written. The format is described in -[songs.md](songs.md). +## music-data + +``` +nnnotes music-data (--master-files DIR | --apk-master) [--full] [--no-deck] [--seeds N] [--workers N] + [--no-bgm] [--jackets DIR] -o FILE +``` + +Writes one JSON file with every `MasterLiveMusic` song and its charts for one master data version: titles, readings +and credits in the five text languages, bands (and a song's own band name), vocal characters, category, tags, +release time, jacket, Gekisou missions, score ranks, the whole `MasterLiveMusic` row, the live BGM's cue and length +(read from the cue sheet's ACB, without decoding audio); per difficulty the chart facts (level and display level, +full combo count, note counts, BPM, note times, the live's music length, skill event times, fever ranges) and the +chart's deck statistics: the no-skill score and the weight of every score-up skill kind at every performance +position, measured by the deck model ournotes-deck (built into nnnotes as `nnnotes._deck`) on its whole-live +simulation and checked against the chart facts. `--full` also writes the deck model's input: every +`MasterLiveMusicScore` row's chart as the client builds it at runtime (notes, skill events, fever ranges) and the +master data tables about cards, skills, bonuses, scores and events. `--no-deck` skips the deck model (every chart's +`deck` is null); `--seeds N` (default 8) and `--workers N` (default: every processor) set its seeds on charts with a +luck range and its threads. The master data is decoded from the files as served: `--master-files DIR` reads +`DIR/MasterManifest.json` and the `.bin` files it lists (`master download`; the file's region is `[catalog] region`), +`--apk-master` the same files inside `[paths] apk` (region `embedded`); each file is checked against the manifest's +SHA-256. `--no-bgm` skips the cue sheets (every `bgm.length` is null). `--jackets DIR` also writes every song's +jacket as `DIR/.webp` (at most 320 px on the longer side). `FILE` ending in `.gz` is written +gzip-compressed; the file is canonical: the same inputs and nnnotes version give the same bytes. Prints `{out, +format, region, masterSource, masterVersion, songs, charts, deck, unplayable, full, bgm, jackets, bytes, fileBytes, +sha256}`. A missing or unreadable input (a master data file, a column, a text id, a chart asset, a cue sheet or cue, +a jacket), a chart the deck model cannot measure, or deck statistics that disagree with the chart facts stop the +command with exit status 1 before the file is written. The format is described in [music-data.md](music-data.md). diff --git a/docs/configuration.md b/docs/configuration.md index 4b21ee1..b8a65c4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -155,7 +155,7 @@ story text); it is not a setting. | `web --pair` / `--all` | as `live`, plus `[paths] player`, `node`; with `--region` / `--all-regions` each region's `[servers.]` table (its `cdn` for what must be downloaded) and master data (`[servers.] master`; `[paths] master` for at most one region) | | `web --live2d` / `--all-live2d` | catalog (bundles from the CDN of the site's first region), `[paths] apk`, `[paths] player`; not `node`; master data only for the model names (optional: without it `models.json` has no names) | | `web --player-only` / `--reingest-json` | `[paths] player` | -| `deck-data` | catalog, `[master] key` + `iv`; `--master-files` also `[catalog] region`; `--apk-master` also `[paths] apk`; `[paths] apk` (optional otherwise) for the client version | +| `music-data` | catalog, `[master] key` + `iv`; `--master-files` also `[catalog] region`; `--apk-master` also `[paths] apk`; `[paths] apk` (optional otherwise) for the client version | | `export`, `plan` | the store (`[paths] store` or `[paths] cache`); catalog (bundles are fetched into the cache); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`); master data for `--views`; with `--catalog-version` an imported catalog version instead of the current catalog | | `run-stage` | the store; `[paths] cache` for inputs located in the cache; `--fetch` also what fetching needs (region, `cdn`, bundle key, `[paths] apk`) | | `catalogs list` / `import` / `diff`, `store verify` | the store; `import` reads the APK's catalog from `[paths] apk` when it is set | diff --git a/docs/contracts.md b/docs/contracts.md index e3a3d48..2cf4e7f 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -226,9 +226,9 @@ The catalog versions of a store, `nnnotes.catalogs/1` ([schema/catalogs.schema.j ([schema/catalog-diff.schema.json](schema/catalog-diff.schema.json), `nnnotes catalogs diff --json`), are written by the commands. -The deck data file of `nnnotes deck-data`, `nnnotes.deck-data/1` -([schema/deck-data.schema.json](schema/deck-data.schema.json)), names its format in a `format` field; it is -described in [deck-data.md](deck-data.md). +The music data file of `nnnotes music-data`, `nnnotes.music-data/1` +([schema/music-data.schema.json](schema/music-data.schema.json)), names its format in a `format` field; it is +described in [music-data.md](music-data.md). ## Store diff --git a/docs/deck-data.md b/docs/deck-data.md deleted file mode 100644 index 62c9a40..0000000 --- a/docs/deck-data.md +++ /dev/null @@ -1,160 +0,0 @@ -# Deck data - -`nnnotes deck-data` writes one JSON file with the game data that deck-building tools read: every live chart as the -client builds it at runtime, and the master data tables about cards, skills, bonuses, scores and events. One file -describes one master data version. The file holds facts as the game has them (master data values as served, notes -as the client's chart converter creates them), not formulas or values derived from them. The format is -`nnnotes.deck-data/1`; its JSON Schema is [schema/deck-data.schema.json](schema/deck-data.schema.json). - -``` -nnnotes deck-data (--master-files DIR | --apk-master) -o FILE -``` - -- `--master-files DIR`: master data files as served, `DIR/MasterManifest.json` and the `.bin` files it lists - (`nnnotes master download`). The file's `region` is `[catalog] region` (`--region`). -- `--apk-master`: the master data files the APK ships (`assets/Master/` of `[paths] apk`, the same layout). The - file's `region` is `embedded`. - -Each master data file is checked against the SHA-256 the manifest lists and decoded with `[master] key` and `iv`. -Charts are read from the catalog of `[catalog] language` (bundles fetched into the cache as for every command). -`FILE` ending in `.gz` is written gzip-compressed. The command prints a summary (`charts`, `notes`, `tables`, -`rows`, `bytes`, `sha256` of the file, ...). - -The command writes the file only when every table and every chart was read: a missing or mismatching master data -file, a table without a column the file exports, a `MasterLiveMusicScore` row whose chart asset is missing or cannot -be converted, a note id that occurs twice in a chart, or a note whose operate type has no judgement type stops it with -exit status 1 and a line naming the input. The file is written through a temporary file and a rename. - -## Layout - -```json -{ - "format": "nnnotes.deck-data/1", - "provenance": {"region": ..., "client": {...}, "catalog": {...}, "master": {...}, "exporter": {...}}, - "master": {"": {"columns": ["_id", ...], "rows": [[...], ...]}, ...}, - "charts": [{"scoreId": ..., "asset": {...}, "notes": {...}, "skillEvents": {...}, "fevers": {...}}, ...] -} -``` - -The file is minified UTF-8 with one line feed at the end. Keys are always in the order shown here and in the tables -below, so the same inputs with the same nnnotes version give the same bytes. A gzip file has no file name and a zero -modification time in its header. - -### provenance - -| Field | Content | -|---|---| -| `region` | the region whose master data this is (a configured region name), or `embedded` for the APK's master data | -| `client.versionName`, `client.versionCode` | the APK's version name and code (null without `[paths] apk`) | -| `catalog.resourceVersion` | the resource version recorded for the catalog in the catalog store (`nnnotes catalogs fetch` / `import`), null when none is recorded | -| `catalog.sha256` | SHA-256 of the remote catalog file the charts were read with | -| `master.source` | `api` (`--master-files`) or `embedded` (`--apk-master`) | -| `master.version` | the `version` of the master data manifest | -| `master.tables.
.sha256` | SHA-256 of the table's file as served, before decoding | -| `exporter.name`, `exporter.version` | `nnnotes` and its version | -| `exporter.chartFormat` | the format of the chart converter the notes come from, `nnnotes.live-score/1` | - -### master - -Each table is `{"columns": [...], "rows": [[...], ...]}`: one array per row, its values in the order of `columns`, -rows in the order the master data lists them. Values are as decoded: integers, strings, booleans, arrays. A number -the master data writes with a fraction or an exponent is a binary32 (single precision) value in the game; it is -written as the shortest decimal that reads back as the same binary32 value, so a reader should parse such columns -as 32-bit floats. Infinity is written `1e999` / `-1e999`. - -Tables with a column list export those columns, and every row must have them. Tables marked "all" export every -column their rows have, in the order the rows first have them; when such a table has no rows, `columns` is empty -(the master data carries no field list for an empty table). A reader treats a table missing from `master` as empty. - -| Table | Columns | -|---|---| -| `MasterMemberCard` | `_id` `_characterID` `_rarity` `_cardType` `_bestMusicTagIDs` `_performancePowerMax` `_technicPowerMax` `_visualPowerMax` `_memberCardLevelGroup` `_memberCardAwakeGroup` `_memberCardRankGroup` `_leaderSkillID` `_liveSkillID` `_gekisouSkillID` | -| `MasterMemberCardLevel` | `_id` `_group` `_level` `_exp` `_performanceRate` `_technicRate` `_visualRate` | -| `MasterMemberCardLevelLimit` | `_id` `_rarity` `_awakeCount` `_limitLevel` | -| `MasterMemberCardAwake` | `_id` `_group` `_awakeCount` `_performanceRate` `_technicRate` `_visualRate` | -| `MasterMemberCardRank` | `_id` `_group` `_rank` `_performanceRate` `_technicRate` `_visualRate` `_leaderSkillLevel` `_musicTypeBonusRate` `_musicTagBonusRate` | -| `MasterSupportCard` | `_id` `_characterIDs` `_rarity` `_cardType` `_performancePowerMax` `_technicPowerMax` `_visualPowerMax` `_supportCardLevelGroup` `_supportCardRankGroup` `_supportSkillId01` `_supportSkillId02` `_gekisouSupportSkillId01` `_gekisouSupportSkillId02` | -| `MasterSupportCardLevel` | `_id` `_group` `_level` `_exp` `_performanceRate` `_technicRate` `_visualRate` | -| `MasterSupportCardRank` | `_id` `_group` `_rank` `_limitLevel` `_cardTypeLinkBonusRate` `_supportSkill01Level` `_supportSkill02Level` `_gekisouSupportSkill01Level` `_gekisouSupportSkill02Level` | -| `MasterCharacter` | `_id` `_bandID` | -| `MasterBand` | `_id` | -| `MasterCharacterRank` | `_id` `_rank` `_bonus` | -| `MasterCharacterTotalRank` | `_id` `_totalRank` `_bonus` | -| `MasterBandItemSkillEffect` | `_id` `_bandItemId` `_level` `_skillTargetIDs` `_skillEffectType` `_effectValue` | -| `MasterBandItem` | `_id` `_bandId` | -| `MasterBandItemLevel` | `_id` `_bandItemId` `_level` `_playerRank` | -| `MasterVipRankBonus` | `_id` `_vipRank` `_vipBonusType` `_value` | -| `MasterMemoryMusicGroup` | `_id` `_skillTargetIds` | -| `MasterMemoryMusic` | `_id` `_groupId` | -| `MasterMemoryMusicBonus` | `_id` `_groupId` `_scoreRank` `_performance` `_technic` `_visual` | -| `MasterMemoryMemberLevel` | `_id` `_point` `_performance` `_technic` `_visual` | -| `MasterMemorySupportLevel` | `_id` `_point` `_performance` `_technic` `_visual` | -| `MasterSkillTarget` | `_id` `_skillTargetType` `_characterID` `_bandID` `_cardType` `_tagID` `_judgement` `_liveMusicType` `_gekisouMissionType` `_liveSkillCategories` `_gekisouSkillCategories` | -| `MasterSkillCondition` | `_id` `_conditionType` `_conditionValues` `_isPositive` `_conditionTargetIDs` | -| `MasterSkillConditionSet` | `_id` `_group` `_conditionIds` | -| `MasterSkillCumulativeCondition` | `_id` `_skillCumulativeConditionType` `_conditionValues` `_conditionTargetIDs` `_maxCumulativeCount` | -| `MasterSkillEffectSetting` | `_id` `_skillEffectType` `_phase` | -| `MasterLeaderSkillEffect` | `_id` `_leaderSkillID` `_level` `_skillConditionGroup` `_skillTargetIDs` `_skillEffectType` `_effectValue` `_skillCumulativeConditionID` | -| `MasterLiveSkill` | `_id` `_skillCategories` | -| `MasterLiveSkillEffect` | `_id` `_liveSkillID` `_level` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | -| `MasterSupportSkill` | `_id` | -| `MasterSupportSkillEffect` | `_id` `_supportSkillID` `_level` `_skillTriggerConditionGroup` `_skillTriggerType` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | -| `MasterGekisouSkill` | `_id` `_gekisouMissionType` `_skillCategories` | -| `MasterGekisouSkillEffect` | `_id` `_gekisouSkillID` `_level` `_skillTriggerConditionGroup` `_skillTriggerType` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | -| `MasterGekisouSupportSkill` | `_id` `_gekisouSupportSkillExecTiming` `_gekisouMissionType` | -| `MasterGekisouSupportSkillEffect` | `_id` `_gekisouSupportSkillID` `_level` `_skillTriggerConditionGroup` `_skillTriggerType` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | -| `MasterLiveNoteParameter` | `_id` `_noteOperateType` `_scorePercent` | -| `MasterLiveJudgementParameter` | `_id` `_noteSimulateJudgement` `_scorePercent` `_damage` | -| `MasterLiveJudgementTiming` | `_id` `_assistLevel` `_judgementPriority` `_noteJudgementType` `_noteSimulateJudgement` `_beforeMs` `_afterMs` | -| `MasterLiveComboScoreBonus` | `_id` `_comboBonusType` `_requiredComboCount` `_bonusFactor` | -| `MasterLiveSettings` | all | -| `MasterParameter` | all | -| `MasterLiveGekisouLuckBasePoint` | `_id` `_noteCategory` `_noteSimulateJudgement` `_weight` `_basePoint` | -| `MasterLiveGekisouLuckBonusLot` | `_id` `_chanceLotType` `_lotResult` `_weight` | -| `MasterLiveGekisouRankingScoreBonus` | `_id` `_missionPattern` `_rank` `_count` `_scoreBonusPercent` | -| `MasterLiveMusic` | `_id` `_musicType` `_bestMusicTagIDs` `_liveScoreRankGroup` `_easyID` `_normalID` `_hardID` `_expertID` `_gekisouMission1` `_gekisouMission2` `_gekisouMission3` | -| `MasterLiveMusicScore` | `_id` `_musicScoreTextFileName` `_musicScoreLevel` `_fullComboCount` | -| `MasterArenaMusic` | all | -| `MasterChallengeMusic` | all | -| `MasterLiveScoreRank` | `_id` `_group` `_liveScoreRank` `_requiredScore` `_battleLiveRequiredScore` | -| `MasterEvent` | all | -| `MasterEventEffect` | all | -| `MasterLiveEventPoint` | all | -| `MasterChallengeLiveEventPoint` | all | -| `MasterLiveChallengePoint` | all | -| `MasterLiveMusicBoostBonus` | `_id` `_consumedLiveBoostCount` `_liveMusicRewardRate` `_playerExpRate` `_memberCardExpRate` `_friendshipExpRate` `_eventPointRate` | -| `MasterChallengeMusicBoostBonus` | all | - -### charts - -One chart per `MasterLiveMusicScore` row, sorted by `scoreId`. Song and score facts (level, full combo count, song -type, the song's difficulties) are not copied into the charts: they are in `MasterLiveMusicScore` and -`MasterLiveMusic`, joined by `scoreId` (`MasterLiveMusicScore._id`, and `MasterLiveMusic._easyID` .. `_expertID`). - -| Field | Content | -|---|---| -| `scoreId` | `MasterLiveMusicScore._id` | -| `asset.key` | `Live/MusicScore/<_musicScoreTextFileName>`, the chart's TextAsset | -| `asset.sha256` | SHA-256 of the TextAsset's bytes as shipped | -| `notes.id` | note id, unique within the chart | -| `notes.op` | `NoteOperateType` | -| `notes.judgementType` | `NoteJudgementType` (from the operate type and the critical flag) | -| `notes.timeMs` | note time in chart milliseconds | -| `skillEvents.timeMs` | skill event times in chart order; the position in the array is the event index (the times need not ascend) | -| `fevers.startMs`, `fevers.endMs` | fever ranges sorted by start; the position in the arrays is the range index | - -The notes are columns: `notes.id[i]`, `notes.op[i]`, `notes.judgementType[i]` and `notes.timeMs[i]` describe the -same note, and the four arrays have the same length; `fevers.startMs[i]` and `fevers.endMs[i]` are one range. The -notes are every note the client creates at runtime, including hidden notes, guide notes, and the combo ticks of -slides (operate types Combo and ComboSkip), in the order the client enumerates its note dictionary. That order is -not the time order: the combo ticks of a slide that fall on a time no other note has come after the slide's end. -Order by `timeMs` (then `id`) where time order is needed. - -Readers can derive further counts from these fields (for example the number of judged notes from `notes.op`); the -file does not contain derived values. - -## Versions - -`format` names the major version. Within `nnnotes.deck-data/1`, fields and tables are only added, never renamed, -removed or given another meaning, and readers ignore keys they do not know. A reader rejects a file with a major -version it does not know. A change that breaks readers is a new major version, `nnnotes.deck-data/2`. diff --git a/docs/music-data.md b/docs/music-data.md new file mode 100644 index 0000000..3fa9c84 --- /dev/null +++ b/docs/music-data.md @@ -0,0 +1,323 @@ +# Music data + +`nnnotes music-data` writes one JSON file with every live song and chart of one master data version: what a song +listing shows (titles and credits in every language, bands, vocal characters, category, tags, release time, score +ranks, jacket, BGM length), per difficulty the chart facts (level, note counts, BPM, chart times, skill events, +fevers), and per chart its **deck statistics**: what the chart contributes to the live score whatever the deck, +measured by the deck model [ournotes-deck](https://github.com/empty-sekai/ournotes-deck), which nnnotes carries as +its extension module `nnnotes._deck`. The format is `nnnotes.music-data/1`; its JSON Schema is +[schema/music-data.schema.json](schema/music-data.schema.json). + +``` +nnnotes music-data (--master-files DIR | --apk-master) [--full] [--no-deck] [--seeds N] [--workers N] + [--no-bgm] [--jackets DIR] -o FILE +``` + +- `--master-files DIR`: master data files as served, `DIR/MasterManifest.json` and the `.bin` files it lists + (`nnnotes master download`). The file's `region` is `[catalog] region` (`--region`). +- `--apk-master`: the master data files the APK ships (`assets/Master/` of `[paths] apk`, the same layout). The + file's `region` is `embedded`. +- `--full`: also write the deck model's input, every chart's runtime notes and the master data tables about cards, + skills, bonuses, scores and events ([the deck input](#the-deck-input---full)), for tools that run a deck model of + their own. +- `--no-deck`: do not run the deck model; every chart's `deck` is null. The deck model measures every chart on a + whole-live simulation, dozens of lives per chart: a full run takes processor time in proportion to the number of + charts. `--workers N` sets the threads it uses (default: every processor), `--seeds N` the seeds measured on a chart + with a luck range (default 8). +- `--no-bgm`: do not read the cue sheets (every `bgm.length` is null). +- `--jackets DIR`: also write every song's jacket, the Texture2D `Image/Jacket/`, as `DIR/.webp` + (WebP quality 88, scaled down with Lanczos to at most 320 pixels on the longer side, without alpha when opaque); a + page next to the file finds a song's jacket at `/.webp`. + +Each master data file is checked against the SHA-256 the manifest lists and decoded with `[master] key` and `iv`. +Charts are read from the catalog of `[catalog] language` (bundles fetched into the cache as for every command); the +BGM length from the cue sheet's ACB (its `CueTable` and `WaveformTable`, no audio is decoded). `FILE` ending in `.gz` +is written gzip-compressed. The command prints a summary (`songs`, `charts`, `deck`: the deck model's commit, +`unplayable`, `bytes`, `sha256` of the file, ...). + +The command writes the file only when every table, chart and cue sheet was read and every chart measured: a missing +or mismatching master data file, a table without a column the file exports, a text id that `MasterText` does not +have, a score id that `MasterLiveMusicScore` does not have, a missing or unreadable chart asset, a note id that +occurs twice in a chart, a cue sheet without the song's cue, (with `--jackets`) a missing jacket texture, a chart the +deck model cannot measure or whose check deck fails, or deck statistics that disagree with the chart facts stops it +with exit status 1 and a line naming the input. The file is written through a temporary file and a rename. + +An installation without the extension module (a source checkout that was not built) runs only with `--no-deck`; +the wheels on PyPI carry it. See [Building](#building). + +## Layout + +```json +{ + "format": "nnnotes.music-data/1", + "provenance": {"region": ..., "client": {...}, "catalog": {...}, "master": {...}, "exporter": {...}, "deck": {...}}, + "languages": ["ja", "en", "zh-Hant", "zh-Hans", "ko"], + "bands": [...], "characters": [...], "tags": [...], "categories": [...], + "deck": {"model": {...}, "kinds": [...]}, + "songs": [{"id": 100001, ..., "charts": [{..., "deck": {...}}, ...]}, ...], + "master": {...}, "charts": [...] +} +``` + +`master` and `charts` are present with `--full` only. The file is minified UTF-8 with one line feed at the end. Keys +are always in the order shown here and in the tables below, so the same inputs with the same nnnotes version (and so +the same deck model) give the same bytes. Master data values that are binary32 (single precision) in the game, a +number the master data writes with a fraction or an exponent, are written as the shortest decimal that reads back as +the same binary32 value (infinity as `1e999` / `-1e999`); the deck model's numbers are binary64 and written as it +writes them. A gzip file has no file name and a zero modification time in its header. + +A **text** is an object with one string per language of `languages` (`{"ja": ..., "en": ..., "zh-Hant": ..., +"zh-Hans": ..., "ko": ...}`), the `MasterText` row of the id; a text field is null when the master data gives no id +(an empty string). A language's string may be empty when the game has no text in that language. + +### provenance + +| Field | Content | +|---|---| +| `region` | the region whose master data this is (a configured region name), or `embedded` for the APK's master data | +| `client.versionName`, `client.versionCode` | the APK's version name and code (null without `[paths] apk`) | +| `catalog.resourceVersion` | the resource version recorded for the catalog in the catalog store (`nnnotes catalogs fetch` / `import`), null when none is recorded | +| `catalog.sha256` | SHA-256 of the remote catalog file the charts were read with | +| `master.source` | `api` (`--master-files`) or `embedded` (`--apk-master`) | +| `master.version` | the `version` of the master data manifest | +| `master.tables.
.sha256` | SHA-256 of each table's file as served, before decoding: the song tables (`MasterLiveMusic`, `MasterLiveMusicScore`, `MasterText`, `MasterBand`, `MasterCharacter`, `MasterTag`, `MasterLiveMusicCategory`, `MasterSound`, `MasterSoundCueSheet`, `MasterLiveScoreRank`) and, when the deck model runs or with `--full`, the tables of [the deck input](#the-deck-input---full) | +| `exporter.name`, `exporter.version` | `nnnotes` and its version | +| `exporter.chartFormat` | the format of the chart converter the notes come from, `nnnotes.live-score/1` | +| `deck` | the deck model: `{name, version, source, commit, format}`, `ournotes-deck`, its package version, repository, the git commit nnnotes is built with and the statistics format (`ournotes-deck.chart-stats/2`); null with `--no-deck` | + +### bands, characters, tags, categories + +| Field | Content | +|---|---| +| `bands[].id`, `.name`, `.mainColor`, `.subColor` | `MasterBand`: id, name text, color codes | +| `characters[].id`, `.bandId`, `.name`, `.shortName`, `.mainColor` | `MasterCharacter` | +| `tags[].id`, `.name` | `MasterTag` (the ids of `bestMusicTagIds`) | +| `categories[].id`, `.musicCategories`, `.name` | `MasterLiveMusicCategory` (the listing's category tabs; `musicCategories` are the song category values it shows) | + +### songs + +One entry per `MasterLiveMusic` row, sorted by `id`. + +| Field | Content | +|---|---| +| `id`, `sortOrder`, `startAt`, `defaultUnlock` | the row's `_id`, `_sortOrder`, `_startAt` (as served, server time), `_defaultUnlock` | +| `title`, `ruby`, `phonetic` | texts of `_titleTextID`, `_rubyTitleTextID`, `_phoneticTextID` | +| `bandIds`, `bandName` | `_bandIDs`; the text of `_bandNameTextID`, a name the song shows instead of its first band's (null: none) | +| `vocalCharacterIds` | `_vocalCharacterIDs` | +| `lyricist`, `composer`, `arranger` | texts of `_lyricistTextID`, `_composerTextID`, `_arrangerTextID` | +| `musicType`, `musicCategories`, `bestMusicTagIds` | `_musicType` (the song type that card type bonuses match), `_musicCategories`, `_bestMusicTagIDs` | +| `jacket` | `_jacketAssetName` (the jacket sprite `Image/Jacket/`) | +| `gekisouMissions` | `[_gekisouMission1, _gekisouMission2, _gekisouMission3]`: the missions of the song's Gekisou ranges (1 combo, 2 luck, 3 Just count) | +| `bgm.soundId`, `.cueSheet`, `.cue` | `_musicSoundID` and its `MasterSound` / `MasterSoundCueSheet` cue | +| `bgm.length` | `{lengthMs, samples, sampleRate, durationMs}`: the cue's `Length` in the ACB `CueTable`, the sample count and rate of its first waveform, and `samples * 1000 // sampleRate`; null with `--no-bgm` | +| `scoreRanks` | `[{rank, requiredScore, battleRequiredScore}]`: the `MasterLiveScoreRank` rows of `_liveScoreRankGroup` in required-score order (rank `E` .. `SS`; `_requiredScore`, `_battleLiveRequiredScore`). The group is the song's, so every difficulty shares these thresholds; a live's rank is the last row whose required score its score reaches | +| `charts` | one entry per difficulty the song has (`_easyID` .. `_expertID` not 0), in the order easy, normal, hard, expert | +| `master.MasterLiveMusic`, `master.MasterLiveScoreRank` | the whole `MasterLiveMusic` row and its score rank rows as decoded | + +### charts + +| Field | Content | +|---|---| +| `difficulty` | `easy`, `normal`, `hard` or `expert` | +| `scoreId`, `level`, `displayLevel`, `fullComboCount` | `MasterLiveMusicScore`: `_id`, `_musicScoreLevel`, `_musicScoreDisplayLevel`, `_fullComboCount` | +| `asset.key`, `asset.sha256` | the chart TextAsset `Live/MusicScore/<_musicScoreTextFileName>` and the SHA-256 of its bytes as shipped | +| `notes.judged` | notes that are judged (and count for a full combo) | +| `notes.total` | every runtime note, including hidden notes, guide notes and slide combo ticks | +| `notes.byOperateType` | `{"": count}` over every runtime note, keys in ascending order | +| `bpm.changes` | every BPM change `{timeMs, bpm}` in time order | +| `bpm.main`, `.min`, `.max` | over the played span (first to last judged note): the BPM that holds longest (the earliest on a tie), the lowest and the highest | +| `firstNoteMs`, `lastJudgedNoteMs` | times of the first and the last judged note | +| `lastNoteMs` | the latest time of any runtime note (the time the score code calls the last timing note) | +| `musicLengthMs` | `lastNoteMs + 1000`: the live's music length on the game's score path (skill effects end at it at the latest) | +| `skillEventsMs` | skill event times in chart order; event `i` fires the skill of the member at performance position `i` | +| `fevers` | fever ranges `[startMs, endMs]` sorted by start | +| `deck` | the chart's deck statistics (below); null with `--no-deck` | + +Chart times are milliseconds of chart time. The two lengths are different facts: `bgm.length` is how long the music +plays, `musicLengthMs` is the length the score code uses; a listing chooses the one it needs. + +## Deck statistics + +The deck model plays every chart of a song on its whole-live simulation, with Gekisou on as the game plays every +live, solo (rank 1), in the theoretical best play: every judged note at its exact time, Just inside the Just-count +ranges and Perfect elsewhere, the frame times of the game's default schedule. It measures, at deck power +`model.power` (300000): + +- `score`: the exact score without skills, the rank bonuses of the Gekisou ranges included; +- for every **score-up kind** (`deck.kinds`) and every performance position `k`, `weights[kind][k]`: the exact score + a deck gains when its position-`k` member has one effect of that kind at factor 1, divided by the deck power. The + effect runs through the simulation's own updaters, conditions, frames and appliers, so the weight carries every rule + of the game (execute and finish frames, the 40 ms score frames, combo and Gekisou combo factors, Just scores, luck + rushes, the rank bonuses of the ranges it overlaps). + +A deck whose live skills are all score-up kinds then scores, up to the floors, + +``` +P * (score / power + sum over positions k of factor_k * weights[kind_k][k]) +``` + +with `P` the deck power and `factor_k` the effect's factor as the game's applier converts its value: effect type 2000 +`floor(value / 10000f * 1e5) / 1e5`, 2005 `floor(value / -10000f * 1e5) / 1e5`, 2002 and 2004 the same quotient +rounded half to even (`value / 10000f` in binary32). Each seed also plays a **check deck**, random kinds at real +master values at another power (`model.checkPower`), and the command fails when its exact score leaves the bound of +the formula. Effects of other types (cumulative score 2001 / 2003, life, judgement conversion, Gekisou and snap +skills) are not linear in the chart alone and have no weights: a deck's score with them comes from the simulation. + +A luck range draws lottery results from the play's random seed, so the measurements are given **per seed**: one seed +(0) when no range is a luck range, else the first `--seeds` seeds of the deck model's published seed set. The seed +set is not the game's seed law (which is unknown); a mean over it is not the game's expectation. + +### deck + +| Field | Content | +|---|---| +| `model` | the deck model's description of the measurement: `engine`, `play`, `score` (the formula), `power`, `checkPower`, `unitValue` (the effect value of factor 1, 10000), `seeds` | +| `kinds[]` | the score-up kinds of the master data: `MasterLiveSkillEffect` rows of type 2000, 2002, 2004 or 2005 without a cumulative condition, grouped by what shapes their score. `id` (the index in `weights`), `effectType`, `activationTimeSecond`, `durationMs` (`ceil(activationTimeSecond * 1000f)`), `skillTargetIds`, `skillConditionGroup`, `skillReleaseConditionGroup`, `effectLimitCount`, `effectExecuteLimitCount`, `effectExecuteLimitResetConditionGroup`; `rows` (master rows of the kind) and `values` (their distinct `_effectValue`s, ascending) | + +The kind of a card's live skill is found by matching its `MasterLiveSkillEffect` row (at the skill level) on these +fields; `values` lists what the master data uses. + +### charts[].deck + +| Field | Content | +|---|---| +| `convertedNoteCount` | the note count the score formula divides by (converted notes) | +| `skip` | score per unit of deck power of a skipped live (every note Great, combo 0, no skills) | +| `events` | `[[position, timeMs], ...]`: the skill events in chart order with the performance position each fires | +| `positions` | the performance positions the events fire (the largest position + 1): the length of every `weights[kind]` | +| `ranges[]` | the Gekisou ranges: `index`, `mission` (1 combo, 2 luck, 3 Just count), `startMs`, `endMs`, `rankBonusPercent` (the solo rank bonus percentage of the song's mission pattern) | +| `justNotes` | notes judged Just on the play | +| `seeds[]` | per seed: `seed`; `score`; `ranges[]` (`rangeScore`: the score gained inside the range, `rankBonus`, `maxCombo`, `justCount`, `lotResults`: lottery results Miss, Hit, Super Hit, Critical); `weights[kind][position]`; `check` (`deck`: `[kind, value]` or null per position, `exact`, `predicted`, `bound`) | +| `unplayable` | null, or why the game cannot play the chart (more than three fevers: the game fails when the fourth starts); `seeds` is then empty | + +The deck model's chart facts are checked against the file's: the song, difficulty, level, judged note count, last +note time, music length, Gekisou missions, skill event times and fever ranges must agree, and are not repeated in +`deck`. + +## The deck input (`--full`) + +With `--full` the file ends with the input the deck model reads: `master`, the master data tables about cards, +skills, bonuses, scores and events, and `charts`, every chart as the client builds it at runtime. They hold facts as +the game has them (master data values as served, notes as the client's chart converter creates them), not values +derived from them. + +### master + +Each table is `{"columns": [...], "rows": [[...], ...]}`: one array per row, its values in the order of `columns`, +rows in the order the master data lists them. Values are as decoded: integers, strings, booleans, arrays; a binary32 +column should be parsed as 32-bit floats. + +Tables with a column list export those columns, and every row must have them. Tables marked "all" export every +column their rows have, in the order the rows first have them; when such a table has no rows, `columns` is empty +(the master data carries no field list for an empty table). A reader treats a table missing from `master` as empty. + +| Table | Columns | +|---|---| +| `MasterMemberCard` | `_id` `_characterID` `_rarity` `_cardType` `_bestMusicTagIDs` `_performancePowerMax` `_technicPowerMax` `_visualPowerMax` `_memberCardLevelGroup` `_memberCardAwakeGroup` `_memberCardRankGroup` `_leaderSkillID` `_liveSkillID` `_gekisouSkillID` | +| `MasterMemberCardLevel` | `_id` `_group` `_level` `_exp` `_performanceRate` `_technicRate` `_visualRate` | +| `MasterMemberCardLevelLimit` | `_id` `_rarity` `_awakeCount` `_limitLevel` | +| `MasterMemberCardAwake` | `_id` `_group` `_awakeCount` `_performanceRate` `_technicRate` `_visualRate` | +| `MasterMemberCardRank` | `_id` `_group` `_rank` `_performanceRate` `_technicRate` `_visualRate` `_leaderSkillLevel` `_musicTypeBonusRate` `_musicTagBonusRate` | +| `MasterSupportCard` | `_id` `_characterIDs` `_rarity` `_cardType` `_performancePowerMax` `_technicPowerMax` `_visualPowerMax` `_supportCardLevelGroup` `_supportCardRankGroup` `_supportSkillId01` `_supportSkillId02` `_gekisouSupportSkillId01` `_gekisouSupportSkillId02` | +| `MasterSupportCardLevel` | `_id` `_group` `_level` `_exp` `_performanceRate` `_technicRate` `_visualRate` | +| `MasterSupportCardRank` | `_id` `_group` `_rank` `_limitLevel` `_cardTypeLinkBonusRate` `_supportSkill01Level` `_supportSkill02Level` `_gekisouSupportSkill01Level` `_gekisouSupportSkill02Level` | +| `MasterCharacter` | `_id` `_bandID` | +| `MasterBand` | `_id` | +| `MasterCharacterRank` | `_id` `_rank` `_bonus` | +| `MasterCharacterTotalRank` | `_id` `_totalRank` `_bonus` | +| `MasterBandItemSkillEffect` | `_id` `_bandItemId` `_level` `_skillTargetIDs` `_skillEffectType` `_effectValue` | +| `MasterBandItem` | `_id` `_bandId` | +| `MasterBandItemLevel` | `_id` `_bandItemId` `_level` `_playerRank` | +| `MasterVipRankBonus` | `_id` `_vipRank` `_vipBonusType` `_value` | +| `MasterMemoryMusicGroup` | `_id` `_skillTargetIds` | +| `MasterMemoryMusic` | `_id` `_groupId` | +| `MasterMemoryMusicBonus` | `_id` `_groupId` `_scoreRank` `_performance` `_technic` `_visual` | +| `MasterMemoryMemberLevel` | `_id` `_point` `_performance` `_technic` `_visual` | +| `MasterMemorySupportLevel` | `_id` `_point` `_performance` `_technic` `_visual` | +| `MasterSkillTarget` | `_id` `_skillTargetType` `_characterID` `_bandID` `_cardType` `_tagID` `_judgement` `_liveMusicType` `_gekisouMissionType` `_liveSkillCategories` `_gekisouSkillCategories` | +| `MasterSkillCondition` | `_id` `_conditionType` `_conditionValues` `_isPositive` `_conditionTargetIDs` | +| `MasterSkillConditionSet` | `_id` `_group` `_conditionIds` | +| `MasterSkillCumulativeCondition` | `_id` `_skillCumulativeConditionType` `_conditionValues` `_conditionTargetIDs` `_maxCumulativeCount` | +| `MasterSkillEffectSetting` | `_id` `_skillEffectType` `_phase` | +| `MasterLeaderSkillEffect` | `_id` `_leaderSkillID` `_level` `_skillConditionGroup` `_skillTargetIDs` `_skillEffectType` `_effectValue` `_skillCumulativeConditionID` | +| `MasterLiveSkill` | `_id` `_skillCategories` | +| `MasterLiveSkillEffect` | `_id` `_liveSkillID` `_level` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | +| `MasterSupportSkill` | `_id` | +| `MasterSupportSkillEffect` | `_id` `_supportSkillID` `_level` `_skillTriggerConditionGroup` `_skillTriggerType` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | +| `MasterGekisouSkill` | `_id` `_gekisouMissionType` `_skillCategories` | +| `MasterGekisouSkillEffect` | `_id` `_gekisouSkillID` `_level` `_skillTriggerConditionGroup` `_skillTriggerType` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | +| `MasterGekisouSupportSkill` | `_id` `_gekisouSupportSkillExecTiming` `_gekisouMissionType` | +| `MasterGekisouSupportSkillEffect` | `_id` `_gekisouSupportSkillID` `_level` `_skillTriggerConditionGroup` `_skillTriggerType` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | +| `MasterLiveNoteParameter` | `_id` `_noteOperateType` `_scorePercent` | +| `MasterLiveJudgementParameter` | `_id` `_noteSimulateJudgement` `_scorePercent` `_damage` | +| `MasterLiveJudgementTiming` | `_id` `_assistLevel` `_judgementPriority` `_noteJudgementType` `_noteSimulateJudgement` `_beforeMs` `_afterMs` | +| `MasterLiveComboScoreBonus` | `_id` `_comboBonusType` `_requiredComboCount` `_bonusFactor` | +| `MasterLiveSettings` | all | +| `MasterParameter` | all | +| `MasterLiveGekisouLuckBasePoint` | `_id` `_noteCategory` `_noteSimulateJudgement` `_weight` `_basePoint` | +| `MasterLiveGekisouLuckBonusLot` | `_id` `_chanceLotType` `_lotResult` `_weight` | +| `MasterLiveGekisouRankingScoreBonus` | `_id` `_missionPattern` `_rank` `_count` `_scoreBonusPercent` | +| `MasterLiveMusic` | `_id` `_musicType` `_bestMusicTagIDs` `_liveScoreRankGroup` `_easyID` `_normalID` `_hardID` `_expertID` `_gekisouMission1` `_gekisouMission2` `_gekisouMission3` | +| `MasterLiveMusicScore` | `_id` `_musicScoreTextFileName` `_musicScoreLevel` `_fullComboCount` | +| `MasterArenaMusic` | all | +| `MasterChallengeMusic` | all | +| `MasterLiveScoreRank` | `_id` `_group` `_liveScoreRank` `_requiredScore` `_battleLiveRequiredScore` | +| `MasterEvent` | all | +| `MasterEventEffect` | all | +| `MasterEventAchievementReward` | `_id` `_eventId` `_eventPoint` `_rewardIds` | +| `MasterEventAchievementLoopReward` | `_id` `_eventId` `_loopStartEventPoint` `_loopEventPoint` `_rewardIds` | +| `MasterLiveEventReward` | `_id` `_group` `_eventGroup` `_scoreRank` `_resourceType` `_resourceId` `_resourceCount` `_probability` | +| `MasterChallengeLiveEventReward` | `_id` `_group` `_eventGroup` `_scoreRank` `_resourceType` `_resourceId` `_resourceCount` `_probability` | +| `MasterLiveEventPoint` | all | +| `MasterChallengeLiveEventPoint` | all | +| `MasterLiveChallengePoint` | all | +| `MasterLiveMusicBoostBonus` | `_id` `_consumedLiveBoostCount` `_liveMusicRewardRate` `_playerExpRate` `_memberCardExpRate` `_friendshipExpRate` `_eventPointRate` | +| `MasterChallengeMusicBoostBonus` | all | + +### charts + +One chart per `MasterLiveMusicScore` row, sorted by `scoreId`, including charts of no song. Song and score facts +are not copied into these records: they are in `songs` and in `master`, joined by `scoreId`. + +| Field | Content | +|---|---| +| `scoreId` | `MasterLiveMusicScore._id` | +| `asset.key` | `Live/MusicScore/<_musicScoreTextFileName>`, the chart's TextAsset | +| `asset.sha256` | SHA-256 of the TextAsset's bytes as shipped | +| `notes.id` | note id, unique within the chart | +| `notes.op` | `NoteOperateType` | +| `notes.judgementType` | `NoteJudgementType` (from the operate type and the critical flag) | +| `notes.timeMs` | note time in chart milliseconds | +| `skillEvents.timeMs` | skill event times in chart order; the position in the array is the event index (the times need not ascend) | +| `fevers.startMs`, `fevers.endMs` | fever ranges sorted by start; the position in the arrays is the range index | + +The notes are columns: `notes.id[i]`, `notes.op[i]`, `notes.judgementType[i]` and `notes.timeMs[i]` describe the +same note, and the four arrays have the same length; `fevers.startMs[i]` and `fevers.endMs[i]` are one range. The +notes are every note the client creates at runtime, including hidden notes, guide notes, and the combo ticks of +slides (operate types Combo and ComboSkip), in the order the client enumerates its note dictionary. That order is +not the time order: the combo ticks of a slide that fall on a time no other note has come after the slide's end. +Order by `timeMs` (then `id`) where time order is needed. + +The deck model reads the same content under the format name `nnnotes.deck-data/1` (`format`, `provenance`, +`master`, `charts`); nnnotes hands it over in memory. + +## Building + +The deck model is the Rust crate ournotes-deck, pinned by commit in `rust/Cargo.toml` (and `rust/Cargo.lock`) and +built into the extension module `nnnotes._deck` with [maturin](https://www.maturin.rs/) (PyO3, the stable ABI of +Python 3.11 and later: one wheel per platform). The release workflow builds the wheels; `pip install .` or +`pip install -e .` in a checkout builds the module with the Rust toolchain. The same nnnotes version always carries +the same deck model: the commit moves only through a pull request (`.github/workflows/deck.yml` opens one when +ournotes-deck's `main` moves), and `provenance.deck.commit` names it in every file. + +## Versions + +`format` names the major version. Within `nnnotes.music-data/1`, fields and tables are only added, never renamed, +removed or given another meaning, and readers ignore keys they do not know. A reader rejects a file with a major +version it does not know. A change that breaks readers is a new major version, `nnnotes.music-data/2`. The deck +statistics follow the deck model's format (`provenance.deck.format`): a new major version of it is a new major +version of this file. + +`nnnotes.music-data/1` replaces the `nnnotes.songs/1` file of `nnnotes songs` (its fields are the songs, charts and +their facts here) and the `nnnotes.deck-data/1` file of `nnnotes deck-data` (its content is the deck input of +`--full`). diff --git a/docs/schema/deck-data.schema.json b/docs/schema/deck-data.schema.json deleted file mode 100644 index d56c1e5..0000000 --- a/docs/schema/deck-data.schema.json +++ /dev/null @@ -1,273 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:nnnotes:schema:deck-data", - "title": "nnnotes.deck-data/1", - "description": "Data for deck-building tools, for one master data version: every live chart as the client builds it at runtime (notes in the order the client enumerates them, skill events, fever ranges) and a subset of the master data tables as columns and rows, with the provenance of both. Written by `nnnotes deck-data`; described in docs/deck-data.md. Within the arrays of a chart's `notes` (and of its `fevers`) the i-th elements belong to the same note (range): the arrays have equal lengths, which a schema cannot state. Readers ignore keys they do not know.", - "type": "object", - "required": [ - "format", - "provenance", - "master", - "charts" - ], - "properties": { - "format": { - "const": "nnnotes.deck-data/1" - }, - "provenance": { - "type": "object", - "required": [ - "region", - "client", - "catalog", - "master", - "exporter" - ], - "properties": { - "region": { - "type": "string", - "description": "the region whose master data this is (a configured region name), or `embedded` for the master data the APK ships" - }, - "client": { - "type": "object", - "required": [ - "versionName", - "versionCode" - ], - "properties": { - "versionName": { - "type": [ - "string", - "null" - ], - "description": "versionName of the APK (null without one)" - }, - "versionCode": { - "type": [ - "integer", - "null" - ], - "description": "versionCode of the APK (null without one)" - } - } - }, - "catalog": { - "type": "object", - "required": [ - "resourceVersion", - "sha256" - ], - "properties": { - "resourceVersion": { - "type": [ - "string", - "null" - ], - "description": "the resource version recorded for the catalog in the catalog store, null when none is recorded" - }, - "sha256": { - "$ref": "#/$defs/sha256", - "description": "SHA-256 of the remote catalog file the charts were read with" - } - } - }, - "master": { - "type": "object", - "required": [ - "source", - "version", - "tables" - ], - "properties": { - "source": { - "enum": [ - "api", - "embedded" - ], - "description": "api: files as downloaded from the region; embedded: the files the APK ships" - }, - "version": { - "type": [ - "string", - "null" - ], - "description": "`version` of the master data manifest" - }, - "tables": { - "type": "object", - "description": "per exported table, the SHA-256 of its file as served (before decoding)", - "additionalProperties": { - "type": "object", - "required": [ - "sha256" - ], - "properties": { - "sha256": { - "$ref": "#/$defs/sha256" - } - } - } - } - } - }, - "exporter": { - "type": "object", - "required": [ - "name", - "version", - "chartFormat" - ], - "properties": { - "name": { - "const": "nnnotes" - }, - "version": { - "type": "string" - }, - "chartFormat": { - "type": "string", - "description": "format of the chart converter the notes come from (nnnotes.live-score/1)" - } - } - } - } - }, - "master": { - "type": "object", - "description": "tables by name; a table not present reads as empty", - "additionalProperties": { - "$ref": "#/$defs/table" - } - }, - "charts": { - "type": "array", - "description": "one chart per MasterLiveMusicScore row, sorted by scoreId", - "items": { - "$ref": "#/$defs/chart" - } - } - }, - "$defs": { - "sha256": { - "type": "string", - "pattern": "^[0-9a-f]{64}$" - }, - "ints": { - "type": "array", - "items": { - "type": "integer" - } - }, - "table": { - "type": "object", - "required": [ - "columns", - "rows" - ], - "properties": { - "columns": { - "type": "array", - "items": { - "type": "string" - }, - "uniqueItems": true - }, - "rows": { - "type": "array", - "items": { - "type": "array" - }, - "description": "one array per row, one value per column in the order of `columns`" - } - } - }, - "chart": { - "type": "object", - "required": [ - "scoreId", - "asset", - "notes", - "skillEvents", - "fevers" - ], - "properties": { - "scoreId": { - "type": "integer", - "description": "MasterLiveMusicScore._id" - }, - "asset": { - "type": "object", - "required": [ - "key", - "sha256" - ], - "properties": { - "key": { - "type": "string", - "description": "Live/MusicScore/<_musicScoreTextFileName>" - }, - "sha256": { - "$ref": "#/$defs/sha256", - "description": "SHA-256 of the chart TextAsset's bytes as shipped" - } - } - }, - "notes": { - "type": "object", - "required": [ - "id", - "op", - "judgementType", - "timeMs" - ], - "properties": { - "id": { - "$ref": "#/$defs/ints", - "description": "note id, unique within the chart" - }, - "op": { - "$ref": "#/$defs/ints", - "description": "NoteOperateType" - }, - "judgementType": { - "$ref": "#/$defs/ints", - "description": "NoteJudgementType" - }, - "timeMs": { - "$ref": "#/$defs/ints", - "description": "note time in chart milliseconds" - } - } - }, - "skillEvents": { - "type": "object", - "required": [ - "timeMs" - ], - "properties": { - "timeMs": { - "$ref": "#/$defs/ints", - "description": "skill event times in chart order (the index is the event index)" - } - } - }, - "fevers": { - "type": "object", - "required": [ - "startMs", - "endMs" - ], - "properties": { - "startMs": { - "$ref": "#/$defs/ints", - "description": "range starts, sorted (the index is the range index)" - }, - "endMs": { - "$ref": "#/$defs/ints" - } - } - } - } - } - } -} diff --git a/docs/schema/music-data.schema.json b/docs/schema/music-data.schema.json new file mode 100644 index 0000000..18dcba2 --- /dev/null +++ b/docs/schema/music-data.schema.json @@ -0,0 +1,1034 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nnnotes:schema:music-data", + "title": "nnnotes.music-data/1", + "description": "Every live song and chart of one master data version: song metadata in every language, chart facts and the deck model's chart statistics, with --full also the deck model's input (every chart's runtime notes and a subset of the master data tables as columns and rows). Written by `nnnotes music-data`; described in docs/music-data.md. Within the arrays of a deck input chart's `notes` (and of its `fevers`) the i-th elements belong to the same note (range): the arrays have equal lengths, which a schema cannot state. Readers ignore keys they do not know.", + "type": "object", + "required": [ + "format", + "provenance", + "languages", + "bands", + "characters", + "tags", + "categories", + "deck", + "songs" + ], + "properties": { + "format": { + "const": "nnnotes.music-data/1" + }, + "provenance": { + "type": "object", + "required": [ + "region", + "client", + "catalog", + "master", + "exporter", + "deck" + ], + "properties": { + "region": { + "type": "string", + "description": "the region whose master data this is (a configured region name), or `embedded` for the master data the APK ships" + }, + "client": { + "type": "object", + "required": [ + "versionName", + "versionCode" + ], + "properties": { + "versionName": { + "type": [ + "string", + "null" + ], + "description": "versionName of the APK (null without one)" + }, + "versionCode": { + "type": [ + "integer", + "null" + ], + "description": "versionCode of the APK (null without one)" + } + } + }, + "catalog": { + "type": "object", + "required": [ + "resourceVersion", + "sha256" + ], + "properties": { + "resourceVersion": { + "type": [ + "string", + "null" + ], + "description": "the resource version recorded for the catalog in the catalog store, null when none is recorded" + }, + "sha256": { + "$ref": "#/$defs/sha256", + "description": "SHA-256 of the remote catalog file the charts were read with" + } + } + }, + "master": { + "type": "object", + "required": [ + "source", + "version", + "tables" + ], + "properties": { + "source": { + "enum": [ + "api", + "embedded" + ], + "description": "api: files as downloaded from the region; embedded: the files the APK ships" + }, + "version": { + "type": [ + "string", + "null" + ], + "description": "`version` of the master data manifest" + }, + "tables": { + "type": "object", + "description": "per table read, the SHA-256 of its file as served (before decoding)", + "additionalProperties": { + "type": "object", + "required": [ + "sha256" + ], + "properties": { + "sha256": { + "$ref": "#/$defs/sha256" + } + } + } + } + } + }, + "exporter": { + "type": "object", + "required": [ + "name", + "version", + "chartFormat" + ], + "properties": { + "name": { + "const": "nnnotes" + }, + "version": { + "type": "string" + }, + "chartFormat": { + "type": "string", + "description": "format of the chart converter the notes come from (nnnotes.live-score/1)" + } + } + }, + "deck": { + "type": [ + "object", + "null" + ], + "description": "the deck model the statistics come from; null without it (--no-deck)", + "required": [ + "name", + "version", + "source", + "commit", + "format" + ], + "properties": { + "name": { + "const": "ournotes-deck" + }, + "version": { + "type": "string" + }, + "source": { + "type": "string", + "description": "the repository" + }, + "commit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "format": { + "type": "string", + "description": "the statistics format, ournotes-deck.chart-stats/2" + } + } + } + } + }, + "languages": { + "type": "array", + "items": { + "type": "string" + } + }, + "bands": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "integer" + }, + "name": { + "$ref": "#/$defs/text" + }, + "mainColor": { + "type": [ + "string", + "null" + ] + }, + "subColor": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "characters": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "integer" + }, + "name": { + "$ref": "#/$defs/text" + }, + "bandId": { + "type": [ + "integer", + "null" + ] + }, + "shortName": { + "$ref": "#/$defs/text" + }, + "mainColor": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "tags": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "integer" + }, + "name": { + "$ref": "#/$defs/text" + } + } + } + }, + "categories": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "integer" + }, + "name": { + "$ref": "#/$defs/text" + }, + "musicCategories": { + "$ref": "#/$defs/ints" + } + } + } + }, + "deck": { + "type": [ + "object", + "null" + ], + "required": [ + "model", + "kinds" + ], + "description": "the deck model's measurement and score-up kinds; null with --no-deck", + "properties": { + "model": { + "type": "object" + }, + "kinds": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "effectType" + ], + "properties": { + "id": { + "type": "integer" + }, + "effectType": { + "type": "integer" + } + } + } + } + } + }, + "songs": { + "type": "array", + "items": { + "$ref": "#/$defs/song" + } + }, + "master": { + "type": "object", + "description": "--full: the deck input's tables by name; a table not present reads as empty", + "additionalProperties": { + "$ref": "#/$defs/table" + } + }, + "charts": { + "type": "array", + "description": "--full: one chart per MasterLiveMusicScore row, sorted by scoreId", + "items": { + "$ref": "#/$defs/chart" + } + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "ints": { + "type": "array", + "items": { + "type": "integer" + } + }, + "table": { + "type": "object", + "required": [ + "columns", + "rows" + ], + "properties": { + "columns": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "rows": { + "type": "array", + "items": { + "type": "array" + }, + "description": "one array per row, one value per column in the order of `columns`" + } + } + }, + "chart": { + "type": "object", + "required": [ + "scoreId", + "asset", + "notes", + "skillEvents", + "fevers" + ], + "properties": { + "scoreId": { + "type": "integer", + "description": "MasterLiveMusicScore._id" + }, + "asset": { + "type": "object", + "required": [ + "key", + "sha256" + ], + "properties": { + "key": { + "type": "string", + "description": "Live/MusicScore/<_musicScoreTextFileName>" + }, + "sha256": { + "$ref": "#/$defs/sha256", + "description": "SHA-256 of the chart TextAsset's bytes as shipped" + } + } + }, + "notes": { + "type": "object", + "required": [ + "id", + "op", + "judgementType", + "timeMs" + ], + "properties": { + "id": { + "$ref": "#/$defs/ints", + "description": "note id, unique within the chart" + }, + "op": { + "$ref": "#/$defs/ints", + "description": "NoteOperateType" + }, + "judgementType": { + "$ref": "#/$defs/ints", + "description": "NoteJudgementType" + }, + "timeMs": { + "$ref": "#/$defs/ints", + "description": "note time in chart milliseconds" + } + } + }, + "skillEvents": { + "type": "object", + "required": [ + "timeMs" + ], + "properties": { + "timeMs": { + "$ref": "#/$defs/ints", + "description": "skill event times in chart order (the index is the event index)" + } + } + }, + "fevers": { + "type": "object", + "required": [ + "startMs", + "endMs" + ], + "properties": { + "startMs": { + "$ref": "#/$defs/ints", + "description": "range starts, sorted (the index is the range index)" + }, + "endMs": { + "$ref": "#/$defs/ints" + } + } + } + } + }, + "text": { + "type": [ + "object", + "null" + ], + "description": "one string per language of `languages`; null: no text id", + "required": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ], + "additionalProperties": { + "type": [ + "string", + "null" + ] + } + }, + "deckChart": { + "type": [ + "object", + "null" + ], + "description": "the chart's deck statistics (docs/music-data.md, Deck statistics); null with --no-deck", + "required": [ + "convertedNoteCount", + "skip", + "events", + "positions", + "ranges", + "justNotes", + "seeds", + "unplayable" + ], + "properties": { + "convertedNoteCount": { + "type": "integer" + }, + "skip": { + "type": "number" + }, + "events": { + "type": "array", + "items": { + "type": "array", + "prefixItems": [ + { + "type": "integer" + }, + { + "type": "integer" + } + ], + "minItems": 2, + "maxItems": 2 + }, + "description": "[performance position, time ms] per skill event, in chart order" + }, + "positions": { + "type": "integer", + "minimum": 0 + }, + "ranges": { + "type": "array", + "items": { + "type": "object", + "required": [ + "index", + "mission", + "startMs", + "endMs", + "rankBonusPercent" + ], + "properties": { + "index": { + "type": "integer" + }, + "mission": { + "type": "integer" + }, + "startMs": { + "type": "integer" + }, + "endMs": { + "type": "integer" + }, + "rankBonusPercent": { + "type": "integer" + } + } + } + }, + "justNotes": { + "type": "integer" + }, + "seeds": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seed", + "score", + "ranges", + "weights", + "check" + ], + "properties": { + "seed": { + "type": "integer" + }, + "score": { + "type": "integer", + "description": "the exact no-skill score at model.power" + }, + "ranges": { + "type": "array", + "items": { + "type": "object", + "required": [ + "rangeScore", + "rankBonus", + "maxCombo", + "justCount", + "lotResults" + ], + "properties": { + "rangeScore": { + "type": "integer" + }, + "rankBonus": { + "type": "integer" + }, + "maxCombo": { + "type": "integer" + }, + "justCount": { + "type": "integer" + }, + "lotResults": { + "type": "array", + "items": { + "type": "integer" + }, + "minItems": 4, + "maxItems": 4 + } + } + } + }, + "weights": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + }, + "description": "weights[kind][position]: score per unit of deck power and of factor" + }, + "check": { + "type": "object", + "required": [ + "deck", + "exact", + "predicted", + "bound" + ], + "properties": { + "deck": { + "type": "array", + "items": { + "type": [ + "array", + "null" + ] + } + }, + "exact": { + "type": "integer" + }, + "predicted": { + "type": "number" + }, + "bound": { + "type": "number" + } + } + } + } + } + }, + "unplayable": { + "type": [ + "string", + "null" + ] + } + } + }, + "songChart": { + "type": "object", + "required": [ + "difficulty", + "scoreId", + "level", + "displayLevel", + "fullComboCount", + "asset", + "notes", + "bpm", + "firstNoteMs", + "lastJudgedNoteMs", + "lastNoteMs", + "musicLengthMs", + "skillEventsMs", + "fevers", + "deck" + ], + "properties": { + "difficulty": { + "enum": [ + "easy", + "normal", + "hard", + "expert" + ] + }, + "scoreId": { + "type": "integer" + }, + "level": { + "type": "integer" + }, + "displayLevel": { + "type": [ + "number", + "null" + ] + }, + "fullComboCount": { + "type": "integer" + }, + "asset": { + "type": "object", + "required": [ + "key", + "sha256" + ], + "properties": { + "key": { + "type": "string", + "description": "Live/MusicScore/<_musicScoreTextFileName>" + }, + "sha256": { + "$ref": "#/$defs/sha256", + "description": "SHA-256 of the chart TextAsset's bytes as shipped" + } + } + }, + "notes": { + "type": "object", + "required": [ + "judged", + "total", + "byOperateType" + ], + "properties": { + "judged": { + "type": "integer" + }, + "total": { + "type": "integer" + }, + "byOperateType": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + } + } + }, + "bpm": { + "type": "object", + "required": [ + "main", + "min", + "max", + "changes" + ], + "properties": { + "main": { + "type": "number" + }, + "min": { + "type": "number" + }, + "max": { + "type": "number" + }, + "changes": { + "type": "array", + "items": { + "type": "object", + "required": [ + "timeMs", + "bpm" + ], + "properties": { + "timeMs": { + "type": "integer" + }, + "bpm": { + "type": "number" + } + } + } + } + } + }, + "firstNoteMs": { + "type": "integer" + }, + "lastJudgedNoteMs": { + "type": "integer" + }, + "lastNoteMs": { + "type": "integer" + }, + "musicLengthMs": { + "type": "integer" + }, + "skillEventsMs": { + "$ref": "#/$defs/ints" + }, + "fevers": { + "type": "array", + "items": { + "$ref": "#/$defs/ints", + "minItems": 2, + "maxItems": 2 + } + }, + "deck": { + "$ref": "#/$defs/deckChart" + } + } + }, + "song": { + "type": "object", + "required": [ + "id", + "sortOrder", + "startAt", + "defaultUnlock", + "title", + "ruby", + "phonetic", + "bandIds", + "bandName", + "vocalCharacterIds", + "lyricist", + "composer", + "arranger", + "musicType", + "musicCategories", + "bestMusicTagIds", + "jacket", + "gekisouMissions", + "bgm", + "scoreRanks", + "charts", + "master" + ], + "properties": { + "id": { + "type": "integer" + }, + "sortOrder": { + "type": [ + "integer", + "null" + ] + }, + "startAt": { + "type": [ + "string", + "null" + ] + }, + "defaultUnlock": { + "type": [ + "boolean", + "null" + ] + }, + "title": { + "$ref": "#/$defs/text" + }, + "ruby": { + "$ref": "#/$defs/text" + }, + "phonetic": { + "$ref": "#/$defs/text" + }, + "bandName": { + "$ref": "#/$defs/text" + }, + "lyricist": { + "$ref": "#/$defs/text" + }, + "composer": { + "$ref": "#/$defs/text" + }, + "arranger": { + "$ref": "#/$defs/text" + }, + "bandIds": { + "$ref": "#/$defs/ints" + }, + "vocalCharacterIds": { + "$ref": "#/$defs/ints" + }, + "musicType": { + "type": [ + "integer", + "null" + ] + }, + "musicCategories": { + "$ref": "#/$defs/ints" + }, + "bestMusicTagIds": { + "$ref": "#/$defs/ints" + }, + "jacket": { + "type": [ + "string", + "null" + ] + }, + "gekisouMissions": { + "type": "array", + "items": { + "type": [ + "integer", + "null" + ] + }, + "minItems": 3, + "maxItems": 3 + }, + "bgm": { + "type": "object", + "required": [ + "soundId", + "cueSheet", + "cue", + "length" + ], + "properties": { + "soundId": { + "type": "integer" + }, + "cueSheet": { + "type": "string" + }, + "cue": { + "type": "string" + }, + "length": { + "type": [ + "object", + "null" + ], + "required": [ + "lengthMs", + "samples", + "sampleRate", + "durationMs" + ], + "properties": { + "lengthMs": { + "type": [ + "integer", + "null" + ] + }, + "samples": { + "type": [ + "integer", + "null" + ] + }, + "sampleRate": { + "type": [ + "integer", + "null" + ] + }, + "durationMs": { + "type": [ + "integer", + "null" + ] + } + } + } + } + }, + "scoreRanks": { + "type": "array", + "items": { + "type": "object", + "required": [ + "rank", + "requiredScore", + "battleRequiredScore" + ], + "properties": { + "rank": { + "enum": [ + "E", + "D", + "C", + "B", + "A", + "S", + "SS" + ] + }, + "requiredScore": { + "type": [ + "integer", + "null" + ] + }, + "battleRequiredScore": { + "type": [ + "integer", + "null" + ] + } + } + } + }, + "charts": { + "type": "array", + "items": { + "$ref": "#/$defs/songChart" + } + }, + "master": { + "type": "object", + "required": [ + "MasterLiveMusic", + "MasterLiveScoreRank" + ], + "properties": { + "MasterLiveMusic": { + "type": "object" + }, + "MasterLiveScoreRank": { + "type": "array", + "items": { + "type": "object" + } + } + } + } + } + } + } +} diff --git a/docs/songs.md b/docs/songs.md deleted file mode 100644 index ea2456a..0000000 --- a/docs/songs.md +++ /dev/null @@ -1,98 +0,0 @@ -# Songs - -`nnnotes songs` writes one JSON file with every live song's metadata for one master data version: what a song -listing shows (titles and credits in every language, bands, vocal characters, category, tags, release time, jacket, -BGM length) and, per difficulty, the chart facts (level, note counts, BPM, chart times, skill events, fevers). The -format is `nnnotes.songs/1`. - -``` -nnnotes songs (--master-files DIR | --apk-master) [--no-bgm] [--jackets DIR] -o FILE -``` - -The master data is read as for [deck data](deck-data.md): the files as served, each checked against the manifest's -SHA-256. A songs file and a deck data file made from the same master data version join by `scoreId` -(`MasterLiveMusicScore._id`). Charts are read from the catalog of `[catalog] language`; the BGM length from the cue -sheet's ACB (its `CueTable` and `WaveformTable`, no audio is decoded). `--no-bgm` does not read the cue sheets. -`--jackets DIR` also writes every song's jacket, the Texture2D `Image/Jacket/`, as `DIR/.webp` -(WebP quality 88, scaled down with Lanczos to at most 320 pixels on the longer side, without alpha when opaque); a -page next to the songs file finds a song's jacket at `/.webp`. - -The command writes the file only when every table, chart and cue sheet was read: a missing master data file, a text -id that `MasterText` does not have, a score id that `MasterLiveMusicScore` does not have, a missing or unreadable -chart asset, a cue sheet without the song's cue, or (with `--jackets`) a missing jacket texture stops it with exit status 1 and a line naming the input. - -## Layout - -```json -{ - "format": "nnnotes.songs/1", - "provenance": {"region": ..., "client": {...}, "catalog": {...}, "master": {...}, "exporter": {...}}, - "languages": ["ja", "en", "zh-Hant", "zh-Hans", "ko"], - "bands": [...], "characters": [...], "tags": [...], "categories": [...], - "songs": [{"id": 100001, ..., "charts": [...]}, ...] -} -``` - -The file is minified UTF-8 with one line feed at the end, keys in the order shown, and canonical in the same way as -deck data (same inputs, same nnnotes version: same bytes; floats are the shortest decimal of their binary32 value). -`provenance` is the deck data provenance, with `master.tables` listing the tables this file reads: `MasterLiveMusic`, -`MasterLiveMusicScore`, `MasterText`, `MasterBand`, `MasterCharacter`, `MasterTag`, `MasterLiveMusicCategory`, -`MasterSound`, `MasterSoundCueSheet`. - -A **text** is an object with one string per language of `languages` (`{"ja": ..., "en": ..., "zh-Hant": ..., -"zh-Hans": ..., "ko": ...}`), the `MasterText` row of the id; a text field is null when the master data gives no id -(an empty string). A language's string may be empty when the game has no text in that language. - -### bands, characters, tags, categories - -| Field | Content | -|---|---| -| `bands[].id`, `.name`, `.mainColor`, `.subColor` | `MasterBand`: id, name text, color codes | -| `characters[].id`, `.bandId`, `.name`, `.shortName`, `.mainColor` | `MasterCharacter` | -| `tags[].id`, `.name` | `MasterTag` (the ids of `bestMusicTagIds`) | -| `categories[].id`, `.musicCategories`, `.name` | `MasterLiveMusicCategory` (the listing's category tabs; `musicCategories` are the song category values it shows) | - -### songs - -One entry per `MasterLiveMusic` row, sorted by `id`. - -| Field | Content | -|---|---| -| `id`, `sortOrder`, `startAt`, `defaultUnlock` | the row's `_id`, `_sortOrder`, `_startAt` (as served, server time), `_defaultUnlock` | -| `title`, `ruby`, `phonetic` | texts of `_titleTextID`, `_rubyTitleTextID`, `_phoneticTextID` | -| `bandIds`, `bandName` | `_bandIDs`; the text of `_bandNameTextID`, a name the song shows instead of its first band's (null: none) | -| `vocalCharacterIds` | `_vocalCharacterIDs` | -| `lyricist`, `composer`, `arranger` | texts of `_lyricistTextID`, `_composerTextID`, `_arrangerTextID` | -| `musicType`, `musicCategories`, `bestMusicTagIds` | `_musicType` (the song type that card type bonuses match), `_musicCategories`, `_bestMusicTagIDs` | -| `jacket` | `_jacketAssetName` (the jacket sprite `Image/Jacket/`) | -| `gekisouMissions` | `[_gekisouMission1, _gekisouMission2, _gekisouMission3]` | -| `bgm.soundId`, `.cueSheet`, `.cue` | `_musicSoundID` and its `MasterSound` / `MasterSoundCueSheet` cue | -| `bgm.length` | `{lengthMs, samples, sampleRate, durationMs}`: the cue's `Length` in the ACB `CueTable`, the sample count and rate of its first waveform, and `samples * 1000 // sampleRate`; null with `--no-bgm` | -| `charts` | one entry per difficulty the song has (`_easyID` .. `_expertID` not 0), in the order easy, normal, hard, expert | -| `master.MasterLiveMusic` | the whole `MasterLiveMusic` row as decoded | - -### charts - -| Field | Content | -|---|---| -| `difficulty` | `easy`, `normal`, `hard` or `expert` | -| `scoreId`, `level`, `displayLevel`, `fullComboCount` | `MasterLiveMusicScore`: `_id`, `_musicScoreLevel`, `_musicScoreDisplayLevel`, `_fullComboCount` | -| `asset.key`, `asset.sha256` | the chart TextAsset `Live/MusicScore/<_musicScoreTextFileName>` and the SHA-256 of its bytes as shipped | -| `notes.judged` | notes that are judged (and count for a full combo) | -| `notes.total` | every runtime note, including hidden notes, guide notes and slide combo ticks | -| `notes.byOperateType` | `{"": count}` over every runtime note, keys in ascending order | -| `bpm.changes` | every BPM change `{timeMs, bpm}` in time order | -| `bpm.main`, `.min`, `.max` | over the played span (first to last judged note): the BPM that holds longest (the earliest on a tie), the lowest and the highest | -| `firstNoteMs`, `lastJudgedNoteMs` | times of the first and the last judged note | -| `lastNoteMs` | the latest time of any runtime note (the time the score code calls the last timing note) | -| `musicLengthMs` | `lastNoteMs + 1000`: the live's music length on the game's score path (skill effects end at it at the latest) | -| `skillEventsMs` | skill event times in chart order; event `i` fires the skill of the member at performance position `i` | -| `fevers` | fever ranges `[startMs, endMs]` sorted by start | - -Chart times are milliseconds of chart time. The two lengths are different facts: `bgm.length` is how long the music -plays, `musicLengthMs` is the length the score code uses; a listing chooses the one it needs. - -## Versions - -As for deck data: within `nnnotes.songs/1` fields are only added, never renamed, removed or given another meaning; -readers ignore keys they do not know and reject a major version they do not know. diff --git a/pyproject.toml b/pyproject.toml index 001a9d2..2b91240 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] -requires = ["setuptools>=77"] -build-backend = "setuptools.build_meta" +requires = ["maturin>=1.9,<2"] +build-backend = "maturin" [project] name = "nnnotes" @@ -37,14 +37,14 @@ Homepage = "https://github.com/MetaSekaiLab/nnnotes" Source = "https://github.com/MetaSekaiLab/nnnotes" Issues = "https://github.com/MetaSekaiLab/nnnotes/issues" -[tool.setuptools.dynamic] -version = { attr = "nnnotes.__version__" } - -[tool.setuptools.packages.find] -where = ["src"] - -[tool.setuptools.package-data] -nnnotes = ["typetrees/*.json", "viewrules.json", "nnnotes.example.toml"] +[tool.maturin] +# the pure Python package src/nnnotes with the extension module nnnotes._deck (rust/: ournotes-deck, pinned by +# commit); the version is rust/Cargo.toml's, kept equal to nnnotes.__version__ (tests/test_musicdata.py) +python-source = "src" +module-name = "nnnotes._deck" +manifest-path = "rust/Cargo.toml" +profile = "release" +locked = true [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 0000000..9213264 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,215 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "nnnotes-deck" +version = "0.1.2" +dependencies = [ + "ournotes-deck", + "pyo3", + "serde_json", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "ournotes-deck" +version = "0.0.1" +source = "git+https://github.com/empty-sekai/ournotes-deck?rev=7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6#7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 0000000..3a28e84 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "nnnotes-deck" +# the nnnotes version (src/nnnotes/__init__.py); the wheel takes its version from here +version = "0.1.2" +edition = "2024" +rust-version = "1.85" +description = "nnnotes._deck: the ournotes-deck chart statistics as a Python extension module" +license = "MIT" +publish = false + +[lib] +name = "_deck" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = { version = "0.29", features = ["abi3-py311"] } +# pinned by commit; .github/workflows/deck.yml opens a pull request when ournotes-deck moves +ournotes-deck = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6" } +serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } + +[profile.release] +lto = "thin" +codegen-units = 1 diff --git a/rust/build.rs b/rust/build.rs new file mode 100644 index 0000000..580abea --- /dev/null +++ b/rust/build.rs @@ -0,0 +1,33 @@ +//! Records the ournotes-deck package the module is built with (version and git commit, from Cargo.lock) as the +//! environment variables DECK_VERSION, DECK_SOURCE and DECK_COMMIT of the crate. + +use std::path::Path; + +fn main() { + let dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR"); + let lock = Path::new(&dir).join("Cargo.lock"); + println!("cargo:rerun-if-changed={}", lock.display()); + let text = std::fs::read_to_string(&lock).expect("rust/Cargo.lock (the deck commit is read from it)"); + let (mut version, mut source) = (None, None); + for block in text.split("[[package]]") { + let field = |k: &str| { + block.lines().find_map(|l| { + let v = l.trim().strip_prefix(k)?.trim_start().strip_prefix('=')?.trim(); + Some(v.trim_matches('"').to_string()) + }) + }; + if field("name").as_deref() == Some("ournotes-deck") { + version = field("version"); + source = field("source"); + } + } + let version = version.expect("Cargo.lock has no ournotes-deck package"); + let source = source.expect("ournotes-deck has no source in Cargo.lock"); + // git+https://github.com/empty-sekai/ournotes-deck?rev=# + let (url, commit) = source.split_once('#').expect("ournotes-deck is not a git dependency"); + let url = url.strip_prefix("git+").unwrap_or(url); + let url = url.split_once('?').map_or(url, |(u, _)| u); + println!("cargo:rustc-env=DECK_VERSION={version}"); + println!("cargo:rustc-env=DECK_SOURCE={url}"); + println!("cargo:rustc-env=DECK_COMMIT={commit}"); +} diff --git a/rust/rustfmt.toml b/rust/rustfmt.toml new file mode 100644 index 0000000..c775577 --- /dev/null +++ b/rust/rustfmt.toml @@ -0,0 +1,2 @@ +max_width = 120 +use_small_heuristics = "Max" diff --git a/rust/src/lib.rs b/rust/src/lib.rs new file mode 100644 index 0000000..476e951 --- /dev/null +++ b/rust/src/lib.rs @@ -0,0 +1,98 @@ +//! `nnnotes._deck`: the chart statistics of ournotes-deck (`ournotes-deck.chart-stats/2`) on a deck data document +//! held in memory, the charts measured in parallel. + +use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use ournotes_deck::chartstats::{self, ChartStats, GEKISOU_SEEDS}; +use ournotes_deck::data::DeckData; +use ournotes_deck::error::Error; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +fn value_error(e: impl std::fmt::Display) -> PyErr { + PyValueError::new_err(e.to_string()) +} + +/// The statistics of every chart, in chart order, on `workers` threads. +fn measure(data: &DeckData, seeds: usize, workers: usize) -> Result, Error> { + let kinds = chartstats::kinds(&data.master); + let n = data.charts.len(); + let next = AtomicUsize::new(0); + let out: Mutex>>> = Mutex::new((0..n).map(|_| None).collect()); + std::thread::scope(|s| { + for _ in 0..workers.clamp(1, n.max(1)) { + s.spawn(|| { + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + if i >= n { + break; + } + let c = &data.charts[i]; + // as chartstats::document names a chart's domain errors + let r = chartstats::chart_stats(&data.master, c, &kinds, seeds).map_err(|e| match e { + Error::Domain(m) => Error::Domain(format!("chart {}: {m}", c.score_id)), + e => e, + }); + let failed = r.is_err(); + out.lock().expect("results")[i] = Some(r); + if failed { + next.store(n, Ordering::Relaxed); + } + } + }); + } + }); + // charts are taken in order and a taken chart always completes, so every chart before the first failure has + // its result: the error returned is the first in chart order, as the command's + let mut stats = Vec::with_capacity(n); + for r in out.into_inner().expect("results") { + stats.push(r.expect("a chart before the first failure has run")?); + } + Ok(stats) +} + +/// chart_stats(data, seeds=None, workers=None) -> str +/// +/// The `ournotes-deck.chart-stats/2` document (JSON text) of a deck data document (`nnnotes.deck-data/1` JSON text), +/// as `ournotes-deck chart-stats` writes it. `seeds`: the size of the seed set of charts with a luck range (default +/// 8); `workers`: threads measuring charts (default: the available parallelism). Raises ValueError for data the +/// deck model cannot read or a chart whose check deck fails. +#[pyfunction] +#[pyo3(signature = (data, seeds=None, workers=None))] +fn chart_stats(py: Python<'_>, data: &str, seeds: Option, workers: Option) -> PyResult { + let seeds = seeds.unwrap_or(GEKISOU_SEEDS); + let workers = workers.unwrap_or_else(|| std::thread::available_parallelism().map_or(1, |n| n.get())); + py.detach(|| { + let mut data = DeckData::from_json(data).map_err(value_error)?; + let charts = std::mem::take(&mut data.charts); + // the document of no chart: the format, source, model and kinds, exactly as the command writes them + let mut doc = chartstats::document(&data, Some(seeds)).map_err(value_error)?; + data.charts = charts; + let stats = measure(&data, seeds, workers).map_err(value_error)?; + doc["charts"] = serde_json::to_value(stats).map_err(value_error)?; + serde_json::to_string(&doc).map_err(value_error) + }) +} + +/// info() -> dict: the ournotes-deck package of this module (name, version, source, commit) and the formats it reads +/// and writes. +#[pyfunction] +fn info(py: Python<'_>) -> PyResult> { + let d = PyDict::new(py); + d.set_item("name", "ournotes-deck")?; + d.set_item("version", env!("DECK_VERSION"))?; + d.set_item("source", env!("DECK_SOURCE"))?; + d.set_item("commit", env!("DECK_COMMIT"))?; + d.set_item("dataFormat", ournotes_deck::data::FORMAT)?; + d.set_item("format", chartstats::FORMAT)?; + Ok(d) +} + +#[pymodule] +fn _deck(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(chart_stats, m)?)?; + m.add_function(wrap_pyfunction!(info, m)?)?; + Ok(()) +} diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index d9bb7db..dfbf4af 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -36,8 +36,8 @@ [--story 10462 [--story ...] | --all-stories] [--story-languages en,ja] [--font en=] [--font emoji=] [--region [--region ...] | --all-regions] - nnnotes deck-data --master-files | --apk-master -o out/deck-data.json[.gz] - nnnotes songs --master-files | --apk-master [--no-bgm] [--jackets DIR] -o out/songs.json[.gz] + nnnotes music-data --master-files | --apk-master [--full] [--no-deck] [--no-bgm] + [--jackets DIR] -o out/music-data.json[.gz] nnnotes export -o out/assets [--select group: | key: | bundle: ...] [--layout original,cas] nnnotes plan [--select ...] [--json] [--check] [--emit-tasks ] nnnotes run-stage [...] @@ -637,44 +637,27 @@ def cmd_web(args, cfg): sys.exit(1) -def cmd_deck_data(args, cfg): - from . import deckdata +def cmd_music_data(args, cfg): + from . import deckdata, musicdata if args.apk_master: cfg.require_path("paths", "apk") # the master data files ship in the APK apk = _existing(cfg, "paths", "apk") try: + deck = None if args.no_deck else musicdata.Deck(seeds=args.seeds, workers=args.workers) if args.apk_master: src, region = deckdata.apk_master(apk), deckdata.EMBEDDED else: src, region = deckdata.master_files(Path(args.master_files)), cfg.region() key = master_key(cfg) cat = open_catalog(cfg) - r = deckdata.export(Path(args.out), src, key, deckdata.catalog_fetch(cat), region=region, - client=deckdata.apk_client(apk) if apk is not None else {}, - catalog=deckdata.catalog_info(cat, cli_assets.store_root(args, cfg))) - except deckdata.DeckDataError as e: - sys.exit(f"nnnotes: {e}") - _print_json(r) - - -def cmd_songs(args, cfg): - from . import deckdata, songs - if args.apk_master: - cfg.require_path("paths", "apk") # the master data files ship in the APK - apk = _existing(cfg, "paths", "apk") - try: - if args.apk_master: - src, region = deckdata.apk_master(apk), deckdata.EMBEDDED - else: - src, region = deckdata.master_files(Path(args.master_files)), cfg.region() - key = master_key(cfg) - cat = open_catalog(cfg) - r = songs.export(Path(args.out), src, key, deckdata.catalog_fetch(cat), - None if args.no_bgm else songs.catalog_bgm(cat), region=region, - client=deckdata.apk_client(apk) if apk is not None else {}, - catalog=deckdata.catalog_info(cat, cli_assets.store_root(args, cfg)), - jacket=songs.catalog_jacket(cat) if args.jackets else None, jackets_dir=args.jackets) - except (deckdata.DeckDataError, songs.SongsError) as e: + r = musicdata.export(Path(args.out), src, key, deckdata.catalog_fetch(cat), + None if args.no_bgm else musicdata.catalog_bgm(cat), region=region, + client=deckdata.apk_client(apk) if apk is not None else {}, + catalog=deckdata.catalog_info(cat, cli_assets.store_root(args, cfg)), + deck=deck, full=args.full, + jacket=musicdata.catalog_jacket(cat) if args.jackets else None, + jackets_dir=args.jackets) + except (deckdata.DeckDataError, musicdata.MusicDataError) as e: sys.exit(f"nnnotes: {e}") _print_json(r) @@ -943,29 +926,28 @@ def target(m, what): _live_option_arg(c) c.set_defaults(func=cmd_web, usage=c.error) - c = sub.add_parser("deck-data", help="every live chart and the master data tables deck-building tools read -> " - "one JSON file") - g = c.add_mutually_exclusive_group(required=True) - g.add_argument("--master-files", metavar="DIR", - help="master data files as served: MasterManifest.json and the .bin files it lists " - "(`master download`)") - g.add_argument("--apk-master", action="store_true", help="the master data files of base.apk ([paths] apk)") - _out(c, "output file (.json, or .json.gz for gzip)") - c.set_defaults(func=cmd_deck_data, usage=c.error) - - c = sub.add_parser("songs", help="every live song's metadata (titles, credits, bands, BGM length) and chart facts " - "(levels, note counts, BPM, times) -> one JSON file") + c = sub.add_parser("music-data", help="every live song and chart: metadata in every language, chart facts and " + "the deck model's chart statistics -> one JSON file") g = c.add_mutually_exclusive_group(required=True) g.add_argument("--master-files", metavar="DIR", help="master data files as served: MasterManifest.json and the .bin files it lists " "(`master download`)") g.add_argument("--apk-master", action="store_true", help="the master data files of base.apk ([paths] apk)") + c.add_argument("--full", action="store_true", + help="also write the deck model's input: every chart's runtime notes and the master data tables " + "about cards, skills, bonuses, scores and events") + c.add_argument("--no-deck", action="store_true", + help="do not run the deck model (every chart's deck is null)") + c.add_argument("--seeds", type=int, default=8, metavar="N", + help="seeds measured on a chart with a luck range (default 8)") + c.add_argument("--workers", type=int, metavar="N", + help="threads measuring charts (default: every processor)") c.add_argument("--no-bgm", action="store_true", help="do not read the BGM cue sheets (every song's bgm.length is null)") c.add_argument("--jackets", metavar="DIR", help="also write every song's jacket as DIR/.webp (at most 320 px on the longer side)") _out(c, "output file (.json, or .json.gz for gzip)") - c.set_defaults(func=cmd_songs, usage=c.error) + c.set_defaults(func=cmd_music_data, usage=c.error) cli_assets.register(sub, argparse.Namespace(open_catalog=open_catalog, print_json=_print_json)) voices.register(sub, argparse.Namespace(open_catalog=open_catalog, master_dir=master_dir)) diff --git a/src/nnnotes/deckdata.py b/src/nnnotes/deckdata.py index 462b065..af7a1f0 100644 --- a/src/nnnotes/deckdata.py +++ b/src/nnnotes/deckdata.py @@ -1,5 +1,6 @@ -"""Deck data: one JSON file with every live chart as the client builds it at runtime and the master data tables that -deck-building tools read, for one master data version (format `nnnotes.deck-data/1`, docs/deck-data.md). +"""Deck input: every live chart as the client builds it at runtime and the master data tables the deck model +(ournotes-deck) reads, for one master data version. `nnnotes music-data` hands it to the deck model in memory +(DECK_FORMAT, the reader's format) and writes it into the music data file with `--full` (docs/music-data.md). Master data is decoded from the files as served: a directory with `MasterManifest.json` and the `.bin` files it lists (`nnnotes master download`), or the same layout inside the APK (`assets/Master/`). Each file is checked against @@ -7,11 +8,11 @@ every MasterLiveMusicScore row, read from the catalog and converted by score.runtime_score; notes are listed in the order the client enumerates them. -The output is canonical: minified UTF-8 with one trailing LF, keys in a fixed order, charts sorted by score id, -numbers the master data writes with a fraction or exponent as the shortest decimal that reads back as the same -binary32 value (infinity as `1e999` / `-1e999`; a NaN is an error). The same inputs give the same bytes; a `.gz` -output is gzip with no file name and a zero modification time. The file is written only when every table and chart -was read; any missing or unreadable input is a DeckDataError naming it. +The encoding is canonical (encode): minified UTF-8 with one trailing LF, keys in a fixed order, charts sorted by +score id, numbers the master data writes with a fraction or exponent as the shortest decimal that reads back as the +same binary32 value (infinity as `1e999` / `-1e999`; a NaN is an error). The same inputs give the same bytes; a `.gz` +output is gzip with no file name and a zero modification time. Any missing or unreadable input is a DeckDataError +naming it. """ from __future__ import annotations @@ -24,7 +25,7 @@ from pathlib import Path from typing import Callable -FORMAT = "nnnotes.deck-data/1" +DECK_FORMAT = "nnnotes.deck-data/1" # the deck model's input format (ournotes-deck data::FORMAT) CHART_FORMAT = "nnnotes.live-score/1" # score.convert's format: the converter the notes come from CHART_PREFIX = "Live/MusicScore/" # + MasterLiveMusicScore._musicScoreTextFileName MANIFEST = "MasterManifest.json" @@ -106,6 +107,12 @@ ("MasterLiveScoreRank", "_id _group _liveScoreRank _requiredScore _battleLiveRequiredScore"), ("MasterEvent", _WHOLE), ("MasterEventEffect", _WHOLE), + ("MasterEventAchievementReward", "_id _eventId _eventPoint _rewardIds"), + ("MasterEventAchievementLoopReward", "_id _eventId _loopStartEventPoint _loopEventPoint _rewardIds"), + ("MasterLiveEventReward", "_id _group _eventGroup _scoreRank _resourceType _resourceId _resourceCount " + "_probability"), + ("MasterChallengeLiveEventReward", "_id _group _eventGroup _scoreRank _resourceType _resourceId " + "_resourceCount _probability"), ("MasterLiveEventPoint", _WHOLE), ("MasterChallengeLiveEventPoint", _WHOLE), ("MasterLiveChallengePoint", _WHOLE), @@ -371,25 +378,11 @@ def resource_version(store_root, remote_sha: str) -> str | None: return None -# ---------------------------------------------------------------- document -def build(tables: dict[str, list[dict]], table_sha: dict[str, str], fetch: Callable[[str], bytes], *, region: str, - client: dict, catalog: dict, master_source: str, master_version: str | None) -> dict: - """The deck data document. `catalog`: {resourceVersion, sha256} of the catalog the charts come from.""" - from . import __version__ - subset = master_subset(tables) - return { - "format": FORMAT, - "provenance": { - "region": region, - "client": {"versionName": client.get("versionName"), "versionCode": client.get("versionCode")}, - "catalog": {"resourceVersion": catalog.get("resourceVersion"), "sha256": catalog.get("sha256")}, - "master": {"source": master_source, "version": master_version, - "tables": {t: {"sha256": table_sha[t]} for t, _ in TABLES}}, - "exporter": {"name": "nnnotes", "version": __version__, "chartFormat": CHART_FORMAT}, - }, - "master": subset, - "charts": charts(tables["MasterLiveMusicScore"], fetch), - } +# ---------------------------------------------------------------- the deck model's input +def build(tables: dict[str, list[dict]], chart_records: list[dict], provenance: dict) -> dict: + """The deck model's input document (DECK_FORMAT): the TABLES subset of `tables` and chart records (charts).""" + return {"format": DECK_FORMAT, "provenance": provenance, "master": master_subset(tables), + "charts": chart_records} def _emit(v, out: list) -> None: @@ -441,22 +434,3 @@ def encode(doc: dict) -> bytes: def file_bytes(data: bytes, gz: bool) -> bytes: """The bytes written: `data`, or with `gz` gzip with no file name and modification time 0.""" return gzip.compress(data, compresslevel=9, mtime=0) if gz else data - - -def export(out, src: MasterSource, key, fetch: Callable[[str], bytes], *, region: str, client: dict, - catalog: dict) -> dict: - """Read the master data and every chart, then write the file `out` (gzip when it ends in `.gz`) through a - temporary file and a rename. Returns the summary.""" - from .cache import write_atomic - out = Path(out) - tables, shas = read_master(src, key) - doc = build(tables, shas, fetch, region=region, client=client, catalog=catalog, master_source=src.source, - master_version=src.version) - data = encode(doc) - written = file_bytes(data, out.name.endswith(".gz")) - out.parent.mkdir(parents=True, exist_ok=True) - write_atomic(out, written) - return {"out": str(out), "format": FORMAT, "region": region, "masterSource": src.source, - "masterVersion": src.version, "tables": len(TABLES), "rows": sum(len(r) for r in tables.values()), - "charts": len(doc["charts"]), "notes": sum(len(c["notes"]["id"]) for c in doc["charts"]), - "bytes": len(data), "fileBytes": len(written), "sha256": hashlib.sha256(written).hexdigest()} diff --git a/src/nnnotes/musicdata.py b/src/nnnotes/musicdata.py new file mode 100644 index 0000000..04ffa64 --- /dev/null +++ b/src/nnnotes/musicdata.py @@ -0,0 +1,460 @@ +"""Music data: one JSON file with every live song and chart of one master data version (format +`nnnotes.music-data/1`, docs/music-data.md): titles and credits in every language, bands, vocal characters, category, +tags, release time, score ranks, the live BGM's length, per difficulty the chart facts (level, note counts, BPM, chart +times, skill events, fever ranges) and the chart's deck statistics, what the chart contributes to the live score +whatever the deck, measured by the deck model ournotes-deck (the extension module nnnotes._deck). + +Master data is read from the files as served (deckdata.master_files / apk_master: SHA-256 checked against the +manifest, decoded with master.decode). Charts are the TextAssets `Live/MusicScore/` converted by +score.runtime_score; the BGM length is read from the cue sheet's ACB (cue `Length` and the stream's sample count), +without decoding audio. The deck model reads the deck input (deckdata.build: the charts' runtime notes and the master +data tables it needs) in memory; its statistics are checked against the chart facts. With `full` the file also +carries that deck input (`master`, `charts`), every chart's runtime notes and the tables. With a jackets directory, +every song's jacket (the Texture2D `Image/Jacket/`) is written there as `.webp`, scaled to at most +JACKET_SIZE pixels on its longer side. + +The output is canonical (deckdata.encode): minified UTF-8 with one trailing LF, keys in a fixed order, songs sorted +by id, master data floats as the shortest decimal of their binary32 value, the deck model's numbers as it writes them. +The same inputs and deck model give the same bytes. The file is written only when every table, chart and cue sheet +was read and every chart measured; any missing or unreadable input is a MusicDataError naming it. +""" +from __future__ import annotations + +import hashlib +import io +import json +from collections import Counter +from pathlib import Path +from typing import Callable + +from . import deckdata +from .languages import LANGUAGES + +FORMAT = "nnnotes.music-data/1" +DIFFICULTIES = ("easy", "normal", "hard", "expert") +MUSIC_LENGTH_TAIL_MS = 1000 # the live's music length: the last note time + 1000 ms (LiveScore skip path) + +# the tables of the song metadata; the deck model's are deckdata.TABLES +SONG_TABLES = ("MasterLiveMusic", "MasterLiveMusicScore", "MasterText", "MasterBand", "MasterCharacter", + "MasterTag", "MasterLiveMusicCategory", "MasterSound", "MasterSoundCueSheet", "MasterLiveScoreRank") +SCORE_RANKS = {1: "E", 2: "D", 3: "C", 4: "B", 5: "A", 6: "S", 7: "SS"} # LiveScoreRank + + +class MusicDataError(ValueError): + """An input the music data file cannot be made from (the message names it).""" + + +# ---------------------------------------------------------------- texts +class Texts: + """MasterText rows by id -> {language: text} in every language (languages.LANGUAGES).""" + + def __init__(self, rows: list[dict]): + self.rows = {r.get("_id"): r for r in rows} + + def get(self, text_id) -> dict | None: + """{language: text} of a text id; None for an empty id; a MusicDataError for an id MasterText does not have.""" + if not text_id: + return None + r = self.rows.get(text_id) + if r is None: + raise MusicDataError(f"MasterText has no text {text_id!r}") + return {code: r.get(col) for code, (_, col) in LANGUAGES.items()} + + +# ---------------------------------------------------------------- charts +def _f(v) -> float: + return float(v) + + +def bpm_facts(bpm_events: list, first_ms: int, last_ms: int) -> dict: + """{main, min, max, changes} of a chart's BPM changes [(bpm, Pos)] (tick order). `changes` lists every change + as {timeMs, bpm}; main / min / max are taken over the played span [first_ms, last_ms] (the first and the last + judged note): main is the BPM that holds longest in the span (the earliest on a tie), min / max the lowest and + the highest that hold in it. A span of one instant takes the BPM at that time.""" + changes = sorted(((int(p.ms), _f(b)) for b, p in bpm_events), key=lambda x: x[0]) + if not changes: + raise MusicDataError("no BPM change") + held: dict[float, int] = {} + order: list[float] = [] + for i, (t, b) in enumerate(changes): + end = changes[i + 1][0] if i + 1 < len(changes) else None + lo = max(t, first_ms) + hi = last_ms if end is None else min(end, last_ms) + at_span = (end is None or end > first_ms) and t <= last_ms + if not at_span: + continue + if b not in held: + order.append(b) + held[b] = 0 + held[b] += max(0, hi - lo) + if not order: # every change after the span: the first one holds before it + order, held = [changes[0][1]], {changes[0][1]: 0} + main = max(order, key=lambda b: (held[b], -order.index(b))) + return {"main": main, "min": min(order), "max": max(order), + "changes": [{"timeMs": t, "bpm": b} for t, b in changes]} + + +def chart_facts(score_row: dict, key: str, raw: bytes) -> dict: + """The facts of one chart: level, note counts, BPM, chart times, skill events and fevers.""" + from . import score + try: + root = score.load_bytes(raw) + except (ValueError, OSError, EOFError): + raise MusicDataError(f"chart {key}: not a chart (gzip or JSON cannot be read)") from None + try: + rs = score.runtime_score(root) + except Exception as e: # the converter's own errors and malformed chart fields + raise MusicDataError(f"chart {key}: cannot be converted ({type(e).__name__}: {e})") from None + notes = rs.notes + if not notes: + raise MusicDataError(f"chart {key}: no notes") + judged = sorted(int(n.pos.ms) for n in notes if score.is_judgement_note(n.op)) + if not judged: + raise MusicDataError(f"chart {key}: no judged note") + last = max(int(n.pos.ms) for n in notes) + by_type = Counter(int(n.op) for n in notes) + try: + bpm = bpm_facts(rs.bpm_events, judged[0], judged[-1]) + except MusicDataError as e: + raise MusicDataError(f"chart {key}: {e}") from None + return { + "scoreId": score_row["_id"], + "level": score_row["_musicScoreLevel"], + "displayLevel": score_row.get("_musicScoreDisplayLevel"), + "fullComboCount": score_row["_fullComboCount"], + "asset": {"key": key, "sha256": hashlib.sha256(raw).hexdigest()}, + "notes": {"judged": len(judged), "total": len(notes), + "byOperateType": {str(op): by_type[op] for op in sorted(by_type)}}, + "bpm": bpm, + "firstNoteMs": judged[0], + "lastJudgedNoteMs": judged[-1], + "lastNoteMs": last, + "musicLengthMs": last + MUSIC_LENGTH_TAIL_MS, + "skillEventsMs": [int(p.ms) for _, p in rs.skills], + "fevers": [[int(a.ms), int(b.ms)] for _, a, b in rs.fevers], + } + + +# ---------------------------------------------------------------- BGM +def cue_length(acb: bytes, cue: str, where: str) -> dict: + """{lengthMs, samples, sampleRate, durationMs} of a cue of an ACB: the CueTable `Length` and the first stream's + sample count and rate (durationMs = samples * 1000 // sampleRate).""" + from . import acb as acbmod + try: + cues = acbmod.cue_streams(acb) + except Exception as e: + raise MusicDataError(f"{where}: the ACB cannot be read ({type(e).__name__}: {e})") from None + c = cues.get(cue) + if c is None: + raise MusicDataError(f"{where}: no cue {cue!r}") + s = (c.get("streams") or [None])[0] or {} + samples, rate = s.get("samples"), s.get("sampleRate") + return {"lengthMs": c.get("lengthMs"), "samples": samples, "sampleRate": rate, + "durationMs": samples * 1000 // rate if isinstance(samples, int) and isinstance(rate, int) and rate + else None} + + +def catalog_bgm(cat) -> Callable[[str, str], dict]: + """bgm(cue sheet, cue) for a catalog: the cue sheet's ACB (cri.acb_data) -> cue_length.""" + from . import cri + + def bgm(sheet: str, cue: str) -> dict: + try: + files, _ = cri.acb_data(cat, sheet) + except KeyError: + raise MusicDataError(f"cue sheet {sheet}: no such asset") from None + except Exception as e: + raise MusicDataError(f"cue sheet {sheet}: cannot be read ({type(e).__name__}: {e})") from None + return cue_length(files["acb"], cue, f"cue sheet {sheet}") + return bgm + + +# ---------------------------------------------------------------- document +JACKET_KEY = "Image/Jacket/{jacket}" +JACKET_SIZE = 320 # longer side of a written jacket, pixels +JACKET_QUALITY = 88 # WebP quality + + +def jacket_webp(image, size: int = JACKET_SIZE) -> bytes: + """WebP bytes of a jacket image (a Pillow image), scaled down (Lanczos) to at most `size` pixels on its longer + side; opaque images are written without alpha.""" + from PIL import Image + img = image.convert("RGBA") + if img.getextrema()[3][0] == 255: + img = img.convert("RGB") + w, h = img.size + if max(w, h) > size: + img = img.resize((max(1, round(w * size / max(w, h))), max(1, round(h * size / max(w, h)))), + Image.LANCZOS) + buf = io.BytesIO() + img.save(buf, format="WEBP", quality=JACKET_QUALITY, method=6) + return buf.getvalue() + + +def catalog_jacket(cat, size: int = JACKET_SIZE) -> Callable[[str], bytes]: + """jacket(name) for a catalog: the WebP bytes (jacket_webp) of the Texture2D `Image/Jacket/`.""" + import tempfile + from .export import Exporter + ex = Exporter(cat, Path(tempfile.gettempdir()), textures="deferred") + + def jacket(name: str) -> bytes: + key = JACKET_KEY.format(jacket=name) + try: + o = ex.key_object(key) + except KeyError: + raise MusicDataError(f"jacket {name}: no asset {key}") from None + except Exception as e: + raise MusicDataError(f"jacket {name}: {key} cannot be read ({type(e).__name__}: {e})") from None + if o is None or o.type.name != "Texture2D": + raise MusicDataError(f"jacket {name}: {key} is not a Texture2D") + return jacket_webp(o.read().image, size) + return jacket + + +def _by_id(rows: list[dict], table: str) -> dict: + out = {} + for r in rows: + if r.get("_id") in out: + raise MusicDataError(f"{table}: _id {r.get('_id')} occurs twice") + out[r.get("_id")] = r + return out + + +# ---------------------------------------------------------------- the deck model +DECK_SEEDS = 8 # seed set size of a chart with a luck range (ournotes-deck's default) + + +class Deck: + """The deck model (nnnotes._deck): chart statistics of a deck input document, on `workers` threads (None: the + available parallelism), `seeds` seeds for a chart with a luck range.""" + + def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=None): + if module is None: + try: + from . import _deck as module + except ImportError: + raise MusicDataError("the deck model (nnnotes._deck) is not built into this installation: install " + "nnnotes from a wheel or build it (maturin), or pass --no-deck") from None + self.module, self.seeds, self.workers = module, seeds, workers + + def info(self) -> dict: + """{name, version, source, commit, format} of the deck model.""" + i = self.module.info() + return {k: i[k] for k in ("name", "version", "source", "commit", "format")} + + def stats(self, deck_input: dict) -> dict: + """The chart statistics document of a deck input document (deckdata.build), its numbers as written.""" + try: + text = self.module.chart_stats(deckdata.encode(deck_input).decode("utf-8"), self.seeds, self.workers) + except ValueError as e: + raise MusicDataError(f"deck model: {e}") from None + doc = json.loads(text, parse_float=deckdata._Num) + if doc.get("format") != self.info()["format"]: + raise MusicDataError(f"deck model: wrote {doc.get('format')!r}, expected {self.info()['format']!r}") + return doc + + +# the keys of a chart's statistics carried by `deck` (the others are checked against the chart facts) +DECK_CHART_KEYS = ("convertedNoteCount", "skip", "events", "positions", "ranges", "justNotes", "seeds", + "unplayable") + + +def chart_deck(song: dict, chart: dict, stats: dict) -> dict: + """A chart's `deck` from its statistics, after checking them against the song and the chart facts.""" + where = f"chart {chart['scoreId']} ({song['id']} {chart['difficulty']})" + checks = ( + ("music id", stats["musicId"], song["id"]), + ("difficulty", stats["difficulty"], chart["difficulty"]), + ("level", stats["level"], chart["level"]), + ("judged note count", stats["judgedNotes"], chart["notes"]["judged"]), + ("last note time", stats["lastNoteMs"], chart["lastNoteMs"]), + ("music length", stats["musicLengthMs"], chart["musicLengthMs"]), + ("Gekisou missions", stats["missions"], song["gekisouMissions"]), + ("skill event times", [t for _, t in stats["events"]], chart["skillEventsMs"]), + ("fevers", [[r["startMs"], r["endMs"]] for r in stats["ranges"]], chart["fevers"][:len(stats["ranges"])]), + ) + for name, deck, facts in checks: + if deck != facts: + raise MusicDataError(f"{where}: the deck model's {name} {deck!r} differs from the chart's {facts!r}") + return {k: stats.get(k) for k in DECK_CHART_KEYS} + + +# ---------------------------------------------------------------- document +def build(tables: dict[str, list[dict]], table_sha: dict[str, str], fetch: Callable[[str], bytes], + bgm: Callable[[str, str], dict] | None, *, region: str, client: dict, catalog: dict, master_source: str, + master_version: str | None, deck: Deck | None = None, full: bool = False) -> dict: + """The music data document. `fetch(file name)`: a chart TextAsset's bytes (KeyError when there is none); + `bgm(cue sheet, cue)`: the BGM length (catalog_bgm), None to leave every song's `bgm.length` null; `deck`: the + deck model measuring the songs' charts, None to leave every chart's `deck` null; `full`: add the deck input + (`master`, `charts`: deckdata.TABLES and every chart's runtime notes). `tables`: tables_of(deck, full).""" + from . import __version__ + raws: dict[str, bytes] = {} + + def raw_chart(row: dict) -> tuple[str, bytes]: # (key, bytes); each chart asset is read once + name = row["_musicScoreTextFileName"] + key = deckdata.chart_key(name) + if name not in raws: + try: + raws[name] = fetch(name) + except KeyError: + raise MusicDataError(f"chart {key} (MasterLiveMusicScore {row['_id']}): no such asset") from None + return key, raws[name] + + text = Texts(tables["MasterText"]) + scores = _by_id(tables["MasterLiveMusicScore"], "MasterLiveMusicScore") + sounds = _by_id(tables["MasterSound"], "MasterSound") + sheets = _by_id(tables["MasterSoundCueSheet"], "MasterSoundCueSheet") + musics = sorted(_by_id(tables["MasterLiveMusic"], "MasterLiveMusic").values(), key=lambda r: r["_id"]) + rank_groups: dict = {} # _group -> rows in required-score order, as the client reads + for r in sorted(tables["MasterLiveScoreRank"], key=lambda r: (r.get("_requiredScore") or 0, r.get("_id") or 0)): + if r.get("_liveScoreRank") not in SCORE_RANKS: + raise MusicDataError(f"MasterLiveScoreRank {r.get('_id')}: unknown rank {r.get('_liveScoreRank')!r}") + rank_groups.setdefault(r.get("_group"), []).append(r) + + bands = [{"id": b["_id"], "name": text.get(b.get("_nameTextID")), "mainColor": b.get("_mainColorCode"), + "subColor": b.get("_subColorCode")} for b in sorted(tables["MasterBand"], key=lambda r: r["_id"])] + characters = [{"id": c["_id"], "bandId": c.get("_bandID"), "name": text.get(c.get("_nameTextID")), + "shortName": text.get(c.get("_shortNameTextID")), "mainColor": c.get("_mainColorCode")} + for c in sorted(tables["MasterCharacter"], key=lambda r: r["_id"])] + tags = [{"id": t["_id"], "name": text.get(t.get("_nameTextID"))} + for t in sorted(tables["MasterTag"], key=lambda r: r["_id"])] + categories = [{"id": c["_id"], "musicCategories": list(c.get("_musicCategories") or []), + "name": text.get(c.get("_textKey"))} + for c in sorted(tables["MasterLiveMusicCategory"], key=lambda r: r["_id"])] + + songs = [] + for m in musics: + where = f"MasterLiveMusic {m['_id']}" + charts = [] + for d in DIFFICULTIES: + sid = m.get(f"_{d}ID") + if not sid: + continue + row = scores.get(sid) + if row is None: + raise MusicDataError(f"{where}: {d} score {sid} is not in MasterLiveMusicScore") + charts.append({"difficulty": d, **chart_facts(row, *raw_chart(row)), "deck": None}) + snd = sounds.get(m.get("_musicSoundID")) + if snd is None: + raise MusicDataError(f"{where}: sound {m.get('_musicSoundID')} is not in MasterSound") + sheet = sheets.get(snd.get("_soundCueSheetID")) + if sheet is None: + raise MusicDataError(f"{where}: cue sheet {snd.get('_soundCueSheetID')} is not in MasterSoundCueSheet") + length = bgm(sheet["_cueSheetName"], snd["_cueName"]) if bgm is not None else None + rank_rows = rank_groups.get(m.get("_liveScoreRankGroup"), []) + songs.append({ + "id": m["_id"], + "sortOrder": m.get("_sortOrder"), + "startAt": m.get("_startAt"), + "defaultUnlock": m.get("_defaultUnlock"), + "title": text.get(m.get("_titleTextID")), + "ruby": text.get(m.get("_rubyTitleTextID")), + "phonetic": text.get(m.get("_phoneticTextID")), + "bandIds": list(m.get("_bandIDs") or []), + "bandName": text.get(m.get("_bandNameTextID")), + "vocalCharacterIds": list(m.get("_vocalCharacterIDs") or []), + "lyricist": text.get(m.get("_lyricistTextID")), + "composer": text.get(m.get("_composerTextID")), + "arranger": text.get(m.get("_arrangerTextID")), + "musicType": m.get("_musicType"), + "musicCategories": list(m.get("_musicCategories") or []), + "bestMusicTagIds": list(m.get("_bestMusicTagIDs") or []), + "jacket": m.get("_jacketAssetName"), + "gekisouMissions": [m.get("_gekisouMission1"), m.get("_gekisouMission2"), m.get("_gekisouMission3")], + "bgm": {"soundId": snd["_id"], "cueSheet": sheet["_cueSheetName"], "cue": snd["_cueName"], + "length": length}, + "scoreRanks": [{"rank": SCORE_RANKS[r["_liveScoreRank"]], "requiredScore": r.get("_requiredScore"), + "battleRequiredScore": r.get("_battleLiveRequiredScore")} for r in rank_rows], + "charts": charts, + "master": {"MasterLiveMusic": m, "MasterLiveScoreRank": rank_rows}, + }) + + exporter = {"name": "nnnotes", "version": __version__, "chartFormat": deckdata.CHART_FORMAT} + records: dict[int, dict] = {} # score id -> the chart's deck input record + + def record(sid: int) -> dict: + if sid not in records: + records[sid] = deckdata.chart_record(sid, *raw_chart(scores[sid])) + return records[sid] + + deck_doc = None + if deck is not None: + measured = sorted({c["scoreId"] for s in songs for c in s["charts"]}) + stats = deck.stats(deckdata.build( + tables, [record(i) for i in measured], + {"region": region, "master": {"source": master_source, "version": master_version}, + "exporter": exporter})) + by_score = {c["scoreId"]: c for c in stats["charts"]} + if sorted(by_score) != measured: + raise MusicDataError("deck model: the charts measured differ from the songs' charts") + for s in songs: + for c in s["charts"]: + c["deck"] = chart_deck(s, c, by_score[c["scoreId"]]) + deck_doc = {"model": stats["model"], "kinds": stats["kinds"]} + + read = [t for t in tables_of(deck is not None, full) if t in tables] + doc = { + "format": FORMAT, + "provenance": { + "region": region, + "client": {"versionName": client.get("versionName"), "versionCode": client.get("versionCode")}, + "catalog": {"resourceVersion": catalog.get("resourceVersion"), "sha256": catalog.get("sha256")}, + "master": {"source": master_source, "version": master_version, + "tables": {t: {"sha256": table_sha[t]} for t in read}}, + "exporter": exporter, + "deck": deck.info() if deck is not None else None, + }, + "languages": list(LANGUAGES), + "bands": bands, + "characters": characters, + "tags": tags, + "categories": categories, + "deck": deck_doc, + "songs": songs, + } + if full: + doc["master"] = deckdata.master_subset(tables) + doc["charts"] = [record(sid) for sid in sorted(scores)] + return doc + + +def tables_of(deck: bool, full: bool) -> tuple[str, ...]: + """The master data tables read: SONG_TABLES, and deckdata.TABLES with the deck model or `full`.""" + if not (deck or full): + return SONG_TABLES + return SONG_TABLES + tuple(t for t, _ in deckdata.TABLES if t not in SONG_TABLES) + + +def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes], + bgm: Callable[[str, str], dict] | None, *, region: str, client: dict, catalog: dict, + deck: Deck | None = None, full: bool = False, jacket: Callable[[str], bytes] | None = None, + jackets_dir=None) -> dict: + """Read the master data, every chart and every BGM cue sheet, measure the charts with `deck`, then write the file + `out` (gzip when it ends in `.gz`) through a temporary file and a rename; with `jacket` and `jackets_dir`, first + every song's jacket as `/.webp`. Returns the summary.""" + from .cache import write_atomic + out = Path(out) + try: + tables, shas = deckdata.read_master(src, key, tables_of(deck is not None, full)) + doc = build(tables, shas, fetch, bgm, region=region, client=client, catalog=catalog, + master_source=src.source, master_version=src.version, deck=deck, full=full) + jackets = sorted({s["jacket"] for s in doc["songs"] if s["jacket"]}) if jacket is not None else [] + images = {name: jacket(name) for name in jackets} + data = deckdata.encode(deckdata._value(doc, "music data")) + except deckdata.DeckDataError as e: + raise MusicDataError(str(e)) from None + if jackets: + d = Path(jackets_dir) + d.mkdir(parents=True, exist_ok=True) + for name, image in images.items(): + write_atomic(d / f"{name}.webp", image) + written = deckdata.file_bytes(data, out.name.endswith(".gz")) + out.parent.mkdir(parents=True, exist_ok=True) + write_atomic(out, written) + charts = [c for s in doc["songs"] for c in s["charts"]] + return {"out": str(out), "format": FORMAT, "region": region, "masterSource": src.source, + "masterVersion": src.version, "songs": len(doc["songs"]), "charts": len(charts), + "deck": doc["provenance"]["deck"]["commit"] if deck is not None else None, + "unplayable": sum(1 for c in charts if c["deck"] and c["deck"]["unplayable"]), + "full": full, "bgm": bgm is not None, "jackets": len(jackets), + "bytes": len(data), "fileBytes": len(written), "sha256": hashlib.sha256(written).hexdigest()} diff --git a/src/nnnotes/songs.py b/src/nnnotes/songs.py deleted file mode 100644 index 40f373a..0000000 --- a/src/nnnotes/songs.py +++ /dev/null @@ -1,338 +0,0 @@ -"""Songs: one JSON file with every live song's metadata for one master data version (format `nnnotes.songs/1`, -docs/songs.md): titles and credits in every language, bands, vocal characters, category, tags, release time, the -live BGM's length, and per difficulty the chart facts a song listing shows (level, note counts, BPM, chart times, -skill events, fever ranges). - -Master data is read from the files as served, like deck data (deckdata.master_files / apk_master: SHA-256 checked -against the manifest, decoded with master.decode), so a songs file and a deck data file of the same master data -version join by `scoreId`. Charts are the TextAssets `Live/MusicScore/` converted by score.runtime_score; -the BGM length is read from the cue sheet's ACB (cue `Length` and the stream's sample count), without decoding -audio. With a jackets directory, every song's jacket (the Texture2D `Image/Jacket/`) is written there as -`.webp`, scaled to at most JACKET_SIZE pixels on its longer side. - -The output is canonical in the same way as deck data (deckdata.encode): minified UTF-8 with one trailing LF, keys -in a fixed order, songs sorted by id, floats as the shortest decimal of their binary32 value. The file is written -only when every table, chart and cue sheet was read; any missing or unreadable input is a SongsError naming it. -""" -from __future__ import annotations - -import hashlib -import io -from collections import Counter -from pathlib import Path -from typing import Callable - -from . import deckdata -from .languages import LANGUAGES - -FORMAT = "nnnotes.songs/1" -DIFFICULTIES = ("easy", "normal", "hard", "expert") -MUSIC_LENGTH_TAIL_MS = 1000 # the live's music length: the last note time + 1000 ms (LiveScore skip path) - -TABLES = ("MasterLiveMusic", "MasterLiveMusicScore", "MasterText", "MasterBand", "MasterCharacter", "MasterTag", - "MasterLiveMusicCategory", "MasterSound", "MasterSoundCueSheet") - - -class SongsError(ValueError): - """An input the songs file cannot be made from (the message names it).""" - - -# ---------------------------------------------------------------- texts -class Texts: - """MasterText rows by id -> {language: text} in every language (languages.LANGUAGES).""" - - def __init__(self, rows: list[dict]): - self.rows = {r.get("_id"): r for r in rows} - - def get(self, text_id) -> dict | None: - """{language: text} of a text id; None for an empty id; a SongsError for an id MasterText does not have.""" - if not text_id: - return None - r = self.rows.get(text_id) - if r is None: - raise SongsError(f"MasterText has no text {text_id!r}") - return {code: r.get(col) for code, (_, col) in LANGUAGES.items()} - - -# ---------------------------------------------------------------- charts -def _f(v) -> float: - return float(v) - - -def bpm_facts(bpm_events: list, first_ms: int, last_ms: int) -> dict: - """{main, min, max, changes} of a chart's BPM changes [(bpm, Pos)] (tick order). `changes` lists every change - as {timeMs, bpm}; main / min / max are taken over the played span [first_ms, last_ms] (the first and the last - judged note): main is the BPM that holds longest in the span (the earliest on a tie), min / max the lowest and - the highest that hold in it. A span of one instant takes the BPM at that time.""" - changes = sorted(((int(p.ms), _f(b)) for b, p in bpm_events), key=lambda x: x[0]) - if not changes: - raise SongsError("no BPM change") - held: dict[float, int] = {} - order: list[float] = [] - for i, (t, b) in enumerate(changes): - end = changes[i + 1][0] if i + 1 < len(changes) else None - lo = max(t, first_ms) - hi = last_ms if end is None else min(end, last_ms) - at_span = (end is None or end > first_ms) and t <= last_ms - if not at_span: - continue - if b not in held: - order.append(b) - held[b] = 0 - held[b] += max(0, hi - lo) - if not order: # every change after the span: the first one holds before it - order, held = [changes[0][1]], {changes[0][1]: 0} - main = max(order, key=lambda b: (held[b], -order.index(b))) - return {"main": main, "min": min(order), "max": max(order), - "changes": [{"timeMs": t, "bpm": b} for t, b in changes]} - - -def chart_facts(score_row: dict, key: str, raw: bytes) -> dict: - """The facts of one chart: level, note counts, BPM, chart times, skill events and fevers.""" - from . import score - try: - root = score.load_bytes(raw) - except (ValueError, OSError, EOFError): - raise SongsError(f"chart {key}: not a chart (gzip or JSON cannot be read)") from None - try: - rs = score.runtime_score(root) - except Exception as e: # the converter's own errors and malformed chart fields - raise SongsError(f"chart {key}: cannot be converted ({type(e).__name__}: {e})") from None - notes = rs.notes - if not notes: - raise SongsError(f"chart {key}: no notes") - judged = sorted(int(n.pos.ms) for n in notes if score.is_judgement_note(n.op)) - if not judged: - raise SongsError(f"chart {key}: no judged note") - last = max(int(n.pos.ms) for n in notes) - by_type = Counter(int(n.op) for n in notes) - try: - bpm = bpm_facts(rs.bpm_events, judged[0], judged[-1]) - except SongsError as e: - raise SongsError(f"chart {key}: {e}") from None - return { - "scoreId": score_row["_id"], - "level": score_row["_musicScoreLevel"], - "displayLevel": score_row.get("_musicScoreDisplayLevel"), - "fullComboCount": score_row["_fullComboCount"], - "asset": {"key": key, "sha256": hashlib.sha256(raw).hexdigest()}, - "notes": {"judged": len(judged), "total": len(notes), - "byOperateType": {str(op): by_type[op] for op in sorted(by_type)}}, - "bpm": bpm, - "firstNoteMs": judged[0], - "lastJudgedNoteMs": judged[-1], - "lastNoteMs": last, - "musicLengthMs": last + MUSIC_LENGTH_TAIL_MS, - "skillEventsMs": [int(p.ms) for _, p in rs.skills], - "fevers": [[int(a.ms), int(b.ms)] for _, a, b in rs.fevers], - } - - -# ---------------------------------------------------------------- BGM -def cue_length(acb: bytes, cue: str, where: str) -> dict: - """{lengthMs, samples, sampleRate, durationMs} of a cue of an ACB: the CueTable `Length` and the first stream's - sample count and rate (durationMs = samples * 1000 // sampleRate).""" - from . import acb as acbmod - try: - cues = acbmod.cue_streams(acb) - except Exception as e: - raise SongsError(f"{where}: the ACB cannot be read ({type(e).__name__}: {e})") from None - c = cues.get(cue) - if c is None: - raise SongsError(f"{where}: no cue {cue!r}") - s = (c.get("streams") or [None])[0] or {} - samples, rate = s.get("samples"), s.get("sampleRate") - return {"lengthMs": c.get("lengthMs"), "samples": samples, "sampleRate": rate, - "durationMs": samples * 1000 // rate if isinstance(samples, int) and isinstance(rate, int) and rate - else None} - - -def catalog_bgm(cat) -> Callable[[str, str], dict]: - """bgm(cue sheet, cue) for a catalog: the cue sheet's ACB (cri.acb_data) -> cue_length.""" - from . import cri - - def bgm(sheet: str, cue: str) -> dict: - try: - files, _ = cri.acb_data(cat, sheet) - except KeyError: - raise SongsError(f"cue sheet {sheet}: no such asset") from None - except Exception as e: - raise SongsError(f"cue sheet {sheet}: cannot be read ({type(e).__name__}: {e})") from None - return cue_length(files["acb"], cue, f"cue sheet {sheet}") - return bgm - - -# ---------------------------------------------------------------- document -JACKET_KEY = "Image/Jacket/{jacket}" -JACKET_SIZE = 320 # longer side of a written jacket, pixels -JACKET_QUALITY = 88 # WebP quality - - -def jacket_webp(image, size: int = JACKET_SIZE) -> bytes: - """WebP bytes of a jacket image (a Pillow image), scaled down (Lanczos) to at most `size` pixels on its longer - side; opaque images are written without alpha.""" - from PIL import Image - img = image.convert("RGBA") - if img.getextrema()[3][0] == 255: - img = img.convert("RGB") - w, h = img.size - if max(w, h) > size: - img = img.resize((max(1, round(w * size / max(w, h))), max(1, round(h * size / max(w, h)))), - Image.LANCZOS) - buf = io.BytesIO() - img.save(buf, format="WEBP", quality=JACKET_QUALITY, method=6) - return buf.getvalue() - - -def catalog_jacket(cat, size: int = JACKET_SIZE) -> Callable[[str], bytes]: - """jacket(name) for a catalog: the WebP bytes (jacket_webp) of the Texture2D `Image/Jacket/`.""" - import tempfile - from .export import Exporter - ex = Exporter(cat, Path(tempfile.gettempdir()), textures="deferred") - - def jacket(name: str) -> bytes: - key = JACKET_KEY.format(jacket=name) - try: - o = ex.key_object(key) - except KeyError: - raise SongsError(f"jacket {name}: no asset {key}") from None - except Exception as e: - raise SongsError(f"jacket {name}: {key} cannot be read ({type(e).__name__}: {e})") from None - if o is None or o.type.name != "Texture2D": - raise SongsError(f"jacket {name}: {key} is not a Texture2D") - return jacket_webp(o.read().image, size) - return jacket - - -def _by_id(rows: list[dict], table: str) -> dict: - out = {} - for r in rows: - if r.get("_id") in out: - raise SongsError(f"{table}: _id {r.get('_id')} occurs twice") - out[r.get("_id")] = r - return out - - -def build(tables: dict[str, list[dict]], table_sha: dict[str, str], fetch: Callable[[str], bytes], - bgm: Callable[[str, str], dict] | None, *, region: str, client: dict, catalog: dict, master_source: str, - master_version: str | None) -> dict: - """The songs document. `fetch(file name)`: a chart TextAsset's bytes (KeyError when there is none); `bgm(cue - sheet, cue)`: the BGM length (catalog_bgm), None to leave every song's `bgm.length` null.""" - from . import __version__ - text = Texts(tables["MasterText"]) - scores = _by_id(tables["MasterLiveMusicScore"], "MasterLiveMusicScore") - sounds = _by_id(tables["MasterSound"], "MasterSound") - sheets = _by_id(tables["MasterSoundCueSheet"], "MasterSoundCueSheet") - musics = sorted(_by_id(tables["MasterLiveMusic"], "MasterLiveMusic").values(), key=lambda r: r["_id"]) - - bands = [{"id": b["_id"], "name": text.get(b.get("_nameTextID")), "mainColor": b.get("_mainColorCode"), - "subColor": b.get("_subColorCode")} for b in sorted(tables["MasterBand"], key=lambda r: r["_id"])] - characters = [{"id": c["_id"], "bandId": c.get("_bandID"), "name": text.get(c.get("_nameTextID")), - "shortName": text.get(c.get("_shortNameTextID")), "mainColor": c.get("_mainColorCode")} - for c in sorted(tables["MasterCharacter"], key=lambda r: r["_id"])] - tags = [{"id": t["_id"], "name": text.get(t.get("_nameTextID"))} - for t in sorted(tables["MasterTag"], key=lambda r: r["_id"])] - categories = [{"id": c["_id"], "musicCategories": list(c.get("_musicCategories") or []), - "name": text.get(c.get("_textKey"))} - for c in sorted(tables["MasterLiveMusicCategory"], key=lambda r: r["_id"])] - - songs = [] - for m in musics: - where = f"MasterLiveMusic {m['_id']}" - charts = [] - for d in DIFFICULTIES: - sid = m.get(f"_{d}ID") - if not sid: - continue - row = scores.get(sid) - if row is None: - raise SongsError(f"{where}: {d} score {sid} is not in MasterLiveMusicScore") - key = deckdata.chart_key(row["_musicScoreTextFileName"]) - try: - raw = fetch(row["_musicScoreTextFileName"]) - except KeyError: - raise SongsError(f"chart {key} (MasterLiveMusicScore {sid}): no such asset") from None - charts.append({"difficulty": d, **chart_facts(row, key, raw)}) - snd = sounds.get(m.get("_musicSoundID")) - if snd is None: - raise SongsError(f"{where}: sound {m.get('_musicSoundID')} is not in MasterSound") - sheet = sheets.get(snd.get("_soundCueSheetID")) - if sheet is None: - raise SongsError(f"{where}: cue sheet {snd.get('_soundCueSheetID')} is not in MasterSoundCueSheet") - length = bgm(sheet["_cueSheetName"], snd["_cueName"]) if bgm is not None else None - songs.append({ - "id": m["_id"], - "sortOrder": m.get("_sortOrder"), - "startAt": m.get("_startAt"), - "defaultUnlock": m.get("_defaultUnlock"), - "title": text.get(m.get("_titleTextID")), - "ruby": text.get(m.get("_rubyTitleTextID")), - "phonetic": text.get(m.get("_phoneticTextID")), - "bandIds": list(m.get("_bandIDs") or []), - "bandName": text.get(m.get("_bandNameTextID")), - "vocalCharacterIds": list(m.get("_vocalCharacterIDs") or []), - "lyricist": text.get(m.get("_lyricistTextID")), - "composer": text.get(m.get("_composerTextID")), - "arranger": text.get(m.get("_arrangerTextID")), - "musicType": m.get("_musicType"), - "musicCategories": list(m.get("_musicCategories") or []), - "bestMusicTagIds": list(m.get("_bestMusicTagIDs") or []), - "jacket": m.get("_jacketAssetName"), - "gekisouMissions": [m.get("_gekisouMission1"), m.get("_gekisouMission2"), m.get("_gekisouMission3")], - "bgm": {"soundId": snd["_id"], "cueSheet": sheet["_cueSheetName"], "cue": snd["_cueName"], - "length": length}, - "charts": charts, - "master": {"MasterLiveMusic": m}, - }) - return { - "format": FORMAT, - "provenance": { - "region": region, - "client": {"versionName": client.get("versionName"), "versionCode": client.get("versionCode")}, - "catalog": {"resourceVersion": catalog.get("resourceVersion"), "sha256": catalog.get("sha256")}, - "master": {"source": master_source, "version": master_version, - "tables": {t: {"sha256": table_sha[t]} for t in TABLES}}, - "exporter": {"name": "nnnotes", "version": __version__, "chartFormat": deckdata.CHART_FORMAT}, - }, - "languages": list(LANGUAGES), - "bands": bands, - "characters": characters, - "tags": tags, - "categories": categories, - "songs": songs, - } - - -def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes], - bgm: Callable[[str, str], dict] | None, *, region: str, client: dict, catalog: dict, - jacket: Callable[[str], bytes] | None = None, jackets_dir=None) -> dict: - """Read the master data, every chart and every BGM cue sheet, then write the file `out` (gzip when it ends in - `.gz`) through a temporary file and a rename; with `jacket` and `jackets_dir`, first every song's jacket as - `/.webp`. Returns the summary.""" - from .cache import write_atomic - out = Path(out) - try: - tables, shas = deckdata.read_master(src, key, TABLES) - except deckdata.DeckDataError as e: - raise SongsError(str(e)) from None - doc = build(tables, shas, fetch, bgm, region=region, client=client, catalog=catalog, master_source=src.source, - master_version=src.version) - jackets = sorted({s["jacket"] for s in doc["songs"] if s["jacket"]}) if jacket is not None else [] - images = {name: jacket(name) for name in jackets} - if jackets: - d = Path(jackets_dir) - d.mkdir(parents=True, exist_ok=True) - for name, data in images.items(): - write_atomic(d / f"{name}.webp", data) - try: - data = deckdata.encode(deckdata._value(doc, "songs")) - except deckdata.DeckDataError as e: - raise SongsError(str(e)) from None - written = deckdata.file_bytes(data, out.name.endswith(".gz")) - out.parent.mkdir(parents=True, exist_ok=True) - write_atomic(out, written) - return {"out": str(out), "format": FORMAT, "region": region, "masterSource": src.source, - "masterVersion": src.version, "songs": len(doc["songs"]), - "charts": sum(len(s["charts"]) for s in doc["songs"]), "bgm": bgm is not None, - "jackets": len(jackets), - "bytes": len(data), "fileBytes": len(written), "sha256": hashlib.sha256(written).hexdigest()} diff --git a/tests/test_deckdata.py b/tests/test_deckdata.py index cb9c0e3..7bce071 100644 --- a/tests/test_deckdata.py +++ b/tests/test_deckdata.py @@ -1,5 +1,6 @@ -"""Deck data export (deckdata), the runtime note order of the chart converter (score.runtime_score) and the APK -version code, on synthetic master data, charts and manifests.""" +"""The deck input (deckdata: master data as served, chart records, canonical encoding), the runtime note order of the +chart converter (score.runtime_score) and the APK version code, on synthetic master data, charts and manifests. The +music data file that carries it is tested in test_musicdata.""" import gzip import hashlib import json @@ -40,15 +41,6 @@ JUDGEMENTS = [10, 1, 1, 21, 1, 11, 21, 21, 21, 21, 21, 21, 2, 1, 1, 21, 5] -def schema_validator(): - jsonschema = pytest.importorskip("jsonschema") - from pathlib import Path - doc = json.loads((Path(__file__).resolve().parents[1] / "docs" / "schema" / "deck-data.schema.json") - .read_text(encoding="utf-8")) - jsonschema.Draft202012Validator.check_schema(doc) - return jsonschema.Draft202012Validator(doc) - - # ---------------------------------------------------------------- converter order def test_runtime_notes_in_note_dictionary_order(): rs = score.runtime_score(CHART) @@ -138,33 +130,31 @@ def fetch(name): "catalog": {"resourceVersion": None, "sha256": "ab" * 32}} -def export(tmp_path, out="deck.json", rows=rows_of, charts=CHARTS, **kw): +def export(tmp_path, out="deck.json", rows=rows_of, charts=CHARTS, src=None, key=None, **kw): + """Read the master data and every chart, build the deck input and write it canonically to `out` (as + `music-data --full` carries it); the document.""" d = master_dir(tmp_path, rows) if not (tmp_path / "m").exists() else tmp_path / "m" - return deckdata.export(tmp_path / out, deckdata.master_files(d), KEY, fetcher(charts), **dict(PROV, **kw)) + tables, shas = deckdata.read_master(src or deckdata.master_files(d), key or KEY) + prov = dict(PROV, **kw) + doc = deckdata.build(tables, deckdata.charts(tables["MasterLiveMusicScore"], fetcher(charts)), + {"region": prov["region"], "tables": shas}) + data = deckdata.encode(doc) + (tmp_path / out).write_bytes(deckdata.file_bytes(data, out.endswith(".gz"))) + return doc # ---------------------------------------------------------------- the document -def test_export_document(tmp_path): - r = export(tmp_path) +def test_deck_input_document(tmp_path): + doc = export(tmp_path) raw = (tmp_path / "deck.json").read_bytes() assert raw.endswith(b"}\n") and raw.count(b"\n") == 1 and b"\r" not in raw - assert r["bytes"] == len(raw) and r["sha256"] == hashlib.sha256(raw).hexdigest() - assert (r["charts"], r["notes"], r["tables"]) == (3, 21, len(deckdata.TABLES)) - doc = json.loads(raw) + assert json.loads(raw) == json.loads(deckdata.encode(doc)) assert list(doc) == ["format", "provenance", "master", "charts"] and doc["format"] == "nnnotes.deck-data/1" - p = doc["provenance"] - assert list(p) == ["region", "client", "catalog", "master", "exporter"] - assert p["region"] == "xx" and p["client"] == {"versionName": "9.9.9", "versionCode": 99} - assert p["catalog"] == {"resourceVersion": None, "sha256": "ab" * 32} - assert p["master"]["source"] == "api" and p["master"]["version"] == "v-test" - names = [t for t, _ in deckdata.TABLES] - assert list(p["master"]["tables"]) == names == list(doc["master"]) - for t in names: + assert list(doc["master"]) == [t for t, _ in deckdata.TABLES] + assert len(doc["charts"]) == 3 and sum(len(c["notes"]["id"]) for c in doc["charts"]) == 21 + for t, _ in deckdata.TABLES: served = (tmp_path / "m" / f"{t}.bin").read_bytes() - assert p["master"]["tables"][t]["sha256"] == hashlib.sha256(served).hexdigest() - assert p["exporter"] == {"name": "nnnotes", "version": cli.__version__, - "chartFormat": score.convert(SMALL)["format"]} - schema_validator().validate(doc) + assert doc["provenance"]["tables"][t] == hashlib.sha256(served).hexdigest() def test_master_columns_and_values(tmp_path): @@ -225,9 +215,9 @@ def test_charts(tmp_path): def test_output_is_deterministic_and_gzip_has_no_name_or_time(tmp_path): - a = export(tmp_path, "a.json") - b = export(tmp_path, "b.json") - assert (tmp_path / "a.json").read_bytes() == (tmp_path / "b.json").read_bytes() and a["sha256"] == b["sha256"] + export(tmp_path, "a.json") + export(tmp_path, "b.json") + assert (tmp_path / "a.json").read_bytes() == (tmp_path / "b.json").read_bytes() export(tmp_path, "a.json.gz") export(tmp_path, "b.json.gz") gz = (tmp_path / "a.json.gz").read_bytes() @@ -250,8 +240,7 @@ def nan(name): return [{"_id": 1, "_comboBonusType": 1, "_requiredComboCount": 10, "_bonusFactor": float("nan")}] return rows_of(name) with pytest.raises(deckdata.DeckDataError, match="MasterLiveComboScoreBonus row 0 _bonusFactor: NaN"): - deckdata.export(tmp_path / "n.json", deckdata.master_files(master_dir(tmp_path, nan, name="n")), KEY, - fetcher(CHARTS), **PROV) + export(tmp_path, "n.json", src=deckdata.master_files(master_dir(tmp_path, nan, name="n"))) assert not (tmp_path / "n.json").exists() @@ -260,8 +249,8 @@ def failing(tmp_path, match, *, src=None, charts=CHARTS, key=KEY): if src is None: src = deckdata.master_files(tmp_path / "m" if (tmp_path / "m").exists() else master_dir(tmp_path)) with pytest.raises(deckdata.DeckDataError, match=match): - deckdata.export(tmp_path / "out.json", src, key, fetcher(charts), **PROV) - assert not (tmp_path / "out.json").exists() and list(tmp_path.glob("out.json*")) == [] + export(tmp_path, "out.json", src=src, key=key, charts=charts) + assert not (tmp_path / "out.json").exists() def test_missing_chart_asset(tmp_path): @@ -369,11 +358,8 @@ def test_apk_master_and_client(tmp_path): assert (src.source, src.version) == ("embedded", "v-test") client = deckdata.apk_client(apk) assert client == {"versionName": "9.9.9", "versionCode": 99} - deckdata.export(tmp_path / "e.json", src, KEY, fetcher(CHARTS), region=deckdata.EMBEDDED, client=client, - catalog=PROV["catalog"]) - export(tmp_path, "f.json") - e, f = (json.loads((tmp_path / n).read_bytes()) for n in ("e.json", "f.json")) - assert e["provenance"]["master"]["source"] == "embedded" and e["provenance"]["region"] == "embedded" + e = export(tmp_path, "e.json", src=src) + f = export(tmp_path, "f.json") assert e["master"] == f["master"] and e["charts"] == f["charts"] with zipfile.ZipFile(tmp_path / "empty.apk", "w") as z: z.writestr("x", b"") @@ -419,40 +405,9 @@ def run(argv, capsys): return code, out, err + message -def test_command(tmp_path, capsys, monkeypatch): - d = master_dir(tmp_path) - out = tmp_path / "o" / "deck.json.gz" - code, _, err = run(["deck-data", "-o", str(out)], capsys) - assert code == 2 and "--master-files" in err and "--apk-master" in err - code, _, err = run(["deck-data", "--master-files", str(d), "-o", str(out)], capsys) - assert code == 2 and "catalog.region" in err - code, _, err = run(["--region", "xx", "deck-data", "--master-files", str(d), "-o", str(out)], capsys) - assert code == 2 and "master.key" in err - code, _, err = run(["deck-data", "--apk-master", "-o", str(out)], capsys) - assert code == 2 and "paths.apk" in err - monkeypatch.setenv("NNNOTES_MASTER_KEY", synth.MASTER_KEY.hex()) - monkeypatch.setenv("NNNOTES_MASTER_IV", synth.MASTER_IV.hex()) - monkeypatch.setattr(cli, "open_catalog", lambda cfg, **kw: FakeCatalog(CHARTS)) - monkeypatch.setattr(score, "fetch_chart", lambda cat, name: cat.charts[name]) - code, stdout, err = run(["--region", "xx", "--cache", str(tmp_path / "cache"), "deck-data", "--master-files", - str(d), "-o", str(out)], capsys) - assert code == 0, err - r = json.loads(stdout) - assert (r["charts"], r["masterVersion"], r["region"]) == (3, "v-test", "xx") - doc = json.loads(gzip.decompress(out.read_bytes())) - assert doc["provenance"]["catalog"] == {"resourceVersion": None, - "sha256": hashlib.sha256(b"remote catalog").hexdigest()} - assert doc["provenance"]["client"] == {"versionName": None, "versionCode": None} - assert synth.MASTER_KEY.hex() not in stdout + err - monkeypatch.setattr(cli, "open_catalog", lambda cfg, **kw: FakeCatalog({})) - code, _, err = run(["--region", "xx", "--cache", str(tmp_path / "cache"), "deck-data", "--master-files", - str(d), "-o", str(tmp_path / "x.json")], capsys) - assert code == 1 and "no such asset" in err and not (tmp_path / "x.json").exists() - - def test_the_tables_are_documented(): from pathlib import Path - text = (Path(__file__).resolve().parents[1] / "docs" / "deck-data.md").read_text(encoding="utf-8") + text = (Path(__file__).resolve().parents[1] / "docs" / "music-data.md").read_text(encoding="utf-8") documented = [] for line in text.splitlines(): if line.startswith("| `Master"): diff --git a/tests/test_musicdata.py b/tests/test_musicdata.py new file mode 100644 index 0000000..b292ba6 --- /dev/null +++ b/tests/test_musicdata.py @@ -0,0 +1,433 @@ +"""Music data export (musicdata): metadata, chart facts, BGM length, the deck model's chart statistics and the deck +input on synthetic master data, charts and ACBs. The deck model is a stand-in (FakeDeck) except where the built +extension module is tested itself.""" +import gzip +import hashlib +import io +import json +import re +import sys +from pathlib import Path + +import pytest + +import synth +from nnnotes import cli, deckdata, musicdata, score +from nnnotes.master import MasterKey +from test_deckdata import CHART, SMALL, FakeCatalog, rows_of, run +from test_voices import simple_acb + +KEY = MasterKey(synth.MASTER_KEY, synth.MASTER_IV) +ROOT = Path(__file__).resolve().parents[1] + + +def text(tid, stem): + return {"_id": tid, "_japanese": f"{stem}-ja", "_english": f"{stem}-en", "_traditionalChinese": f"{stem}-tw", + "_simplifiedChinese": f"{stem}-cn", "_korean": f"{stem}-ko"} + + +MUSIC = {"_id": 100002, "_sortOrder": 2, "_startAt": "2026/01/01 0:00:00", "_defaultUnlock": True, + "_titleTextID": "T2", "_rubyTitleTextID": "", "_phoneticTextID": "P2", "_bandIDs": [1], + "_bandNameTextID": "", "_vocalCharacterIDs": [1], "_lyricistTextID": "L2", "_composerTextID": "C2", + "_arrangerTextID": "", "_musicType": 4, "_musicCategories": [1], "_bestMusicTagIDs": [1], + "_jacketAssetName": "jkt_2", "_gekisouMission1": 1, "_gekisouMission2": 3, "_gekisouMission3": 3, + "_musicSoundID": 13, "_easyID": 20, "_normalID": 0, "_hardID": 0, "_expertID": 30, "_liveScoreRankGroup": 9, + "_extra": [7]} +MUSIC1 = dict(MUSIC, _id=100001, _sortOrder=1, _titleTextID="T1", _bandNameTextID="BN", _easyID=10, _expertID=0, + _liveScoreRankGroup=7) +TABLE_ROWS = { + "MasterLiveMusic": [MUSIC, MUSIC1], + "MasterLiveMusicScore": [ + {"_id": 30, "_musicScoreTextFileName": "c/c_03", "_musicScoreLevel": 20, "_fullComboCount": 2, + "_musicScoreDisplayLevel": 20.5}, + {"_id": 10, "_musicScoreTextFileName": "c/c_01", "_musicScoreLevel": 5, "_fullComboCount": 11, + "_musicScoreDisplayLevel": 5.0}, + {"_id": 20, "_musicScoreTextFileName": "c/c_02", "_musicScoreLevel": 9, "_fullComboCount": 2, + "_musicScoreDisplayLevel": 9.0}, + {"_id": 40, "_musicScoreTextFileName": "c/c_04", "_musicScoreLevel": 1, "_fullComboCount": 2, + "_musicScoreDisplayLevel": 1.0}], # a chart of no song + "MasterText": [text("T1", "one"), text("T2", "two"), text("P2", "pho"), text("L2", "lyr"), text("C2", "com"), + text("BN", "crychic"), text("Band1", "mygo"), text("Ch1", "tomori"), text("Ch1s", "tomo"), + text("Tag1", "tag"), text("Cat1", "original")], + "MasterBand": [{"_id": 1, "_nameTextID": "Band1", "_mainColorCode": "#3388BB", "_subColorCode": "#FFFFFF"}], + "MasterCharacter": [{"_id": 1, "_nameTextID": "Ch1", "_shortNameTextID": "Ch1s", "_bandID": 1, + "_mainColorCode": "#77BBDD"}], + "MasterTag": [{"_id": 1, "_nameTextID": "Tag1"}], + "MasterLiveMusicCategory": [{"_id": 1, "_musicCategories": [1], "_textKey": "Cat1"}], + "MasterSound": [{"_id": 13, "_soundCueSheetID": 5, "_cueName": "song2"}], + "MasterSoundCueSheet": [{"_id": 5, "_cueSheetName": "Bgm2"}], + "MasterLiveScoreRank": [ + {"_id": 3, "_group": 9, "_liveScoreRank": 7, "_requiredScore": 900, "_battleLiveRequiredScore": 1800}, + {"_id": 1, "_group": 9, "_liveScoreRank": 2, "_requiredScore": 0, "_battleLiveRequiredScore": 0}, + {"_id": 2, "_group": 8, "_liveScoreRank": 6, "_requiredScore": 5, "_battleLiveRequiredScore": 6}], +} +CHARTS = {"c/c_01": gzip.compress(json.dumps(CHART).encode("utf-8"), mtime=0), + "c/c_02": json.dumps(SMALL).encode("utf-8"), + "c/c_03": gzip.compress(json.dumps(SMALL).encode("utf-8"), mtime=0), + "c/c_04": json.dumps(SMALL).encode("utf-8")} +ACB = simple_acb({"song2": [1]}, [1], {1: 48000 * 90 + 24}) # 90.0005 s at 48 kHz, cue Length 100 +PROV = {"region": "xx", "client": {"versionName": "9.9.9", "versionCode": 99}, + "catalog": {"resourceVersion": None, "sha256": "ab" * 32}} + + +def all_rows(name): + """The song tables above; the deck model's tables of test_deckdata.""" + return TABLE_ROWS[name] if name in TABLE_ROWS else rows_of(name) + + +def master_dir(tmp_path, rows=TABLE_ROWS): + d = tmp_path / "m" + d.mkdir() + files = [] + for t in musicdata.tables_of(True, True): + data = synth.master_file({"_allData": rows[t] if t in rows else rows_of(t)}) + (d / f"{t}.bin").write_bytes(data) + files.append({"name": f"{t}.bin", "hash": hashlib.sha256(data).hexdigest(), "size": len(data)}) + (d / "MasterManifest.json").write_text(json.dumps({"version": "v-test", "files": files}), encoding="utf-8") + return d + + +def bgm(sheet, cue): + assert sheet == "Bgm2" + return musicdata.cue_length(ACB, cue, f"cue sheet {sheet}") + + +# ---------------------------------------------------------------- a stand-in deck model +WEIGHT = "0.30000000000000004" # a binary64 value: kept as the deck model writes it + + +class FakeDeck: + """The interface of nnnotes._deck: statistics made from the deck input as the model reports them.""" + COMMIT = "7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6" + + def __init__(self, change=None, fail=None): + self.change, self.fail, self.inputs = change, fail, [] + + def info(self): + return {"name": "ournotes-deck", "version": "0.0.1", "source": "https://github.com/empty-sekai/ournotes-deck", + "commit": self.COMMIT, "dataFormat": deckdata.DECK_FORMAT, "format": "ournotes-deck.chart-stats/2"} + + def chart_stats(self, data, seeds, workers): + if self.fail: + raise ValueError(self.fail) + doc = json.loads(data) + self.inputs.append((doc, seeds, workers)) + assert doc["format"] == deckdata.DECK_FORMAT and list(doc["master"]) == [t for t, _ in deckdata.TABLES] + m = doc["master"] + + def table(name): + return [dict(zip(m[name]["columns"], r)) for r in m[name]["rows"]] + levels = {r["_id"]: r["_musicScoreLevel"] for r in table("MasterLiveMusicScore")} + songs = {} + for r in table("MasterLiveMusic"): + for d in musicdata.DIFFICULTIES: + if r[f"_{d}ID"]: + songs[r[f"_{d}ID"]] = (r["_id"], d, [r["_gekisouMission1"], r["_gekisouMission2"], + r["_gekisouMission3"]]) + charts = [] + for c in doc["charts"]: + music, d, missions = songs[c["scoreId"]] + last = max(c["notes"]["timeMs"]) + fevers = list(zip(c["fevers"]["startMs"], c["fevers"]["endMs"]))[:3] + s = {"scoreId": c["scoreId"], "musicId": music, "difficulty": d, "level": levels[c["scoreId"]], + "judgedNotes": sum(score.is_judgement_note(op) for op in c["notes"]["op"]), + "convertedNoteCount": len(c["notes"]["id"]), "lastNoteMs": last, "musicLengthMs": last + 1000, + "skip": 1.5, "events": [[i % 5, t] for i, t in enumerate(c["skillEvents"]["timeMs"])], + "positions": min(len(c["skillEvents"]["timeMs"]), 5), "missions": missions, + "ranges": [{"index": i, "mission": missions[i], "startMs": a, "endMs": b, "rankBonusPercent": 10} + for i, (a, b) in enumerate(fevers)], + "justNotes": 0, + "seeds": [{"seed": 0, "score": 1234, "ranges": [], "weights": [["W"]], + "check": {"deck": [[0, 5000]], "exact": 2000, "predicted": 2000.25, "bound": 7.0}}]} + if self.change: + self.change(s) + charts.append(s) + out = json.dumps({"format": "ournotes-deck.chart-stats/2", "source": {}, "model": {"power": 300000}, + "kinds": [{"id": 0, "effectType": 2000, "activationTimeSecond": 7.5}], "charts": charts}) + return out.replace('"W"', WEIGHT) + + +def export(tmp_path, rows=TABLE_ROWS, charts=CHARTS, bgm=bgm, out="music.json", deck=None, **kw): + d = master_dir(tmp_path, rows) + return musicdata.export(tmp_path / out, deckdata.master_files(d), KEY, charts.__getitem__, bgm, **PROV, + deck=musicdata.Deck(module=deck, workers=3) if deck is not None else None, **kw) + + +def schema_validator(): + jsonschema = pytest.importorskip("jsonschema") + doc = json.loads((ROOT / "docs" / "schema" / "music-data.schema.json").read_text(encoding="utf-8")) + jsonschema.Draft202012Validator.check_schema(doc) + return jsonschema.Draft202012Validator(doc) + + +# ---------------------------------------------------------------- the document +def test_document(tmp_path): + r = export(tmp_path) + raw = (tmp_path / "music.json").read_bytes() + assert raw.endswith(b"}\n") and raw.count(b"\n") == 1 + assert (r["songs"], r["charts"], r["deck"], r["full"]) == (2, 3, None, False) + assert r["sha256"] == hashlib.sha256(raw).hexdigest() + doc = json.loads(raw) + assert list(doc) == ["format", "provenance", "languages", "bands", "characters", "tags", "categories", "deck", + "songs"] + assert doc["format"] == "nnnotes.music-data/1" and doc["languages"] == ["ja", "en", "zh-Hant", "zh-Hans", "ko"] + p = doc["provenance"] + assert list(p) == ["region", "client", "catalog", "master", "exporter", "deck"] and p["deck"] is None + assert list(p["master"]["tables"]) == list(musicdata.SONG_TABLES) # without the deck model: its tables only + assert p["exporter"] == {"name": "nnnotes", "version": cli.__version__, "chartFormat": deckdata.CHART_FORMAT} + assert doc["deck"] is None + assert doc["bands"] == [{"id": 1, "name": {"ja": "mygo-ja", "en": "mygo-en", "zh-Hant": "mygo-tw", + "zh-Hans": "mygo-cn", "ko": "mygo-ko"}, + "mainColor": "#3388BB", "subColor": "#FFFFFF"}] + assert doc["characters"][0]["name"]["ja"] == "tomori-ja" and doc["characters"][0]["bandId"] == 1 + assert doc["categories"] == [{"id": 1, "musicCategories": [1], "name": dict(doc["categories"][0]["name"])}] + one, two = doc["songs"] + assert (one["id"], two["id"]) == (100001, 100002) # sorted by id + assert one["bandName"]["ja"] == "crychic-ja" and two["bandName"] is None + assert two["title"]["zh-Hans"] == "two-cn" and two["ruby"] is None and two["arranger"] is None + assert two["lyricist"]["en"] == "lyr-en" and two["gekisouMissions"] == [1, 3, 3] + assert two["bgm"] == {"soundId": 13, "cueSheet": "Bgm2", "cue": "song2", + "length": {"lengthMs": 100, "samples": 48000 * 90 + 24, "sampleRate": 48000, + "durationMs": 90000}} + assert two["master"]["MasterLiveMusic"]["_extra"] == [7] # the whole row + assert two["scoreRanks"] == [{"rank": "D", "requiredScore": 0, "battleRequiredScore": 0}, # by required score + {"rank": "SS", "requiredScore": 900, "battleRequiredScore": 1800}] + assert [r["_id"] for r in two["master"]["MasterLiveScoreRank"]] == [1, 3] + assert one["scoreRanks"] == [] and one["master"]["MasterLiveScoreRank"] == [] # a group without rows + assert [c["difficulty"] for c in two["charts"]] == ["easy", "expert"] # difficulties without a score are left out + assert [c["difficulty"] for c in one["charts"]] == ["easy"] + assert all(c["deck"] is None for s in doc["songs"] for c in s["charts"]) + schema_validator().validate(doc) + + +def test_chart_facts(tmp_path): + export(tmp_path) + doc = json.loads((tmp_path / "music.json").read_bytes()) + c = doc["songs"][0]["charts"][0] # CHART at 120 BPM + assert (c["scoreId"], c["level"], c["displayLevel"], c["fullComboCount"]) == (10, 5, 5.0, 11) + assert c["asset"] == {"key": "Live/MusicScore/c/c_01", "sha256": hashlib.sha256(CHARTS["c/c_01"]).hexdigest()} + rs = score.runtime_score(CHART) + judged = sorted(n.pos.ms for n in rs.notes if score.is_judgement_note(n.op)) + assert c["notes"]["judged"] == len(judged) and c["notes"]["total"] == len(rs.notes) == 17 + assert sum(c["notes"]["byOperateType"].values()) == 17 and list(c["notes"]["byOperateType"])[0] == "1" + assert c["bpm"] == {"main": 120, "min": 120, "max": 120, "changes": [{"timeMs": 0, "bpm": 120}]} + assert (c["firstNoteMs"], c["lastJudgedNoteMs"], c["lastNoteMs"], c["musicLengthMs"]) == (0, 5000, 5000, 6000) + assert c["skillEventsMs"] == [2500, 500] and c["fevers"] == [[0, 1000], [2000, 2500]] + assert list(c)[-1] == "deck" + + +def test_bpm_facts(): + class P: + def __init__(self, ms): + self.ms = ms + ev = [(100.0, P(0)), (200.0, P(1000)), (150.0, P(1500)), (300.0, P(9000))] + f = musicdata.bpm_facts(ev, 500, 4000) # 100 for 500 ms, 200 for 500, 150 for 2500; 300 after + assert (f["main"], f["min"], f["max"]) == (150.0, 100.0, 200.0) + assert [x["timeMs"] for x in f["changes"]] == [0, 1000, 1500, 9000] + assert musicdata.bpm_facts(ev, 1200, 1200)["main"] == 200.0 # one instant: the BPM at that time + tie = [(120.0, P(0)), (180.0, P(1000))] + assert musicdata.bpm_facts(tie, 0, 2000)["main"] == 120.0 # equal time: the earliest + with pytest.raises(musicdata.MusicDataError): + musicdata.bpm_facts([], 0, 1) + + +# ---------------------------------------------------------------- the deck model +def test_deck(tmp_path): + fake = FakeDeck() + r = export(tmp_path, deck=fake) + raw = (tmp_path / "music.json").read_bytes() + doc = json.loads(raw) + assert r["deck"] == FakeDeck.COMMIT and r["unplayable"] == 0 + (deck_input, seeds, workers), = fake.inputs + assert (seeds, workers) == (musicdata.DECK_SEEDS, 3) + assert [c["scoreId"] for c in deck_input["charts"]] == [10, 20, 30] # the songs' charts, not chart 40 + assert deck_input["provenance"]["master"] == {"source": "api", "version": "v-test"} + p = doc["provenance"] + assert p["deck"] == {k: fake.info()[k] for k in ("name", "version", "source", "commit", "format")} + assert list(p["master"]["tables"]) == list(musicdata.tables_of(True, False)) + assert doc["deck"] == {"model": {"power": 300000}, + "kinds": [{"id": 0, "effectType": 2000, "activationTimeSecond": 7.5}]} + c = doc["songs"][0]["charts"][0] + assert list(c["deck"]) == list(musicdata.DECK_CHART_KEYS) + assert c["deck"]["events"] == [[0, 2500], [1, 500]] and c["deck"]["ranges"][1]["startMs"] == 2000 + assert c["deck"]["unplayable"] is None and c["deck"]["skip"] == 1.5 + # the deck model's numbers as it writes them: a binary64 value is not narrowed to binary32 + assert b'"weights":[[' + WEIGHT.encode() + b']]' in raw and b'"predicted":2000.25' in raw + assert "master" not in doc and "charts" not in doc + schema_validator().validate(doc) + (tmp_path / "m").rename(tmp_path / "m0") + export(tmp_path, deck=FakeDeck(), out="again.json") + assert (tmp_path / "again.json").read_bytes() == raw + + +@pytest.mark.parametrize("change, match", [ + (lambda s: s.update(judgedNotes=s["judgedNotes"] + 1), "chart 10 .*judged note count"), + (lambda s: s.update(musicLengthMs=0), "music length 0 differs"), + (lambda s: s.update(level=99), "level 99 differs"), + (lambda s: s["missions"].reverse(), "Gekisou missions"), + (lambda s: s["events"].pop(), "skill event times"), + (lambda s: s["ranges"] and s["ranges"][0].update(endMs=1), "fevers"), + (lambda s: s.update(difficulty="hard"), "difficulty 'hard' differs"), +]) +def test_deck_checks(tmp_path, change, match): + with pytest.raises(musicdata.MusicDataError, match=match): + export(tmp_path, deck=FakeDeck(change)) + assert not (tmp_path / "music.json").exists() + + +def test_deck_errors(tmp_path, monkeypatch): + with pytest.raises(musicdata.MusicDataError, match="deck model: chart 30: the check deck scores"): + export(tmp_path, deck=FakeDeck(fail="chart 30: the check deck scores 1, the chart statistics predict 2")) + assert not (tmp_path / "music.json").exists() + monkeypatch.setitem(sys.modules, "nnnotes._deck", None) # an installation without the module + with pytest.raises(musicdata.MusicDataError, match="not built into this installation.*--no-deck"): + musicdata.Deck() + + +def test_deck_module(): + deck = pytest.importorskip("nnnotes._deck") + info = deck.info() + assert info["name"] == "ournotes-deck" and info["format"] == "ournotes-deck.chart-stats/2" + assert info["dataFormat"] == deckdata.DECK_FORMAT and re.fullmatch(r"[0-9a-f]{40}", info["commit"]) + lock = (ROOT / "rust" / "Cargo.lock").read_text(encoding="utf-8") + assert f"#{info['commit']}\"" in lock # the commit Cargo.lock pins + with pytest.raises(ValueError, match="not a deck data file"): + deck.chart_stats('{"format": "x"}') + assert musicdata.Deck().info()["commit"] == info["commit"] + + +def test_the_extension_version_is_the_package_version(): + from nnnotes import __version__ + cargo = (ROOT / "rust" / "Cargo.toml").read_text(encoding="utf-8") + assert re.search(r'^version = "([^"]+)"', cargo, re.M).group(1) == __version__ + + +# ---------------------------------------------------------------- the deck input (--full) +def test_full(tmp_path): + asked = [] + + def fetch(name): + asked.append(name) + return CHARTS[name] + r = export(tmp_path, charts=type("C", (), {"__getitem__": staticmethod(fetch)})(), full=True, deck=FakeDeck()) + doc = json.loads((tmp_path / "music.json").read_bytes()) + assert r["full"] is True and sorted(asked) == sorted(CHARTS) # every chart asset read once + assert list(doc)[-2:] == ["master", "charts"] + assert list(doc["master"]) == [t for t, _ in deckdata.TABLES] + assert [c["scoreId"] for c in doc["charts"]] == [10, 20, 30, 40] # every chart, songs or not + tables, _ = deckdata.read_master(deckdata.master_files(tmp_path / "m"), KEY) + assert doc["master"] == json.loads(deckdata.encode({"m": deckdata.master_subset(tables)}))["m"] + assert doc["charts"][0] == deckdata.chart_record(10, "Live/MusicScore/c/c_01", CHARTS["c/c_01"]) + schema_validator().validate(doc) + (tmp_path / "x").mkdir() + export(tmp_path / "x", full=True) # the deck input without the deck model + assert json.loads((tmp_path / "x" / "music.json").read_bytes())["charts"] == doc["charts"] + + +# ---------------------------------------------------------------- output, jackets and failures +def test_deterministic_and_no_bgm(tmp_path): + export(tmp_path, out="a.json") + (tmp_path / "m").rename(tmp_path / "m0") + export(tmp_path, out="b.json") + assert (tmp_path / "a.json").read_bytes() == (tmp_path / "b.json").read_bytes() + (tmp_path / "m").rename(tmp_path / "m1") + r = export(tmp_path, bgm=None, out="c.json.gz") + doc = json.loads(gzip.decompress((tmp_path / "c.json.gz").read_bytes())) + assert r["bgm"] is False and all(s["bgm"]["length"] is None for s in doc["songs"]) + + +def test_jacket_webp(): + from PIL import Image + img = Image.new("RGBA", (1024, 512), (10, 200, 30, 255)) + out = Image.open(io.BytesIO(musicdata.jacket_webp(img))) + assert (out.format, out.size, out.mode) == ("WEBP", (320, 160), "RGB") + img.putpixel((0, 0), (0, 0, 0, 0)) + out = Image.open(io.BytesIO(musicdata.jacket_webp(img, size=2000))) + assert (out.size, out.mode) == ((1024, 512), "RGBA") + + +def test_jackets(tmp_path): + from PIL import Image + asked = [] + + def jacket(name): + asked.append(name) + return musicdata.jacket_webp(Image.new("RGB", (64, 64), (1, 2, 3))) + r = export(tmp_path, jacket=jacket, jackets_dir=tmp_path / "j") + assert r["jackets"] == 1 and asked == ["jkt_2"] # the two songs share one jacket + assert Image.open(tmp_path / "j" / "jkt_2.webp").size == (64, 64) + (tmp_path / "x").mkdir() + assert export(tmp_path / "x", out="b.json")["jackets"] == 0 + + def missing(name): + raise musicdata.MusicDataError(f"jacket {name}: no asset") + (tmp_path / "y").mkdir() + with pytest.raises(musicdata.MusicDataError, match="jacket jkt_2"): + export(tmp_path / "y", jacket=missing, jackets_dir=tmp_path / "y" / "j") + assert not (tmp_path / "y" / "music.json").exists() + + +@pytest.mark.parametrize("change, match", [ + (lambda r: r["MasterText"].pop(0), "MasterText has no text 'T1'"), + (lambda r: r["MasterLiveMusicScore"].pop(0), "expert score 30 is not in MasterLiveMusicScore"), + (lambda r: r["MasterSound"].clear(), "sound 13 is not in MasterSound"), + (lambda r: r["MasterLiveMusic"].append(dict(MUSIC)), "MasterLiveMusic: _id 100002 occurs twice"), + (lambda r: r["MasterLiveScoreRank"][0].update(_liveScoreRank=8), "MasterLiveScoreRank 3: unknown rank 8"), +]) +def test_master_errors(tmp_path, change, match): + rows = json.loads(json.dumps(TABLE_ROWS)) + change(rows) + with pytest.raises(musicdata.MusicDataError, match=match): + export(tmp_path, rows=rows) + assert not (tmp_path / "music.json").exists() + + +def test_input_errors(tmp_path): + with pytest.raises(musicdata.MusicDataError, match="c/c_03 .*no such asset"): + export(tmp_path, charts={k: v for k, v in CHARTS.items() if k != "c/c_03"}) + (tmp_path / "m").rename(tmp_path / "m0") + with pytest.raises(musicdata.MusicDataError, match="not a chart"): + export(tmp_path, charts=dict(CHARTS, **{"c/c_02": b"\x1f\x8bnot"})) + (tmp_path / "m").rename(tmp_path / "m1") + with pytest.raises(musicdata.MusicDataError, match="no cue 'song2'"): + export(tmp_path, bgm=lambda s, c: musicdata.cue_length(simple_acb({"x": [1]}, [1], {1: 5}), c, s)) + (tmp_path / "m").rename(tmp_path / "m2") + with pytest.raises(musicdata.MusicDataError, match=r"c/c_04 \(MasterLiveMusicScore 40\): no such asset"): + export(tmp_path, charts={k: v for k, v in CHARTS.items() if k != "c/c_04"}, full=True) + + +# ---------------------------------------------------------------- command line +def test_command(tmp_path, capsys, monkeypatch): + d = master_dir(tmp_path) + out = tmp_path / "o" / "music.json" + code, _, err = run(["music-data", "-o", str(out)], capsys) + assert code == 2 and "--master-files" in err and "--apk-master" in err + code, _, err = run(["music-data", "--master-files", str(d), "--no-deck", "-o", str(out)], capsys) + assert code == 2 and "catalog.region" in err + code, _, err = run(["--region", "xx", "music-data", "--master-files", str(d), "--no-deck", "-o", str(out)], + capsys) + assert code == 2 and "master.key" in err + monkeypatch.setenv("NNNOTES_MASTER_KEY", synth.MASTER_KEY.hex()) + monkeypatch.setenv("NNNOTES_MASTER_IV", synth.MASTER_IV.hex()) + monkeypatch.setattr(cli, "open_catalog", lambda cfg, **kw: FakeCatalog(CHARTS)) + monkeypatch.setattr(score, "fetch_chart", lambda cat, name: cat.charts[name]) + base = ["--region", "xx", "--cache", str(tmp_path / "cache"), "music-data", "--master-files", str(d), "--no-bgm"] + code, stdout, err = run(base + ["--no-deck", "-o", str(out)], capsys) + assert code == 0, err + r = json.loads(stdout) + assert (r["songs"], r["charts"], r["bgm"], r["region"], r["deck"]) == (2, 3, False, "xx", None) + assert synth.MASTER_KEY.hex() not in stdout + err + fake = FakeDeck() + real = musicdata.Deck + monkeypatch.setattr(musicdata, "Deck", lambda **kw: real(module=fake, **kw)) + code, stdout, err = run(base + ["--full", "--seeds", "2", "--workers", "4", "-o", str(out) + ".gz"], capsys) + assert code == 0, err + r = json.loads(stdout) + assert (r["deck"], r["full"]) == (FakeDeck.COMMIT, True) and fake.inputs[0][1:] == (2, 4) + doc = json.loads(gzip.decompress(Path(str(out) + ".gz").read_bytes())) + assert doc["provenance"]["catalog"] == {"resourceVersion": None, + "sha256": hashlib.sha256(b"remote catalog").hexdigest()} + assert len(doc["charts"]) == 4 + monkeypatch.setattr(musicdata, "Deck", lambda **kw: real(module=FakeDeck(fail="boom"), **kw)) + code, _, err = run(base + ["-o", str(tmp_path / "x.json")], capsys) + assert code == 1 and "deck model: boom" in err and not (tmp_path / "x.json").exists() diff --git a/tests/test_songs.py b/tests/test_songs.py deleted file mode 100644 index 7d806b0..0000000 --- a/tests/test_songs.py +++ /dev/null @@ -1,218 +0,0 @@ -"""Songs export (songs): metadata, chart facts and BGM length on synthetic master data, charts and ACBs.""" -import gzip -import hashlib -import io -import json - -import pytest - -import synth -from nnnotes import cli, deckdata, score, songs -from nnnotes.master import MasterKey -from test_deckdata import CHART, SMALL, FakeCatalog, run -from test_voices import simple_acb - -KEY = MasterKey(synth.MASTER_KEY, synth.MASTER_IV) - - -def text(tid, stem): - return {"_id": tid, "_japanese": f"{stem}-ja", "_english": f"{stem}-en", "_traditionalChinese": f"{stem}-tw", - "_simplifiedChinese": f"{stem}-cn", "_korean": f"{stem}-ko"} - - -MUSIC = {"_id": 100002, "_sortOrder": 2, "_startAt": "2026/01/01 0:00:00", "_defaultUnlock": True, - "_titleTextID": "T2", "_rubyTitleTextID": "", "_phoneticTextID": "P2", "_bandIDs": [1], - "_bandNameTextID": "", "_vocalCharacterIDs": [1], "_lyricistTextID": "L2", "_composerTextID": "C2", - "_arrangerTextID": "", "_musicType": 4, "_musicCategories": [1], "_bestMusicTagIDs": [1], - "_jacketAssetName": "jkt_2", "_gekisouMission1": 1, "_gekisouMission2": 3, "_gekisouMission3": 3, - "_musicSoundID": 13, "_easyID": 20, "_normalID": 0, "_hardID": 0, "_expertID": 30, "_extra": [7]} -MUSIC1 = dict(MUSIC, _id=100001, _sortOrder=1, _titleTextID="T1", _bandNameTextID="BN", _easyID=10, _expertID=0) -TABLE_ROWS = { - "MasterLiveMusic": [MUSIC, MUSIC1], - "MasterLiveMusicScore": [ - {"_id": 30, "_musicScoreTextFileName": "c/c_03", "_musicScoreLevel": 20, "_fullComboCount": 2, - "_musicScoreDisplayLevel": 20.5}, - {"_id": 10, "_musicScoreTextFileName": "c/c_01", "_musicScoreLevel": 5, "_fullComboCount": 11, - "_musicScoreDisplayLevel": 5.0}, - {"_id": 20, "_musicScoreTextFileName": "c/c_02", "_musicScoreLevel": 9, "_fullComboCount": 2, - "_musicScoreDisplayLevel": 9.0}], - "MasterText": [text("T1", "one"), text("T2", "two"), text("P2", "pho"), text("L2", "lyr"), text("C2", "com"), - text("BN", "crychic"), text("Band1", "mygo"), text("Ch1", "tomori"), text("Ch1s", "tomo"), - text("Tag1", "tag"), text("Cat1", "original")], - "MasterBand": [{"_id": 1, "_nameTextID": "Band1", "_mainColorCode": "#3388BB", "_subColorCode": "#FFFFFF"}], - "MasterCharacter": [{"_id": 1, "_nameTextID": "Ch1", "_shortNameTextID": "Ch1s", "_bandID": 1, - "_mainColorCode": "#77BBDD"}], - "MasterTag": [{"_id": 1, "_nameTextID": "Tag1"}], - "MasterLiveMusicCategory": [{"_id": 1, "_musicCategories": [1], "_textKey": "Cat1"}], - "MasterSound": [{"_id": 13, "_soundCueSheetID": 5, "_cueName": "song2"}], - "MasterSoundCueSheet": [{"_id": 5, "_cueSheetName": "Bgm2"}], -} -CHARTS = {"c/c_01": gzip.compress(json.dumps(CHART).encode("utf-8"), mtime=0), - "c/c_02": json.dumps(SMALL).encode("utf-8"), - "c/c_03": gzip.compress(json.dumps(SMALL).encode("utf-8"), mtime=0)} -ACB = simple_acb({"song2": [1]}, [1], {1: 48000 * 90 + 24}) # 90.0005 s at 48 kHz, cue Length 100 -PROV = {"region": "xx", "client": {"versionName": "9.9.9", "versionCode": 99}, - "catalog": {"resourceVersion": None, "sha256": "ab" * 32}} - - -def master_dir(tmp_path, rows=TABLE_ROWS): - d = tmp_path / "m" - d.mkdir() - files = [] - for t in songs.TABLES: - data = synth.master_file({"_allData": rows.get(t, [])}) - (d / f"{t}.bin").write_bytes(data) - files.append({"name": f"{t}.bin", "hash": hashlib.sha256(data).hexdigest(), "size": len(data)}) - (d / "MasterManifest.json").write_text(json.dumps({"version": "v-test", "files": files}), encoding="utf-8") - return d - - -def bgm(sheet, cue): - assert sheet == "Bgm2" - return songs.cue_length(ACB, cue, f"cue sheet {sheet}") - - -def export(tmp_path, rows=TABLE_ROWS, charts=CHARTS, bgm=bgm, out="songs.json", **kw): - d = master_dir(tmp_path, rows) - return songs.export(tmp_path / out, deckdata.master_files(d), KEY, charts.__getitem__, bgm, **PROV, **kw) - - -def test_document(tmp_path): - r = export(tmp_path) - raw = (tmp_path / "songs.json").read_bytes() - assert raw.endswith(b"}\n") and raw.count(b"\n") == 1 - assert (r["songs"], r["charts"], r["sha256"]) == (2, 3, hashlib.sha256(raw).hexdigest()) - doc = json.loads(raw) - assert list(doc) == ["format", "provenance", "languages", "bands", "characters", "tags", "categories", "songs"] - assert doc["format"] == "nnnotes.songs/1" and doc["languages"] == ["ja", "en", "zh-Hant", "zh-Hans", "ko"] - assert list(doc["provenance"]["master"]["tables"]) == list(songs.TABLES) - assert doc["bands"] == [{"id": 1, "name": {"ja": "mygo-ja", "en": "mygo-en", "zh-Hant": "mygo-tw", - "zh-Hans": "mygo-cn", "ko": "mygo-ko"}, - "mainColor": "#3388BB", "subColor": "#FFFFFF"}] - assert doc["characters"][0]["name"]["ja"] == "tomori-ja" and doc["characters"][0]["bandId"] == 1 - assert doc["categories"] == [{"id": 1, "musicCategories": [1], "name": dict(doc["categories"][0]["name"])}] - one, two = doc["songs"] - assert (one["id"], two["id"]) == (100001, 100002) # sorted by id - assert one["bandName"]["ja"] == "crychic-ja" and two["bandName"] is None - assert two["title"]["zh-Hans"] == "two-cn" and two["ruby"] is None and two["arranger"] is None - assert two["lyricist"]["en"] == "lyr-en" and two["gekisouMissions"] == [1, 3, 3] - assert two["bgm"] == {"soundId": 13, "cueSheet": "Bgm2", "cue": "song2", - "length": {"lengthMs": 100, "samples": 48000 * 90 + 24, "sampleRate": 48000, - "durationMs": 90000}} - assert two["master"]["MasterLiveMusic"]["_extra"] == [7] # the whole row - assert [c["difficulty"] for c in two["charts"]] == ["easy", "expert"] # difficulties without a score are left out - assert [c["difficulty"] for c in one["charts"]] == ["easy"] - - -def test_chart_facts(tmp_path): - export(tmp_path) - doc = json.loads((tmp_path / "songs.json").read_bytes()) - c = doc["songs"][0]["charts"][0] # CHART at 120 BPM - assert (c["scoreId"], c["level"], c["displayLevel"], c["fullComboCount"]) == (10, 5, 5.0, 11) - assert c["asset"] == {"key": "Live/MusicScore/c/c_01", "sha256": hashlib.sha256(CHARTS["c/c_01"]).hexdigest()} - rs = score.runtime_score(CHART) - judged = sorted(n.pos.ms for n in rs.notes if score.is_judgement_note(n.op)) - assert c["notes"]["judged"] == len(judged) and c["notes"]["total"] == len(rs.notes) == 17 - assert sum(c["notes"]["byOperateType"].values()) == 17 and list(c["notes"]["byOperateType"])[0] == "1" - assert c["bpm"] == {"main": 120, "min": 120, "max": 120, "changes": [{"timeMs": 0, "bpm": 120}]} - assert (c["firstNoteMs"], c["lastJudgedNoteMs"], c["lastNoteMs"], c["musicLengthMs"]) == (0, 5000, 5000, 6000) - assert c["skillEventsMs"] == [2500, 500] and c["fevers"] == [[0, 1000], [2000, 2500]] - - -def test_bpm_facts(): - class P: - def __init__(self, ms): - self.ms = ms - ev = [(100.0, P(0)), (200.0, P(1000)), (150.0, P(1500)), (300.0, P(9000))] - f = songs.bpm_facts(ev, 500, 4000) # 100 for 500 ms, 200 for 500, 150 for 2500; 300 after - assert (f["main"], f["min"], f["max"]) == (150.0, 100.0, 200.0) - assert [x["timeMs"] for x in f["changes"]] == [0, 1000, 1500, 9000] - assert songs.bpm_facts(ev, 1200, 1200)["main"] == 200.0 # one instant: the BPM at that time - tie = [(120.0, P(0)), (180.0, P(1000))] - assert songs.bpm_facts(tie, 0, 2000)["main"] == 120.0 # equal time: the earliest - with pytest.raises(songs.SongsError): - songs.bpm_facts([], 0, 1) - - -def test_deterministic_and_no_bgm(tmp_path): - export(tmp_path, out="a.json") - (tmp_path / "m").rename(tmp_path / "m0") - export(tmp_path, out="b.json") - assert (tmp_path / "a.json").read_bytes() == (tmp_path / "b.json").read_bytes() - (tmp_path / "m").rename(tmp_path / "m1") - r = export(tmp_path, bgm=None, out="c.json.gz") - doc = json.loads(gzip.decompress((tmp_path / "c.json.gz").read_bytes())) - assert r["bgm"] is False and all(s["bgm"]["length"] is None for s in doc["songs"]) - - -def test_jacket_webp(): - from PIL import Image - img = Image.new("RGBA", (1024, 512), (10, 200, 30, 255)) - out = Image.open(io.BytesIO(songs.jacket_webp(img))) - assert (out.format, out.size, out.mode) == ("WEBP", (320, 160), "RGB") - img.putpixel((0, 0), (0, 0, 0, 0)) - out = Image.open(io.BytesIO(songs.jacket_webp(img, size=2000))) - assert (out.size, out.mode) == ((1024, 512), "RGBA") - - -def test_jackets(tmp_path): - from PIL import Image - asked = [] - - def jacket(name): - asked.append(name) - return songs.jacket_webp(Image.new("RGB", (64, 64), (1, 2, 3))) - r = export(tmp_path, jacket=jacket, jackets_dir=tmp_path / "j") - assert r["jackets"] == 1 and asked == ["jkt_2"] # the two songs share one jacket - assert Image.open(tmp_path / "j" / "jkt_2.webp").size == (64, 64) - (tmp_path / "x").mkdir() - assert export(tmp_path / "x", out="b.json")["jackets"] == 0 - - def missing(name): - raise songs.SongsError(f"jacket {name}: no asset") - (tmp_path / "y").mkdir() - with pytest.raises(songs.SongsError, match="jacket jkt_2"): - export(tmp_path / "y", jacket=missing, jackets_dir=tmp_path / "y" / "j") - assert not (tmp_path / "y" / "songs.json").exists() - - -@pytest.mark.parametrize("change, match", [ - (lambda r: r["MasterText"].pop(0), "MasterText has no text 'T1'"), - (lambda r: r["MasterLiveMusicScore"].pop(0), "expert score 30 is not in MasterLiveMusicScore"), - (lambda r: r["MasterSound"].clear(), "sound 13 is not in MasterSound"), - (lambda r: r["MasterLiveMusic"].append(dict(MUSIC)), "MasterLiveMusic: _id 100002 occurs twice"), -]) -def test_master_errors(tmp_path, change, match): - rows = json.loads(json.dumps(TABLE_ROWS)) - change(rows) - with pytest.raises(songs.SongsError, match=match): - export(tmp_path, rows=rows) - assert not (tmp_path / "songs.json").exists() - - -def test_input_errors(tmp_path): - with pytest.raises(songs.SongsError, match="c/c_03 .*no such asset"): - export(tmp_path, charts={k: v for k, v in CHARTS.items() if k != "c/c_03"}) - (tmp_path / "m").rename(tmp_path / "m0") - with pytest.raises(songs.SongsError, match="not a chart"): - export(tmp_path, charts=dict(CHARTS, **{"c/c_02": b"\x1f\x8bnot"})) - (tmp_path / "m").rename(tmp_path / "m1") - with pytest.raises(songs.SongsError, match="no cue 'song2'"): - export(tmp_path, bgm=lambda s, c: songs.cue_length(simple_acb({"x": [1]}, [1], {1: 5}), c, s)) - - -def test_command(tmp_path, capsys, monkeypatch): - d = master_dir(tmp_path) - out = tmp_path / "o" / "songs.json" - code, _, err = run(["songs", "-o", str(out)], capsys) - assert code == 2 and "--master-files" in err - monkeypatch.setenv("NNNOTES_MASTER_KEY", synth.MASTER_KEY.hex()) - monkeypatch.setenv("NNNOTES_MASTER_IV", synth.MASTER_IV.hex()) - monkeypatch.setattr(cli, "open_catalog", lambda cfg, **kw: FakeCatalog(CHARTS)) - monkeypatch.setattr(score, "fetch_chart", lambda cat, name: cat.charts[name]) - code, stdout, err = run(["--region", "xx", "--cache", str(tmp_path / "cache"), "songs", "--master-files", - str(d), "--no-bgm", "-o", str(out)], capsys) - assert code == 0, err - r = json.loads(stdout) - assert (r["songs"], r["charts"], r["bgm"], r["region"]) == (2, 3, False, "xx") - assert synth.MASTER_KEY.hex() not in stdout + err From a03591e7fd4f186df7ddbe161c787c7cad138b41 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 30 Sep 2026 00:46:59 +0800 Subject: [PATCH 10/11] feat(music-data): play scenarios: Gekisou off, every rank, the Perfect play The deck model moves to ournotes-deck d4abb52 (chart-stats scenarios). Every chart's `deck` carries `offSeeds`, the chart measured with Gekisou off as a solo live plays; its ranges carry `rankBonusPercents` and its seeds `rangeWeights`, `rankCheck`, `scorePerfect` and `rangeScorePerfect`, from which a page derives a Gekisou live at any ranks and a Just rate. The statistics are checked against the master data and themselves: the rank bonus percentages against the MasterLiveGekisouRankingScoreBonus rows of the song's mission pattern, every rank bonus against trunc(rangeScore * percent / 100), the Perfect play of a chart without Just notes against the play, the array shapes, one Gekisou off seed, and every check and rank check within its bound. docs/music-data.md and the schema describe the new fields (optional in the schema: older files do not have them). Co-Authored-By: Claude Opus 5.5 --- docs/commands.md | 5 +- docs/music-data.md | 73 +++++++++++---- docs/schema/music-data.schema.json | 138 ++++++++++++++++++++++++++++- rust/Cargo.lock | 2 +- rust/Cargo.toml | 2 +- src/nnnotes/musicdata.py | 108 ++++++++++++++++++++-- tests/test_musicdata.py | 81 +++++++++++++++-- 7 files changed, 376 insertions(+), 33 deletions(-) diff --git a/docs/commands.md b/docs/commands.md index 8271439..0af1540 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -680,8 +680,9 @@ release time, jacket, Gekisou missions, score ranks, the whole `MasterLiveMusic` (read from the cue sheet's ACB, without decoding audio); per difficulty the chart facts (level and display level, full combo count, note counts, BPM, note times, the live's music length, skill event times, fever ranges) and the chart's deck statistics: the no-skill score and the weight of every score-up skill kind at every performance -position, measured by the deck model ournotes-deck (built into nnnotes as `nnnotes._deck`) on its whole-live -simulation and checked against the chart facts. `--full` also writes the deck model's input: every +position, with Gekisou on (a Gekisou live at rank 1, with what every other rank and the Perfect play need) and off (a +solo live), measured by the deck model ournotes-deck (built into nnnotes as `nnnotes._deck`) on its whole-live +simulation and checked against the chart facts and the master data. `--full` also writes the deck model's input: every `MasterLiveMusicScore` row's chart as the client builds it at runtime (notes, skill events, fever ranges) and the master data tables about cards, skills, bonuses, scores and events. `--no-deck` skips the deck model (every chart's `deck` is null); `--seeds N` (default 8) and `--workers N` (default: every processor) set its seeds on charts with a diff --git a/docs/music-data.md b/docs/music-data.md index 3fa9c84..ddbb563 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -3,8 +3,8 @@ `nnnotes music-data` writes one JSON file with every live song and chart of one master data version: what a song listing shows (titles and credits in every language, bands, vocal characters, category, tags, release time, score ranks, jacket, BGM length), per difficulty the chart facts (level, note counts, BPM, chart times, skill events, -fevers), and per chart its **deck statistics**: what the chart contributes to the live score whatever the deck, -measured by the deck model [ournotes-deck](https://github.com/empty-sekai/ournotes-deck), which nnnotes carries as +fevers), and per chart its **deck statistics**: what the chart contributes to the live score whatever the deck, in a +solo live (Gekisou off) and in a Gekisou live at every rank, measured by the deck model [ournotes-deck](https://github.com/empty-sekai/ournotes-deck), which nnnotes carries as its extension module `nnnotes._deck`. The format is `nnnotes.music-data/1`; its JSON Schema is [schema/music-data.schema.json](schema/music-data.schema.json). @@ -39,7 +39,8 @@ The command writes the file only when every table, chart and cue sheet was read or mismatching master data file, a table without a column the file exports, a text id that `MasterText` does not have, a score id that `MasterLiveMusicScore` does not have, a missing or unreadable chart asset, a note id that occurs twice in a chart, a cue sheet without the song's cue, (with `--jackets`) a missing jacket texture, a chart the -deck model cannot measure or whose check deck fails, or deck statistics that disagree with the chart facts stops it +deck model cannot measure or whose check deck fails, or deck statistics that disagree with the chart facts or the +master data stops it with exit status 1 and a line naming the input. The file is written through a temporary file and a rename. An installation without the extension module (a source checkout that was not built) runs only with `--no-deck`; @@ -138,12 +139,18 @@ plays, `musicLengthMs` is the length the score code uses; a listing chooses the ## Deck statistics -The deck model plays every chart of a song on its whole-live simulation, with Gekisou on as the game plays every -live, solo (rank 1), in the theoretical best play: every judged note at its exact time, Just inside the Just-count -ranges and Perfect elsewhere, the frame times of the game's default schedule. It measures, at deck power -`model.power` (300000): +The deck model plays every chart of a song on its whole-live simulation in the theoretical best play, the frame +times of the game's default schedule, in two scenarios: -- `score`: the exact score without skills, the rank bonuses of the Gekisou ranges included; +- **Gekisou on** (`seeds`), as a Gekisou live (Battle Live, up to five players) plays: every judged note at its exact + time, Just inside the Just-count ranges and Perfect elsewhere, rank 1 in every Gekisou range. Other ranks follow + from the same numbers (below). +- **Gekisou off** (`offSeeds`), as a solo live (Free Live, Challenge Live) plays: every judged note Perfect at its + exact time, seed 0, no Just, luck, Gekisou combo or rank bonus. A chart with more than three fevers plays here too. + +In each it measures, at deck power `model.power` (300000): + +- `score`: the exact score without skills (with Gekisou on, the rank 1 bonuses of the Gekisou ranges included); - for every **score-up kind** (`deck.kinds`) and every performance position `k`, `weights[kind][k]`: the exact score a deck gains when its position-`k` member has one effect of that kind at factor 1, divided by the deck power. The effect runs through the simulation's own updaters, conditions, frames and appliers, so the weight carries every rule @@ -163,15 +170,42 @@ master values at another power (`model.checkPower`), and the command fails when the formula. Effects of other types (cumulative score 2001 / 2003, life, judgement conversion, Gekisou and snap skills) are not linear in the chart alone and have no weights: a deck's score with them comes from the simulation. -A luck range draws lottery results from the play's random seed, so the measurements are given **per seed**: one seed -(0) when no range is a luck range, else the first `--seeds` seeds of the deck model's published seed set. The seed -set is not the game's seed law (which is unknown); a mean over it is not the game's expectation. +A luck range draws lottery results from the play's random seed, so the Gekisou on measurements are given **per +seed**: one seed (0) when no range is a luck range, else the first `--seeds` seeds of the deck model's published seed +set. The seed set is not the game's seed law (which is unknown); a mean over it is not the game's expectation. + +### Ranks + +In a Gekisou live, range `i` takes a rank `r_i` from 1 to 5 among the room's players, and its rank bonus is +`trunc(rangeScore_i * p_i(r_i) / 100)` with `p_i(r) = ranges[i].rankBonusPercents[r - 1]`. The bonus is a fixed score +at the range's end: it changes no factor and no note score, and a later range's score holds it at both ends, so the +range scores do not depend on the ranks. At ranks `r` a seed's numbers are therefore + +``` +score_r = score - sum_i rankBonus_i + sum_i trunc(rangeScore_i * p_i(r_i) / 100) (exact) +weights_r[kind][k] = weights[kind][k] + sum_i (p_i(r_i) - p_i(1)) / 100 * rangeWeights[kind][k][i] +``` + +(`rankBonus_i`, `rangeScore_i`: `seeds[].ranges[i]`), and a deck scores `P * (score_r / power + sum_k factor_k * +weights_r[kind_k][k])` as above; `weights_r` is within `2 * ranges / power` per unit of factor of the exact weight at +those ranks. At rank 1 everywhere these are `score` and `weights`. `rangeWeights` is null when a range's bonus can fall +inside another range's score frames (overlapping ranges), and a kind's entry is null when its conditions read the +confirmed rank (condition 7012): the ranks do not follow linearly there. `rankCheck` plays the seed's check deck at +random ranks through the simulation's explicit rank confirmations and bounds it against these formulas. + +### Just rate + +`scorePerfect` and `ranges[i].rangeScorePerfect` are the no-skill score (rank 1 bonuses included) and the range +scores of the same play with every Just judged Perfect (the Just judgement is enabled only inside the Just-count +ranges, so nothing else changes). Between a Just rate of 1 (`score`, `rangeScore`) and 0 (`scorePerfect`, +`rangeScorePerfect`) a page can interpolate; the rank bonus of the Perfect play is `trunc(rangeScorePerfect_i * +p_i(r_i) / 100)`. A chart without Just notes (`justNotes` 0) has `scorePerfect == score`. ### deck | Field | Content | |---|---| -| `model` | the deck model's description of the measurement: `engine`, `play`, `score` (the formula), `power`, `checkPower`, `unitValue` (the effect value of factor 1, 10000), `seeds` | +| `model` | the deck model's description of the measurement: `engine`, `play`, `score` (the formula), `power`, `checkPower`, `unitValue` (the effect value of factor 1, 10000), `seeds`, `ranks` (the rank formulas), `perfect` (the Perfect play), `off` (the Gekisou off scenario) | | `kinds[]` | the score-up kinds of the master data: `MasterLiveSkillEffect` rows of type 2000, 2002, 2004 or 2005 without a cumulative condition, grouped by what shapes their score. `id` (the index in `weights`), `effectType`, `activationTimeSecond`, `durationMs` (`ceil(activationTimeSecond * 1000f)`), `skillTargetIds`, `skillConditionGroup`, `skillReleaseConditionGroup`, `effectLimitCount`, `effectExecuteLimitCount`, `effectExecuteLimitResetConditionGroup`; `rows` (master rows of the kind) and `values` (their distinct `_effectValue`s, ascending) | The kind of a card's live skill is found by matching its `MasterLiveSkillEffect` row (at the skill level) on these @@ -185,14 +219,19 @@ fields; `values` lists what the master data uses. | `skip` | score per unit of deck power of a skipped live (every note Great, combo 0, no skills) | | `events` | `[[position, timeMs], ...]`: the skill events in chart order with the performance position each fires | | `positions` | the performance positions the events fire (the largest position + 1): the length of every `weights[kind]` | -| `ranges[]` | the Gekisou ranges: `index`, `mission` (1 combo, 2 luck, 3 Just count), `startMs`, `endMs`, `rankBonusPercent` (the solo rank bonus percentage of the song's mission pattern) | -| `justNotes` | notes judged Just on the play | -| `seeds[]` | per seed: `seed`; `score`; `ranges[]` (`rangeScore`: the score gained inside the range, `rankBonus`, `maxCombo`, `justCount`, `lotResults`: lottery results Miss, Hit, Super Hit, Critical); `weights[kind][position]`; `check` (`deck`: `[kind, value]` or null per position, `exact`, `predicted`, `bound`) | -| `unplayable` | null, or why the game cannot play the chart (more than three fevers: the game fails when the fourth starts); `seeds` is then empty | +| `ranges[]` | the Gekisou ranges: `index`, `mission` (1 combo, 2 luck, 3 Just count), `startMs`, `endMs`, `rankBonusPercents` (the rank bonus percentages of ranks 1..5 of the song's mission pattern, `MasterLiveGekisouRankingScoreBonus`), `rankBonusPercent` (the rank 1 percentage, `rankBonusPercents[0]`) | +| `justNotes` | notes judged Just on the Gekisou on play | +| `seeds[]` | Gekisou on, per seed: `seed`; `score` (points at `model.power`, rank 1 bonuses included); `ranges[]` (`rangeScore`: the points gained inside the range, `rankBonus`: its rank 1 bonus in points, `maxCombo`, `justCount`, `lotResults`: lottery results Miss, Hit, Super Hit, Critical, `rangeScorePerfect`: `rangeScore` on the Perfect play); `weights[kind][position]` (points per unit of deck power and of factor); `check` (`deck`: `[kind, value]` or null per position, `exact`, `predicted`, `bound`: points at `model.checkPower`); `scorePerfect` (`score` on the Perfect play); `rangeWeights[kind][position][range]` (range points per unit of deck power and of factor, or null; a kind null); `rankCheck` (`ranks`: 1..5 per range, `exact`, `predicted`, `bound`; null without ranges or range weights) | +| `offSeeds[]` | Gekisou off, one seed: `seed` (0), `score`, `weights[kind][position]` (a kind null when its conditions read the Gekisou state, which a solo live does not have) and `check`, as in `seeds[]` | +| `unplayable` | null, or why the game cannot play the chart with Gekisou (more than three fevers: the game fails when the fourth starts); `seeds` is then empty, `offSeeds` is not | The deck model's chart facts are checked against the file's: the song, difficulty, level, judged note count, last note time, music length, Gekisou missions, skill event times and fever ranges must agree, and are not repeated in -`deck`. +`deck`. Its numbers are checked against the master data and themselves: every range's `rankBonusPercents` are the +`MasterLiveGekisouRankingScoreBonus` rows of the song's mission pattern (0 without a row), every `rankBonus` is +`trunc(rangeScore * rankBonusPercent / 100)`, a chart without Just notes has the same scores on the Perfect play, +every array has its shape (`[kind][position]`, `[kind][position][range]`, one range result per range, one Gekisou off +seed), and every check and rank check is within its bound. ## The deck input (`--full`) diff --git a/docs/schema/music-data.schema.json b/docs/schema/music-data.schema.json index 18dcba2..42e8eda 100644 --- a/docs/schema/music-data.schema.json +++ b/docs/schema/music-data.schema.json @@ -546,7 +546,17 @@ "type": "integer" }, "rankBonusPercent": { - "type": "integer" + "type": "integer", + "description": "the rank 1 bonus percentage (rankBonusPercents[0])" + }, + "rankBonusPercents": { + "type": "array", + "items": { + "type": "integer" + }, + "minItems": 5, + "maxItems": 5, + "description": "the rank bonus percentages of ranks 1..5 of the song's mission pattern" } } } @@ -604,6 +614,10 @@ }, "minItems": 4, "maxItems": 4 + }, + "rangeScorePerfect": { + "type": "integer", + "description": "the range score on the Perfect play (every Just judged Perfect)" } } } @@ -618,6 +632,125 @@ }, "description": "weights[kind][position]: score per unit of deck power and of factor" }, + "check": { + "type": "object", + "required": [ + "deck", + "exact", + "predicted", + "bound" + ], + "properties": { + "deck": { + "type": "array", + "items": { + "type": [ + "array", + "null" + ] + } + }, + "exact": { + "type": "integer" + }, + "predicted": { + "type": "number" + }, + "bound": { + "type": "number" + } + } + }, + "scorePerfect": { + "type": "integer", + "description": "the exact no-skill score at model.power on the Perfect play (every Just judged Perfect), its rank 1 bonuses included" + }, + "rangeWeights": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "array", + "null" + ], + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + "description": "rangeWeights[kind][position][range]: range score per unit of deck power and of factor; null when the ranks do not follow linearly on the chart, a kind null when its conditions read the confirmed rank" + }, + "rankCheck": { + "type": [ + "object", + "null" + ], + "required": [ + "ranks", + "exact", + "predicted", + "bound" + ], + "description": "the check deck played at random ranks; null without ranges or range weights", + "properties": { + "ranks": { + "type": "array", + "items": { + "type": "integer", + "minimum": 1, + "maximum": 5 + } + }, + "exact": { + "type": "integer" + }, + "predicted": { + "type": "number" + }, + "bound": { + "type": "number" + } + } + } + } + } + }, + "offSeeds": { + "type": "array", + "description": "the measurements with Gekisou off (a solo live), one seed", + "items": { + "type": "object", + "required": [ + "seed", + "score", + "weights", + "check" + ], + "properties": { + "seed": { + "type": "integer" + }, + "score": { + "type": "integer", + "description": "the exact no-skill score at model.power, Gekisou off" + }, + "weights": { + "type": "array", + "items": { + "type": [ + "array", + "null" + ], + "items": { + "type": "number" + } + }, + "description": "weights[kind][position], Gekisou off; a kind null when its conditions read the Gekisou state" + }, "check": { "type": "object", "required": [ @@ -654,7 +787,8 @@ "type": [ "string", "null" - ] + ], + "description": "why the game cannot play the chart with Gekisou (seeds is then empty; offSeeds is not)" } } }, diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 9213264..9a76560 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -44,7 +44,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "ournotes-deck" version = "0.0.1" -source = "git+https://github.com/empty-sekai/ournotes-deck?rev=7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6#7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6" +source = "git+https://github.com/empty-sekai/ournotes-deck?rev=d4abb52ff75059a13e99a88fd6812ec1bdaa5dd2#d4abb52ff75059a13e99a88fd6812ec1bdaa5dd2" dependencies = [ "serde", "serde_json", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 3a28e84..92a1db1 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -15,7 +15,7 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.29", features = ["abi3-py311"] } # pinned by commit; .github/workflows/deck.yml opens a pull request when ournotes-deck moves -ournotes-deck = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6" } +ournotes-deck = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "d4abb52ff75059a13e99a88fd6812ec1bdaa5dd2" } serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } [profile.release] diff --git a/src/nnnotes/musicdata.py b/src/nnnotes/musicdata.py index 04ffa64..24157dd 100644 --- a/src/nnnotes/musicdata.py +++ b/src/nnnotes/musicdata.py @@ -2,7 +2,8 @@ `nnnotes.music-data/1`, docs/music-data.md): titles and credits in every language, bands, vocal characters, category, tags, release time, score ranks, the live BGM's length, per difficulty the chart facts (level, note counts, BPM, chart times, skill events, fever ranges) and the chart's deck statistics, what the chart contributes to the live score -whatever the deck, measured by the deck model ournotes-deck (the extension module nnnotes._deck). +whatever the deck, with Gekisou on (a Gekisou live, every rank) and off (a solo live), measured by the deck model +ournotes-deck (the extension module nnnotes._deck). Master data is read from the files as served (deckdata.master_files / apk_master: SHA-256 checked against the manifest, decoded with master.decode). Charts are the TextAssets `Live/MusicScore/` converted by @@ -255,12 +256,89 @@ def stats(self, deck_input: dict) -> dict: # the keys of a chart's statistics carried by `deck` (the others are checked against the chart facts) -DECK_CHART_KEYS = ("convertedNoteCount", "skip", "events", "positions", "ranges", "justNotes", "seeds", +DECK_CHART_KEYS = ("convertedNoteCount", "skip", "events", "positions", "ranges", "justNotes", "seeds", "offSeeds", "unplayable") +RANKS = 5 # the ranks of a Gekisou range (a Gekisou live has up to five players) +GEKISOU_RANGES = 3 # the Gekisou ranges of a live (its first three fevers) + + +def mission_pattern(missions) -> int: + """The mission pattern of a song's three Gekisou missions, the `_missionPattern` of its rank bonus rows: 0 when a + mission is missing, 1 all the same, 2 all different, 3 otherwise.""" + a, b, c = missions + if not (a and b and c): + return 0 + if a == b: + return 1 if a == c else 3 + return 2 if b != c and a != c else 3 + + +def rank_bonus_percents(rows: list[dict], missions) -> list[list]: + """The rank bonus percentages [range][rank - 1] of a song's missions from the MasterLiveGekisouRankingScoreBonus + rows of its mission pattern (`_count`: the range 1..3, `_rank` 1..5; a later row wins; 0 without a row).""" + pattern = mission_pattern(missions) + out = [[0] * RANKS for _ in range(GEKISOU_RANGES)] + for r in rows: + c, k = (r.get("_count") or 0) - 1, (r.get("_rank") or 0) - 1 + if r.get("_missionPattern") == pattern and 0 <= c < GEKISOU_RANGES and 0 <= k < RANKS: + out[c][k] = r.get("_scoreBonusPercent") + return out -def chart_deck(song: dict, chart: dict, stats: dict) -> dict: - """A chart's `deck` from its statistics, after checking them against the song and the chart facts.""" +def _trunc_percent(score: int, percent: int) -> int: + """trunc(score * percent / 100), the rank bonus of a range score.""" + p = score * percent + return p // 100 if p >= 0 else -(-p // 100) + + +def _numbers(v, n: int) -> bool: + """Whether v is a list of n numbers (as the deck model writes them).""" + return isinstance(v, list) and len(v) == n and all( + isinstance(x, (int, deckdata._Num)) and not isinstance(x, bool) for x in v) + + +def _check_seed_shapes(stats: dict, kinds: int, where: str) -> None: + """The array shapes and the checks of a chart's statistics: `weights[kind][position]` (a kind null only with + Gekisou off), `rangeWeights[kind][position][range]` (null, or a kind null), one Gekisou off seed, the seeds' + ranges one per range, and every check (and rank check) within its bound.""" + positions, n = stats["positions"], len(stats["ranges"]) + + def check(c, what): + if abs(c["exact"] - float(c["predicted"])) > float(c["bound"]): + raise MusicDataError(f"{where}: {what} scores {c['exact']}, predicted {c['predicted']} beyond the bound " + f"{c['bound']}") + + for seed in stats["seeds"]: + s = f"seed {seed['seed']}" + w = seed.get("weights") + if not (isinstance(w, list) and len(w) == kinds and all(_numbers(x, positions) for x in w)): + raise MusicDataError(f"{where}: {s}: weights are not [kind][position]") + if len(seed["ranges"]) != n: + raise MusicDataError(f"{where}: {s}: {len(seed['ranges'])} range results for {n} ranges") + rw = seed.get("rangeWeights") + if rw is not None and not (isinstance(rw, list) and len(rw) == kinds and all( + k is None or (isinstance(k, list) and len(k) == positions and all(_numbers(x, n) for x in k)) + for k in rw)): + raise MusicDataError(f"{where}: {s}: rangeWeights are not [kind][position][range]") + check(seed["check"], f"{s}: the check deck") + rc = seed.get("rankCheck") + if rc is not None: + if len(rc["ranks"]) != n or not all(1 <= r <= RANKS for r in rc["ranks"]): + raise MusicDataError(f"{where}: {s}: rank check ranks {rc['ranks']!r}") + check(rc, f"{s}: the check deck at ranks {rc['ranks']!r}") + off = stats.get("offSeeds") + if not isinstance(off, list) or len(off) != 1: + raise MusicDataError(f"{where}: the deck model gives no Gekisou off statistics (offSeeds)") + for seed in off: + w = seed.get("weights") + if not (isinstance(w, list) and len(w) == kinds and all(x is None or _numbers(x, positions) for x in w)): + raise MusicDataError(f"{where}: Gekisou off: weights are not [kind][position]") + check(seed["check"], "Gekisou off: the check deck") + + +def chart_deck(song: dict, chart: dict, stats: dict, kinds: int, percents: list[list]) -> dict: + """A chart's `deck` from its statistics, after checking them against the song, the chart facts and the song's + rank bonus percentages (`percents`: rank_bonus_percents), `kinds` the number of score-up kinds.""" where = f"chart {chart['scoreId']} ({song['id']} {chart['difficulty']})" checks = ( ("music id", stats["musicId"], song["id"]), @@ -276,6 +354,24 @@ def chart_deck(song: dict, chart: dict, stats: dict) -> dict: for name, deck, facts in checks: if deck != facts: raise MusicDataError(f"{where}: the deck model's {name} {deck!r} differs from the chart's {facts!r}") + ranges = stats["ranges"] + for i, r in enumerate(ranges): + want = percents[i] if i < len(percents) else [0] * RANKS + got = r.get("rankBonusPercents") + if got != want or r["rankBonusPercent"] != want[0]: + raise MusicDataError(f"{where}: range {i}: the deck model's rank bonus percentages {got!r} " + f"({r['rankBonusPercent']!r}) differ from MasterLiveGekisouRankingScoreBonus {want!r}") + _check_seed_shapes(stats, kinds, where) + for seed in stats["seeds"]: + for i, (r, info) in enumerate(zip(seed["ranges"], ranges)): + bonus = _trunc_percent(r["rangeScore"], info["rankBonusPercent"]) + if r["rankBonus"] != bonus: + raise MusicDataError(f"{where}: seed {seed['seed']} range {i}: rank bonus {r['rankBonus']} is not " + f"trunc({r['rangeScore']} * {info['rankBonusPercent']} / 100) = {bonus}") + if stats["justNotes"] == 0 and (seed.get("scorePerfect") != seed["score"] or any( + r.get("rangeScorePerfect") != r["rangeScore"] for r in seed["ranges"])): + raise MusicDataError(f"{where}: seed {seed['seed']}: a chart without Just notes scores otherwise on the " + f"Perfect play ({seed.get('scorePerfect')} for {seed['score']})") return {k: stats.get(k) for k in DECK_CHART_KEYS} @@ -387,9 +483,11 @@ def record(sid: int) -> dict: by_score = {c["scoreId"]: c for c in stats["charts"]} if sorted(by_score) != measured: raise MusicDataError("deck model: the charts measured differ from the songs' charts") + bonus_rows = tables.get("MasterLiveGekisouRankingScoreBonus", []) for s in songs: + percents = rank_bonus_percents(bonus_rows, s["gekisouMissions"]) for c in s["charts"]: - c["deck"] = chart_deck(s, c, by_score[c["scoreId"]]) + c["deck"] = chart_deck(s, c, by_score[c["scoreId"]], len(stats["kinds"]), percents) deck_doc = {"model": stats["model"], "kinds": stats["kinds"]} read = [t for t in tables_of(deck is not None, full) if t in tables] diff --git a/tests/test_musicdata.py b/tests/test_musicdata.py index b292ba6..4e9cd20 100644 --- a/tests/test_musicdata.py +++ b/tests/test_musicdata.py @@ -60,6 +60,11 @@ def text(tid, stem): {"_id": 3, "_group": 9, "_liveScoreRank": 7, "_requiredScore": 900, "_battleLiveRequiredScore": 1800}, {"_id": 1, "_group": 9, "_liveScoreRank": 2, "_requiredScore": 0, "_battleLiveRequiredScore": 0}, {"_id": 2, "_group": 8, "_liveScoreRank": 6, "_requiredScore": 5, "_battleLiveRequiredScore": 6}], + # the songs' missions [1, 3, 3] are pattern 3; pattern 2 rows are another song's + "MasterLiveGekisouRankingScoreBonus": [ + {"_id": 100 * p + 10 * c + k, "_missionPattern": p, "_count": c, "_rank": k, + "_scoreBonusPercent": (6 - k) * c + 10 * p} + for p in (2, 3) for c in (1, 2, 3) for k in (1, 2, 3, 4, 5)], } CHARTS = {"c/c_01": gzip.compress(json.dumps(CHART).encode("utf-8"), mtime=0), "c/c_02": json.dumps(SMALL).encode("utf-8"), @@ -96,6 +101,10 @@ def bgm(sheet, cue): WEIGHT = "0.30000000000000004" # a binary64 value: kept as the deck model writes it +def trunc_percent(score, percent): + return int(score * percent / 100) + + class FakeDeck: """The interface of nnnotes._deck: statistics made from the deck input as the model reports them.""" COMMIT = "7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6" @@ -118,6 +127,8 @@ def chart_stats(self, data, seeds, workers): def table(name): return [dict(zip(m[name]["columns"], r)) for r in m[name]["rows"]] levels = {r["_id"]: r["_musicScoreLevel"] for r in table("MasterLiveMusicScore")} + bonus = {(r["_missionPattern"], r["_count"], r["_rank"]): r["_scoreBonusPercent"] + for r in table("MasterLiveGekisouRankingScoreBonus")} songs = {} for r in table("MasterLiveMusic"): for d in musicdata.DIFFICULTIES: @@ -129,16 +140,28 @@ def table(name): music, d, missions = songs[c["scoreId"]] last = max(c["notes"]["timeMs"]) fevers = list(zip(c["fevers"]["startMs"], c["fevers"]["endMs"]))[:3] + positions = min(len(c["skillEvents"]["timeMs"]), 5) + pattern = 3 if missions == [1, 3, 3] else 0 + percents = [[bonus.get((pattern, i + 1, k), 0) for k in range(1, 6)] for i in range(len(fevers))] + check = {"deck": [[0, 5000]], "exact": 2000, "predicted": 2000.25, "bound": 7.0} s = {"scoreId": c["scoreId"], "musicId": music, "difficulty": d, "level": levels[c["scoreId"]], "judgedNotes": sum(score.is_judgement_note(op) for op in c["notes"]["op"]), "convertedNoteCount": len(c["notes"]["id"]), "lastNoteMs": last, "musicLengthMs": last + 1000, "skip": 1.5, "events": [[i % 5, t] for i, t in enumerate(c["skillEvents"]["timeMs"])], - "positions": min(len(c["skillEvents"]["timeMs"]), 5), "missions": missions, - "ranges": [{"index": i, "mission": missions[i], "startMs": a, "endMs": b, "rankBonusPercent": 10} + "positions": positions, "missions": missions, + "ranges": [{"index": i, "mission": missions[i], "startMs": a, "endMs": b, + "rankBonusPercent": percents[i][0], "rankBonusPercents": percents[i]} for i, (a, b) in enumerate(fevers)], "justNotes": 0, - "seeds": [{"seed": 0, "score": 1234, "ranges": [], "weights": [["W"]], - "check": {"deck": [[0, 5000]], "exact": 2000, "predicted": 2000.25, "bound": 7.0}}]} + "seeds": [{"seed": 0, "score": 1234, + "ranges": [{"rangeScore": 101 * (i + 1), "rankBonus": trunc_percent(101 * (i + 1), p[0]), + "maxCombo": 1, "justCount": 0, "lotResults": [0, 0, 0, 0], + "rangeScorePerfect": 101 * (i + 1)} for i, p in enumerate(percents)], + "weights": [["W"] * positions], "check": check, "scorePerfect": 1234, + "rangeWeights": [[["W"] * len(fevers)] * positions], + "rankCheck": {"ranks": [2] * len(fevers), "exact": 2001, "predicted": 2000.5, + "bound": 7.0} if fevers else None}], + "offSeeds": [{"seed": 0, "score": 1000, "weights": [["W"] * positions], "check": dict(check)}]} if self.change: self.change(s) charts.append(s) @@ -251,8 +274,14 @@ def test_deck(tmp_path): assert list(c["deck"]) == list(musicdata.DECK_CHART_KEYS) assert c["deck"]["events"] == [[0, 2500], [1, 500]] and c["deck"]["ranges"][1]["startMs"] == 2000 assert c["deck"]["unplayable"] is None and c["deck"]["skip"] == 1.5 + # the ranks: the song's mission pattern (3) rows of every range + assert [r["rankBonusPercents"] for r in c["deck"]["ranges"]] == [[35, 34, 33, 32, 31], [40, 38, 36, 34, 32]] + seed = c["deck"]["seeds"][0] + assert seed["rankCheck"]["ranks"] == [2, 2] and seed["scorePerfect"] == seed["score"] + assert [r["rankBonus"] for r in seed["ranges"]] == [35, 80] + assert c["deck"]["offSeeds"][0]["score"] == 1000 # the deck model's numbers as it writes them: a binary64 value is not narrowed to binary32 - assert b'"weights":[[' + WEIGHT.encode() + b']]' in raw and b'"predicted":2000.25' in raw + assert b'"weights":[[' + WEIGHT.encode() + b',' + WEIGHT.encode() + b']]' in raw and b'"predicted":2000.25' in raw assert "master" not in doc and "charts" not in doc schema_validator().validate(doc) (tmp_path / "m").rename(tmp_path / "m0") @@ -268,6 +297,21 @@ def test_deck(tmp_path): (lambda s: s["events"].pop(), "skill event times"), (lambda s: s["ranges"] and s["ranges"][0].update(endMs=1), "fevers"), (lambda s: s.update(difficulty="hard"), "difficulty 'hard' differs"), + (lambda s: s["ranges"] and s["ranges"][0]["rankBonusPercents"].__setitem__(2, 99), "range 0: .*percentages"), + (lambda s: s["ranges"] and s["ranges"][0].update(rankBonusPercent=1), "range 0: .*percentages"), + (lambda s: s["ranges"] and s["seeds"][0]["ranges"][0].update(rankBonus=-1), "rank bonus -1 is not trunc"), + (lambda s: s["seeds"][0]["ranges"].pop(), "range results for"), + (lambda s: s.pop("offSeeds"), "no Gekisou off statistics"), + (lambda s: s["offSeeds"].append(s["offSeeds"][0]), "no Gekisou off statistics"), + (lambda s: s["offSeeds"][0]["weights"].append(None), "Gekisou off: weights"), + (lambda s: s["offSeeds"][0]["check"].update(exact=0), "Gekisou off: the check deck scores 0"), + (lambda s: s["seeds"][0].update(scorePerfect=1), "otherwise on the Perfect play"), + (lambda s: s["seeds"][0]["weights"][0].pop(), "weights are not"), + (lambda s: s["seeds"][0]["weights"].append(None), "weights are not"), + (lambda s: s["seeds"][0].update(rangeWeights=[[]]), "rangeWeights are not"), + (lambda s: s["seeds"][0]["check"].update(bound=0.1), "the check deck scores 2000"), + (lambda s: s["ranges"] and s["seeds"][0]["rankCheck"].update(exact=0), "at ranks"), + (lambda s: s["ranges"] and s["seeds"][0]["rankCheck"].update(ranks=[6, 1]), "rank check ranks"), ]) def test_deck_checks(tmp_path, change, match): with pytest.raises(musicdata.MusicDataError, match=match): @@ -275,6 +319,33 @@ def test_deck_checks(tmp_path, change, match): assert not (tmp_path / "music.json").exists() +def test_deck_without_range_weights(tmp_path): + """A chart without range weights, a kind without them and a kind without Gekisou off weights are carried.""" + def change(s): + s["seeds"][0]["rangeWeights"] = None if s["scoreId"] == 10 else [None] + s["seeds"][0]["rankCheck"] = None + s["offSeeds"][0]["weights"] = [None] + export(tmp_path, deck=FakeDeck(change)) + doc = json.loads((tmp_path / "music.json").read_bytes()) + decks = {c["scoreId"]: c["deck"] for s in doc["songs"] for c in s["charts"]} + assert decks[10]["seeds"][0]["rangeWeights"] is None and decks[20]["seeds"][0]["rangeWeights"] == [None] + assert decks[30]["offSeeds"][0]["weights"] == [None] + schema_validator().validate(doc) + + +def test_rank_bonus_percents(): + assert [musicdata.mission_pattern(m) for m in ([1, 2, 0], [2, 2, 2], [1, 2, 3], [1, 1, 3], [1, 3, 3], + [3, 1, 3])] == [0, 1, 2, 3, 3, 3] + rows = [{"_missionPattern": 2, "_count": 1, "_rank": 1, "_scoreBonusPercent": 30}, + {"_missionPattern": 2, "_count": 3, "_rank": 5, "_scoreBonusPercent": 4}, + {"_missionPattern": 2, "_count": 3, "_rank": 5, "_scoreBonusPercent": 5}, # a later row wins + {"_missionPattern": 2, "_count": 4, "_rank": 1, "_scoreBonusPercent": 9}, # no fourth range + {"_missionPattern": 2, "_count": 1, "_rank": 6, "_scoreBonusPercent": 9}, # no sixth rank + {"_missionPattern": 1, "_count": 2, "_rank": 1, "_scoreBonusPercent": 9}] # another pattern + assert musicdata.rank_bonus_percents(rows, [1, 2, 3]) == [[30, 0, 0, 0, 0], [0] * 5, [0, 0, 0, 0, 5]] + assert musicdata.rank_bonus_percents(rows, [0, 0, 0]) == [[0] * 5] * 3 + + def test_deck_errors(tmp_path, monkeypatch): with pytest.raises(musicdata.MusicDataError, match="deck model: chart 30: the check deck scores"): export(tmp_path, deck=FakeDeck(fail="chart 30: the check deck scores 1, the chart statistics predict 2")) From 12df2a67b71ade5e41b51e6602d97d6b5733bd99 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 30 Sep 2026 01:12:37 +0800 Subject: [PATCH 11/11] feat(music-data): read decoded master data with its manifest (--decoded-master) `nnnotes music-data --decoded-master` reads master data decoded elsewhere, the directory the other commands read (`[paths] master`, `--master`, `[servers.] master`): one `
.json` per table and the `MasterManifest.json` of the files they were decoded from, as a master data snapshot published with its manifest carries it. No master key is needed, so a workflow can build the file from such a snapshot without the key. The master data version and each table's SHA-256 are the manifest's (the decoded tables cannot be checked against the files as served); `master.source` stays `api`, and the file is byte for byte the one `--master-files` writes from the same files. A missing manifest, an unlisted table, a table without a SHA-256 in the manifest, a missing or unreadable table stop the command as before. Co-Authored-By: Claude Opus 5.5 --- README.en.md | 2 +- README.md | 2 +- docs/commands.md | 21 +++++----- docs/configuration.md | 4 +- docs/music-data.md | 13 +++++-- docs/schema/music-data.schema.json | 4 +- src/nnnotes/cli.py | 11 ++++-- src/nnnotes/deckdata.py | 62 ++++++++++++++++++++---------- tests/test_deckdata.py | 47 +++++++++++++++++++++- tests/test_musicdata.py | 28 ++++++++++++++ 10 files changed, 153 insertions(+), 41 deletions(-) diff --git a/README.en.md b/README.en.md index 34ca1f0..6906a25 100644 --- a/README.en.md +++ b/README.en.md @@ -49,7 +49,7 @@ nnnotes config check # each setting's origin and whether it is valid | `player` | APK | render-related global settings (color space, quality levels, renderers) as JSON | | `live` | music ID + difficulty | a full chart directory: chart and runtime notes, 3D scene, note and effect assets, BGM and sounds, sound routing | | `web` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; the Live2D models of the stories are built first as models, listed in `models.json`, and the story manifests reference them; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`); compressible assets (JSON, shaders, moc3, ...) are stored gzip encoded by default (`--compress br` / `none`) | -| `music-data` | master data files (`--master-files` directory or `--apk-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (built into nnnotes) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) | +| `music-data` | master data files (`--master-files` directory or `--apk-master`), or decoded master data with its manifest (`--decoded-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (built into nnnotes) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) | Export conventions: diff --git a/README.md b/README.md index 4d9397b..96b94dc 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ nnnotes config check # 每项设置的来源和格式是否有效， | `player` | APK | 渲染相关的全局设置（色彩空间、画质等级、渲染器）JSON | | `live` | 曲目 ID + 难度 | 完整谱面目录：谱面与运行时音符、3D 场景、音符与特效资源、BGM 与音效、声音路由 | | `web` | `--pair 曲目:难度`（可重复）或 `--all`；`--live2d 模型`（可重复）或 `--all-live2d`；`--story 剧情 ID`（可重复）或 `--all-stories`；`--region 区服`（可重复）或 `--all-regions` | ournotes-player 静态站点：共享播放器 + 每谱 / 每模型 / 每集剧情清单 + 内容寻址资源；剧情用到的 Live2D 模型先按模型构建并列入 `models.json`，剧情清单引用它们；一个站点可服务多个区服，列表文本含五种语言；剧情的界面文字按语言分组，字形由开源字体生成 TextMeshPro 字体资源（`--fonts game` 时用游戏字体）；可压缩的资源（JSON、着色器、moc3 等）默认以 gzip 存储，`--compress br` / `none` 可改 | -| `music-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`） | 全部歌曲与谱面的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、评级线、BGM 时长；每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间；以及组卡模型 ournotes-deck（内置于 nnnotes）在整场模拟上实测的谱面统计（无技能得分、每种加分技能在每个演出位的权重）。`--full` 另附组卡模型的输入：每张谱面的运行时音符与卡牌、技能、加成、分数、活动相关的 masterdata 表（[格式](docs/music-data.md)） | +| `music-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`），或带清单的解码 masterdata（`--decoded-master`） | 全部歌曲与谱面的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、评级线、BGM 时长；每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间；以及组卡模型 ournotes-deck（内置于 nnnotes）在整场模拟上实测的谱面统计（无技能得分、每种加分技能在每个演出位的权重）。`--full` 另附组卡模型的输入：每张谱面的运行时音符与卡牌、技能、加成、分数、活动相关的 masterdata 表（[格式](docs/music-data.md)） | 导出约定： diff --git a/docs/commands.md b/docs/commands.md index 0af1540..bd03f0f 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -670,8 +670,8 @@ in the printed summary and in `SITE.failures.json`, models that fail in the summ ## music-data ``` -nnnotes music-data (--master-files DIR | --apk-master) [--full] [--no-deck] [--seeds N] [--workers N] - [--no-bgm] [--jackets DIR] -o FILE +nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] + [--workers N] [--no-bgm] [--jackets DIR] -o FILE ``` Writes one JSON file with every `MasterLiveMusic` song and its charts for one master data version: titles, readings @@ -689,10 +689,13 @@ master data tables about cards, skills, bonuses, scores and events. `--no-deck` luck range and its threads. The master data is decoded from the files as served: `--master-files DIR` reads `DIR/MasterManifest.json` and the `.bin` files it lists (`master download`; the file's region is `[catalog] region`), `--apk-master` the same files inside `[paths] apk` (region `embedded`); each file is checked against the manifest's -SHA-256. `--no-bgm` skips the cue sheets (every `bgm.length` is null). `--jackets DIR` also writes every song's -jacket as `DIR/.webp` (at most 320 px on the longer side). `FILE` ending in `.gz` is written -gzip-compressed; the file is canonical: the same inputs and nnnotes version give the same bytes. Prints `{out, -format, region, masterSource, masterVersion, songs, charts, deck, unplayable, full, bgm, jackets, bytes, fileBytes, -sha256}`. A missing or unreadable input (a master data file, a column, a text id, a chart asset, a cue sheet or cue, -a jacket), a chart the deck model cannot measure, or deck statistics that disagree with the chart facts stop the -command with exit status 1 before the file is written. The format is described in [music-data.md](music-data.md). +SHA-256. `--decoded-master` reads master data decoded elsewhere instead, without the master key: the `
.json` +files of the master data directory (`[paths] master`, `--master`) and the `MasterManifest.json` of the files they were +decoded from, whose version and SHA-256 the file records (region `[catalog] region`). `--no-bgm` skips the cue sheets +(every `bgm.length` is null). `--jackets DIR` also writes every song's jacket as `DIR/.webp` (at most 320 px +on the longer side). `FILE` ending in `.gz` is written gzip-compressed; the file is canonical: the same inputs and +nnnotes version give the same bytes. Prints `{out, format, region, masterSource, masterVersion, songs, charts, deck, +unplayable, full, bgm, jackets, bytes, fileBytes, sha256}`. A missing or unreadable input (a master data file, a +column, a text id, a chart asset, a cue sheet or cue, a jacket), a chart the deck model cannot measure, or deck +statistics that disagree with the chart facts stop the command with exit status 1 before the file is written. The +format is described in [music-data.md](music-data.md). diff --git a/docs/configuration.md b/docs/configuration.md index b8a65c4..c98fbd7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -155,7 +155,7 @@ story text); it is not a setting. | `web --pair` / `--all` | as `live`, plus `[paths] player`, `node`; with `--region` / `--all-regions` each region's `[servers.]` table (its `cdn` for what must be downloaded) and master data (`[servers.] master`; `[paths] master` for at most one region) | | `web --live2d` / `--all-live2d` | catalog (bundles from the CDN of the site's first region), `[paths] apk`, `[paths] player`; not `node`; master data only for the model names (optional: without it `models.json` has no names) | | `web --player-only` / `--reingest-json` | `[paths] player` | -| `music-data` | catalog, `[master] key` + `iv`; `--master-files` also `[catalog] region`; `--apk-master` also `[paths] apk`; `[paths] apk` (optional otherwise) for the client version | +| `music-data` | catalog, `[master] key` + `iv` (not with `--decoded-master`); `--master-files` also `[catalog] region`; `--apk-master` also `[paths] apk`; `--decoded-master` also `[catalog] region` and master data with its `MasterManifest.json`; `[paths] apk` (optional otherwise) for the client version | | `export`, `plan` | the store (`[paths] store` or `[paths] cache`); catalog (bundles are fetched into the cache); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`); master data for `--views`; with `--catalog-version` an imported catalog version instead of the current catalog | | `run-stage` | the store; `[paths] cache` for inputs located in the cache; `--fetch` also what fetching needs (region, `cdn`, bundle key, `[paths] apk`) | | `catalogs list` / `import` / `diff`, `store verify` | the store; `import` reads the APK's catalog from `[paths] apk` when it is set | @@ -216,6 +216,8 @@ nnnotes: game API call Version to [servers.tw] api failed: UNAVAILABLE (server u `nnnotes master download --latest` (or `--version `) or on the game client's own files. `[servers.] master` the same for one region: `nnnotes --region master download --latest -o `, then `nnnotes master decode -o ` (the regions serve different master data versions). + `music-data --decoded-master` also reads the `MasterManifest.json` of the decoded files there (copy it from the + download directory; a published master data snapshot may carry it). - **CRI HCA keycode**: not a setting. `audio`, `story`, `live` and `web` read it from the APK's boot data; `nnnotes crikey` shows whether one was found and can write it as a `.hcakey` file for vgmstream. - **Tools**: [vgmstream](https://vgmstream.org/) (`vgmstream-cli`), [FFmpeg](https://ffmpeg.org/) and, for diff --git a/docs/music-data.md b/docs/music-data.md index ddbb563..2294e41 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -9,14 +9,18 @@ its extension module `nnnotes._deck`. The format is `nnnotes.music-data/1`; its [schema/music-data.schema.json](schema/music-data.schema.json). ``` -nnnotes music-data (--master-files DIR | --apk-master) [--full] [--no-deck] [--seeds N] [--workers N] - [--no-bgm] [--jackets DIR] -o FILE +nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] + [--workers N] [--no-bgm] [--jackets DIR] -o FILE ``` - `--master-files DIR`: master data files as served, `DIR/MasterManifest.json` and the `.bin` files it lists (`nnnotes master download`). The file's `region` is `[catalog] region` (`--region`). - `--apk-master`: the master data files the APK ships (`assets/Master/` of `[paths] apk`, the same layout). The file's `region` is `embedded`. +- `--decoded-master`: master data decoded elsewhere, the directory the other commands read (`[paths] master`, + `--master`, `[servers.] master`): one `
.json` per table (`nnnotes master decode`) and the + `MasterManifest.json` of the files they were decoded from, as a master data snapshot published with its manifest + carries it. The file's `region` is `[catalog] region`; no master key is needed. - `--full`: also write the deck model's input, every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([the deck input](#the-deck-input---full)), for tools that run a deck model of their own. @@ -30,6 +34,9 @@ nnnotes music-data (--master-files DIR | --apk-master) [--full] [--no-deck] [--s page next to the file finds a song's jacket at `/.webp`. Each master data file is checked against the SHA-256 the manifest lists and decoded with `[master] key` and `iv`. +With `--decoded-master` the tables are read as decoded, and the master data version and each file's SHA-256 are the +manifest's: the decoded tables cannot be checked against the files as served, so the file records what the manifest +lists (the same values as `--master-files` on those files). Charts are read from the catalog of `[catalog] language` (bundles fetched into the cache as for every command); the BGM length from the cue sheet's ACB (its `CueTable` and `WaveformTable`, no audio is decoded). `FILE` ending in `.gz` is written gzip-compressed. The command prints a summary (`songs`, `charts`, `deck`: the deck model's commit, @@ -79,7 +86,7 @@ A **text** is an object with one string per language of `languages` (`{"ja": ... | `client.versionName`, `client.versionCode` | the APK's version name and code (null without `[paths] apk`) | | `catalog.resourceVersion` | the resource version recorded for the catalog in the catalog store (`nnnotes catalogs fetch` / `import`), null when none is recorded | | `catalog.sha256` | SHA-256 of the remote catalog file the charts were read with | -| `master.source` | `api` (`--master-files`) or `embedded` (`--apk-master`) | +| `master.source` | `api` (`--master-files`, `--decoded-master`: the region's files) or `embedded` (`--apk-master`) | | `master.version` | the `version` of the master data manifest | | `master.tables.
.sha256` | SHA-256 of each table's file as served, before decoding: the song tables (`MasterLiveMusic`, `MasterLiveMusicScore`, `MasterText`, `MasterBand`, `MasterCharacter`, `MasterTag`, `MasterLiveMusicCategory`, `MasterSound`, `MasterSoundCueSheet`, `MasterLiveScoreRank`) and, when the deck model runs or with `--full`, the tables of [the deck input](#the-deck-input---full) | | `exporter.name`, `exporter.version` | `nnnotes` and its version | diff --git a/docs/schema/music-data.schema.json b/docs/schema/music-data.schema.json index 42e8eda..dad6a10 100644 --- a/docs/schema/music-data.schema.json +++ b/docs/schema/music-data.schema.json @@ -90,7 +90,7 @@ "api", "embedded" ], - "description": "api: files as downloaded from the region; embedded: the files the APK ships" + "description": "api: the region's files (downloaded, or with --decoded-master decoded elsewhere with their manifest); embedded: the files the APK ships" }, "version": { "type": [ @@ -101,7 +101,7 @@ }, "tables": { "type": "object", - "description": "per table read, the SHA-256 of its file as served (before decoding)", + "description": "per table read, the SHA-256 of its file as served (before decoding; with --decoded-master as the manifest lists it)", "additionalProperties": { "type": "object", "required": [ diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index dfbf4af..b14ead1 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -36,8 +36,8 @@ [--story 10462 [--story ...] | --all-stories] [--story-languages en,ja] [--font en=] [--font emoji=] [--region [--region ...] | --all-regions] - nnnotes music-data --master-files | --apk-master [--full] [--no-deck] [--no-bgm] - [--jackets DIR] -o out/music-data.json[.gz] + nnnotes music-data --master-files | --apk-master | --decoded-master [--full] [--no-deck] + [--no-bgm] [--jackets DIR] -o out/music-data.json[.gz] nnnotes export -o out/assets [--select group: | key: | bundle: ...] [--layout original,cas] nnnotes plan [--select ...] [--json] [--check] [--emit-tasks ] nnnotes run-stage [...] @@ -646,9 +646,11 @@ def cmd_music_data(args, cfg): deck = None if args.no_deck else musicdata.Deck(seeds=args.seeds, workers=args.workers) if args.apk_master: src, region = deckdata.apk_master(apk), deckdata.EMBEDDED + elif args.decoded_master: # decoded elsewhere: no master key + src, region = deckdata.decoded_master(master_dir(cfg)), cfg.region() else: src, region = deckdata.master_files(Path(args.master_files)), cfg.region() - key = master_key(cfg) + key = None if src.decoded else master_key(cfg) cat = open_catalog(cfg) r = musicdata.export(Path(args.out), src, key, deckdata.catalog_fetch(cat), None if args.no_bgm else musicdata.catalog_bgm(cat), region=region, @@ -933,6 +935,9 @@ def target(m, what): help="master data files as served: MasterManifest.json and the .bin files it lists " "(`master download`)") g.add_argument("--apk-master", action="store_true", help="the master data files of base.apk ([paths] apk)") + g.add_argument("--decoded-master", action="store_true", + help="decoded master data ([paths] master or --master) with the MasterManifest.json of the files " + "it was decoded from; no master key") c.add_argument("--full", action="store_true", help="also write the deck model's input: every chart's runtime notes and the master data tables " "about cards, skills, bonuses, scores and events") diff --git a/src/nnnotes/deckdata.py b/src/nnnotes/deckdata.py index af7a1f0..4cd6905 100644 --- a/src/nnnotes/deckdata.py +++ b/src/nnnotes/deckdata.py @@ -4,9 +4,11 @@ Master data is decoded from the files as served: a directory with `MasterManifest.json` and the `.bin` files it lists (`nnnotes master download`), or the same layout inside the APK (`assets/Master/`). Each file is checked against -the manifest's SHA-256 and decoded with master.decode. Charts are the TextAssets `Live/MusicScore/` of -every MasterLiveMusicScore row, read from the catalog and converted by score.runtime_score; notes are listed in the -order the client enumerates them. +the manifest's SHA-256 and decoded with master.decode. Master data decoded elsewhere is read as it is: a directory of +decoded tables (`
.json`) with the `MasterManifest.json` of the files they were decoded from, whose version and +SHA-256 are taken as the manifest lists them (decoded_master). Charts are the TextAssets +`Live/MusicScore/` of every MasterLiveMusicScore row, read from the catalog and converted by +score.runtime_score; notes are listed in the order the client enumerates them. The encoding is canonical (encode): minified UTF-8 with one trailing LF, keys in a fixed order, charts sorted by score id, numbers the master data writes with a fraction or exponent as the shortest decimal that reads back as the @@ -19,6 +21,7 @@ import gzip import hashlib import json +import re import zipfile from collections import Counter from dataclasses import dataclass @@ -130,12 +133,13 @@ class DeckDataError(ValueError): @dataclass(frozen=True) class MasterSource: """Master data files as served: `version` and the SHA-256 of each file name from the manifest, and a reader of - files by name.""" + files by name. `decoded`: the reader has the decoded tables instead (`
.json` for `
.bin`).""" source: str # API or EMBEDDED where: str # the directory or APK, for messages version: str | None hashes: dict[str, str] # file name -> sha256 (lowercase hex; "" when not listed) read: Callable[[list[str]], dict[str, bytes]] # file names -> {name: bytes}; a missing file raises KeyError + decoded: bool = False def _manifest(raw: bytes, where: str) -> tuple[str | None, dict[str, str]]: @@ -149,12 +153,10 @@ def _manifest(raw: bytes, where: str) -> tuple[str | None, dict[str, str]]: return (str(version) if version is not None else None), hashes -def master_files(directory) -> MasterSource: - """The master data files of a directory written by `nnnotes master download` (MasterManifest.json + .bin).""" - d = Path(directory) +def _directory(d: Path, what: str, decoded: bool = False) -> MasterSource: m = d / MANIFEST if not m.is_file(): - raise DeckDataError(f"{d}: no {MANIFEST} (a directory written by `nnnotes master download`)") + raise DeckDataError(f"{d}: no {MANIFEST} ({what})") version, hashes = _manifest(m.read_bytes(), str(d)) def read(names): @@ -165,7 +167,21 @@ def read(names): except FileNotFoundError: raise KeyError(n) from None return out - return MasterSource(API, str(d), version, hashes, read) + return MasterSource(API, str(d), version, hashes, read, decoded) + + +def master_files(directory) -> MasterSource: + """The master data files of a directory written by `nnnotes master download` (MasterManifest.json + .bin).""" + return _directory(Path(directory), "a directory written by `nnnotes master download`") + + +def decoded_master(directory) -> MasterSource: + """Decoded master data: a directory of decoded tables, `
.json` with the `_allData` rows (as `nnnotes master + decode` writes them), and the `MasterManifest.json` of the files they were decoded from (a master data snapshot + published with its manifest). The version and each file's SHA-256 are the manifest's: the decoded tables cannot + be checked against the files as served.""" + return _directory(Path(directory), "decoded master data needs the manifest of the files it was decoded from", + decoded=True) def apk_master(apk) -> MasterSource: @@ -192,31 +208,37 @@ def read(names): def read_master(src: MasterSource, key, tables=None) -> tuple[dict[str, list[dict]], dict[str, str]]: """The rows (`_allData`) of every table of `tables` (default: TABLES) and the SHA-256 of each file as served. - `key`: a master.MasterKey.""" + `key`: a master.MasterKey (unused for decoded master data: None).""" from . import master names = {t: f"{t}.bin" for t in (tables if tables is not None else (t for t, _ in TABLES))} unlisted = [t for t, n in names.items() if n not in src.hashes] if unlisted: raise DeckDataError(f"master data {src.where}: {MANIFEST} lists no {', '.join(unlisted)}") + files = {t: f"{t}.json" if src.decoded else n for t, n in names.items()} try: - data = src.read(list(names.values())) + data = src.read(list(files.values())) except KeyError as e: raise DeckDataError(f"master data {src.where}: no file {e.args[0]}") from None - rk = master.round_keys(key.key) + rk = None if src.decoded else master.round_keys(key.key) tables, shas = {}, {} for t, n in names.items(): - raw = data[n] - sha = hashlib.sha256(raw).hexdigest() - if src.hashes[n] and sha != src.hashes[n]: - raise DeckDataError(f"master data {src.where}: {n}: sha256 differs from the manifest") + raw, f = data[files[t]], files[t] + if src.decoded: # the manifest's SHA-256 of the file as served + sha = src.hashes[n] + if not re.fullmatch(r"[0-9a-f]{64}", sha): + raise DeckDataError(f"master data {src.where}: {MANIFEST} lists no SHA-256 for {n}") + else: + sha = hashlib.sha256(raw).hexdigest() + if src.hashes[n] and sha != src.hashes[n]: + raise DeckDataError(f"master data {src.where}: {n}: sha256 differs from the manifest") try: - doc = json.loads(master.decode(raw, key, rk).decode("utf-8")) + doc = json.loads((raw if src.decoded else master.decode(raw, key, rk)).decode("utf-8")) except Exception as e: # padding, gzip, UTF-8 or JSON: the file cannot be read - raise DeckDataError(f"master data {src.where}: {n} cannot be decoded ({type(e).__name__}: " - f"{str(e)[:120]})") from None + raise DeckDataError(f"master data {src.where}: {f} cannot be {'read' if src.decoded else 'decoded'} " + f"({type(e).__name__}: {str(e)[:120]})") from None rows = doc.get("_allData") if isinstance(doc, dict) else None if not isinstance(rows, list) or not all(isinstance(r, dict) for r in rows): - raise DeckDataError(f"master data {src.where}: {n} has no `_allData` rows") + raise DeckDataError(f"master data {src.where}: {f} has no `_allData` rows") tables[t], shas[t] = rows, sha return tables, shas diff --git a/tests/test_deckdata.py b/tests/test_deckdata.py index 7bce071..8d44d9b 100644 --- a/tests/test_deckdata.py +++ b/tests/test_deckdata.py @@ -4,6 +4,7 @@ import gzip import hashlib import json +import shutil import struct import zipfile @@ -11,7 +12,7 @@ import pytest import synth -from nnnotes import cli, deckdata, jsonio, player, score +from nnnotes import cli, deckdata, jsonio, master, player, score from nnnotes.catalogdb import CatalogDB from nnnotes.master import MasterKey @@ -372,6 +373,50 @@ def test_apk_master_and_client(tmp_path): assert deckdata.apk_client(tmp_path / "empty.apk") == {"versionName": None, "versionCode": None} +# ---------------------------------------------------------------- decoded master data +def decoded_dir(tmp_path, d, name="dec"): + """The decoded tables of a master_dir (`master decode`) with its manifest, as a published snapshot carries them.""" + out = tmp_path / name + assert not master.decode_files(sorted(d.glob("*.bin")), out, KEY)["failed"] + shutil.copy(d / "MasterManifest.json", out / "MasterManifest.json") + return out + + +def test_decoded_master(tmp_path): + d = master_dir(tmp_path) + src = deckdata.decoded_master(decoded_dir(tmp_path, d)) + assert (src.source, src.version, src.decoded) == ("api", "v-test", True) + # the same rows and the manifest's SHA-256 of the files as served, without the key + assert deckdata.read_master(src, None) == deckdata.read_master(deckdata.master_files(d), KEY) + export(tmp_path, "e.json", src=src) + export(tmp_path, "f.json") + assert (tmp_path / "e.json").read_bytes() == (tmp_path / "f.json").read_bytes() + + +def test_decoded_master_checks(tmp_path): + d = master_dir(tmp_path) + with pytest.raises(deckdata.DeckDataError, match="no MasterManifest.json .decoded master data needs the manifest"): + deckdata.decoded_master(tmp_path) + dec = decoded_dir(tmp_path, d, "a") + (dec / "MasterBand.json").unlink() + failing(tmp_path, "no file MasterBand.json", src=deckdata.decoded_master(dec), key=None) + dec = decoded_dir(tmp_path, d, "b") + (dec / "MasterBand.json").write_text("{", encoding="utf-8") + failing(tmp_path, r"MasterBand.json cannot be read \(JSONDecodeError", src=deckdata.decoded_master(dec), key=None) + (dec / "MasterBand.json").write_text('{"x": 1}', encoding="utf-8") + failing(tmp_path, "MasterBand.json has no `_allData` rows", src=deckdata.decoded_master(dec), key=None) + dec = decoded_dir(tmp_path, d, "c") + m = json.loads((dec / "MasterManifest.json").read_text(encoding="utf-8")) + for f in m["files"]: + if f["name"] == "MasterBand.bin": + f["hash"] = "" + (dec / "MasterManifest.json").write_text(json.dumps(m), encoding="utf-8") + failing(tmp_path, "MasterManifest.json lists no SHA-256 for MasterBand.bin", src=deckdata.decoded_master(dec), + key=None) + dec = decoded_dir(tmp_path, master_dir(tmp_path, skip={"MasterEvent"}, name="m2"), "e") + failing(tmp_path, "lists no MasterEvent", src=deckdata.decoded_master(dec), key=None) + + def test_resource_version_from_the_catalog_store(tmp_path): remote = synth.CatalogWriter().build([("a_01.bundle", synth.remote("a_01.bundle"), [])]) sha = hashlib.sha256(remote).hexdigest() diff --git a/tests/test_musicdata.py b/tests/test_musicdata.py index 4e9cd20..8735be7 100644 --- a/tests/test_musicdata.py +++ b/tests/test_musicdata.py @@ -502,3 +502,31 @@ def test_command(tmp_path, capsys, monkeypatch): monkeypatch.setattr(musicdata, "Deck", lambda **kw: real(module=FakeDeck(fail="boom"), **kw)) code, _, err = run(base + ["-o", str(tmp_path / "x.json")], capsys) assert code == 1 and "deck model: boom" in err and not (tmp_path / "x.json").exists() + + +def test_command_decoded_master(tmp_path, capsys, monkeypatch): + from test_deckdata import decoded_dir + d = master_dir(tmp_path) + dec = decoded_dir(tmp_path, d) + out = tmp_path / "o" / "music.json" + code, _, err = run(["--region", "xx", "music-data", "--decoded-master", "--no-deck", "-o", str(out)], capsys) + assert code == 2 and "paths.master" in err + monkeypatch.setattr(cli, "open_catalog", lambda cfg, **kw: FakeCatalog(CHARTS)) + monkeypatch.setattr(score, "fetch_chart", lambda cat, name: cat.charts[name]) + fake = FakeDeck() + real = musicdata.Deck + monkeypatch.setattr(musicdata, "Deck", lambda **kw: real(module=fake, **kw)) + common = ["--region", "xx", "--cache", str(tmp_path / "cache")] + options = ["--no-bgm", "--full", "-o"] + code, stdout, err = run(common + ["--master", str(dec), "music-data", "--decoded-master"] + options + [str(out)], + capsys) # no master key is set + assert code == 0, err + r = json.loads(stdout) + assert (r["masterSource"], r["masterVersion"], r["region"], r["songs"]) == ("api", "v-test", "xx", 2) + # the file of the master data files as served, byte for byte + monkeypatch.setenv("NNNOTES_MASTER_KEY", synth.MASTER_KEY.hex()) + monkeypatch.setenv("NNNOTES_MASTER_IV", synth.MASTER_IV.hex()) + code, _, err = run(common + ["music-data", "--master-files", str(d)] + options + [str(tmp_path / "f.json")], + capsys) + assert code == 0, err + assert out.read_bytes() == (tmp_path / "f.json").read_bytes()