From cea696f5e47edffab0eb93f3a1d69891ab45002a Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 30 Sep 2026 01:12:37 +0800 Subject: [PATCH] feat(music-data): read decoded master data with its manifest (--decoded-master) `nnnotes music-data --decoded-master` reads master data decoded elsewhere, the directory the other commands read (`[paths] master`, `--master`, `[servers.] master`): one `.json` per table and the `MasterManifest.json` of the files they were decoded from, as a master data snapshot published with its manifest carries it. No master key is needed, so a workflow can build the file from such a snapshot without the key. The master data version and each table's SHA-256 are the manifest's (the decoded tables cannot be checked against the files as served); `master.source` stays `api`, and the file is byte for byte the one `--master-files` writes from the same files. A missing manifest, an unlisted table, a table without a SHA-256 in the manifest, a missing or unreadable table stop the command as before. Co-Authored-By: Claude Opus 5.5 --- README.en.md | 2 +- README.md | 2 +- docs/commands.md | 21 +++++----- docs/configuration.md | 4 +- docs/music-data.md | 13 +++++-- docs/schema/music-data.schema.json | 4 +- src/nnnotes/cli.py | 11 ++++-- src/nnnotes/deckdata.py | 62 ++++++++++++++++++++---------- tests/test_deckdata.py | 47 +++++++++++++++++++++- tests/test_musicdata.py | 28 ++++++++++++++ 10 files changed, 153 insertions(+), 41 deletions(-) diff --git a/README.en.md b/README.en.md index 34ca1f0..6906a25 100644 --- a/README.en.md +++ b/README.en.md @@ -49,7 +49,7 @@ nnnotes config check # each setting's origin and whether it is valid | `player` | APK | render-related global settings (color space, quality levels, renderers) as JSON | | `live` | music ID + difficulty | a full chart directory: chart and runtime notes, 3D scene, note and effect assets, BGM and sounds, sound routing | | `web` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; the Live2D models of the stories are built first as models, listed in `models.json`, and the story manifests reference them; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`); compressible assets (JSON, shaders, moc3, ...) are stored gzip encoded by default (`--compress br` / `none`) | -| `music-data` | master data files (`--master-files` directory or `--apk-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (built into nnnotes) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) | +| `music-data` | master data files (`--master-files` directory or `--apk-master`), or decoded master data with its manifest (`--decoded-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (built into nnnotes) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) | Export conventions: diff --git a/README.md b/README.md index 4d9397b..96b94dc 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ nnnotes config check # 每项设置的来源和格式是否有效， | `player` | APK | 渲染相关的全局设置（色彩空间、画质等级、渲染器）JSON | | `live` | 曲目 ID + 难度 | 完整谱面目录：谱面与运行时音符、3D 场景、音符与特效资源、BGM 与音效、声音路由 | | `web` | `--pair 曲目:难度`（可重复）或 `--all`；`--live2d 模型`（可重复）或 `--all-live2d`；`--story 剧情 ID`（可重复）或 `--all-stories`；`--region 区服`（可重复）或 `--all-regions` | ournotes-player 静态站点：共享播放器 + 每谱 / 每模型 / 每集剧情清单 + 内容寻址资源；剧情用到的 Live2D 模型先按模型构建并列入 `models.json`，剧情清单引用它们；一个站点可服务多个区服，列表文本含五种语言；剧情的界面文字按语言分组，字形由开源字体生成 TextMeshPro 字体资源（`--fonts game` 时用游戏字体）；可压缩的资源（JSON、着色器、moc3 等）默认以 gzip 存储，`--compress br` / `none` 可改 | -| `music-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`） | 全部歌曲与谱面的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、评级线、BGM 时长；每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间；以及组卡模型 ournotes-deck（内置于 nnnotes）在整场模拟上实测的谱面统计（无技能得分、每种加分技能在每个演出位的权重）。`--full` 另附组卡模型的输入：每张谱面的运行时音符与卡牌、技能、加成、分数、活动相关的 masterdata 表（[格式](docs/music-data.md)） | +| `music-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`），或带清单的解码 masterdata（`--decoded-master`） | 全部歌曲与谱面的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、评级线、BGM 时长；每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间；以及组卡模型 ournotes-deck（内置于 nnnotes）在整场模拟上实测的谱面统计（无技能得分、每种加分技能在每个演出位的权重）。`--full` 另附组卡模型的输入：每张谱面的运行时音符与卡牌、技能、加成、分数、活动相关的 masterdata 表（[格式](docs/music-data.md)） | 导出约定： diff --git a/docs/commands.md b/docs/commands.md index 0af1540..bd03f0f 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -670,8 +670,8 @@ in the printed summary and in `SITE.failures.json`, models that fail in the summ ## music-data ``` -nnnotes music-data (--master-files DIR | --apk-master) [--full] [--no-deck] [--seeds N] [--workers N] - [--no-bgm] [--jackets DIR] -o FILE +nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] + [--workers N] [--no-bgm] [--jackets DIR] -o FILE ``` Writes one JSON file with every `MasterLiveMusic` song and its charts for one master data version: titles, readings @@ -689,10 +689,13 @@ master data tables about cards, skills, bonuses, scores and events. `--no-deck` luck range and its threads. The master data is decoded from the files as served: `--master-files DIR` reads `DIR/MasterManifest.json` and the `.bin` files it lists (`master download`; the file's region is `[catalog] region`), `--apk-master` the same files inside `[paths] apk` (region `embedded`); each file is checked against the manifest's -SHA-256. `--no-bgm` skips the cue sheets (every `bgm.length` is null). `--jackets DIR` also writes every song's -jacket as `DIR/.webp` (at most 320 px on the longer side). `FILE` ending in `.gz` is written -gzip-compressed; the file is canonical: the same inputs and nnnotes version give the same bytes. Prints `{out, -format, region, masterSource, masterVersion, songs, charts, deck, unplayable, full, bgm, jackets, bytes, fileBytes, -sha256}`. A missing or unreadable input (a master data file, a column, a text id, a chart asset, a cue sheet or cue, -a jacket), a chart the deck model cannot measure, or deck statistics that disagree with the chart facts stop the -command with exit status 1 before the file is written. The format is described in [music-data.md](music-data.md). +SHA-256. `--decoded-master` reads master data decoded elsewhere instead, without the master key: the `
.json` +files of the master data directory (`[paths] master`, `--master`) and the `MasterManifest.json` of the files they were +decoded from, whose version and SHA-256 the file records (region `[catalog] region`). `--no-bgm` skips the cue sheets +(every `bgm.length` is null). `--jackets DIR` also writes every song's jacket as `DIR/.webp` (at most 320 px +on the longer side). `FILE` ending in `.gz` is written gzip-compressed; the file is canonical: the same inputs and +nnnotes version give the same bytes. Prints `{out, format, region, masterSource, masterVersion, songs, charts, deck, +unplayable, full, bgm, jackets, bytes, fileBytes, sha256}`. A missing or unreadable input (a master data file, a +column, a text id, a chart asset, a cue sheet or cue, a jacket), a chart the deck model cannot measure, or deck +statistics that disagree with the chart facts stop the command with exit status 1 before the file is written. The +format is described in [music-data.md](music-data.md). diff --git a/docs/configuration.md b/docs/configuration.md index b8a65c4..c98fbd7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -155,7 +155,7 @@ story text); it is not a setting. | `web --pair` / `--all` | as `live`, plus `[paths] player`, `node`; with `--region` / `--all-regions` each region's `[servers.]` table (its `cdn` for what must be downloaded) and master data (`[servers.] master`; `[paths] master` for at most one region) | | `web --live2d` / `--all-live2d` | catalog (bundles from the CDN of the site's first region), `[paths] apk`, `[paths] player`; not `node`; master data only for the model names (optional: without it `models.json` has no names) | | `web --player-only` / `--reingest-json` | `[paths] player` | -| `music-data` | catalog, `[master] key` + `iv`; `--master-files` also `[catalog] region`; `--apk-master` also `[paths] apk`; `[paths] apk` (optional otherwise) for the client version | +| `music-data` | catalog, `[master] key` + `iv` (not with `--decoded-master`); `--master-files` also `[catalog] region`; `--apk-master` also `[paths] apk`; `--decoded-master` also `[catalog] region` and master data with its `MasterManifest.json`; `[paths] apk` (optional otherwise) for the client version | | `export`, `plan` | the store (`[paths] store` or `[paths] cache`); catalog (bundles are fetched into the cache); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`); master data for `--views`; with `--catalog-version` an imported catalog version instead of the current catalog | | `run-stage` | the store; `[paths] cache` for inputs located in the cache; `--fetch` also what fetching needs (region, `cdn`, bundle key, `[paths] apk`) | | `catalogs list` / `import` / `diff`, `store verify` | the store; `import` reads the APK's catalog from `[paths] apk` when it is set | @@ -216,6 +216,8 @@ nnnotes: game API call Version to [servers.tw] api failed: UNAVAILABLE (server u `nnnotes master download --latest` (or `--version `) or on the game client's own files. `[servers.] master` the same for one region: `nnnotes --region master download --latest -o `, then `nnnotes master decode -o ` (the regions serve different master data versions). + `music-data --decoded-master` also reads the `MasterManifest.json` of the decoded files there (copy it from the + download directory; a published master data snapshot may carry it). - **CRI HCA keycode**: not a setting. `audio`, `story`, `live` and `web` read it from the APK's boot data; `nnnotes crikey` shows whether one was found and can write it as a `.hcakey` file for vgmstream. - **Tools**: [vgmstream](https://vgmstream.org/) (`vgmstream-cli`), [FFmpeg](https://ffmpeg.org/) and, for diff --git a/docs/music-data.md b/docs/music-data.md index ddbb563..2294e41 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -9,14 +9,18 @@ its extension module `nnnotes._deck`. The format is `nnnotes.music-data/1`; its [schema/music-data.schema.json](schema/music-data.schema.json). ``` -nnnotes music-data (--master-files DIR | --apk-master) [--full] [--no-deck] [--seeds N] [--workers N] - [--no-bgm] [--jackets DIR] -o FILE +nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] + [--workers N] [--no-bgm] [--jackets DIR] -o FILE ``` - `--master-files DIR`: master data files as served, `DIR/MasterManifest.json` and the `.bin` files it lists (`nnnotes master download`). The file's `region` is `[catalog] region` (`--region`). - `--apk-master`: the master data files the APK ships (`assets/Master/` of `[paths] apk`, the same layout). The file's `region` is `embedded`. +- `--decoded-master`: master data decoded elsewhere, the directory the other commands read (`[paths] master`, + `--master`, `[servers.] master`): one `
.json` per table (`nnnotes master decode`) and the + `MasterManifest.json` of the files they were decoded from, as a master data snapshot published with its manifest + carries it. The file's `region` is `[catalog] region`; no master key is needed. - `--full`: also write the deck model's input, every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([the deck input](#the-deck-input---full)), for tools that run a deck model of their own. @@ -30,6 +34,9 @@ nnnotes music-data (--master-files DIR | --apk-master) [--full] [--no-deck] [--s page next to the file finds a song's jacket at `/.webp`. Each master data file is checked against the SHA-256 the manifest lists and decoded with `[master] key` and `iv`. +With `--decoded-master` the tables are read as decoded, and the master data version and each file's SHA-256 are the +manifest's: the decoded tables cannot be checked against the files as served, so the file records what the manifest +lists (the same values as `--master-files` on those files). Charts are read from the catalog of `[catalog] language` (bundles fetched into the cache as for every command); the BGM length from the cue sheet's ACB (its `CueTable` and `WaveformTable`, no audio is decoded). `FILE` ending in `.gz` is written gzip-compressed. The command prints a summary (`songs`, `charts`, `deck`: the deck model's commit, @@ -79,7 +86,7 @@ A **text** is an object with one string per language of `languages` (`{"ja": ... | `client.versionName`, `client.versionCode` | the APK's version name and code (null without `[paths] apk`) | | `catalog.resourceVersion` | the resource version recorded for the catalog in the catalog store (`nnnotes catalogs fetch` / `import`), null when none is recorded | | `catalog.sha256` | SHA-256 of the remote catalog file the charts were read with | -| `master.source` | `api` (`--master-files`) or `embedded` (`--apk-master`) | +| `master.source` | `api` (`--master-files`, `--decoded-master`: the region's files) or `embedded` (`--apk-master`) | | `master.version` | the `version` of the master data manifest | | `master.tables.
.sha256` | SHA-256 of each table's file as served, before decoding: the song tables (`MasterLiveMusic`, `MasterLiveMusicScore`, `MasterText`, `MasterBand`, `MasterCharacter`, `MasterTag`, `MasterLiveMusicCategory`, `MasterSound`, `MasterSoundCueSheet`, `MasterLiveScoreRank`) and, when the deck model runs or with `--full`, the tables of [the deck input](#the-deck-input---full) | | `exporter.name`, `exporter.version` | `nnnotes` and its version | diff --git a/docs/schema/music-data.schema.json b/docs/schema/music-data.schema.json index 42e8eda..dad6a10 100644 --- a/docs/schema/music-data.schema.json +++ b/docs/schema/music-data.schema.json @@ -90,7 +90,7 @@ "api", "embedded" ], - "description": "api: files as downloaded from the region; embedded: the files the APK ships" + "description": "api: the region's files (downloaded, or with --decoded-master decoded elsewhere with their manifest); embedded: the files the APK ships" }, "version": { "type": [ @@ -101,7 +101,7 @@ }, "tables": { "type": "object", - "description": "per table read, the SHA-256 of its file as served (before decoding)", + "description": "per table read, the SHA-256 of its file as served (before decoding; with --decoded-master as the manifest lists it)", "additionalProperties": { "type": "object", "required": [ diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index dfbf4af..b14ead1 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -36,8 +36,8 @@ [--story 10462 [--story ...] | --all-stories] [--story-languages en,ja] [--font en=] [--font emoji=] [--region [--region ...] | --all-regions] - nnnotes music-data --master-files | --apk-master [--full] [--no-deck] [--no-bgm] - [--jackets DIR] -o out/music-data.json[.gz] + nnnotes music-data --master-files | --apk-master | --decoded-master [--full] [--no-deck] + [--no-bgm] [--jackets DIR] -o out/music-data.json[.gz] nnnotes export -o out/assets [--select group: | key: | bundle: ...] [--layout original,cas] nnnotes plan [--select ...] [--json] [--check] [--emit-tasks ] nnnotes run-stage [...] @@ -646,9 +646,11 @@ def cmd_music_data(args, cfg): deck = None if args.no_deck else musicdata.Deck(seeds=args.seeds, workers=args.workers) if args.apk_master: src, region = deckdata.apk_master(apk), deckdata.EMBEDDED + elif args.decoded_master: # decoded elsewhere: no master key + src, region = deckdata.decoded_master(master_dir(cfg)), cfg.region() else: src, region = deckdata.master_files(Path(args.master_files)), cfg.region() - key = master_key(cfg) + key = None if src.decoded else master_key(cfg) cat = open_catalog(cfg) r = musicdata.export(Path(args.out), src, key, deckdata.catalog_fetch(cat), None if args.no_bgm else musicdata.catalog_bgm(cat), region=region, @@ -933,6 +935,9 @@ def target(m, what): help="master data files as served: MasterManifest.json and the .bin files it lists " "(`master download`)") g.add_argument("--apk-master", action="store_true", help="the master data files of base.apk ([paths] apk)") + g.add_argument("--decoded-master", action="store_true", + help="decoded master data ([paths] master or --master) with the MasterManifest.json of the files " + "it was decoded from; no master key") c.add_argument("--full", action="store_true", help="also write the deck model's input: every chart's runtime notes and the master data tables " "about cards, skills, bonuses, scores and events") diff --git a/src/nnnotes/deckdata.py b/src/nnnotes/deckdata.py index af7a1f0..4cd6905 100644 --- a/src/nnnotes/deckdata.py +++ b/src/nnnotes/deckdata.py @@ -4,9 +4,11 @@ Master data is decoded from the files as served: a directory with `MasterManifest.json` and the `.bin` files it lists (`nnnotes master download`), or the same layout inside the APK (`assets/Master/`). Each file is checked against -the manifest's SHA-256 and decoded with master.decode. Charts are the TextAssets `Live/MusicScore/` of -every MasterLiveMusicScore row, read from the catalog and converted by score.runtime_score; notes are listed in the -order the client enumerates them. +the manifest's SHA-256 and decoded with master.decode. Master data decoded elsewhere is read as it is: a directory of +decoded tables (`
.json`) with the `MasterManifest.json` of the files they were decoded from, whose version and +SHA-256 are taken as the manifest lists them (decoded_master). Charts are the TextAssets +`Live/MusicScore/` of every MasterLiveMusicScore row, read from the catalog and converted by +score.runtime_score; notes are listed in the order the client enumerates them. The encoding is canonical (encode): minified UTF-8 with one trailing LF, keys in a fixed order, charts sorted by score id, numbers the master data writes with a fraction or exponent as the shortest decimal that reads back as the @@ -19,6 +21,7 @@ import gzip import hashlib import json +import re import zipfile from collections import Counter from dataclasses import dataclass @@ -130,12 +133,13 @@ class DeckDataError(ValueError): @dataclass(frozen=True) class MasterSource: """Master data files as served: `version` and the SHA-256 of each file name from the manifest, and a reader of - files by name.""" + files by name. `decoded`: the reader has the decoded tables instead (`
.json` for `
.bin`).""" source: str # API or EMBEDDED where: str # the directory or APK, for messages version: str | None hashes: dict[str, str] # file name -> sha256 (lowercase hex; "" when not listed) read: Callable[[list[str]], dict[str, bytes]] # file names -> {name: bytes}; a missing file raises KeyError + decoded: bool = False def _manifest(raw: bytes, where: str) -> tuple[str | None, dict[str, str]]: @@ -149,12 +153,10 @@ def _manifest(raw: bytes, where: str) -> tuple[str | None, dict[str, str]]: return (str(version) if version is not None else None), hashes -def master_files(directory) -> MasterSource: - """The master data files of a directory written by `nnnotes master download` (MasterManifest.json + .bin).""" - d = Path(directory) +def _directory(d: Path, what: str, decoded: bool = False) -> MasterSource: m = d / MANIFEST if not m.is_file(): - raise DeckDataError(f"{d}: no {MANIFEST} (a directory written by `nnnotes master download`)") + raise DeckDataError(f"{d}: no {MANIFEST} ({what})") version, hashes = _manifest(m.read_bytes(), str(d)) def read(names): @@ -165,7 +167,21 @@ def read(names): except FileNotFoundError: raise KeyError(n) from None return out - return MasterSource(API, str(d), version, hashes, read) + return MasterSource(API, str(d), version, hashes, read, decoded) + + +def master_files(directory) -> MasterSource: + """The master data files of a directory written by `nnnotes master download` (MasterManifest.json + .bin).""" + return _directory(Path(directory), "a directory written by `nnnotes master download`") + + +def decoded_master(directory) -> MasterSource: + """Decoded master data: a directory of decoded tables, `
.json` with the `_allData` rows (as `nnnotes master + decode` writes them), and the `MasterManifest.json` of the files they were decoded from (a master data snapshot + published with its manifest). The version and each file's SHA-256 are the manifest's: the decoded tables cannot + be checked against the files as served.""" + return _directory(Path(directory), "decoded master data needs the manifest of the files it was decoded from", + decoded=True) def apk_master(apk) -> MasterSource: @@ -192,31 +208,37 @@ def read(names): def read_master(src: MasterSource, key, tables=None) -> tuple[dict[str, list[dict]], dict[str, str]]: """The rows (`_allData`) of every table of `tables` (default: TABLES) and the SHA-256 of each file as served. - `key`: a master.MasterKey.""" + `key`: a master.MasterKey (unused for decoded master data: None).""" from . import master names = {t: f"{t}.bin" for t in (tables if tables is not None else (t for t, _ in TABLES))} unlisted = [t for t, n in names.items() if n not in src.hashes] if unlisted: raise DeckDataError(f"master data {src.where}: {MANIFEST} lists no {', '.join(unlisted)}") + files = {t: f"{t}.json" if src.decoded else n for t, n in names.items()} try: - data = src.read(list(names.values())) + data = src.read(list(files.values())) except KeyError as e: raise DeckDataError(f"master data {src.where}: no file {e.args[0]}") from None - rk = master.round_keys(key.key) + rk = None if src.decoded else master.round_keys(key.key) tables, shas = {}, {} for t, n in names.items(): - raw = data[n] - sha = hashlib.sha256(raw).hexdigest() - if src.hashes[n] and sha != src.hashes[n]: - raise DeckDataError(f"master data {src.where}: {n}: sha256 differs from the manifest") + raw, f = data[files[t]], files[t] + if src.decoded: # the manifest's SHA-256 of the file as served + sha = src.hashes[n] + if not re.fullmatch(r"[0-9a-f]{64}", sha): + raise DeckDataError(f"master data {src.where}: {MANIFEST} lists no SHA-256 for {n}") + else: + sha = hashlib.sha256(raw).hexdigest() + if src.hashes[n] and sha != src.hashes[n]: + raise DeckDataError(f"master data {src.where}: {n}: sha256 differs from the manifest") try: - doc = json.loads(master.decode(raw, key, rk).decode("utf-8")) + doc = json.loads((raw if src.decoded else master.decode(raw, key, rk)).decode("utf-8")) except Exception as e: # padding, gzip, UTF-8 or JSON: the file cannot be read - raise DeckDataError(f"master data {src.where}: {n} cannot be decoded ({type(e).__name__}: " - f"{str(e)[:120]})") from None + raise DeckDataError(f"master data {src.where}: {f} cannot be {'read' if src.decoded else 'decoded'} " + f"({type(e).__name__}: {str(e)[:120]})") from None rows = doc.get("_allData") if isinstance(doc, dict) else None if not isinstance(rows, list) or not all(isinstance(r, dict) for r in rows): - raise DeckDataError(f"master data {src.where}: {n} has no `_allData` rows") + raise DeckDataError(f"master data {src.where}: {f} has no `_allData` rows") tables[t], shas[t] = rows, sha return tables, shas diff --git a/tests/test_deckdata.py b/tests/test_deckdata.py index 7bce071..8d44d9b 100644 --- a/tests/test_deckdata.py +++ b/tests/test_deckdata.py @@ -4,6 +4,7 @@ import gzip import hashlib import json +import shutil import struct import zipfile @@ -11,7 +12,7 @@ import pytest import synth -from nnnotes import cli, deckdata, jsonio, player, score +from nnnotes import cli, deckdata, jsonio, master, player, score from nnnotes.catalogdb import CatalogDB from nnnotes.master import MasterKey @@ -372,6 +373,50 @@ def test_apk_master_and_client(tmp_path): assert deckdata.apk_client(tmp_path / "empty.apk") == {"versionName": None, "versionCode": None} +# ---------------------------------------------------------------- decoded master data +def decoded_dir(tmp_path, d, name="dec"): + """The decoded tables of a master_dir (`master decode`) with its manifest, as a published snapshot carries them.""" + out = tmp_path / name + assert not master.decode_files(sorted(d.glob("*.bin")), out, KEY)["failed"] + shutil.copy(d / "MasterManifest.json", out / "MasterManifest.json") + return out + + +def test_decoded_master(tmp_path): + d = master_dir(tmp_path) + src = deckdata.decoded_master(decoded_dir(tmp_path, d)) + assert (src.source, src.version, src.decoded) == ("api", "v-test", True) + # the same rows and the manifest's SHA-256 of the files as served, without the key + assert deckdata.read_master(src, None) == deckdata.read_master(deckdata.master_files(d), KEY) + export(tmp_path, "e.json", src=src) + export(tmp_path, "f.json") + assert (tmp_path / "e.json").read_bytes() == (tmp_path / "f.json").read_bytes() + + +def test_decoded_master_checks(tmp_path): + d = master_dir(tmp_path) + with pytest.raises(deckdata.DeckDataError, match="no MasterManifest.json .decoded master data needs the manifest"): + deckdata.decoded_master(tmp_path) + dec = decoded_dir(tmp_path, d, "a") + (dec / "MasterBand.json").unlink() + failing(tmp_path, "no file MasterBand.json", src=deckdata.decoded_master(dec), key=None) + dec = decoded_dir(tmp_path, d, "b") + (dec / "MasterBand.json").write_text("{", encoding="utf-8") + failing(tmp_path, r"MasterBand.json cannot be read \(JSONDecodeError", src=deckdata.decoded_master(dec), key=None) + (dec / "MasterBand.json").write_text('{"x": 1}', encoding="utf-8") + failing(tmp_path, "MasterBand.json has no `_allData` rows", src=deckdata.decoded_master(dec), key=None) + dec = decoded_dir(tmp_path, d, "c") + m = json.loads((dec / "MasterManifest.json").read_text(encoding="utf-8")) + for f in m["files"]: + if f["name"] == "MasterBand.bin": + f["hash"] = "" + (dec / "MasterManifest.json").write_text(json.dumps(m), encoding="utf-8") + failing(tmp_path, "MasterManifest.json lists no SHA-256 for MasterBand.bin", src=deckdata.decoded_master(dec), + key=None) + dec = decoded_dir(tmp_path, master_dir(tmp_path, skip={"MasterEvent"}, name="m2"), "e") + failing(tmp_path, "lists no MasterEvent", src=deckdata.decoded_master(dec), key=None) + + def test_resource_version_from_the_catalog_store(tmp_path): remote = synth.CatalogWriter().build([("a_01.bundle", synth.remote("a_01.bundle"), [])]) sha = hashlib.sha256(remote).hexdigest() diff --git a/tests/test_musicdata.py b/tests/test_musicdata.py index 4e9cd20..8735be7 100644 --- a/tests/test_musicdata.py +++ b/tests/test_musicdata.py @@ -502,3 +502,31 @@ def test_command(tmp_path, capsys, monkeypatch): monkeypatch.setattr(musicdata, "Deck", lambda **kw: real(module=FakeDeck(fail="boom"), **kw)) code, _, err = run(base + ["-o", str(tmp_path / "x.json")], capsys) assert code == 1 and "deck model: boom" in err and not (tmp_path / "x.json").exists() + + +def test_command_decoded_master(tmp_path, capsys, monkeypatch): + from test_deckdata import decoded_dir + d = master_dir(tmp_path) + dec = decoded_dir(tmp_path, d) + out = tmp_path / "o" / "music.json" + code, _, err = run(["--region", "xx", "music-data", "--decoded-master", "--no-deck", "-o", str(out)], capsys) + assert code == 2 and "paths.master" in err + monkeypatch.setattr(cli, "open_catalog", lambda cfg, **kw: FakeCatalog(CHARTS)) + monkeypatch.setattr(score, "fetch_chart", lambda cat, name: cat.charts[name]) + fake = FakeDeck() + real = musicdata.Deck + monkeypatch.setattr(musicdata, "Deck", lambda **kw: real(module=fake, **kw)) + common = ["--region", "xx", "--cache", str(tmp_path / "cache")] + options = ["--no-bgm", "--full", "-o"] + code, stdout, err = run(common + ["--master", str(dec), "music-data", "--decoded-master"] + options + [str(out)], + capsys) # no master key is set + assert code == 0, err + r = json.loads(stdout) + assert (r["masterSource"], r["masterVersion"], r["region"], r["songs"]) == ("api", "v-test", "xx", 2) + # the file of the master data files as served, byte for byte + monkeypatch.setenv("NNNOTES_MASTER_KEY", synth.MASTER_KEY.hex()) + monkeypatch.setenv("NNNOTES_MASTER_IV", synth.MASTER_IV.hex()) + code, _, err = run(common + ["music-data", "--master-files", str(d)] + options + [str(tmp_path / "f.json")], + capsys) + assert code == 0, err + assert out.read_bytes() == (tmp_path / "f.json").read_bytes()