diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89a9a6b..fc8d103 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,16 +20,19 @@ jobs: matrix: python-version: ["3.11", "3.12", "3.13"] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} cache: pip cache-dependency-path: pyproject.toml + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust - uses: astral-sh/setup-uv@v7 with: enable-cache: true - - name: Install + - name: Install (builds the extension module nnnotes._deck) run: uv pip install --system -e ".[test]" - name: Lint run: python -m pyflakes src tests @@ -43,14 +46,17 @@ jobs: # versions installed an output change without a stage or converter version bump fails instead of being skipped. runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.13" cache: pip cache-dependency-path: | pyproject.toml .github/goldens-constraints.txt + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust - uses: astral-sh/setup-uv@v7 with: enable-cache: true @@ -60,3 +66,17 @@ jobs: env: GOLDENS_STRICT: "1" run: python -m pytest tests/test_goldens.py tests/test_objexport.py + + rust: + # the extension module's own code (rust/); ournotes-deck is tested in its repository + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust + - name: Format and lints + working-directory: rust + run: | + cargo fmt --check + cargo clippy --locked --all-targets -- -D warnings diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index 1bec0c5..8bdfb96 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -13,12 +13,12 @@ jobs: commitlint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: - node-version: 20 + node-version: 24 - name: Install commitlint run: npm install --no-save --no-package-lock @commitlint/cli@19 @commitlint/config-conventional@19 - name: Pull request title diff --git a/.github/workflows/deck.yml b/.github/workflows/deck.yml new file mode 100644 index 0000000..688eabf --- /dev/null +++ b/.github/workflows/deck.yml @@ -0,0 +1,50 @@ +name: Deck model + +# nnnotes builds the deck model ournotes-deck at the commit rust/Cargo.toml pins, so that one nnnotes version always +# carries one deck model. When ournotes-deck's main moves past that commit, this opens a pull request that moves the +# pin; CI then builds and tests it like any change. +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + update: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Compare the pinned commit with ournotes-deck's main + id: deck + env: + GH_TOKEN: ${{ github.token }} + run: | + pinned=$(sed -nE 's/^ournotes-deck = .*rev = "([0-9a-f]{40})".*/\1/p' rust/Cargo.toml) + head=$(git ls-remote https://github.com/empty-sekai/ournotes-deck refs/heads/main | cut -f1) + # ahead: main contains the pinned commit and more; behind or diverged: leave the pin to a person + status=$(gh api "repos/empty-sekai/ournotes-deck/compare/$pinned...$head" --jq .status) + echo "pinned $pinned, main $head: $status" + echo "pinned=$pinned" >> "$GITHUB_OUTPUT" + echo "head=$head" >> "$GITHUB_OUTPUT" + echo "move=$([ "$status" = ahead ] && echo true || echo false)" >> "$GITHUB_OUTPUT" + - name: Move the pin + if: steps.deck.outputs.move == 'true' + working-directory: rust + run: | + sed -i -E 's/(rev = ")${{ steps.deck.outputs.pinned }}(")/\1${{ steps.deck.outputs.head }}\2/' Cargo.toml + cargo update -p ournotes-deck + - uses: peter-evans/create-pull-request@v7 + if: steps.deck.outputs.move == 'true' + with: + branch: deck/update + commit-message: "build(deck): ournotes-deck ${{ steps.deck.outputs.head }}" + title: "build(deck): ournotes-deck ${{ steps.deck.outputs.head }}" + body: | + Moves the deck model to ournotes-deck's main, + https://github.com/empty-sekai/ournotes-deck/compare/${{ steps.deck.outputs.pinned }}...${{ steps.deck.outputs.head }} + + Music data files made after this change name the new commit in `provenance.deck.commit`; their deck + statistics may differ. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3866318..9808b4d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,26 +8,80 @@ permissions: contents: read jobs: - build: + check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.13" - - name: Check the tag against the package version + - uses: actions/checkout@v7 + - name: Check the tag against the package and extension versions run: | - version=$(python -c 'import re, pathlib; print(re.search(r"__version__ = \"([^\"]+)\"", pathlib.Path("src/nnnotes/__init__.py").read_text()).group(1))') - if [ "v$version" != "$GITHUB_REF_NAME" ]; then - echo "tag $GITHUB_REF_NAME does not match the package version $version" >&2 + version=$(python3 -c 'import re, pathlib; print(re.search(r"__version__ = \"([^\"]+)\"", pathlib.Path("src/nnnotes/__init__.py").read_text()).group(1))') + cargo=$(python3 -c 'import re, pathlib; print(re.search(r"^version = \"([^\"]+)\"", pathlib.Path("rust/Cargo.toml").read_text(), re.M).group(1))') + if [ "v$version" != "$GITHUB_REF_NAME" ] || [ "$cargo" != "$version" ]; then + echo "tag $GITHUB_REF_NAME, package $version and rust/Cargo.toml $cargo differ" >&2 exit 1 fi - - name: Build sdist and wheel + + # one abi3 wheel (CPython 3.11 and later) per platform with the extension module nnnotes._deck, and the sdist + wheels: + needs: check + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + include: + - {os: ubuntu-latest, target: x86_64, manylinux: "2_28"} + - {os: ubuntu-24.04-arm, target: aarch64, manylinux: "2_28"} + - {os: windows-latest, target: x64} + - {os: macos-latest, target: aarch64} + - {os: macos-15-intel, target: x86_64} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux || 'auto' }} + args: --release --locked --out dist + - uses: actions/upload-artifact@v7 + with: + name: wheel-${{ matrix.os }}-${{ matrix.target }} + path: dist/ + if-no-files-found: error + + sdist: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + - uses: actions/upload-artifact@v7 + with: + name: sdist + path: dist/ + if-no-files-found: error + + build: + needs: [wheels, sdist] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v8 + with: + path: dist/ + merge-multiple: true + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - name: Check the distributions run: | - python -m pip install --upgrade pip build twine - python -m build + python -m pip install --upgrade pip twine python -m twine check --strict dist/* - - uses: actions/upload-artifact@v4 + ls dist + - uses: actions/upload-artifact@v7 with: name: dist path: dist/ @@ -42,7 +96,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: name: dist path: dist/ @@ -57,7 +111,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: Release notes @@ -68,12 +122,12 @@ jobs: env: OUTPUT: RELEASE_NOTES.md GITHUB_REPO: ${{ github.repository }} - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: name: dist path: dist/ - name: GitHub release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: body_path: RELEASE_NOTES.md files: | diff --git a/.github/workflows/story-site.yml b/.github/workflows/story-site.yml index 3c4e950..5a933cd 100644 --- a/.github/workflows/story-site.yml +++ b/.github/workflows/story-site.yml @@ -104,6 +104,9 @@ jobs: # The runner's preinstalled SDKs take most of its disk; a story build keeps its bundles and work files. run: sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /opt/hostedtoolcache/CodeQL && df -h "$GITHUB_WORKSPACE" + - uses: Swatinem/rust-cache@v2 # the install builds the extension module nnnotes._deck (rust/) + with: + workspaces: rust - uses: astral-sh/setup-uv@v7 with: enable-cache: true diff --git a/.gitignore b/.gitignore index a179109..0c92583 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ nnnotes.toml .env out/ cache/ +rust/target/ +src/nnnotes/_deck*.pyd +src/nnnotes/_deck*.so diff --git a/README.en.md b/README.en.md index 951b73b..6906a25 100644 --- a/README.en.md +++ b/README.en.md @@ -49,7 +49,7 @@ nnnotes config check # each setting's origin and whether it is valid | `player` | APK | render-related global settings (color space, quality levels, renderers) as JSON | | `live` | music ID + difficulty | a full chart directory: chart and runtime notes, 3D scene, note and effect assets, BGM and sounds, sound routing | | `web` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; the Live2D models of the stories are built first as models, listed in `models.json`, and the story manifests reference them; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`); compressible assets (JSON, shaders, moc3, ...) are stored gzip encoded by default (`--compress br` / `none`) | -| `deck-data` | master data files (`--master-files` directory or `--apk-master`) | one JSON file for deck-building tools: every chart's runtime notes, skill events and fever ranges, and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/deck-data.md)) | +| `music-data` | master data files (`--master-files` directory or `--apk-master`), or decoded master data with its manifest (`--decoded-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (built into nnnotes) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) | Export conventions: @@ -74,13 +74,13 @@ Based on all data of the Taiwan server, version 1.0.1 (zh-Hant): | catalog / bundle decryption / dependency closure | working | | master data decoding | working | | stories, `adv` | 946 / 946 episodes | -| stories, `story` (full directory) | 946 / 946 episodes have all their resources in the catalog and of supported kinds (the resource closure equals the game's own per-episode download list); 759 exported one by one, the other 187 (with frames, effects, post effects, stills and the like) not yet one by one | +| stories, `story` (full directory) | 946 / 946 episodes have all their resources in the catalog and of supported kinds (the resource closure equals the game's own per-episode download list); 946 / 946 exported and checked one by one: the files, models and audio the output refers to exist, every JSON parses and every path it refers to resolves; the 235 episodes with frames, effects, post effects, stills, chats, talk windows or videos are byte-identical across two exports except for the models directory path | | Live2D models | 239 / 239 (every model of the catalog; episodes use 185 of them) | | CRI audio | 681 / 681 cue sheets | | charts, `live` | 336 / 336 (music, difficulty) pairs | -| web site, `web` | 336 / 336 charts, 239 / 239 Live2D models, 946 / 946 story episodes (two full builds in English without audio are byte-identical and 946 / 946 pass the data validation; a 27-episode sample in five languages with AAC audio is likewise identical across two builds and valid) | -| spots, `spot` / `room` | one spot verified, the others not individually checked | -| other regions (en / kr) and languages | checked: the regions serve the same catalog for a language and the same bundles, and the keys are shared; the master tables the charts use are the same in the three regions, and the text tables have all five languages; the chart exports checked match the Taiwan server's. A full multi-region site build is not verified yet | +| web site, `web` | 336 / 336 charts, 239 / 239 Live2D models, 946 / 946 story episodes (two full builds in English without audio are byte-identical and 946 / 946 pass the data validation); two full builds of the three regions in five languages with AAC audio (`--all-regions`, with the live master data: 340 charts of 85 musics) are likewise byte-identical and pass the data validation in full | +| spots, `spot` / `room` | 39 / 39 spots, every reference resolves; `room` exports 20 / 20 room backgrounds of the catalog. Every glb passes the Khronos glTF-Validator with 0 errors | +| other regions (en / kr) and languages | checked: the regions serve the same catalog for a language and the same bundles, and the keys are shared; the master tables the charts use are the same in the three regions, and the text tables have all five languages; the chart exports checked match the Taiwan server's. A full multi-region site build is verified (the row above) | ## Requirements diff --git a/README.md b/README.md index 35beea1..96b94dc 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ nnnotes config check # 每项设置的来源和格式是否有效， | `player` | APK | 渲染相关的全局设置（色彩空间、画质等级、渲染器）JSON | | `live` | 曲目 ID + 难度 | 完整谱面目录：谱面与运行时音符、3D 场景、音符与特效资源、BGM 与音效、声音路由 | | `web` | `--pair 曲目:难度`（可重复）或 `--all`；`--live2d 模型`（可重复）或 `--all-live2d`；`--story 剧情 ID`（可重复）或 `--all-stories`；`--region 区服`（可重复）或 `--all-regions` | ournotes-player 静态站点：共享播放器 + 每谱 / 每模型 / 每集剧情清单 + 内容寻址资源；剧情用到的 Live2D 模型先按模型构建并列入 `models.json`，剧情清单引用它们；一个站点可服务多个区服，列表文本含五种语言；剧情的界面文字按语言分组，字形由开源字体生成 TextMeshPro 字体资源（`--fonts game` 时用游戏字体）；可压缩的资源（JSON、着色器、moc3 等）默认以 gzip 存储，`--compress br` / `none` 可改 | -| `deck-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`） | 供组卡工具读取的单个 JSON：每张谱面的运行时音符、技能事件与 fever 区间，以及卡牌、技能、加成、分数与活动相关的 masterdata 表（[格式](docs/deck-data.md)） | +| `music-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`），或带清单的解码 masterdata（`--decoded-master`） | 全部歌曲与谱面的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、评级线、BGM 时长；每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间；以及组卡模型 ournotes-deck（内置于 nnnotes）在整场模拟上实测的谱面统计（无技能得分、每种加分技能在每个演出位的权重）。`--full` 另附组卡模型的输入：每张谱面的运行时音符与卡牌、技能、加成、分数、活动相关的 masterdata 表（[格式](docs/music-data.md)） | 导出约定： @@ -59,13 +59,13 @@ nnnotes config check # 每项设置的来源和格式是否有效， | catalog / 资源包解密 / 依赖闭包 | 可用 | | masterdata 解码 | 可用 | | 剧情 `adv` | 946 / 946 集可导出 | -| 剧情 `story`（完整目录） | 946 / 946 集的资源都在 catalog 中且类型均受支持（资源闭包与游戏自带的每集下载清单一致）；已逐集导出验证 759 集，其余 187 集（用到 Frame / Effect / PostEffect / Still 等资源）未逐一验证 | +| 剧情 `story`（完整目录） | 946 / 946 集的资源都在 catalog 中且类型均受支持（资源闭包与游戏自带的每集下载清单一致）；946 / 946 集已逐集导出验证：输出中引用的文件、模型与音频均存在，JSON 均可解析，引用的路径均可解析；其中 235 集用到 Frame / Effect / PostEffect / Still / Chat / TalkWindow / 视频资源，两次导出除模型目录路径外逐字节一致 | | Live2D 模型 | 239 / 239 个可导出（catalog 中的全部模型，剧情用到其中 185 个） | | CRI 音频 | 681 / 681 个 cue sheet 可解码 | | 谱面 `live` | 336 / 336 个（曲目, 难度）组合可导出 | -| 网页站点 `web` | 336 / 336 张谱面，239 / 239 个 Live2D 模型，946 / 946 集剧情（英语、无音频的全量构建两次结果逐字节一致，946 / 946 集通过数据校验；五种语言含 AAC 音频的 27 集抽样同样一致，并通过数据校验） | -| 据点 `spot` / `room` | 已验证单个据点，其余未逐一验证 | -| 其他区服（en / kr）与其他语言 | 已核对：各区服同一语言的 catalog 相同、资源包相同，密钥通用；谱面用到的 masterdata 表在三个区服间一致，文本表五种语言齐全；抽查的谱面导出与台服一致。多区服整站构建未做全量验证 | +| 网页站点 `web` | 336 / 336 张谱面，239 / 239 个 Live2D 模型，946 / 946 集剧情（英语、无音频的全量构建两次结果逐字节一致，946 / 946 集通过数据校验）；三个区服、五种语言、含 AAC 音频的全量构建（`--all-regions`，用线上 masterdata：85 首曲目的 340 张谱面）两次结果同样逐字节一致，全部通过数据校验 | +| 据点 `spot` / `room` | 39 / 39 个据点可导出，引用均可解析；catalog 中 20 / 20 个房间背景可由 `room` 导出。生成的 glb 均通过 Khronos glTF-Validator，0 个错误 | +| 其他区服（en / kr）与其他语言 | 已核对：各区服同一语言的 catalog 相同、资源包相同，密钥通用；谱面用到的 masterdata 表在三个区服间一致，文本表五种语言齐全；抽查的谱面导出与台服一致。多区服整站已全量构建验证（见上一行） | ## 需要准备 diff --git a/docs/commands.md b/docs/commands.md index c489b58..bd03f0f 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -177,7 +177,9 @@ OUT/story.json index of the above (a file the episode does not need i story.json `models` maps the key of each model to its id (`` of `Character/Live2D///model/`) and `modelsDir` is the path from OUT to MODELS. A model directory that exists is used as it is, so the stories of one models directory export each model once; `--force` exports the episode's models again. The printed summary lists -the models exported (`modelsBuilt`) and those used as they were (`modelsSkipped`). +the models exported (`modelsBuilt`) and those used as they were (`modelsSkipped`). Several `story` processes may +share a models directory: a model appears in it in one rename, whole, and a model two processes export at once is +kept from the first to finish (the other's export, the same bytes, is dropped and counted in `modelsSkipped`). `--format` (default `flac`) is the audio format and `--flac-level` its FLAC compression level (as for `audio`); `--no-audio` leaves the cue sheets undecoded (no `audio/`, `audio` @@ -665,19 +667,35 @@ byte-identical outputs; the encoded assets also need the same zlib and brotli ve in the printed summary and in `SITE.failures.json`, models that fail in the summary and in `SITE.model-failures.json`, stories in `SITE.story-failures.json`; the exit status is then 1. -## deck-data - -``` -nnnotes deck-data (--master-files DIR | --apk-master) -o FILE -``` - -Writes one JSON file for deck-building tools, for one master data version: every `MasterLiveMusicScore` row's chart -as the client builds it at runtime (notes, skill events, fever ranges) and the master data tables about cards, -skills, bonuses, scores and events, with their provenance. The master data is decoded from the files as served: -`--master-files DIR` reads `DIR/MasterManifest.json` and the `.bin` files it lists (`master download`; the file's -region is `[catalog] region`), `--apk-master` the same files inside `[paths] apk` (region `embedded`); each file is -checked against the manifest's SHA-256. `FILE` ending in `.gz` is written gzip-compressed. The file is canonical: the -same inputs give the same bytes. Prints `{out, format, region, masterSource, masterVersion, tables, rows, charts, -notes, bytes, fileBytes, sha256}`. A missing or mismatching input (a master data file, a column, a chart asset) -stops the command with exit status 1 before the file is written. The format is described in -[deck-data.md](deck-data.md). +## music-data + +``` +nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] + [--workers N] [--no-bgm] [--jackets DIR] -o FILE +``` + +Writes one JSON file with every `MasterLiveMusic` song and its charts for one master data version: titles, readings +and credits in the five text languages, bands (and a song's own band name), vocal characters, category, tags, +release time, jacket, Gekisou missions, score ranks, the whole `MasterLiveMusic` row, the live BGM's cue and length +(read from the cue sheet's ACB, without decoding audio); per difficulty the chart facts (level and display level, +full combo count, note counts, BPM, note times, the live's music length, skill event times, fever ranges) and the +chart's deck statistics: the no-skill score and the weight of every score-up skill kind at every performance +position, with Gekisou on (a Gekisou live at rank 1, with what every other rank and the Perfect play need) and off (a +solo live), measured by the deck model ournotes-deck (built into nnnotes as `nnnotes._deck`) on its whole-live +simulation and checked against the chart facts and the master data. `--full` also writes the deck model's input: every +`MasterLiveMusicScore` row's chart as the client builds it at runtime (notes, skill events, fever ranges) and the +master data tables about cards, skills, bonuses, scores and events. `--no-deck` skips the deck model (every chart's +`deck` is null); `--seeds N` (default 8) and `--workers N` (default: every processor) set its seeds on charts with a +luck range and its threads. The master data is decoded from the files as served: `--master-files DIR` reads +`DIR/MasterManifest.json` and the `.bin` files it lists (`master download`; the file's region is `[catalog] region`), +`--apk-master` the same files inside `[paths] apk` (region `embedded`); each file is checked against the manifest's +SHA-256. `--decoded-master` reads master data decoded elsewhere instead, without the master key: the `.json` +files of the master data directory (`[paths] master`, `--master`) and the `MasterManifest.json` of the files they were +decoded from, whose version and SHA-256 the file records (region `[catalog] region`). `--no-bgm` skips the cue sheets +(every `bgm.length` is null). `--jackets DIR` also writes every song's jacket as `DIR/.webp` (at most 320 px +on the longer side). `FILE` ending in `.gz` is written gzip-compressed; the file is canonical: the same inputs and +nnnotes version give the same bytes. Prints `{out, format, region, masterSource, masterVersion, songs, charts, deck, +unplayable, full, bgm, jackets, bytes, fileBytes, sha256}`. A missing or unreadable input (a master data file, a +column, a text id, a chart asset, a cue sheet or cue, a jacket), a chart the deck model cannot measure, or deck +statistics that disagree with the chart facts stop the command with exit status 1 before the file is written. The +format is described in [music-data.md](music-data.md). diff --git a/docs/configuration.md b/docs/configuration.md index 4b21ee1..c98fbd7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -155,7 +155,7 @@ story text); it is not a setting. | `web --pair` / `--all` | as `live`, plus `[paths] player`, `node`; with `--region` / `--all-regions` each region's `[servers.]` table (its `cdn` for what must be downloaded) and master data (`[servers.] master`; `[paths] master` for at most one region) | | `web --live2d` / `--all-live2d` | catalog (bundles from the CDN of the site's first region), `[paths] apk`, `[paths] player`; not `node`; master data only for the model names (optional: without it `models.json` has no names) | | `web --player-only` / `--reingest-json` | `[paths] player` | -| `deck-data` | catalog, `[master] key` + `iv`; `--master-files` also `[catalog] region`; `--apk-master` also `[paths] apk`; `[paths] apk` (optional otherwise) for the client version | +| `music-data` | catalog, `[master] key` + `iv` (not with `--decoded-master`); `--master-files` also `[catalog] region`; `--apk-master` also `[paths] apk`; `--decoded-master` also `[catalog] region` and master data with its `MasterManifest.json`; `[paths] apk` (optional otherwise) for the client version | | `export`, `plan` | the store (`[paths] store` or `[paths] cache`); catalog (bundles are fetched into the cache); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`); master data for `--views`; with `--catalog-version` an imported catalog version instead of the current catalog | | `run-stage` | the store; `[paths] cache` for inputs located in the cache; `--fetch` also what fetching needs (region, `cdn`, bundle key, `[paths] apk`) | | `catalogs list` / `import` / `diff`, `store verify` | the store; `import` reads the APK's catalog from `[paths] apk` when it is set | @@ -216,6 +216,8 @@ nnnotes: game API call Version to [servers.tw] api failed: UNAVAILABLE (server u `nnnotes master download --latest` (or `--version `) or on the game client's own files. `[servers.] master` the same for one region: `nnnotes --region master download --latest -o `, then `nnnotes master decode -o ` (the regions serve different master data versions). + `music-data --decoded-master` also reads the `MasterManifest.json` of the decoded files there (copy it from the + download directory; a published master data snapshot may carry it). - **CRI HCA keycode**: not a setting. `audio`, `story`, `live` and `web` read it from the APK's boot data; `nnnotes crikey` shows whether one was found and can write it as a `.hcakey` file for vgmstream. - **Tools**: [vgmstream](https://vgmstream.org/) (`vgmstream-cli`), [FFmpeg](https://ffmpeg.org/) and, for diff --git a/docs/contracts.md b/docs/contracts.md index e3a3d48..2cf4e7f 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -226,9 +226,9 @@ The catalog versions of a store, `nnnotes.catalogs/1` ([schema/catalogs.schema.j ([schema/catalog-diff.schema.json](schema/catalog-diff.schema.json), `nnnotes catalogs diff --json`), are written by the commands. -The deck data file of `nnnotes deck-data`, `nnnotes.deck-data/1` -([schema/deck-data.schema.json](schema/deck-data.schema.json)), names its format in a `format` field; it is -described in [deck-data.md](deck-data.md). +The music data file of `nnnotes music-data`, `nnnotes.music-data/1` +([schema/music-data.schema.json](schema/music-data.schema.json)), names its format in a `format` field; it is +described in [music-data.md](music-data.md). ## Store diff --git a/docs/deck-data.md b/docs/deck-data.md deleted file mode 100644 index 62c9a40..0000000 --- a/docs/deck-data.md +++ /dev/null @@ -1,160 +0,0 @@ -# Deck data - -`nnnotes deck-data` writes one JSON file with the game data that deck-building tools read: every live chart as the -client builds it at runtime, and the master data tables about cards, skills, bonuses, scores and events. One file -describes one master data version. The file holds facts as the game has them (master data values as served, notes -as the client's chart converter creates them), not formulas or values derived from them. The format is -`nnnotes.deck-data/1`; its JSON Schema is [schema/deck-data.schema.json](schema/deck-data.schema.json). - -``` -nnnotes deck-data (--master-files DIR | --apk-master) -o FILE -``` - -- `--master-files DIR`: master data files as served, `DIR/MasterManifest.json` and the `.bin` files it lists - (`nnnotes master download`). The file's `region` is `[catalog] region` (`--region`). -- `--apk-master`: the master data files the APK ships (`assets/Master/` of `[paths] apk`, the same layout). The - file's `region` is `embedded`. - -Each master data file is checked against the SHA-256 the manifest lists and decoded with `[master] key` and `iv`. -Charts are read from the catalog of `[catalog] language` (bundles fetched into the cache as for every command). -`FILE` ending in `.gz` is written gzip-compressed. The command prints a summary (`charts`, `notes`, `tables`, -`rows`, `bytes`, `sha256` of the file, ...). - -The command writes the file only when every table and every chart was read: a missing or mismatching master data -file, a table without a column the file exports, a `MasterLiveMusicScore` row whose chart asset is missing or cannot -be converted, a note id that occurs twice in a chart, or a note whose operate type has no judgement type stops it with -exit status 1 and a line naming the input. The file is written through a temporary file and a rename. - -## Layout - -```json -{ - "format": "nnnotes.deck-data/1", - "provenance": {"region": ..., "client": {...}, "catalog": {...}, "master": {...}, "exporter": {...}}, - "master": {"
": {"columns": ["_id", ...], "rows": [[...], ...]}, ...}, - "charts": [{"scoreId": ..., "asset": {...}, "notes": {...}, "skillEvents": {...}, "fevers": {...}}, ...] -} -``` - -The file is minified UTF-8 with one line feed at the end. Keys are always in the order shown here and in the tables -below, so the same inputs with the same nnnotes version give the same bytes. A gzip file has no file name and a zero -modification time in its header. - -### provenance - -| Field | Content | -|---|---| -| `region` | the region whose master data this is (a configured region name), or `embedded` for the APK's master data | -| `client.versionName`, `client.versionCode` | the APK's version name and code (null without `[paths] apk`) | -| `catalog.resourceVersion` | the resource version recorded for the catalog in the catalog store (`nnnotes catalogs fetch` / `import`), null when none is recorded | -| `catalog.sha256` | SHA-256 of the remote catalog file the charts were read with | -| `master.source` | `api` (`--master-files`) or `embedded` (`--apk-master`) | -| `master.version` | the `version` of the master data manifest | -| `master.tables.
.sha256` | SHA-256 of the table's file as served, before decoding | -| `exporter.name`, `exporter.version` | `nnnotes` and its version | -| `exporter.chartFormat` | the format of the chart converter the notes come from, `nnnotes.live-score/1` | - -### master - -Each table is `{"columns": [...], "rows": [[...], ...]}`: one array per row, its values in the order of `columns`, -rows in the order the master data lists them. Values are as decoded: integers, strings, booleans, arrays. A number -the master data writes with a fraction or an exponent is a binary32 (single precision) value in the game; it is -written as the shortest decimal that reads back as the same binary32 value, so a reader should parse such columns -as 32-bit floats. Infinity is written `1e999` / `-1e999`. - -Tables with a column list export those columns, and every row must have them. Tables marked "all" export every -column their rows have, in the order the rows first have them; when such a table has no rows, `columns` is empty -(the master data carries no field list for an empty table). A reader treats a table missing from `master` as empty. - -| Table | Columns | -|---|---| -| `MasterMemberCard` | `_id` `_characterID` `_rarity` `_cardType` `_bestMusicTagIDs` `_performancePowerMax` `_technicPowerMax` `_visualPowerMax` `_memberCardLevelGroup` `_memberCardAwakeGroup` `_memberCardRankGroup` `_leaderSkillID` `_liveSkillID` `_gekisouSkillID` | -| `MasterMemberCardLevel` | `_id` `_group` `_level` `_exp` `_performanceRate` `_technicRate` `_visualRate` | -| `MasterMemberCardLevelLimit` | `_id` `_rarity` `_awakeCount` `_limitLevel` | -| `MasterMemberCardAwake` | `_id` `_group` `_awakeCount` `_performanceRate` `_technicRate` `_visualRate` | -| `MasterMemberCardRank` | `_id` `_group` `_rank` `_performanceRate` `_technicRate` `_visualRate` `_leaderSkillLevel` `_musicTypeBonusRate` `_musicTagBonusRate` | -| `MasterSupportCard` | `_id` `_characterIDs` `_rarity` `_cardType` `_performancePowerMax` `_technicPowerMax` `_visualPowerMax` `_supportCardLevelGroup` `_supportCardRankGroup` `_supportSkillId01` `_supportSkillId02` `_gekisouSupportSkillId01` `_gekisouSupportSkillId02` | -| `MasterSupportCardLevel` | `_id` `_group` `_level` `_exp` `_performanceRate` `_technicRate` `_visualRate` | -| `MasterSupportCardRank` | `_id` `_group` `_rank` `_limitLevel` `_cardTypeLinkBonusRate` `_supportSkill01Level` `_supportSkill02Level` `_gekisouSupportSkill01Level` `_gekisouSupportSkill02Level` | -| `MasterCharacter` | `_id` `_bandID` | -| `MasterBand` | `_id` | -| `MasterCharacterRank` | `_id` `_rank` `_bonus` | -| `MasterCharacterTotalRank` | `_id` `_totalRank` `_bonus` | -| `MasterBandItemSkillEffect` | `_id` `_bandItemId` `_level` `_skillTargetIDs` `_skillEffectType` `_effectValue` | -| `MasterBandItem` | `_id` `_bandId` | -| `MasterBandItemLevel` | `_id` `_bandItemId` `_level` `_playerRank` | -| `MasterVipRankBonus` | `_id` `_vipRank` `_vipBonusType` `_value` | -| `MasterMemoryMusicGroup` | `_id` `_skillTargetIds` | -| `MasterMemoryMusic` | `_id` `_groupId` | -| `MasterMemoryMusicBonus` | `_id` `_groupId` `_scoreRank` `_performance` `_technic` `_visual` | -| `MasterMemoryMemberLevel` | `_id` `_point` `_performance` `_technic` `_visual` | -| `MasterMemorySupportLevel` | `_id` `_point` `_performance` `_technic` `_visual` | -| `MasterSkillTarget` | `_id` `_skillTargetType` `_characterID` `_bandID` `_cardType` `_tagID` `_judgement` `_liveMusicType` `_gekisouMissionType` `_liveSkillCategories` `_gekisouSkillCategories` | -| `MasterSkillCondition` | `_id` `_conditionType` `_conditionValues` `_isPositive` `_conditionTargetIDs` | -| `MasterSkillConditionSet` | `_id` `_group` `_conditionIds` | -| `MasterSkillCumulativeCondition` | `_id` `_skillCumulativeConditionType` `_conditionValues` `_conditionTargetIDs` `_maxCumulativeCount` | -| `MasterSkillEffectSetting` | `_id` `_skillEffectType` `_phase` | -| `MasterLeaderSkillEffect` | `_id` `_leaderSkillID` `_level` `_skillConditionGroup` `_skillTargetIDs` `_skillEffectType` `_effectValue` `_skillCumulativeConditionID` | -| `MasterLiveSkill` | `_id` `_skillCategories` | -| `MasterLiveSkillEffect` | `_id` `_liveSkillID` `_level` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | -| `MasterSupportSkill` | `_id` | -| `MasterSupportSkillEffect` | `_id` `_supportSkillID` `_level` `_skillTriggerConditionGroup` `_skillTriggerType` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | -| `MasterGekisouSkill` | `_id` `_gekisouMissionType` `_skillCategories` | -| `MasterGekisouSkillEffect` | `_id` `_gekisouSkillID` `_level` `_skillTriggerConditionGroup` `_skillTriggerType` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | -| `MasterGekisouSupportSkill` | `_id` `_gekisouSupportSkillExecTiming` `_gekisouMissionType` | -| `MasterGekisouSupportSkillEffect` | `_id` `_gekisouSupportSkillID` `_level` `_skillTriggerConditionGroup` `_skillTriggerType` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | -| `MasterLiveNoteParameter` | `_id` `_noteOperateType` `_scorePercent` | -| `MasterLiveJudgementParameter` | `_id` `_noteSimulateJudgement` `_scorePercent` `_damage` | -| `MasterLiveJudgementTiming` | `_id` `_assistLevel` `_judgementPriority` `_noteJudgementType` `_noteSimulateJudgement` `_beforeMs` `_afterMs` | -| `MasterLiveComboScoreBonus` | `_id` `_comboBonusType` `_requiredComboCount` `_bonusFactor` | -| `MasterLiveSettings` | all | -| `MasterParameter` | all | -| `MasterLiveGekisouLuckBasePoint` | `_id` `_noteCategory` `_noteSimulateJudgement` `_weight` `_basePoint` | -| `MasterLiveGekisouLuckBonusLot` | `_id` `_chanceLotType` `_lotResult` `_weight` | -| `MasterLiveGekisouRankingScoreBonus` | `_id` `_missionPattern` `_rank` `_count` `_scoreBonusPercent` | -| `MasterLiveMusic` | `_id` `_musicType` `_bestMusicTagIDs` `_liveScoreRankGroup` `_easyID` `_normalID` `_hardID` `_expertID` `_gekisouMission1` `_gekisouMission2` `_gekisouMission3` | -| `MasterLiveMusicScore` | `_id` `_musicScoreTextFileName` `_musicScoreLevel` `_fullComboCount` | -| `MasterArenaMusic` | all | -| `MasterChallengeMusic` | all | -| `MasterLiveScoreRank` | `_id` `_group` `_liveScoreRank` `_requiredScore` `_battleLiveRequiredScore` | -| `MasterEvent` | all | -| `MasterEventEffect` | all | -| `MasterLiveEventPoint` | all | -| `MasterChallengeLiveEventPoint` | all | -| `MasterLiveChallengePoint` | all | -| `MasterLiveMusicBoostBonus` | `_id` `_consumedLiveBoostCount` `_liveMusicRewardRate` `_playerExpRate` `_memberCardExpRate` `_friendshipExpRate` `_eventPointRate` | -| `MasterChallengeMusicBoostBonus` | all | - -### charts - -One chart per `MasterLiveMusicScore` row, sorted by `scoreId`. Song and score facts (level, full combo count, song -type, the song's difficulties) are not copied into the charts: they are in `MasterLiveMusicScore` and -`MasterLiveMusic`, joined by `scoreId` (`MasterLiveMusicScore._id`, and `MasterLiveMusic._easyID` .. `_expertID`). - -| Field | Content | -|---|---| -| `scoreId` | `MasterLiveMusicScore._id` | -| `asset.key` | `Live/MusicScore/<_musicScoreTextFileName>`, the chart's TextAsset | -| `asset.sha256` | SHA-256 of the TextAsset's bytes as shipped | -| `notes.id` | note id, unique within the chart | -| `notes.op` | `NoteOperateType` | -| `notes.judgementType` | `NoteJudgementType` (from the operate type and the critical flag) | -| `notes.timeMs` | note time in chart milliseconds | -| `skillEvents.timeMs` | skill event times in chart order; the position in the array is the event index (the times need not ascend) | -| `fevers.startMs`, `fevers.endMs` | fever ranges sorted by start; the position in the arrays is the range index | - -The notes are columns: `notes.id[i]`, `notes.op[i]`, `notes.judgementType[i]` and `notes.timeMs[i]` describe the -same note, and the four arrays have the same length; `fevers.startMs[i]` and `fevers.endMs[i]` are one range. The -notes are every note the client creates at runtime, including hidden notes, guide notes, and the combo ticks of -slides (operate types Combo and ComboSkip), in the order the client enumerates its note dictionary. That order is -not the time order: the combo ticks of a slide that fall on a time no other note has come after the slide's end. -Order by `timeMs` (then `id`) where time order is needed. - -Readers can derive further counts from these fields (for example the number of judged notes from `notes.op`); the -file does not contain derived values. - -## Versions - -`format` names the major version. Within `nnnotes.deck-data/1`, fields and tables are only added, never renamed, -removed or given another meaning, and readers ignore keys they do not know. A reader rejects a file with a major -version it does not know. A change that breaks readers is a new major version, `nnnotes.deck-data/2`. diff --git a/docs/music-data.md b/docs/music-data.md new file mode 100644 index 0000000..2294e41 --- /dev/null +++ b/docs/music-data.md @@ -0,0 +1,369 @@ +# Music data + +`nnnotes music-data` writes one JSON file with every live song and chart of one master data version: what a song +listing shows (titles and credits in every language, bands, vocal characters, category, tags, release time, score +ranks, jacket, BGM length), per difficulty the chart facts (level, note counts, BPM, chart times, skill events, +fevers), and per chart its **deck statistics**: what the chart contributes to the live score whatever the deck, in a +solo live (Gekisou off) and in a Gekisou live at every rank, measured by the deck model [ournotes-deck](https://github.com/empty-sekai/ournotes-deck), which nnnotes carries as +its extension module `nnnotes._deck`. The format is `nnnotes.music-data/1`; its JSON Schema is +[schema/music-data.schema.json](schema/music-data.schema.json). + +``` +nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] + [--workers N] [--no-bgm] [--jackets DIR] -o FILE +``` + +- `--master-files DIR`: master data files as served, `DIR/MasterManifest.json` and the `.bin` files it lists + (`nnnotes master download`). The file's `region` is `[catalog] region` (`--region`). +- `--apk-master`: the master data files the APK ships (`assets/Master/` of `[paths] apk`, the same layout). The + file's `region` is `embedded`. +- `--decoded-master`: master data decoded elsewhere, the directory the other commands read (`[paths] master`, + `--master`, `[servers.] master`): one `
.json` per table (`nnnotes master decode`) and the + `MasterManifest.json` of the files they were decoded from, as a master data snapshot published with its manifest + carries it. The file's `region` is `[catalog] region`; no master key is needed. +- `--full`: also write the deck model's input, every chart's runtime notes and the master data tables about cards, + skills, bonuses, scores and events ([the deck input](#the-deck-input---full)), for tools that run a deck model of + their own. +- `--no-deck`: do not run the deck model; every chart's `deck` is null. The deck model measures every chart on a + whole-live simulation, dozens of lives per chart: a full run takes processor time in proportion to the number of + charts. `--workers N` sets the threads it uses (default: every processor), `--seeds N` the seeds measured on a chart + with a luck range (default 8). +- `--no-bgm`: do not read the cue sheets (every `bgm.length` is null). +- `--jackets DIR`: also write every song's jacket, the Texture2D `Image/Jacket/`, as `DIR/.webp` + (WebP quality 88, scaled down with Lanczos to at most 320 pixels on the longer side, without alpha when opaque); a + page next to the file finds a song's jacket at `/.webp`. + +Each master data file is checked against the SHA-256 the manifest lists and decoded with `[master] key` and `iv`. +With `--decoded-master` the tables are read as decoded, and the master data version and each file's SHA-256 are the +manifest's: the decoded tables cannot be checked against the files as served, so the file records what the manifest +lists (the same values as `--master-files` on those files). +Charts are read from the catalog of `[catalog] language` (bundles fetched into the cache as for every command); the +BGM length from the cue sheet's ACB (its `CueTable` and `WaveformTable`, no audio is decoded). `FILE` ending in `.gz` +is written gzip-compressed. The command prints a summary (`songs`, `charts`, `deck`: the deck model's commit, +`unplayable`, `bytes`, `sha256` of the file, ...). + +The command writes the file only when every table, chart and cue sheet was read and every chart measured: a missing +or mismatching master data file, a table without a column the file exports, a text id that `MasterText` does not +have, a score id that `MasterLiveMusicScore` does not have, a missing or unreadable chart asset, a note id that +occurs twice in a chart, a cue sheet without the song's cue, (with `--jackets`) a missing jacket texture, a chart the +deck model cannot measure or whose check deck fails, or deck statistics that disagree with the chart facts or the +master data stops it +with exit status 1 and a line naming the input. The file is written through a temporary file and a rename. + +An installation without the extension module (a source checkout that was not built) runs only with `--no-deck`; +the wheels on PyPI carry it. See [Building](#building). + +## Layout + +```json +{ + "format": "nnnotes.music-data/1", + "provenance": {"region": ..., "client": {...}, "catalog": {...}, "master": {...}, "exporter": {...}, "deck": {...}}, + "languages": ["ja", "en", "zh-Hant", "zh-Hans", "ko"], + "bands": [...], "characters": [...], "tags": [...], "categories": [...], + "deck": {"model": {...}, "kinds": [...]}, + "songs": [{"id": 100001, ..., "charts": [{..., "deck": {...}}, ...]}, ...], + "master": {...}, "charts": [...] +} +``` + +`master` and `charts` are present with `--full` only. The file is minified UTF-8 with one line feed at the end. Keys +are always in the order shown here and in the tables below, so the same inputs with the same nnnotes version (and so +the same deck model) give the same bytes. Master data values that are binary32 (single precision) in the game, a +number the master data writes with a fraction or an exponent, are written as the shortest decimal that reads back as +the same binary32 value (infinity as `1e999` / `-1e999`); the deck model's numbers are binary64 and written as it +writes them. A gzip file has no file name and a zero modification time in its header. + +A **text** is an object with one string per language of `languages` (`{"ja": ..., "en": ..., "zh-Hant": ..., +"zh-Hans": ..., "ko": ...}`), the `MasterText` row of the id; a text field is null when the master data gives no id +(an empty string). A language's string may be empty when the game has no text in that language. + +### provenance + +| Field | Content | +|---|---| +| `region` | the region whose master data this is (a configured region name), or `embedded` for the APK's master data | +| `client.versionName`, `client.versionCode` | the APK's version name and code (null without `[paths] apk`) | +| `catalog.resourceVersion` | the resource version recorded for the catalog in the catalog store (`nnnotes catalogs fetch` / `import`), null when none is recorded | +| `catalog.sha256` | SHA-256 of the remote catalog file the charts were read with | +| `master.source` | `api` (`--master-files`, `--decoded-master`: the region's files) or `embedded` (`--apk-master`) | +| `master.version` | the `version` of the master data manifest | +| `master.tables.
.sha256` | SHA-256 of each table's file as served, before decoding: the song tables (`MasterLiveMusic`, `MasterLiveMusicScore`, `MasterText`, `MasterBand`, `MasterCharacter`, `MasterTag`, `MasterLiveMusicCategory`, `MasterSound`, `MasterSoundCueSheet`, `MasterLiveScoreRank`) and, when the deck model runs or with `--full`, the tables of [the deck input](#the-deck-input---full) | +| `exporter.name`, `exporter.version` | `nnnotes` and its version | +| `exporter.chartFormat` | the format of the chart converter the notes come from, `nnnotes.live-score/1` | +| `deck` | the deck model: `{name, version, source, commit, format}`, `ournotes-deck`, its package version, repository, the git commit nnnotes is built with and the statistics format (`ournotes-deck.chart-stats/2`); null with `--no-deck` | + +### bands, characters, tags, categories + +| Field | Content | +|---|---| +| `bands[].id`, `.name`, `.mainColor`, `.subColor` | `MasterBand`: id, name text, color codes | +| `characters[].id`, `.bandId`, `.name`, `.shortName`, `.mainColor` | `MasterCharacter` | +| `tags[].id`, `.name` | `MasterTag` (the ids of `bestMusicTagIds`) | +| `categories[].id`, `.musicCategories`, `.name` | `MasterLiveMusicCategory` (the listing's category tabs; `musicCategories` are the song category values it shows) | + +### songs + +One entry per `MasterLiveMusic` row, sorted by `id`. + +| Field | Content | +|---|---| +| `id`, `sortOrder`, `startAt`, `defaultUnlock` | the row's `_id`, `_sortOrder`, `_startAt` (as served, server time), `_defaultUnlock` | +| `title`, `ruby`, `phonetic` | texts of `_titleTextID`, `_rubyTitleTextID`, `_phoneticTextID` | +| `bandIds`, `bandName` | `_bandIDs`; the text of `_bandNameTextID`, a name the song shows instead of its first band's (null: none) | +| `vocalCharacterIds` | `_vocalCharacterIDs` | +| `lyricist`, `composer`, `arranger` | texts of `_lyricistTextID`, `_composerTextID`, `_arrangerTextID` | +| `musicType`, `musicCategories`, `bestMusicTagIds` | `_musicType` (the song type that card type bonuses match), `_musicCategories`, `_bestMusicTagIDs` | +| `jacket` | `_jacketAssetName` (the jacket sprite `Image/Jacket/`) | +| `gekisouMissions` | `[_gekisouMission1, _gekisouMission2, _gekisouMission3]`: the missions of the song's Gekisou ranges (1 combo, 2 luck, 3 Just count) | +| `bgm.soundId`, `.cueSheet`, `.cue` | `_musicSoundID` and its `MasterSound` / `MasterSoundCueSheet` cue | +| `bgm.length` | `{lengthMs, samples, sampleRate, durationMs}`: the cue's `Length` in the ACB `CueTable`, the sample count and rate of its first waveform, and `samples * 1000 // sampleRate`; null with `--no-bgm` | +| `scoreRanks` | `[{rank, requiredScore, battleRequiredScore}]`: the `MasterLiveScoreRank` rows of `_liveScoreRankGroup` in required-score order (rank `E` .. `SS`; `_requiredScore`, `_battleLiveRequiredScore`). The group is the song's, so every difficulty shares these thresholds; a live's rank is the last row whose required score its score reaches | +| `charts` | one entry per difficulty the song has (`_easyID` .. `_expertID` not 0), in the order easy, normal, hard, expert | +| `master.MasterLiveMusic`, `master.MasterLiveScoreRank` | the whole `MasterLiveMusic` row and its score rank rows as decoded | + +### charts + +| Field | Content | +|---|---| +| `difficulty` | `easy`, `normal`, `hard` or `expert` | +| `scoreId`, `level`, `displayLevel`, `fullComboCount` | `MasterLiveMusicScore`: `_id`, `_musicScoreLevel`, `_musicScoreDisplayLevel`, `_fullComboCount` | +| `asset.key`, `asset.sha256` | the chart TextAsset `Live/MusicScore/<_musicScoreTextFileName>` and the SHA-256 of its bytes as shipped | +| `notes.judged` | notes that are judged (and count for a full combo) | +| `notes.total` | every runtime note, including hidden notes, guide notes and slide combo ticks | +| `notes.byOperateType` | `{"": count}` over every runtime note, keys in ascending order | +| `bpm.changes` | every BPM change `{timeMs, bpm}` in time order | +| `bpm.main`, `.min`, `.max` | over the played span (first to last judged note): the BPM that holds longest (the earliest on a tie), the lowest and the highest | +| `firstNoteMs`, `lastJudgedNoteMs` | times of the first and the last judged note | +| `lastNoteMs` | the latest time of any runtime note (the time the score code calls the last timing note) | +| `musicLengthMs` | `lastNoteMs + 1000`: the live's music length on the game's score path (skill effects end at it at the latest) | +| `skillEventsMs` | skill event times in chart order; event `i` fires the skill of the member at performance position `i` | +| `fevers` | fever ranges `[startMs, endMs]` sorted by start | +| `deck` | the chart's deck statistics (below); null with `--no-deck` | + +Chart times are milliseconds of chart time. The two lengths are different facts: `bgm.length` is how long the music +plays, `musicLengthMs` is the length the score code uses; a listing chooses the one it needs. + +## Deck statistics + +The deck model plays every chart of a song on its whole-live simulation in the theoretical best play, the frame +times of the game's default schedule, in two scenarios: + +- **Gekisou on** (`seeds`), as a Gekisou live (Battle Live, up to five players) plays: every judged note at its exact + time, Just inside the Just-count ranges and Perfect elsewhere, rank 1 in every Gekisou range. Other ranks follow + from the same numbers (below). +- **Gekisou off** (`offSeeds`), as a solo live (Free Live, Challenge Live) plays: every judged note Perfect at its + exact time, seed 0, no Just, luck, Gekisou combo or rank bonus. A chart with more than three fevers plays here too. + +In each it measures, at deck power `model.power` (300000): + +- `score`: the exact score without skills (with Gekisou on, the rank 1 bonuses of the Gekisou ranges included); +- for every **score-up kind** (`deck.kinds`) and every performance position `k`, `weights[kind][k]`: the exact score + a deck gains when its position-`k` member has one effect of that kind at factor 1, divided by the deck power. The + effect runs through the simulation's own updaters, conditions, frames and appliers, so the weight carries every rule + of the game (execute and finish frames, the 40 ms score frames, combo and Gekisou combo factors, Just scores, luck + rushes, the rank bonuses of the ranges it overlaps). + +A deck whose live skills are all score-up kinds then scores, up to the floors, + +``` +P * (score / power + sum over positions k of factor_k * weights[kind_k][k]) +``` + +with `P` the deck power and `factor_k` the effect's factor as the game's applier converts its value: effect type 2000 +`floor(value / 10000f * 1e5) / 1e5`, 2005 `floor(value / -10000f * 1e5) / 1e5`, 2002 and 2004 the same quotient +rounded half to even (`value / 10000f` in binary32). Each seed also plays a **check deck**, random kinds at real +master values at another power (`model.checkPower`), and the command fails when its exact score leaves the bound of +the formula. Effects of other types (cumulative score 2001 / 2003, life, judgement conversion, Gekisou and snap +skills) are not linear in the chart alone and have no weights: a deck's score with them comes from the simulation. + +A luck range draws lottery results from the play's random seed, so the Gekisou on measurements are given **per +seed**: one seed (0) when no range is a luck range, else the first `--seeds` seeds of the deck model's published seed +set. The seed set is not the game's seed law (which is unknown); a mean over it is not the game's expectation. + +### Ranks + +In a Gekisou live, range `i` takes a rank `r_i` from 1 to 5 among the room's players, and its rank bonus is +`trunc(rangeScore_i * p_i(r_i) / 100)` with `p_i(r) = ranges[i].rankBonusPercents[r - 1]`. The bonus is a fixed score +at the range's end: it changes no factor and no note score, and a later range's score holds it at both ends, so the +range scores do not depend on the ranks. At ranks `r` a seed's numbers are therefore + +``` +score_r = score - sum_i rankBonus_i + sum_i trunc(rangeScore_i * p_i(r_i) / 100) (exact) +weights_r[kind][k] = weights[kind][k] + sum_i (p_i(r_i) - p_i(1)) / 100 * rangeWeights[kind][k][i] +``` + +(`rankBonus_i`, `rangeScore_i`: `seeds[].ranges[i]`), and a deck scores `P * (score_r / power + sum_k factor_k * +weights_r[kind_k][k])` as above; `weights_r` is within `2 * ranges / power` per unit of factor of the exact weight at +those ranks. At rank 1 everywhere these are `score` and `weights`. `rangeWeights` is null when a range's bonus can fall +inside another range's score frames (overlapping ranges), and a kind's entry is null when its conditions read the +confirmed rank (condition 7012): the ranks do not follow linearly there. `rankCheck` plays the seed's check deck at +random ranks through the simulation's explicit rank confirmations and bounds it against these formulas. + +### Just rate + +`scorePerfect` and `ranges[i].rangeScorePerfect` are the no-skill score (rank 1 bonuses included) and the range +scores of the same play with every Just judged Perfect (the Just judgement is enabled only inside the Just-count +ranges, so nothing else changes). Between a Just rate of 1 (`score`, `rangeScore`) and 0 (`scorePerfect`, +`rangeScorePerfect`) a page can interpolate; the rank bonus of the Perfect play is `trunc(rangeScorePerfect_i * +p_i(r_i) / 100)`. A chart without Just notes (`justNotes` 0) has `scorePerfect == score`. + +### deck + +| Field | Content | +|---|---| +| `model` | the deck model's description of the measurement: `engine`, `play`, `score` (the formula), `power`, `checkPower`, `unitValue` (the effect value of factor 1, 10000), `seeds`, `ranks` (the rank formulas), `perfect` (the Perfect play), `off` (the Gekisou off scenario) | +| `kinds[]` | the score-up kinds of the master data: `MasterLiveSkillEffect` rows of type 2000, 2002, 2004 or 2005 without a cumulative condition, grouped by what shapes their score. `id` (the index in `weights`), `effectType`, `activationTimeSecond`, `durationMs` (`ceil(activationTimeSecond * 1000f)`), `skillTargetIds`, `skillConditionGroup`, `skillReleaseConditionGroup`, `effectLimitCount`, `effectExecuteLimitCount`, `effectExecuteLimitResetConditionGroup`; `rows` (master rows of the kind) and `values` (their distinct `_effectValue`s, ascending) | + +The kind of a card's live skill is found by matching its `MasterLiveSkillEffect` row (at the skill level) on these +fields; `values` lists what the master data uses. + +### charts[].deck + +| Field | Content | +|---|---| +| `convertedNoteCount` | the note count the score formula divides by (converted notes) | +| `skip` | score per unit of deck power of a skipped live (every note Great, combo 0, no skills) | +| `events` | `[[position, timeMs], ...]`: the skill events in chart order with the performance position each fires | +| `positions` | the performance positions the events fire (the largest position + 1): the length of every `weights[kind]` | +| `ranges[]` | the Gekisou ranges: `index`, `mission` (1 combo, 2 luck, 3 Just count), `startMs`, `endMs`, `rankBonusPercents` (the rank bonus percentages of ranks 1..5 of the song's mission pattern, `MasterLiveGekisouRankingScoreBonus`), `rankBonusPercent` (the rank 1 percentage, `rankBonusPercents[0]`) | +| `justNotes` | notes judged Just on the Gekisou on play | +| `seeds[]` | Gekisou on, per seed: `seed`; `score` (points at `model.power`, rank 1 bonuses included); `ranges[]` (`rangeScore`: the points gained inside the range, `rankBonus`: its rank 1 bonus in points, `maxCombo`, `justCount`, `lotResults`: lottery results Miss, Hit, Super Hit, Critical, `rangeScorePerfect`: `rangeScore` on the Perfect play); `weights[kind][position]` (points per unit of deck power and of factor); `check` (`deck`: `[kind, value]` or null per position, `exact`, `predicted`, `bound`: points at `model.checkPower`); `scorePerfect` (`score` on the Perfect play); `rangeWeights[kind][position][range]` (range points per unit of deck power and of factor, or null; a kind null); `rankCheck` (`ranks`: 1..5 per range, `exact`, `predicted`, `bound`; null without ranges or range weights) | +| `offSeeds[]` | Gekisou off, one seed: `seed` (0), `score`, `weights[kind][position]` (a kind null when its conditions read the Gekisou state, which a solo live does not have) and `check`, as in `seeds[]` | +| `unplayable` | null, or why the game cannot play the chart with Gekisou (more than three fevers: the game fails when the fourth starts); `seeds` is then empty, `offSeeds` is not | + +The deck model's chart facts are checked against the file's: the song, difficulty, level, judged note count, last +note time, music length, Gekisou missions, skill event times and fever ranges must agree, and are not repeated in +`deck`. Its numbers are checked against the master data and themselves: every range's `rankBonusPercents` are the +`MasterLiveGekisouRankingScoreBonus` rows of the song's mission pattern (0 without a row), every `rankBonus` is +`trunc(rangeScore * rankBonusPercent / 100)`, a chart without Just notes has the same scores on the Perfect play, +every array has its shape (`[kind][position]`, `[kind][position][range]`, one range result per range, one Gekisou off +seed), and every check and rank check is within its bound. + +## The deck input (`--full`) + +With `--full` the file ends with the input the deck model reads: `master`, the master data tables about cards, +skills, bonuses, scores and events, and `charts`, every chart as the client builds it at runtime. They hold facts as +the game has them (master data values as served, notes as the client's chart converter creates them), not values +derived from them. + +### master + +Each table is `{"columns": [...], "rows": [[...], ...]}`: one array per row, its values in the order of `columns`, +rows in the order the master data lists them. Values are as decoded: integers, strings, booleans, arrays; a binary32 +column should be parsed as 32-bit floats. + +Tables with a column list export those columns, and every row must have them. Tables marked "all" export every +column their rows have, in the order the rows first have them; when such a table has no rows, `columns` is empty +(the master data carries no field list for an empty table). A reader treats a table missing from `master` as empty. + +| Table | Columns | +|---|---| +| `MasterMemberCard` | `_id` `_characterID` `_rarity` `_cardType` `_bestMusicTagIDs` `_performancePowerMax` `_technicPowerMax` `_visualPowerMax` `_memberCardLevelGroup` `_memberCardAwakeGroup` `_memberCardRankGroup` `_leaderSkillID` `_liveSkillID` `_gekisouSkillID` | +| `MasterMemberCardLevel` | `_id` `_group` `_level` `_exp` `_performanceRate` `_technicRate` `_visualRate` | +| `MasterMemberCardLevelLimit` | `_id` `_rarity` `_awakeCount` `_limitLevel` | +| `MasterMemberCardAwake` | `_id` `_group` `_awakeCount` `_performanceRate` `_technicRate` `_visualRate` | +| `MasterMemberCardRank` | `_id` `_group` `_rank` `_performanceRate` `_technicRate` `_visualRate` `_leaderSkillLevel` `_musicTypeBonusRate` `_musicTagBonusRate` | +| `MasterSupportCard` | `_id` `_characterIDs` `_rarity` `_cardType` `_performancePowerMax` `_technicPowerMax` `_visualPowerMax` `_supportCardLevelGroup` `_supportCardRankGroup` `_supportSkillId01` `_supportSkillId02` `_gekisouSupportSkillId01` `_gekisouSupportSkillId02` | +| `MasterSupportCardLevel` | `_id` `_group` `_level` `_exp` `_performanceRate` `_technicRate` `_visualRate` | +| `MasterSupportCardRank` | `_id` `_group` `_rank` `_limitLevel` `_cardTypeLinkBonusRate` `_supportSkill01Level` `_supportSkill02Level` `_gekisouSupportSkill01Level` `_gekisouSupportSkill02Level` | +| `MasterCharacter` | `_id` `_bandID` | +| `MasterBand` | `_id` | +| `MasterCharacterRank` | `_id` `_rank` `_bonus` | +| `MasterCharacterTotalRank` | `_id` `_totalRank` `_bonus` | +| `MasterBandItemSkillEffect` | `_id` `_bandItemId` `_level` `_skillTargetIDs` `_skillEffectType` `_effectValue` | +| `MasterBandItem` | `_id` `_bandId` | +| `MasterBandItemLevel` | `_id` `_bandItemId` `_level` `_playerRank` | +| `MasterVipRankBonus` | `_id` `_vipRank` `_vipBonusType` `_value` | +| `MasterMemoryMusicGroup` | `_id` `_skillTargetIds` | +| `MasterMemoryMusic` | `_id` `_groupId` | +| `MasterMemoryMusicBonus` | `_id` `_groupId` `_scoreRank` `_performance` `_technic` `_visual` | +| `MasterMemoryMemberLevel` | `_id` `_point` `_performance` `_technic` `_visual` | +| `MasterMemorySupportLevel` | `_id` `_point` `_performance` `_technic` `_visual` | +| `MasterSkillTarget` | `_id` `_skillTargetType` `_characterID` `_bandID` `_cardType` `_tagID` `_judgement` `_liveMusicType` `_gekisouMissionType` `_liveSkillCategories` `_gekisouSkillCategories` | +| `MasterSkillCondition` | `_id` `_conditionType` `_conditionValues` `_isPositive` `_conditionTargetIDs` | +| `MasterSkillConditionSet` | `_id` `_group` `_conditionIds` | +| `MasterSkillCumulativeCondition` | `_id` `_skillCumulativeConditionType` `_conditionValues` `_conditionTargetIDs` `_maxCumulativeCount` | +| `MasterSkillEffectSetting` | `_id` `_skillEffectType` `_phase` | +| `MasterLeaderSkillEffect` | `_id` `_leaderSkillID` `_level` `_skillConditionGroup` `_skillTargetIDs` `_skillEffectType` `_effectValue` `_skillCumulativeConditionID` | +| `MasterLiveSkill` | `_id` `_skillCategories` | +| `MasterLiveSkillEffect` | `_id` `_liveSkillID` `_level` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | +| `MasterSupportSkill` | `_id` | +| `MasterSupportSkillEffect` | `_id` `_supportSkillID` `_level` `_skillTriggerConditionGroup` `_skillTriggerType` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | +| `MasterGekisouSkill` | `_id` `_gekisouMissionType` `_skillCategories` | +| `MasterGekisouSkillEffect` | `_id` `_gekisouSkillID` `_level` `_skillTriggerConditionGroup` `_skillTriggerType` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | +| `MasterGekisouSupportSkill` | `_id` `_gekisouSupportSkillExecTiming` `_gekisouMissionType` | +| `MasterGekisouSupportSkillEffect` | `_id` `_gekisouSupportSkillID` `_level` `_skillTriggerConditionGroup` `_skillTriggerType` `_skillConditionGroup` `_skillReleaseConditionGroup` `_skillTargetIDs` `_skillEffectType` `_activationTimeSecond` `_effectValue` `_maxEffectValue` `_effectLimitCount` `_skillCumulativeConditionID` `_effectExecuteLimitCount` `_effectExecuteLimitResetConditionGroup` | +| `MasterLiveNoteParameter` | `_id` `_noteOperateType` `_scorePercent` | +| `MasterLiveJudgementParameter` | `_id` `_noteSimulateJudgement` `_scorePercent` `_damage` | +| `MasterLiveJudgementTiming` | `_id` `_assistLevel` `_judgementPriority` `_noteJudgementType` `_noteSimulateJudgement` `_beforeMs` `_afterMs` | +| `MasterLiveComboScoreBonus` | `_id` `_comboBonusType` `_requiredComboCount` `_bonusFactor` | +| `MasterLiveSettings` | all | +| `MasterParameter` | all | +| `MasterLiveGekisouLuckBasePoint` | `_id` `_noteCategory` `_noteSimulateJudgement` `_weight` `_basePoint` | +| `MasterLiveGekisouLuckBonusLot` | `_id` `_chanceLotType` `_lotResult` `_weight` | +| `MasterLiveGekisouRankingScoreBonus` | `_id` `_missionPattern` `_rank` `_count` `_scoreBonusPercent` | +| `MasterLiveMusic` | `_id` `_musicType` `_bestMusicTagIDs` `_liveScoreRankGroup` `_easyID` `_normalID` `_hardID` `_expertID` `_gekisouMission1` `_gekisouMission2` `_gekisouMission3` | +| `MasterLiveMusicScore` | `_id` `_musicScoreTextFileName` `_musicScoreLevel` `_fullComboCount` | +| `MasterArenaMusic` | all | +| `MasterChallengeMusic` | all | +| `MasterLiveScoreRank` | `_id` `_group` `_liveScoreRank` `_requiredScore` `_battleLiveRequiredScore` | +| `MasterEvent` | all | +| `MasterEventEffect` | all | +| `MasterEventAchievementReward` | `_id` `_eventId` `_eventPoint` `_rewardIds` | +| `MasterEventAchievementLoopReward` | `_id` `_eventId` `_loopStartEventPoint` `_loopEventPoint` `_rewardIds` | +| `MasterLiveEventReward` | `_id` `_group` `_eventGroup` `_scoreRank` `_resourceType` `_resourceId` `_resourceCount` `_probability` | +| `MasterChallengeLiveEventReward` | `_id` `_group` `_eventGroup` `_scoreRank` `_resourceType` `_resourceId` `_resourceCount` `_probability` | +| `MasterLiveEventPoint` | all | +| `MasterChallengeLiveEventPoint` | all | +| `MasterLiveChallengePoint` | all | +| `MasterLiveMusicBoostBonus` | `_id` `_consumedLiveBoostCount` `_liveMusicRewardRate` `_playerExpRate` `_memberCardExpRate` `_friendshipExpRate` `_eventPointRate` | +| `MasterChallengeMusicBoostBonus` | all | + +### charts + +One chart per `MasterLiveMusicScore` row, sorted by `scoreId`, including charts of no song. Song and score facts +are not copied into these records: they are in `songs` and in `master`, joined by `scoreId`. + +| Field | Content | +|---|---| +| `scoreId` | `MasterLiveMusicScore._id` | +| `asset.key` | `Live/MusicScore/<_musicScoreTextFileName>`, the chart's TextAsset | +| `asset.sha256` | SHA-256 of the TextAsset's bytes as shipped | +| `notes.id` | note id, unique within the chart | +| `notes.op` | `NoteOperateType` | +| `notes.judgementType` | `NoteJudgementType` (from the operate type and the critical flag) | +| `notes.timeMs` | note time in chart milliseconds | +| `skillEvents.timeMs` | skill event times in chart order; the position in the array is the event index (the times need not ascend) | +| `fevers.startMs`, `fevers.endMs` | fever ranges sorted by start; the position in the arrays is the range index | + +The notes are columns: `notes.id[i]`, `notes.op[i]`, `notes.judgementType[i]` and `notes.timeMs[i]` describe the +same note, and the four arrays have the same length; `fevers.startMs[i]` and `fevers.endMs[i]` are one range. The +notes are every note the client creates at runtime, including hidden notes, guide notes, and the combo ticks of +slides (operate types Combo and ComboSkip), in the order the client enumerates its note dictionary. That order is +not the time order: the combo ticks of a slide that fall on a time no other note has come after the slide's end. +Order by `timeMs` (then `id`) where time order is needed. + +The deck model reads the same content under the format name `nnnotes.deck-data/1` (`format`, `provenance`, +`master`, `charts`); nnnotes hands it over in memory. + +## Building + +The deck model is the Rust crate ournotes-deck, pinned by commit in `rust/Cargo.toml` (and `rust/Cargo.lock`) and +built into the extension module `nnnotes._deck` with [maturin](https://www.maturin.rs/) (PyO3, the stable ABI of +Python 3.11 and later: one wheel per platform). The release workflow builds the wheels; `pip install .` or +`pip install -e .` in a checkout builds the module with the Rust toolchain. The same nnnotes version always carries +the same deck model: the commit moves only through a pull request (`.github/workflows/deck.yml` opens one when +ournotes-deck's `main` moves), and `provenance.deck.commit` names it in every file. + +## Versions + +`format` names the major version. Within `nnnotes.music-data/1`, fields and tables are only added, never renamed, +removed or given another meaning, and readers ignore keys they do not know. A reader rejects a file with a major +version it does not know. A change that breaks readers is a new major version, `nnnotes.music-data/2`. The deck +statistics follow the deck model's format (`provenance.deck.format`): a new major version of it is a new major +version of this file. + +`nnnotes.music-data/1` replaces the `nnnotes.songs/1` file of `nnnotes songs` (its fields are the songs, charts and +their facts here) and the `nnnotes.deck-data/1` file of `nnnotes deck-data` (its content is the deck input of +`--full`). diff --git a/docs/schema/deck-data.schema.json b/docs/schema/deck-data.schema.json deleted file mode 100644 index d56c1e5..0000000 --- a/docs/schema/deck-data.schema.json +++ /dev/null @@ -1,273 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:nnnotes:schema:deck-data", - "title": "nnnotes.deck-data/1", - "description": "Data for deck-building tools, for one master data version: every live chart as the client builds it at runtime (notes in the order the client enumerates them, skill events, fever ranges) and a subset of the master data tables as columns and rows, with the provenance of both. Written by `nnnotes deck-data`; described in docs/deck-data.md. Within the arrays of a chart's `notes` (and of its `fevers`) the i-th elements belong to the same note (range): the arrays have equal lengths, which a schema cannot state. Readers ignore keys they do not know.", - "type": "object", - "required": [ - "format", - "provenance", - "master", - "charts" - ], - "properties": { - "format": { - "const": "nnnotes.deck-data/1" - }, - "provenance": { - "type": "object", - "required": [ - "region", - "client", - "catalog", - "master", - "exporter" - ], - "properties": { - "region": { - "type": "string", - "description": "the region whose master data this is (a configured region name), or `embedded` for the master data the APK ships" - }, - "client": { - "type": "object", - "required": [ - "versionName", - "versionCode" - ], - "properties": { - "versionName": { - "type": [ - "string", - "null" - ], - "description": "versionName of the APK (null without one)" - }, - "versionCode": { - "type": [ - "integer", - "null" - ], - "description": "versionCode of the APK (null without one)" - } - } - }, - "catalog": { - "type": "object", - "required": [ - "resourceVersion", - "sha256" - ], - "properties": { - "resourceVersion": { - "type": [ - "string", - "null" - ], - "description": "the resource version recorded for the catalog in the catalog store, null when none is recorded" - }, - "sha256": { - "$ref": "#/$defs/sha256", - "description": "SHA-256 of the remote catalog file the charts were read with" - } - } - }, - "master": { - "type": "object", - "required": [ - "source", - "version", - "tables" - ], - "properties": { - "source": { - "enum": [ - "api", - "embedded" - ], - "description": "api: files as downloaded from the region; embedded: the files the APK ships" - }, - "version": { - "type": [ - "string", - "null" - ], - "description": "`version` of the master data manifest" - }, - "tables": { - "type": "object", - "description": "per exported table, the SHA-256 of its file as served (before decoding)", - "additionalProperties": { - "type": "object", - "required": [ - "sha256" - ], - "properties": { - "sha256": { - "$ref": "#/$defs/sha256" - } - } - } - } - } - }, - "exporter": { - "type": "object", - "required": [ - "name", - "version", - "chartFormat" - ], - "properties": { - "name": { - "const": "nnnotes" - }, - "version": { - "type": "string" - }, - "chartFormat": { - "type": "string", - "description": "format of the chart converter the notes come from (nnnotes.live-score/1)" - } - } - } - } - }, - "master": { - "type": "object", - "description": "tables by name; a table not present reads as empty", - "additionalProperties": { - "$ref": "#/$defs/table" - } - }, - "charts": { - "type": "array", - "description": "one chart per MasterLiveMusicScore row, sorted by scoreId", - "items": { - "$ref": "#/$defs/chart" - } - } - }, - "$defs": { - "sha256": { - "type": "string", - "pattern": "^[0-9a-f]{64}$" - }, - "ints": { - "type": "array", - "items": { - "type": "integer" - } - }, - "table": { - "type": "object", - "required": [ - "columns", - "rows" - ], - "properties": { - "columns": { - "type": "array", - "items": { - "type": "string" - }, - "uniqueItems": true - }, - "rows": { - "type": "array", - "items": { - "type": "array" - }, - "description": "one array per row, one value per column in the order of `columns`" - } - } - }, - "chart": { - "type": "object", - "required": [ - "scoreId", - "asset", - "notes", - "skillEvents", - "fevers" - ], - "properties": { - "scoreId": { - "type": "integer", - "description": "MasterLiveMusicScore._id" - }, - "asset": { - "type": "object", - "required": [ - "key", - "sha256" - ], - "properties": { - "key": { - "type": "string", - "description": "Live/MusicScore/<_musicScoreTextFileName>" - }, - "sha256": { - "$ref": "#/$defs/sha256", - "description": "SHA-256 of the chart TextAsset's bytes as shipped" - } - } - }, - "notes": { - "type": "object", - "required": [ - "id", - "op", - "judgementType", - "timeMs" - ], - "properties": { - "id": { - "$ref": "#/$defs/ints", - "description": "note id, unique within the chart" - }, - "op": { - "$ref": "#/$defs/ints", - "description": "NoteOperateType" - }, - "judgementType": { - "$ref": "#/$defs/ints", - "description": "NoteJudgementType" - }, - "timeMs": { - "$ref": "#/$defs/ints", - "description": "note time in chart milliseconds" - } - } - }, - "skillEvents": { - "type": "object", - "required": [ - "timeMs" - ], - "properties": { - "timeMs": { - "$ref": "#/$defs/ints", - "description": "skill event times in chart order (the index is the event index)" - } - } - }, - "fevers": { - "type": "object", - "required": [ - "startMs", - "endMs" - ], - "properties": { - "startMs": { - "$ref": "#/$defs/ints", - "description": "range starts, sorted (the index is the range index)" - }, - "endMs": { - "$ref": "#/$defs/ints" - } - } - } - } - } - } -} diff --git a/docs/schema/music-data.schema.json b/docs/schema/music-data.schema.json new file mode 100644 index 0000000..dad6a10 --- /dev/null +++ b/docs/schema/music-data.schema.json @@ -0,0 +1,1168 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nnnotes:schema:music-data", + "title": "nnnotes.music-data/1", + "description": "Every live song and chart of one master data version: song metadata in every language, chart facts and the deck model's chart statistics, with --full also the deck model's input (every chart's runtime notes and a subset of the master data tables as columns and rows). Written by `nnnotes music-data`; described in docs/music-data.md. Within the arrays of a deck input chart's `notes` (and of its `fevers`) the i-th elements belong to the same note (range): the arrays have equal lengths, which a schema cannot state. Readers ignore keys they do not know.", + "type": "object", + "required": [ + "format", + "provenance", + "languages", + "bands", + "characters", + "tags", + "categories", + "deck", + "songs" + ], + "properties": { + "format": { + "const": "nnnotes.music-data/1" + }, + "provenance": { + "type": "object", + "required": [ + "region", + "client", + "catalog", + "master", + "exporter", + "deck" + ], + "properties": { + "region": { + "type": "string", + "description": "the region whose master data this is (a configured region name), or `embedded` for the master data the APK ships" + }, + "client": { + "type": "object", + "required": [ + "versionName", + "versionCode" + ], + "properties": { + "versionName": { + "type": [ + "string", + "null" + ], + "description": "versionName of the APK (null without one)" + }, + "versionCode": { + "type": [ + "integer", + "null" + ], + "description": "versionCode of the APK (null without one)" + } + } + }, + "catalog": { + "type": "object", + "required": [ + "resourceVersion", + "sha256" + ], + "properties": { + "resourceVersion": { + "type": [ + "string", + "null" + ], + "description": "the resource version recorded for the catalog in the catalog store, null when none is recorded" + }, + "sha256": { + "$ref": "#/$defs/sha256", + "description": "SHA-256 of the remote catalog file the charts were read with" + } + } + }, + "master": { + "type": "object", + "required": [ + "source", + "version", + "tables" + ], + "properties": { + "source": { + "enum": [ + "api", + "embedded" + ], + "description": "api: the region's files (downloaded, or with --decoded-master decoded elsewhere with their manifest); embedded: the files the APK ships" + }, + "version": { + "type": [ + "string", + "null" + ], + "description": "`version` of the master data manifest" + }, + "tables": { + "type": "object", + "description": "per table read, the SHA-256 of its file as served (before decoding; with --decoded-master as the manifest lists it)", + "additionalProperties": { + "type": "object", + "required": [ + "sha256" + ], + "properties": { + "sha256": { + "$ref": "#/$defs/sha256" + } + } + } + } + } + }, + "exporter": { + "type": "object", + "required": [ + "name", + "version", + "chartFormat" + ], + "properties": { + "name": { + "const": "nnnotes" + }, + "version": { + "type": "string" + }, + "chartFormat": { + "type": "string", + "description": "format of the chart converter the notes come from (nnnotes.live-score/1)" + } + } + }, + "deck": { + "type": [ + "object", + "null" + ], + "description": "the deck model the statistics come from; null without it (--no-deck)", + "required": [ + "name", + "version", + "source", + "commit", + "format" + ], + "properties": { + "name": { + "const": "ournotes-deck" + }, + "version": { + "type": "string" + }, + "source": { + "type": "string", + "description": "the repository" + }, + "commit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "format": { + "type": "string", + "description": "the statistics format, ournotes-deck.chart-stats/2" + } + } + } + } + }, + "languages": { + "type": "array", + "items": { + "type": "string" + } + }, + "bands": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "integer" + }, + "name": { + "$ref": "#/$defs/text" + }, + "mainColor": { + "type": [ + "string", + "null" + ] + }, + "subColor": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "characters": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "integer" + }, + "name": { + "$ref": "#/$defs/text" + }, + "bandId": { + "type": [ + "integer", + "null" + ] + }, + "shortName": { + "$ref": "#/$defs/text" + }, + "mainColor": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "tags": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "integer" + }, + "name": { + "$ref": "#/$defs/text" + } + } + } + }, + "categories": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "integer" + }, + "name": { + "$ref": "#/$defs/text" + }, + "musicCategories": { + "$ref": "#/$defs/ints" + } + } + } + }, + "deck": { + "type": [ + "object", + "null" + ], + "required": [ + "model", + "kinds" + ], + "description": "the deck model's measurement and score-up kinds; null with --no-deck", + "properties": { + "model": { + "type": "object" + }, + "kinds": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "effectType" + ], + "properties": { + "id": { + "type": "integer" + }, + "effectType": { + "type": "integer" + } + } + } + } + } + }, + "songs": { + "type": "array", + "items": { + "$ref": "#/$defs/song" + } + }, + "master": { + "type": "object", + "description": "--full: the deck input's tables by name; a table not present reads as empty", + "additionalProperties": { + "$ref": "#/$defs/table" + } + }, + "charts": { + "type": "array", + "description": "--full: one chart per MasterLiveMusicScore row, sorted by scoreId", + "items": { + "$ref": "#/$defs/chart" + } + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "ints": { + "type": "array", + "items": { + "type": "integer" + } + }, + "table": { + "type": "object", + "required": [ + "columns", + "rows" + ], + "properties": { + "columns": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "rows": { + "type": "array", + "items": { + "type": "array" + }, + "description": "one array per row, one value per column in the order of `columns`" + } + } + }, + "chart": { + "type": "object", + "required": [ + "scoreId", + "asset", + "notes", + "skillEvents", + "fevers" + ], + "properties": { + "scoreId": { + "type": "integer", + "description": "MasterLiveMusicScore._id" + }, + "asset": { + "type": "object", + "required": [ + "key", + "sha256" + ], + "properties": { + "key": { + "type": "string", + "description": "Live/MusicScore/<_musicScoreTextFileName>" + }, + "sha256": { + "$ref": "#/$defs/sha256", + "description": "SHA-256 of the chart TextAsset's bytes as shipped" + } + } + }, + "notes": { + "type": "object", + "required": [ + "id", + "op", + "judgementType", + "timeMs" + ], + "properties": { + "id": { + "$ref": "#/$defs/ints", + "description": "note id, unique within the chart" + }, + "op": { + "$ref": "#/$defs/ints", + "description": "NoteOperateType" + }, + "judgementType": { + "$ref": "#/$defs/ints", + "description": "NoteJudgementType" + }, + "timeMs": { + "$ref": "#/$defs/ints", + "description": "note time in chart milliseconds" + } + } + }, + "skillEvents": { + "type": "object", + "required": [ + "timeMs" + ], + "properties": { + "timeMs": { + "$ref": "#/$defs/ints", + "description": "skill event times in chart order (the index is the event index)" + } + } + }, + "fevers": { + "type": "object", + "required": [ + "startMs", + "endMs" + ], + "properties": { + "startMs": { + "$ref": "#/$defs/ints", + "description": "range starts, sorted (the index is the range index)" + }, + "endMs": { + "$ref": "#/$defs/ints" + } + } + } + } + }, + "text": { + "type": [ + "object", + "null" + ], + "description": "one string per language of `languages`; null: no text id", + "required": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ], + "additionalProperties": { + "type": [ + "string", + "null" + ] + } + }, + "deckChart": { + "type": [ + "object", + "null" + ], + "description": "the chart's deck statistics (docs/music-data.md, Deck statistics); null with --no-deck", + "required": [ + "convertedNoteCount", + "skip", + "events", + "positions", + "ranges", + "justNotes", + "seeds", + "unplayable" + ], + "properties": { + "convertedNoteCount": { + "type": "integer" + }, + "skip": { + "type": "number" + }, + "events": { + "type": "array", + "items": { + "type": "array", + "prefixItems": [ + { + "type": "integer" + }, + { + "type": "integer" + } + ], + "minItems": 2, + "maxItems": 2 + }, + "description": "[performance position, time ms] per skill event, in chart order" + }, + "positions": { + "type": "integer", + "minimum": 0 + }, + "ranges": { + "type": "array", + "items": { + "type": "object", + "required": [ + "index", + "mission", + "startMs", + "endMs", + "rankBonusPercent" + ], + "properties": { + "index": { + "type": "integer" + }, + "mission": { + "type": "integer" + }, + "startMs": { + "type": "integer" + }, + "endMs": { + "type": "integer" + }, + "rankBonusPercent": { + "type": "integer", + "description": "the rank 1 bonus percentage (rankBonusPercents[0])" + }, + "rankBonusPercents": { + "type": "array", + "items": { + "type": "integer" + }, + "minItems": 5, + "maxItems": 5, + "description": "the rank bonus percentages of ranks 1..5 of the song's mission pattern" + } + } + } + }, + "justNotes": { + "type": "integer" + }, + "seeds": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seed", + "score", + "ranges", + "weights", + "check" + ], + "properties": { + "seed": { + "type": "integer" + }, + "score": { + "type": "integer", + "description": "the exact no-skill score at model.power" + }, + "ranges": { + "type": "array", + "items": { + "type": "object", + "required": [ + "rangeScore", + "rankBonus", + "maxCombo", + "justCount", + "lotResults" + ], + "properties": { + "rangeScore": { + "type": "integer" + }, + "rankBonus": { + "type": "integer" + }, + "maxCombo": { + "type": "integer" + }, + "justCount": { + "type": "integer" + }, + "lotResults": { + "type": "array", + "items": { + "type": "integer" + }, + "minItems": 4, + "maxItems": 4 + }, + "rangeScorePerfect": { + "type": "integer", + "description": "the range score on the Perfect play (every Just judged Perfect)" + } + } + } + }, + "weights": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + }, + "description": "weights[kind][position]: score per unit of deck power and of factor" + }, + "check": { + "type": "object", + "required": [ + "deck", + "exact", + "predicted", + "bound" + ], + "properties": { + "deck": { + "type": "array", + "items": { + "type": [ + "array", + "null" + ] + } + }, + "exact": { + "type": "integer" + }, + "predicted": { + "type": "number" + }, + "bound": { + "type": "number" + } + } + }, + "scorePerfect": { + "type": "integer", + "description": "the exact no-skill score at model.power on the Perfect play (every Just judged Perfect), its rank 1 bonuses included" + }, + "rangeWeights": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "array", + "null" + ], + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + "description": "rangeWeights[kind][position][range]: range score per unit of deck power and of factor; null when the ranks do not follow linearly on the chart, a kind null when its conditions read the confirmed rank" + }, + "rankCheck": { + "type": [ + "object", + "null" + ], + "required": [ + "ranks", + "exact", + "predicted", + "bound" + ], + "description": "the check deck played at random ranks; null without ranges or range weights", + "properties": { + "ranks": { + "type": "array", + "items": { + "type": "integer", + "minimum": 1, + "maximum": 5 + } + }, + "exact": { + "type": "integer" + }, + "predicted": { + "type": "number" + }, + "bound": { + "type": "number" + } + } + } + } + } + }, + "offSeeds": { + "type": "array", + "description": "the measurements with Gekisou off (a solo live), one seed", + "items": { + "type": "object", + "required": [ + "seed", + "score", + "weights", + "check" + ], + "properties": { + "seed": { + "type": "integer" + }, + "score": { + "type": "integer", + "description": "the exact no-skill score at model.power, Gekisou off" + }, + "weights": { + "type": "array", + "items": { + "type": [ + "array", + "null" + ], + "items": { + "type": "number" + } + }, + "description": "weights[kind][position], Gekisou off; a kind null when its conditions read the Gekisou state" + }, + "check": { + "type": "object", + "required": [ + "deck", + "exact", + "predicted", + "bound" + ], + "properties": { + "deck": { + "type": "array", + "items": { + "type": [ + "array", + "null" + ] + } + }, + "exact": { + "type": "integer" + }, + "predicted": { + "type": "number" + }, + "bound": { + "type": "number" + } + } + } + } + } + }, + "unplayable": { + "type": [ + "string", + "null" + ], + "description": "why the game cannot play the chart with Gekisou (seeds is then empty; offSeeds is not)" + } + } + }, + "songChart": { + "type": "object", + "required": [ + "difficulty", + "scoreId", + "level", + "displayLevel", + "fullComboCount", + "asset", + "notes", + "bpm", + "firstNoteMs", + "lastJudgedNoteMs", + "lastNoteMs", + "musicLengthMs", + "skillEventsMs", + "fevers", + "deck" + ], + "properties": { + "difficulty": { + "enum": [ + "easy", + "normal", + "hard", + "expert" + ] + }, + "scoreId": { + "type": "integer" + }, + "level": { + "type": "integer" + }, + "displayLevel": { + "type": [ + "number", + "null" + ] + }, + "fullComboCount": { + "type": "integer" + }, + "asset": { + "type": "object", + "required": [ + "key", + "sha256" + ], + "properties": { + "key": { + "type": "string", + "description": "Live/MusicScore/<_musicScoreTextFileName>" + }, + "sha256": { + "$ref": "#/$defs/sha256", + "description": "SHA-256 of the chart TextAsset's bytes as shipped" + } + } + }, + "notes": { + "type": "object", + "required": [ + "judged", + "total", + "byOperateType" + ], + "properties": { + "judged": { + "type": "integer" + }, + "total": { + "type": "integer" + }, + "byOperateType": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + } + } + }, + "bpm": { + "type": "object", + "required": [ + "main", + "min", + "max", + "changes" + ], + "properties": { + "main": { + "type": "number" + }, + "min": { + "type": "number" + }, + "max": { + "type": "number" + }, + "changes": { + "type": "array", + "items": { + "type": "object", + "required": [ + "timeMs", + "bpm" + ], + "properties": { + "timeMs": { + "type": "integer" + }, + "bpm": { + "type": "number" + } + } + } + } + } + }, + "firstNoteMs": { + "type": "integer" + }, + "lastJudgedNoteMs": { + "type": "integer" + }, + "lastNoteMs": { + "type": "integer" + }, + "musicLengthMs": { + "type": "integer" + }, + "skillEventsMs": { + "$ref": "#/$defs/ints" + }, + "fevers": { + "type": "array", + "items": { + "$ref": "#/$defs/ints", + "minItems": 2, + "maxItems": 2 + } + }, + "deck": { + "$ref": "#/$defs/deckChart" + } + } + }, + "song": { + "type": "object", + "required": [ + "id", + "sortOrder", + "startAt", + "defaultUnlock", + "title", + "ruby", + "phonetic", + "bandIds", + "bandName", + "vocalCharacterIds", + "lyricist", + "composer", + "arranger", + "musicType", + "musicCategories", + "bestMusicTagIds", + "jacket", + "gekisouMissions", + "bgm", + "scoreRanks", + "charts", + "master" + ], + "properties": { + "id": { + "type": "integer" + }, + "sortOrder": { + "type": [ + "integer", + "null" + ] + }, + "startAt": { + "type": [ + "string", + "null" + ] + }, + "defaultUnlock": { + "type": [ + "boolean", + "null" + ] + }, + "title": { + "$ref": "#/$defs/text" + }, + "ruby": { + "$ref": "#/$defs/text" + }, + "phonetic": { + "$ref": "#/$defs/text" + }, + "bandName": { + "$ref": "#/$defs/text" + }, + "lyricist": { + "$ref": "#/$defs/text" + }, + "composer": { + "$ref": "#/$defs/text" + }, + "arranger": { + "$ref": "#/$defs/text" + }, + "bandIds": { + "$ref": "#/$defs/ints" + }, + "vocalCharacterIds": { + "$ref": "#/$defs/ints" + }, + "musicType": { + "type": [ + "integer", + "null" + ] + }, + "musicCategories": { + "$ref": "#/$defs/ints" + }, + "bestMusicTagIds": { + "$ref": "#/$defs/ints" + }, + "jacket": { + "type": [ + "string", + "null" + ] + }, + "gekisouMissions": { + "type": "array", + "items": { + "type": [ + "integer", + "null" + ] + }, + "minItems": 3, + "maxItems": 3 + }, + "bgm": { + "type": "object", + "required": [ + "soundId", + "cueSheet", + "cue", + "length" + ], + "properties": { + "soundId": { + "type": "integer" + }, + "cueSheet": { + "type": "string" + }, + "cue": { + "type": "string" + }, + "length": { + "type": [ + "object", + "null" + ], + "required": [ + "lengthMs", + "samples", + "sampleRate", + "durationMs" + ], + "properties": { + "lengthMs": { + "type": [ + "integer", + "null" + ] + }, + "samples": { + "type": [ + "integer", + "null" + ] + }, + "sampleRate": { + "type": [ + "integer", + "null" + ] + }, + "durationMs": { + "type": [ + "integer", + "null" + ] + } + } + } + } + }, + "scoreRanks": { + "type": "array", + "items": { + "type": "object", + "required": [ + "rank", + "requiredScore", + "battleRequiredScore" + ], + "properties": { + "rank": { + "enum": [ + "E", + "D", + "C", + "B", + "A", + "S", + "SS" + ] + }, + "requiredScore": { + "type": [ + "integer", + "null" + ] + }, + "battleRequiredScore": { + "type": [ + "integer", + "null" + ] + } + } + } + }, + "charts": { + "type": "array", + "items": { + "$ref": "#/$defs/songChart" + } + }, + "master": { + "type": "object", + "required": [ + "MasterLiveMusic", + "MasterLiveScoreRank" + ], + "properties": { + "MasterLiveMusic": { + "type": "object" + }, + "MasterLiveScoreRank": { + "type": "array", + "items": { + "type": "object" + } + } + } + } + } + } + } +} diff --git a/pyproject.toml b/pyproject.toml index 3baadeb..ef4dedc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] -requires = ["setuptools>=77"] -build-backend = "setuptools.build_meta" +requires = ["maturin>=1.9,<2"] +build-backend = "maturin" [project] name = "nnnotes" @@ -39,14 +39,14 @@ Homepage = "https://github.com/MetaSekaiLab/nnnotes" Source = "https://github.com/MetaSekaiLab/nnnotes" Issues = "https://github.com/MetaSekaiLab/nnnotes/issues" -[tool.setuptools.dynamic] -version = { attr = "nnnotes.__version__" } - -[tool.setuptools.packages.find] -where = ["src"] - -[tool.setuptools.package-data] -nnnotes = ["typetrees/*.json", "viewrules.json", "nnnotes.example.toml"] +[tool.maturin] +# the pure Python package src/nnnotes with the extension module nnnotes._deck (rust/: ournotes-deck, pinned by +# commit); the version is rust/Cargo.toml's, kept equal to nnnotes.__version__ (tests/test_musicdata.py) +python-source = "src" +module-name = "nnnotes._deck" +manifest-path = "rust/Cargo.toml" +profile = "release" +locked = true [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 0000000..9a76560 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,215 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "nnnotes-deck" +version = "0.1.2" +dependencies = [ + "ournotes-deck", + "pyo3", + "serde_json", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "ournotes-deck" +version = "0.0.1" +source = "git+https://github.com/empty-sekai/ournotes-deck?rev=d4abb52ff75059a13e99a88fd6812ec1bdaa5dd2#d4abb52ff75059a13e99a88fd6812ec1bdaa5dd2" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 0000000..92a1db1 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "nnnotes-deck" +# the nnnotes version (src/nnnotes/__init__.py); the wheel takes its version from here +version = "0.1.2" +edition = "2024" +rust-version = "1.85" +description = "nnnotes._deck: the ournotes-deck chart statistics as a Python extension module" +license = "MIT" +publish = false + +[lib] +name = "_deck" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = { version = "0.29", features = ["abi3-py311"] } +# pinned by commit; .github/workflows/deck.yml opens a pull request when ournotes-deck moves +ournotes-deck = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "d4abb52ff75059a13e99a88fd6812ec1bdaa5dd2" } +serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } + +[profile.release] +lto = "thin" +codegen-units = 1 diff --git a/rust/build.rs b/rust/build.rs new file mode 100644 index 0000000..580abea --- /dev/null +++ b/rust/build.rs @@ -0,0 +1,33 @@ +//! Records the ournotes-deck package the module is built with (version and git commit, from Cargo.lock) as the +//! environment variables DECK_VERSION, DECK_SOURCE and DECK_COMMIT of the crate. + +use std::path::Path; + +fn main() { + let dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR"); + let lock = Path::new(&dir).join("Cargo.lock"); + println!("cargo:rerun-if-changed={}", lock.display()); + let text = std::fs::read_to_string(&lock).expect("rust/Cargo.lock (the deck commit is read from it)"); + let (mut version, mut source) = (None, None); + for block in text.split("[[package]]") { + let field = |k: &str| { + block.lines().find_map(|l| { + let v = l.trim().strip_prefix(k)?.trim_start().strip_prefix('=')?.trim(); + Some(v.trim_matches('"').to_string()) + }) + }; + if field("name").as_deref() == Some("ournotes-deck") { + version = field("version"); + source = field("source"); + } + } + let version = version.expect("Cargo.lock has no ournotes-deck package"); + let source = source.expect("ournotes-deck has no source in Cargo.lock"); + // git+https://github.com/empty-sekai/ournotes-deck?rev=# + let (url, commit) = source.split_once('#').expect("ournotes-deck is not a git dependency"); + let url = url.strip_prefix("git+").unwrap_or(url); + let url = url.split_once('?').map_or(url, |(u, _)| u); + println!("cargo:rustc-env=DECK_VERSION={version}"); + println!("cargo:rustc-env=DECK_SOURCE={url}"); + println!("cargo:rustc-env=DECK_COMMIT={commit}"); +} diff --git a/rust/rustfmt.toml b/rust/rustfmt.toml new file mode 100644 index 0000000..c775577 --- /dev/null +++ b/rust/rustfmt.toml @@ -0,0 +1,2 @@ +max_width = 120 +use_small_heuristics = "Max" diff --git a/rust/src/lib.rs b/rust/src/lib.rs new file mode 100644 index 0000000..476e951 --- /dev/null +++ b/rust/src/lib.rs @@ -0,0 +1,98 @@ +//! `nnnotes._deck`: the chart statistics of ournotes-deck (`ournotes-deck.chart-stats/2`) on a deck data document +//! held in memory, the charts measured in parallel. + +use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use ournotes_deck::chartstats::{self, ChartStats, GEKISOU_SEEDS}; +use ournotes_deck::data::DeckData; +use ournotes_deck::error::Error; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +fn value_error(e: impl std::fmt::Display) -> PyErr { + PyValueError::new_err(e.to_string()) +} + +/// The statistics of every chart, in chart order, on `workers` threads. +fn measure(data: &DeckData, seeds: usize, workers: usize) -> Result, Error> { + let kinds = chartstats::kinds(&data.master); + let n = data.charts.len(); + let next = AtomicUsize::new(0); + let out: Mutex>>> = Mutex::new((0..n).map(|_| None).collect()); + std::thread::scope(|s| { + for _ in 0..workers.clamp(1, n.max(1)) { + s.spawn(|| { + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + if i >= n { + break; + } + let c = &data.charts[i]; + // as chartstats::document names a chart's domain errors + let r = chartstats::chart_stats(&data.master, c, &kinds, seeds).map_err(|e| match e { + Error::Domain(m) => Error::Domain(format!("chart {}: {m}", c.score_id)), + e => e, + }); + let failed = r.is_err(); + out.lock().expect("results")[i] = Some(r); + if failed { + next.store(n, Ordering::Relaxed); + } + } + }); + } + }); + // charts are taken in order and a taken chart always completes, so every chart before the first failure has + // its result: the error returned is the first in chart order, as the command's + let mut stats = Vec::with_capacity(n); + for r in out.into_inner().expect("results") { + stats.push(r.expect("a chart before the first failure has run")?); + } + Ok(stats) +} + +/// chart_stats(data, seeds=None, workers=None) -> str +/// +/// The `ournotes-deck.chart-stats/2` document (JSON text) of a deck data document (`nnnotes.deck-data/1` JSON text), +/// as `ournotes-deck chart-stats` writes it. `seeds`: the size of the seed set of charts with a luck range (default +/// 8); `workers`: threads measuring charts (default: the available parallelism). Raises ValueError for data the +/// deck model cannot read or a chart whose check deck fails. +#[pyfunction] +#[pyo3(signature = (data, seeds=None, workers=None))] +fn chart_stats(py: Python<'_>, data: &str, seeds: Option, workers: Option) -> PyResult { + let seeds = seeds.unwrap_or(GEKISOU_SEEDS); + let workers = workers.unwrap_or_else(|| std::thread::available_parallelism().map_or(1, |n| n.get())); + py.detach(|| { + let mut data = DeckData::from_json(data).map_err(value_error)?; + let charts = std::mem::take(&mut data.charts); + // the document of no chart: the format, source, model and kinds, exactly as the command writes them + let mut doc = chartstats::document(&data, Some(seeds)).map_err(value_error)?; + data.charts = charts; + let stats = measure(&data, seeds, workers).map_err(value_error)?; + doc["charts"] = serde_json::to_value(stats).map_err(value_error)?; + serde_json::to_string(&doc).map_err(value_error) + }) +} + +/// info() -> dict: the ournotes-deck package of this module (name, version, source, commit) and the formats it reads +/// and writes. +#[pyfunction] +fn info(py: Python<'_>) -> PyResult> { + let d = PyDict::new(py); + d.set_item("name", "ournotes-deck")?; + d.set_item("version", env!("DECK_VERSION"))?; + d.set_item("source", env!("DECK_SOURCE"))?; + d.set_item("commit", env!("DECK_COMMIT"))?; + d.set_item("dataFormat", ournotes_deck::data::FORMAT)?; + d.set_item("format", chartstats::FORMAT)?; + Ok(d) +} + +#[pymodule] +fn _deck(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(chart_stats, m)?)?; + m.add_function(wrap_pyfunction!(info, m)?)?; + Ok(()) +} diff --git a/src/nnnotes/__init__.py b/src/nnnotes/__init__.py index 60485bb..9bf2a64 100644 --- a/src/nnnotes/__init__.py +++ b/src/nnnotes/__init__.py @@ -4,4 +4,4 @@ models, spots, shaders, CRI audio and live charts as JSON and common file formats. Game files, keys and server addresses are supplied by the user through the configuration; nothing of the game is included. """ -__version__ = "0.1.1" +__version__ = "0.1.2" diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index 4056856..b14ead1 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -36,7 +36,8 @@ [--story 10462 [--story ...] | --all-stories] [--story-languages en,ja] [--font en=] [--font emoji=] [--region [--region ...] | --all-regions] - nnnotes deck-data --master-files | --apk-master -o out/deck-data.json[.gz] + nnnotes music-data --master-files | --apk-master | --decoded-master [--full] [--no-deck] + [--no-bgm] [--jackets DIR] -o out/music-data.json[.gz] nnnotes export -o out/assets [--select group: | key: | bundle: ...] [--layout original,cas] nnnotes plan [--select ...] [--json] [--check] [--emit-tasks ] nnnotes run-stage [...] @@ -636,22 +637,29 @@ def cmd_web(args, cfg): sys.exit(1) -def cmd_deck_data(args, cfg): - from . import deckdata +def cmd_music_data(args, cfg): + from . import deckdata, musicdata if args.apk_master: cfg.require_path("paths", "apk") # the master data files ship in the APK apk = _existing(cfg, "paths", "apk") try: + deck = None if args.no_deck else musicdata.Deck(seeds=args.seeds, workers=args.workers) if args.apk_master: src, region = deckdata.apk_master(apk), deckdata.EMBEDDED + elif args.decoded_master: # decoded elsewhere: no master key + src, region = deckdata.decoded_master(master_dir(cfg)), cfg.region() else: src, region = deckdata.master_files(Path(args.master_files)), cfg.region() - key = master_key(cfg) + key = None if src.decoded else master_key(cfg) cat = open_catalog(cfg) - r = deckdata.export(Path(args.out), src, key, deckdata.catalog_fetch(cat), region=region, - client=deckdata.apk_client(apk) if apk is not None else {}, - catalog=deckdata.catalog_info(cat, cli_assets.store_root(args, cfg))) - except deckdata.DeckDataError as e: + r = musicdata.export(Path(args.out), src, key, deckdata.catalog_fetch(cat), + None if args.no_bgm else musicdata.catalog_bgm(cat), region=region, + client=deckdata.apk_client(apk) if apk is not None else {}, + catalog=deckdata.catalog_info(cat, cli_assets.store_root(args, cfg)), + deck=deck, full=args.full, + jacket=musicdata.catalog_jacket(cat) if args.jackets else None, + jackets_dir=args.jackets) + except (deckdata.DeckDataError, musicdata.MusicDataError) as e: sys.exit(f"nnnotes: {e}") _print_json(r) @@ -920,15 +928,31 @@ def target(m, what): _live_option_arg(c) c.set_defaults(func=cmd_web, usage=c.error) - c = sub.add_parser("deck-data", help="every live chart and the master data tables deck-building tools read -> " - "one JSON file") + c = sub.add_parser("music-data", help="every live song and chart: metadata in every language, chart facts and " + "the deck model's chart statistics -> one JSON file") g = c.add_mutually_exclusive_group(required=True) g.add_argument("--master-files", metavar="DIR", help="master data files as served: MasterManifest.json and the .bin files it lists " "(`master download`)") g.add_argument("--apk-master", action="store_true", help="the master data files of base.apk ([paths] apk)") + g.add_argument("--decoded-master", action="store_true", + help="decoded master data ([paths] master or --master) with the MasterManifest.json of the files " + "it was decoded from; no master key") + c.add_argument("--full", action="store_true", + help="also write the deck model's input: every chart's runtime notes and the master data tables " + "about cards, skills, bonuses, scores and events") + c.add_argument("--no-deck", action="store_true", + help="do not run the deck model (every chart's deck is null)") + c.add_argument("--seeds", type=int, default=8, metavar="N", + help="seeds measured on a chart with a luck range (default 8)") + c.add_argument("--workers", type=int, metavar="N", + help="threads measuring charts (default: every processor)") + c.add_argument("--no-bgm", action="store_true", + help="do not read the BGM cue sheets (every song's bgm.length is null)") + c.add_argument("--jackets", metavar="DIR", + help="also write every song's jacket as DIR/.webp (at most 320 px on the longer side)") _out(c, "output file (.json, or .json.gz for gzip)") - c.set_defaults(func=cmd_deck_data, usage=c.error) + c.set_defaults(func=cmd_music_data, usage=c.error) cli_assets.register(sub, argparse.Namespace(open_catalog=open_catalog, print_json=_print_json)) voices.register(sub, argparse.Namespace(open_catalog=open_catalog, master_dir=master_dir)) diff --git a/src/nnnotes/deckdata.py b/src/nnnotes/deckdata.py index 447364b..4cd6905 100644 --- a/src/nnnotes/deckdata.py +++ b/src/nnnotes/deckdata.py @@ -1,30 +1,34 @@ -"""Deck data: one JSON file with every live chart as the client builds it at runtime and the master data tables that -deck-building tools read, for one master data version (format `nnnotes.deck-data/1`, docs/deck-data.md). +"""Deck input: every live chart as the client builds it at runtime and the master data tables the deck model +(ournotes-deck) reads, for one master data version. `nnnotes music-data` hands it to the deck model in memory +(DECK_FORMAT, the reader's format) and writes it into the music data file with `--full` (docs/music-data.md). Master data is decoded from the files as served: a directory with `MasterManifest.json` and the `.bin` files it lists (`nnnotes master download`), or the same layout inside the APK (`assets/Master/`). Each file is checked against -the manifest's SHA-256 and decoded with master.decode. Charts are the TextAssets `Live/MusicScore/` of -every MasterLiveMusicScore row, read from the catalog and converted by score.runtime_score; notes are listed in the -order the client enumerates them. - -The output is canonical: minified UTF-8 with one trailing LF, keys in a fixed order, charts sorted by score id, -numbers the master data writes with a fraction or exponent as the shortest decimal that reads back as the same -binary32 value (infinity as `1e999` / `-1e999`; a NaN is an error). The same inputs give the same bytes; a `.gz` -output is gzip with no file name and a zero modification time. The file is written only when every table and chart -was read; any missing or unreadable input is a DeckDataError naming it. +the manifest's SHA-256 and decoded with master.decode. Master data decoded elsewhere is read as it is: a directory of +decoded tables (`
.json`) with the `MasterManifest.json` of the files they were decoded from, whose version and +SHA-256 are taken as the manifest lists them (decoded_master). Charts are the TextAssets +`Live/MusicScore/` of every MasterLiveMusicScore row, read from the catalog and converted by +score.runtime_score; notes are listed in the order the client enumerates them. + +The encoding is canonical (encode): minified UTF-8 with one trailing LF, keys in a fixed order, charts sorted by +score id, numbers the master data writes with a fraction or exponent as the shortest decimal that reads back as the +same binary32 value (infinity as `1e999` / `-1e999`; a NaN is an error). The same inputs give the same bytes; a `.gz` +output is gzip with no file name and a zero modification time. Any missing or unreadable input is a DeckDataError +naming it. """ from __future__ import annotations import gzip import hashlib import json +import re import zipfile from collections import Counter from dataclasses import dataclass from pathlib import Path from typing import Callable -FORMAT = "nnnotes.deck-data/1" +DECK_FORMAT = "nnnotes.deck-data/1" # the deck model's input format (ournotes-deck data::FORMAT) CHART_FORMAT = "nnnotes.live-score/1" # score.convert's format: the converter the notes come from CHART_PREFIX = "Live/MusicScore/" # + MasterLiveMusicScore._musicScoreTextFileName MANIFEST = "MasterManifest.json" @@ -106,6 +110,12 @@ ("MasterLiveScoreRank", "_id _group _liveScoreRank _requiredScore _battleLiveRequiredScore"), ("MasterEvent", _WHOLE), ("MasterEventEffect", _WHOLE), + ("MasterEventAchievementReward", "_id _eventId _eventPoint _rewardIds"), + ("MasterEventAchievementLoopReward", "_id _eventId _loopStartEventPoint _loopEventPoint _rewardIds"), + ("MasterLiveEventReward", "_id _group _eventGroup _scoreRank _resourceType _resourceId _resourceCount " + "_probability"), + ("MasterChallengeLiveEventReward", "_id _group _eventGroup _scoreRank _resourceType _resourceId " + "_resourceCount _probability"), ("MasterLiveEventPoint", _WHOLE), ("MasterChallengeLiveEventPoint", _WHOLE), ("MasterLiveChallengePoint", _WHOLE), @@ -123,12 +133,13 @@ class DeckDataError(ValueError): @dataclass(frozen=True) class MasterSource: """Master data files as served: `version` and the SHA-256 of each file name from the manifest, and a reader of - files by name.""" + files by name. `decoded`: the reader has the decoded tables instead (`
.json` for `
.bin`).""" source: str # API or EMBEDDED where: str # the directory or APK, for messages version: str | None hashes: dict[str, str] # file name -> sha256 (lowercase hex; "" when not listed) read: Callable[[list[str]], dict[str, bytes]] # file names -> {name: bytes}; a missing file raises KeyError + decoded: bool = False def _manifest(raw: bytes, where: str) -> tuple[str | None, dict[str, str]]: @@ -142,12 +153,10 @@ def _manifest(raw: bytes, where: str) -> tuple[str | None, dict[str, str]]: return (str(version) if version is not None else None), hashes -def master_files(directory) -> MasterSource: - """The master data files of a directory written by `nnnotes master download` (MasterManifest.json + .bin).""" - d = Path(directory) +def _directory(d: Path, what: str, decoded: bool = False) -> MasterSource: m = d / MANIFEST if not m.is_file(): - raise DeckDataError(f"{d}: no {MANIFEST} (a directory written by `nnnotes master download`)") + raise DeckDataError(f"{d}: no {MANIFEST} ({what})") version, hashes = _manifest(m.read_bytes(), str(d)) def read(names): @@ -158,7 +167,21 @@ def read(names): except FileNotFoundError: raise KeyError(n) from None return out - return MasterSource(API, str(d), version, hashes, read) + return MasterSource(API, str(d), version, hashes, read, decoded) + + +def master_files(directory) -> MasterSource: + """The master data files of a directory written by `nnnotes master download` (MasterManifest.json + .bin).""" + return _directory(Path(directory), "a directory written by `nnnotes master download`") + + +def decoded_master(directory) -> MasterSource: + """Decoded master data: a directory of decoded tables, `
.json` with the `_allData` rows (as `nnnotes master + decode` writes them), and the `MasterManifest.json` of the files they were decoded from (a master data snapshot + published with its manifest). The version and each file's SHA-256 are the manifest's: the decoded tables cannot + be checked against the files as served.""" + return _directory(Path(directory), "decoded master data needs the manifest of the files it was decoded from", + decoded=True) def apk_master(apk) -> MasterSource: @@ -183,33 +206,39 @@ def read(names): return MasterSource(EMBEDDED, f"{apk} {APK_MASTER}", version, hashes, read) -def read_master(src: MasterSource, key) -> tuple[dict[str, list[dict]], dict[str, str]]: - """The rows (`_allData`) of every table of TABLES and the SHA-256 of each file as served. `key`: a - master.MasterKey.""" +def read_master(src: MasterSource, key, tables=None) -> tuple[dict[str, list[dict]], dict[str, str]]: + """The rows (`_allData`) of every table of `tables` (default: TABLES) and the SHA-256 of each file as served. + `key`: a master.MasterKey (unused for decoded master data: None).""" from . import master - names = {t: f"{t}.bin" for t, _ in TABLES} + names = {t: f"{t}.bin" for t in (tables if tables is not None else (t for t, _ in TABLES))} unlisted = [t for t, n in names.items() if n not in src.hashes] if unlisted: raise DeckDataError(f"master data {src.where}: {MANIFEST} lists no {', '.join(unlisted)}") + files = {t: f"{t}.json" if src.decoded else n for t, n in names.items()} try: - data = src.read(list(names.values())) + data = src.read(list(files.values())) except KeyError as e: raise DeckDataError(f"master data {src.where}: no file {e.args[0]}") from None - rk = master.round_keys(key.key) + rk = None if src.decoded else master.round_keys(key.key) tables, shas = {}, {} for t, n in names.items(): - raw = data[n] - sha = hashlib.sha256(raw).hexdigest() - if src.hashes[n] and sha != src.hashes[n]: - raise DeckDataError(f"master data {src.where}: {n}: sha256 differs from the manifest") + raw, f = data[files[t]], files[t] + if src.decoded: # the manifest's SHA-256 of the file as served + sha = src.hashes[n] + if not re.fullmatch(r"[0-9a-f]{64}", sha): + raise DeckDataError(f"master data {src.where}: {MANIFEST} lists no SHA-256 for {n}") + else: + sha = hashlib.sha256(raw).hexdigest() + if src.hashes[n] and sha != src.hashes[n]: + raise DeckDataError(f"master data {src.where}: {n}: sha256 differs from the manifest") try: - doc = json.loads(master.decode(raw, key, rk).decode("utf-8")) + doc = json.loads((raw if src.decoded else master.decode(raw, key, rk)).decode("utf-8")) except Exception as e: # padding, gzip, UTF-8 or JSON: the file cannot be read - raise DeckDataError(f"master data {src.where}: {n} cannot be decoded ({type(e).__name__}: " - f"{str(e)[:120]})") from None + raise DeckDataError(f"master data {src.where}: {f} cannot be {'read' if src.decoded else 'decoded'} " + f"({type(e).__name__}: {str(e)[:120]})") from None rows = doc.get("_allData") if isinstance(doc, dict) else None if not isinstance(rows, list) or not all(isinstance(r, dict) for r in rows): - raise DeckDataError(f"master data {src.where}: {n} has no `_allData` rows") + raise DeckDataError(f"master data {src.where}: {f} has no `_allData` rows") tables[t], shas[t] = rows, sha return tables, shas @@ -371,25 +400,11 @@ def resource_version(store_root, remote_sha: str) -> str | None: return None -# ---------------------------------------------------------------- document -def build(tables: dict[str, list[dict]], table_sha: dict[str, str], fetch: Callable[[str], bytes], *, region: str, - client: dict, catalog: dict, master_source: str, master_version: str | None) -> dict: - """The deck data document. `catalog`: {resourceVersion, sha256} of the catalog the charts come from.""" - from . import __version__ - subset = master_subset(tables) - return { - "format": FORMAT, - "provenance": { - "region": region, - "client": {"versionName": client.get("versionName"), "versionCode": client.get("versionCode")}, - "catalog": {"resourceVersion": catalog.get("resourceVersion"), "sha256": catalog.get("sha256")}, - "master": {"source": master_source, "version": master_version, - "tables": {t: {"sha256": table_sha[t]} for t, _ in TABLES}}, - "exporter": {"name": "nnnotes", "version": __version__, "chartFormat": CHART_FORMAT}, - }, - "master": subset, - "charts": charts(tables["MasterLiveMusicScore"], fetch), - } +# ---------------------------------------------------------------- the deck model's input +def build(tables: dict[str, list[dict]], chart_records: list[dict], provenance: dict) -> dict: + """The deck model's input document (DECK_FORMAT): the TABLES subset of `tables` and chart records (charts).""" + return {"format": DECK_FORMAT, "provenance": provenance, "master": master_subset(tables), + "charts": chart_records} def _emit(v, out: list) -> None: @@ -441,22 +456,3 @@ def encode(doc: dict) -> bytes: def file_bytes(data: bytes, gz: bool) -> bytes: """The bytes written: `data`, or with `gz` gzip with no file name and modification time 0.""" return gzip.compress(data, compresslevel=9, mtime=0) if gz else data - - -def export(out, src: MasterSource, key, fetch: Callable[[str], bytes], *, region: str, client: dict, - catalog: dict) -> dict: - """Read the master data and every chart, then write the file `out` (gzip when it ends in `.gz`) through a - temporary file and a rename. Returns the summary.""" - from .cache import write_atomic - out = Path(out) - tables, shas = read_master(src, key) - doc = build(tables, shas, fetch, region=region, client=client, catalog=catalog, master_source=src.source, - master_version=src.version) - data = encode(doc) - written = file_bytes(data, out.name.endswith(".gz")) - out.parent.mkdir(parents=True, exist_ok=True) - write_atomic(out, written) - return {"out": str(out), "format": FORMAT, "region": region, "masterSource": src.source, - "masterVersion": src.version, "tables": len(TABLES), "rows": sum(len(r) for r in tables.values()), - "charts": len(doc["charts"]), "notes": sum(len(c["notes"]["id"]) for c in doc["charts"]), - "bytes": len(data), "fileBytes": len(written), "sha256": hashlib.sha256(written).hexdigest()} diff --git a/src/nnnotes/musicdata.py b/src/nnnotes/musicdata.py new file mode 100644 index 0000000..24157dd --- /dev/null +++ b/src/nnnotes/musicdata.py @@ -0,0 +1,558 @@ +"""Music data: one JSON file with every live song and chart of one master data version (format +`nnnotes.music-data/1`, docs/music-data.md): titles and credits in every language, bands, vocal characters, category, +tags, release time, score ranks, the live BGM's length, per difficulty the chart facts (level, note counts, BPM, chart +times, skill events, fever ranges) and the chart's deck statistics, what the chart contributes to the live score +whatever the deck, with Gekisou on (a Gekisou live, every rank) and off (a solo live), measured by the deck model +ournotes-deck (the extension module nnnotes._deck). + +Master data is read from the files as served (deckdata.master_files / apk_master: SHA-256 checked against the +manifest, decoded with master.decode). Charts are the TextAssets `Live/MusicScore/` converted by +score.runtime_score; the BGM length is read from the cue sheet's ACB (cue `Length` and the stream's sample count), +without decoding audio. The deck model reads the deck input (deckdata.build: the charts' runtime notes and the master +data tables it needs) in memory; its statistics are checked against the chart facts. With `full` the file also +carries that deck input (`master`, `charts`), every chart's runtime notes and the tables. With a jackets directory, +every song's jacket (the Texture2D `Image/Jacket/`) is written there as `.webp`, scaled to at most +JACKET_SIZE pixels on its longer side. + +The output is canonical (deckdata.encode): minified UTF-8 with one trailing LF, keys in a fixed order, songs sorted +by id, master data floats as the shortest decimal of their binary32 value, the deck model's numbers as it writes them. +The same inputs and deck model give the same bytes. The file is written only when every table, chart and cue sheet +was read and every chart measured; any missing or unreadable input is a MusicDataError naming it. +""" +from __future__ import annotations + +import hashlib +import io +import json +from collections import Counter +from pathlib import Path +from typing import Callable + +from . import deckdata +from .languages import LANGUAGES + +FORMAT = "nnnotes.music-data/1" +DIFFICULTIES = ("easy", "normal", "hard", "expert") +MUSIC_LENGTH_TAIL_MS = 1000 # the live's music length: the last note time + 1000 ms (LiveScore skip path) + +# the tables of the song metadata; the deck model's are deckdata.TABLES +SONG_TABLES = ("MasterLiveMusic", "MasterLiveMusicScore", "MasterText", "MasterBand", "MasterCharacter", + "MasterTag", "MasterLiveMusicCategory", "MasterSound", "MasterSoundCueSheet", "MasterLiveScoreRank") +SCORE_RANKS = {1: "E", 2: "D", 3: "C", 4: "B", 5: "A", 6: "S", 7: "SS"} # LiveScoreRank + + +class MusicDataError(ValueError): + """An input the music data file cannot be made from (the message names it).""" + + +# ---------------------------------------------------------------- texts +class Texts: + """MasterText rows by id -> {language: text} in every language (languages.LANGUAGES).""" + + def __init__(self, rows: list[dict]): + self.rows = {r.get("_id"): r for r in rows} + + def get(self, text_id) -> dict | None: + """{language: text} of a text id; None for an empty id; a MusicDataError for an id MasterText does not have.""" + if not text_id: + return None + r = self.rows.get(text_id) + if r is None: + raise MusicDataError(f"MasterText has no text {text_id!r}") + return {code: r.get(col) for code, (_, col) in LANGUAGES.items()} + + +# ---------------------------------------------------------------- charts +def _f(v) -> float: + return float(v) + + +def bpm_facts(bpm_events: list, first_ms: int, last_ms: int) -> dict: + """{main, min, max, changes} of a chart's BPM changes [(bpm, Pos)] (tick order). `changes` lists every change + as {timeMs, bpm}; main / min / max are taken over the played span [first_ms, last_ms] (the first and the last + judged note): main is the BPM that holds longest in the span (the earliest on a tie), min / max the lowest and + the highest that hold in it. A span of one instant takes the BPM at that time.""" + changes = sorted(((int(p.ms), _f(b)) for b, p in bpm_events), key=lambda x: x[0]) + if not changes: + raise MusicDataError("no BPM change") + held: dict[float, int] = {} + order: list[float] = [] + for i, (t, b) in enumerate(changes): + end = changes[i + 1][0] if i + 1 < len(changes) else None + lo = max(t, first_ms) + hi = last_ms if end is None else min(end, last_ms) + at_span = (end is None or end > first_ms) and t <= last_ms + if not at_span: + continue + if b not in held: + order.append(b) + held[b] = 0 + held[b] += max(0, hi - lo) + if not order: # every change after the span: the first one holds before it + order, held = [changes[0][1]], {changes[0][1]: 0} + main = max(order, key=lambda b: (held[b], -order.index(b))) + return {"main": main, "min": min(order), "max": max(order), + "changes": [{"timeMs": t, "bpm": b} for t, b in changes]} + + +def chart_facts(score_row: dict, key: str, raw: bytes) -> dict: + """The facts of one chart: level, note counts, BPM, chart times, skill events and fevers.""" + from . import score + try: + root = score.load_bytes(raw) + except (ValueError, OSError, EOFError): + raise MusicDataError(f"chart {key}: not a chart (gzip or JSON cannot be read)") from None + try: + rs = score.runtime_score(root) + except Exception as e: # the converter's own errors and malformed chart fields + raise MusicDataError(f"chart {key}: cannot be converted ({type(e).__name__}: {e})") from None + notes = rs.notes + if not notes: + raise MusicDataError(f"chart {key}: no notes") + judged = sorted(int(n.pos.ms) for n in notes if score.is_judgement_note(n.op)) + if not judged: + raise MusicDataError(f"chart {key}: no judged note") + last = max(int(n.pos.ms) for n in notes) + by_type = Counter(int(n.op) for n in notes) + try: + bpm = bpm_facts(rs.bpm_events, judged[0], judged[-1]) + except MusicDataError as e: + raise MusicDataError(f"chart {key}: {e}") from None + return { + "scoreId": score_row["_id"], + "level": score_row["_musicScoreLevel"], + "displayLevel": score_row.get("_musicScoreDisplayLevel"), + "fullComboCount": score_row["_fullComboCount"], + "asset": {"key": key, "sha256": hashlib.sha256(raw).hexdigest()}, + "notes": {"judged": len(judged), "total": len(notes), + "byOperateType": {str(op): by_type[op] for op in sorted(by_type)}}, + "bpm": bpm, + "firstNoteMs": judged[0], + "lastJudgedNoteMs": judged[-1], + "lastNoteMs": last, + "musicLengthMs": last + MUSIC_LENGTH_TAIL_MS, + "skillEventsMs": [int(p.ms) for _, p in rs.skills], + "fevers": [[int(a.ms), int(b.ms)] for _, a, b in rs.fevers], + } + + +# ---------------------------------------------------------------- BGM +def cue_length(acb: bytes, cue: str, where: str) -> dict: + """{lengthMs, samples, sampleRate, durationMs} of a cue of an ACB: the CueTable `Length` and the first stream's + sample count and rate (durationMs = samples * 1000 // sampleRate).""" + from . import acb as acbmod + try: + cues = acbmod.cue_streams(acb) + except Exception as e: + raise MusicDataError(f"{where}: the ACB cannot be read ({type(e).__name__}: {e})") from None + c = cues.get(cue) + if c is None: + raise MusicDataError(f"{where}: no cue {cue!r}") + s = (c.get("streams") or [None])[0] or {} + samples, rate = s.get("samples"), s.get("sampleRate") + return {"lengthMs": c.get("lengthMs"), "samples": samples, "sampleRate": rate, + "durationMs": samples * 1000 // rate if isinstance(samples, int) and isinstance(rate, int) and rate + else None} + + +def catalog_bgm(cat) -> Callable[[str, str], dict]: + """bgm(cue sheet, cue) for a catalog: the cue sheet's ACB (cri.acb_data) -> cue_length.""" + from . import cri + + def bgm(sheet: str, cue: str) -> dict: + try: + files, _ = cri.acb_data(cat, sheet) + except KeyError: + raise MusicDataError(f"cue sheet {sheet}: no such asset") from None + except Exception as e: + raise MusicDataError(f"cue sheet {sheet}: cannot be read ({type(e).__name__}: {e})") from None + return cue_length(files["acb"], cue, f"cue sheet {sheet}") + return bgm + + +# ---------------------------------------------------------------- document +JACKET_KEY = "Image/Jacket/{jacket}" +JACKET_SIZE = 320 # longer side of a written jacket, pixels +JACKET_QUALITY = 88 # WebP quality + + +def jacket_webp(image, size: int = JACKET_SIZE) -> bytes: + """WebP bytes of a jacket image (a Pillow image), scaled down (Lanczos) to at most `size` pixels on its longer + side; opaque images are written without alpha.""" + from PIL import Image + img = image.convert("RGBA") + if img.getextrema()[3][0] == 255: + img = img.convert("RGB") + w, h = img.size + if max(w, h) > size: + img = img.resize((max(1, round(w * size / max(w, h))), max(1, round(h * size / max(w, h)))), + Image.LANCZOS) + buf = io.BytesIO() + img.save(buf, format="WEBP", quality=JACKET_QUALITY, method=6) + return buf.getvalue() + + +def catalog_jacket(cat, size: int = JACKET_SIZE) -> Callable[[str], bytes]: + """jacket(name) for a catalog: the WebP bytes (jacket_webp) of the Texture2D `Image/Jacket/`.""" + import tempfile + from .export import Exporter + ex = Exporter(cat, Path(tempfile.gettempdir()), textures="deferred") + + def jacket(name: str) -> bytes: + key = JACKET_KEY.format(jacket=name) + try: + o = ex.key_object(key) + except KeyError: + raise MusicDataError(f"jacket {name}: no asset {key}") from None + except Exception as e: + raise MusicDataError(f"jacket {name}: {key} cannot be read ({type(e).__name__}: {e})") from None + if o is None or o.type.name != "Texture2D": + raise MusicDataError(f"jacket {name}: {key} is not a Texture2D") + return jacket_webp(o.read().image, size) + return jacket + + +def _by_id(rows: list[dict], table: str) -> dict: + out = {} + for r in rows: + if r.get("_id") in out: + raise MusicDataError(f"{table}: _id {r.get('_id')} occurs twice") + out[r.get("_id")] = r + return out + + +# ---------------------------------------------------------------- the deck model +DECK_SEEDS = 8 # seed set size of a chart with a luck range (ournotes-deck's default) + + +class Deck: + """The deck model (nnnotes._deck): chart statistics of a deck input document, on `workers` threads (None: the + available parallelism), `seeds` seeds for a chart with a luck range.""" + + def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=None): + if module is None: + try: + from . import _deck as module + except ImportError: + raise MusicDataError("the deck model (nnnotes._deck) is not built into this installation: install " + "nnnotes from a wheel or build it (maturin), or pass --no-deck") from None + self.module, self.seeds, self.workers = module, seeds, workers + + def info(self) -> dict: + """{name, version, source, commit, format} of the deck model.""" + i = self.module.info() + return {k: i[k] for k in ("name", "version", "source", "commit", "format")} + + def stats(self, deck_input: dict) -> dict: + """The chart statistics document of a deck input document (deckdata.build), its numbers as written.""" + try: + text = self.module.chart_stats(deckdata.encode(deck_input).decode("utf-8"), self.seeds, self.workers) + except ValueError as e: + raise MusicDataError(f"deck model: {e}") from None + doc = json.loads(text, parse_float=deckdata._Num) + if doc.get("format") != self.info()["format"]: + raise MusicDataError(f"deck model: wrote {doc.get('format')!r}, expected {self.info()['format']!r}") + return doc + + +# the keys of a chart's statistics carried by `deck` (the others are checked against the chart facts) +DECK_CHART_KEYS = ("convertedNoteCount", "skip", "events", "positions", "ranges", "justNotes", "seeds", "offSeeds", + "unplayable") +RANKS = 5 # the ranks of a Gekisou range (a Gekisou live has up to five players) +GEKISOU_RANGES = 3 # the Gekisou ranges of a live (its first three fevers) + + +def mission_pattern(missions) -> int: + """The mission pattern of a song's three Gekisou missions, the `_missionPattern` of its rank bonus rows: 0 when a + mission is missing, 1 all the same, 2 all different, 3 otherwise.""" + a, b, c = missions + if not (a and b and c): + return 0 + if a == b: + return 1 if a == c else 3 + return 2 if b != c and a != c else 3 + + +def rank_bonus_percents(rows: list[dict], missions) -> list[list]: + """The rank bonus percentages [range][rank - 1] of a song's missions from the MasterLiveGekisouRankingScoreBonus + rows of its mission pattern (`_count`: the range 1..3, `_rank` 1..5; a later row wins; 0 without a row).""" + pattern = mission_pattern(missions) + out = [[0] * RANKS for _ in range(GEKISOU_RANGES)] + for r in rows: + c, k = (r.get("_count") or 0) - 1, (r.get("_rank") or 0) - 1 + if r.get("_missionPattern") == pattern and 0 <= c < GEKISOU_RANGES and 0 <= k < RANKS: + out[c][k] = r.get("_scoreBonusPercent") + return out + + +def _trunc_percent(score: int, percent: int) -> int: + """trunc(score * percent / 100), the rank bonus of a range score.""" + p = score * percent + return p // 100 if p >= 0 else -(-p // 100) + + +def _numbers(v, n: int) -> bool: + """Whether v is a list of n numbers (as the deck model writes them).""" + return isinstance(v, list) and len(v) == n and all( + isinstance(x, (int, deckdata._Num)) and not isinstance(x, bool) for x in v) + + +def _check_seed_shapes(stats: dict, kinds: int, where: str) -> None: + """The array shapes and the checks of a chart's statistics: `weights[kind][position]` (a kind null only with + Gekisou off), `rangeWeights[kind][position][range]` (null, or a kind null), one Gekisou off seed, the seeds' + ranges one per range, and every check (and rank check) within its bound.""" + positions, n = stats["positions"], len(stats["ranges"]) + + def check(c, what): + if abs(c["exact"] - float(c["predicted"])) > float(c["bound"]): + raise MusicDataError(f"{where}: {what} scores {c['exact']}, predicted {c['predicted']} beyond the bound " + f"{c['bound']}") + + for seed in stats["seeds"]: + s = f"seed {seed['seed']}" + w = seed.get("weights") + if not (isinstance(w, list) and len(w) == kinds and all(_numbers(x, positions) for x in w)): + raise MusicDataError(f"{where}: {s}: weights are not [kind][position]") + if len(seed["ranges"]) != n: + raise MusicDataError(f"{where}: {s}: {len(seed['ranges'])} range results for {n} ranges") + rw = seed.get("rangeWeights") + if rw is not None and not (isinstance(rw, list) and len(rw) == kinds and all( + k is None or (isinstance(k, list) and len(k) == positions and all(_numbers(x, n) for x in k)) + for k in rw)): + raise MusicDataError(f"{where}: {s}: rangeWeights are not [kind][position][range]") + check(seed["check"], f"{s}: the check deck") + rc = seed.get("rankCheck") + if rc is not None: + if len(rc["ranks"]) != n or not all(1 <= r <= RANKS for r in rc["ranks"]): + raise MusicDataError(f"{where}: {s}: rank check ranks {rc['ranks']!r}") + check(rc, f"{s}: the check deck at ranks {rc['ranks']!r}") + off = stats.get("offSeeds") + if not isinstance(off, list) or len(off) != 1: + raise MusicDataError(f"{where}: the deck model gives no Gekisou off statistics (offSeeds)") + for seed in off: + w = seed.get("weights") + if not (isinstance(w, list) and len(w) == kinds and all(x is None or _numbers(x, positions) for x in w)): + raise MusicDataError(f"{where}: Gekisou off: weights are not [kind][position]") + check(seed["check"], "Gekisou off: the check deck") + + +def chart_deck(song: dict, chart: dict, stats: dict, kinds: int, percents: list[list]) -> dict: + """A chart's `deck` from its statistics, after checking them against the song, the chart facts and the song's + rank bonus percentages (`percents`: rank_bonus_percents), `kinds` the number of score-up kinds.""" + where = f"chart {chart['scoreId']} ({song['id']} {chart['difficulty']})" + checks = ( + ("music id", stats["musicId"], song["id"]), + ("difficulty", stats["difficulty"], chart["difficulty"]), + ("level", stats["level"], chart["level"]), + ("judged note count", stats["judgedNotes"], chart["notes"]["judged"]), + ("last note time", stats["lastNoteMs"], chart["lastNoteMs"]), + ("music length", stats["musicLengthMs"], chart["musicLengthMs"]), + ("Gekisou missions", stats["missions"], song["gekisouMissions"]), + ("skill event times", [t for _, t in stats["events"]], chart["skillEventsMs"]), + ("fevers", [[r["startMs"], r["endMs"]] for r in stats["ranges"]], chart["fevers"][:len(stats["ranges"])]), + ) + for name, deck, facts in checks: + if deck != facts: + raise MusicDataError(f"{where}: the deck model's {name} {deck!r} differs from the chart's {facts!r}") + ranges = stats["ranges"] + for i, r in enumerate(ranges): + want = percents[i] if i < len(percents) else [0] * RANKS + got = r.get("rankBonusPercents") + if got != want or r["rankBonusPercent"] != want[0]: + raise MusicDataError(f"{where}: range {i}: the deck model's rank bonus percentages {got!r} " + f"({r['rankBonusPercent']!r}) differ from MasterLiveGekisouRankingScoreBonus {want!r}") + _check_seed_shapes(stats, kinds, where) + for seed in stats["seeds"]: + for i, (r, info) in enumerate(zip(seed["ranges"], ranges)): + bonus = _trunc_percent(r["rangeScore"], info["rankBonusPercent"]) + if r["rankBonus"] != bonus: + raise MusicDataError(f"{where}: seed {seed['seed']} range {i}: rank bonus {r['rankBonus']} is not " + f"trunc({r['rangeScore']} * {info['rankBonusPercent']} / 100) = {bonus}") + if stats["justNotes"] == 0 and (seed.get("scorePerfect") != seed["score"] or any( + r.get("rangeScorePerfect") != r["rangeScore"] for r in seed["ranges"])): + raise MusicDataError(f"{where}: seed {seed['seed']}: a chart without Just notes scores otherwise on the " + f"Perfect play ({seed.get('scorePerfect')} for {seed['score']})") + return {k: stats.get(k) for k in DECK_CHART_KEYS} + + +# ---------------------------------------------------------------- document +def build(tables: dict[str, list[dict]], table_sha: dict[str, str], fetch: Callable[[str], bytes], + bgm: Callable[[str, str], dict] | None, *, region: str, client: dict, catalog: dict, master_source: str, + master_version: str | None, deck: Deck | None = None, full: bool = False) -> dict: + """The music data document. `fetch(file name)`: a chart TextAsset's bytes (KeyError when there is none); + `bgm(cue sheet, cue)`: the BGM length (catalog_bgm), None to leave every song's `bgm.length` null; `deck`: the + deck model measuring the songs' charts, None to leave every chart's `deck` null; `full`: add the deck input + (`master`, `charts`: deckdata.TABLES and every chart's runtime notes). `tables`: tables_of(deck, full).""" + from . import __version__ + raws: dict[str, bytes] = {} + + def raw_chart(row: dict) -> tuple[str, bytes]: # (key, bytes); each chart asset is read once + name = row["_musicScoreTextFileName"] + key = deckdata.chart_key(name) + if name not in raws: + try: + raws[name] = fetch(name) + except KeyError: + raise MusicDataError(f"chart {key} (MasterLiveMusicScore {row['_id']}): no such asset") from None + return key, raws[name] + + text = Texts(tables["MasterText"]) + scores = _by_id(tables["MasterLiveMusicScore"], "MasterLiveMusicScore") + sounds = _by_id(tables["MasterSound"], "MasterSound") + sheets = _by_id(tables["MasterSoundCueSheet"], "MasterSoundCueSheet") + musics = sorted(_by_id(tables["MasterLiveMusic"], "MasterLiveMusic").values(), key=lambda r: r["_id"]) + rank_groups: dict = {} # _group -> rows in required-score order, as the client reads + for r in sorted(tables["MasterLiveScoreRank"], key=lambda r: (r.get("_requiredScore") or 0, r.get("_id") or 0)): + if r.get("_liveScoreRank") not in SCORE_RANKS: + raise MusicDataError(f"MasterLiveScoreRank {r.get('_id')}: unknown rank {r.get('_liveScoreRank')!r}") + rank_groups.setdefault(r.get("_group"), []).append(r) + + bands = [{"id": b["_id"], "name": text.get(b.get("_nameTextID")), "mainColor": b.get("_mainColorCode"), + "subColor": b.get("_subColorCode")} for b in sorted(tables["MasterBand"], key=lambda r: r["_id"])] + characters = [{"id": c["_id"], "bandId": c.get("_bandID"), "name": text.get(c.get("_nameTextID")), + "shortName": text.get(c.get("_shortNameTextID")), "mainColor": c.get("_mainColorCode")} + for c in sorted(tables["MasterCharacter"], key=lambda r: r["_id"])] + tags = [{"id": t["_id"], "name": text.get(t.get("_nameTextID"))} + for t in sorted(tables["MasterTag"], key=lambda r: r["_id"])] + categories = [{"id": c["_id"], "musicCategories": list(c.get("_musicCategories") or []), + "name": text.get(c.get("_textKey"))} + for c in sorted(tables["MasterLiveMusicCategory"], key=lambda r: r["_id"])] + + songs = [] + for m in musics: + where = f"MasterLiveMusic {m['_id']}" + charts = [] + for d in DIFFICULTIES: + sid = m.get(f"_{d}ID") + if not sid: + continue + row = scores.get(sid) + if row is None: + raise MusicDataError(f"{where}: {d} score {sid} is not in MasterLiveMusicScore") + charts.append({"difficulty": d, **chart_facts(row, *raw_chart(row)), "deck": None}) + snd = sounds.get(m.get("_musicSoundID")) + if snd is None: + raise MusicDataError(f"{where}: sound {m.get('_musicSoundID')} is not in MasterSound") + sheet = sheets.get(snd.get("_soundCueSheetID")) + if sheet is None: + raise MusicDataError(f"{where}: cue sheet {snd.get('_soundCueSheetID')} is not in MasterSoundCueSheet") + length = bgm(sheet["_cueSheetName"], snd["_cueName"]) if bgm is not None else None + rank_rows = rank_groups.get(m.get("_liveScoreRankGroup"), []) + songs.append({ + "id": m["_id"], + "sortOrder": m.get("_sortOrder"), + "startAt": m.get("_startAt"), + "defaultUnlock": m.get("_defaultUnlock"), + "title": text.get(m.get("_titleTextID")), + "ruby": text.get(m.get("_rubyTitleTextID")), + "phonetic": text.get(m.get("_phoneticTextID")), + "bandIds": list(m.get("_bandIDs") or []), + "bandName": text.get(m.get("_bandNameTextID")), + "vocalCharacterIds": list(m.get("_vocalCharacterIDs") or []), + "lyricist": text.get(m.get("_lyricistTextID")), + "composer": text.get(m.get("_composerTextID")), + "arranger": text.get(m.get("_arrangerTextID")), + "musicType": m.get("_musicType"), + "musicCategories": list(m.get("_musicCategories") or []), + "bestMusicTagIds": list(m.get("_bestMusicTagIDs") or []), + "jacket": m.get("_jacketAssetName"), + "gekisouMissions": [m.get("_gekisouMission1"), m.get("_gekisouMission2"), m.get("_gekisouMission3")], + "bgm": {"soundId": snd["_id"], "cueSheet": sheet["_cueSheetName"], "cue": snd["_cueName"], + "length": length}, + "scoreRanks": [{"rank": SCORE_RANKS[r["_liveScoreRank"]], "requiredScore": r.get("_requiredScore"), + "battleRequiredScore": r.get("_battleLiveRequiredScore")} for r in rank_rows], + "charts": charts, + "master": {"MasterLiveMusic": m, "MasterLiveScoreRank": rank_rows}, + }) + + exporter = {"name": "nnnotes", "version": __version__, "chartFormat": deckdata.CHART_FORMAT} + records: dict[int, dict] = {} # score id -> the chart's deck input record + + def record(sid: int) -> dict: + if sid not in records: + records[sid] = deckdata.chart_record(sid, *raw_chart(scores[sid])) + return records[sid] + + deck_doc = None + if deck is not None: + measured = sorted({c["scoreId"] for s in songs for c in s["charts"]}) + stats = deck.stats(deckdata.build( + tables, [record(i) for i in measured], + {"region": region, "master": {"source": master_source, "version": master_version}, + "exporter": exporter})) + by_score = {c["scoreId"]: c for c in stats["charts"]} + if sorted(by_score) != measured: + raise MusicDataError("deck model: the charts measured differ from the songs' charts") + bonus_rows = tables.get("MasterLiveGekisouRankingScoreBonus", []) + for s in songs: + percents = rank_bonus_percents(bonus_rows, s["gekisouMissions"]) + for c in s["charts"]: + c["deck"] = chart_deck(s, c, by_score[c["scoreId"]], len(stats["kinds"]), percents) + deck_doc = {"model": stats["model"], "kinds": stats["kinds"]} + + read = [t for t in tables_of(deck is not None, full) if t in tables] + doc = { + "format": FORMAT, + "provenance": { + "region": region, + "client": {"versionName": client.get("versionName"), "versionCode": client.get("versionCode")}, + "catalog": {"resourceVersion": catalog.get("resourceVersion"), "sha256": catalog.get("sha256")}, + "master": {"source": master_source, "version": master_version, + "tables": {t: {"sha256": table_sha[t]} for t in read}}, + "exporter": exporter, + "deck": deck.info() if deck is not None else None, + }, + "languages": list(LANGUAGES), + "bands": bands, + "characters": characters, + "tags": tags, + "categories": categories, + "deck": deck_doc, + "songs": songs, + } + if full: + doc["master"] = deckdata.master_subset(tables) + doc["charts"] = [record(sid) for sid in sorted(scores)] + return doc + + +def tables_of(deck: bool, full: bool) -> tuple[str, ...]: + """The master data tables read: SONG_TABLES, and deckdata.TABLES with the deck model or `full`.""" + if not (deck or full): + return SONG_TABLES + return SONG_TABLES + tuple(t for t, _ in deckdata.TABLES if t not in SONG_TABLES) + + +def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes], + bgm: Callable[[str, str], dict] | None, *, region: str, client: dict, catalog: dict, + deck: Deck | None = None, full: bool = False, jacket: Callable[[str], bytes] | None = None, + jackets_dir=None) -> dict: + """Read the master data, every chart and every BGM cue sheet, measure the charts with `deck`, then write the file + `out` (gzip when it ends in `.gz`) through a temporary file and a rename; with `jacket` and `jackets_dir`, first + every song's jacket as `/.webp`. Returns the summary.""" + from .cache import write_atomic + out = Path(out) + try: + tables, shas = deckdata.read_master(src, key, tables_of(deck is not None, full)) + doc = build(tables, shas, fetch, bgm, region=region, client=client, catalog=catalog, + master_source=src.source, master_version=src.version, deck=deck, full=full) + jackets = sorted({s["jacket"] for s in doc["songs"] if s["jacket"]}) if jacket is not None else [] + images = {name: jacket(name) for name in jackets} + data = deckdata.encode(deckdata._value(doc, "music data")) + except deckdata.DeckDataError as e: + raise MusicDataError(str(e)) from None + if jackets: + d = Path(jackets_dir) + d.mkdir(parents=True, exist_ok=True) + for name, image in images.items(): + write_atomic(d / f"{name}.webp", image) + written = deckdata.file_bytes(data, out.name.endswith(".gz")) + out.parent.mkdir(parents=True, exist_ok=True) + write_atomic(out, written) + charts = [c for s in doc["songs"] for c in s["charts"]] + return {"out": str(out), "format": FORMAT, "region": region, "masterSource": src.source, + "masterVersion": src.version, "songs": len(doc["songs"]), "charts": len(charts), + "deck": doc["provenance"]["deck"]["commit"] if deck is not None else None, + "unplayable": sum(1 for c in charts if c["deck"] and c["deck"]["unplayable"]), + "full": full, "bgm": bgm is not None, "jackets": len(jackets), + "bytes": len(data), "fileBytes": len(written), "sha256": hashlib.sha256(written).hexdigest()} diff --git a/src/nnnotes/room.py b/src/nnnotes/room.py index c0f4421..7a4001f 100644 --- a/src/nnnotes/room.py +++ b/src/nnnotes/room.py @@ -258,23 +258,29 @@ def add_mesh(self, name, pos, nrm, uv, prims, active=True) -> None: "extras": {"unityActive": active}}) def write(self, path: Path, extras: dict): + """glTF requires the arrays it has to be non-empty and a buffer to hold bytes: an empty array is left out, + and a room without geometry is a scene without nodes and a file without the BIN chunk.""" + scene = {"nodes": list(range(len(self.nodes)))} if self.nodes else {} gltf = { "asset": {"version": "2.0", "generator": "nnnotes/room", "extras": extras}, - "scene": 0, "scenes": [{"nodes": list(range(len(self.nodes)))}], + "scene": 0, "scenes": [scene], "nodes": self.nodes, "meshes": self.meshes, "accessors": self.accessors, - "bufferViews": self.bufferViews, "buffers": [{"byteLength": len(self.bin)}], + "bufferViews": self.bufferViews, "buffers": [{"byteLength": len(self.bin)}] if self.bin else [], "materials": self.materials, "textures": self.textures, "images": self.images, "samplers": self.samplers, } + gltf = {k: v for k, v in gltf.items() if v != []} if self.ext_used: gltf["extensionsUsed"] = sorted(self.ext_used) js = dumps(gltf, separators=(",", ":")).encode("utf-8") js += b" " * (-len(js) % 4) self.bin.extend(b"\0" * (-len(self.bin) % 4)) + size = 12 + 8 + len(js) + (8 + len(self.bin) if self.bin else 0) with open(path, "wb") as f: - f.write(struct.pack(" TypeTreeNode | None: return head -def script_class(obj) -> str: - """Class name of a MonoBehaviour (needs the monoscript bundle loaded). Reads the header up to m_Script with the - object's own typetree (not its fields); the whole object when that is not possible.""" +def script_class(obj) -> str | None: + """Class name of a MonoBehaviour (needs the monoscript bundle loaded); None for a missing script (null m_Script). + Reads the header up to m_Script with the object's own typetree (not its fields); the whole object when that is + not possible.""" try: head = _head_node(obj._get_typetree_node()) if head is not None: - ms = deref(obj, obj.read_typetree(head, check_read=False)["m_Script"]) - if ms is not None: - return ms.read().m_ClassName + pptr = obj.read_typetree(head, check_read=False)["m_Script"] + if not pptr["m_PathID"]: + return None + return deref(obj, pptr).read().m_ClassName except Exception: pass - return obj.read().m_Script.read().m_ClassName + script = obj.read().m_Script + if not script.m_PathID: + return None + return script.read().m_ClassName def strip_pptrs(tt: dict) -> dict: diff --git a/src/nnnotes/webmodel.py b/src/nnnotes/webmodel.py index 6bd83e6..c2922a4 100644 --- a/src/nnnotes/webmodel.py +++ b/src/nnnotes/webmodel.py @@ -321,7 +321,13 @@ def source_entry(address: str, doc: dict, where: str) -> dict: class ModelDir: """The models of `root`: each model in //, the files of a model manifest (read_files; the shader index reduced to the listed GLES3 programs as web.collect reduces it) in the layout export_model writes. ensure exports - a model whose directory does not exist, or every model once when `force`; `built` / `skipped`: the ids.""" + a model whose directory does not exist, or every model once when `force`; `built` / `skipped`: the ids (a model + another process installed meanwhile is skipped). + + Processes may share `root` (several `nnnotes story` at once): a model directory appears in one rename, whole, and + the first process to install a model wins; the others drop their export (the same bytes: exports are + deterministic) and use it. `force` moves a previous directory aside before installing; a process that finds the + directory gone in between exports the model itself.""" def __init__(self, cat, player, root: Path, force: bool = False): self.cat, self.player, self.root, self.force = cat, player, Path(root), force @@ -333,32 +339,42 @@ def ensure(self, address: str) -> dict: mid = model_id(address) if mid not in self._done: d = self.root / mid - if self.force or not d.exists(): - self._export(address, d) - self.built.append(mid) - else: - self.skipped.append(mid) index = d / MODEL_INDEX - if not index.is_file(): + built = (self.force or not d.exists()) and self._export(address, d) + if not index.is_file() and not d.exists(): # moved aside by a forced export of another process + built = self._export(address, d) + (self.built if built else self.skipped).append(mid) + try: + doc = json.loads(index.read_text(encoding="utf-8")) + except FileNotFoundError: raise RuntimeError(f"{d}: no {MODEL_INDEX} (not a model directory); remove it or export again " - f"(--force)") - self._done[mid] = source_entry(address, json.loads(index.read_text(encoding="utf-8")), str(index)) + f"(--force)") from None + self._done[mid] = source_entry(address, doc, str(index)) return self._done[mid] - def _export(self, address: str, d: Path) -> None: - """The model into a temporary directory next to `d`, then renamed to `d` (a previous `d` removed).""" + def _export(self, address: str, d: Path) -> bool: + """The model into a temporary directory in `root`, then renamed to `d`; False when another process's `d` is + there instead (kept: without `force` any `d`, with `force` one installed after the previous `d` was moved + aside).""" self.root.mkdir(parents=True, exist_ok=True) work = Path(tempfile.mkdtemp(prefix=f".{d.name}-", dir=self.root)) try: summary = export_model(self.cat, self.player, address, work / "export") text, binary = collect(work / "export", summary["files"]) files = {**{p: t.encode("utf-8") for p, t in text.items()}, **binary} + model = work / "model" for rel, data in sorted(files.items()): - (work / "model" / rel).parent.mkdir(parents=True, exist_ok=True) - (work / "model" / rel).write_bytes(data) - if d.exists(): - shutil.rmtree(d) - (work / "model").rename(d) + (model / rel).parent.mkdir(parents=True, exist_ok=True) + (model / rel).write_bytes(data) + if _install(model, d): + return True + if not self.force: + return False + try: + d.rename(work / "previous") + except FileNotFoundError: # moved aside by another process + pass + return _install(model, d) finally: shutil.rmtree(work, ignore_errors=True) @@ -367,6 +383,18 @@ def story_fields(self, story_dir: Path) -> dict: return {"modelsDir": Path(os.path.relpath(os.path.abspath(self.root), os.path.abspath(story_dir))).as_posix()} +def _install(src: Path, d: Path) -> bool: + """Rename the directory `src` to `d`; False when `d` exists (a non-empty directory: POSIX refuses to replace it, + Windows any directory).""" + try: + src.rename(d) + return True + except OSError: + if not d.exists(): + raise + return False + + class SiteModels: """The models of the site at `site`: ensure reads model.json from the model manifest models/.json (the models are built before the stories: storysite.build).""" diff --git a/tests/test_deckdata.py b/tests/test_deckdata.py index cb9c0e3..8d44d9b 100644 --- a/tests/test_deckdata.py +++ b/tests/test_deckdata.py @@ -1,8 +1,10 @@ -"""Deck data export (deckdata), the runtime note order of the chart converter (score.runtime_score) and the APK -version code, on synthetic master data, charts and manifests.""" +"""The deck input (deckdata: master data as served, chart records, canonical encoding), the runtime note order of the +chart converter (score.runtime_score) and the APK version code, on synthetic master data, charts and manifests. The +music data file that carries it is tested in test_musicdata.""" import gzip import hashlib import json +import shutil import struct import zipfile @@ -10,7 +12,7 @@ import pytest import synth -from nnnotes import cli, deckdata, jsonio, player, score +from nnnotes import cli, deckdata, jsonio, master, player, score from nnnotes.catalogdb import CatalogDB from nnnotes.master import MasterKey @@ -40,15 +42,6 @@ JUDGEMENTS = [10, 1, 1, 21, 1, 11, 21, 21, 21, 21, 21, 21, 2, 1, 1, 21, 5] -def schema_validator(): - jsonschema = pytest.importorskip("jsonschema") - from pathlib import Path - doc = json.loads((Path(__file__).resolve().parents[1] / "docs" / "schema" / "deck-data.schema.json") - .read_text(encoding="utf-8")) - jsonschema.Draft202012Validator.check_schema(doc) - return jsonschema.Draft202012Validator(doc) - - # ---------------------------------------------------------------- converter order def test_runtime_notes_in_note_dictionary_order(): rs = score.runtime_score(CHART) @@ -138,33 +131,31 @@ def fetch(name): "catalog": {"resourceVersion": None, "sha256": "ab" * 32}} -def export(tmp_path, out="deck.json", rows=rows_of, charts=CHARTS, **kw): +def export(tmp_path, out="deck.json", rows=rows_of, charts=CHARTS, src=None, key=None, **kw): + """Read the master data and every chart, build the deck input and write it canonically to `out` (as + `music-data --full` carries it); the document.""" d = master_dir(tmp_path, rows) if not (tmp_path / "m").exists() else tmp_path / "m" - return deckdata.export(tmp_path / out, deckdata.master_files(d), KEY, fetcher(charts), **dict(PROV, **kw)) + tables, shas = deckdata.read_master(src or deckdata.master_files(d), key or KEY) + prov = dict(PROV, **kw) + doc = deckdata.build(tables, deckdata.charts(tables["MasterLiveMusicScore"], fetcher(charts)), + {"region": prov["region"], "tables": shas}) + data = deckdata.encode(doc) + (tmp_path / out).write_bytes(deckdata.file_bytes(data, out.endswith(".gz"))) + return doc # ---------------------------------------------------------------- the document -def test_export_document(tmp_path): - r = export(tmp_path) +def test_deck_input_document(tmp_path): + doc = export(tmp_path) raw = (tmp_path / "deck.json").read_bytes() assert raw.endswith(b"}\n") and raw.count(b"\n") == 1 and b"\r" not in raw - assert r["bytes"] == len(raw) and r["sha256"] == hashlib.sha256(raw).hexdigest() - assert (r["charts"], r["notes"], r["tables"]) == (3, 21, len(deckdata.TABLES)) - doc = json.loads(raw) + assert json.loads(raw) == json.loads(deckdata.encode(doc)) assert list(doc) == ["format", "provenance", "master", "charts"] and doc["format"] == "nnnotes.deck-data/1" - p = doc["provenance"] - assert list(p) == ["region", "client", "catalog", "master", "exporter"] - assert p["region"] == "xx" and p["client"] == {"versionName": "9.9.9", "versionCode": 99} - assert p["catalog"] == {"resourceVersion": None, "sha256": "ab" * 32} - assert p["master"]["source"] == "api" and p["master"]["version"] == "v-test" - names = [t for t, _ in deckdata.TABLES] - assert list(p["master"]["tables"]) == names == list(doc["master"]) - for t in names: + assert list(doc["master"]) == [t for t, _ in deckdata.TABLES] + assert len(doc["charts"]) == 3 and sum(len(c["notes"]["id"]) for c in doc["charts"]) == 21 + for t, _ in deckdata.TABLES: served = (tmp_path / "m" / f"{t}.bin").read_bytes() - assert p["master"]["tables"][t]["sha256"] == hashlib.sha256(served).hexdigest() - assert p["exporter"] == {"name": "nnnotes", "version": cli.__version__, - "chartFormat": score.convert(SMALL)["format"]} - schema_validator().validate(doc) + assert doc["provenance"]["tables"][t] == hashlib.sha256(served).hexdigest() def test_master_columns_and_values(tmp_path): @@ -225,9 +216,9 @@ def test_charts(tmp_path): def test_output_is_deterministic_and_gzip_has_no_name_or_time(tmp_path): - a = export(tmp_path, "a.json") - b = export(tmp_path, "b.json") - assert (tmp_path / "a.json").read_bytes() == (tmp_path / "b.json").read_bytes() and a["sha256"] == b["sha256"] + export(tmp_path, "a.json") + export(tmp_path, "b.json") + assert (tmp_path / "a.json").read_bytes() == (tmp_path / "b.json").read_bytes() export(tmp_path, "a.json.gz") export(tmp_path, "b.json.gz") gz = (tmp_path / "a.json.gz").read_bytes() @@ -250,8 +241,7 @@ def nan(name): return [{"_id": 1, "_comboBonusType": 1, "_requiredComboCount": 10, "_bonusFactor": float("nan")}] return rows_of(name) with pytest.raises(deckdata.DeckDataError, match="MasterLiveComboScoreBonus row 0 _bonusFactor: NaN"): - deckdata.export(tmp_path / "n.json", deckdata.master_files(master_dir(tmp_path, nan, name="n")), KEY, - fetcher(CHARTS), **PROV) + export(tmp_path, "n.json", src=deckdata.master_files(master_dir(tmp_path, nan, name="n"))) assert not (tmp_path / "n.json").exists() @@ -260,8 +250,8 @@ def failing(tmp_path, match, *, src=None, charts=CHARTS, key=KEY): if src is None: src = deckdata.master_files(tmp_path / "m" if (tmp_path / "m").exists() else master_dir(tmp_path)) with pytest.raises(deckdata.DeckDataError, match=match): - deckdata.export(tmp_path / "out.json", src, key, fetcher(charts), **PROV) - assert not (tmp_path / "out.json").exists() and list(tmp_path.glob("out.json*")) == [] + export(tmp_path, "out.json", src=src, key=key, charts=charts) + assert not (tmp_path / "out.json").exists() def test_missing_chart_asset(tmp_path): @@ -369,11 +359,8 @@ def test_apk_master_and_client(tmp_path): assert (src.source, src.version) == ("embedded", "v-test") client = deckdata.apk_client(apk) assert client == {"versionName": "9.9.9", "versionCode": 99} - deckdata.export(tmp_path / "e.json", src, KEY, fetcher(CHARTS), region=deckdata.EMBEDDED, client=client, - catalog=PROV["catalog"]) - export(tmp_path, "f.json") - e, f = (json.loads((tmp_path / n).read_bytes()) for n in ("e.json", "f.json")) - assert e["provenance"]["master"]["source"] == "embedded" and e["provenance"]["region"] == "embedded" + e = export(tmp_path, "e.json", src=src) + f = export(tmp_path, "f.json") assert e["master"] == f["master"] and e["charts"] == f["charts"] with zipfile.ZipFile(tmp_path / "empty.apk", "w") as z: z.writestr("x", b"") @@ -386,6 +373,50 @@ def test_apk_master_and_client(tmp_path): assert deckdata.apk_client(tmp_path / "empty.apk") == {"versionName": None, "versionCode": None} +# ---------------------------------------------------------------- decoded master data +def decoded_dir(tmp_path, d, name="dec"): + """The decoded tables of a master_dir (`master decode`) with its manifest, as a published snapshot carries them.""" + out = tmp_path / name + assert not master.decode_files(sorted(d.glob("*.bin")), out, KEY)["failed"] + shutil.copy(d / "MasterManifest.json", out / "MasterManifest.json") + return out + + +def test_decoded_master(tmp_path): + d = master_dir(tmp_path) + src = deckdata.decoded_master(decoded_dir(tmp_path, d)) + assert (src.source, src.version, src.decoded) == ("api", "v-test", True) + # the same rows and the manifest's SHA-256 of the files as served, without the key + assert deckdata.read_master(src, None) == deckdata.read_master(deckdata.master_files(d), KEY) + export(tmp_path, "e.json", src=src) + export(tmp_path, "f.json") + assert (tmp_path / "e.json").read_bytes() == (tmp_path / "f.json").read_bytes() + + +def test_decoded_master_checks(tmp_path): + d = master_dir(tmp_path) + with pytest.raises(deckdata.DeckDataError, match="no MasterManifest.json .decoded master data needs the manifest"): + deckdata.decoded_master(tmp_path) + dec = decoded_dir(tmp_path, d, "a") + (dec / "MasterBand.json").unlink() + failing(tmp_path, "no file MasterBand.json", src=deckdata.decoded_master(dec), key=None) + dec = decoded_dir(tmp_path, d, "b") + (dec / "MasterBand.json").write_text("{", encoding="utf-8") + failing(tmp_path, r"MasterBand.json cannot be read \(JSONDecodeError", src=deckdata.decoded_master(dec), key=None) + (dec / "MasterBand.json").write_text('{"x": 1}', encoding="utf-8") + failing(tmp_path, "MasterBand.json has no `_allData` rows", src=deckdata.decoded_master(dec), key=None) + dec = decoded_dir(tmp_path, d, "c") + m = json.loads((dec / "MasterManifest.json").read_text(encoding="utf-8")) + for f in m["files"]: + if f["name"] == "MasterBand.bin": + f["hash"] = "" + (dec / "MasterManifest.json").write_text(json.dumps(m), encoding="utf-8") + failing(tmp_path, "MasterManifest.json lists no SHA-256 for MasterBand.bin", src=deckdata.decoded_master(dec), + key=None) + dec = decoded_dir(tmp_path, master_dir(tmp_path, skip={"MasterEvent"}, name="m2"), "e") + failing(tmp_path, "lists no MasterEvent", src=deckdata.decoded_master(dec), key=None) + + def test_resource_version_from_the_catalog_store(tmp_path): remote = synth.CatalogWriter().build([("a_01.bundle", synth.remote("a_01.bundle"), [])]) sha = hashlib.sha256(remote).hexdigest() @@ -419,40 +450,9 @@ def run(argv, capsys): return code, out, err + message -def test_command(tmp_path, capsys, monkeypatch): - d = master_dir(tmp_path) - out = tmp_path / "o" / "deck.json.gz" - code, _, err = run(["deck-data", "-o", str(out)], capsys) - assert code == 2 and "--master-files" in err and "--apk-master" in err - code, _, err = run(["deck-data", "--master-files", str(d), "-o", str(out)], capsys) - assert code == 2 and "catalog.region" in err - code, _, err = run(["--region", "xx", "deck-data", "--master-files", str(d), "-o", str(out)], capsys) - assert code == 2 and "master.key" in err - code, _, err = run(["deck-data", "--apk-master", "-o", str(out)], capsys) - assert code == 2 and "paths.apk" in err - monkeypatch.setenv("NNNOTES_MASTER_KEY", synth.MASTER_KEY.hex()) - monkeypatch.setenv("NNNOTES_MASTER_IV", synth.MASTER_IV.hex()) - monkeypatch.setattr(cli, "open_catalog", lambda cfg, **kw: FakeCatalog(CHARTS)) - monkeypatch.setattr(score, "fetch_chart", lambda cat, name: cat.charts[name]) - code, stdout, err = run(["--region", "xx", "--cache", str(tmp_path / "cache"), "deck-data", "--master-files", - str(d), "-o", str(out)], capsys) - assert code == 0, err - r = json.loads(stdout) - assert (r["charts"], r["masterVersion"], r["region"]) == (3, "v-test", "xx") - doc = json.loads(gzip.decompress(out.read_bytes())) - assert doc["provenance"]["catalog"] == {"resourceVersion": None, - "sha256": hashlib.sha256(b"remote catalog").hexdigest()} - assert doc["provenance"]["client"] == {"versionName": None, "versionCode": None} - assert synth.MASTER_KEY.hex() not in stdout + err - monkeypatch.setattr(cli, "open_catalog", lambda cfg, **kw: FakeCatalog({})) - code, _, err = run(["--region", "xx", "--cache", str(tmp_path / "cache"), "deck-data", "--master-files", - str(d), "-o", str(tmp_path / "x.json")], capsys) - assert code == 1 and "no such asset" in err and not (tmp_path / "x.json").exists() - - def test_the_tables_are_documented(): from pathlib import Path - text = (Path(__file__).resolve().parents[1] / "docs" / "deck-data.md").read_text(encoding="utf-8") + text = (Path(__file__).resolve().parents[1] / "docs" / "music-data.md").read_text(encoding="utf-8") documented = [] for line in text.splitlines(): if line.startswith("| `Master"): diff --git a/tests/test_musicdata.py b/tests/test_musicdata.py new file mode 100644 index 0000000..8735be7 --- /dev/null +++ b/tests/test_musicdata.py @@ -0,0 +1,532 @@ +"""Music data export (musicdata): metadata, chart facts, BGM length, the deck model's chart statistics and the deck +input on synthetic master data, charts and ACBs. The deck model is a stand-in (FakeDeck) except where the built +extension module is tested itself.""" +import gzip +import hashlib +import io +import json +import re +import sys +from pathlib import Path + +import pytest + +import synth +from nnnotes import cli, deckdata, musicdata, score +from nnnotes.master import MasterKey +from test_deckdata import CHART, SMALL, FakeCatalog, rows_of, run +from test_voices import simple_acb + +KEY = MasterKey(synth.MASTER_KEY, synth.MASTER_IV) +ROOT = Path(__file__).resolve().parents[1] + + +def text(tid, stem): + return {"_id": tid, "_japanese": f"{stem}-ja", "_english": f"{stem}-en", "_traditionalChinese": f"{stem}-tw", + "_simplifiedChinese": f"{stem}-cn", "_korean": f"{stem}-ko"} + + +MUSIC = {"_id": 100002, "_sortOrder": 2, "_startAt": "2026/01/01 0:00:00", "_defaultUnlock": True, + "_titleTextID": "T2", "_rubyTitleTextID": "", "_phoneticTextID": "P2", "_bandIDs": [1], + "_bandNameTextID": "", "_vocalCharacterIDs": [1], "_lyricistTextID": "L2", "_composerTextID": "C2", + "_arrangerTextID": "", "_musicType": 4, "_musicCategories": [1], "_bestMusicTagIDs": [1], + "_jacketAssetName": "jkt_2", "_gekisouMission1": 1, "_gekisouMission2": 3, "_gekisouMission3": 3, + "_musicSoundID": 13, "_easyID": 20, "_normalID": 0, "_hardID": 0, "_expertID": 30, "_liveScoreRankGroup": 9, + "_extra": [7]} +MUSIC1 = dict(MUSIC, _id=100001, _sortOrder=1, _titleTextID="T1", _bandNameTextID="BN", _easyID=10, _expertID=0, + _liveScoreRankGroup=7) +TABLE_ROWS = { + "MasterLiveMusic": [MUSIC, MUSIC1], + "MasterLiveMusicScore": [ + {"_id": 30, "_musicScoreTextFileName": "c/c_03", "_musicScoreLevel": 20, "_fullComboCount": 2, + "_musicScoreDisplayLevel": 20.5}, + {"_id": 10, "_musicScoreTextFileName": "c/c_01", "_musicScoreLevel": 5, "_fullComboCount": 11, + "_musicScoreDisplayLevel": 5.0}, + {"_id": 20, "_musicScoreTextFileName": "c/c_02", "_musicScoreLevel": 9, "_fullComboCount": 2, + "_musicScoreDisplayLevel": 9.0}, + {"_id": 40, "_musicScoreTextFileName": "c/c_04", "_musicScoreLevel": 1, "_fullComboCount": 2, + "_musicScoreDisplayLevel": 1.0}], # a chart of no song + "MasterText": [text("T1", "one"), text("T2", "two"), text("P2", "pho"), text("L2", "lyr"), text("C2", "com"), + text("BN", "crychic"), text("Band1", "mygo"), text("Ch1", "tomori"), text("Ch1s", "tomo"), + text("Tag1", "tag"), text("Cat1", "original")], + "MasterBand": [{"_id": 1, "_nameTextID": "Band1", "_mainColorCode": "#3388BB", "_subColorCode": "#FFFFFF"}], + "MasterCharacter": [{"_id": 1, "_nameTextID": "Ch1", "_shortNameTextID": "Ch1s", "_bandID": 1, + "_mainColorCode": "#77BBDD"}], + "MasterTag": [{"_id": 1, "_nameTextID": "Tag1"}], + "MasterLiveMusicCategory": [{"_id": 1, "_musicCategories": [1], "_textKey": "Cat1"}], + "MasterSound": [{"_id": 13, "_soundCueSheetID": 5, "_cueName": "song2"}], + "MasterSoundCueSheet": [{"_id": 5, "_cueSheetName": "Bgm2"}], + "MasterLiveScoreRank": [ + {"_id": 3, "_group": 9, "_liveScoreRank": 7, "_requiredScore": 900, "_battleLiveRequiredScore": 1800}, + {"_id": 1, "_group": 9, "_liveScoreRank": 2, "_requiredScore": 0, "_battleLiveRequiredScore": 0}, + {"_id": 2, "_group": 8, "_liveScoreRank": 6, "_requiredScore": 5, "_battleLiveRequiredScore": 6}], + # the songs' missions [1, 3, 3] are pattern 3; pattern 2 rows are another song's + "MasterLiveGekisouRankingScoreBonus": [ + {"_id": 100 * p + 10 * c + k, "_missionPattern": p, "_count": c, "_rank": k, + "_scoreBonusPercent": (6 - k) * c + 10 * p} + for p in (2, 3) for c in (1, 2, 3) for k in (1, 2, 3, 4, 5)], +} +CHARTS = {"c/c_01": gzip.compress(json.dumps(CHART).encode("utf-8"), mtime=0), + "c/c_02": json.dumps(SMALL).encode("utf-8"), + "c/c_03": gzip.compress(json.dumps(SMALL).encode("utf-8"), mtime=0), + "c/c_04": json.dumps(SMALL).encode("utf-8")} +ACB = simple_acb({"song2": [1]}, [1], {1: 48000 * 90 + 24}) # 90.0005 s at 48 kHz, cue Length 100 +PROV = {"region": "xx", "client": {"versionName": "9.9.9", "versionCode": 99}, + "catalog": {"resourceVersion": None, "sha256": "ab" * 32}} + + +def all_rows(name): + """The song tables above; the deck model's tables of test_deckdata.""" + return TABLE_ROWS[name] if name in TABLE_ROWS else rows_of(name) + + +def master_dir(tmp_path, rows=TABLE_ROWS): + d = tmp_path / "m" + d.mkdir() + files = [] + for t in musicdata.tables_of(True, True): + data = synth.master_file({"_allData": rows[t] if t in rows else rows_of(t)}) + (d / f"{t}.bin").write_bytes(data) + files.append({"name": f"{t}.bin", "hash": hashlib.sha256(data).hexdigest(), "size": len(data)}) + (d / "MasterManifest.json").write_text(json.dumps({"version": "v-test", "files": files}), encoding="utf-8") + return d + + +def bgm(sheet, cue): + assert sheet == "Bgm2" + return musicdata.cue_length(ACB, cue, f"cue sheet {sheet}") + + +# ---------------------------------------------------------------- a stand-in deck model +WEIGHT = "0.30000000000000004" # a binary64 value: kept as the deck model writes it + + +def trunc_percent(score, percent): + return int(score * percent / 100) + + +class FakeDeck: + """The interface of nnnotes._deck: statistics made from the deck input as the model reports them.""" + COMMIT = "7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6" + + def __init__(self, change=None, fail=None): + self.change, self.fail, self.inputs = change, fail, [] + + def info(self): + return {"name": "ournotes-deck", "version": "0.0.1", "source": "https://github.com/empty-sekai/ournotes-deck", + "commit": self.COMMIT, "dataFormat": deckdata.DECK_FORMAT, "format": "ournotes-deck.chart-stats/2"} + + def chart_stats(self, data, seeds, workers): + if self.fail: + raise ValueError(self.fail) + doc = json.loads(data) + self.inputs.append((doc, seeds, workers)) + assert doc["format"] == deckdata.DECK_FORMAT and list(doc["master"]) == [t for t, _ in deckdata.TABLES] + m = doc["master"] + + def table(name): + return [dict(zip(m[name]["columns"], r)) for r in m[name]["rows"]] + levels = {r["_id"]: r["_musicScoreLevel"] for r in table("MasterLiveMusicScore")} + bonus = {(r["_missionPattern"], r["_count"], r["_rank"]): r["_scoreBonusPercent"] + for r in table("MasterLiveGekisouRankingScoreBonus")} + songs = {} + for r in table("MasterLiveMusic"): + for d in musicdata.DIFFICULTIES: + if r[f"_{d}ID"]: + songs[r[f"_{d}ID"]] = (r["_id"], d, [r["_gekisouMission1"], r["_gekisouMission2"], + r["_gekisouMission3"]]) + charts = [] + for c in doc["charts"]: + music, d, missions = songs[c["scoreId"]] + last = max(c["notes"]["timeMs"]) + fevers = list(zip(c["fevers"]["startMs"], c["fevers"]["endMs"]))[:3] + positions = min(len(c["skillEvents"]["timeMs"]), 5) + pattern = 3 if missions == [1, 3, 3] else 0 + percents = [[bonus.get((pattern, i + 1, k), 0) for k in range(1, 6)] for i in range(len(fevers))] + check = {"deck": [[0, 5000]], "exact": 2000, "predicted": 2000.25, "bound": 7.0} + s = {"scoreId": c["scoreId"], "musicId": music, "difficulty": d, "level": levels[c["scoreId"]], + "judgedNotes": sum(score.is_judgement_note(op) for op in c["notes"]["op"]), + "convertedNoteCount": len(c["notes"]["id"]), "lastNoteMs": last, "musicLengthMs": last + 1000, + "skip": 1.5, "events": [[i % 5, t] for i, t in enumerate(c["skillEvents"]["timeMs"])], + "positions": positions, "missions": missions, + "ranges": [{"index": i, "mission": missions[i], "startMs": a, "endMs": b, + "rankBonusPercent": percents[i][0], "rankBonusPercents": percents[i]} + for i, (a, b) in enumerate(fevers)], + "justNotes": 0, + "seeds": [{"seed": 0, "score": 1234, + "ranges": [{"rangeScore": 101 * (i + 1), "rankBonus": trunc_percent(101 * (i + 1), p[0]), + "maxCombo": 1, "justCount": 0, "lotResults": [0, 0, 0, 0], + "rangeScorePerfect": 101 * (i + 1)} for i, p in enumerate(percents)], + "weights": [["W"] * positions], "check": check, "scorePerfect": 1234, + "rangeWeights": [[["W"] * len(fevers)] * positions], + "rankCheck": {"ranks": [2] * len(fevers), "exact": 2001, "predicted": 2000.5, + "bound": 7.0} if fevers else None}], + "offSeeds": [{"seed": 0, "score": 1000, "weights": [["W"] * positions], "check": dict(check)}]} + if self.change: + self.change(s) + charts.append(s) + out = json.dumps({"format": "ournotes-deck.chart-stats/2", "source": {}, "model": {"power": 300000}, + "kinds": [{"id": 0, "effectType": 2000, "activationTimeSecond": 7.5}], "charts": charts}) + return out.replace('"W"', WEIGHT) + + +def export(tmp_path, rows=TABLE_ROWS, charts=CHARTS, bgm=bgm, out="music.json", deck=None, **kw): + d = master_dir(tmp_path, rows) + return musicdata.export(tmp_path / out, deckdata.master_files(d), KEY, charts.__getitem__, bgm, **PROV, + deck=musicdata.Deck(module=deck, workers=3) if deck is not None else None, **kw) + + +def schema_validator(): + jsonschema = pytest.importorskip("jsonschema") + doc = json.loads((ROOT / "docs" / "schema" / "music-data.schema.json").read_text(encoding="utf-8")) + jsonschema.Draft202012Validator.check_schema(doc) + return jsonschema.Draft202012Validator(doc) + + +# ---------------------------------------------------------------- the document +def test_document(tmp_path): + r = export(tmp_path) + raw = (tmp_path / "music.json").read_bytes() + assert raw.endswith(b"}\n") and raw.count(b"\n") == 1 + assert (r["songs"], r["charts"], r["deck"], r["full"]) == (2, 3, None, False) + assert r["sha256"] == hashlib.sha256(raw).hexdigest() + doc = json.loads(raw) + assert list(doc) == ["format", "provenance", "languages", "bands", "characters", "tags", "categories", "deck", + "songs"] + assert doc["format"] == "nnnotes.music-data/1" and doc["languages"] == ["ja", "en", "zh-Hant", "zh-Hans", "ko"] + p = doc["provenance"] + assert list(p) == ["region", "client", "catalog", "master", "exporter", "deck"] and p["deck"] is None + assert list(p["master"]["tables"]) == list(musicdata.SONG_TABLES) # without the deck model: its tables only + assert p["exporter"] == {"name": "nnnotes", "version": cli.__version__, "chartFormat": deckdata.CHART_FORMAT} + assert doc["deck"] is None + assert doc["bands"] == [{"id": 1, "name": {"ja": "mygo-ja", "en": "mygo-en", "zh-Hant": "mygo-tw", + "zh-Hans": "mygo-cn", "ko": "mygo-ko"}, + "mainColor": "#3388BB", "subColor": "#FFFFFF"}] + assert doc["characters"][0]["name"]["ja"] == "tomori-ja" and doc["characters"][0]["bandId"] == 1 + assert doc["categories"] == [{"id": 1, "musicCategories": [1], "name": dict(doc["categories"][0]["name"])}] + one, two = doc["songs"] + assert (one["id"], two["id"]) == (100001, 100002) # sorted by id + assert one["bandName"]["ja"] == "crychic-ja" and two["bandName"] is None + assert two["title"]["zh-Hans"] == "two-cn" and two["ruby"] is None and two["arranger"] is None + assert two["lyricist"]["en"] == "lyr-en" and two["gekisouMissions"] == [1, 3, 3] + assert two["bgm"] == {"soundId": 13, "cueSheet": "Bgm2", "cue": "song2", + "length": {"lengthMs": 100, "samples": 48000 * 90 + 24, "sampleRate": 48000, + "durationMs": 90000}} + assert two["master"]["MasterLiveMusic"]["_extra"] == [7] # the whole row + assert two["scoreRanks"] == [{"rank": "D", "requiredScore": 0, "battleRequiredScore": 0}, # by required score + {"rank": "SS", "requiredScore": 900, "battleRequiredScore": 1800}] + assert [r["_id"] for r in two["master"]["MasterLiveScoreRank"]] == [1, 3] + assert one["scoreRanks"] == [] and one["master"]["MasterLiveScoreRank"] == [] # a group without rows + assert [c["difficulty"] for c in two["charts"]] == ["easy", "expert"] # difficulties without a score are left out + assert [c["difficulty"] for c in one["charts"]] == ["easy"] + assert all(c["deck"] is None for s in doc["songs"] for c in s["charts"]) + schema_validator().validate(doc) + + +def test_chart_facts(tmp_path): + export(tmp_path) + doc = json.loads((tmp_path / "music.json").read_bytes()) + c = doc["songs"][0]["charts"][0] # CHART at 120 BPM + assert (c["scoreId"], c["level"], c["displayLevel"], c["fullComboCount"]) == (10, 5, 5.0, 11) + assert c["asset"] == {"key": "Live/MusicScore/c/c_01", "sha256": hashlib.sha256(CHARTS["c/c_01"]).hexdigest()} + rs = score.runtime_score(CHART) + judged = sorted(n.pos.ms for n in rs.notes if score.is_judgement_note(n.op)) + assert c["notes"]["judged"] == len(judged) and c["notes"]["total"] == len(rs.notes) == 17 + assert sum(c["notes"]["byOperateType"].values()) == 17 and list(c["notes"]["byOperateType"])[0] == "1" + assert c["bpm"] == {"main": 120, "min": 120, "max": 120, "changes": [{"timeMs": 0, "bpm": 120}]} + assert (c["firstNoteMs"], c["lastJudgedNoteMs"], c["lastNoteMs"], c["musicLengthMs"]) == (0, 5000, 5000, 6000) + assert c["skillEventsMs"] == [2500, 500] and c["fevers"] == [[0, 1000], [2000, 2500]] + assert list(c)[-1] == "deck" + + +def test_bpm_facts(): + class P: + def __init__(self, ms): + self.ms = ms + ev = [(100.0, P(0)), (200.0, P(1000)), (150.0, P(1500)), (300.0, P(9000))] + f = musicdata.bpm_facts(ev, 500, 4000) # 100 for 500 ms, 200 for 500, 150 for 2500; 300 after + assert (f["main"], f["min"], f["max"]) == (150.0, 100.0, 200.0) + assert [x["timeMs"] for x in f["changes"]] == [0, 1000, 1500, 9000] + assert musicdata.bpm_facts(ev, 1200, 1200)["main"] == 200.0 # one instant: the BPM at that time + tie = [(120.0, P(0)), (180.0, P(1000))] + assert musicdata.bpm_facts(tie, 0, 2000)["main"] == 120.0 # equal time: the earliest + with pytest.raises(musicdata.MusicDataError): + musicdata.bpm_facts([], 0, 1) + + +# ---------------------------------------------------------------- the deck model +def test_deck(tmp_path): + fake = FakeDeck() + r = export(tmp_path, deck=fake) + raw = (tmp_path / "music.json").read_bytes() + doc = json.loads(raw) + assert r["deck"] == FakeDeck.COMMIT and r["unplayable"] == 0 + (deck_input, seeds, workers), = fake.inputs + assert (seeds, workers) == (musicdata.DECK_SEEDS, 3) + assert [c["scoreId"] for c in deck_input["charts"]] == [10, 20, 30] # the songs' charts, not chart 40 + assert deck_input["provenance"]["master"] == {"source": "api", "version": "v-test"} + p = doc["provenance"] + assert p["deck"] == {k: fake.info()[k] for k in ("name", "version", "source", "commit", "format")} + assert list(p["master"]["tables"]) == list(musicdata.tables_of(True, False)) + assert doc["deck"] == {"model": {"power": 300000}, + "kinds": [{"id": 0, "effectType": 2000, "activationTimeSecond": 7.5}]} + c = doc["songs"][0]["charts"][0] + assert list(c["deck"]) == list(musicdata.DECK_CHART_KEYS) + assert c["deck"]["events"] == [[0, 2500], [1, 500]] and c["deck"]["ranges"][1]["startMs"] == 2000 + assert c["deck"]["unplayable"] is None and c["deck"]["skip"] == 1.5 + # the ranks: the song's mission pattern (3) rows of every range + assert [r["rankBonusPercents"] for r in c["deck"]["ranges"]] == [[35, 34, 33, 32, 31], [40, 38, 36, 34, 32]] + seed = c["deck"]["seeds"][0] + assert seed["rankCheck"]["ranks"] == [2, 2] and seed["scorePerfect"] == seed["score"] + assert [r["rankBonus"] for r in seed["ranges"]] == [35, 80] + assert c["deck"]["offSeeds"][0]["score"] == 1000 + # the deck model's numbers as it writes them: a binary64 value is not narrowed to binary32 + assert b'"weights":[[' + WEIGHT.encode() + b',' + WEIGHT.encode() + b']]' in raw and b'"predicted":2000.25' in raw + assert "master" not in doc and "charts" not in doc + schema_validator().validate(doc) + (tmp_path / "m").rename(tmp_path / "m0") + export(tmp_path, deck=FakeDeck(), out="again.json") + assert (tmp_path / "again.json").read_bytes() == raw + + +@pytest.mark.parametrize("change, match", [ + (lambda s: s.update(judgedNotes=s["judgedNotes"] + 1), "chart 10 .*judged note count"), + (lambda s: s.update(musicLengthMs=0), "music length 0 differs"), + (lambda s: s.update(level=99), "level 99 differs"), + (lambda s: s["missions"].reverse(), "Gekisou missions"), + (lambda s: s["events"].pop(), "skill event times"), + (lambda s: s["ranges"] and s["ranges"][0].update(endMs=1), "fevers"), + (lambda s: s.update(difficulty="hard"), "difficulty 'hard' differs"), + (lambda s: s["ranges"] and s["ranges"][0]["rankBonusPercents"].__setitem__(2, 99), "range 0: .*percentages"), + (lambda s: s["ranges"] and s["ranges"][0].update(rankBonusPercent=1), "range 0: .*percentages"), + (lambda s: s["ranges"] and s["seeds"][0]["ranges"][0].update(rankBonus=-1), "rank bonus -1 is not trunc"), + (lambda s: s["seeds"][0]["ranges"].pop(), "range results for"), + (lambda s: s.pop("offSeeds"), "no Gekisou off statistics"), + (lambda s: s["offSeeds"].append(s["offSeeds"][0]), "no Gekisou off statistics"), + (lambda s: s["offSeeds"][0]["weights"].append(None), "Gekisou off: weights"), + (lambda s: s["offSeeds"][0]["check"].update(exact=0), "Gekisou off: the check deck scores 0"), + (lambda s: s["seeds"][0].update(scorePerfect=1), "otherwise on the Perfect play"), + (lambda s: s["seeds"][0]["weights"][0].pop(), "weights are not"), + (lambda s: s["seeds"][0]["weights"].append(None), "weights are not"), + (lambda s: s["seeds"][0].update(rangeWeights=[[]]), "rangeWeights are not"), + (lambda s: s["seeds"][0]["check"].update(bound=0.1), "the check deck scores 2000"), + (lambda s: s["ranges"] and s["seeds"][0]["rankCheck"].update(exact=0), "at ranks"), + (lambda s: s["ranges"] and s["seeds"][0]["rankCheck"].update(ranks=[6, 1]), "rank check ranks"), +]) +def test_deck_checks(tmp_path, change, match): + with pytest.raises(musicdata.MusicDataError, match=match): + export(tmp_path, deck=FakeDeck(change)) + assert not (tmp_path / "music.json").exists() + + +def test_deck_without_range_weights(tmp_path): + """A chart without range weights, a kind without them and a kind without Gekisou off weights are carried.""" + def change(s): + s["seeds"][0]["rangeWeights"] = None if s["scoreId"] == 10 else [None] + s["seeds"][0]["rankCheck"] = None + s["offSeeds"][0]["weights"] = [None] + export(tmp_path, deck=FakeDeck(change)) + doc = json.loads((tmp_path / "music.json").read_bytes()) + decks = {c["scoreId"]: c["deck"] for s in doc["songs"] for c in s["charts"]} + assert decks[10]["seeds"][0]["rangeWeights"] is None and decks[20]["seeds"][0]["rangeWeights"] == [None] + assert decks[30]["offSeeds"][0]["weights"] == [None] + schema_validator().validate(doc) + + +def test_rank_bonus_percents(): + assert [musicdata.mission_pattern(m) for m in ([1, 2, 0], [2, 2, 2], [1, 2, 3], [1, 1, 3], [1, 3, 3], + [3, 1, 3])] == [0, 1, 2, 3, 3, 3] + rows = [{"_missionPattern": 2, "_count": 1, "_rank": 1, "_scoreBonusPercent": 30}, + {"_missionPattern": 2, "_count": 3, "_rank": 5, "_scoreBonusPercent": 4}, + {"_missionPattern": 2, "_count": 3, "_rank": 5, "_scoreBonusPercent": 5}, # a later row wins + {"_missionPattern": 2, "_count": 4, "_rank": 1, "_scoreBonusPercent": 9}, # no fourth range + {"_missionPattern": 2, "_count": 1, "_rank": 6, "_scoreBonusPercent": 9}, # no sixth rank + {"_missionPattern": 1, "_count": 2, "_rank": 1, "_scoreBonusPercent": 9}] # another pattern + assert musicdata.rank_bonus_percents(rows, [1, 2, 3]) == [[30, 0, 0, 0, 0], [0] * 5, [0, 0, 0, 0, 5]] + assert musicdata.rank_bonus_percents(rows, [0, 0, 0]) == [[0] * 5] * 3 + + +def test_deck_errors(tmp_path, monkeypatch): + with pytest.raises(musicdata.MusicDataError, match="deck model: chart 30: the check deck scores"): + export(tmp_path, deck=FakeDeck(fail="chart 30: the check deck scores 1, the chart statistics predict 2")) + assert not (tmp_path / "music.json").exists() + monkeypatch.setitem(sys.modules, "nnnotes._deck", None) # an installation without the module + with pytest.raises(musicdata.MusicDataError, match="not built into this installation.*--no-deck"): + musicdata.Deck() + + +def test_deck_module(): + deck = pytest.importorskip("nnnotes._deck") + info = deck.info() + assert info["name"] == "ournotes-deck" and info["format"] == "ournotes-deck.chart-stats/2" + assert info["dataFormat"] == deckdata.DECK_FORMAT and re.fullmatch(r"[0-9a-f]{40}", info["commit"]) + lock = (ROOT / "rust" / "Cargo.lock").read_text(encoding="utf-8") + assert f"#{info['commit']}\"" in lock # the commit Cargo.lock pins + with pytest.raises(ValueError, match="not a deck data file"): + deck.chart_stats('{"format": "x"}') + assert musicdata.Deck().info()["commit"] == info["commit"] + + +def test_the_extension_version_is_the_package_version(): + from nnnotes import __version__ + cargo = (ROOT / "rust" / "Cargo.toml").read_text(encoding="utf-8") + assert re.search(r'^version = "([^"]+)"', cargo, re.M).group(1) == __version__ + + +# ---------------------------------------------------------------- the deck input (--full) +def test_full(tmp_path): + asked = [] + + def fetch(name): + asked.append(name) + return CHARTS[name] + r = export(tmp_path, charts=type("C", (), {"__getitem__": staticmethod(fetch)})(), full=True, deck=FakeDeck()) + doc = json.loads((tmp_path / "music.json").read_bytes()) + assert r["full"] is True and sorted(asked) == sorted(CHARTS) # every chart asset read once + assert list(doc)[-2:] == ["master", "charts"] + assert list(doc["master"]) == [t for t, _ in deckdata.TABLES] + assert [c["scoreId"] for c in doc["charts"]] == [10, 20, 30, 40] # every chart, songs or not + tables, _ = deckdata.read_master(deckdata.master_files(tmp_path / "m"), KEY) + assert doc["master"] == json.loads(deckdata.encode({"m": deckdata.master_subset(tables)}))["m"] + assert doc["charts"][0] == deckdata.chart_record(10, "Live/MusicScore/c/c_01", CHARTS["c/c_01"]) + schema_validator().validate(doc) + (tmp_path / "x").mkdir() + export(tmp_path / "x", full=True) # the deck input without the deck model + assert json.loads((tmp_path / "x" / "music.json").read_bytes())["charts"] == doc["charts"] + + +# ---------------------------------------------------------------- output, jackets and failures +def test_deterministic_and_no_bgm(tmp_path): + export(tmp_path, out="a.json") + (tmp_path / "m").rename(tmp_path / "m0") + export(tmp_path, out="b.json") + assert (tmp_path / "a.json").read_bytes() == (tmp_path / "b.json").read_bytes() + (tmp_path / "m").rename(tmp_path / "m1") + r = export(tmp_path, bgm=None, out="c.json.gz") + doc = json.loads(gzip.decompress((tmp_path / "c.json.gz").read_bytes())) + assert r["bgm"] is False and all(s["bgm"]["length"] is None for s in doc["songs"]) + + +def test_jacket_webp(): + from PIL import Image + img = Image.new("RGBA", (1024, 512), (10, 200, 30, 255)) + out = Image.open(io.BytesIO(musicdata.jacket_webp(img))) + assert (out.format, out.size, out.mode) == ("WEBP", (320, 160), "RGB") + img.putpixel((0, 0), (0, 0, 0, 0)) + out = Image.open(io.BytesIO(musicdata.jacket_webp(img, size=2000))) + assert (out.size, out.mode) == ((1024, 512), "RGBA") + + +def test_jackets(tmp_path): + from PIL import Image + asked = [] + + def jacket(name): + asked.append(name) + return musicdata.jacket_webp(Image.new("RGB", (64, 64), (1, 2, 3))) + r = export(tmp_path, jacket=jacket, jackets_dir=tmp_path / "j") + assert r["jackets"] == 1 and asked == ["jkt_2"] # the two songs share one jacket + assert Image.open(tmp_path / "j" / "jkt_2.webp").size == (64, 64) + (tmp_path / "x").mkdir() + assert export(tmp_path / "x", out="b.json")["jackets"] == 0 + + def missing(name): + raise musicdata.MusicDataError(f"jacket {name}: no asset") + (tmp_path / "y").mkdir() + with pytest.raises(musicdata.MusicDataError, match="jacket jkt_2"): + export(tmp_path / "y", jacket=missing, jackets_dir=tmp_path / "y" / "j") + assert not (tmp_path / "y" / "music.json").exists() + + +@pytest.mark.parametrize("change, match", [ + (lambda r: r["MasterText"].pop(0), "MasterText has no text 'T1'"), + (lambda r: r["MasterLiveMusicScore"].pop(0), "expert score 30 is not in MasterLiveMusicScore"), + (lambda r: r["MasterSound"].clear(), "sound 13 is not in MasterSound"), + (lambda r: r["MasterLiveMusic"].append(dict(MUSIC)), "MasterLiveMusic: _id 100002 occurs twice"), + (lambda r: r["MasterLiveScoreRank"][0].update(_liveScoreRank=8), "MasterLiveScoreRank 3: unknown rank 8"), +]) +def test_master_errors(tmp_path, change, match): + rows = json.loads(json.dumps(TABLE_ROWS)) + change(rows) + with pytest.raises(musicdata.MusicDataError, match=match): + export(tmp_path, rows=rows) + assert not (tmp_path / "music.json").exists() + + +def test_input_errors(tmp_path): + with pytest.raises(musicdata.MusicDataError, match="c/c_03 .*no such asset"): + export(tmp_path, charts={k: v for k, v in CHARTS.items() if k != "c/c_03"}) + (tmp_path / "m").rename(tmp_path / "m0") + with pytest.raises(musicdata.MusicDataError, match="not a chart"): + export(tmp_path, charts=dict(CHARTS, **{"c/c_02": b"\x1f\x8bnot"})) + (tmp_path / "m").rename(tmp_path / "m1") + with pytest.raises(musicdata.MusicDataError, match="no cue 'song2'"): + export(tmp_path, bgm=lambda s, c: musicdata.cue_length(simple_acb({"x": [1]}, [1], {1: 5}), c, s)) + (tmp_path / "m").rename(tmp_path / "m2") + with pytest.raises(musicdata.MusicDataError, match=r"c/c_04 \(MasterLiveMusicScore 40\): no such asset"): + export(tmp_path, charts={k: v for k, v in CHARTS.items() if k != "c/c_04"}, full=True) + + +# ---------------------------------------------------------------- command line +def test_command(tmp_path, capsys, monkeypatch): + d = master_dir(tmp_path) + out = tmp_path / "o" / "music.json" + code, _, err = run(["music-data", "-o", str(out)], capsys) + assert code == 2 and "--master-files" in err and "--apk-master" in err + code, _, err = run(["music-data", "--master-files", str(d), "--no-deck", "-o", str(out)], capsys) + assert code == 2 and "catalog.region" in err + code, _, err = run(["--region", "xx", "music-data", "--master-files", str(d), "--no-deck", "-o", str(out)], + capsys) + assert code == 2 and "master.key" in err + monkeypatch.setenv("NNNOTES_MASTER_KEY", synth.MASTER_KEY.hex()) + monkeypatch.setenv("NNNOTES_MASTER_IV", synth.MASTER_IV.hex()) + monkeypatch.setattr(cli, "open_catalog", lambda cfg, **kw: FakeCatalog(CHARTS)) + monkeypatch.setattr(score, "fetch_chart", lambda cat, name: cat.charts[name]) + base = ["--region", "xx", "--cache", str(tmp_path / "cache"), "music-data", "--master-files", str(d), "--no-bgm"] + code, stdout, err = run(base + ["--no-deck", "-o", str(out)], capsys) + assert code == 0, err + r = json.loads(stdout) + assert (r["songs"], r["charts"], r["bgm"], r["region"], r["deck"]) == (2, 3, False, "xx", None) + assert synth.MASTER_KEY.hex() not in stdout + err + fake = FakeDeck() + real = musicdata.Deck + monkeypatch.setattr(musicdata, "Deck", lambda **kw: real(module=fake, **kw)) + code, stdout, err = run(base + ["--full", "--seeds", "2", "--workers", "4", "-o", str(out) + ".gz"], capsys) + assert code == 0, err + r = json.loads(stdout) + assert (r["deck"], r["full"]) == (FakeDeck.COMMIT, True) and fake.inputs[0][1:] == (2, 4) + doc = json.loads(gzip.decompress(Path(str(out) + ".gz").read_bytes())) + assert doc["provenance"]["catalog"] == {"resourceVersion": None, + "sha256": hashlib.sha256(b"remote catalog").hexdigest()} + assert len(doc["charts"]) == 4 + monkeypatch.setattr(musicdata, "Deck", lambda **kw: real(module=FakeDeck(fail="boom"), **kw)) + code, _, err = run(base + ["-o", str(tmp_path / "x.json")], capsys) + assert code == 1 and "deck model: boom" in err and not (tmp_path / "x.json").exists() + + +def test_command_decoded_master(tmp_path, capsys, monkeypatch): + from test_deckdata import decoded_dir + d = master_dir(tmp_path) + dec = decoded_dir(tmp_path, d) + out = tmp_path / "o" / "music.json" + code, _, err = run(["--region", "xx", "music-data", "--decoded-master", "--no-deck", "-o", str(out)], capsys) + assert code == 2 and "paths.master" in err + monkeypatch.setattr(cli, "open_catalog", lambda cfg, **kw: FakeCatalog(CHARTS)) + monkeypatch.setattr(score, "fetch_chart", lambda cat, name: cat.charts[name]) + fake = FakeDeck() + real = musicdata.Deck + monkeypatch.setattr(musicdata, "Deck", lambda **kw: real(module=fake, **kw)) + common = ["--region", "xx", "--cache", str(tmp_path / "cache")] + options = ["--no-bgm", "--full", "-o"] + code, stdout, err = run(common + ["--master", str(dec), "music-data", "--decoded-master"] + options + [str(out)], + capsys) # no master key is set + assert code == 0, err + r = json.loads(stdout) + assert (r["masterSource"], r["masterVersion"], r["region"], r["songs"]) == ("api", "v-test", "xx", 2) + # the file of the master data files as served, byte for byte + monkeypatch.setenv("NNNOTES_MASTER_KEY", synth.MASTER_KEY.hex()) + monkeypatch.setenv("NNNOTES_MASTER_IV", synth.MASTER_IV.hex()) + code, _, err = run(common + ["music-data", "--master-files", str(d)] + options + [str(tmp_path / "f.json")], + capsys) + assert code == 0, err + assert out.read_bytes() == (tmp_path / "f.json").read_bytes() diff --git a/tests/test_room.py b/tests/test_room.py index 2a5bc62..1095343 100644 --- a/tests/test_room.py +++ b/tests/test_room.py @@ -332,6 +332,39 @@ def test_textures_with_the_same_png_bytes_share_one_image(): assert [im["name"] for im in glb.images] == ["a", "c"] and len(glb.bufferViews) == 2 +def glb_chunks(path): + data = path.read_bytes() + magic, version, size = struct.unpack_from("", "m_GameObject"), ("UInt8", "m_Enabled"), + ("PPtr", "m_Script"))] + node = TypeTreeNode(0, "MonoBehaviour", "Base", -1, 1, m_Children=kids) + null = {"m_FileID": 0, "m_PathID": 0} + obj = SimpleNamespace(_get_typetree_node=lambda: node, + read_typetree=lambda *a, **k: {"m_GameObject": null, "m_Enabled": 1, "m_Script": null}) + assert unity.script_class(obj) is None + whole = SimpleNamespace(_get_typetree_node=lambda: None, + read=lambda: SimpleNamespace(m_Script=SimpleNamespace(m_PathID=0))) + assert unity.script_class(whole) is None + + # ---------------------------------------------------------------- typetree values def ref_value(ex, owner, v): if unity.is_pptr(v): diff --git a/tests/test_webmodel.py b/tests/test_webmodel.py index 2d2129d..1585a83 100644 --- a/tests/test_webmodel.py +++ b/tests/test_webmodel.py @@ -642,6 +642,47 @@ def failing(*a): webmodel.ModelDir(None, player, root).ensure(KEY_B) +@pytest.mark.parametrize("force", [False, True]) +def test_model_dir_shared_by_processes_keeps_one_whole_model(tmp_path, monkeypatch, force): + """Another process (another ModelDir on the same root) installs the model while this one exports it: without + force the first installed copy is kept and this export dropped, with force this export replaces it; either way + one whole model directory and no temporary directory is left.""" + player = fake_export(monkeypatch, motion_sync=True) + root, real, other = tmp_path / "live2d", webmodel.export_model, [] + + def racing(*a): + if not other: + other.append(webmodel.ModelDir(None, player, root, force=force)) + other[0].ensure(KEY_A) + return real(*a) + monkeypatch.setattr(webmodel, "export_model", racing) + a = webmodel.ModelDir(None, player, root, force=force) + assert a.ensure(KEY_A) == {"id": "adv_model_a", "motionSync": True} + assert (a.built, a.skipped) == ((["adv_model_a"], []) if force else ([], ["adv_model_a"])) + assert other[0].built == ["adv_model_a"] + assert [p.name for p in root.iterdir()] == ["adv_model_a"] + assert dir_files(root) == [f"adv_model_a/{f}" for f in model_files("adv_model_a")] + + +def test_model_dir_exports_a_model_moved_aside_after_the_check(tmp_path, monkeypatch): + """The model directory existed when checked but is gone when read (a forced export of another process moved it + aside): exported here.""" + player = fake_export(monkeypatch, motion_sync=True) + root = tmp_path / "live2d" + d, real, seen = root / "adv_model_a", webmodel.Path.exists, [] + + def exists(p): + if p == d and not seen: + seen.append(p) + return True + return real(p) + monkeypatch.setattr(webmodel.Path, "exists", exists) + a = webmodel.ModelDir(None, player, root) + assert a.ensure(KEY_A) == {"id": "adv_model_a", "motionSync": True} + assert seen and (a.built, a.skipped) == (["adv_model_a"], []) + assert dir_files(root) == [f"adv_model_a/{f}" for f in model_files("adv_model_a")] + + def test_story_command_exports_its_models_next_to_the_story(tmp_path, capsys, monkeypatch): from nnnotes import story player = fake_export(monkeypatch)