From 36642e4ca3e97b42622d1e9dcca46b926114f3d8 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 30 Sep 2026 03:22:15 +0800 Subject: [PATCH 1/4] feat(music-data): the Gekisou catalog, range luck points and the seed set check The file carries `gekisouCatalog`, what a deck brings to a Gekisou live, from the master data (with --no-deck too): the Gekisou skills and Gekisou support skills with their mission, highest level, name and description, the member cards with their character, band and Gekisou skill, and the snaps with their characters, Gekisou support skills and the level those have at the snap's highest rank. The texts are in every language, as the other text fields. The catalog tables are read and recorded in `provenance.master.tables` with the song tables. Every seed's range results carry `luckPoints` (TotalBonusPoint, the luck mission's rank figure), and the seeds of every chart are checked to be its seed set: none when it is unplayable with Gekisou, seed 0 without a luck range, else the first --seeds seeds of the deck model's published seed set, which nnnotes computes itself. docs/music-data.md and the schema describe the catalog and the luck points (optional in the schema: older files do not have them). Co-Authored-By: Claude Opus 5.5 --- README.en.md | 2 +- README.md | 2 +- docs/commands.md | 13 +- docs/music-data.md | 51 +++++-- docs/schema/music-data.schema.json | 166 +++++++++++++++++++++++ src/nnnotes/musicdata.py | 194 ++++++++++++++++++++++---- tests/test_musicdata.py | 211 ++++++++++++++++++++++++++--- 7 files changed, 569 insertions(+), 70 deletions(-) diff --git a/README.en.md b/README.en.md index 6906a25..a6d5c5d 100644 --- a/README.en.md +++ b/README.en.md @@ -49,7 +49,7 @@ nnnotes config check # each setting's origin and whether it is valid | `player` | APK | render-related global settings (color space, quality levels, renderers) as JSON | | `live` | music ID + difficulty | a full chart directory: chart and runtime notes, 3D scene, note and effect assets, BGM and sounds, sound routing | | `web` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; the Live2D models of the stories are built first as models, listed in `models.json`, and the story manifests reference them; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`); compressible assets (JSON, shaders, moc3, ...) are stored gzip encoded by default (`--compress br` / `none`) | -| `music-data` | master data files (`--master-files` directory or `--apk-master`), or decoded master data with its manifest (`--decoded-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (built into nnnotes) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) | +| `music-data` | master data files (`--master-files` directory or `--apk-master`), or decoded master data with its manifest (`--decoded-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; the Gekisou catalog (member cards, snaps and their Gekisou skills and support skills); per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (built into nnnotes) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) | Export conventions: diff --git a/README.md b/README.md index 96b94dc..d02509e 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ nnnotes config check # 每项设置的来源和格式是否有效， | `player` | APK | 渲染相关的全局设置（色彩空间、画质等级、渲染器）JSON | | `live` | 曲目 ID + 难度 | 完整谱面目录：谱面与运行时音符、3D 场景、音符与特效资源、BGM 与音效、声音路由 | | `web` | `--pair 曲目:难度`（可重复）或 `--all`；`--live2d 模型`（可重复）或 `--all-live2d`；`--story 剧情 ID`（可重复）或 `--all-stories`；`--region 区服`（可重复）或 `--all-regions` | ournotes-player 静态站点：共享播放器 + 每谱 / 每模型 / 每集剧情清单 + 内容寻址资源；剧情用到的 Live2D 模型先按模型构建并列入 `models.json`，剧情清单引用它们；一个站点可服务多个区服，列表文本含五种语言；剧情的界面文字按语言分组，字形由开源字体生成 TextMeshPro 字体资源（`--fonts game` 时用游戏字体）；可压缩的资源（JSON、着色器、moc3 等）默认以 gzip 存储，`--compress br` / `none` 可改 | -| `music-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`），或带清单的解码 masterdata（`--decoded-master`） | 全部歌曲与谱面的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、评级线、BGM 时长；每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间；以及组卡模型 ournotes-deck（内置于 nnnotes）在整场模拟上实测的谱面统计（无技能得分、每种加分技能在每个演出位的权重）。`--full` 另附组卡模型的输入：每张谱面的运行时音符与卡牌、技能、加成、分数、活动相关的 masterdata 表（[格式](docs/music-data.md)） | +| `music-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`），或带清单的解码 masterdata（`--decoded-master`） | 全部歌曲与谱面的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、评级线、BGM 时长；激走目录（成员卡、小卡及其激走技能与激走支援技能）；每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间；以及组卡模型 ournotes-deck（内置于 nnnotes）在整场模拟上实测的谱面统计（无技能得分、每种加分技能在每个演出位的权重）。`--full` 另附组卡模型的输入：每张谱面的运行时音符与卡牌、技能、加成、分数、活动相关的 masterdata 表（[格式](docs/music-data.md)） | 导出约定： diff --git a/docs/commands.md b/docs/commands.md index bd03f0f..189ff66 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -677,12 +677,13 @@ nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--ful Writes one JSON file with every `MasterLiveMusic` song and its charts for one master data version: titles, readings and credits in the five text languages, bands (and a song's own band name), vocal characters, category, tags, release time, jacket, Gekisou missions, score ranks, the whole `MasterLiveMusic` row, the live BGM's cue and length -(read from the cue sheet's ACB, without decoding audio); per difficulty the chart facts (level and display level, -full combo count, note counts, BPM, note times, the live's music length, skill event times, fever ranges) and the -chart's deck statistics: the no-skill score and the weight of every score-up skill kind at every performance -position, with Gekisou on (a Gekisou live at rank 1, with what every other rank and the Perfect play need) and off (a -solo live), measured by the deck model ournotes-deck (built into nnnotes as `nnnotes._deck`) on its whole-live -simulation and checked against the chart facts and the master data. `--full` also writes the deck model's input: every +(read from the cue sheet's ACB, without decoding audio); the Gekisou catalog (member cards and snaps with their +characters, bands and Gekisou skills and support skills: missions, highest levels, names and descriptions); per +difficulty the chart facts (level and display level, full combo count, note counts, BPM, note times, the live's music +length, skill event times, fever ranges) and the chart's deck statistics: the no-skill score and the weight of every +score-up skill kind at every performance position, with Gekisou on (a Gekisou live at rank 1, with what every other +rank and the Perfect play need) and off (a solo live), measured by the deck model ournotes-deck (built into nnnotes as +`nnnotes._deck`) on its whole-live simulation and checked against the chart facts and the master data. `--full` also writes the deck model's input: every `MasterLiveMusicScore` row's chart as the client builds it at runtime (notes, skill events, fever ranges) and the master data tables about cards, skills, bonuses, scores and events. `--no-deck` skips the deck model (every chart's `deck` is null); `--seeds N` (default 8) and `--workers N` (default: every processor) set its seeds on charts with a diff --git a/docs/music-data.md b/docs/music-data.md index 2294e41..97f3611 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -2,10 +2,11 @@ `nnnotes music-data` writes one JSON file with every live song and chart of one master data version: what a song listing shows (titles and credits in every language, bands, vocal characters, category, tags, release time, score -ranks, jacket, BGM length), per difficulty the chart facts (level, note counts, BPM, chart times, skill events, -fevers), and per chart its **deck statistics**: what the chart contributes to the live score whatever the deck, in a -solo live (Gekisou off) and in a Gekisou live at every rank, measured by the deck model [ournotes-deck](https://github.com/empty-sekai/ournotes-deck), which nnnotes carries as -its extension module `nnnotes._deck`. The format is `nnnotes.music-data/1`; its JSON Schema is +ranks, jacket, BGM length), the Gekisou catalog (member cards, snaps and their Gekisou skills), per difficulty the +chart facts (level, note counts, BPM, chart times, skill events, fevers), and per chart its **deck statistics**: what +the chart contributes to the live score whatever the deck, in a solo live (Gekisou off) and in a Gekisou live at every +rank, measured by the deck model [ournotes-deck](https://github.com/empty-sekai/ournotes-deck), which nnnotes carries +as its extension module `nnnotes._deck`. The format is `nnnotes.music-data/1`; its JSON Schema is [schema/music-data.schema.json](schema/music-data.schema.json). ``` @@ -45,10 +46,10 @@ is written gzip-compressed. The command prints a summary (`songs`, `charts`, `de The command writes the file only when every table, chart and cue sheet was read and every chart measured: a missing or mismatching master data file, a table without a column the file exports, a text id that `MasterText` does not have, a score id that `MasterLiveMusicScore` does not have, a missing or unreadable chart asset, a note id that -occurs twice in a chart, a cue sheet without the song's cue, (with `--jackets`) a missing jacket texture, a chart the -deck model cannot measure or whose check deck fails, or deck statistics that disagree with the chart facts or the -master data stops it -with exit status 1 and a line naming the input. The file is written through a temporary file and a rename. +occurs twice in a chart, a cue sheet without the song's cue, a member card or snap whose character, Gekisou (support) +skill or rank group the master data does not have, (with `--jackets`) a missing jacket texture, a chart the deck model +cannot measure or whose check deck fails, or deck statistics that disagree with the chart facts or the master data +stops it with exit status 1 and a line naming the input. The file is written through a temporary file and a rename. An installation without the extension module (a source checkout that was not built) runs only with `--no-deck`; the wheels on PyPI carry it. See [Building](#building). @@ -61,6 +62,7 @@ the wheels on PyPI carry it. See [Building](#building). "provenance": {"region": ..., "client": {...}, "catalog": {...}, "master": {...}, "exporter": {...}, "deck": {...}}, "languages": ["ja", "en", "zh-Hant", "zh-Hans", "ko"], "bands": [...], "characters": [...], "tags": [...], "categories": [...], + "gekisouCatalog": {"skills": [...], "supportSkills": [...], "members": [...], "snaps": [...]}, "deck": {"model": {...}, "kinds": [...]}, "songs": [{"id": 100001, ..., "charts": [{..., "deck": {...}}, ...]}, ...], "master": {...}, "charts": [...] @@ -88,7 +90,7 @@ A **text** is an object with one string per language of `languages` (`{"ja": ... | `catalog.sha256` | SHA-256 of the remote catalog file the charts were read with | | `master.source` | `api` (`--master-files`, `--decoded-master`: the region's files) or `embedded` (`--apk-master`) | | `master.version` | the `version` of the master data manifest | -| `master.tables..sha256` | SHA-256 of each table's file as served, before decoding: the song tables (`MasterLiveMusic`, `MasterLiveMusicScore`, `MasterText`, `MasterBand`, `MasterCharacter`, `MasterTag`, `MasterLiveMusicCategory`, `MasterSound`, `MasterSoundCueSheet`, `MasterLiveScoreRank`) and, when the deck model runs or with `--full`, the tables of [the deck input](#the-deck-input---full) | +| `master.tables.
.sha256` | SHA-256 of each table's file as served, before decoding: the song tables (`MasterLiveMusic`, `MasterLiveMusicScore`, `MasterText`, `MasterBand`, `MasterCharacter`, `MasterTag`, `MasterLiveMusicCategory`, `MasterSound`, `MasterSoundCueSheet`, `MasterLiveScoreRank`), the tables of the Gekisou catalog (`MasterMemberCard`, `MasterSupportCard`, `MasterSupportCardRank`, `MasterGekisouSkill`, `MasterGekisouSkillEffect`, `MasterGekisouSupportSkill`, `MasterGekisouSupportSkillEffect`) and, when the deck model runs or with `--full`, the tables of [the deck input](#the-deck-input---full) | | `exporter.name`, `exporter.version` | `nnnotes` and its version | | `exporter.chartFormat` | the format of the chart converter the notes come from, `nnnotes.live-score/1` | | `deck` | the deck model: `{name, version, source, commit, format}`, `ournotes-deck`, its package version, repository, the git commit nnnotes is built with and the statistics format (`ournotes-deck.chart-stats/2`); null with `--no-deck` | @@ -102,6 +104,24 @@ A **text** is an object with one string per language of `languages` (`{"ja": ... | `tags[].id`, `.name` | `MasterTag` (the ids of `bestMusicTagIds`) | | `categories[].id`, `.musicCategories`, `.name` | `MasterLiveMusicCategory` (the listing's category tabs; `musicCategories` are the song category values it shows) | +### gekisouCatalog + +The Gekisou skills a deck brings to a Gekisou live and the cards that carry them, from the master data (written with +`--no-deck` too): the member cards' Gekisou skills, the snaps' Gekisou support skills, the member cards and the +snaps, every list sorted by `id`. Texts as above; `description` is the master data's format text with its +placeholders (`{effects[0].value}`, ...), which a page fills from the effect rows or leaves out. + +| Field | Content | +|---|---| +| `skills[]` | `MasterGekisouSkill`, the member cards' Gekisou skills: `id`, `mission` (`_gekisouMissionType`: 1 combo, 2 luck, 3 Just count, 4 every mission), `maxLevel` (the highest `_level` of its `MasterGekisouSkillEffect` rows, 0 without one), `name` (`_nameTextID`), `description` (`_descriptionTextFormatID`) | +| `supportSkills[]` | `MasterGekisouSupportSkill`, the snaps' Gekisou support skills: the same fields (`MasterGekisouSupportSkillEffect` for `maxLevel`) | +| `members[]` | `MasterMemberCard`: `id`, `characterId`, `bandId` (the character's `MasterCharacter._bandID`), `rarity`, `gekisouSkillId` (`_gekisouSkillID`, null for 0), `name` (`_nameTextID`), `subtitle` (`_subtitleTextID`, the card's title) | +| `snaps[]` | `MasterSupportCard`: `id`, `characterIds`, `rarity`, `gekisouSupportSkillIds` (`_gekisouSupportSkillId01`, `_gekisouSupportSkillId02` that are not 0), `supportSkillLevel` (their level at the snap's highest rank: `_gekisouSupportSkill01Level` / `02Level` of the `MasterSupportCardRank` row of its rank group with the highest `_rank`), `name` (`_nameTextID`), `subtitle` (`_descriptionTextID`, the snap's title) | + +A member card's character, Gekisou skill and a snap's Gekisou support skills must be in their tables, and a snap's +rank group must have a row; a snap whose two Gekisou support skills have different levels at its highest rank stops +the command (the catalog has one level per snap). + ### songs One entry per `MasterLiveMusic` row, sorted by `id`. @@ -179,7 +199,10 @@ skills) are not linear in the chart alone and have no weights: a deck's score wi A luck range draws lottery results from the play's random seed, so the Gekisou on measurements are given **per seed**: one seed (0) when no range is a luck range, else the first `--seeds` seeds of the deck model's published seed -set. The seed set is not the game's seed law (which is unknown); a mean over it is not the game's expectation. +set. That set is a fixed sequence: candidate `k` is the low 32 bits (signed) of output `k + 1` of SplitMix64 started at +`0x6765_6B69_736F_7531` (`gekisou1`), kept when its pair of stream seeds (`|b|` and `|b ^ 0x9E3779B9|`, -2^31 as +2^31 - 1) differs from those of every seed kept before it, so a smaller set is a prefix of a larger one. The seed +set is not the game's seed law (which is unknown); a mean over it is not the game's expectation. ### Ranks @@ -228,7 +251,7 @@ fields; `values` lists what the master data uses. | `positions` | the performance positions the events fire (the largest position + 1): the length of every `weights[kind]` | | `ranges[]` | the Gekisou ranges: `index`, `mission` (1 combo, 2 luck, 3 Just count), `startMs`, `endMs`, `rankBonusPercents` (the rank bonus percentages of ranks 1..5 of the song's mission pattern, `MasterLiveGekisouRankingScoreBonus`), `rankBonusPercent` (the rank 1 percentage, `rankBonusPercents[0]`) | | `justNotes` | notes judged Just on the Gekisou on play | -| `seeds[]` | Gekisou on, per seed: `seed`; `score` (points at `model.power`, rank 1 bonuses included); `ranges[]` (`rangeScore`: the points gained inside the range, `rankBonus`: its rank 1 bonus in points, `maxCombo`, `justCount`, `lotResults`: lottery results Miss, Hit, Super Hit, Critical, `rangeScorePerfect`: `rangeScore` on the Perfect play); `weights[kind][position]` (points per unit of deck power and of factor); `check` (`deck`: `[kind, value]` or null per position, `exact`, `predicted`, `bound`: points at `model.checkPower`); `scorePerfect` (`score` on the Perfect play); `rangeWeights[kind][position][range]` (range points per unit of deck power and of factor, or null; a kind null); `rankCheck` (`ranks`: 1..5 per range, `exact`, `predicted`, `bound`; null without ranges or range weights) | +| `seeds[]` | Gekisou on, per seed: `seed`; `score` (points at `model.power`, rank 1 bonuses included); `ranges[]` (`rangeScore`: the points gained inside the range, `rankBonus`: its rank 1 bonus in points, `maxCombo`, `justCount`, `luckPoints`: the luck points gained (`TotalBonusPoint`; a combo, Just count or luck range ranks the room by `maxCombo`, `justCount` or `luckPoints`), `lotResults`: lottery results Miss, Hit, Super Hit, Critical, `rangeScorePerfect`: `rangeScore` on the Perfect play); `weights[kind][position]` (points per unit of deck power and of factor); `check` (`deck`: `[kind, value]` or null per position, `exact`, `predicted`, `bound`: points at `model.checkPower`); `scorePerfect` (`score` on the Perfect play); `rangeWeights[kind][position][range]` (range points per unit of deck power and of factor, or null; a kind null); `rankCheck` (`ranks`: 1..5 per range, `exact`, `predicted`, `bound`; null without ranges or range weights) | | `offSeeds[]` | Gekisou off, one seed: `seed` (0), `score`, `weights[kind][position]` (a kind null when its conditions read the Gekisou state, which a solo live does not have) and `check`, as in `seeds[]` | | `unplayable` | null, or why the game cannot play the chart with Gekisou (more than three fevers: the game fails when the fourth starts); `seeds` is then empty, `offSeeds` is not | @@ -237,8 +260,10 @@ note time, music length, Gekisou missions, skill event times and fever ranges mu `deck`. Its numbers are checked against the master data and themselves: every range's `rankBonusPercents` are the `MasterLiveGekisouRankingScoreBonus` rows of the song's mission pattern (0 without a row), every `rankBonus` is `trunc(rangeScore * rankBonusPercent / 100)`, a chart without Just notes has the same scores on the Perfect play, -every array has its shape (`[kind][position]`, `[kind][position][range]`, one range result per range, one Gekisou off -seed), and every check and rank check is within its bound. +every array has its shape (`[kind][position]`, `[kind][position][range]`, one range result per range with its luck +points, one Gekisou off seed), the seeds are the chart's seed set (none on a chart unplayable with Gekisou, seed 0 +without a luck range, else the first `--seeds` seeds of the published seed set, which nnnotes computes itself), and +every check and rank check is within its bound. ## The deck input (`--full`) diff --git a/docs/schema/music-data.schema.json b/docs/schema/music-data.schema.json index dad6a10..51f7e64 100644 --- a/docs/schema/music-data.schema.json +++ b/docs/schema/music-data.schema.json @@ -280,6 +280,133 @@ } } }, + "gekisouCatalog": { + "type": "object", + "description": "the Gekisou skills a deck brings to a Gekisou live and the cards that carry them, every list sorted by id (docs/music-data.md, gekisouCatalog)", + "required": [ + "skills", + "supportSkills", + "members", + "snaps" + ], + "properties": { + "skills": { + "type": "array", + "description": "MasterGekisouSkill: the member cards' Gekisou skills", + "items": { + "$ref": "#/$defs/gekisouSkill" + } + }, + "supportSkills": { + "type": "array", + "description": "MasterGekisouSupportSkill: the snaps' Gekisou support skills", + "items": { + "$ref": "#/$defs/gekisouSkill" + } + }, + "members": { + "type": "array", + "description": "MasterMemberCard", + "items": { + "type": "object", + "required": [ + "id", + "characterId", + "bandId", + "rarity", + "gekisouSkillId", + "name", + "subtitle" + ], + "properties": { + "id": { + "type": "integer" + }, + "characterId": { + "type": "integer" + }, + "bandId": { + "type": [ + "integer", + "null" + ], + "description": "the character's band" + }, + "rarity": { + "type": [ + "integer", + "null" + ] + }, + "gekisouSkillId": { + "type": [ + "integer", + "null" + ], + "description": "_gekisouSkillID (a skills[] id); null for 0" + }, + "name": { + "$ref": "#/$defs/text", + "description": "_nameTextID" + }, + "subtitle": { + "$ref": "#/$defs/text", + "description": "_subtitleTextID: the card's title" + } + } + } + }, + "snaps": { + "type": "array", + "description": "MasterSupportCard", + "items": { + "type": "object", + "required": [ + "id", + "characterIds", + "rarity", + "gekisouSupportSkillIds", + "supportSkillLevel", + "name", + "subtitle" + ], + "properties": { + "id": { + "type": "integer" + }, + "characterIds": { + "$ref": "#/$defs/ints" + }, + "rarity": { + "type": [ + "integer", + "null" + ] + }, + "gekisouSupportSkillIds": { + "$ref": "#/$defs/ints", + "description": "_gekisouSupportSkillId01 / 02 that are not 0 (supportSkills[] ids)" + }, + "supportSkillLevel": { + "type": [ + "integer", + "null" + ], + "description": "the level of its Gekisou support skills at the snap's highest rank (MasterSupportCardRank)" + }, + "name": { + "$ref": "#/$defs/text", + "description": "_nameTextID" + }, + "subtitle": { + "$ref": "#/$defs/text", + "description": "_descriptionTextID: the snap's title" + } + } + } + } + } + }, "deck": { "type": [ "object", @@ -477,6 +604,41 @@ ] } }, + "gekisouSkill": { + "type": "object", + "required": [ + "id", + "mission", + "maxLevel", + "name", + "description" + ], + "properties": { + "id": { + "type": "integer" + }, + "mission": { + "type": [ + "integer", + "null" + ], + "description": "_gekisouMissionType: 1 combo, 2 luck, 3 Just count, 4 every mission" + }, + "maxLevel": { + "type": "integer", + "minimum": 0, + "description": "the highest _level of the skill's effect rows, 0 without one" + }, + "name": { + "$ref": "#/$defs/text", + "description": "_nameTextID" + }, + "description": { + "$ref": "#/$defs/text", + "description": "_descriptionTextFormatID: the format text with its placeholders" + } + } + }, "deckChart": { "type": [ "object", @@ -607,6 +769,10 @@ "justCount": { "type": "integer" }, + "luckPoints": { + "type": "integer", + "description": "the luck points gained in the range (TotalBonusPoint), a luck range's rank figure" + }, "lotResults": { "type": "array", "items": { diff --git a/src/nnnotes/musicdata.py b/src/nnnotes/musicdata.py index 24157dd..f6d54ef 100644 --- a/src/nnnotes/musicdata.py +++ b/src/nnnotes/musicdata.py @@ -1,18 +1,19 @@ """Music data: one JSON file with every live song and chart of one master data version (format `nnnotes.music-data/1`, docs/music-data.md): titles and credits in every language, bands, vocal characters, category, -tags, release time, score ranks, the live BGM's length, per difficulty the chart facts (level, note counts, BPM, chart -times, skill events, fever ranges) and the chart's deck statistics, what the chart contributes to the live score -whatever the deck, with Gekisou on (a Gekisou live, every rank) and off (a solo live), measured by the deck model -ournotes-deck (the extension module nnnotes._deck). +tags, release time, score ranks, the live BGM's length, the Gekisou catalog (member cards, snaps and their Gekisou +skills), per difficulty the chart facts (level, note counts, BPM, chart times, skill events, fever ranges) and the +chart's deck statistics, what the chart contributes to the live score whatever the deck, with Gekisou on (a Gekisou +live, every rank) and off (a solo live), measured by the deck model ournotes-deck (the extension module +nnnotes._deck). Master data is read from the files as served (deckdata.master_files / apk_master: SHA-256 checked against the manifest, decoded with master.decode). Charts are the TextAssets `Live/MusicScore/` converted by score.runtime_score; the BGM length is read from the cue sheet's ACB (cue `Length` and the stream's sample count), without decoding audio. The deck model reads the deck input (deckdata.build: the charts' runtime notes and the master -data tables it needs) in memory; its statistics are checked against the chart facts. With `full` the file also -carries that deck input (`master`, `charts`), every chart's runtime notes and the tables. With a jackets directory, -every song's jacket (the Texture2D `Image/Jacket/`) is written there as `.webp`, scaled to at most -JACKET_SIZE pixels on its longer side. +data tables it needs) in memory; its statistics are checked against the chart facts and the master data. With `full` +the file also carries that deck input (`master`, `charts`), every chart's runtime notes and the tables. With a jackets +directory, every song's jacket (the Texture2D `Image/Jacket/`) is written there as `.webp`, scaled to +at most JACKET_SIZE pixels on its longer side. The output is canonical (deckdata.encode): minified UTF-8 with one trailing LF, keys in a fixed order, songs sorted by id, master data floats as the shortest decimal of their binary32 value, the deck model's numbers as it writes them. @@ -38,6 +39,9 @@ # the tables of the song metadata; the deck model's are deckdata.TABLES SONG_TABLES = ("MasterLiveMusic", "MasterLiveMusicScore", "MasterText", "MasterBand", "MasterCharacter", "MasterTag", "MasterLiveMusicCategory", "MasterSound", "MasterSoundCueSheet", "MasterLiveScoreRank") +# the tables of the Gekisou catalog (gekisou_catalog): member cards, snaps and their Gekisou (support) skills +CATALOG_TABLES = ("MasterMemberCard", "MasterSupportCard", "MasterSupportCardRank", "MasterGekisouSkill", + "MasterGekisouSkillEffect", "MasterGekisouSupportSkill", "MasterGekisouSupportSkillEffect") SCORE_RANKS = {1: "E", 2: "D", 3: "C", 4: "B", 5: "A", 6: "S", 7: "SS"} # LiveScoreRank @@ -221,6 +225,78 @@ def _by_id(rows: list[dict], table: str) -> dict: return out +# ---------------------------------------------------------------- the Gekisou catalog +MISSION_LUCK = 2 # GekisouMissionType: 1 combo, 2 luck, 3 Just count, 4 all + + +def _max_levels(rows: list[dict], key: str) -> dict: + """skill id -> the highest `_level` of its effect rows.""" + out: dict = {} + for r in rows: + out[r.get(key)] = max(out.get(r.get(key), 0), r.get("_level") or 0) + return out + + +def gekisou_catalog(tables: dict[str, list[dict]], text: Texts) -> dict: + """The Gekisou catalog `{skills, supportSkills, members, snaps}`, each sorted by id: the Gekisou skills + (MasterGekisouSkill) and Gekisou support skills (MasterGekisouSupportSkill) with their mission and highest level + (the highest `_level` of their effect rows, 0 without one), the member cards with their character, band and + Gekisou skill, the snaps (MasterSupportCard) with their characters, Gekisou support skills and the level those have + at the snap's highest rank (MasterSupportCardRank of its rank group).""" + characters = _by_id(tables["MasterCharacter"], "MasterCharacter") + skill_rows = _by_id(tables["MasterGekisouSkill"], "MasterGekisouSkill") + support_rows = _by_id(tables["MasterGekisouSupportSkill"], "MasterGekisouSupportSkill") + levels = _max_levels(tables["MasterGekisouSkillEffect"], "_gekisouSkillID") + support_levels = _max_levels(tables["MasterGekisouSupportSkillEffect"], "_gekisouSupportSkillID") + top: dict = {} # rank group -> its highest rank row (a later row on a tie) + for r in tables["MasterSupportCardRank"]: + g = r.get("_group") + if g not in top or (r.get("_rank") or 0) >= (top[g].get("_rank") or 0): + top[g] = r + + def skill(r: dict, max_level: dict) -> dict: + return {"id": r["_id"], "mission": r.get("_gekisouMissionType"), "maxLevel": max_level.get(r["_id"], 0), + "name": text.get(r.get("_nameTextID")), "description": text.get(r.get("_descriptionTextFormatID"))} + + members = [] + for c in sorted(_by_id(tables["MasterMemberCard"], "MasterMemberCard").values(), key=lambda r: r["_id"]): + where = f"MasterMemberCard {c['_id']}" + ch = characters.get(c.get("_characterID")) + if ch is None: + raise MusicDataError(f"{where}: character {c.get('_characterID')} is not in MasterCharacter") + sid = c.get("_gekisouSkillID") or None + if sid is not None and sid not in skill_rows: + raise MusicDataError(f"{where}: Gekisou skill {sid} is not in MasterGekisouSkill") + members.append({"id": c["_id"], "characterId": c.get("_characterID"), "bandId": ch.get("_bandID"), + "rarity": c.get("_rarity"), "gekisouSkillId": sid, "name": text.get(c.get("_nameTextID")), + "subtitle": text.get(c.get("_subtitleTextID"))}) + snaps = [] + for s in sorted(_by_id(tables["MasterSupportCard"], "MasterSupportCard").values(), key=lambda r: r["_id"]): + where = f"MasterSupportCard {s['_id']}" + rank = top.get(s.get("_supportCardRankGroup")) + if rank is None: + raise MusicDataError(f"{where}: rank group {s.get('_supportCardRankGroup')} has no MasterSupportCardRank " + f"row") + ids, at = [], [] + for n in (1, 2): + i = s.get(f"_gekisouSupportSkillId0{n}") + if i: + if i not in support_rows: + raise MusicDataError(f"{where}: Gekisou support skill {i} is not in MasterGekisouSupportSkill") + ids.append(i) + at.append(rank.get(f"_gekisouSupportSkill0{n}Level")) + if len(set(at)) > 1: + raise MusicDataError(f"{where}: its Gekisou support skills {ids} have the levels {at} at the highest " + f"rank; the catalog has one level per snap") + snaps.append({"id": s["_id"], "characterIds": list(s.get("_characterIDs") or []), "rarity": s.get("_rarity"), + "gekisouSupportSkillIds": ids, + "supportSkillLevel": at[0] if at else rank.get("_gekisouSupportSkill01Level"), + "name": text.get(s.get("_nameTextID")), "subtitle": text.get(s.get("_descriptionTextID"))}) + return {"skills": [skill(r, levels) for r in sorted(skill_rows.values(), key=lambda r: r["_id"])], + "supportSkills": [skill(r, support_levels) for r in sorted(support_rows.values(), key=lambda r: r["_id"])], + "members": members, "snaps": snaps} + + # ---------------------------------------------------------------- the deck model DECK_SEEDS = 8 # seed set size of a chart with a luck range (ournotes-deck's default) @@ -291,23 +367,76 @@ def _trunc_percent(score: int, percent: int) -> int: return p // 100 if p >= 0 else -(-p // 100) +def _int(v) -> bool: + return isinstance(v, int) and not isinstance(v, bool) + + def _numbers(v, n: int) -> bool: """Whether v is a list of n numbers (as the deck model writes them).""" return isinstance(v, list) and len(v) == n and all( isinstance(x, (int, deckdata._Num)) and not isinstance(x, bool) for x in v) +def _check_bound(c, what: str, where: str) -> None: + """A check (or rank check) of the deck model: its exact score within `bound` of the predicted one.""" + if not isinstance(c, dict): + raise MusicDataError(f"{where}: {what}: no check") + if abs(c["exact"] - float(c["predicted"])) > float(c["bound"]): + raise MusicDataError(f"{where}: {what} scores {c['exact']}, predicted {c['predicted']} beyond the bound " + f"{c['bound']}") + + +def _check_rank_check(rc: dict, n: int, s: str, where: str) -> None: + if len(rc["ranks"]) != n or not all(1 <= r <= RANKS for r in rc["ranks"]): + raise MusicDataError(f"{where}: {s}: rank check ranks {rc['ranks']!r}") + _check_bound(rc, f"{s}: the check deck at ranks {rc['ranks']!r}", where) + + +# ournotes-deck's published seed set (live::seeds): candidate k is the low 32 bits (a signed integer) of output k + 1 +# of SplitMix64 started at SEED_ORIGIN (the ASCII bytes of `gekisou1`); a candidate is kept when its pair of effective +# stream seeds (|b| and |b ^ 0x9E3779B9|, -2^31 taken as 2^31 - 1) differs from that of every seed kept before it +SEED_ORIGIN = 0x6765_6B69_736F_7531 +_GAMMA, _M64, _LUCK_XOR = 0x9E37_79B9_7F4A_7C15, (1 << 64) - 1, 0x9E37_79B9 + + +def _i32(x: int) -> int: + x &= 0xFFFF_FFFF + return x - (1 << 32) if x >> 31 else x + + +def published_seeds(n: int) -> list[int]: + """The first n seeds of ournotes-deck's published seed set (a smaller set is a prefix of a larger one).""" + def mix(z: int) -> int: + z = ((z ^ (z >> 30)) * 0xBF58_476D_1CE4_E5B9) & _M64 + z = ((z ^ (z >> 27)) * 0x94D0_49BB_1331_11EB) & _M64 + return z ^ (z >> 31) + + def eff(x: int) -> int: + return (1 << 31) - 1 if x == -(1 << 31) else abs(x) + keys, out, k = set(), [], 0 + while len(out) < n: + b = _i32(mix((SEED_ORIGIN + _GAMMA * (k + 1)) & _M64)) + key = (eff(b), eff(_i32(b ^ _LUCK_XOR))) + if key not in keys: + keys.add(key) + out.append(b) + k += 1 + return out + + +def chart_seeds(stats: dict, seeds: int) -> list[int]: + """The seeds of a chart's Gekisou on statistics: none when it is unplayable with Gekisou, seed 0 when no range is + a luck range (the play draws nothing), else the first `seeds` published seeds.""" + if stats.get("unplayable") is not None: + return [] + return published_seeds(seeds) if any(r["mission"] == MISSION_LUCK for r in stats["ranges"]) else [0] + + def _check_seed_shapes(stats: dict, kinds: int, where: str) -> None: """The array shapes and the checks of a chart's statistics: `weights[kind][position]` (a kind null only with Gekisou off), `rangeWeights[kind][position][range]` (null, or a kind null), one Gekisou off seed, the seeds' - ranges one per range, and every check (and rank check) within its bound.""" + ranges one per range with their luck points, and every check (and rank check) within its bound.""" positions, n = stats["positions"], len(stats["ranges"]) - - def check(c, what): - if abs(c["exact"] - float(c["predicted"])) > float(c["bound"]): - raise MusicDataError(f"{where}: {what} scores {c['exact']}, predicted {c['predicted']} beyond the bound " - f"{c['bound']}") - for seed in stats["seeds"]: s = f"seed {seed['seed']}" w = seed.get("weights") @@ -315,17 +444,18 @@ def check(c, what): raise MusicDataError(f"{where}: {s}: weights are not [kind][position]") if len(seed["ranges"]) != n: raise MusicDataError(f"{where}: {s}: {len(seed['ranges'])} range results for {n} ranges") + for i, r in enumerate(seed["ranges"]): + if not _int(r.get("luckPoints")): + raise MusicDataError(f"{where}: {s} range {i}: no luck points (luckPoints)") rw = seed.get("rangeWeights") if rw is not None and not (isinstance(rw, list) and len(rw) == kinds and all( k is None or (isinstance(k, list) and len(k) == positions and all(_numbers(x, n) for x in k)) for k in rw)): raise MusicDataError(f"{where}: {s}: rangeWeights are not [kind][position][range]") - check(seed["check"], f"{s}: the check deck") + _check_bound(seed["check"], f"{s}: the check deck", where) rc = seed.get("rankCheck") if rc is not None: - if len(rc["ranks"]) != n or not all(1 <= r <= RANKS for r in rc["ranks"]): - raise MusicDataError(f"{where}: {s}: rank check ranks {rc['ranks']!r}") - check(rc, f"{s}: the check deck at ranks {rc['ranks']!r}") + _check_rank_check(rc, n, s, where) off = stats.get("offSeeds") if not isinstance(off, list) or len(off) != 1: raise MusicDataError(f"{where}: the deck model gives no Gekisou off statistics (offSeeds)") @@ -333,12 +463,14 @@ def check(c, what): w = seed.get("weights") if not (isinstance(w, list) and len(w) == kinds and all(x is None or _numbers(x, positions) for x in w)): raise MusicDataError(f"{where}: Gekisou off: weights are not [kind][position]") - check(seed["check"], "Gekisou off: the check deck") + _check_bound(seed["check"], "Gekisou off: the check deck", where) -def chart_deck(song: dict, chart: dict, stats: dict, kinds: int, percents: list[list]) -> dict: - """A chart's `deck` from its statistics, after checking them against the song, the chart facts and the song's - rank bonus percentages (`percents`: rank_bonus_percents), `kinds` the number of score-up kinds.""" +def chart_deck(song: dict, chart: dict, stats: dict, kinds: int, percents: list[list], + seeds: int = DECK_SEEDS) -> dict: + """A chart's `deck` from its statistics, after checking them against the song, the chart facts, the song's rank + bonus percentages (`percents`: rank_bonus_percents) and the seed set (chart_seeds: `seeds` seeds with a luck + range), `kinds` the number of score-up kinds.""" where = f"chart {chart['scoreId']} ({song['id']} {chart['difficulty']})" checks = ( ("music id", stats["musicId"], song["id"]), @@ -361,6 +493,10 @@ def chart_deck(song: dict, chart: dict, stats: dict, kinds: int, percents: list[ if got != want or r["rankBonusPercent"] != want[0]: raise MusicDataError(f"{where}: range {i}: the deck model's rank bonus percentages {got!r} " f"({r['rankBonusPercent']!r}) differ from MasterLiveGekisouRankingScoreBonus {want!r}") + want = chart_seeds(stats, seeds) + if [s["seed"] for s in stats["seeds"]] != want: + raise MusicDataError(f"{where}: seeds {[s['seed'] for s in stats['seeds']]} are not the chart's seed set " + f"{want}") _check_seed_shapes(stats, kinds, where) for seed in stats["seeds"]: for i, (r, info) in enumerate(zip(seed["ranges"], ranges)): @@ -417,6 +553,7 @@ def raw_chart(row: dict) -> tuple[str, bytes]: # (key, bytes); each chart a categories = [{"id": c["_id"], "musicCategories": list(c.get("_musicCategories") or []), "name": text.get(c.get("_textKey"))} for c in sorted(tables["MasterLiveMusicCategory"], key=lambda r: r["_id"])] + gk_catalog = gekisou_catalog(tables, text) songs = [] for m in musics: @@ -487,7 +624,7 @@ def record(sid: int) -> dict: for s in songs: percents = rank_bonus_percents(bonus_rows, s["gekisouMissions"]) for c in s["charts"]: - c["deck"] = chart_deck(s, c, by_score[c["scoreId"]], len(stats["kinds"]), percents) + c["deck"] = chart_deck(s, c, by_score[c["scoreId"]], len(stats["kinds"]), percents, deck.seeds) deck_doc = {"model": stats["model"], "kinds": stats["kinds"]} read = [t for t in tables_of(deck is not None, full) if t in tables] @@ -507,6 +644,7 @@ def record(sid: int) -> dict: "characters": characters, "tags": tags, "categories": categories, + "gekisouCatalog": gk_catalog, "deck": deck_doc, "songs": songs, } @@ -517,10 +655,12 @@ def record(sid: int) -> dict: def tables_of(deck: bool, full: bool) -> tuple[str, ...]: - """The master data tables read: SONG_TABLES, and deckdata.TABLES with the deck model or `full`.""" + """The master data tables read: SONG_TABLES and CATALOG_TABLES, and deckdata.TABLES with the deck model or + `full`.""" + base = SONG_TABLES + tuple(t for t in CATALOG_TABLES if t not in SONG_TABLES) if not (deck or full): - return SONG_TABLES - return SONG_TABLES + tuple(t for t, _ in deckdata.TABLES if t not in SONG_TABLES) + return base + return base + tuple(t for t, _ in deckdata.TABLES if t not in base) def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes], diff --git a/tests/test_musicdata.py b/tests/test_musicdata.py index 8735be7..c434d16 100644 --- a/tests/test_musicdata.py +++ b/tests/test_musicdata.py @@ -35,6 +35,60 @@ def text(tid, stem): "_extra": [7]} MUSIC1 = dict(MUSIC, _id=100001, _sortOrder=1, _titleTextID="T1", _bandNameTextID="BN", _easyID=10, _expertID=0, _liveScoreRankGroup=7) + + +def columns(table, **values): + """A row of a deck input table: every column 0 (an id list empty), then `values`.""" + cols = dict(deckdata.TABLES)[table] + return {**{c: [] if c.endswith(("IDs", "Ids", "Categories")) else 0 for c in cols}, **values} + + +def member_card(i, character, skill, **values): + return columns("MasterMemberCard", _id=i, _characterID=character, _rarity=3, _cardType=1, _bestMusicTagIDs=[1], + _gekisouSkillID=skill, _nameTextID=f"Card{i}", _subtitleTextID=f"Sub{i}" if i <= 2 else "", + **values) + + +def snap(i, character, support, **values): + return columns("MasterSupportCard", _id=i, _characterIDs=[character], _rarity=2, _supportCardRankGroup=1, + _gekisouSupportSkillId01=support, _nameTextID=f"Snap{i}", + _descriptionTextID="SnapSub41" if i == 41 else "", **values) + + +def effect(table, key, i, skill, level, group=0): + return columns(table, _id=i, **{key: skill}, _level=level, _skillConditionGroup=group, _skillEffectType=2000, + _effectValue=100 * level) + + +# Gekisou skills 1 (combo, levels 1..5), 2 (Just count, 1..3), 3 (luck, 1..5); Gekisou support skills 31 (Just count, +# two rows per level: a member of band 1 or not) and 32 (combo); member cards 1..8 (card 6 without a Gekisou skill); +# snaps 41..44 (43 without a Gekisou support skill), highest rank 5 +GEKISOU_ROWS = { + "MasterMemberCard": [member_card(1, 1, 1), member_card(2, 2, 2), member_card(3, 3, 1), member_card(4, 4, 3), + member_card(5, 5, 2), member_card(6, 6, 0), member_card(7, 3, 3), member_card(8, 1, 2)], + "MasterGekisouSkill": [ + {"_id": 1, "_nameTextID": "GS1", "_descriptionTextFormatID": "GSD1", "_gekisouMissionType": 1, + "_skillCategories": [1]}, + {"_id": 2, "_nameTextID": "GS2", "_descriptionTextFormatID": "GSD2", "_gekisouMissionType": 3, + "_skillCategories": [2]}, + {"_id": 3, "_nameTextID": "GS3", "_descriptionTextFormatID": "", "_gekisouMissionType": 2, + "_skillCategories": [3]}], + "MasterGekisouSkillEffect": [ + effect("MasterGekisouSkillEffect", "_gekisouSkillID", 10 * s + lv, s, lv) + for s, top in ((1, 5), (2, 3), (3, 5)) for lv in range(1, top + 1)], + "MasterGekisouSupportSkill": [ + {"_id": 31, "_nameTextID": "GS31", "_descriptionTextFormatID": "GSD31", "_gekisouSupportSkillExecTiming": 1, + "_gekisouMissionType": 3}, + {"_id": 32, "_nameTextID": "GS32", "_descriptionTextFormatID": "", "_gekisouSupportSkillExecTiming": 1, + "_gekisouMissionType": 1}], + "MasterGekisouSupportSkillEffect": [ + effect("MasterGekisouSupportSkillEffect", "_gekisouSupportSkillID", 100 * g + 10 * s + lv, s, lv, g) + for s, groups in ((31, (100, 101)), (32, (0,))) for lv in range(1, 6) for g in groups], + "MasterSupportCard": [snap(41, 1, 31), snap(42, 2, 32), snap(43, 3, 0), snap(44, 4, 31)], + "MasterSupportCardRank": [ + columns("MasterSupportCardRank", _id=r, _group=1, _rank=r, _gekisouSupportSkill01Level=r, + _gekisouSupportSkill02Level=r) for r in range(1, 6)], +} TABLE_ROWS = { "MasterLiveMusic": [MUSIC, MUSIC1], "MasterLiveMusicScore": [ @@ -47,11 +101,21 @@ def text(tid, stem): {"_id": 40, "_musicScoreTextFileName": "c/c_04", "_musicScoreLevel": 1, "_fullComboCount": 2, "_musicScoreDisplayLevel": 1.0}], # a chart of no song "MasterText": [text("T1", "one"), text("T2", "two"), text("P2", "pho"), text("L2", "lyr"), text("C2", "com"), - text("BN", "crychic"), text("Band1", "mygo"), text("Ch1", "tomori"), text("Ch1s", "tomo"), - text("Tag1", "tag"), text("Cat1", "original")], - "MasterBand": [{"_id": 1, "_nameTextID": "Band1", "_mainColorCode": "#3388BB", "_subColorCode": "#FFFFFF"}], + text("BN", "crychic"), text("Band1", "mygo"), text("Band2", "mujica"), text("Ch1", "tomori"), + text("Ch1s", "tomo"), text("Tag1", "tag"), text("Cat1", "original")] + + [text(f"Ch{i}", f"char{i}") for i in range(2, 7)] + + [text(f"Card{i}", f"card{i}") for i in range(1, 9)] + [text(f"Sub{i}", f"sub{i}") for i in (1, 2)] + + [text(f"Snap{i}", f"snap{i}") for i in (41, 42, 43, 44)] + [text("SnapSub41", "snapsub41")] + + [text(f"GS{i}", f"gs{i}") for i in (1, 2, 3, 31, 32)] + [text(f"GSD{i}", f"gsd{i}") + for i in (1, 2, 31)], + "MasterBand": [{"_id": 1, "_nameTextID": "Band1", "_mainColorCode": "#3388BB", "_subColorCode": "#FFFFFF"}, + {"_id": 2, "_nameTextID": "Band2", "_mainColorCode": "#881122", "_subColorCode": "#000000"}], + # characters 1..4 of band 1, 5 and 6 of band 2 "MasterCharacter": [{"_id": 1, "_nameTextID": "Ch1", "_shortNameTextID": "Ch1s", "_bandID": 1, - "_mainColorCode": "#77BBDD"}], + "_mainColorCode": "#77BBDD"}] + + [{"_id": i, "_nameTextID": f"Ch{i}", "_shortNameTextID": "", "_bandID": 1 if i <= 4 else 2, + "_mainColorCode": "#000000"} for i in range(2, 7)], + **GEKISOU_ROWS, "MasterTag": [{"_id": 1, "_nameTextID": "Tag1"}], "MasterLiveMusicCategory": [{"_id": 1, "_musicCategories": [1], "_textKey": "Cat1"}], "MasterSound": [{"_id": 13, "_soundCueSheetID": 5, "_cueName": "song2"}], @@ -108,13 +172,14 @@ def trunc_percent(score, percent): class FakeDeck: """The interface of nnnotes._deck: statistics made from the deck input as the model reports them.""" COMMIT = "7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6" + FORMAT = "ournotes-deck.chart-stats/2" def __init__(self, change=None, fail=None): self.change, self.fail, self.inputs = change, fail, [] def info(self): return {"name": "ournotes-deck", "version": "0.0.1", "source": "https://github.com/empty-sekai/ournotes-deck", - "commit": self.COMMIT, "dataFormat": deckdata.DECK_FORMAT, "format": "ournotes-deck.chart-stats/2"} + "commit": self.COMMIT, "dataFormat": deckdata.DECK_FORMAT, "format": self.FORMAT} def chart_stats(self, data, seeds, workers): if self.fail: @@ -141,9 +206,20 @@ def table(name): last = max(c["notes"]["timeMs"]) fevers = list(zip(c["fevers"]["startMs"], c["fevers"]["endMs"]))[:3] positions = min(len(c["skillEvents"]["timeMs"]), 5) - pattern = 3 if missions == [1, 3, 3] else 0 + pattern = musicdata.mission_pattern(missions) percents = [[bonus.get((pattern, i + 1, k), 0) for k in range(1, 6)] for i in range(len(fevers))] check = {"deck": [[0, 5000]], "exact": 2000, "predicted": 2000.25, "bound": 7.0} + + def seed_stats(seed): + return {"seed": seed, "score": 1234, + "ranges": [{"rangeScore": 101 * (i + 1), "rankBonus": trunc_percent(101 * (i + 1), p[0]), + "maxCombo": 1, "justCount": 0, "luckPoints": 0, "lotResults": [0, 0, 0, 0], + "rangeScorePerfect": 101 * (i + 1)} for i, p in enumerate(percents)], + "weights": [["W"] * positions], "check": dict(check), "scorePerfect": 1234, + "rangeWeights": [[["W"] * len(fevers)] * positions], + "rankCheck": {"ranks": [2] * len(fevers), "exact": 2001, "predicted": 2000.5, + "bound": 7.0} if fevers else None} + luck = musicdata.MISSION_LUCK in missions[:len(fevers)] s = {"scoreId": c["scoreId"], "musicId": music, "difficulty": d, "level": levels[c["scoreId"]], "judgedNotes": sum(score.is_judgement_note(op) for op in c["notes"]["op"]), "convertedNoteCount": len(c["notes"]["id"]), "lastNoteMs": last, "musicLengthMs": last + 1000, @@ -153,19 +229,12 @@ def table(name): "rankBonusPercent": percents[i][0], "rankBonusPercents": percents[i]} for i, (a, b) in enumerate(fevers)], "justNotes": 0, - "seeds": [{"seed": 0, "score": 1234, - "ranges": [{"rangeScore": 101 * (i + 1), "rankBonus": trunc_percent(101 * (i + 1), p[0]), - "maxCombo": 1, "justCount": 0, "lotResults": [0, 0, 0, 0], - "rangeScorePerfect": 101 * (i + 1)} for i, p in enumerate(percents)], - "weights": [["W"] * positions], "check": check, "scorePerfect": 1234, - "rangeWeights": [[["W"] * len(fevers)] * positions], - "rankCheck": {"ranks": [2] * len(fevers), "exact": 2001, "predicted": 2000.5, - "bound": 7.0} if fevers else None}], + "seeds": [seed_stats(x) for x in (musicdata.published_seeds(seeds) if luck else [0])], "offSeeds": [{"seed": 0, "score": 1000, "weights": [["W"] * positions], "check": dict(check)}]} if self.change: self.change(s) charts.append(s) - out = json.dumps({"format": "ournotes-deck.chart-stats/2", "source": {}, "model": {"power": 300000}, + out = json.dumps({"format": self.FORMAT, "source": {}, "model": {"power": 300000}, "kinds": [{"id": 0, "effectType": 2000, "activationTimeSecond": 7.5}], "charts": charts}) return out.replace('"W"', WEIGHT) @@ -191,17 +260,19 @@ def test_document(tmp_path): assert (r["songs"], r["charts"], r["deck"], r["full"]) == (2, 3, None, False) assert r["sha256"] == hashlib.sha256(raw).hexdigest() doc = json.loads(raw) - assert list(doc) == ["format", "provenance", "languages", "bands", "characters", "tags", "categories", "deck", - "songs"] + assert list(doc) == ["format", "provenance", "languages", "bands", "characters", "tags", "categories", + "gekisouCatalog", "deck", "songs"] assert doc["format"] == "nnnotes.music-data/1" and doc["languages"] == ["ja", "en", "zh-Hant", "zh-Hans", "ko"] p = doc["provenance"] assert list(p) == ["region", "client", "catalog", "master", "exporter", "deck"] and p["deck"] is None - assert list(p["master"]["tables"]) == list(musicdata.SONG_TABLES) # without the deck model: its tables only + # without the deck model: the song and the Gekisou catalog tables only + assert list(p["master"]["tables"]) == list(musicdata.SONG_TABLES + musicdata.CATALOG_TABLES) assert p["exporter"] == {"name": "nnnotes", "version": cli.__version__, "chartFormat": deckdata.CHART_FORMAT} assert doc["deck"] is None - assert doc["bands"] == [{"id": 1, "name": {"ja": "mygo-ja", "en": "mygo-en", "zh-Hant": "mygo-tw", - "zh-Hans": "mygo-cn", "ko": "mygo-ko"}, - "mainColor": "#3388BB", "subColor": "#FFFFFF"}] + assert doc["bands"][0] == {"id": 1, "name": {"ja": "mygo-ja", "en": "mygo-en", "zh-Hant": "mygo-tw", + "zh-Hans": "mygo-cn", "ko": "mygo-ko"}, + "mainColor": "#3388BB", "subColor": "#FFFFFF"} + assert [b["id"] for b in doc["bands"]] == [1, 2] assert doc["characters"][0]["name"]["ja"] == "tomori-ja" and doc["characters"][0]["bandId"] == 1 assert doc["categories"] == [{"id": 1, "musicCategories": [1], "name": dict(doc["categories"][0]["name"])}] one, two = doc["songs"] @@ -278,7 +349,7 @@ def test_deck(tmp_path): assert [r["rankBonusPercents"] for r in c["deck"]["ranges"]] == [[35, 34, 33, 32, 31], [40, 38, 36, 34, 32]] seed = c["deck"]["seeds"][0] assert seed["rankCheck"]["ranks"] == [2, 2] and seed["scorePerfect"] == seed["score"] - assert [r["rankBonus"] for r in seed["ranges"]] == [35, 80] + assert [r["rankBonus"] for r in seed["ranges"]] == [35, 80] and [r["luckPoints"] for r in seed["ranges"]] == [0, 0] assert c["deck"]["offSeeds"][0]["score"] == 1000 # the deck model's numbers as it writes them: a binary64 value is not narrowed to binary32 assert b'"weights":[[' + WEIGHT.encode() + b',' + WEIGHT.encode() + b']]' in raw and b'"predicted":2000.25' in raw @@ -312,6 +383,9 @@ def test_deck(tmp_path): (lambda s: s["seeds"][0]["check"].update(bound=0.1), "the check deck scores 2000"), (lambda s: s["ranges"] and s["seeds"][0]["rankCheck"].update(exact=0), "at ranks"), (lambda s: s["ranges"] and s["seeds"][0]["rankCheck"].update(ranks=[6, 1]), "rank check ranks"), + (lambda s: s["seeds"][0]["ranges"][0].pop("luckPoints"), "chart 10 .*seed 0 range 0: no luck points"), + (lambda s: s["seeds"][0].update(seed=3), r"chart 10 .*seeds \[3\] are not the chart's seed set \[0\]"), + (lambda s: s["seeds"].append(dict(s["seeds"][0], seed=5)), r"chart 10 .*seeds \[0, 5\] are not the chart's seed"), ]) def test_deck_checks(tmp_path, change, match): with pytest.raises(musicdata.MusicDataError, match=match): @@ -333,6 +407,99 @@ def change(s): schema_validator().validate(doc) +def test_gekisou_catalog(tmp_path): + export(tmp_path) # without the deck model as with it + cat = json.loads((tmp_path / "music.json").read_bytes())["gekisouCatalog"] + assert list(cat) == ["skills", "supportSkills", "members", "snaps"] + assert cat["skills"][0] == {"id": 1, "mission": 1, "maxLevel": 5, + "name": {"ja": "gs1-ja", "en": "gs1-en", "zh-Hant": "gs1-tw", "zh-Hans": "gs1-cn", + "ko": "gs1-ko"}, + "description": {"ja": "gsd1-ja", "en": "gsd1-en", "zh-Hant": "gsd1-tw", + "zh-Hans": "gsd1-cn", "ko": "gsd1-ko"}} + assert [(s["id"], s["mission"], s["maxLevel"]) for s in cat["skills"]] == [(1, 1, 5), (2, 3, 3), (3, 2, 5)] + assert cat["skills"][2]["description"] is None # no text id + assert [(s["id"], s["mission"], s["maxLevel"]) for s in cat["supportSkills"]] == [(31, 3, 5), (32, 1, 5)] + assert cat["supportSkills"][0]["name"]["en"] == "gs31-en" and cat["supportSkills"][1]["description"] is None + assert cat["members"][0] == {"id": 1, "characterId": 1, "bandId": 1, "rarity": 3, "gekisouSkillId": 1, + "name": {"ja": "card1-ja", "en": "card1-en", "zh-Hant": "card1-tw", + "zh-Hans": "card1-cn", "ko": "card1-ko"}, + "subtitle": {"ja": "sub1-ja", "en": "sub1-en", "zh-Hant": "sub1-tw", + "zh-Hans": "sub1-cn", "ko": "sub1-ko"}} + assert [(m["id"], m["characterId"], m["bandId"], m["gekisouSkillId"]) for m in cat["members"]] == [ + (1, 1, 1, 1), (2, 2, 1, 2), (3, 3, 1, 1), (4, 4, 1, 3), (5, 5, 2, 2), (6, 6, 2, None), (7, 3, 1, 3), + (8, 1, 1, 2)] + assert cat["members"][2]["subtitle"] is None + assert [(s["id"], s["characterIds"], s["rarity"], s["gekisouSupportSkillIds"], s["supportSkillLevel"]) + for s in cat["snaps"]] == [(41, [1], 2, [31], 5), (42, [2], 2, [32], 5), (43, [3], 2, [], 5), + (44, [4], 2, [31], 5)] + assert cat["snaps"][0]["name"]["ja"] == "snap41-ja" and cat["snaps"][0]["subtitle"]["ko"] == "snapsub41-ko" + assert cat["snaps"][1]["subtitle"] is None + schema_validator().validate(json.loads((tmp_path / "music.json").read_bytes())) + + +def _rows(change): + rows = json.loads(json.dumps(TABLE_ROWS)) + change(rows) + return rows + + +@pytest.mark.parametrize("change, match", [ + (lambda r: r["MasterMemberCard"][0].update(_characterID=9), "MasterMemberCard 1: character 9 is not in"), + (lambda r: r["MasterMemberCard"][0].update(_gekisouSkillID=9), "MasterMemberCard 1: Gekisou skill 9 is not in"), + (lambda r: r["MasterMemberCard"].append(r["MasterMemberCard"][0]), "MasterMemberCard: _id 1 occurs twice"), + (lambda r: r["MasterSupportCard"][0].update(_gekisouSupportSkillId01=99), + "MasterSupportCard 41: Gekisou support skill 99 is not in"), + (lambda r: r["MasterSupportCard"][0].update(_supportCardRankGroup=7), + "MasterSupportCard 41: rank group 7 has no MasterSupportCardRank row"), + (lambda r: (r["MasterSupportCard"][0].update(_gekisouSupportSkillId02=32), + r["MasterSupportCardRank"][4].update(_gekisouSupportSkill02Level=4)), + r"MasterSupportCard 41: its Gekisou support skills \[31, 32\] have the levels \[5, 4\]"), + (lambda r: r["MasterGekisouSkill"][0].update(_nameTextID="GSX"), "MasterText has no text 'GSX'"), + (lambda r: r["MasterGekisouSupportSkill"].append(r["MasterGekisouSupportSkill"][0]), + "MasterGekisouSupportSkill: _id 31 occurs twice"), +]) +def test_gekisou_catalog_errors(tmp_path, change, match): + with pytest.raises(musicdata.MusicDataError, match=match): + export(tmp_path, rows=_rows(change)) + assert not (tmp_path / "music.json").exists() + + +def test_gekisou_catalog_levels(tmp_path): + """A snap's level is its rank group's highest rank's; two Gekisou support skills of one level are one snap.""" + def change(r): + r["MasterSupportCard"][0]["_gekisouSupportSkillId02"] = 32 + r["MasterSupportCardRank"].append(columns("MasterSupportCardRank", _id=9, _group=1, _rank=6, + _gekisouSupportSkill01Level=3, _gekisouSupportSkill02Level=3)) + export(tmp_path, rows=_rows(change)) + snaps = json.loads((tmp_path / "music.json").read_bytes())["gekisouCatalog"]["snaps"] + assert (snaps[0]["gekisouSupportSkillIds"], snaps[0]["supportSkillLevel"]) == ([31, 32], 3) + + +def test_published_seeds(): + # ournotes-deck tests/gekisou_play.rs: published_seeds_are_fixed_and_nested + assert musicdata.published_seeds(8) == [-70152769, -452351740, 156766337, -1696681451, -1283484205, -895566507, + 322491774, 2099122494] + assert musicdata.published_seeds(64)[:8] == musicdata.published_seeds(8) + + +def on(score_id, change): + """A FakeDeck change of one chart.""" + return lambda s: change(s) if s["scoreId"] == score_id else None + + +def test_luck_chart(tmp_path): + """A chart with a luck range: its seeds are the first published seeds.""" + rows = _rows(lambda r: r["MasterLiveMusic"][1].update(_gekisouMission1=2)) # song 100001: [2, 3, 3] + export(tmp_path, rows=rows, deck=FakeDeck()) + doc = json.loads((tmp_path / "music.json").read_bytes()) + deck = doc["songs"][0]["charts"][0]["deck"] + assert [s["seed"] for s in deck["seeds"]] == musicdata.published_seeds(8) + schema_validator().validate(doc) + (tmp_path / "m").rename(tmp_path / "m1") + with pytest.raises(musicdata.MusicDataError, match=r"chart 10 .*seeds .* are not the chart's seed set"): + export(tmp_path, rows=rows, deck=FakeDeck(on(10, lambda s: s["seeds"].pop()))) + + def test_rank_bonus_percents(): assert [musicdata.mission_pattern(m) for m in ([1, 2, 0], [2, 2, 2], [1, 2, 3], [1, 1, 3], [1, 3, 3], [3, 1, 3])] == [0, 1, 2, 3, 3, 3] From 4623856cd102e5c9fcf0d465b55eb7bbc581e2ce Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 30 Sep 2026 04:38:19 +0800 Subject: [PATCH 2/4] feat(music-data): export and validate single-skill Gekisou aptitude Carry the shape catalog and chart aptitude, validate master references and sampling results, and expose aptitude controls through the CLI and PyO3 binding. Pin the deck feature commit and document the additive schema. Co-Authored-By: Claude Code --- README.en.md | 2 +- README.md | 2 +- docs/commands.md | 7 +- docs/music-data.md | 61 ++- docs/schema/music-data.schema.json | 587 +++++++++++++++++++++++++++++ rust/Cargo.lock | 2 +- rust/Cargo.toml | 2 +- rust/src/lib.rs | 36 +- src/nnnotes/cli.py | 12 +- src/nnnotes/musicdata.py | 439 ++++++++++++++++++++- tests/test_musicdata.py | 282 ++++++++++++-- 11 files changed, 1373 insertions(+), 59 deletions(-) diff --git a/README.en.md b/README.en.md index a6d5c5d..70fce55 100644 --- a/README.en.md +++ b/README.en.md @@ -49,7 +49,7 @@ nnnotes config check # each setting's origin and whether it is valid | `player` | APK | render-related global settings (color space, quality levels, renderers) as JSON | | `live` | music ID + difficulty | a full chart directory: chart and runtime notes, 3D scene, note and effect assets, BGM and sounds, sound routing | | `web` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; the Live2D models of the stories are built first as models, listed in `models.json`, and the story manifests reference them; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`); compressible assets (JSON, shaders, moc3, ...) are stored gzip encoded by default (`--compress br` / `none`) | -| `music-data` | master data files (`--master-files` directory or `--apk-master`), or decoded master data with its manifest (`--decoded-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; the Gekisou catalog (member cards, snaps and their Gekisou skills and support skills); per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (built into nnnotes) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) | +| `music-data` | master data files (`--master-files` directory or `--apk-master`), or decoded master data with its manifest (`--decoded-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; the Gekisou catalog (member cards, snaps and their Gekisou skills and support skills); per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (built into nnnotes) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position), plus single-shape Gekisou aptitude (increments and standard errors, not an optimal deck; omit with `--no-gekisou-aptitude`). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) | Export conventions: diff --git a/README.md b/README.md index d02509e..df023bc 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ nnnotes config check # 每项设置的来源和格式是否有效， | `player` | APK | 渲染相关的全局设置（色彩空间、画质等级、渲染器）JSON | | `live` | 曲目 ID + 难度 | 完整谱面目录：谱面与运行时音符、3D 场景、音符与特效资源、BGM 与音效、声音路由 | | `web` | `--pair 曲目:难度`（可重复）或 `--all`；`--live2d 模型`（可重复）或 `--all-live2d`；`--story 剧情 ID`（可重复）或 `--all-stories`；`--region 区服`（可重复）或 `--all-regions` | ournotes-player 静态站点：共享播放器 + 每谱 / 每模型 / 每集剧情清单 + 内容寻址资源；剧情用到的 Live2D 模型先按模型构建并列入 `models.json`，剧情清单引用它们；一个站点可服务多个区服，列表文本含五种语言；剧情的界面文字按语言分组，字形由开源字体生成 TextMeshPro 字体资源（`--fonts game` 时用游戏字体）；可压缩的资源（JSON、着色器、moc3 等）默认以 gzip 存储，`--compress br` / `none` 可改 | -| `music-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`），或带清单的解码 masterdata（`--decoded-master`） | 全部歌曲与谱面的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、评级线、BGM 时长；激走目录（成员卡、小卡及其激走技能与激走支援技能）；每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间；以及组卡模型 ournotes-deck（内置于 nnnotes）在整场模拟上实测的谱面统计（无技能得分、每种加分技能在每个演出位的权重）。`--full` 另附组卡模型的输入：每张谱面的运行时音符与卡牌、技能、加成、分数、活动相关的 masterdata 表（[格式](docs/music-data.md)） | +| `music-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`），或带清单的解码 masterdata（`--decoded-master`） | 全部歌曲与谱面的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、评级线、BGM 时长；激走目录（成员卡、小卡及其激走技能与激走支援技能）；每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间；以及组卡模型 ournotes-deck（内置于 nnnotes）在整场模拟上实测的谱面统计（无技能得分、每种加分技能在每个演出位的权重），以及单个激走技能形状的谱面适性（增量与标准误，不选最佳编成；`--no-gekisou-aptitude` 可跳过）。`--full` 另附组卡模型的输入：每张谱面的运行时音符与卡牌、技能、加成、分数、活动相关的 masterdata 表（[格式](docs/music-data.md)） | 导出约定： diff --git a/docs/commands.md b/docs/commands.md index 189ff66..6da9cbf 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -671,7 +671,8 @@ in the printed summary and in `SITE.failures.json`, models that fail in the summ ``` nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] - [--workers N] [--no-bgm] [--jackets DIR] -o FILE + [--workers N] [--no-gekisou-aptitude] [--aptitude-max-seeds N] [--aptitude-cross-seeds N] + [--no-bgm] [--jackets DIR] -o FILE ``` Writes one JSON file with every `MasterLiveMusic` song and its charts for one master data version: titles, readings @@ -685,7 +686,9 @@ score-up skill kind at every performance position, with Gekisou on (a Gekisou li rank and the Perfect play need) and off (a solo live), measured by the deck model ournotes-deck (built into nnnotes as `nnnotes._deck`) on its whole-live simulation and checked against the chart facts and the master data. `--full` also writes the deck model's input: every `MasterLiveMusicScore` row's chart as the client builds it at runtime (notes, skill events, fever ranges) and the -master data tables about cards, skills, bonuses, scores and events. `--no-deck` skips the deck model (every chart's +master data tables about cards, skills, bonuses, scores and events. Single-skill Gekisou aptitude is included by +default (not an optimal deck); `--no-gekisou-aptitude` omits it, `--aptitude-max-seeds N` (1024) and +`--aptitude-cross-seeds N` (64) cap its sampling and cross terms. `--no-deck` skips the deck model (every chart's `deck` is null); `--seeds N` (default 8) and `--workers N` (default: every processor) set its seeds on charts with a luck range and its threads. The master data is decoded from the files as served: `--master-files DIR` reads `DIR/MasterManifest.json` and the `.bin` files it lists (`master download`; the file's region is `[catalog] region`), diff --git a/docs/music-data.md b/docs/music-data.md index 97f3611..f6216ac 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -11,7 +11,8 @@ as its extension module `nnnotes._deck`. The format is `nnnotes.music-data/1`; i ``` nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] - [--workers N] [--no-bgm] [--jackets DIR] -o FILE + [--workers N] [--no-gekisou-aptitude] [--aptitude-max-seeds N] [--aptitude-cross-seeds N] + [--no-bgm] [--jackets DIR] -o FILE ``` - `--master-files DIR`: master data files as served, `DIR/MasterManifest.json` and the `.bin` files it lists @@ -29,6 +30,9 @@ nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--ful whole-live simulation, dozens of lives per chart: a full run takes processor time in proportion to the number of charts. `--workers N` sets the threads it uses (default: every processor), `--seeds N` the seeds measured on a chart with a luck range (default 8). +- `--no-gekisou-aptitude`: keep the existing deck statistics but skip single-skill aptitude measurements. + `--aptitude-max-seeds N` caps their samples (default 1024), and `--aptitude-cross-seeds N` caps cross-term samples + (default 64). Lower caps reduce work but may leave the standard-error target unmet. - `--no-bgm`: do not read the cue sheets (every `bgm.length` is null). - `--jackets DIR`: also write every song's jacket, the Texture2D `Image/Jacket/`, as `DIR/.webp` (WebP quality 88, scaled down with Lanczos to at most 320 pixels on the longer side, without alpha when opaque); a @@ -63,7 +67,7 @@ the wheels on PyPI carry it. See [Building](#building). "languages": ["ja", "en", "zh-Hant", "zh-Hans", "ko"], "bands": [...], "characters": [...], "tags": [...], "categories": [...], "gekisouCatalog": {"skills": [...], "supportSkills": [...], "members": [...], "snaps": [...]}, - "deck": {"model": {...}, "kinds": [...]}, + "deck": {"model": {...}, "kinds": [...], "gekisouAptitude": {...}}, "songs": [{"id": 100001, ..., "charts": [{..., "deck": {...}}, ...]}, ...], "master": {...}, "charts": [...] } @@ -265,6 +269,59 @@ points, one Gekisou off seed), the seeds are the chart's seed set (none on a cha without a luck range, else the first `--seeds` seeds of the published seed set, which nnnotes computes itself), and every check and rank check is within its bound. +### Gekisou skill aptitude + +`deck.gekisouAptitude` describes the shapes measured, and each chart's `deck.gekisouAptitude` describes how its +score changes with **one** such shape equipped. It does not select a deck, and increments measured separately must +not be added to estimate several skills together. The existing `deck.seeds` still measures Gekisou **without card +Gekisou skills**; a solo/free live uses `offSeeds`, without Gekisou. These are model results, not a guarantee that +they reproduce the game. + +The file header has `plainKind` (the ordinary, unconditional, whole-team five-second score-up kind used for cross +terms, or null), `host` (how a support skill's paired member is measured), `seedRule` and `shapes`. A shape has a +continuous zero-based `id`, `source` (`member` or `support`), `mission` (1 combo, 2 luck, 3 Just, 4 all), +`bandCondition`, normalized `effects`, and `skills`. Each skill is `{id, level, memberTargetIds, bandIds}`; join its +id to `gekisouCatalog.skills` or `supportSkills` for its name. Members use their skill's highest level; snaps use +the support skill level at their highest rank. Effect rows preserve the master order and expose their effect, +trigger, duration, value, limits, targets, four condition groups and cumulative condition (the exact fields are in +the JSON Schema). Equivalent parameters share a shape. Support condition 5000's target is normalized away; +`memberTargetIds` and `bandIds` retain each skill's targets and bands for matching, or are null without that condition. + +Each chart has `{factors, variants}`, or null when aptitude is disabled, the chart is unplayable with Gekisou, there +are no ranges, or the master has no shapes. `factors`, in range order, has `judgedNotes`, `justNotes`, `perfectNotes`, +`tailNotes`, `comboAtStart`, and `lotteries`: the mean and standard error of the number of lotteries without a skill, +on `deck.seeds` (zero outside luck ranges). + +Variants are in shape-id order, only for the chart's missions or mission 4. A band-conditioned shape appears twice, +`bandMatch: true` then `false`; otherwise `bandMatch` is null. Each variant has: + +| Field | Content | +|---|---| +| `shape`, `bandMatch` | shape and the measured band condition | +| `deterministic`, `seeds`, `seTargetMet`, `crossSeeds` | whether random dependencies were excluded and four initial plays agreed, number of measured seeds, whether the standard-error target was met, number used for cross terms | +| `score`, `scorePerfect`, `tail`, `tailPerfect`, `converted` | score increments, the increments after the ranges, and the conversion increment; each `[mean, standard error]` | +| `ranges` | one object per range: `rangeScore`, `rankBonus`, `rangeScorePerfect`, `maxCombo`, `justCount`, `luckPoints`, each `[mean, standard error]` | +| `weights` | cross terms `[position][mean, standard error]`, null without a plain kind | +| `rangeWeights` | cross terms `[position][range][mean, standard error]`, null without a plain kind or a linear rank model | +| `check` | first seed's check: `seed`, `ranks`, `deck` (plain kind/value or null per slot), `exact`, `predicted`, `bound` | + +Point increments are measured at `deck.model.power`, using the same seed for the equipped and unequipped plays. +`score.mean = tail.mean + sum(rangeScore.mean + rankBonus.mean)`; **standard errors cannot be added this way**. +Point means and errors are rounded to 0.001. With R ranges this equality allows `(2+2R)*0.0005 + 1e-8` points of +independent rounding error. Weights, lottery statistics and the check's prediction and bound keep binary64 precision. +A deterministic result uses one seed and zero standard errors. Charts with luck ranges, effects 11000–11005, +or probability condition 4011 in any condition group always use random sampling, even if the first observations +agree; four matching observations alone do not establish determinism. Others expand through `seedRule.batches` (normally +32, 64, 128, 256, 512, 1024), stopping when the unrounded score error is at most the larger of `relative` (0.01) times +the absolute mean increment and `baseline` (0.001) times the no-skill mean on the same seeds. A result at the cap may +have `seTargetMet: false`. Cross terms use the first `min(seeds, seedRule.crossSeeds)` seeds (normally capped at 64). +The sampled increment is an estimate, not an exact expectation; consult its error and the check bound. + +The exporter checks the shape table against independently read master data, its skill/level coverage and bands, +the variants' references, mission and band coverage, finite `[mean, se]` pairs with nonnegative errors, deterministic +zero errors, array lengths, the tail identity and each check's bound. New exports always include the aptitude keys; +the Schema still accepts older `/1` files without them. + ## The deck input (`--full`) With `--full` the file ends with the input the deck model reads: `master`, the master data tables about cards, diff --git a/docs/schema/music-data.schema.json b/docs/schema/music-data.schema.json index 51f7e64..2dc97a3 100644 --- a/docs/schema/music-data.schema.json +++ b/docs/schema/music-data.schema.json @@ -438,6 +438,9 @@ } } } + }, + "gekisouAptitude": { + "$ref": "#/$defs/gekisouAptitude" } } }, @@ -955,6 +958,9 @@ "null" ], "description": "why the game cannot play the chart with Gekisou (seeds is then empty; offSeeds is not)" + }, + "gekisouAptitude": { + "$ref": "#/$defs/chartGekisouAptitude" } } }, @@ -1329,6 +1335,587 @@ } } } + }, + "aptitudeMeanSE": { + "type": "array", + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number", + "minimum": 0 + } + ], + "items": false, + "minItems": 2, + "maxItems": 2 + }, + "aptitudeCondition": { + "type": "object", + "required": [ + "type", + "values", + "positive", + "targetIds" + ], + "properties": { + "type": { + "type": "integer" + }, + "values": { + "type": "array", + "items": { + "type": "integer" + } + }, + "positive": { + "type": "boolean" + }, + "targetIds": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + } + }, + { + "type": "null" + } + ] + } + } + }, + "aptitudeEffect": { + "type": "object", + "required": [ + "effectType", + "triggerType", + "effectValue", + "maxEffectValue", + "effectLimitCount", + "effectExecuteLimitCount", + "activationTimeSecond", + "skillTargetIds", + "trigger", + "condition", + "release", + "reset", + "cumulative" + ], + "properties": { + "effectType": { + "type": "integer" + }, + "triggerType": { + "type": "integer" + }, + "effectValue": { + "type": "integer" + }, + "maxEffectValue": { + "type": "integer" + }, + "effectLimitCount": { + "type": "integer" + }, + "effectExecuteLimitCount": { + "type": "integer" + }, + "activationTimeSecond": { + "type": "number" + }, + "skillTargetIds": { + "type": "array", + "items": { + "type": "integer" + } + }, + "trigger": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/aptitudeCondition" + } + } + }, + "condition": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/aptitudeCondition" + } + } + }, + "release": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/aptitudeCondition" + } + } + }, + "reset": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/aptitudeCondition" + } + } + }, + "cumulative": { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "values", + "targetIds", + "maxCumulativeCount" + ], + "properties": { + "type": { + "type": "integer" + }, + "values": { + "type": "array", + "items": { + "type": "integer" + } + }, + "targetIds": { + "type": "array", + "items": { + "type": "integer" + } + }, + "maxCumulativeCount": { + "type": "integer" + } + } + }, + { + "type": "null" + } + ] + } + } + }, + "aptitudeShape": { + "type": "object", + "required": [ + "id", + "source", + "mission", + "bandCondition", + "effects", + "skills" + ], + "properties": { + "id": { + "type": "integer", + "minimum": 0 + }, + "source": { + "enum": [ + "member", + "support" + ] + }, + "mission": { + "enum": [ + 1, + 2, + 3, + 4 + ] + }, + "bandCondition": { + "type": "boolean" + }, + "effects": { + "type": "array", + "items": { + "$ref": "#/$defs/aptitudeEffect" + } + }, + "skills": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "level", + "memberTargetIds", + "bandIds" + ], + "properties": { + "id": { + "type": "integer" + }, + "level": { + "type": "integer", + "minimum": 1 + }, + "memberTargetIds": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + } + }, + { + "type": "null" + } + ] + }, + "bandIds": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + } + }, + { + "type": "null" + } + ] + } + } + }, + "minItems": 1 + } + } + }, + "gekisouAptitude": { + "anyOf": [ + { + "type": "object", + "required": [ + "plainKind", + "host", + "seedRule", + "shapes" + ], + "properties": { + "plainKind": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ] + }, + "host": { + "type": "string", + "minLength": 1 + }, + "seedRule": { + "type": "object", + "required": [ + "deterministicTest", + "batches", + "relative", + "baseline", + "crossSeeds" + ], + "properties": { + "deterministicTest": { + "type": "integer", + "minimum": 1 + }, + "batches": { + "type": "array", + "items": { + "type": "integer", + "minimum": 1 + }, + "minItems": 1 + }, + "relative": { + "type": "number", + "minimum": 0 + }, + "baseline": { + "type": "number", + "minimum": 0 + }, + "crossSeeds": { + "type": "integer", + "minimum": 1 + } + } + }, + "shapes": { + "type": "array", + "items": { + "$ref": "#/$defs/aptitudeShape" + } + } + } + }, + { + "type": "null" + } + ] + }, + "aptitudeVariant": { + "type": "object", + "required": [ + "shape", + "bandMatch", + "deterministic", + "seeds", + "seTargetMet", + "crossSeeds", + "score", + "scorePerfect", + "tail", + "tailPerfect", + "converted", + "ranges", + "weights", + "rangeWeights", + "check" + ], + "properties": { + "shape": { + "type": "integer", + "minimum": 0 + }, + "bandMatch": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "deterministic": { + "type": "boolean" + }, + "seeds": { + "type": "integer", + "minimum": 1 + }, + "seTargetMet": { + "type": "boolean" + }, + "crossSeeds": { + "type": "integer", + "minimum": 1 + }, + "score": { + "$ref": "#/$defs/aptitudeMeanSE" + }, + "scorePerfect": { + "$ref": "#/$defs/aptitudeMeanSE" + }, + "tail": { + "$ref": "#/$defs/aptitudeMeanSE" + }, + "tailPerfect": { + "$ref": "#/$defs/aptitudeMeanSE" + }, + "converted": { + "$ref": "#/$defs/aptitudeMeanSE" + }, + "ranges": { + "type": "array", + "items": { + "type": "object", + "required": [ + "rangeScore", + "rankBonus", + "rangeScorePerfect", + "maxCombo", + "justCount", + "luckPoints" + ], + "properties": { + "rangeScore": { + "$ref": "#/$defs/aptitudeMeanSE" + }, + "rankBonus": { + "$ref": "#/$defs/aptitudeMeanSE" + }, + "rangeScorePerfect": { + "$ref": "#/$defs/aptitudeMeanSE" + }, + "maxCombo": { + "$ref": "#/$defs/aptitudeMeanSE" + }, + "justCount": { + "$ref": "#/$defs/aptitudeMeanSE" + }, + "luckPoints": { + "$ref": "#/$defs/aptitudeMeanSE" + } + } + } + }, + "weights": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/aptitudeMeanSE" + } + }, + { + "type": "null" + } + ] + }, + "rangeWeights": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/aptitudeMeanSE" + } + } + }, + { + "type": "null" + } + ] + }, + "check": { + "type": "object", + "required": [ + "seed", + "ranks", + "deck", + "exact", + "predicted", + "bound" + ], + "properties": { + "seed": { + "type": "integer" + }, + "ranks": { + "type": "array", + "items": { + "type": "integer", + "minimum": 1, + "maximum": 5 + } + }, + "deck": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "array", + "prefixItems": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "integer" + } + ], + "items": false, + "minItems": 2, + "maxItems": 2 + }, + { + "type": "null" + } + ] + } + }, + "exact": { + "type": "integer" + }, + "predicted": { + "type": "number" + }, + "bound": { + "type": "number", + "minimum": 0 + } + } + } + } + }, + "chartGekisouAptitude": { + "anyOf": [ + { + "type": "object", + "required": [ + "factors", + "variants" + ], + "properties": { + "factors": { + "type": "array", + "items": { + "type": "object", + "required": [ + "judgedNotes", + "justNotes", + "perfectNotes", + "tailNotes", + "comboAtStart", + "lotteries" + ], + "properties": { + "judgedNotes": { + "type": "integer", + "minimum": 0 + }, + "justNotes": { + "type": "integer", + "minimum": 0 + }, + "perfectNotes": { + "type": "integer", + "minimum": 0 + }, + "tailNotes": { + "type": "integer", + "minimum": 0 + }, + "comboAtStart": { + "type": "integer", + "minimum": 0 + }, + "lotteries": { + "$ref": "#/$defs/aptitudeMeanSE" + } + } + } + }, + "variants": { + "type": "array", + "items": { + "$ref": "#/$defs/aptitudeVariant" + } + } + } + }, + { + "type": "null" + } + ] } } } diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 9a76560..7375f82 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -44,7 +44,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "ournotes-deck" version = "0.0.1" -source = "git+https://github.com/empty-sekai/ournotes-deck?rev=d4abb52ff75059a13e99a88fd6812ec1bdaa5dd2#d4abb52ff75059a13e99a88fd6812ec1bdaa5dd2" +source = "git+https://github.com/empty-sekai/ournotes-deck?rev=94281d18ce50c991d7c8a082d2a32a4522050454#94281d18ce50c991d7c8a082d2a32a4522050454" dependencies = [ "serde", "serde_json", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 92a1db1..cc698c5 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -15,7 +15,7 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.29", features = ["abi3-py311"] } # pinned by commit; .github/workflows/deck.yml opens a pull request when ournotes-deck moves -ournotes-deck = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "d4abb52ff75059a13e99a88fd6812ec1bdaa5dd2" } +ournotes-deck = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "94281d18ce50c991d7c8a082d2a32a4522050454" } serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } [profile.release] diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 476e951..d4ca68f 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -4,7 +4,7 @@ use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; -use ournotes_deck::chartstats::{self, ChartStats, GEKISOU_SEEDS}; +use ournotes_deck::chartstats::{self, AptitudeOptions, ChartStats, GEKISOU_SEEDS, Options}; use ournotes_deck::data::DeckData; use ournotes_deck::error::Error; use pyo3::exceptions::PyValueError; @@ -16,7 +16,7 @@ fn value_error(e: impl std::fmt::Display) -> PyErr { } /// The statistics of every chart, in chart order, on `workers` threads. -fn measure(data: &DeckData, seeds: usize, workers: usize) -> Result, Error> { +fn measure(data: &DeckData, options: &Options, workers: usize) -> Result, Error> { let kinds = chartstats::kinds(&data.master); let n = data.charts.len(); let next = AtomicUsize::new(0); @@ -31,7 +31,7 @@ fn measure(data: &DeckData, seeds: usize, workers: usize) -> Result Error::Domain(format!("chart {}: {m}", c.score_id)), e => e, }); @@ -53,24 +53,42 @@ fn measure(data: &DeckData, seeds: usize, workers: usize) -> Result str +/// chart_stats(data, seeds=None, workers=None, aptitude=True, aptitude_max_seeds=None, +/// aptitude_cross_seeds=None) -> str /// /// The `ournotes-deck.chart-stats/2` document (JSON text) of a deck data document (`nnnotes.deck-data/1` JSON text), /// as `ournotes-deck chart-stats` writes it. `seeds`: the size of the seed set of charts with a luck range (default /// 8); `workers`: threads measuring charts (default: the available parallelism). Raises ValueError for data the -/// deck model cannot read or a chart whose check deck fails. +/// deck model cannot read or a chart whose check deck fails. `aptitude`: measure single Gekisou skill shapes +/// (default true); `aptitude_max_seeds` and `aptitude_cross_seeds`: sample caps (defaults 1024 and 64). #[pyfunction] -#[pyo3(signature = (data, seeds=None, workers=None))] -fn chart_stats(py: Python<'_>, data: &str, seeds: Option, workers: Option) -> PyResult { +#[pyo3(signature = (data, seeds=None, workers=None, aptitude=true, aptitude_max_seeds=None, aptitude_cross_seeds=None))] +fn chart_stats( + py: Python<'_>, + data: &str, + seeds: Option, + workers: Option, + aptitude: bool, + aptitude_max_seeds: Option, + aptitude_cross_seeds: Option, +) -> PyResult { let seeds = seeds.unwrap_or(GEKISOU_SEEDS); + let defaults = AptitudeOptions::default(); + let options = Options { + seeds, + aptitude: aptitude.then_some(AptitudeOptions { + max_seeds: aptitude_max_seeds.unwrap_or(defaults.max_seeds), + cross_seeds: aptitude_cross_seeds.unwrap_or(defaults.cross_seeds), + }), + }; let workers = workers.unwrap_or_else(|| std::thread::available_parallelism().map_or(1, |n| n.get())); py.detach(|| { let mut data = DeckData::from_json(data).map_err(value_error)?; let charts = std::mem::take(&mut data.charts); // the document of no chart: the format, source, model and kinds, exactly as the command writes them - let mut doc = chartstats::document(&data, Some(seeds)).map_err(value_error)?; + let mut doc = chartstats::document_with(&data, &options).map_err(value_error)?; data.charts = charts; - let stats = measure(&data, seeds, workers).map_err(value_error)?; + let stats = measure(&data, &options, workers).map_err(value_error)?; doc["charts"] = serde_json::to_value(stats).map_err(value_error)?; serde_json::to_string(&doc).map_err(value_error) }) diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index b14ead1..7b94b7c 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -37,7 +37,7 @@ [--font emoji=] [--region [--region ...] | --all-regions] nnnotes music-data --master-files | --apk-master | --decoded-master [--full] [--no-deck] - [--no-bgm] [--jackets DIR] -o out/music-data.json[.gz] + [--no-gekisou-aptitude] [--no-bgm] [--jackets DIR] -o out/music-data.json[.gz] nnnotes export -o out/assets [--select group: | key: | bundle: ...] [--layout original,cas] nnnotes plan [--select ...] [--json] [--check] [--emit-tasks ] nnnotes run-stage [...] @@ -643,7 +643,9 @@ def cmd_music_data(args, cfg): cfg.require_path("paths", "apk") # the master data files ship in the APK apk = _existing(cfg, "paths", "apk") try: - deck = None if args.no_deck else musicdata.Deck(seeds=args.seeds, workers=args.workers) + deck = None if args.no_deck else musicdata.Deck( + seeds=args.seeds, workers=args.workers, aptitude=not args.no_gekisou_aptitude, + aptitude_max_seeds=args.aptitude_max_seeds, aptitude_cross_seeds=args.aptitude_cross_seeds) if args.apk_master: src, region = deckdata.apk_master(apk), deckdata.EMBEDDED elif args.decoded_master: # decoded elsewhere: no master key @@ -947,6 +949,12 @@ def target(m, what): help="seeds measured on a chart with a luck range (default 8)") c.add_argument("--workers", type=int, metavar="N", help="threads measuring charts (default: every processor)") + c.add_argument("--no-gekisou-aptitude", action="store_true", + help="leave out the charts' Gekisou aptitude (every gekisouAptitude is null)") + c.add_argument("--aptitude-max-seeds", type=int, metavar="N", + help="seeds of a Gekisou aptitude variant at most (default: the deck model's, 1024)") + c.add_argument("--aptitude-cross-seeds", type=int, metavar="N", + help="seeds of a Gekisou aptitude variant's cross terms (default: the deck model's, 64)") c.add_argument("--no-bgm", action="store_true", help="do not read the BGM cue sheets (every song's bgm.length is null)") c.add_argument("--jackets", metavar="DIR", diff --git a/src/nnnotes/musicdata.py b/src/nnnotes/musicdata.py index f6d54ef..8925776 100644 --- a/src/nnnotes/musicdata.py +++ b/src/nnnotes/musicdata.py @@ -25,6 +25,7 @@ import hashlib import io import json +import math from collections import Counter from pathlib import Path from typing import Callable @@ -226,7 +227,7 @@ def _by_id(rows: list[dict], table: str) -> dict: # ---------------------------------------------------------------- the Gekisou catalog -MISSION_LUCK = 2 # GekisouMissionType: 1 combo, 2 luck, 3 Just count, 4 all +MISSION_LUCK, MISSION_JUST, MISSION_ALL = 2, 3, 4 # GekisouMissionType: 1 combo, 2 luck, 3 Just count, 4 all def _max_levels(rows: list[dict], key: str) -> dict: @@ -303,9 +304,12 @@ def skill(r: dict, max_level: dict) -> dict: class Deck: """The deck model (nnnotes._deck): chart statistics of a deck input document, on `workers` threads (None: the - available parallelism), `seeds` seeds for a chart with a luck range.""" + available parallelism), `seeds` seeds for a chart with a luck range; with `aptitude` every chart's Gekisou aptitude + too, a variant on at most `aptitude_max_seeds` seeds and its cross terms on at most `aptitude_cross_seeds` (None: + the deck model's defaults).""" - def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=None): + def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=None, aptitude: bool = True, + aptitude_max_seeds: int | None = None, aptitude_cross_seeds: int | None = None): if module is None: try: from . import _deck as module @@ -313,6 +317,8 @@ def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=N raise MusicDataError("the deck model (nnnotes._deck) is not built into this installation: install " "nnnotes from a wheel or build it (maturin), or pass --no-deck") from None self.module, self.seeds, self.workers = module, seeds, workers + self.aptitude, self.aptitude_max_seeds, self.aptitude_cross_seeds = \ + aptitude, aptitude_max_seeds, aptitude_cross_seeds def info(self) -> dict: """{name, version, source, commit, format} of the deck model.""" @@ -322,7 +328,9 @@ def info(self) -> dict: def stats(self, deck_input: dict) -> dict: """The chart statistics document of a deck input document (deckdata.build), its numbers as written.""" try: - text = self.module.chart_stats(deckdata.encode(deck_input).decode("utf-8"), self.seeds, self.workers) + text = self.module.chart_stats(deckdata.encode(deck_input).decode("utf-8"), self.seeds, self.workers, + aptitude=self.aptitude, aptitude_max_seeds=self.aptitude_max_seeds, + aptitude_cross_seeds=self.aptitude_cross_seeds) except ValueError as e: raise MusicDataError(f"deck model: {e}") from None doc = json.loads(text, parse_float=deckdata._Num) @@ -333,7 +341,7 @@ def stats(self, deck_input: dict) -> dict: # the keys of a chart's statistics carried by `deck` (the others are checked against the chart facts) DECK_CHART_KEYS = ("convertedNoteCount", "skip", "events", "positions", "ranges", "justNotes", "seeds", "offSeeds", - "unplayable") + "unplayable", "gekisouAptitude") RANKS = 5 # the ranks of a Gekisou range (a Gekisou live has up to five players) GEKISOU_RANGES = 3 # the Gekisou ranges of a live (its first three fevers) @@ -381,6 +389,9 @@ def _check_bound(c, what: str, where: str) -> None: """A check (or rank check) of the deck model: its exact score within `bound` of the predicted one.""" if not isinstance(c, dict): raise MusicDataError(f"{where}: {what}: no check") + if not (_int(c.get("exact")) and _number(c.get("predicted")) and _number(c.get("bound")) + and float(c["bound"]) >= 0): + raise MusicDataError(f"{where}: {what}: invalid exact, predicted or bound") if abs(c["exact"] - float(c["predicted"])) > float(c["bound"]): raise MusicDataError(f"{where}: {what} scores {c['exact']}, predicted {c['predicted']} beyond the bound " f"{c['bound']}") @@ -466,11 +477,403 @@ def _check_seed_shapes(stats: dict, kinds: int, where: str) -> None: _check_bound(seed["check"], "Gekisou off: the check deck", where) -def chart_deck(song: dict, chart: dict, stats: dict, kinds: int, percents: list[list], - seeds: int = DECK_SEEDS) -> dict: +# ---------------------------------------------------------------- the Gekisou aptitude +PLAIN_EFFECT_TYPE, PLAIN_MS = 2000, 5000 # the plain kind: score up on the whole deck for 5 s, nothing else +CONDITION_MEMBER_TARGET = 5000 # the skill condition on the member a snap's support skill is paired with +APTITUDE_KEYS = ("plainKind", "host", "seedRule", "shapes") +SEED_RULE_KEYS = ("deterministicTest", "batches", "relative", "baseline", "crossSeeds") +SHAPE_KEYS = ("id", "source", "mission", "bandCondition", "effects", "skills") +FACTOR_COUNTS = ("judgedNotes", "justNotes", "perfectNotes", "tailNotes", "comboAtStart") +VARIANT_MEANS = ("score", "scorePerfect", "tail", "tailPerfect", "converted") +RANGE_MEANS = ("rangeScore", "rankBonus", "rangeScorePerfect", "maxCombo", "justCount", "luckPoints") +CHECK_KEYS = ("seed", "ranks", "deck", "exact", "predicted", "bound") +CONDITION_GROUPS = (("trigger", "_skillTriggerConditionGroup"), ("condition", "_skillConditionGroup"), + ("release", "_skillReleaseConditionGroup"), ("reset", "_effectExecuteLimitResetConditionGroup")) +MEAN_TOLERANCE = 1e-9 # relative: a mean against its recomputation (binary64 sums) + + +def plain_kind(kinds: list[dict]) -> int | None: + """The id of the plain score-up kind of `deck.kinds` (the page's plainKind): effect type 2000 without targets, + conditions or limits for 5 s (a kind without `durationMs` counts as 5 s); None when there is none.""" + for k in kinds: + ms = k.get("durationMs") + if (k.get("effectType") == PLAIN_EFFECT_TYPE and not (k.get("skillTargetIds") or []) + and not k.get("skillConditionGroup") and not k.get("skillReleaseConditionGroup") + and not k.get("effectLimitCount") and not k.get("effectExecuteLimitCount") + and (PLAIN_MS if ms is None else ms) == PLAIN_MS): + return k["id"] + return None + + +def _number(v) -> bool: + """Whether v is a finite number as the deck model writes it.""" + if isinstance(v, bool) or not isinstance(v, (int, deckdata._Num)): + return False + try: + return math.isfinite(float(v)) + except (OverflowError, ValueError): + return False + + +def _mean_se(v) -> bool: + """Whether v is `[mean, standard error]`: two finite numbers, the error not negative.""" + return isinstance(v, list) and len(v) == 2 and all(_number(x) for x in v) and float(v[1]) >= 0 + + +def _same(a: float, b: float, scale: float = 1.0) -> bool: + return abs(a - b) <= MEAN_TOLERANCE * max(1.0, abs(scale), abs(a), abs(b)) + + +def mean_se(xs: list[int]) -> tuple[float, float]: + """The mean of xs and its standard error (the sample standard deviation / sqrt(n); 0 for one value).""" + n = len(xs) + m = sum(xs) / n + return m, (math.sqrt(sum((x - m) ** 2 for x in xs) / (n - 1) / n) if n > 1 else 0.0) + + +def _f32(v) -> float: + import numpy as np + return float(np.float32(float(v))) + + +class Aptitude: + """The Gekisou aptitude of the deck model checked against the Gekisou catalog and the master data it was made + from: the file's shape table (`deck.gekisouAptitude`, check_header) and every chart's variants (chart).""" + + def __init__(self, tables: dict[str, list[dict]], catalog: dict): + self.skills = {s["id"]: s for s in catalog["skills"]} + self.supports = {s["id"]: s for s in catalog["supportSkills"]} + self.conditions = {r["_id"]: r for r in tables["MasterSkillCondition"]} + self.cumulative = {r["_id"]: r for r in tables["MasterSkillCumulativeCondition"]} + self.targets = {r["_id"]: r for r in tables["MasterSkillTarget"]} + self.sets: dict = {} + for r in tables["MasterSkillConditionSet"]: + self.sets.setdefault(r.get("_group"), []).append(r) + self.rows: dict = {} # (source, skill id, level) -> effect rows in master order + for source, table, key in (("member", "MasterGekisouSkillEffect", "_gekisouSkillID"), + ("support", "MasterGekisouSupportSkillEffect", "_gekisouSupportSkillID")): + for r in tables[table]: + self.rows.setdefault((source, r.get(key), r.get("_level")), []).append(r) + # what the shapes cover: every member card's Gekisou skill at its highest level, every snap's Gekisou support + # skills at the snap's highest rank + expected = {("member", m["gekisouSkillId"], self.skills[m["gekisouSkillId"]]["maxLevel"]) + for m in catalog["members"] if m["gekisouSkillId"] is not None} + expected |= {("support", i, s["supportSkillLevel"]) for s in catalog["snaps"] + for i in s["gekisouSupportSkillIds"]} + self.expected = {k for k in expected if k in self.rows} # a skill level with effect rows + self.header = self.shapes = None + + # ------------------------------------------------ the shape table + def _group(self, g) -> list: + """A condition group as the shapes write it: its condition sets in master order, each a list of conditions + `{type, values, positive, targetIds}` (the member target of condition 5000 null); [] for group 0.""" + if not g: + return [] + out = [] + for s in self.sets.get(g, []): + conds = [] + for cid in s.get("_conditionIds") or []: + c = self.conditions.get(cid) + if c is None: + raise MusicDataError(f"MasterSkillConditionSet {s.get('_id')}: condition {cid} is not in " + f"MasterSkillCondition") + t = c.get("_conditionType") + conds.append({"type": t, "values": list(c.get("_conditionValues") or []), + "positive": bool(c.get("_isPositive")), + "targetIds": (None if t == CONDITION_MEMBER_TARGET + else list(c.get("_conditionTargetIDs") or []))}) + out.append(conds) + return out + + def effects(self, source: str, skill: int, level: int) -> list[dict]: + """The effect rows of a Gekisou (support) skill at a level as the shapes write them.""" + out = [] + for r in self.rows.get((source, skill, level), []): + cid = r.get("_skillCumulativeConditionID") + c = self.cumulative.get(cid) if cid else None + if cid and c is None: + raise MusicDataError(f"Gekisou {source} skill {skill} level {level}: cumulative condition {cid} is not " + f"in MasterSkillCumulativeCondition") + e = {"effectType": r.get("_skillEffectType"), "triggerType": r.get("_skillTriggerType"), + "activationTimeSecond": r.get("_activationTimeSecond"), "effectValue": r.get("_effectValue"), + "maxEffectValue": r.get("_maxEffectValue"), "effectLimitCount": r.get("_effectLimitCount"), + "effectExecuteLimitCount": r.get("_effectExecuteLimitCount"), + "skillTargetIds": list(r.get("_skillTargetIDs") or [])} + e.update({k: self._group(r.get(col)) for k, col in CONDITION_GROUPS}) + e["cumulative"] = None if c is None else { + "type": c.get("_skillCumulativeConditionType"), "values": list(c.get("_conditionValues") or []), + "targetIds": list(c.get("_conditionTargetIDs") or []), + "maxCumulativeCount": c.get("_maxCumulativeCount")} + out.append(e) + return out + + def member_targets(self, source: str, skill: int, level: int) -> tuple[list | None, list | None]: + """(memberTargetIds, bandIds) of a skill at a level: the target ids of every condition 5000 of its effect rows + (unique, ascending) and their bands; (None, None) without such a condition.""" + ids = set() + found = False + for r in self.rows.get((source, skill, level), []): + for _, col in CONDITION_GROUPS: + for s in self.sets.get(r.get(col), []) if r.get(col) else []: + for cid in s.get("_conditionIds") or []: + c = self.conditions.get(cid) or {} + if c.get("_conditionType") == CONDITION_MEMBER_TARGET: + found = True + ids.update(c.get("_conditionTargetIDs") or []) + if not found: + return None, None + bands = sorted({(self.targets.get(t) or {}).get("_bandID") or 0 for t in ids} - {0}) + return sorted(ids), bands + + @staticmethod + def _effects_equal(got, want) -> bool: + if not (isinstance(got, list) and len(got) == len(want)): + return False + for g, w in zip(got, want): + if not isinstance(g, dict) or any(k not in g for k in w): + return False + for k, v in w.items(): + if k == "activationTimeSecond": + if not (_number(g[k]) and _f32(g[k]) == _f32(v)): + return False + elif g[k] != v: + return False + return True + + def check_header(self, info, kinds: list[dict], model: dict) -> dict | None: + """`deck.gekisouAptitude` after checking it (None when the deck model leaves the aptitude out): the plain kind + of `kinds`, a description of the host (and `model.gekisouAptitude`), the seed rule, and the shape table: + ids 0..n-1, each shape's skills in the catalog (a member card's skill at its highest level, a snap's support + skill at its highest rank level) with the shape's mission, their effect rows as the master data has them, their + member targets and bands, and every skill and level of the catalog in exactly one shape.""" + if info is None: + return None + where = "deck model: gekisouAptitude" + if not isinstance(info, dict) or any(k not in info for k in APTITUDE_KEYS): + raise MusicDataError(f"{where}: not {{{', '.join(APTITUDE_KEYS)}}}") + want = plain_kind(kinds) + if info["plainKind"] != want: + raise MusicDataError(f"{where}: plain kind {info['plainKind']!r} is not the plain kind of the kinds, " + f"{want!r}") + if not (isinstance(info["host"], str) and info["host"]): + raise MusicDataError(f"{where}: the support skills' host is not described (host)") + if not (isinstance(model.get("gekisouAptitude"), str) and model["gekisouAptitude"]): + raise MusicDataError("deck model: the model does not describe the aptitude (model.gekisouAptitude)") + rule = info["seedRule"] + if not (isinstance(rule, dict) and all(k in rule for k in SEED_RULE_KEYS) + and _int(rule["deterministicTest"]) and rule["deterministicTest"] >= 1 + and isinstance(rule["batches"], list) and rule["batches"] and all(_int(b) for b in rule["batches"]) + and rule["batches"][0] >= 2 and all(a < b for a, b in zip(rule["batches"], rule["batches"][1:])) + and _number(rule["relative"]) and float(rule["relative"]) >= 0 + and _number(rule["baseline"]) and float(rule["baseline"]) >= 0 + and _int(rule["crossSeeds"]) and rule["crossSeeds"] >= 1): + raise MusicDataError(f"{where}: seed rule {rule!r}") + shapes = info["shapes"] + if not isinstance(shapes, list) or [s.get("id") if isinstance(s, dict) else s for s in shapes] != list( + range(len(shapes))): + ids = [s.get("id") if isinstance(s, dict) else s for s in shapes] if isinstance(shapes, list) else shapes + raise MusicDataError(f"{where}: shape ids {ids!r} are not 0, 1, ... in order") + seen: dict = {} + for shape in shapes: + at = f"{where}: shape {shape['id']}" + if not _int(shape["id"]) or any(k not in shape for k in SHAPE_KEYS) or shape["source"] not in ("member", "support") \ + or not _int(shape["mission"]) or shape["mission"] not in (1, 2, 3, 4) \ + or not isinstance(shape["bandCondition"], bool) \ + or not isinstance(shape["skills"], list) or not shape["skills"]: + raise MusicDataError(f"{at}: not a shape {{{', '.join(SHAPE_KEYS)}}} of a source, a mission and " + f"skills") + source = shape["source"] + catalog = self.skills if source == "member" else self.supports + for sk in shape["skills"]: + if not isinstance(sk, dict) or any(k not in sk for k in ("id", "level", "memberTargetIds", "bandIds")) or not ( + _int(sk.get("id")) and _int(sk.get("level"))): + raise MusicDataError(f"{at}: skill {sk!r}") + key = (source, sk["id"], sk["level"]) + what = f"{at}: Gekisou {'skill' if source == 'member' else 'support skill'} {sk['id']} level " \ + f"{sk['level']}" + if key in seen: + raise MusicDataError(f"{what} is in shape {seen[key]} too") + seen[key] = shape["id"] + if key not in self.expected: + raise MusicDataError(f"{what} is not a skill and level of the Gekisou catalog (a member card's " + f"Gekisou skill at its highest level, a snap's support skill at its highest " + f"rank)") + if catalog[sk["id"]]["mission"] != shape["mission"]: + raise MusicDataError(f"{what} has the mission {catalog[sk['id']]['mission']!r}, the shape " + f"{shape['mission']!r}") + targets = self.member_targets(*key) + if (sk.get("memberTargetIds"), sk.get("bandIds")) != targets: + raise MusicDataError(f"{what}: member targets {sk.get('memberTargetIds')!r} and bands " + f"{sk.get('bandIds')!r}, the master data has {targets[0]!r} and " + f"{targets[1]!r}") + if shape["bandCondition"] != (targets[0] is not None): + raise MusicDataError(f"{what}: band condition {shape['bandCondition']}, the master data has " + f"{'one' if targets[0] is not None else 'none'}") + if not self._effects_equal(shape["effects"], self.effects(*key)): + raise MusicDataError(f"{what}: the shape's effect rows are not the master data's") + missing = sorted(self.expected - set(seen), key=repr) + if missing: + raise MusicDataError(f"{where}: no shape has " + ", ".join(f"the {s} skill {i} level {lv}" + for s, i, lv in missing)) + self.header, self.shapes = info, shapes + return {k: info[k] for k in APTITUDE_KEYS} + + # ------------------------------------------------ a chart's aptitude + def chart(self, where: str, stats: dict, plain: int | None) -> dict | None: + """A chart's `gekisouAptitude` after checking it: null exactly when the aptitude is left out, the master data + has no shape, the chart is unplayable with Gekisou or has no Gekisou range; else a factor per range and a + variant per shape of the chart's missions (both band match results of a shape with a band condition), each + with its means shaped and consistent (the tail, a deterministic variant's rank bonuses) and its check within + its bound.""" + apt = stats.get("gekisouAptitude") + if self.header is None or not self.shapes or stats.get("unplayable") is not None or not stats["ranges"]: + if apt is not None: + why = ("the aptitude left out" if self.header is None else "no shape" if not self.shapes + else "a chart unplayable with Gekisou" if stats.get("unplayable") is not None + else "a chart without Gekisou ranges") + raise MusicDataError(f"{where}: the deck model gives a Gekisou aptitude with {why}") + return None + where = f"{where}: Gekisou aptitude" + if not (isinstance(apt, dict) and isinstance(apt.get("factors"), list) and isinstance(apt.get("variants"), + list)): + raise MusicDataError(f"{where}: none (gekisouAptitude {{factors, variants}})") + ranges, base = stats["ranges"], stats["seeds"] + self._factors(where, apt["factors"], stats) + missions = {r["mission"] for r in ranges} + want = [(s["id"], b) for s in self.shapes if s["mission"] == MISSION_ALL or s["mission"] in missions + for b in ((True, False) if s["bandCondition"] else (None,))] + if any(not isinstance(v, dict) or not _int(v.get("shape")) + or v.get("bandMatch") is not None and not isinstance(v.get("bandMatch"), bool) + or "bandMatch" not in v for v in apt["variants"]): + raise MusicDataError(f"{where}: invalid shape or bandMatch") + got = [(v.get("shape"), v.get("bandMatch")) if isinstance(v, dict) else v for v in apt["variants"]] + if got != want: + raise MusicDataError(f"{where}: variants (shape, bandMatch) {got!r}, the shapes of the chart's missions " + f"{sorted(missions)} give {want!r}") + for v in apt["variants"]: + self._variant(f"{where}: shape {v['shape']} band match {v['bandMatch']!r}", v, stats, base, plain) + return {"factors": apt["factors"], "variants": apt["variants"]} + + def _factors(self, where: str, factors: list, stats: dict) -> None: + ranges, base = stats["ranges"], stats["seeds"] + if len(factors) != len(ranges): + raise MusicDataError(f"{where}: {len(factors)} factors for {len(ranges)} ranges") + just = 0 + for j, (f, r) in enumerate(zip(factors, ranges)): + at = f"{where}: range {j} factors" + if not (isinstance(f, dict) and all(_int(f.get(k)) and f[k] >= 0 for k in FACTOR_COUNTS) + and _mean_se(f.get("lotteries"))): + raise MusicDataError(f"{at}: not {{{', '.join(FACTOR_COUNTS)}, lotteries}}: {f!r}") + if r["mission"] != MISSION_JUST and (f["justNotes"], f["perfectNotes"]) != (0, 0): + raise MusicDataError(f"{at}: {f['justNotes']} Just and {f['perfectNotes']} Perfect notes outside a " + f"Just count range") + if f["justNotes"] + f["perfectNotes"] > f["judgedNotes"] or f["comboAtStart"] > stats["judgedNotes"]: + raise MusicDataError(f"{at}: note counts {f!r} beyond the range's or the chart's") + just += f["justNotes"] + m, se = mean_se([sum(s["ranges"][j]["lotResults"]) for s in base]) if base else (0.0, 0.0) + if r["mission"] != MISSION_LUCK: + m, se = 0.0, 0.0 + if not (_same(float(f["lotteries"][0]), m) and _same(float(f["lotteries"][1]), se, m)): + raise MusicDataError(f"{at}: lotteries {f['lotteries']}, the chart's seeds give [{m!r}, {se!r}]") + if just > stats["justNotes"]: + raise MusicDataError(f"{where}: {just} Just notes in the ranges, {stats['justNotes']} on the play") + + def _variant(self, at: str, v: dict, stats: dict, base: list, plain: int | None) -> None: + rule, n, positions = self.header["seedRule"], len(stats["ranges"]), stats["positions"] + if not (isinstance(v.get("deterministic"), bool) and isinstance(v.get("seTargetMet"), bool) + and _int(v.get("seeds")) and _int(v.get("crossSeeds"))): + raise MusicDataError(f"{at}: deterministic, seeds, seTargetMet, crossSeeds " + f"{[v.get(k) for k in ('deterministic', 'seeds', 'seTargetMet', 'crossSeeds')]!r}") + det = v["deterministic"] + effects = self.shapes[v["shape"]]["effects"] + random = any(r["mission"] == MISSION_LUCK for r in stats["ranges"]) or any( + 11000 <= e["effectType"] <= 11005 or any( + c["type"] == 4011 for key, _ in CONDITION_GROUPS for group in e[key] for c in group) + for e in effects) + if det and random: + raise MusicDataError(f"{at}: a deterministic variant with a random dependency") + if det and (v["seeds"], v["seTargetMet"]) != (1, True): + raise MusicDataError(f"{at}: deterministic on {v['seeds']} seeds (seTargetMet {v['seTargetMet']})") + if not det and (v["seeds"] not in rule["batches"] + or not v["seTargetMet"] and v["seeds"] != rule["batches"][-1]): + raise MusicDataError(f"{at}: {v['seeds']} seeds (seTargetMet {v['seTargetMet']}), not a batch of the seed " + f"rule {rule['batches']}") + if v["crossSeeds"] != min(v["seeds"], rule["crossSeeds"]): + raise MusicDataError(f"{at}: {v['crossSeeds']} cross seeds, not min({v['seeds']}, {rule['crossSeeds']})") + pairs = [(k, v.get(k)) for k in VARIANT_MEANS] + rs = v.get("ranges") + if not (isinstance(rs, list) and len(rs) == n and all(isinstance(r, dict) for r in rs)): + raise MusicDataError(f"{at}: {len(rs) if isinstance(rs, list) else rs!r} range results for {n} ranges") + pairs += [(f"ranges[{j}].{k}", r.get(k)) for j, r in enumerate(rs) for k in RANGE_MEANS] + w, rw = v.get("weights"), v.get("rangeWeights") + linear = bool(base) and base[0].get("rangeWeights") is not None + if plain is None: + if w is not None or rw is not None: + raise MusicDataError(f"{at}: weights without a plain kind") + else: + if not (isinstance(w, list) and len(w) == positions): + raise MusicDataError(f"{at}: weights are not [position] of [mean, se]") + pairs += [(f"weights[{k}]", x) for k, x in enumerate(w)] + if (rw is not None) != linear: + raise MusicDataError(f"{at}: rangeWeights {'given' if rw is not None else 'null'}, the chart's seeds " + f"have {'them' if linear else 'none'}") + if rw is not None: + if not (isinstance(rw, list) and len(rw) == positions and all(isinstance(x, list) and len(x) == n + for x in rw)): + raise MusicDataError(f"{at}: rangeWeights are not [position][range] of [mean, se]") + pairs += [(f"rangeWeights[{k}][{j}]", y) for k, x in enumerate(rw) for j, y in enumerate(x)] + for name, x in pairs: + if not _mean_se(x): + raise MusicDataError(f"{at}: {name} {x!r} is not [mean, standard error]") + if det and float(x[1]) != 0: + raise MusicDataError(f"{at}: {name} {x!r}: a deterministic variant with a standard error") + m = {k: float(v[k][0]) for k in VARIANT_MEANS} + terms = [float(r[k][0]) for r in rs for k in ("rangeScore", "rankBonus")] + # Every serialized point mean is independently rounded to 0.001 (score, tail and two terms per range). + if abs(m["tail"] - (m["score"] - sum(terms))) > (len(terms) + 2) * 0.0005 + 1e-8: + raise MusicDataError(f"{at}: tail {v['tail'][0]} is not score {v['score'][0]} - {sum(terms)!r} (the " + f"ranges' scores and rank bonuses)") + if det: + ints = [float(x[0]) for name, x in pairs if not name.startswith(("weights", "rangeWeights"))] + if not all(x.is_integer() for x in ints): + raise MusicDataError(f"{at}: a deterministic variant with a fraction of a point") + perfect = 0 # the Perfect play's range scores and rank bonuses + for j, (r, info, r0) in enumerate(zip(rs, stats["ranges"], base[0]["ranges"])): + p = info["rankBonusPercent"] + want = _trunc_percent(r0["rangeScore"] + int(float(r["rangeScore"][0])), p) - r0["rankBonus"] + if int(float(r["rankBonus"][0])) != want: + raise MusicDataError(f"{at}: range {j}: rank bonus {r['rankBonus'][0]} is not trunc((" + f"{r0['rangeScore']} + {r['rangeScore'][0]}) * {p} / 100) - " + f"{r0['rankBonus']} = {want}") + d, r0p = int(float(r["rangeScorePerfect"][0])), r0["rangeScorePerfect"] + perfect += d + _trunc_percent(r0p + d, p) - _trunc_percent(r0p, p) + if m["tailPerfect"] != m["scorePerfect"] - perfect: + raise MusicDataError(f"{at}: tail on the Perfect play {v['tailPerfect'][0]} is not scorePerfect " + f"{v['scorePerfect'][0]} - {perfect} (the ranges' Perfect play scores and rank " + f"bonuses)") + c = v.get("check") + if not (isinstance(c, dict) and all(k in c for k in CHECK_KEYS)): + raise MusicDataError(f"{at}: no check {{{', '.join(CHECK_KEYS)}}}") + first = base[0]["seed"] if det else published_seeds(1)[0] + if not _int(c["seed"]) or c["seed"] != first: + raise MusicDataError(f"{at}: the check plays seed {c['seed']!r}, not the variant's first {first}") + if not (isinstance(c["ranks"], list) and len(c["ranks"]) == n and all(_int(r) and 1 <= r <= RANKS + for r in c["ranks"]) + and (linear or all(r == 1 for r in c["ranks"]))): + raise MusicDataError(f"{at}: check ranks {c['ranks']!r}") + if not (isinstance(c["deck"], list) and len(c["deck"]) == positions and all( + x is None or (plain is not None and isinstance(x, list) and len(x) == 2 and x[0] == plain + and _int(x[1])) for x in c["deck"])): + raise MusicDataError(f"{at}: check deck {c['deck']!r} is not [plain kind ({plain}), value] or null per " + f"position") + _check_bound(c, "the check deck", at) + + +def chart_deck(song: dict, chart: dict, stats: dict, kinds: int, percents: list[list], seeds: int, + aptitude: Aptitude, plain: int | None) -> dict: """A chart's `deck` from its statistics, after checking them against the song, the chart facts, the song's rank - bonus percentages (`percents`: rank_bonus_percents) and the seed set (chart_seeds: `seeds` seeds with a luck - range), `kinds` the number of score-up kinds.""" + bonus percentages (`percents`: rank_bonus_percents), the seed set (chart_seeds: `seeds` seeds with a luck range) + and its Gekisou aptitude (Aptitude.chart, `plain` the plain kind), `kinds` the number of score-up kinds.""" where = f"chart {chart['scoreId']} ({song['id']} {chart['difficulty']})" checks = ( ("music id", stats["musicId"], song["id"]), @@ -508,7 +911,7 @@ def chart_deck(song: dict, chart: dict, stats: dict, kinds: int, percents: list[ r.get("rangeScorePerfect") != r["rangeScore"] for r in seed["ranges"])): raise MusicDataError(f"{where}: seed {seed['seed']}: a chart without Just notes scores otherwise on the " f"Perfect play ({seed.get('scorePerfect')} for {seed['score']})") - return {k: stats.get(k) for k in DECK_CHART_KEYS} + return dict({k: stats.get(k) for k in DECK_CHART_KEYS}, gekisouAptitude=aptitude.chart(where, stats, plain)) # ---------------------------------------------------------------- document @@ -517,8 +920,9 @@ def build(tables: dict[str, list[dict]], table_sha: dict[str, str], fetch: Calla master_version: str | None, deck: Deck | None = None, full: bool = False) -> dict: """The music data document. `fetch(file name)`: a chart TextAsset's bytes (KeyError when there is none); `bgm(cue sheet, cue)`: the BGM length (catalog_bgm), None to leave every song's `bgm.length` null; `deck`: the - deck model measuring the songs' charts, None to leave every chart's `deck` null; `full`: add the deck input - (`master`, `charts`: deckdata.TABLES and every chart's runtime notes). `tables`: tables_of(deck, full).""" + deck model measuring the songs' charts (and their Gekisou aptitude), None to leave every chart's `deck` null; + `full`: add the deck input (`master`, `charts`: deckdata.TABLES and every chart's runtime notes). `tables`: + tables_of(deck, full).""" from . import __version__ raws: dict[str, bytes] = {} @@ -620,12 +1024,19 @@ def record(sid: int) -> dict: by_score = {c["scoreId"]: c for c in stats["charts"]} if sorted(by_score) != measured: raise MusicDataError("deck model: the charts measured differ from the songs' charts") + aptitude = Aptitude(tables, gk_catalog) + header = aptitude.check_header(stats.get("gekisouAptitude"), stats["kinds"], stats["model"]) + if (header is not None) != deck.aptitude: + raise MusicDataError(f"deck model: {'no' if deck.aptitude else 'a'} Gekisou aptitude (gekisouAptitude), " + f"asked for {'it' if deck.aptitude else 'none'}") + plain = header["plainKind"] if header is not None else None bonus_rows = tables.get("MasterLiveGekisouRankingScoreBonus", []) for s in songs: percents = rank_bonus_percents(bonus_rows, s["gekisouMissions"]) for c in s["charts"]: - c["deck"] = chart_deck(s, c, by_score[c["scoreId"]], len(stats["kinds"]), percents, deck.seeds) - deck_doc = {"model": stats["model"], "kinds": stats["kinds"]} + c["deck"] = chart_deck(s, c, by_score[c["scoreId"]], len(stats["kinds"]), percents, deck.seeds, + aptitude, plain) + deck_doc = {"model": stats["model"], "kinds": stats["kinds"], "gekisouAptitude": header} read = [t for t in tables_of(deck is not None, full) if t in tables] doc = { diff --git a/tests/test_musicdata.py b/tests/test_musicdata.py index c434d16..53fa54d 100644 --- a/tests/test_musicdata.py +++ b/tests/test_musicdata.py @@ -60,9 +60,9 @@ def effect(table, key, i, skill, level, group=0): _effectValue=100 * level) -# Gekisou skills 1 (combo, levels 1..5), 2 (Just count, 1..3), 3 (luck, 1..5); Gekisou support skills 31 (Just count, -# two rows per level: a member of band 1 or not) and 32 (combo); member cards 1..8 (card 6 without a Gekisou skill); -# snaps 41..44 (43 without a Gekisou support skill), highest rank 5 +# Gekisou skills 1 (combo, levels 1..5), 2 (Just count, 1..3), 3 (luck, 1..5); Gekisou support skills 31 and 33 (Just +# count, two rows per level: a member of band 1 (31) or 2 (33), or not) and 32 (combo); member cards 1..8 (card 6 +# without a Gekisou skill); snaps 41..45 (43 without a Gekisou support skill), highest rank 5 GEKISOU_ROWS = { "MasterMemberCard": [member_card(1, 1, 1), member_card(2, 2, 2), member_card(3, 3, 1), member_card(4, 4, 3), member_card(5, 5, 2), member_card(6, 6, 0), member_card(7, 3, 3), member_card(8, 1, 2)], @@ -80,14 +80,23 @@ def effect(table, key, i, skill, level, group=0): {"_id": 31, "_nameTextID": "GS31", "_descriptionTextFormatID": "GSD31", "_gekisouSupportSkillExecTiming": 1, "_gekisouMissionType": 3}, {"_id": 32, "_nameTextID": "GS32", "_descriptionTextFormatID": "", "_gekisouSupportSkillExecTiming": 1, - "_gekisouMissionType": 1}], + "_gekisouMissionType": 1}, + {"_id": 33, "_nameTextID": "GS33", "_descriptionTextFormatID": "", "_gekisouSupportSkillExecTiming": 1, + "_gekisouMissionType": 3}], "MasterGekisouSupportSkillEffect": [ effect("MasterGekisouSupportSkillEffect", "_gekisouSupportSkillID", 100 * g + 10 * s + lv, s, lv, g) - for s, groups in ((31, (100, 101)), (32, (0,))) for lv in range(1, 6) for g in groups], - "MasterSupportCard": [snap(41, 1, 31), snap(42, 2, 32), snap(43, 3, 0), snap(44, 4, 31)], + for s, groups in ((31, (100, 101)), (32, (0,)), (33, (102, 103))) for lv in range(1, 6) for g in groups], + "MasterSupportCard": [snap(41, 1, 31), snap(42, 2, 32), snap(43, 3, 0), snap(44, 4, 31), snap(45, 5, 33)], "MasterSupportCardRank": [ columns("MasterSupportCardRank", _id=r, _group=1, _rank=r, _gekisouSupportSkill01Level=r, _gekisouSupportSkill02Level=r) for r in range(1, 6)], + # the band conditions: group 100 (102) a member of band 1 (2), group 101 (103) not + "MasterSkillConditionSet": [{"_id": i, "_group": 99 + i, "_conditionIds": [i]} for i in (1, 2, 3, 4)], + "MasterSkillCondition": [ + {"_id": i, "_conditionType": 5000, "_conditionValues": [], "_isPositive": i in (1, 3), + "_conditionTargetIDs": [3 if i <= 2 else 4]} for i in (1, 2, 3, 4)], + "MasterSkillTarget": [columns("MasterSkillTarget", _id=t, _skillTargetType=3, _bandID=t - 2, _judgement=-1) + for t in (3, 4)], } TABLE_ROWS = { "MasterLiveMusic": [MUSIC, MUSIC1], @@ -105,9 +114,9 @@ def effect(table, key, i, skill, level, group=0): text("Ch1s", "tomo"), text("Tag1", "tag"), text("Cat1", "original")] + [text(f"Ch{i}", f"char{i}") for i in range(2, 7)] + [text(f"Card{i}", f"card{i}") for i in range(1, 9)] + [text(f"Sub{i}", f"sub{i}") for i in (1, 2)] - + [text(f"Snap{i}", f"snap{i}") for i in (41, 42, 43, 44)] + [text("SnapSub41", "snapsub41")] - + [text(f"GS{i}", f"gs{i}") for i in (1, 2, 3, 31, 32)] + [text(f"GSD{i}", f"gsd{i}") - for i in (1, 2, 31)], + + [text(f"Snap{i}", f"snap{i}") for i in (41, 42, 43, 44, 45)] + [text("SnapSub41", "snapsub41")] + + [text(f"GS{i}", f"gs{i}") for i in (1, 2, 3, 31, 32, 33)] + [text(f"GSD{i}", f"gsd{i}") + for i in (1, 2, 31)], "MasterBand": [{"_id": 1, "_nameTextID": "Band1", "_mainColorCode": "#3388BB", "_subColorCode": "#FFFFFF"}, {"_id": 2, "_nameTextID": "Band2", "_mainColorCode": "#881122", "_subColorCode": "#000000"}], # characters 1..4 of band 1, 5 and 6 of band 2 @@ -169,25 +178,106 @@ def trunc_percent(score, percent): return int(score * percent / 100) +def effect_row(level, *band): + """A Gekisou (support) skill effect row of GEKISOU_ROWS as the shapes write it: score up by 100 * level, with a + band condition (positive or not) when given.""" + condition = [[{"type": 5000, "values": [], "positive": b, "targetIds": None}] for b in band] + return {"effectType": 2000, "triggerType": 0, "activationTimeSecond": 0, "effectValue": 100 * level, + "maxEffectValue": 0, "effectLimitCount": 0, "effectExecuteLimitCount": 0, "skillTargetIds": [], + "trigger": [], "condition": condition, "release": [], "reset": [], "cumulative": None} + + +def shape_skill(i, level, targets=None, bands=None): + return {"id": i, "level": level, "memberTargetIds": targets, "bandIds": bands} + + +# the shape table of GEKISOU_ROWS: support skills 31 and 33 differ in their band only +SHAPES = [ + {"id": 0, "source": "member", "mission": 1, "bandCondition": False, "effects": [effect_row(5)], + "skills": [shape_skill(1, 5)]}, + {"id": 1, "source": "member", "mission": 3, "bandCondition": False, "effects": [effect_row(3)], + "skills": [shape_skill(2, 3)]}, + {"id": 2, "source": "member", "mission": 2, "bandCondition": False, "effects": [effect_row(5)], + "skills": [shape_skill(3, 5)]}, + {"id": 3, "source": "support", "mission": 3, "bandCondition": True, + "effects": [effect_row(5, True), effect_row(5, False)], + "skills": [shape_skill(31, 5, [3], [1]), shape_skill(33, 5, [4], [2])]}, + {"id": 4, "source": "support", "mission": 1, "bandCondition": False, "effects": [effect_row(5)], + "skills": [shape_skill(32, 5)]}, +] +SEED_RULE = {"deterministicTest": 4, "batches": [32, 64, 128, 256, 512, 1024], "relative": 0.01, "baseline": 0.001, + "crossSeeds": 64} +NONDETERMINISTIC = {2, 3} # the shapes the stand-in measures on 32 seeds + + +def mean_se(xs): + m = sum(xs) / len(xs) + return [m, (sum((x - m) ** 2 for x in xs) / (len(xs) - 1) / len(xs)) ** 0.5 if len(xs) > 1 else 0.0] + + +def aptitude_of(s, positions): + """The stand-in's Gekisou aptitude of a chart's statistics `s`: per range 5 judged notes (Perfect in a Just count + range), per variant +10 * (j + 1) in range j and 7 after the ranges; a nondeterministic shape's means a quarter + point off with a standard error of 0.5.""" + ranges, base = s["ranges"], s["seeds"][0] + missions = {r["mission"] for r in ranges} + factors = [{"judgedNotes": 5, "justNotes": 0, "perfectNotes": 5 if r["mission"] == 3 else 0, "tailNotes": 1, + "comboAtStart": 0, + "lotteries": mean_se([sum(x["ranges"][j]["lotResults"]) for x in s["seeds"]]) if r["mission"] == 2 + else [0, 0]} for j, r in enumerate(ranges)] + variants = [] + for shape in SHAPES: + if shape["mission"] != 4 and shape["mission"] not in missions: + continue + for band in ((True, False) if shape["bandCondition"] else (None,)): + det = shape["id"] not in NONDETERMINISTIC and 2 not in missions + off, se = (0, 0) if det else (0.25, 0.5) + rs, inside, inside_p = [], 0, 0 + for j, (r, r0) in enumerate(zip(ranges, base["ranges"])): + d = 10 * (j + 1) + rb = trunc_percent(r0["rangeScore"] + d, r["rankBonusPercent"]) - r0["rankBonus"] + rbp = trunc_percent(r0["rangeScorePerfect"] + d, r["rankBonusPercent"]) - \ + trunc_percent(r0["rangeScorePerfect"], r["rankBonusPercent"]) + rs.append({"rangeScore": [d + off, se], "rankBonus": [rb + off, se], "rangeScorePerfect": [d + off, se], + "maxCombo": [0, 0], "justCount": [1 + off, se], "luckPoints": [off, se]}) + inside += d + rb + 2 * off + inside_p += d + rbp + seeds = 1 if det else 32 + variants.append({ + "shape": shape["id"], "bandMatch": band, "deterministic": det, "seeds": seeds, "seTargetMet": True, + "crossSeeds": min(seeds, SEED_RULE["crossSeeds"]), + "score": [inside + 7, se], "scorePerfect": [inside_p + 7 + off, se], "tail": [7, se], + "tailPerfect": [7, se], "converted": [0, 0], "ranges": rs, + "weights": [[0.01, se / 100]] * positions, + "rangeWeights": [[[0.001, 0]] * len(ranges)] * positions if base["rangeWeights"] is not None else None, + "check": {"seed": base["seed"] if det else musicdata.published_seeds(1)[0], "ranks": [2 if base["rangeWeights"] is not None else 1] * len(ranges), + "deck": [[0, 5000]] + [None] * (positions - 1), "exact": 2000, "predicted": 2000.25, + "bound": 7.0}}) + return {"factors": factors, "variants": variants} + + class FakeDeck: - """The interface of nnnotes._deck: statistics made from the deck input as the model reports them.""" + """The interface of nnnotes._deck: statistics made from the deck input as the model reports them, with the Gekisou + aptitude (SHAPES; `aptitude`) unless it is left out.""" COMMIT = "7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6" FORMAT = "ournotes-deck.chart-stats/2" - def __init__(self, change=None, fail=None): - self.change, self.fail, self.inputs = change, fail, [] + def __init__(self, change=None, fail=None, header=None): + self.change, self.fail, self.header, self.inputs, self.options = change, fail, header, [], [] def info(self): return {"name": "ournotes-deck", "version": "0.0.1", "source": "https://github.com/empty-sekai/ournotes-deck", "commit": self.COMMIT, "dataFormat": deckdata.DECK_FORMAT, "format": self.FORMAT} - def chart_stats(self, data, seeds, workers): + def chart_stats(self, data, seeds, workers, aptitude=True, aptitude_max_seeds=None, aptitude_cross_seeds=None): if self.fail: raise ValueError(self.fail) doc = json.loads(data) self.inputs.append((doc, seeds, workers)) + self.options.append((aptitude, aptitude_max_seeds, aptitude_cross_seeds)) assert doc["format"] == deckdata.DECK_FORMAT and list(doc["master"]) == [t for t, _ in deckdata.TABLES] m = doc["master"] + with_aptitude = aptitude def table(name): return [dict(zip(m[name]["columns"], r)) for r in m[name]["rows"]] @@ -209,17 +299,18 @@ def table(name): pattern = musicdata.mission_pattern(missions) percents = [[bonus.get((pattern, i + 1, k), 0) for k in range(1, 6)] for i in range(len(fevers))] check = {"deck": [[0, 5000]], "exact": 2000, "predicted": 2000.25, "bound": 7.0} + luck = musicdata.MISSION_LUCK in missions[:len(fevers)] - def seed_stats(seed): + def seed_stats(i, seed): return {"seed": seed, "score": 1234, - "ranges": [{"rangeScore": 101 * (i + 1), "rankBonus": trunc_percent(101 * (i + 1), p[0]), - "maxCombo": 1, "justCount": 0, "luckPoints": 0, "lotResults": [0, 0, 0, 0], - "rangeScorePerfect": 101 * (i + 1)} for i, p in enumerate(percents)], + "ranges": [{"rangeScore": 101 * (j + 1), "rankBonus": trunc_percent(101 * (j + 1), p[0]), + "maxCombo": 1, "justCount": 0, "luckPoints": 0, + "lotResults": [i % 3, 1, 0, 0] if luck else [0, 0, 0, 0], + "rangeScorePerfect": 101 * (j + 1)} for j, p in enumerate(percents)], "weights": [["W"] * positions], "check": dict(check), "scorePerfect": 1234, "rangeWeights": [[["W"] * len(fevers)] * positions], "rankCheck": {"ranks": [2] * len(fevers), "exact": 2001, "predicted": 2000.5, "bound": 7.0} if fevers else None} - luck = musicdata.MISSION_LUCK in missions[:len(fevers)] s = {"scoreId": c["scoreId"], "musicId": music, "difficulty": d, "level": levels[c["scoreId"]], "judgedNotes": sum(score.is_judgement_note(op) for op in c["notes"]["op"]), "convertedNoteCount": len(c["notes"]["id"]), "lastNoteMs": last, "musicLengthMs": last + 1000, @@ -229,14 +320,21 @@ def seed_stats(seed): "rankBonusPercent": percents[i][0], "rankBonusPercents": percents[i]} for i, (a, b) in enumerate(fevers)], "justNotes": 0, - "seeds": [seed_stats(x) for x in (musicdata.published_seeds(seeds) if luck else [0])], + "seeds": [seed_stats(i, x) for i, x in enumerate(musicdata.published_seeds(seeds) if luck else [0])], "offSeeds": [{"seed": 0, "score": 1000, "weights": [["W"] * positions], "check": dict(check)}]} + s["gekisouAptitude"] = aptitude_of(s, positions) if with_aptitude and fevers else None if self.change: self.change(s) charts.append(s) - out = json.dumps({"format": self.FORMAT, "source": {}, "model": {"power": 300000}, - "kinds": [{"id": 0, "effectType": 2000, "activationTimeSecond": 7.5}], "charts": charts}) - return out.replace('"W"', WEIGHT) + out = {"format": self.FORMAT, "source": {}, + "model": {"power": 300000, "gekisouAptitude": "each Gekisou (support) skill shape alone"}, + "gekisouAptitude": {"plainKind": 0, "host": "a synthetic member without a Gekisou skill", + "seedRule": SEED_RULE, "shapes": SHAPES} if aptitude else None, + "kinds": [{"id": 0, "effectType": 2000, "activationTimeSecond": 7.5}], "charts": charts} + out = json.loads(json.dumps(out)) + if self.header: + self.header(out) + return json.dumps(out).replace('"W"', WEIGHT) def export(tmp_path, rows=TABLE_ROWS, charts=CHARTS, bgm=bgm, out="music.json", deck=None, **kw): @@ -339,8 +437,8 @@ def test_deck(tmp_path): p = doc["provenance"] assert p["deck"] == {k: fake.info()[k] for k in ("name", "version", "source", "commit", "format")} assert list(p["master"]["tables"]) == list(musicdata.tables_of(True, False)) - assert doc["deck"] == {"model": {"power": 300000}, - "kinds": [{"id": 0, "effectType": 2000, "activationTimeSecond": 7.5}]} + assert doc["deck"]["model"]["power"] == 300000 + assert doc["deck"]["gekisouAptitude"]["shapes"] == SHAPES c = doc["songs"][0]["charts"][0] assert list(c["deck"]) == list(musicdata.DECK_CHART_KEYS) assert c["deck"]["events"] == [[0, 2500], [1, 500]] and c["deck"]["ranges"][1]["startMs"] == 2000 @@ -399,6 +497,10 @@ def change(s): s["seeds"][0]["rangeWeights"] = None if s["scoreId"] == 10 else [None] s["seeds"][0]["rankCheck"] = None s["offSeeds"][0]["weights"] = [None] + if s["scoreId"] == 10: + for v in s["gekisouAptitude"]["variants"]: + v["rangeWeights"] = None + v["check"]["ranks"] = [1] * len(s["ranges"]) export(tmp_path, deck=FakeDeck(change)) doc = json.loads((tmp_path / "music.json").read_bytes()) decks = {c["scoreId"]: c["deck"] for s in doc["songs"] for c in s["charts"]} @@ -418,7 +520,8 @@ def test_gekisou_catalog(tmp_path): "zh-Hans": "gsd1-cn", "ko": "gsd1-ko"}} assert [(s["id"], s["mission"], s["maxLevel"]) for s in cat["skills"]] == [(1, 1, 5), (2, 3, 3), (3, 2, 5)] assert cat["skills"][2]["description"] is None # no text id - assert [(s["id"], s["mission"], s["maxLevel"]) for s in cat["supportSkills"]] == [(31, 3, 5), (32, 1, 5)] + assert [(s["id"], s["mission"], s["maxLevel"]) for s in cat["supportSkills"]] == [(31, 3, 5), (32, 1, 5), + (33, 3, 5)] assert cat["supportSkills"][0]["name"]["en"] == "gs31-en" and cat["supportSkills"][1]["description"] is None assert cat["members"][0] == {"id": 1, "characterId": 1, "bandId": 1, "rarity": 3, "gekisouSkillId": 1, "name": {"ja": "card1-ja", "en": "card1-en", "zh-Hant": "card1-tw", @@ -431,7 +534,7 @@ def test_gekisou_catalog(tmp_path): assert cat["members"][2]["subtitle"] is None assert [(s["id"], s["characterIds"], s["rarity"], s["gekisouSupportSkillIds"], s["supportSkillLevel"]) for s in cat["snaps"]] == [(41, [1], 2, [31], 5), (42, [2], 2, [32], 5), (43, [3], 2, [], 5), - (44, [4], 2, [31], 5)] + (44, [4], 2, [31], 5), (45, [5], 2, [33], 5)] assert cat["snaps"][0]["name"]["ja"] == "snap41-ja" and cat["snaps"][0]["subtitle"]["ko"] == "snapsub41-ko" assert cat["snaps"][1]["subtitle"] is None schema_validator().validate(json.loads((tmp_path / "music.json").read_bytes())) @@ -697,3 +800,130 @@ def test_command_decoded_master(tmp_path, capsys, monkeypatch): capsys) assert code == 0, err assert out.read_bytes() == (tmp_path / "f.json").read_bytes() + + +# ---------------------------------------------------------------- Gekisou aptitude gates +@pytest.fixture +def aptitude_fixture(): + """A header and a two-range chart, independently authored above, through JSON as the extension returns it.""" + tables = {t: TABLE_ROWS.get(t, rows_of(t)) for t in musicdata.tables_of(True, False)} + a = musicdata.Aptitude(tables, musicdata.gekisou_catalog(tables, musicdata.Texts(tables['MasterText']))) + header = json.loads(json.dumps({'plainKind': 0, 'host': 'synthetic host', 'seedRule': SEED_RULE, 'shapes': SHAPES}), + parse_float=deckdata._Num) + chart = {'positions': 2, 'judgedNotes': 20, 'justNotes': 0, 'ranges': [ + {'mission': 1, 'rankBonusPercent': 35}, {'mission': 3, 'rankBonusPercent': 40}], + 'seeds': [{'seed': 0, 'rangeWeights': [], 'ranges': [ + {'rangeScore': 101, 'rangeScorePerfect': 101, 'rankBonus': 35, 'lotResults': [0, 0, 0, 0]}, + {'rangeScore': 202, 'rangeScorePerfect': 202, 'rankBonus': 80, 'lotResults': [0, 0, 0, 0]}]}]} + chart['gekisouAptitude'] = aptitude_of(chart, 2) + chart = json.loads(json.dumps(chart), parse_float=deckdata._Num) + return a, header, chart + + +def aptitude_header(a, h): + return a.check_header(h, [{'id': 0, 'effectType': 2000}], {'gekisouAptitude': 'single shape'}) + + +def test_aptitude_fixture(aptitude_fixture): + a, h, c = aptitude_fixture + aptitude_header(a, h) + assert a.chart('synthetic', c, 0) == c['gekisouAptitude'] + assert [(v['shape'], v['bandMatch']) for v in c['gekisouAptitude']['variants']] == [ + (0, None), (1, None), (3, True), (3, False), (4, None)] + assert h['shapes'][3]['skills'][1]['bandIds'] == [2] + + +@pytest.mark.parametrize('change', [ + lambda h: h['shapes'][0].update(id=8), + lambda h: h['shapes'][0].update(id=False), + lambda h: h['shapes'][0].update(mission=4), + lambda h: h['shapes'][0].update(mission=8), + lambda h: h['shapes'][0].update(source='unknown'), + lambda h: h['shapes'][0].update(bandCondition=True), + lambda h: h['shapes'][0]['skills'][0].update(level=4), + lambda h: h['shapes'][0]['skills'][0].update(id=99), + lambda h: h['shapes'][0]['skills'][0].pop('bandIds'), + lambda h: h['shapes'][0]['skills'].append(h['shapes'][0]['skills'][0]), + lambda h: h['shapes'][3]['skills'][0].update(bandIds=[2]), + lambda h: h['shapes'][3]['skills'][0].update(memberTargetIds=[4]), + lambda h: h['shapes'][0]['effects'][0].update(effectValue=999), + lambda h: h['shapes'].pop(), + lambda h: h.update(plainKind=1), + lambda h: h.update(host=''), + lambda h: h['seedRule'].update(batches=[32, 16]), + lambda h: h['seedRule'].update(crossSeeds=0), + lambda h: h['seedRule'].update(relative=deckdata._Num('NaN')), +]) +def test_aptitude_header_rejects(aptitude_fixture, change): + a, h, _ = aptitude_fixture + change(h) + with pytest.raises(musicdata.MusicDataError): + aptitude_header(a, h) + + +@pytest.mark.parametrize('change', [ + lambda c: c.update(gekisouAptitude=None), + lambda c: c.update(unplayable='no table'), + lambda c: c['gekisouAptitude']['factors'].pop(), + lambda c: c['gekisouAptitude']['factors'][0].update(judgedNotes=-1), + lambda c: c['gekisouAptitude']['factors'][0].update(lotteries=[1, 0]), + lambda c: c['gekisouAptitude']['variants'].pop(), + lambda c: c['gekisouAptitude']['variants'][0].update(shape=99), + lambda c: c['gekisouAptitude']['variants'][0].update(shape=False), + lambda c: c['gekisouAptitude']['variants'][2].update(bandMatch=False), + lambda c: c['gekisouAptitude']['variants'][0].update(score=[1]), + lambda c: c['gekisouAptitude']['variants'][0].update(score=[deckdata._Num('NaN'), 0]), + lambda c: c['gekisouAptitude']['variants'][0].update(score=[10**400, 0]), + lambda c: c['gekisouAptitude']['variants'][0].update(score=[1, -1]), + lambda c: c['gekisouAptitude']['variants'][0].update(tail=[7, 1]), + lambda c: c['gekisouAptitude']['variants'][0].update(tail=[8, 0]), + lambda c: c['gekisouAptitude']['variants'][0]['ranges'].pop(), + lambda c: c['gekisouAptitude']['variants'][0].update(seeds=2), + lambda c: c['gekisouAptitude']['variants'][0].update(crossSeeds=2), + lambda c: c['gekisouAptitude']['variants'][2].update(seeds=31), + lambda c: c['gekisouAptitude']['variants'][2].update(seTargetMet=False), + lambda c: c['gekisouAptitude']['variants'][0]['check'].update(bound=-1), + lambda c: c['gekisouAptitude']['variants'][0]['check'].update(predicted=deckdata._Num('NaN')), + lambda c: c['gekisouAptitude']['variants'][0]['check'].update(exact=0), + lambda c: c['gekisouAptitude']['variants'][0]['check'].update(ranks=[0, 2]), + lambda c: c['gekisouAptitude']['variants'][0]['check'].update(deck=[[9, 100], None]), +]) +def test_aptitude_chart_rejects(aptitude_fixture, change): + a, h, c = aptitude_fixture + aptitude_header(a, h) + change(c) + with pytest.raises(musicdata.MusicDataError): + a.chart('synthetic', c, 0) + + +def test_aptitude_rounding(aptitude_fixture): + a, h, c = aptitude_fixture + aptitude_header(a, h) + v = c['gekisouAptitude']['variants'][2] + v['tail'][0] = deckdata._Num('7.002') + a.chart('synthetic', c, 0) # independently rounded terms, two ranges: 0.00300001 absolute tolerance + v['tail'][0] = deckdata._Num('7.004') + with pytest.raises(musicdata.MusicDataError, match='tail'): + a.chart('synthetic', c, 0) + + +def test_aptitude_no_plain(aptitude_fixture): + a, h, c = aptitude_fixture + h['plainKind'] = None + a.check_header(h, [], {'gekisouAptitude': 'single shape'}) + for v in c['gekisouAptitude']['variants']: + v.update(weights=None, rangeWeights=None) + v['check']['deck'] = [None, None] + a.chart('synthetic', c, None) + + +def test_aptitude_disabled(tmp_path): + fake = FakeDeck() + d = master_dir(tmp_path) + musicdata.export(tmp_path / 'music.json', deckdata.master_files(d), KEY, CHARTS.__getitem__, bgm, **PROV, + deck=musicdata.Deck(module=fake, aptitude=False, aptitude_max_seeds=32, aptitude_cross_seeds=8)) + doc = json.loads((tmp_path / 'music.json').read_bytes()) + assert fake.options == [(False, 32, 8)] + assert doc['deck']['gekisouAptitude'] is None + assert all(c['deck']['gekisouAptitude'] is None for s in doc['songs'] for c in s['charts']) + schema_validator().validate(doc) From 534a20a871d292390e5033adbe21135f8d1725c5 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 30 Sep 2026 05:03:08 +0800 Subject: [PATCH 3/4] build(deck): pin the final Gekisou aptitude implementation Co-Authored-By: Claude Code --- rust/Cargo.lock | 2 +- rust/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 7375f82..ad82f2b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -44,7 +44,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "ournotes-deck" version = "0.0.1" -source = "git+https://github.com/empty-sekai/ournotes-deck?rev=94281d18ce50c991d7c8a082d2a32a4522050454#94281d18ce50c991d7c8a082d2a32a4522050454" +source = "git+https://github.com/empty-sekai/ournotes-deck?rev=155bd2ad89d077d3bdcfbca9c45c0100475ab538#155bd2ad89d077d3bdcfbca9c45c0100475ab538" dependencies = [ "serde", "serde_json", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index cc698c5..727c272 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -15,7 +15,7 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.29", features = ["abi3-py311"] } # pinned by commit; .github/workflows/deck.yml opens a pull request when ournotes-deck moves -ournotes-deck = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "94281d18ce50c991d7c8a082d2a32a4522050454" } +ournotes-deck = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "155bd2ad89d077d3bdcfbca9c45c0100475ab538" } serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } [profile.release] From c1646206460744aa536ebffd46f5a0288d0f1e6f Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 30 Sep 2026 13:37:22 +0800 Subject: [PATCH 4/4] feat: export versioned Rust replay data with chart statistics Export normalized charts, actual music length and SHA-bound WASM manifests. Pin the native-validated deck model and keep both aptitude precision gates. Publish the shared song-page modules alongside the existing players. Co-Authored-By: Claude Code --- docs/music-data.md | 7 +- docs/replay.md | 34 ++++++++++ docs/schema/music-data.schema.json | 26 ++++++++ rust/Cargo.lock | 2 +- rust/Cargo.toml | 2 +- rust/src/lib.rs | 2 +- src/nnnotes/cli.py | 14 +++- src/nnnotes/musicdata.py | 44 +++++++++++-- src/nnnotes/replaydata.py | 101 +++++++++++++++++++++++++++++ src/nnnotes/web.py | 11 +++- tests/test_replaydata.py | 85 ++++++++++++++++++++++++ tests/test_web.py | 19 ++++++ 12 files changed, 334 insertions(+), 13 deletions(-) create mode 100644 docs/replay.md create mode 100644 src/nnnotes/replaydata.py create mode 100644 tests/test_replaydata.py diff --git a/docs/music-data.md b/docs/music-data.md index f6216ac..357b6e7 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -31,7 +31,7 @@ nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--ful charts. `--workers N` sets the threads it uses (default: every processor), `--seeds N` the seeds measured on a chart with a luck range (default 8). - `--no-gekisou-aptitude`: keep the existing deck statistics but skip single-skill aptitude measurements. - `--aptitude-max-seeds N` caps their samples (default 1024), and `--aptitude-cross-seeds N` caps cross-term samples + `--aptitude-max-seeds N` caps their samples (default 65536), and `--aptitude-cross-seeds N` caps cross-term samples (default 64). Lower caps reduce work but may leave the standard-error target unmet. - `--no-bgm`: do not read the cue sheets (every `bgm.length` is null). - `--jackets DIR`: also write every song's jacket, the Texture2D `Image/Jacket/`, as `DIR/.webp` @@ -449,3 +449,8 @@ version of this file. `nnnotes.music-data/1` replaces the `nnnotes.songs/1` file of `nnnotes songs` (its fields are the songs, charts and their facts here) and the `nnnotes.deck-data/1` file of `nnnotes deck-data` (its content is the deck input of `--full`). + + +Final aptitude exports require the standard-error target for both `score` and `scorePerfect`, each against its own paired no-skill baseline. Sampling extends the same published seed prefix through geometric batches, stopping as soon as both targets agree with the unchanged max(1% of increment, 0.1% of baseline) rule. The 65,536-seed cap is a failure guard, not a requirement to run every seed. An unmet cap aborts a normal export before writing artifacts. `--allow-unconverged-aptitude` is an explicit diagnostic option; it retains real SE values and unmet flags. + +`--replay-dir OUT/replay --replay-engine WASM_PKG` writes normalized runtime inputs and pinned WASM assets as described in [replay.md](replay.md). The music data stays compact and carries the SHA-bound `replay.manifestUrl` pointer. No original chart/master blobs or native binary are included in this artifact bundle. diff --git a/docs/replay.md b/docs/replay.md new file mode 100644 index 0000000..8ab44ea --- /dev/null +++ b/docs/replay.md @@ -0,0 +1,34 @@ +# Shared Rust/WASM replay inputs + +`nnnotes music-data --replay-dir OUT/replay --replay-engine WASM_PKG -o OUT/music-data.json` writes the normalized input of the pinned deck model. The engine package must contain `ournotes_replay.js`, `ournotes_replay_bg.wasm` and `build.json`; its commit and both file hashes must match the measured model. Actual ACB cue length is required. Original encrypted master files, chart blobs and native binaries are not copied. + +The music-data pointer is: + +```json +{"replay":{"format":"nnnotes.replay-manifest/1","manifestUrl":"replay//manifest.json","sha256":"","charts":340}} +``` + +`manifestUrl` is relative to the music-data URL. The manifest hash names its directory, so uploading a new release preserves the preceding document's resource tree. Every manifest resource URL below is relative to the manifest URL: + +```json +{ + "format": "nnnotes.replay-manifest/1", + "deckData": {"format":"nnnotes.deck-data/1","url":"deck-data.json","sha256":"","bytes":0}, + "charts": [{"scoreId":10000303,"url":"charts/10000303.json","sha256":"","bytes":0,"musicLengthMs":120557,"noteCount":0,"assetSha256":""}], + "engine": { + "model": {"name":"ournotes-deck","version":"","source":"","commit":"","format":"ournotes-deck.chart-stats/2"}, + "requestFormat":"ournotes.replay/1","class":"ReplaySession","methods":["describeChart","template","run"], + "js":{"url":"engine/ournotes_replay.js","sha256":"","bytes":0}, + "wasm":{"url":"engine/ournotes_replay_bg.wasm","sha256":"","bytes":0}, + "build":{"url":"engine/build.json","sha256":"","bytes":0} + }, + "unlistedScoreIds":[], + "clock":"Explicit frames from ReplaySession.template; no Python/JS scoring or scheduling" +} +``` + +Numbers and SHA placeholders above illustrate the schema; actual manifests contain measured sizes, complete chart entries and computed hashes. A data-only export can leave `engine` null. A publishable interactive bundle supplies the pinned engine. Normalized runtime rows belonging to no listed live song are omitted and recorded in `unlistedScoreIds`. + +Each chart resource is `{format:"nnnotes.replay-chart/1",scoreId,musicLengthMs,chart}`. `chart` retains the existing DeckData record: `asset:{key,sha256}`, equal-length `notes:{id,op,judgementType,timeMs}` arrays in native enumeration order, `skillEvents:{timeMs}` and `fevers:{startMs,endMs}`. `deck-data.json` uses the existing `nnnotes.deck-data/1` schema and adds `provenance.replay.musicLengthsMs:{"":}`. Score-table length is a separate native value derived by the shared Rust chart implementation; it must not be replaced by audio length. + +The page loads and hashes the module/data once, calls `new ReplaySession(deckDataJson)`, then parses JSON returned by `describeChart(scoreId)`, `template(scoreId,power,fps)` and `run(requestJson)`. Template construction stays in Rust. It preserves simultaneous note order, supplies Perfect for judgement notes and native Pass for other operations, and does not guess Just. The request has `format:"ournotes.replay/1"`, explicit `scoreId`, `power`, audio and score-table lengths, `seed`, five performers, a `skillOrder` permutation, mode, and frames `{timeMs,deltaSeconds,judgements:[{noteId,judgement,judgementTimeMs}]}`. Complete input is required; missing notes are not silently changed to Miss. Advanced skill/rank scenarios use the same request JSON and engine. diff --git a/docs/schema/music-data.schema.json b/docs/schema/music-data.schema.json index 2dc97a3..c344bab 100644 --- a/docs/schema/music-data.schema.json +++ b/docs/schema/music-data.schema.json @@ -463,6 +463,32 @@ "items": { "$ref": "#/$defs/chart" } + }, + "replay": { + "type": "object", + "required": [ + "format", + "manifestUrl", + "sha256", + "charts" + ], + "properties": { + "format": { + "const": "nnnotes.replay-manifest/1" + }, + "manifestUrl": { + "type": "string" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "charts": { + "type": "integer", + "minimum": 0 + } + }, + "description": "Optional normalized replay artifact manifest, URL relative to the music-data URL; shared Rust/WASM engine only." } }, "$defs": { diff --git a/rust/Cargo.lock b/rust/Cargo.lock index ad82f2b..149336a 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -44,7 +44,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "ournotes-deck" version = "0.0.1" -source = "git+https://github.com/empty-sekai/ournotes-deck?rev=155bd2ad89d077d3bdcfbca9c45c0100475ab538#155bd2ad89d077d3bdcfbca9c45c0100475ab538" +source = "git+https://github.com/empty-sekai/ournotes-deck?rev=dbd9cf01a4854808dceb6aedcd4041af372faeee#dbd9cf01a4854808dceb6aedcd4041af372faeee" dependencies = [ "serde", "serde_json", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 727c272..3d8acac 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -15,7 +15,7 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.29", features = ["abi3-py311"] } # pinned by commit; .github/workflows/deck.yml opens a pull request when ournotes-deck moves -ournotes-deck = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "155bd2ad89d077d3bdcfbca9c45c0100475ab538" } +ournotes-deck = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "dbd9cf01a4854808dceb6aedcd4041af372faeee" } serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } [profile.release] diff --git a/rust/src/lib.rs b/rust/src/lib.rs index d4ca68f..65643ca 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -60,7 +60,7 @@ fn measure(data: &DeckData, options: &Options, workers: usize) -> Result dict: """{name, version, source, commit, format} of the deck model.""" @@ -336,6 +338,16 @@ def stats(self, deck_input: dict) -> dict: doc = json.loads(text, parse_float=deckdata._Num) if doc.get("format") != self.info()["format"]: raise MusicDataError(f"deck model: wrote {doc.get('format')!r}, expected {self.info()['format']!r}") + if self.require_convergence and self.aptitude: + unmet = [(c.get("scoreId"), v.get("shape"), v.get("bandMatch"), v.get("seeds")) + for c in doc.get("charts", []) + for v in (c.get("gekisouAptitude") or {}).get("variants", []) + if v.get("seTargetMet") is not True] + if unmet: + chart, shape, band, seeds = unmet[0] + raise MusicDataError(f"deck model: {len(unmet)} aptitude variants did not meet BOTH score SE targets; " + f"chart {chart}, shape {shape}, bandMatch {band}, seeds {seeds}; " + "increase --aptitude-max-seeds; no final artifact written") return doc @@ -1077,16 +1089,35 @@ def tables_of(deck: bool, full: bool) -> tuple[str, ...]: def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes], bgm: Callable[[str, str], dict] | None, *, region: str, client: dict, catalog: dict, deck: Deck | None = None, full: bool = False, jacket: Callable[[str], bytes] | None = None, - jackets_dir=None) -> dict: + jackets_dir=None, replay_dir=None, replay_engine=None) -> dict: """Read the master data, every chart and every BGM cue sheet, measure the charts with `deck`, then write the file `out` (gzip when it ends in `.gz`) through a temporary file and a rename; with `jacket` and `jackets_dir`, first every song's jacket as `/.webp`. Returns the summary.""" from .cache import write_atomic out = Path(out) try: - tables, shas = deckdata.read_master(src, key, tables_of(deck is not None, full)) + if replay_engine is not None and replay_dir is None: + raise deckdata.DeckDataError("--replay-engine needs --replay-dir") + if replay_dir is not None and bgm is None: + raise deckdata.DeckDataError("replay export requires actual BGM lengths; --no-bgm is incompatible") + tables, shas = deckdata.read_master(src, key, tables_of(deck is not None, full or replay_dir is not None)) doc = build(tables, shas, fetch, bgm, region=region, client=client, catalog=catalog, - master_source=src.source, master_version=src.version, deck=deck, full=full) + master_source=src.source, master_version=src.version, deck=deck, full=full or replay_dir is not None) + replay_files = None + if replay_dir is not None: + from . import replaydata + replay_files, replay_manifest = replaydata.bundle(doc, replay_engine) + manifest_sha = hashlib.sha256(replay_files["manifest.json"]).hexdigest() + replay_target = Path(replay_dir) / manifest_sha + try: + relative = replay_target.resolve().relative_to(out.parent.resolve()).as_posix() + except ValueError: + raise deckdata.DeckDataError("replay directory must be under the music-data output directory") from None + doc["replay"] = {"format": replaydata.FORMAT, "manifestUrl": relative + "/manifest.json", + "sha256": manifest_sha, + "charts": len(replay_manifest["charts"])} + if not full: + del doc["master"], doc["charts"] jackets = sorted({s["jacket"] for s in doc["songs"] if s["jacket"]}) if jacket is not None else [] images = {name: jacket(name) for name in jackets} data = deckdata.encode(deckdata._value(doc, "music data")) @@ -1098,6 +1129,8 @@ def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes], for name, image in images.items(): write_atomic(d / f"{name}.webp", image) written = deckdata.file_bytes(data, out.name.endswith(".gz")) + if replay_files is not None: + replaydata.write(replay_target, replay_files) out.parent.mkdir(parents=True, exist_ok=True) write_atomic(out, written) charts = [c for s in doc["songs"] for c in s["charts"]] @@ -1106,4 +1139,5 @@ def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes], "deck": doc["provenance"]["deck"]["commit"] if deck is not None else None, "unplayable": sum(1 for c in charts if c["deck"] and c["deck"]["unplayable"]), "full": full, "bgm": bgm is not None, "jackets": len(jackets), - "bytes": len(data), "fileBytes": len(written), "sha256": hashlib.sha256(written).hexdigest()} + "bytes": len(data), "fileBytes": len(written), "sha256": hashlib.sha256(written).hexdigest(), + **({"replay": doc["replay"]} if replay_files is not None else {})} diff --git a/src/nnnotes/replaydata.py b/src/nnnotes/replaydata.py new file mode 100644 index 0000000..267819e --- /dev/null +++ b/src/nnnotes/replaydata.py @@ -0,0 +1,101 @@ +"""Normalized runtime inputs for the shared Rust/WASM replay; no scoring or frame scheduler here.""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from . import deckdata +from .cache import write_atomic + +FORMAT = "nnnotes.replay-manifest/1" +CHART_FORMAT = "nnnotes.replay-chart/1" +JS = "ournotes_replay.js" +WASM = "ournotes_replay_bg.wasm" + + +def _json(value) -> bytes: + return deckdata.encode(deckdata._value(value, "replay input")) + + +def bundle(music: dict, engine_dir: Path | None = None) -> tuple[dict[str, bytes], dict]: + """Canonical normalized --full inputs and SHA manifest. Actual ACB length is mandatory. + + Runtime arrays retain native enumeration order. ReplaySession.template/run in Rust produce frames and scores. + """ + if "master" not in music or "charts" not in music: + raise deckdata.DeckDataError("replay input needs the normalized --full master/charts") + lengths = {} + for song in music["songs"]: + ms = ((song.get("bgm") or {}).get("length") or {}).get("lengthMs") + if type(ms) is not int or ms <= 0: + raise deckdata.DeckDataError(f"replay song {song['id']}: a positive actual BGM lengthMs is required") + for chart in song["charts"]: + key = str(chart["scoreId"]) + if key in lengths: + raise deckdata.DeckDataError(f"replay chart {key}: duplicate song chart") + lengths[key] = ms + provenance = dict(music["provenance"]) + provenance["replay"] = {"musicLengthsMs": lengths, + "musicLengthSource": "ACB CueTable lengthMs; no last-note estimate"} + source_records = sorted(music["charts"], key=lambda chart: chart["scoreId"]) + records = [c for c in source_records if str(c["scoreId"]) in lengths] + if len(records) != len(lengths) or {str(c["scoreId"]) for c in records} != set(lengths): + raise deckdata.DeckDataError("replay runtime charts differ from the songs' charts") + data = {"format": deckdata.DECK_FORMAT, "provenance": provenance, + "master": music["master"], "charts": records} + files = {"deck-data.json": _json(data)} + + def resource(path: str) -> dict: + return {"url": path, "sha256": hashlib.sha256(files[path]).hexdigest(), "bytes": len(files[path])} + + charts = [] + for record in records: + sid = record["scoreId"] + path = f"charts/{sid}.json" + files[path] = _json({"format": CHART_FORMAT, "scoreId": sid, + "musicLengthMs": lengths[str(sid)], "chart": record}) + charts.append({"scoreId": sid, **resource(path), "musicLengthMs": lengths[str(sid)], + "noteCount": len(record["notes"]["id"]), "assetSha256": record["asset"]["sha256"]}) + model = music["provenance"].get("deck") + engine = None + if engine_dir is not None: + engine_dir = Path(engine_dir) + try: + built = json.loads((engine_dir / "build.json").read_text(encoding="utf8")) + except (OSError, ValueError): + raise deckdata.DeckDataError("replay engine: missing or malformed build.json") from None + if built.get("format") != "ournotes.replay-engine/1" or not model or built.get("commit") != model.get("commit"): + raise deckdata.DeckDataError("replay engine: build.json commit differs from the pinned deck model") + for name in (JS, WASM): + source = engine_dir / name + if not source.is_file(): + raise deckdata.DeckDataError(f"replay engine: missing {name} in {engine_dir}") + files[f"engine/{name}"] = source.read_bytes() + if not files[f"engine/{WASM}"].startswith(b"\x00asm\x01\x00\x00\x00"): + raise deckdata.DeckDataError("replay engine: not a WASM v1 module") + if not model or not model.get("commit"): + raise deckdata.DeckDataError("replay engine: the pinned deck model identity is required") + if built.get("jsSha256") != hashlib.sha256(files[f"engine/{JS}"]).hexdigest() or \ + built.get("wasmSha256") != hashlib.sha256(files[f"engine/{WASM}"]).hexdigest(): + raise deckdata.DeckDataError("replay engine: JS/WASM SHA differs from build.json") + files["engine/build.json"] = _json(built) + engine = {"model": model, "requestFormat": "ournotes.replay/1", "class": "ReplaySession", + "methods": ["describeChart", "template", "run"], + "js": resource(f"engine/{JS}"), "wasm": resource(f"engine/{WASM}"), + "build": resource("engine/build.json")} + manifest = {"format": FORMAT, "deckData": {"format": deckdata.DECK_FORMAT, **resource("deck-data.json")}, + "charts": charts, "engine": engine, + "unlistedScoreIds": [c["scoreId"] for c in source_records if str(c["scoreId"]) not in lengths], + "clock": "Explicit frames from ReplaySession.template; no Python/JS scoring or scheduling"} + files["manifest.json"] = _json(manifest) + return files, manifest + + +def write(directory: Path, files: dict[str, bytes]) -> None: + """Write validated normalized artifacts; callers publish the marker last.""" + directory = Path(directory) + for name, data in files.items(): + path = directory / name + path.parent.mkdir(parents=True, exist_ok=True) + write_atomic(path, data) diff --git a/src/nnnotes/web.py b/src/nnnotes/web.py index 29bc010..37b8406 100644 --- a/src/nnnotes/web.py +++ b/src/nnnotes/web.py @@ -102,6 +102,8 @@ READ_SET_SCRIPT = "scripts/read-set.mjs" # node ', + "songs.js": b'import "./catalog.js"; import "./replay-panel.js";', + "catalog.js": b"export const catalog = {};", "text.js": b"export const text = {};", + "replay-panel.js": b'import "./replay-preset.js";', + "replay-worker.js": b"// worker", "replay-preset.js": b"// preset", + "nested/future-dependency.json": b"{}"} + for name, data in files.items(): + path = player / web.SONGS_PAGE_DIR / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + site = tmp_path / "site" + result = web.write_player(site, player) + assert result["songsPageFiles"] == len(files) + assert {path.relative_to(site / "songs").as_posix(): path.read_bytes() + for path in (site / "songs").rglob("*") if path.is_file()} == files + + def test_player_only_command(tmp_path, capsys): from nnnotes import cli out = tmp_path / "site"