From 96ae15e8ac151009e522e877125bb7fcae9e7f21 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 30 Sep 2026 03:17:00 +0800 Subject: [PATCH 1/2] ci: check the music data seeds, rank bonuses and luck points The deck gate checks the seeds (the one seed 0 on a chart without a luck range, else two or more different seeds, the same on every luck chart: their number is the file's, not fixed), every seed range's rank 1 bonus (trunc(rangeScore * rankBonusPercent / 100)) and its luckPoints, which nnnotes and ournotes-deck add with the Gekisou skills. RECIPE 2. Co-Authored-By: Claude Opus 5.5 --- .github/MUSIC_DATA.md | 9 +++-- .github/scripts/music_data.py | 41 +++++++++++++++++++- .github/scripts/test_music_data.py | 62 ++++++++++++++++++++++-------- 3 files changed, 91 insertions(+), 21 deletions(-) diff --git a/.github/MUSIC_DATA.md b/.github/MUSIC_DATA.md index f8b22ab..9aca4f8 100644 --- a/.github/MUSIC_DATA.md +++ b/.github/MUSIC_DATA.md @@ -68,7 +68,7 @@ Every one must pass, else nothing is published. Warnings go to the job summary a | `schema` | the file against `docs/schema/music-data.schema.json` of the checkout (JSON Schema 2020-12) | | `provenance` | `format`; `region` `tw`; `master.source` `api`; `master.version` equal to the snapshot's and its `MasterManifest.json`'s; every table's SHA-256 the manifest's, every decoded table read the one `index.json` lists; the song tables and the deck model's present; `deck.commit` the one `rust/Cargo.lock` pins; `exporter.version` the installed nnnotes; an APK version; a catalog SHA-256 (warning: the APK is another client version than the snapshot's) | | `counts` | no fewer songs and charts than the published file (warning: ids no longer in it) | -| `deck` | deck statistics on every chart: kinds, a positive power, events and positions matching the chart, seeds unless unplayable (a warning), `weights[kind][position]` numbers, every check deck within its bound | +| `deck` | deck statistics on every chart: kinds, a positive power, events and positions matching the chart, seeds unless unplayable (a warning): the one seed 0 on a chart without a luck range, else two or more different seeds, the same on every luck chart (their number is the file's, not fixed), `weights[kind][position]` numbers, every seed range's `rankBonus` = trunc(`rangeScore` x `rankBonusPercent` / 100) and its `luckPoints` an int, every check deck within its bound | | `scenarios` | the play scenario fields: `offSeeds` exactly one entry (seed 0, score, weights, check within its bound), every range's `rankBonusPercents` five ints (the first `rankBonusPercent`), every seed's `scorePerfect`, `rangeWeights` (`[kind][position][range]`) and `rankCheck` (within its bound), every seed range's `rangeScorePerfect` (warnings, none in TW: a null `rangeWeights`, a null kind in it or in `offSeeds`' weights) | | `finite` | no NaN or infinity (warning: one inside master data rows, `songs[].master`, which the format writes as `1e999`) | | `references` | texts in every language of `languages` (names and titles not empty); unique ids; songs sorted; the songs' bands, vocal characters and tags in the file; a band or a band name; a jacket, and its file in `jackets/`; a BGM cue; score ranks; charts in difficulty order, score ids unique (warnings: a title without a `zh-Hant` text, a music category on no tab, a character of no band) | @@ -89,7 +89,8 @@ python -m pytest -q -p no:cacheprovider .github/scripts/test_music_data.py with, optionally, `MUSIC_DATA_SCHEMA` (a schema file when the checkout has none), `MUSIC_DATA_PAGE` (an `examples/songs` directory: the smoke test), `MUSIC_DATA_SAMPLE` (a real file with the play scenario fields: its -content gates pass) and `MUSIC_DATA_OLD_SAMPLE` (one without them: the scenario gate stops it). +content gates pass; one made before the ranges' `luckPoints`: the deck gate stops it on those alone) and +`MUSIC_DATA_OLD_SAMPLE` (one without the play scenario fields: the scenario gate stops it). ## Settings @@ -112,7 +113,9 @@ Repository variables: - **nnnotes.** The workflow runs this fork's nnnotes. It needs upstream's `music-data` command with the play scenarios (MetaSekaiLab/nnnotes `a03591e`) and `--decoded-master` (MetaSekaiLab/nnnotes#6, `12df2a6`): sync the - fork with upstream first. Until then `plan` stops naming what is missing. + fork with upstream first. Until then `plan` stops naming what is missing. The deck gate also needs the ranges' + `luckPoints`, which come with nnnotes' and ournotes-deck's Gekisou skill changes: until the fork has them every + build stops there. - **The page.** Set `MUSIC_DATA_PLAYER_REF` to the ournotes-player commit of the chart data page that reads the play scenario fields, once that page is merged. - **Publishing.** Set `MUSIC_DATA_PUBLISH` to `true` last, when dry runs pass and the published format is final. diff --git a/.github/scripts/music_data.py b/.github/scripts/music_data.py index 0c336b5..05af9ff 100755 --- a/.github/scripts/music_data.py +++ b/.github/scripts/music_data.py @@ -49,7 +49,7 @@ BUILD_FORMAT = "moenotes.music-data-build/1" # This script's own version of a build: bump it when what it builds or publishes changes, so that the next run builds # although the master data, the deck model and nnnotes are the same. -RECIPE = 1 +RECIPE = 2 FILE, MARKER, JACKETS, ARCHIVE = "music-data.json", "build.json", "jackets/", "archive/" MANIFEST = "MasterManifest.json" SOURCE_PATHS = ("src", "rust", "pyproject.toml") # nnnotes' code: the commit that last changed one of them @@ -66,6 +66,7 @@ BGM_CUE_SLACK_MS = 1000 # |durationMs - lengthMs| BGM_TAIL_MS = 60_000 # BGM after the last note: more is reported RANKS = 5 +LUCK_MISSION = 2 # a range of it draws lots: its chart has several seeds DIFFICULTIES = ("easy", "normal", "hard", "expert") SONG_TABLES = ("MasterLiveMusic", "MasterLiveMusicScore", "MasterText", "MasterBand", "MasterCharacter", "MasterTag", "MasterLiveMusicCategory", "MasterSound", "MasterSoundCueSheet", "MasterLiveScoreRank") @@ -290,6 +291,16 @@ def within(c) -> bool: and abs(c["exact"] - c["predicted"]) <= c["bound"]) +def rank_bonus(range_score: int, percent: int) -> int: + """trunc(rangeScore * percent / 100): a range's rank bonus.""" + q = abs(range_score * percent) // 100 + return q if range_score * percent >= 0 else -q + + +def luck_chart(deck: dict) -> bool: + return any(isinstance(r, dict) and r.get("mission") == LUCK_MISSION for r in deck.get("ranges") or []) + + def gate_schema(doc, ctx: Context, g: Gate): if ctx.schema is None: g.note = "skipped" @@ -410,6 +421,9 @@ def weights_shape(w, kinds: int, positions: int, nullable: bool) -> bool: def gate_deck(doc, ctx: Context, g: Gate): + """The deck statistics. Seeds (deck.model.seeds): the one seed 0 on a chart without a luck range, else two or more + seeds, the same on every luck chart (their number is the file's); every range's rankBonus the rank 1 bonus and + its luckPoints (the range's luck points without skills).""" deck = doc.get("deck") if not isinstance(deck, dict): g.fail("deck is null: no deck statistics (made with --no-deck?)") @@ -421,6 +435,7 @@ def gate_deck(doc, ctx: Context, g: Gate): if not (is_num(power) and power > 0): g.fail(f"deck.model.power {power!r}") n = unplayable = seeds = 0 + luck_seeds = None for song, chart in charts_of(doc): n += 1 w, d = where(song, chart), chart.get("deck") @@ -440,6 +455,17 @@ def gate_deck(doc, ctx: Context, g: Gate): g.fail(f"{w}: unplayable, but has Gekisou on seeds") elif not d.get("seeds"): g.fail(f"{w}: no seeds") + values = [s.get("seed") for s in d.get("seeds") or []] + if values and not luck_chart(d): + if len(values) != 1 or not is_int(values[0]) or values[0] != 0: + g.fail(f"{w}: seeds {values[:4]!r} without a luck range, expected the one seed 0") + elif values: + if not all(map(is_int, values)) or len(values) < 2 or len(set(values)) != len(values): + g.fail(f"{w}: {len(values)} seeds on a luck chart, expected two or more different int seeds") + elif luck_seeds is None: + luck_seeds = values + elif values != luck_seeds: + g.fail(f"{w}: its {len(values)} seeds are not the {len(luck_seeds)} of the first luck chart") for seed in d.get("seeds") or []: seeds += 1 s = f"{w} seed {seed.get('seed')}" @@ -449,9 +475,20 @@ def gate_deck(doc, ctx: Context, g: Gate): g.fail(f"{s}: weights are not [kind][position] numbers") if len(seed.get("ranges") or []) != len(d.get("ranges") or []): g.fail(f"{s}: {len(seed.get('ranges') or [])} range results for {len(d.get('ranges') or [])} ranges") + for i, (r, rr) in enumerate(zip(seed.get("ranges") or [], d.get("ranges") or [], strict=False)): + r, rr = (r if isinstance(r, dict) else {}), (rr if isinstance(rr, dict) else {}) + if not (is_int(r.get("rangeScore")) and is_int(r.get("rankBonus"))): + g.fail(f"{s} range {i}: rangeScore {r.get('rangeScore')!r}, rankBonus {r.get('rankBonus')!r}") + elif is_int(rr.get("rankBonusPercent")) and r["rankBonus"] != rank_bonus(r["rangeScore"], + rr["rankBonusPercent"]): + g.fail(f"{s} range {i}: rankBonus {r['rankBonus']} is not trunc({r['rangeScore']} * " + f"{rr['rankBonusPercent']} / 100)") + if not is_int(r.get("luckPoints")): + g.fail(f"{s} range {i}: luckPoints {'missing' if 'luckPoints' not in r else 'not an int'}") if not within(seed.get("check")): g.fail(f"{s}: the check deck is not within its bound") - g.note = f"{n} charts, {kinds} kinds, {seeds} seeds, {unplayable} unplayable" + g.note = (f"{n} charts, {kinds} kinds, {seeds} seeds, {unplayable} unplayable, " + f"{len(luck_seeds or [])} seeds per luck chart") def gate_scenarios(doc, ctx: Context, g: Gate): diff --git a/.github/scripts/test_music_data.py b/.github/scripts/test_music_data.py index bc86b88..c90400f 100644 --- a/.github/scripts/test_music_data.py +++ b/.github/scripts/test_music_data.py @@ -5,8 +5,9 @@ The JSON Schema gate uses docs/schema/music-data.schema.json (or $MUSIC_DATA_SCHEMA) when the checkout has it; the page smoke test runs when $MUSIC_DATA_PAGE names ournotes-player's examples/songs (and Node.js is installed). Real -files, when named: $MUSIC_DATA_SAMPLE (a file with the play scenario fields: the content gates pass) and -$MUSIC_DATA_OLD_SAMPLE (one without them: the scenario gate stops it). +files, when named: $MUSIC_DATA_SAMPLE (a file with the play scenario fields: the content gates pass; one made before +the ranges' luckPoints, the deck gate stops on those alone) and $MUSIC_DATA_OLD_SAMPLE (one without the play scenario +fields: the scenario gate stops it). """ import copy import hashlib @@ -53,27 +54,31 @@ def check_deck(exact): return {"deck": [[0, 5000], None], "exact": exact, "predicted": exact + 0.25, "bound": 7.0} -def deck_chart(): - seed = {"seed": 0, "score": 120000, "ranges": [{"rangeScore": 4000, "rankBonus": 10000, "maxCombo": 10, - "justCount": 0, "lotResults": [0, 0, 0, 0], - "rangeScorePerfect": 4000}], - "weights": [[0.5, 0.25]], "check": check_deck(2000), "scorePerfect": 120000, - "rangeWeights": [[[0.1], [0.05]]], "rankCheck": dict(check_deck(1900), ranks=[3])} +LUCK_SEEDS = [11, 22] # a luck chart's seeds; else the one seed 0 + + +def deck_chart(luck=False): + def one(n): + return {"seed": n, "score": 120000 + n, "ranges": [{"rangeScore": 4000, "rankBonus": 10000, "maxCombo": 10, + "justCount": 0, "luckPoints": 3 if luck else 0, + "lotResults": [0, 0, 0, 0], "rangeScorePerfect": 4000}], + "weights": [[0.5, 0.25]], "check": check_deck(2000), "scorePerfect": 120000 + n, + "rangeWeights": [[[0.1], [0.05]]], "rankCheck": dict(check_deck(1900), ranks=[3])} return {"convertedNoteCount": 20, "skip": 0.01, "events": [[0, 1000], [1, 3000]], "positions": 2, - "ranges": [{"index": 0, "mission": 1, "startMs": 1000, "endMs": 5000, "rankBonusPercent": 250, - "rankBonusPercents": [250, 190, 160, 100, 100]}], - "justNotes": 0, "seeds": [seed], + "ranges": [{"index": 0, "mission": 2 if luck else 1, "startMs": 1000, "endMs": 5000, + "rankBonusPercent": 250, "rankBonusPercents": [250, 190, 160, 100, 100]}], + "justNotes": 0, "seeds": [one(n) for n in (LUCK_SEEDS if luck else [0])], "offSeeds": [{"seed": 0, "score": 90000, "weights": [[0.4, 0.2]], "check": check_deck(1500)}], "unplayable": None} -def chart(difficulty, score_id, last=60000): +def chart(difficulty, score_id, last=60000, luck=False): return {"difficulty": difficulty, "scoreId": score_id, "level": 10, "displayLevel": 10.5, "fullComboCount": 20, "asset": {"key": f"Live/MusicScore/c/c_{score_id}", "sha256": "ab" * 32}, "notes": {"judged": 20, "total": 22, "byOperateType": {"1": 20, "120": 2}}, "bpm": {"main": 120.0, "min": 120.0, "max": 120.0, "changes": [{"timeMs": 0, "bpm": 120.0}]}, "firstNoteMs": 1000, "lastJudgedNoteMs": last, "lastNoteMs": last, "musicLengthMs": last + 1000, - "skillEventsMs": [1000, 3000], "fevers": [[1000, 5000]], "deck": deck_chart()} + "skillEventsMs": [1000, 3000], "fevers": [[1000, 5000]], "deck": deck_chart(luck)} def song(i, charts): @@ -109,7 +114,8 @@ def sample() -> dict: "tags": [{"id": 1, "name": text("tag")}], "categories": [{"id": 1, "musicCategories": [1], "name": text("cat")}], "deck": {"model": {"power": 300000, "checkPower": 1000003}, "kinds": [kind]}, - "songs": [song(100001, [chart("easy", 10), chart("expert", 30)]), song(100002, [chart("expert", 40)])], + "songs": [song(100001, [chart("easy", 10), chart("expert", 30, luck=True)]), + song(100002, [chart("expert", 40, luck=True)])], } @@ -216,6 +222,14 @@ def unplayable(doc): (lambda d: d["songs"][0]["charts"][0]["deck"].update(seeds=[]), "deck", "no seeds"), (unplayable, "deck", "unplayable, but has Gekisou on seeds"), (lambda d: d["songs"][0]["charts"][0]["deck"].update(events=[[0, 1000]]), "deck", "1 skill events"), + (lambda d: seed(d).update(seed=5), "deck", "seeds [5] without a luck range, expected the one seed 0"), + (lambda d: d["songs"][0]["charts"][1]["deck"]["seeds"].pop(), "deck", "1 seeds on a luck chart"), + (lambda d: d["songs"][0]["charts"][1]["deck"]["seeds"][1].update(seed=11), "deck", "2 seeds on a luck chart"), + (lambda d: d["songs"][1]["charts"][0]["deck"]["seeds"][1].update(seed=33), "deck", + "its 2 seeds are not the 2 of the first luck chart"), + (lambda d: seed(d)["ranges"][0].update(rankBonus=9999), "deck", "rankBonus 9999 is not trunc(4000 * 250 / 100)"), + (lambda d: seed(d)["ranges"][0].pop("luckPoints"), "deck", "seed 0 range 0: luckPoints missing"), + (lambda d: seed(d, 0, 1)["ranges"][0].update(luckPoints=2.5), "deck", "seed 11 range 0: luckPoints not an int"), # numbers (lambda d: seed(d)["weights"][0].__setitem__(1, float("nan")), "finite", "weights[0][1]: not finite"), (lambda d: d["songs"][1]["charts"][0]["bpm"].update(main=float("inf")), "finite", "bpm.main: not finite"), @@ -339,10 +353,23 @@ def real(name): return Path(p).read_bytes() +def luck_points(raw: bytes) -> bool: + """Whether a file's deck.seeds ranges have luckPoints (a file made before them has none).""" + return any("luckPoints" in r for _, c in music_data.charts_of(json.loads(raw)) + for s in (c.get("deck") or {}).get("seeds") or [] for r in s.get("ranges") or []) + + +def before_luck_points(r: dict) -> bool: + """The deck gate stopped only on the ranges' missing luckPoints.""" + g = gate(r, "deck") + return not g["passed"] and all(f.endswith("luckPoints missing") for f in g["failures"]) + + def test_a_real_file_without_the_scenario_fields(): raw = real("MUSIC_DATA_OLD_SAMPLE") r = gates(raw, Context(language="zh-Hant"), only=CONTENT) - assert failures(r) == [("scenarios", gate(r, "scenarios")["failures"])] + assert [n for n, _ in failures(r)] == (["scenarios"] if luck_points(raw) else ["deck", "scenarios"]) + assert luck_points(raw) or before_luck_points(r) g = gate(r, "scenarios") charts = sum(len(s["charts"]) for s in json.loads(raw)["songs"]) assert g["failureCount"] >= charts and all("offSeeds missing" in f or "rankBonusPercents missing" in f @@ -354,7 +381,10 @@ def test_a_real_file_with_the_scenario_fields(tmp_path): (tmp_path / "music-data.json").write_bytes(raw) ctx = Context(language="zh-Hant", page=page_path(), file=tmp_path / "music-data.json", published=raw) r = gates(raw, ctx, only=CONTENT + ("counts", "size", "page")) - assert r["passed"], failures(r) + if luck_points(raw): + assert r["passed"], failures(r) + else: + assert [n for n, _ in failures(r)] == ["deck"] and before_luck_points(r), failures(r) assert gate(r, "scenarios")["warningCount"] == 0 From 4c22bfe2ed93d1cd1410425a8ff574a7200211a6 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 30 Sep 2026 04:49:01 +0800 Subject: [PATCH 2/2] ci: validate Gekisou aptitude and smoke test the page API Check shape references, mission coverage, band variants, mean/error pairs, seed rules, dimensions, tail identities and simulation bounds. Cap gzip size at 2 MB overall and 1.2 MB for aptitude. Exercise page lookups, gains, uncertainty and missing cross terms. Default figures must remain unchanged. Reconstruct only deterministic check seeds with positional master skill factors, never stochastic means. Add a real chart-stats sample test and a necessary rounding bound for stochastic tailPerfect. Keep production PLAYER_REF and publishing unchanged. Co-Authored-By: Claude Code --- .github/MUSIC_DATA.md | 26 +- .github/scripts/music_data.py | 380 ++++++++++++++++++++++++++- .github/scripts/music_data_smoke.mjs | 83 +++++- .github/scripts/test_music_data.py | 255 ++++++++++++++++-- 4 files changed, 719 insertions(+), 25 deletions(-) diff --git a/.github/MUSIC_DATA.md b/.github/MUSIC_DATA.md index 9aca4f8..6c24ae5 100644 --- a/.github/MUSIC_DATA.md +++ b/.github/MUSIC_DATA.md @@ -2,7 +2,8 @@ `.github/workflows/music-data.yml` keeps the music data file of the chart data page (ournotes-player `examples/songs`) up to date: `nnnotes music-data` of the current master data ([docs/music-data.md](../docs/music-data.md): -every song and chart with the deck model's statistics and the play scenarios), checked by quality gates and +every song and chart with the deck model's statistics, the play scenarios and the chart's Gekisou skill aptitude), +checked by quality gates and published into the story site's bucket under `music-data/` (`https://storage.bdon.moe/moenotes/music-data/`): | Object | Content | Cache-Control | @@ -70,10 +71,12 @@ Every one must pass, else nothing is published. Warnings go to the job summary a | `counts` | no fewer songs and charts than the published file (warning: ids no longer in it) | | `deck` | deck statistics on every chart: kinds, a positive power, events and positions matching the chart, seeds unless unplayable (a warning): the one seed 0 on a chart without a luck range, else two or more different seeds, the same on every luck chart (their number is the file's, not fixed), `weights[kind][position]` numbers, every seed range's `rankBonus` = trunc(`rangeScore` x `rankBonusPercent` / 100) and its `luckPoints` an int, every check deck within its bound | | `scenarios` | the play scenario fields: `offSeeds` exactly one entry (seed 0, score, weights, check within its bound), every range's `rankBonusPercents` five ints (the first `rankBonusPercent`), every seed's `scorePerfect`, `rangeWeights` (`[kind][position][range]`) and `rankCheck` (within its bound), every seed range's `rangeScorePerfect` (warnings, none in TW: a null `rangeWeights`, a null kind in it or in `offSeeds`' weights) | +| `aptitude` | the Gekisou skill aptitude (every shape alone on a chart). `deck.model.gekisouAptitude` a text; `deck.gekisouAptitude`: every key, `plainKind` the page's plain kind, a `host` text, the `seedRule` (a deterministic test, increasing batches, the targets, the cross seeds), `shapes` numbered 0, 1, 2, ... (source `member` or `support`, mission 1 to 4, `bandCondition` a support skill's alone and exactly when an effect has condition 5000, effect rows with every key and their condition groups, condition 5000 without targets, skills with a level and, with a band condition alone, member targets and bands). Every chart's `deck.gekisouAptitude`: null exactly when the chart is unplayable with Gekisou on, has no Gekisou range or there is no shape; else `factors` one per range (counts; no Just or Perfect notes outside a Just range; `lotteries` `[0, 0]` outside a luck range, else the mean of `deck.seeds`' `lotResults`) and `variants` one per shape of the chart's missions (or mission 4) in shape order, a band condition shape's `bandMatch` true then false: every `[mean, se]` two finite numbers with se >= 0 (every se 0 when deterministic), ranges one per range, `tail` = `score` less the ranges' `rangeScore` and `rankBonus` (allowing 0.0005 per rounded term plus 1e-6), deterministic point deltas integers and `tailPerfect` checked against the baseline Perfect range bonuses, 1 seed when deterministic else a batch of the seed rule (the last one when `seTargetMet` is false), `crossSeeds` min(seeds, the rule's), `weights` one per position and `rangeWeights` per position and range where the plain kind and `deck.seeds[0].rangeWeights` are, else null, the `check` on `deck.seeds[0]`'s seed, a rank per range (1 where the ranks are not linear), a plain kind value or null per position, within its bound (warning: variants that missed the standard error target) | | `finite` | no NaN or infinity (warning: one inside master data rows, `songs[].master`, which the format writes as `1e999`) | | `references` | texts in every language of `languages` (names and titles not empty); unique ids; songs sorted; the songs' bands, vocal characters and tags in the file; a band or a band name; a jacket, and its file in `jackets/`; a BGM cue; score ranks; charts in difficulty order, score ids unique (warnings: a title without a `zh-Hant` text, a music category on no tab, a character of no band) | | `bgm` | every song's BGM length: `durationMs = samples * 1000 // sampleRate`, 30 s to 10 min, within 1 s of the cue's `lengthMs`, not ending before a chart's last note (warning: more than a minute after it) | | `size` | 0.8 to 2 times the published file | +| `gzip` | the file gzipped at most 2 MB (0.37 MB before the aptitude), its Gekisou skill aptitude gzipped at most 1.2 MB (about 0.4 MB expected) | | `page` | `music_data_smoke.mjs`: the page's `catalog.js` and `ranking.js` in Node.js over the file: a row per chart, a plain score-up kind, data for the free, rank and Just scenarios, finite positive figures for every chart the data covers in seven scenarios (Gekisou Live at several ranks, Just rates and a Great share, Free Live), the ranking, frontier and event figures | | (publish) | read back after upload, SHA-256 checked | @@ -89,9 +92,22 @@ python -m pytest -q -p no:cacheprovider .github/scripts/test_music_data.py with, optionally, `MUSIC_DATA_SCHEMA` (a schema file when the checkout has none), `MUSIC_DATA_PAGE` (an `examples/songs` directory: the smoke test), `MUSIC_DATA_SAMPLE` (a real file with the play scenario fields: its -content gates pass; one made before the ranges' `luckPoints`: the deck gate stops it on those alone) and +content gates pass; one made before the ranges' `luckPoints` and the aptitude: the deck and aptitude gates stop it +on those alone) and `MUSIC_DATA_OLD_SAMPLE` (one without the play scenario fields: the scenario gate stops it). +### Aptitude page smoke + +With the aptitude API (ournotes-player PR #11, `1522c24`), the same Node smoke also checks shape/skill/band +lookups, chart variants, all five battle scenarios, Free Live exclusion, finite gains, raw standard errors, +missing cross terms and the absence of standard errors for transformed or combined figures. Removing aptitude +must not change the default chart figures: default ranking still has no card Gekisou skills. + +Only deterministic variants are reconstructed against their individual `check` seed, using positional cards and +`masterSkillFactor` for the game's float32 conversion. Stochastic means are never used to reconstruct a check. +Older pinned page modules explicitly report `API unavailable (skipped)`; moving `MUSIC_DATA_PLAYER_REF` remains a +separate rollout decision. No browser or page build is needed. + ## Settings Repository secrets: the story site's (`.github/STORY_SITE.md`), no new one: `NNNOTES_BUNDLE_KEY`, @@ -113,9 +129,9 @@ Repository variables: - **nnnotes.** The workflow runs this fork's nnnotes. It needs upstream's `music-data` command with the play scenarios (MetaSekaiLab/nnnotes `a03591e`) and `--decoded-master` (MetaSekaiLab/nnnotes#6, `12df2a6`): sync the - fork with upstream first. Until then `plan` stops naming what is missing. The deck gate also needs the ranges' - `luckPoints`, which come with nnnotes' and ournotes-deck's Gekisou skill changes: until the fork has them every - build stops there. + fork with upstream first. Until then `plan` stops naming what is missing. The deck and aptitude gates also need + the ranges' `luckPoints` and the Gekisou skill aptitude, which come with nnnotes' and ournotes-deck's Gekisou skill + changes: until the fork has them every build stops there. - **The page.** Set `MUSIC_DATA_PLAYER_REF` to the ournotes-player commit of the chart data page that reads the play scenario fields, once that page is merged. - **Publishing.** Set `MUSIC_DATA_PUBLISH` to `true` last, when dry runs pass and the published format is final. diff --git a/.github/scripts/music_data.py b/.github/scripts/music_data.py index 05af9ff..a9116a0 100755 --- a/.github/scripts/music_data.py +++ b/.github/scripts/music_data.py @@ -27,6 +27,7 @@ """ from __future__ import annotations +import gzip import hashlib import json import math @@ -62,11 +63,18 @@ # the gates' bounds (MUSIC_DATA.md) SIZE_RATIO = (0.8, 2.0) # against the published file +FILE_GZIP_MAX = 2_000_000 # the file gzipped, the download (0.37 MB before the aptitude) +APTITUDE_GZIP_MAX = 1_200_000 # the Gekisou skill aptitude gzipped (about 0.4 MB expected) BGM_MS = (30_000, 600_000) # a song's BGM length BGM_CUE_SLACK_MS = 1000 # |durationMs - lengthMs| BGM_TAIL_MS = 60_000 # BGM after the last note: more is reported RANKS = 5 LUCK_MISSION = 2 # a range of it draws lots: its chart has several seeds +JUST_MISSION = 3 # a range of it judges Just +MISSIONS = (1, 2, 3, 4) # a Gekisou skill's: combo, luck, Just, every one +PLAIN_KIND = (2000, 5000) # the page's plain kind (ranking.js plainKind): effect type, ms +BAND_CONDITION = 5000 # the skill condition on the paired member (its band) +APTITUDE_SLACK = 1e-6 # relative: the aptitude's identities on means of integers DIFFICULTIES = ("easy", "normal", "hard", "expert") SONG_TABLES = ("MasterLiveMusic", "MasterLiveMusicScore", "MasterText", "MasterBand", "MasterCharacter", "MasterTag", "MasterLiveMusicCategory", "MasterSound", "MasterSoundCueSheet", "MasterLiveScoreRank") @@ -297,6 +305,18 @@ def rank_bonus(range_score: int, percent: int) -> int: return q if range_score * percent >= 0 else -q +def plain_kind(doc: dict): + """The id of the page's plain score-up kind in deck.kinds (ranking.js plainKind): effect type 2000 on the whole + deck for 5 s, without targets, conditions or limits; None for none.""" + for k in ((doc.get("deck") or {}).get("kinds")) or []: + if (isinstance(k, dict) and k.get("effectType") == PLAIN_KIND[0] and not k.get("skillTargetIds") + and not any(k.get(x) for x in ("skillConditionGroup", "skillReleaseConditionGroup", + "effectLimitCount", "effectExecuteLimitCount")) + and (PLAIN_KIND[1] if k.get("durationMs") is None else k["durationMs"]) == PLAIN_KIND[1]): + return k.get("id") + return None + + def luck_chart(deck: dict) -> bool: return any(isinstance(r, dict) and r.get("mission") == LUCK_MISSION for r in deck.get("ranges") or []) @@ -561,6 +581,359 @@ def gate_scenarios(doc, ctx: Context, g: Gate): g.note = "offSeeds, rankBonusPercents, scorePerfect, rangeWeights, rankCheck, rangeScorePerfect" +APTITUDE_KEYS = ("plainKind", "host", "seedRule", "shapes") +SEED_RULE_KEYS = ("deterministicTest", "batches", "relative", "baseline", "crossSeeds") +SHAPE_KEYS = ("id", "source", "mission", "bandCondition", "effects", "skills") +EFFECT_INTS = ("effectType", "triggerType", "effectValue", "maxEffectValue", "effectLimitCount", + "effectExecuteLimitCount") +EFFECT_GROUPS = ("trigger", "condition", "release", "reset") +FACTOR_INTS = ("judgedNotes", "justNotes", "perfectNotes", "tailNotes", "comboAtStart") +VARIANT_KEYS = ("shape", "bandMatch", "deterministic", "seeds", "seTargetMet", "crossSeeds", "score", "scorePerfect", + "tail", "tailPerfect", "converted", "ranges", "weights", "rangeWeights", "check") +VARIANT_PAIRS = ("score", "scorePerfect", "tail", "tailPerfect", "converted") +VARIANT_RANGE_PAIRS = ("rangeScore", "rankBonus", "rangeScorePerfect", "maxCombo", "justCount", "luckPoints") +APTITUDE_CHECK_KEYS = ("seed", "ranks", "deck", "exact", "predicted", "bound") + + +def pair(v) -> bool: + """A [mean, standard error]: two finite numbers, the error not negative.""" + return isinstance(v, list) and len(v) == 2 and is_num(v[0]) and is_num(v[1]) and v[1] >= 0 + + +def close(a: float, b: float) -> bool: + return abs(a - b) <= APTITUDE_SLACK * max(1.0, abs(a), abs(b)) + + +def gate_aptitude(doc, ctx: Context, g: Gate): + """The charts' Gekisou skill aptitude: deck.gekisouAptitude (its plain kind the page's, the seed rule, shapes + numbered from 0: source, mission, band condition, effect rows, skills); every chart's deck.gekisouAptitude, null + exactly when the chart is unplayable with Gekisou on, has no Gekisou range or there is no shape; else factors per + range and one variant per shape of the chart's missions (or mission 4) in shape order, a band condition shape's + bandMatch true then false: every [mean, se] two finite numbers with se >= 0 (0 when deterministic), ranges per + range, tail = score - the ranges' rangeScore and rankBonus, the seeds and the cross seeds by the seed rule, weights + and rangeWeights where the plain kind and the chart's rank weights are, the check within its bound. Warnings: + variants whose standard error missed the seed rule's target.""" + deck = doc.get("deck") + if not isinstance(deck, dict): + g.fail("deck is null: no Gekisou skill aptitude") + return + plain = plain_kind(doc) + if not (isinstance((deck.get("model") or {}).get("gekisouAptitude"), str) and deck["model"]["gekisouAptitude"]): + g.fail("deck.model.gekisouAptitude: no text") + head = deck.get("gekisouAptitude") + if not isinstance(head, dict): + g.fail("deck.gekisouAptitude missing" if head is None else f"deck.gekisouAptitude {head!r}") + head = {} + elif [k for k in APTITUDE_KEYS if k not in head]: + g.fail(f"deck.gekisouAptitude: no {', '.join(k for k in APTITUDE_KEYS if k not in head)}") + if head: + pk = head.get("plainKind", "missing") + if not (pk is None or is_int(pk)) or pk != plain: + g.fail(f"deck.gekisouAptitude.plainKind {pk!r}, the page's plain kind is {plain!r}") + if not (isinstance(head.get("host"), str) and head["host"].strip()): + g.fail("deck.gekisouAptitude.host: no text") + rule = head.get("seedRule") if isinstance(head.get("seedRule"), dict) else {} + batches = rule.get("batches") + if head and not (all(k in rule for k in SEED_RULE_KEYS) and is_int(rule["deterministicTest"]) + and rule["deterministicTest"] >= 1 and isinstance(batches, list) and batches + and all(map(is_int, batches)) and batches == sorted(set(batches)) and batches[0] >= 1 + and is_num(rule["relative"]) and rule["relative"] >= 0 and is_num(rule["baseline"]) + and rule["baseline"] >= 0 and is_int(rule["crossSeeds"]) and rule["crossSeeds"] >= 1): + g.fail(f"deck.gekisouAptitude.seedRule {rule!r}"[:200]) + rule, batches = {}, None + + # the shapes + shapes = head.get("shapes") if isinstance(head.get("shapes"), list) else [] + if head and not isinstance(head.get("shapes"), list): + g.fail("deck.gekisouAptitude.shapes is not a list") + if any(not isinstance(s, dict) or not is_int(s.get("id")) or s["id"] != i + for i, s in enumerate(shapes)): + g.fail("deck.gekisouAptitude.shapes: ids are not 0, 1, 2, ... in order") + by_id: dict = {} + for s in shapes: + if not isinstance(s, dict): + continue + w = f"shape {s.get('id')}" + absent = [k for k in SHAPE_KEYS if k not in s] + if absent: + g.fail(f"{w}: no {', '.join(absent)}") + continue + if is_int(s["id"]): + by_id[s["id"]] = s + if s["source"] not in ("member", "support"): + g.fail(f"{w}: source {s['source']!r}") + if not is_int(s["mission"]) or s["mission"] not in MISSIONS: + g.fail(f"{w}: mission {s['mission']!r}") + band = s["bandCondition"] + if not isinstance(band, bool) or (band and s["source"] != "support"): + g.fail(f"{w}: bandCondition {band!r} (a support skill's alone)") + effects = s["effects"] + fives = 0 + if not isinstance(effects, list): + g.fail(f"{w}: effects are not a list") + effects = [] + for i, e in enumerate(effects): + if not isinstance(e, dict): + g.fail(f"{w} effect {i}: not an object") + continue + bad = [k for k in EFFECT_INTS if not is_int(e.get(k))] + if not is_num(e.get("activationTimeSecond")): + bad.append("activationTimeSecond") + if not (isinstance(e.get("skillTargetIds"), list) and all(map(is_int, e["skillTargetIds"]))): + bad.append("skillTargetIds") + for k in EFFECT_GROUPS: + sets = e.get(k) + if not (isinstance(sets, list) and all(isinstance(x, list) for x in sets)): + bad.append(k) + continue + for c in (c for x in sets for c in x): + if not (isinstance(c, dict) and is_int(c.get("type")) and isinstance(c.get("values"), list) + and isinstance(c.get("positive"), bool) and "targetIds" in c): + bad.append(k) + elif c["type"] == BAND_CONDITION: + fives += 1 + if c["targetIds"] is not None: + bad.append(f"{k} (condition {BAND_CONDITION} targetIds not null)") + elif not (isinstance(c["targetIds"], list) and all(map(is_int, c["targetIds"]))): + bad.append(k) + cu = e.get("cumulative", "missing") + if cu is not None and not (isinstance(cu, dict) and all( + k in cu for k in ("type", "values", "targetIds", "maxCumulativeCount"))): + bad.append("cumulative") + if bad: + g.fail(f"{w} effect {i}: {', '.join(dict.fromkeys(bad))} missing or malformed") + if isinstance(band, bool) and band != (fives > 0): + g.fail(f"{w}: bandCondition {band}, its effects have {fives} condition {BAND_CONDITION}") + skills = s["skills"] + if not (isinstance(skills, list) and skills): + g.fail(f"{w}: no skills") + continue + for k in skills: + ok = (isinstance(k, dict) and is_int(k.get("id")) and is_int(k.get("level")) and k["level"] >= 1 + and "memberTargetIds" in k and "bandIds" in k) + targets, band_ids = (k.get("memberTargetIds"), k.get("bandIds")) if isinstance(k, dict) else (0, 0) + if band is True: + ok = (ok and isinstance(targets, list) and bool(targets) and all(map(is_int, targets)) + and targets == sorted(set(targets)) and isinstance(band_ids, list) + and all(map(is_int, band_ids))) + else: + ok = ok and targets is None and band_ids is None + if not ok: + g.fail(f"{w}: skill {k!r} is not an id, a level and (with a band condition alone) member targets " + f"and bands"[:240]) + + # the charts + aptitudes = nulls = variants = deterministic = 0 + missed = [] + for song, chart in charts_of(doc): + w, d = where(song, chart), chart.get("deck") + if not isinstance(d, dict): + continue # the deck gate fails it + if "gekisouAptitude" not in d: + g.fail(f"{w}: no deck.gekisouAptitude") + continue + a, ranges, dseeds = d["gekisouAptitude"], d.get("ranges") or [], d.get("seeds") or [] + why = ("unplayable with Gekisou on" if d.get("unplayable") else "without a Gekisou range" if not ranges + else None if shapes else "without a Gekisou skill shape") + if a is None: + nulls += 1 + if why is None: + g.fail(f"{w}: deck.gekisouAptitude is null, but the chart is playable with Gekisou on") + continue + if why is not None: + g.fail(f"{w}: a Gekisou skill aptitude on a chart {why}") + continue + if not (isinstance(a, dict) and isinstance(a.get("factors"), list) and isinstance(a.get("variants"), list)): + g.fail(f"{w}: deck.gekisouAptitude has no factors and variants lists") + continue + aptitudes += 1 + positions = d.get("positions") + missions = {r.get("mission") for r in ranges if isinstance(r, dict)} + linear = bool(dseeds) and dseeds[0].get("rangeWeights") is not None + + # factors + if len(a["factors"]) != len(ranges): + g.fail(f"{w}: {len(a['factors'])} factors for {len(ranges)} ranges") + for j, (f, r) in enumerate(zip(a["factors"], ranges, strict=False)): + f, r = (f if isinstance(f, dict) else {}), (r if isinstance(r, dict) else {}) + bad = [k for k in FACTOR_INTS if not (is_int(f.get(k)) and f[k] >= 0)] + if not pair(f.get("lotteries")): + bad.append("lotteries") + if bad: + g.fail(f"{w} factors {j}: {', '.join(bad)} missing or not counts") + continue + if r.get("mission") != JUST_MISSION and (f["justNotes"] or f["perfectNotes"]): + g.fail(f"{w} factors {j}: Just or Perfect notes in a range without the Just mission") + if f["justNotes"] + f["perfectNotes"] > f["judgedNotes"]: + g.fail(f"{w} factors {j}: {f['justNotes']} Just and {f['perfectNotes']} Perfect notes of " + f"{f['judgedNotes']} judged") + at = [s["ranges"][j] for s in dseeds if isinstance(s.get("ranges"), list) and len(s["ranges"]) > j + and isinstance(s["ranges"][j], dict)] + lots = [sum(x["lotResults"]) for x in at + if isinstance(x.get("lotResults"), list) and all(map(is_int, x["lotResults"]))] + if r.get("mission") != LUCK_MISSION and f["lotteries"] != [0, 0]: + g.fail(f"{w} factors {j}: lotteries {f['lotteries']} in a range without the luck mission") + elif lots and len(lots) == len(dseeds) and not close(f["lotteries"][0], sum(lots) / len(lots)): + g.fail(f"{w} factors {j}: lotteries {f['lotteries'][0]}, deck.seeds' lotResults give " + f"{sum(lots) / len(lots)}") + + # the variants: which, in order + want = [(s["id"], match) for s in sorted(by_id.values(), key=lambda s: s["id"]) + if s["mission"] == 4 or s["mission"] in missions + for match in ((True, False) if s["bandCondition"] is True else (None,))] + got = [(v.get("shape"), v.get("bandMatch")) if isinstance(v, dict) else None for v in a["variants"]] + unknown = [v[0] for v in got if v is not None and not (is_int(v[0]) and v[0] in by_id)] + other = [v[0] for v in got if v is not None and is_int(v[0]) and v[0] in by_id + and by_id[v[0]]["mission"] != 4 and by_id[v[0]]["mission"] not in missions] + if unknown: + g.fail(f"{w}: variants of shapes {sorted(set(map(repr, unknown)))} not in deck.gekisouAptitude.shapes") + if other: + g.fail(f"{w}: variants of shapes {sorted(set(other))} of a mission the chart does not play") + if got != want and not unknown and not other: + lacking = [x for x in want if x not in got] + g.fail(f"{w}: variants {'lack ' + repr(lacking[:6]) if lacking else 'not in shape order, true first'}" + f" ({len(got)} for {len(want)})") + + for v in a["variants"]: + if not isinstance(v, dict): + g.fail(f"{w}: a variant {v!r}") + continue + s = f"{w} shape {v.get('shape')}" + ("" if v.get("bandMatch") is None else f" {v['bandMatch']}") + absent = [k for k in VARIANT_KEYS if k not in v] + if absent: + g.fail(f"{s}: no {', '.join(absent)}") + continue + variants += 1 + shape = by_id.get(v["shape"]) if is_int(v["shape"]) else None + det = v["deterministic"] + if not isinstance(det, bool): + g.fail(f"{s}: deterministic {det!r}") + det = False + deterministic += det + if v["bandMatch"] is not None and not isinstance(v["bandMatch"], bool): + g.fail(f"{s}: bandMatch {v['bandMatch']!r}") + elif shape is not None and (v["bandMatch"] is None) == (shape.get("bandCondition") is True): + g.fail(f"{s}: bandMatch {v['bandMatch']!r} for a shape " + f"{'with' if v['bandMatch'] is None else 'without'} a band condition") + n, cross, met = v["seeds"], v["crossSeeds"], v["seTargetMet"] + if not (is_int(n) and n >= 1 and isinstance(met, bool)): + g.fail(f"{s}: seeds {n!r}, seTargetMet {met!r}") + elif det and (n != 1 or not met): + g.fail(f"{s}: deterministic, but {n} seeds and seTargetMet {met}") + elif not det and batches and (n not in batches or (not met and n != batches[-1])): + g.fail(f"{s}: {n} seeds (seTargetMet {met}), not a batch of the seed rule {batches}") + elif not det and not met: + missed.append(f"{chart.get('scoreId')} shape {v['shape']}") + if (not is_int(cross) or cross < 1 or (is_int(n) and is_int(rule.get("crossSeeds")) + and cross != min(n, rule["crossSeeds"]))): + g.fail(f"{s}: crossSeeds {cross!r}, expected min({n}, {rule['crossSeeds']})") + + # every [mean, se] + values = {k: v[k] for k in VARIANT_PAIRS} + rs = v["ranges"] + if not isinstance(rs, list) or len(rs) != len(ranges): + g.fail(f"{s}: {len(rs) if isinstance(rs, list) else repr(rs)} range results for {len(ranges)} ranges") + rs = [] + for j, r in enumerate(rs): + for k in VARIANT_RANGE_PAIRS: + values[f"ranges[{j}].{k}"] = r.get(k) if isinstance(r, dict) else None + wt, rw = v["weights"], v["rangeWeights"] + if plain is None: + if wt is not None or rw is not None: + g.fail(f"{s}: weights or rangeWeights without a plain kind") + else: + if not (isinstance(wt, list) and len(wt) == positions): + g.fail(f"{s}: weights are not one [mean, se] per position") + else: + values.update({f"weights[{k}]": x for k, x in enumerate(wt)}) + if not linear: + if rw is not None: + g.fail(f"{s}: rangeWeights, but deck.seeds[0].rangeWeights is null") + elif not (isinstance(rw, list) and len(rw) == positions and all( + isinstance(x, list) and len(x) == len(ranges) for x in rw)): + g.fail(f"{s}: rangeWeights are not [position][range] [mean, se]") + else: + values.update({f"rangeWeights[{k}][{j}]": y for k, x in enumerate(rw) for j, y in enumerate(x)}) + bad = [k for k, x in values.items() if not pair(x)] + if bad: + g.fail(f"{s}: {', '.join(bad[:6])}{' ...' if len(bad) > 6 else ''} not [mean, se] (finite, se >= 0)") + elif det and any(x[1] for x in values.values()): + g.fail(f"{s}: deterministic, but a standard error is not 0: " + f"{', '.join(k for k, x in values.items() if x[1])[:160]}") + elif rs and all(pair(r.get(k)) for r in rs for k in ("rangeScore", "rankBonus")): + inside = sum(r["rangeScore"][0] + r["rankBonus"][0] for r in rs) + if abs(v["tail"][0] - (v["score"][0] - inside)) > ( + 0.0005 * (2 + 2 * len(rs)) + APTITUDE_SLACK): + g.fail(f"{s}: tail {v['tail'][0]} is not score {v['score'][0]} less the ranges' rangeScore and " + f"rankBonus {inside}") + + # Perfect rank-bonus deltas are not exported. Only a necessary rounding bound can be checked + # for stochastic means: each difference of two truncated bonuses is within 2 points of delta*p/100. + if not bad and len(rs) == len(ranges): + perfect_terms = [r["rangeScorePerfect"][0] * (1 + info["rankBonusPercent"] / 100) + for r, info in zip(rs, ranges, strict=True)] + rounding = 0.001 + sum(0.0005 * abs(1 + info["rankBonusPercent"] / 100) for info in ranges) + residual = v["scorePerfect"][0] - v["tailPerfect"][0] - sum(perfect_terms) + if abs(residual) > 2 * len(ranges) + rounding + APTITUDE_SLACK: + g.fail(f"{s}: tailPerfect violates the necessary Perfect rank-bonus rounding bound") + + # Deterministic point deltas are integers. Perfect rank bonuses can then be recovered exactly + # from the first baseline seed (stochastic Perfect rank-bonus deltas are not exported). + if det and not bad and rs and dseeds: + points = [x[0] for k, x in values.items() if not k.startswith(("weights", "rangeWeights"))] + if not all(float(x).is_integer() for x in points): + g.fail(f"{s}: deterministic, but a point delta is not an integer") + else: + perfect = 0 + for r, info, base in zip(rs, ranges, dseeds[0]["ranges"], strict=True): + delta, baseline = int(r["rangeScorePerfect"][0]), base["rangeScorePerfect"] + percent = info["rankBonusPercent"] + perfect += delta + rank_bonus(baseline + delta, percent) - rank_bonus(baseline, percent) + if v["tailPerfect"][0] != v["scorePerfect"][0] - perfect: + g.fail(f"{s}: tailPerfect is not scorePerfect less the Perfect range scores and bonuses") + + # the check + c = v["check"] + if not (isinstance(c, dict) and all(k in c for k in APTITUDE_CHECK_KEYS)): + g.fail(f"{s}: the check has no {', '.join(APTITUDE_CHECK_KEYS)}") + continue + if dseeds and c["seed"] != dseeds[0].get("seed"): + g.fail(f"{s}: the check's seed {c['seed']!r} is not deck.seeds[0]'s {dseeds[0].get('seed')!r}") + ranks = c["ranks"] + if not (isinstance(ranks, list) and len(ranks) == len(ranges) + and all(is_int(x) and 1 <= x <= RANKS for x in ranks)): + g.fail(f"{s}: the check's ranks {ranks!r} are not one rank per range") + elif not linear and any(x != 1 for x in ranks): + g.fail(f"{s}: the check's ranks {ranks} on a chart whose ranks are not linear (rank 1 alone)") + cd = c["deck"] + if not (isinstance(cd, list) and len(cd) == positions and all( + x is None or (plain is not None and isinstance(x, list) and len(x) == 2 and x[0] == plain + and is_num(x[1])) for x in cd)): + g.fail(f"{s}: the check deck is not a [plain kind, value] or null per position") + if not within(c): + g.fail(f"{s}: the check is not within its bound") + if missed: + g.warn(f"{len(missed)} variants missed the seed rule's standard error target: {', '.join(missed[:10])}" + + (" ..." if len(missed) > 10 else "")) + g.note = (f"{len(shapes)} shapes; {aptitudes} charts with an aptitude, {nulls} null; {variants} variants, " + f"{deterministic} deterministic; plain kind {plain}") + + +def gate_gzip(doc, ctx: Context, g: Gate, raw: bytes): + """The file's gzip size (what the page downloads) and that of its Gekisou skill aptitude, against loose caps.""" + whole = len(gzip.compress(raw, 6)) + deck = doc.get("deck") if isinstance(doc.get("deck"), dict) else {} + part = {"deck": deck.get("gekisouAptitude"), + "charts": [(c.get("deck") or {}).get("gekisouAptitude") if isinstance(c.get("deck"), dict) else None + for _, c in charts_of(doc)]} + aptitude = len(gzip.compress(json.dumps(part, ensure_ascii=False, separators=(",", ":")).encode("utf-8"), 6)) + if whole > FILE_GZIP_MAX: + g.fail(f"{whole} bytes gzipped, more than {FILE_GZIP_MAX}") + if aptitude > APTITUDE_GZIP_MAX: + g.fail(f"the Gekisou skill aptitude: {aptitude} bytes gzipped, more than {APTITUDE_GZIP_MAX}") + g.note = f"{whole} bytes gzipped, the aptitude {aptitude}" + + def nonfinite(v, path: str, out: list): if isinstance(v, float): if not math.isfinite(v): @@ -725,8 +1098,9 @@ def gate_page(doc, ctx: Context, g: Gate): GATES = (("schema", gate_schema), ("provenance", gate_provenance), ("counts", gate_counts), ("deck", gate_deck), - ("scenarios", gate_scenarios), ("finite", gate_finite), ("references", gate_references), ("bgm", gate_bgm), - ("size", gate_size), ("page", gate_page)) + ("scenarios", gate_scenarios), ("aptitude", gate_aptitude), ("finite", gate_finite), + ("references", gate_references), ("bgm", gate_bgm), ("size", gate_size), ("gzip", gate_gzip), + ("page", gate_page)) def gates(raw: bytes, ctx: Context, only=None) -> dict: @@ -745,7 +1119,7 @@ def gates(raw: bytes, ctx: Context, only=None) -> dict: continue g = Gate() try: - fn(doc, ctx, g, raw) if name == "size" else fn(doc, ctx, g) + fn(doc, ctx, g, raw) if name in ("size", "gzip") else fn(doc, ctx, g) except Exception as e: # a malformed file the gate did not foresee fails the gate g.fail(f"the gate stopped: {type(e).__name__}: {str(e)[:160]}") results.append({"gate": name, "passed": not g.failures, "failures": g.failures[:LISTED], diff --git a/.github/scripts/music_data_smoke.mjs b/.github/scripts/music_data_smoke.mjs index a711edc..635b456 100644 --- a/.github/scripts/music_data_smoke.mjs +++ b/.github/scripts/music_data_smoke.mjs @@ -85,10 +85,91 @@ for (const [name, scenario] of SCENARIOS) { } catalog.histogram(rows, (r) => r.level); +// Aptitude is a separate view, never folded into the default song ranking. Older pinned pages may lack this API; +// report that explicitly until PLAYER_REF is deliberately moved to a page with the aptitude UI. +let aptitudeFigures = 0; +const aptitudeApi = ["aptitudeShapes", "chartVariants", "aptitudeFigures", "aptitudeRate", "aptitudeSe", + "masterSkillFactor"].every((k) => typeof ranking[k] === "function") + && ["gekisouSkill", "shapeSkills", "shapeBands"].every((k) => typeof catalog[k] === "function"); +const near = (a, b) => finite(a) && finite(b) && Math.abs(a - b) <= 1e-8 * Math.max(1, Math.abs(a), Math.abs(b)); +if (aptitudeApi && data.deck?.gekisouAptitude) { + const shapes = ranking.aptitudeShapes(data); + if (shapes.size !== data.deck.gekisouAptitude.shapes.length) problem("aptitudeShapes: missing shapes"); + for (const shape of shapes.values()) { + const skills = catalog.shapeSkills(data, shape, "zh-Hant"); + if (skills.length !== new Set(shape.skills.map((s) => `${s.id}:${s.level}`)).size) { + problem(`shape ${shape.id}: shapeSkills count`); + } + const table = shape.source === "support" ? "supportSkills" : "skills"; + for (const skill of skills) { + const named = catalog.gekisouSkill(data, table, skill.id, "zh-Hant"); + if (!named.name || named.name !== skill.name) problem(`shape ${shape.id}: skill name lookup`); + } + const bands = catalog.shapeBands(data, shape, "zh-Hant"); + if (bands.length !== new Set(shape.skills.flatMap((s) => s.bandIds || [])).size || bands.some((s) => !s)) { + problem(`shape ${shape.id}: shapeBands lookup`); + } + } + const power = data.deck.model.power; + for (const { chart } of charts) { + const d = chart.deck; + if (!d) continue; + const variants = ranking.chartVariants(d); + if (variants.length !== (d.unplayable ? 0 : d.gekisouAptitude?.variants.length || 0)) { + problem(`chart ${chart.scoreId}: chartVariants count`); + } + // Removing aptitude must not alter any default figure. + const baseline = ranking.chartFigures(d, kind, power); + const without = ranking.chartFigures({ ...d, gekisouAptitude: null }, kind, power); + if (JSON.stringify(baseline) !== JSON.stringify(without)) problem(`chart ${chart.scoreId}: aptitude changes default`); + for (const variant of variants) { + for (const [name, scenario] of SCENARIOS) { + const tag = `aptitude chart ${chart.scoreId} shape ${variant.shape}, ${name}`; + const f = ranking.aptitudeFigures(variant, d.ranges, power, scenario); + if (scenario?.mode === "free") { + if (f !== null) problem(`${tag}: Free Live has aptitude`); + continue; + } + aptitudeFigures++; + if (!f || !finite(f.base)) { problem(`${tag}: missing or nonfinite base`); continue; } + if (f.weights !== null && (f.weights.length !== d.positions || !f.weights.every(finite))) { + problem(`${tag}: invalid weights`); + } + const zero = Array(d.positions).fill(0); + if (!near(ranking.aptitudeRate(f, zero), f.base)) problem(`${tag}: no-skill rate`); + const rate = ranking.aptitudeRate(f, SKILLS); + if (f.weights === null ? rate !== null : !finite(rate)) problem(`${tag}: missing cross-term handling`); + if (ranking.aptitudeSe(f, SKILLS) !== null) problem(`${tag}: SE assigned without covariance`); + if (scenario === null) { + if (!near(f.base, variant.score[0] / power) + || !near(ranking.aptitudeSe(f, zero), variant.score[1] / power)) problem(`${tag}: raw mean/SE`); + } else if (ranking.aptitudeSe(f, zero) !== null) problem(`${tag}: transformed SE is not null`); + } + // Only deterministic deltas describe the individual check seed. Never reconstruct a stochastic check + // from sampled means. Ordinary check cards are positional, unlike the UI's random-order expectation. + if (!variant.deterministic || kind === null) continue; + const c = variant.check; + const seed = d.seeds.find((s) => s.seed === c.seed); + const sc = { mode: "battle", ranks: c.ranks, just: 1, great: 0 }; + const base = ranking.scenarioSeed(seed, d.ranges, kind, sc); + const gain = ranking.aptitudeFigures(variant, d.ranges, power, sc); + if (!base || !gain?.weights) continue; + const predicted = data.deck.model.checkPower * ((base.score / power) + gain.base + + c.deck.reduce((sum, card, k) => sum + (card ? ranking.masterSkillFactor(card[1]) + * (base.weights[k] + gain.weights[k]) : 0), 0)); + // Exported weight rounding adds a small reconstruction error on top of the engine's bound. + const rounding = data.deck.model.checkPower * 1e-7 * (1 + d.ranges.length) * d.positions; + if (!finite(predicted) || Math.abs(predicted - c.exact) > c.bound + rounding) { + problem(`aptitude chart ${chart.scoreId} shape ${variant.shape}: deterministic check reconstruction`); + } + } + } +} + if (problems.length) { for (const m of problems.slice(0, 40)) console.log(m); if (problems.length > 40) console.log(`${problems.length - 40} more problems`); process.exit(1); } console.log(`page smoke test: ${rows.length} charts, ${SCENARIOS.length} scenarios, ${figures} chart figures; ` - + `plain kind ${kind}, scenarios free/ranks/just`); + + `plain kind ${kind}, scenarios free/ranks/just; aptitude ${aptitudeApi ? aptitudeFigures + " figures" : "API unavailable (skipped)"}`); diff --git a/.github/scripts/test_music_data.py b/.github/scripts/test_music_data.py index c90400f..3df390c 100644 --- a/.github/scripts/test_music_data.py +++ b/.github/scripts/test_music_data.py @@ -6,8 +6,8 @@ The JSON Schema gate uses docs/schema/music-data.schema.json (or $MUSIC_DATA_SCHEMA) when the checkout has it; the page smoke test runs when $MUSIC_DATA_PAGE names ournotes-player's examples/songs (and Node.js is installed). Real files, when named: $MUSIC_DATA_SAMPLE (a file with the play scenario fields: the content gates pass; one made before -the ranges' luckPoints, the deck gate stops on those alone) and $MUSIC_DATA_OLD_SAMPLE (one without the play scenario -fields: the scenario gate stops it). +the ranges' luckPoints and the Gekisou skill aptitude, the deck and aptitude gates stop it on those alone) and +$MUSIC_DATA_OLD_SAMPLE (one without the play scenario fields: the scenario gate stops it). """ import copy import hashlib @@ -55,21 +55,83 @@ def check_deck(exact): LUCK_SEEDS = [11, 22] # a luck chart's seeds; else the one seed 0 +# the Gekisou skill aptitude's shapes: id, source, mission, band condition +SHAPES = [(0, "member", 1, False), (1, "support", 2, True), (2, "member", 3, False), (3, "support", 4, False)] +SEED_RULE = {"deterministicTest": 4, "batches": [32, 64, 128, 256, 512, 1024], "relative": 0.01, "baseline": 0.001, + "crossSeeds": 64} + + +def effect(band=False): + return {"effectType": 2000, "triggerType": 7010, "activationTimeSecond": 5.0, "effectValue": 1000, + "maxEffectValue": 0, "effectLimitCount": 0, "effectExecuteLimitCount": 0, "skillTargetIds": [], + "trigger": [[{"type": 7010, "values": [1], "positive": True, "targetIds": []}]], + "condition": [[{"type": 5000, "values": [1], "positive": True, "targetIds": None}]] if band else [], + "release": [], "reset": [], "cumulative": None} + + +def aptitude_head(): + return {"plainKind": 0, "host": "one performer with a synthetic empty Gekisou skill", "seedRule": SEED_RULE, + "shapes": [{"id": i, "source": src, "mission": m, "bandCondition": band, "effects": [effect(band)], + "skills": [{"id": 10 + i, "level": 5, "memberTargetIds": [41] if band else None, + "bandIds": [1] if band else None}]} for i, src, m, band in SHAPES]} + + +def variant(deck, shape, match, deterministic): + """One shape's aptitude on a chart: deterministic on a chart without a luck range, else on 32 seeds.""" + se = 0 if deterministic else 12.5 + + def p(m): + return [m, se] + ranges = [{"rangeScore": p(300), "rankBonus": p(750), "rangeScorePerfect": p(300), "maxCombo": p(0), + "justCount": p(0), "luckPoints": p(2)} for _ in deck["ranges"]] + n = 1 if deterministic else 32 + linear = deck["seeds"][0]["rangeWeights"] is not None + result = {"shape": shape, "bandMatch": match, "deterministic": deterministic, "seeds": n, "seTargetMet": True, + "crossSeeds": min(n, SEED_RULE["crossSeeds"]), "score": p(1500), "scorePerfect": p(1500), + "tail": p(1500 - 1050 * len(ranges)), "tailPerfect": p(1500 - 1050 * len(ranges)), + "converted": p(0), "ranges": ranges, + "weights": [p(0.01), p(0.02)], + "rangeWeights": [[p(0.001)] * len(ranges), [p(0.002)] * len(ranges)] if linear else None, + "check": {"seed": deck["seeds"][0]["seed"], "ranks": [2] * len(ranges) if linear else [1] * len(ranges), + "deck": [[0, 7000], None], "exact": 5000, "predicted": 5000.5, "bound": 3.0}} + base = deck["seeds"][0] + score = base["score"] + 1500 + weight = base["weights"][0][0] + 0.01 + if linear: + for r, info in zip(base["ranges"], deck["ranges"], strict=True): + percent = info["rankBonusPercents"][1] + score += int(r["rangeScore"] * percent / 100) - r["rankBonus"] + 300 * (percent - 250) / 100 + weight += (190 - 250) / 100 * (0.1 + 0.001) + predicted = 1000003 * (score / 300000 + 0.7 * weight) + result["check"].update(exact=round(predicted), predicted=predicted) + return result + + +def aptitude(deck, luck): + missions = {r["mission"] for r in deck["ranges"]} + variants = [variant(deck, i, match, deterministic=not luck) for i, _, m, band in SHAPES + if m == 4 or m in missions for match in ((True, False) if band else (None,))] + factors = [{"judgedNotes": 12, "justNotes": 0, "perfectNotes": 0, "tailNotes": 2, "comboAtStart": 5, + "lotteries": [4.0, 0.0] if r["mission"] == 2 else [0, 0]} for r in deck["ranges"]] + return {"factors": factors, "variants": variants} def deck_chart(luck=False): def one(n): return {"seed": n, "score": 120000 + n, "ranges": [{"rangeScore": 4000, "rankBonus": 10000, "maxCombo": 10, "justCount": 0, "luckPoints": 3 if luck else 0, - "lotResults": [0, 0, 0, 0], "rangeScorePerfect": 4000}], + "lotResults": [1, 2, 0, 1] if luck else [0, 0, 0, 0], + "rangeScorePerfect": 4000}], "weights": [[0.5, 0.25]], "check": check_deck(2000), "scorePerfect": 120000 + n, "rangeWeights": [[[0.1], [0.05]]], "rankCheck": dict(check_deck(1900), ranks=[3])} - return {"convertedNoteCount": 20, "skip": 0.01, "events": [[0, 1000], [1, 3000]], "positions": 2, - "ranges": [{"index": 0, "mission": 2 if luck else 1, "startMs": 1000, "endMs": 5000, - "rankBonusPercent": 250, "rankBonusPercents": [250, 190, 160, 100, 100]}], - "justNotes": 0, "seeds": [one(n) for n in (LUCK_SEEDS if luck else [0])], - "offSeeds": [{"seed": 0, "score": 90000, "weights": [[0.4, 0.2]], "check": check_deck(1500)}], - "unplayable": None} + d = {"convertedNoteCount": 20, "skip": 0.01, "events": [[0, 1000], [1, 3000]], "positions": 2, + "ranges": [{"index": 0, "mission": 2 if luck else 1, "startMs": 1000, "endMs": 5000, + "rankBonusPercent": 250, "rankBonusPercents": [250, 190, 160, 100, 100]}], + "justNotes": 0, "seeds": [one(n) for n in (LUCK_SEEDS if luck else [0])], + "offSeeds": [{"seed": 0, "score": 90000, "weights": [[0.4, 0.2]], "check": check_deck(1500)}], + "unplayable": None} + d["gekisouAptitude"] = aptitude(d, luck) + return d def chart(difficulty, score_id, last=60000, luck=False): @@ -113,7 +175,8 @@ def sample() -> dict: "characters": [{"id": 1, "bandId": 1, "name": text("ch"), "shortName": text("c"), "mainColor": "#77BBDD"}], "tags": [{"id": 1, "name": text("tag")}], "categories": [{"id": 1, "musicCategories": [1], "name": text("cat")}], - "deck": {"model": {"power": 300000, "checkPower": 1000003}, "kinds": [kind]}, + "deck": {"model": {"power": 300000, "checkPower": 1000003, "gekisouAptitude": "one skill at a time"}, + "kinds": [kind], "gekisouAptitude": aptitude_head()}, "songs": [song(100001, [chart("easy", 10), chart("expert", 30, luck=True)]), song(100002, [chart("expert", 40, luck=True)])], } @@ -164,6 +227,30 @@ def seed(doc, song=0, chart=0): return doc["songs"][song]["charts"][chart]["deck"]["seeds"][0] +def apt(doc, song=0, chart=0): + return doc["songs"][song]["charts"][chart]["deck"]["gekisouAptitude"] + + +def var(doc, i=0, song=0, chart=0): + return apt(doc, song, chart)["variants"][i] + + +def shape(doc, i): + return doc["deck"]["gekisouAptitude"]["shapes"][i] + + +def nonlinear(doc, song=0, chart=0): + """A chart whose ranks are not linear (overlapping ranges): no rangeWeights, the checks at rank 1.""" + d = doc["songs"][song]["charts"][chart]["deck"] + for s in d["seeds"]: + s.update(rangeWeights=None, rankCheck=None) + for v in d["gekisouAptitude"]["variants"]: + v["rangeWeights"] = None + v["check"]["ranks"] = [1] * len(d["ranges"]) + predicted = 1000003 * ((d["seeds"][0]["score"] + v["score"][0]) / 300000 + 0.7 * 0.51) + v["check"].update(exact=round(predicted), predicted=predicted) + + # ---------------------------------------------------------------- the gates def test_the_sample_passes_every_gate(tmp_path): r = run(tmp_path, sample(), page=page_path()) @@ -194,6 +281,20 @@ def test_page_smoke(tmp_path): assert not g["passed"] and any("free scenario" in f for f in g["failures"]) +def test_page_aptitude_check_reconstruction(tmp_path): + page = page_path() + if page is None or "aptitudeFigures" not in (page / "ranking.js").read_text(encoding="utf-8"): + pytest.skip("page does not yet expose the aptitude API") + doc = sample() + # Stochastic check values are not reconstructible from means; changing them must not trigger reconstruction. + var(doc, 0, 0, 1)["check"].update(exact=99999999, predicted=99999999) + assert gate(run(tmp_path, doc, page=page), "page")["passed"] + # Even a self-consistent exported check is independently rejected when deterministic deltas disagree. + var(doc)["check"].update(exact=99999999, predicted=99999999) + g = gate(run(tmp_path, doc, page=page), "page") + assert any("deterministic check reconstruction" in f for f in g["failures"]), g + + def unplayable(doc): doc["songs"][1]["charts"][0]["deck"]["unplayable"] = "more than three fevers" # its seeds kept @@ -214,6 +315,70 @@ def unplayable(doc): (lambda d: seed(d)["ranges"][0].pop("rangeScorePerfect"), "scenarios", "rangeScorePerfect missing"), (lambda d: seed(d).update(rangeWeights=[[[0.1]]]), "scenarios", "rangeWeights are not"), (lambda d: seed(d)["rankCheck"].update(exact=99999), "scenarios", "rank check deck is not within"), + # the Gekisou skill aptitude + (lambda d: d["deck"].pop("gekisouAptitude"), "aptitude", "deck.gekisouAptitude missing"), + (lambda d: d["deck"]["model"].pop("gekisouAptitude"), "aptitude", "deck.model.gekisouAptitude: no text"), + (lambda d: d["deck"]["gekisouAptitude"].update(plainKind=1), "aptitude", "the page's plain kind is 0"), + (lambda d: d["deck"]["kinds"][0].update(durationMs=6000), "aptitude", "plainKind 0, the page's plain kind is None"), + (lambda d: d["deck"]["kinds"][0].update(durationMs=6000), "aptitude", "weights or rangeWeights without a plain"), + (lambda d: d["deck"]["gekisouAptitude"].pop("host"), "aptitude", "deck.gekisouAptitude: no host"), + (lambda d: d["deck"]["gekisouAptitude"].update(seedRule=dict(SEED_RULE, batches=[64, 32])), "aptitude", + "deck.gekisouAptitude.seedRule"), + (lambda d: shape(d, 1).update(id=5), "aptitude", "ids are not 0, 1, 2, ... in order"), + (lambda d: shape(d, 0).update(source="card"), "aptitude", "shape 0: source 'card'"), + (lambda d: shape(d, 2).update(mission=5), "aptitude", "shape 2: mission 5"), + (lambda d: shape(d, 0).update(bandCondition=True), "aptitude", "bandCondition True (a support skill's alone)"), + (lambda d: shape(d, 3).update(bandCondition=True), "aptitude", + "shape 3: bandCondition True, its effects have 0 condition 5000"), + (lambda d: shape(d, 1)["effects"][0]["condition"][0][0].update(targetIds=[41]), "aptitude", + "condition 5000 targetIds not null"), + (lambda d: shape(d, 0)["effects"][0].pop("cumulative"), "aptitude", "effect 0: cumulative missing or malformed"), + (lambda d: shape(d, 0)["effects"][0].update(effectValue=1.5), "aptitude", "effectValue missing or malformed"), + (lambda d: shape(d, 0)["effects"][0].update(reset=None), "aptitude", "reset missing or malformed"), + (lambda d: shape(d, 1)["skills"][0].update(memberTargetIds=None), "aptitude", "is not an id, a level"), + (lambda d: shape(d, 0)["skills"][0].update(bandIds=[1]), "aptitude", "is not an id, a level"), + (lambda d: shape(d, 0).update(skills=[]), "aptitude", "shape 0: no skills"), + (lambda d: d["songs"][0]["charts"][0]["deck"].pop("gekisouAptitude"), "aptitude", "no deck.gekisouAptitude"), + (lambda d: d["songs"][0]["charts"][0]["deck"].update(gekisouAptitude=None), "aptitude", + "deck.gekisouAptitude is null, but the chart is playable"), + (unplayable, "aptitude", "a Gekisou skill aptitude on a chart unplayable with Gekisou on"), + (lambda d: apt(d)["factors"].pop(), "aptitude", "0 factors for 1 ranges"), + (lambda d: apt(d)["factors"][0].update(justNotes=3), "aptitude", "Just or Perfect notes in a range without"), + (lambda d: apt(d)["factors"][0].update(tailNotes=-1), "aptitude", "tailNotes missing or not counts"), + (lambda d: apt(d)["factors"][0].update(lotteries=[1, 0]), "aptitude", "in a range without the luck mission"), + (lambda d: apt(d, 0, 1)["factors"][0].update(lotteries=[3.0, 0]), "aptitude", "deck.seeds' lotResults give 4.0"), + (lambda d: apt(d)["variants"].pop(0), "aptitude", "variants lack [(0, None)]"), + (lambda d: apt(d)["variants"].reverse(), "aptitude", "not in shape order, true first"), + (lambda d: apt(d, 0, 1)["variants"].pop(1), "aptitude", "variants lack [(1, False)]"), + (lambda d: var(d).update(shape=9), "aptitude", "variants of shapes ['9'] not in deck.gekisouAptitude.shapes"), + (lambda d: var(d).update(shape=2), "aptitude", "variants of shapes [2] of a mission the chart does not play"), + (lambda d: var(d, 0, 0, 1).update(bandMatch=None), "aptitude", "bandMatch None for a shape with a band"), + (lambda d: var(d).update(bandMatch=True), "aptitude", "bandMatch True for a shape without a band"), + (lambda d: var(d).pop("tail"), "aptitude", "shape 0: no tail"), + (lambda d: var(d).update(score=[1500]), "aptitude", "score not [mean, se]"), + (lambda d: var(d).update(score=[1500, -1]), "aptitude", "score not [mean, se]"), + (lambda d: var(d).update(converted=[float("nan"), 0]), "aptitude", "converted not [mean, se]"), + (lambda d: var(d)["ranges"][0].update(luckPoints=None), "aptitude", "ranges[0].luckPoints not [mean, se]"), + (lambda d: var(d).update(score=[1500, 1]), "aptitude", "deterministic, but a standard error is not 0: score"), + (lambda d: var(d).update(seeds=2), "aptitude", "deterministic, but 2 seeds"), + (lambda d: var(d, 0, 0, 1).update(seeds=33), "aptitude", "33 seeds (seTargetMet True), not a batch"), + (lambda d: var(d, 0, 0, 1).update(seTargetMet=False), "aptitude", "32 seeds (seTargetMet False), not a batch"), + (lambda d: var(d, 0, 0, 1).update(crossSeeds=64), "aptitude", "crossSeeds 64, expected min(32, 64)"), + (lambda d: var(d)["ranges"].append(dict(var(d)["ranges"][0])), "aptitude", "2 range results for 1 ranges"), + (lambda d: var(d).update(tail=[451, 0]), "aptitude", "tail 451 is not score 1500 less the ranges' rangeScore"), + (lambda d: var(d)["weights"].append([0, 0]), "aptitude", "weights are not one [mean, se] per position"), + (lambda d: seed(d).update(rangeWeights=None, rankCheck=None), "aptitude", + "rangeWeights, but deck.seeds[0].rangeWeights is null"), + (lambda d: var(d).update(rangeWeights=[[[0, 0]]]), "aptitude", "rangeWeights are not [position][range]"), + (lambda d: (nonlinear(d), var(d)["check"].update(ranks=[2])), "aptitude", "ranks are not linear (rank 1 alone)"), + (lambda d: var(d)["check"].update(seed=5), "aptitude", "the check's seed 5 is not deck.seeds[0]'s 0"), + (lambda d: var(d, 0, 0, 1)["check"].update(seed=22), "aptitude", "the check's seed 22 is not deck.seeds[0]'s 11"), + (lambda d: var(d)["check"].update(ranks=[6]), "aptitude", "the check's ranks [6] are not one rank per range"), + (lambda d: var(d)["check"].update(deck=[[1, 7000], None]), "aptitude", "not a [plain kind, value] or null"), + (lambda d: var(d)["check"].update(exact=99999999), "aptitude", "the check is not within its bound"), + (lambda d: var(d)["check"].pop("bound"), "aptitude", "the check has no seed, ranks"), + (lambda d: var(d).update(tailPerfect=[449, 0]), "aptitude", "tailPerfect is not scorePerfect"), + (lambda d: var(d).update(converted=[0.5, 0]), "aptitude", "a point delta is not an integer"), # the deck statistics (lambda d: d.update(deck=None), "deck", "deck is null"), (lambda d: d["songs"][0]["charts"][0].update(deck=None), "deck", "no deck statistics"), @@ -289,7 +454,7 @@ def test_a_jacket_file_is_missing(tmp_path): def test_warnings_do_not_fail(tmp_path): doc = sample() - seed(doc).update(rangeWeights=None, rankCheck=None) # overlapping ranges + nonlinear(doc) # overlapping ranges seed(doc, 0, 1)["rangeWeights"][0] = None # a kind reading the confirmed rank doc["songs"][1]["charts"][0]["deck"]["offSeeds"][0]["weights"][0] = None doc["songs"][0]["master"]["MasterLiveMusic"]["_rate"] = float("inf") # 1e999 in master data as served @@ -300,6 +465,50 @@ def test_warnings_do_not_fail(tmp_path): assert gate(r, "references")["warnings"] == ["songs without a zh-Hant title: 100002"] +def test_aptitude_warnings(tmp_path): + doc = sample() + var(doc, 0, 0, 1).update(seeds=1024, seTargetMet=False, crossSeeds=64) # the last batch, not the target + r = run(tmp_path, doc) + assert r["passed"], failures(r) + assert gate(r, "aptitude")["warnings"] == [ + "1 variants missed the seed rule's standard error target: 30 shape 1"] + assert gate(r, "aptitude")["note"] == ("4 shapes; 3 charts with an aptitude, 0 null; 8 variants, " + "2 deterministic; plain kind 0") + + +def test_no_gekisou_skill_shapes(tmp_path): + doc = sample() + doc["deck"]["gekisouAptitude"]["shapes"] = [] + kept = apt(doc) + for _, c in music_data.charts_of(doc): + c["deck"]["gekisouAptitude"] = None + r = run(tmp_path, doc, page=page_path()) + assert r["passed"], failures(r) + doc["songs"][0]["charts"][0]["deck"]["gekisouAptitude"] = kept + g = gate(run(tmp_path, doc), "aptitude") + assert g["failures"] == ["chart 10 (100001 easy): a Gekisou skill aptitude on a chart without a Gekisou skill " + "shape"] + + +def test_an_unplayable_chart_has_no_aptitude(tmp_path): + doc = sample() + doc["songs"][1]["charts"][0]["deck"].update(unplayable="more than three fevers", seeds=[], gekisouAptitude=None) + r = run(tmp_path, doc, page=page_path()) + assert r["passed"], failures(r) + assert gate(r, "aptitude")["note"].startswith("4 shapes; 2 charts with an aptitude, 1 null") + + +def test_gzip_caps(tmp_path, monkeypatch): + raw, ctx = context(tmp_path, sample()) + g = gate(gates(raw, ctx, only=("gzip",)), "gzip") + assert g["passed"] and g["note"].startswith(f"{len(music_data.gzip.compress(raw, 6))} bytes gzipped") + monkeypatch.setattr(music_data, "APTITUDE_GZIP_MAX", 100) + g = gate(gates(raw, ctx, only=("gzip",)), "gzip") + assert not g["passed"] and g["failures"][0].startswith("the Gekisou skill aptitude: ") + monkeypatch.setattr(music_data, "FILE_GZIP_MAX", 100) + assert len(gate(gates(raw, ctx, only=("gzip",)), "gzip")["failures"]) == 2 + + def test_against_the_published_file(tmp_path): doc = sample() raw, ctx = context(tmp_path, doc) @@ -380,12 +589,26 @@ def test_a_real_file_with_the_scenario_fields(tmp_path): raw = real("MUSIC_DATA_SAMPLE") (tmp_path / "music-data.json").write_bytes(raw) ctx = Context(language="zh-Hant", page=page_path(), file=tmp_path / "music-data.json", published=raw) - r = gates(raw, ctx, only=CONTENT + ("counts", "size", "page")) - if luck_points(raw): - assert r["passed"], failures(r) - else: - assert [n for n, _ in failures(r)] == ["deck"] and before_luck_points(r), failures(r) + r = gates(raw, ctx, only=CONTENT + ("aptitude", "counts", "size", "gzip", "page")) + new = [n for n, ok in (("deck", luck_points(raw)), ("aptitude", "gekisouAptitude" in json.loads(raw)["deck"])) + if not ok] # the gates of fields made after the file + assert [n for n, _ in failures(r)] == new, failures(r) + assert "deck" not in new or before_luck_points(r) + g = gate(r, "aptitude") + assert "aptitude" not in new or all(f.endswith(("gekisouAptitude missing", "gekisouAptitude: no text", + ": no deck.gekisouAptitude")) for f in g["failures"]), g assert gate(r, "scenarios")["warningCount"] == 0 + print(f"aptitude: {g['note']}; warnings: {g['warnings']}; gzip: {gate(r, 'gzip')['note']}") + + +def test_real_aptitude_deck_sample(): + """Optional real chart-stats output, wrapped without modifying its statistics or touching the source file.""" + stats = json.loads(real("MUSIC_DATA_APTITUDE_DECK_SAMPLE")) + doc = {"deck": {k: stats[k] for k in ("model", "kinds", "gekisouAptitude")}, + "songs": [{"id": c["musicId"], "charts": [{"scoreId": c["scoreId"], "difficulty": c["difficulty"], + "skillEventsMs": [e[1] for e in c["events"]], "deck": c}]} for c in stats["charts"]]} + report = gates(json.dumps(doc).encode(), Context(), only=("deck", "aptitude", "gzip")) + assert report["passed"], failures(report) # ---------------------------------------------------------------- publish (a stand-in bucket)