feat(jp): support Japanese release assets and split APKs

.github/MUSIC_DATA.md

@@ -1,5 +1,9 @@
 # Music data workflow (StarMoe)
 
+JP: `MUSIC_DATA_MASTERDATA_REGION=jp` selects the JP package, catalog and client metadata, with
+`jp/music-data` as the default output prefix. The marker includes the asset hash and the provenance gate rejects
+a catalog from another snapshot. See [Japanese release](../docs/jp.md) for setup and validation limits.
+
 `.github/workflows/music-data.yml` keeps the music data file of the chart data page (ournotes-player
 `examples/songs`) up to date: `nnnotes music-data` of the current master data ([docs/music-data.md](../docs/music-data.md):
 every song and chart with the deck model's statistics and the play scenarios), checked by quality gates and
@@ -14,8 +18,8 @@ published into the story site's bucket under `music-data/` (`https://storage.bdo
 
 A run never deletes anything from the bucket. Its helper steps are `.github/scripts/music_data.py` (with the bucket,
 HTTP and master data helpers of `story_site.py`), `music_data_smoke.mjs`, `songs_page.sh` and `apk.sh`; the gate
-self-test is `test_music_data.py`. Nothing outside `.github/` differs from upstream, so the fork syncs with it as
-before. The Cloudflare Pages preview of the page is not part of the workflow.
+self-tests are `test_music_data.py` and `test_jp_workflow.py`. The fork also contains JP support in `src/nnnotes`.
+The Cloudflare Pages preview of the page is not part of the workflow.
 
 ## A run
 

.github/scripts/music_data.py

@@ -49,7 +49,7 @@
 BUILD_FORMAT = "moenotes.music-data-build/1"
 # This script's own version of a build: bump it when what it builds or publishes changes, so that the next run builds
 # although the master data, the deck model and nnnotes are the same.
-RECIPE = 1
+RECIPE = 2
 FILE, MARKER, JACKETS, ARCHIVE = "music-data.json", "build.json", "jackets/", "archive/"
 MANIFEST = "MasterManifest.json"
 SOURCE_PATHS = ("src", "rust", "pyproject.toml")    # nnnotes' code: the commit that last changed one of them
@@ -58,7 +58,7 @@
 ARCHIVE_CACHE = story_site.ASSET_CACHE              # archive/<version>/<sha256>.json: content-addressed
 FILE_CACHE = "no-cache"                             # music-data.json and build.json change in place
 JACKET_CACHE = "public, max-age=86400"
-SNAPSHOT_KEYS = ("version", "resource_version", "client_version", "verified_at", "manifest_sha256", "table_count")
+SNAPSHOT_KEYS = ("version", "resource_version", "resource_hash", "client_version", "verified_at", "manifest_sha256", "table_count")
 
 # the gates' bounds (MUSIC_DATA.md)
 SIZE_RATIO = (0.8, 2.0)                             # against the published file
@@ -159,6 +159,7 @@ def inputs(entry: dict, root: Path = Path(".")) -> dict:
     """What a build is made of: the master data snapshot, the deck model, nnnotes and this script."""
     return {"masterRegion": env("MASTERDATA_REGION"), "masterVersion": entry.get("version"),
             "resourceVersion": entry.get("resource_version"), "clientVersion": entry.get("client_version"),
+            "resourceHash": entry.get("resource_hash"),
             "deckCommit": deck_commit(root), "nnnotesCommit": nnnotes_commit(root), "recipe": RECIPE}
 
 
@@ -220,6 +221,7 @@ def cmd_master(out: str) -> None:
 
 # ---------------------------------------------------------------- build
 def cmd_build(out: str) -> None:
+    story_site.configure_region()
     o = Path(out).resolve()
     o.mkdir(parents=True, exist_ok=True)
     nnnotes = [sys.executable, "-m", "nnnotes"]
@@ -324,6 +326,10 @@ def gate_provenance(doc, ctx: Context, g: Gate):
         v = ctx.snapshot["entry"].get("version")
         if m.get("version") != v:
             g.fail(f"master.version {m.get('version')!r}, the snapshot's {v!r}")
+        if ctx.snapshot.get("region") == "jp":
+            for expected, actual in (("resource_version", "resourceVersion"), ("resource_hash", "resourceHash")):
+                if (p.get("catalog") or {}).get(actual) != ctx.snapshot["entry"].get(expected):
+                    g.fail(f"JP catalog {actual} differs from the master snapshot")
     if ctx.master is not None:
         manifest = json.loads((ctx.master / MANIFEST).read_bytes())
         if m.get("version") != manifest.get("version"):

.github/scripts/story_site.py

@@ -137,6 +137,32 @@ def master_index() -> tuple[str, dict]:
     return base + region["path"], region
 
 
+def configure_region() -> None:
+    """Keep build inputs from one release; JP client/API/CDN are read from the public snapshot."""
+    region = env("MASTERDATA_REGION")
+    names = {"hk-tw-mo": "tw", "en": "en", "kr": "kr", "jp": "jp"}
+    if region not in names or env("NNNOTES_CATALOG_REGION") != names[region]:
+        sys.exit("story_site: master region and catalog region differ")
+    if region != "jp":
+        return
+    _, snapshot = master_index()
+    entry = snapshot.get("entry") or {}
+    assets = entry.get("assets") or {}
+    upstream = entry.get("upstream") or {}
+    api = assets.get("api_root") or upstream.get("api_root")
+    bundle = assets.get("bundle_root")
+    from urllib.parse import urlsplit
+    cdn = upstream.get("cdn_root")
+    if bundle:
+        u = urlsplit(bundle)
+        cdn = f"{u.scheme}://{u.netloc}"
+    client = entry.get("client_version")
+    if not all(isinstance(v, str) and v for v in (api, cdn, client)):
+        sys.exit("story_site: JP snapshot lacks API/CDN/client metadata")
+    os.environ.update(NNNOTES_SERVERS_JP_API=api, NNNOTES_SERVERS_JP_CDN=cdn,
+                      NNNOTES_SERVERS_JP_CLIENT_VERSION=client, NNNOTES_SERVERS_JP_PROVIDER="jp")
+
+
 def fetch_table(url: str, name: str, digest: str, dest: Path) -> None:
     if not re.fullmatch(r"[A-Za-z0-9_-]+\.json", name):
         sys.exit(f"story_site: unexpected master file name {name!r}")
@@ -215,10 +241,13 @@ def fetch(key: str) -> None:
 
 
 def cmd_build(site_dir: str, ids: list[str]) -> None:
+    configure_region()
     site = Path(site_dir).resolve()
     stories = parse_ids(" ".join(ids))
     tmp = site.parent / f"{site.name}.tmp"
     nnnotes = [sys.executable, "-m", "nnnotes", "web", str(site), "--tmp", str(tmp)]
+    if env("MASTERDATA_REGION") == "jp":
+        nnnotes += ["--story-languages", "ja"]
     status = 0
     if stories:
         cmd = nnnotes + [a for i in stories for a in ("--story", str(i))]

.github/scripts/test_jp_workflow.py

@@ -0,0 +1,52 @@
+"""The workflow selects one release and derives JP endpoints without copying credentials."""
+import os
+from pathlib import Path
+import sys
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent))
+import story_site
+import music_data
+
+
+def test_configure_jp_from_public_snapshot(monkeypatch):
+    monkeypatch.setenv('MASTERDATA_REGION', 'jp')
+    monkeypatch.setenv('NNNOTES_CATALOG_REGION', 'jp')
+    entry = {'client_version': '1.0.4', 'assets': {'api_root': 'https://api.example.test',
+             'bundle_root': 'https://cdn.example.test/asset/1.0.0.300/Android/' + 'a' * 32}}
+    monkeypatch.setattr(story_site, 'master_index', lambda: ('unused', {'entry': entry}))
+    for key in ('API', 'CDN', 'CLIENT_VERSION', 'PROVIDER'):
+        monkeypatch.setenv('NNNOTES_SERVERS_JP_' + key, '')
+    story_site.configure_region()
+    assert os.environ['NNNOTES_SERVERS_JP_CDN'] == 'https://cdn.example.test'
+    assert os.environ['NNNOTES_SERVERS_JP_API'] == 'https://api.example.test'
+    assert os.environ['NNNOTES_SERVERS_JP_CLIENT_VERSION'] == '1.0.4'
+
+
+def test_reject_mixed_master_and_catalog(monkeypatch):
+    monkeypatch.setenv('MASTERDATA_REGION', 'jp')
+    monkeypatch.setenv('NNNOTES_CATALOG_REGION', 'tw')
+    with pytest.raises(SystemExit, match='differ'):
+        story_site.configure_region()
+
+
+def test_asset_hash_changes_build_inputs(monkeypatch):
+    monkeypatch.setenv('MASTERDATA_REGION', 'jp')
+    monkeypatch.setattr(music_data, 'deck_commit', lambda root: 'a' * 40)
+    monkeypatch.setattr(music_data, 'nnnotes_commit', lambda root: 'b' * 40)
+    a = music_data.inputs({'version': 'v', 'resource_hash': 'a' * 32})
+    b = music_data.inputs({'version': 'v', 'resource_hash': 'b' * 32})
+    assert a != b and a['resourceHash'] == 'a' * 32
+
+
+def test_jp_catalog_provenance_matches_snapshot():
+    from test_music_data import sample
+    doc = sample()
+    doc['provenance']['catalog']['resourceVersion'] = '1.0.0.300'
+    doc['provenance']['catalog']['resourceHash'] = 'a' * 32
+    context = music_data.Context(snapshot={'region': 'jp', 'entry': {
+        'version': doc['provenance']['master']['version'], 'resource_version': '1.0.0.300', 'resource_hash': 'b' * 32}})
+    gate = music_data.Gate()
+    music_data.gate_provenance(doc, context, gate)
+    assert any('resourceHash differs' in s for s in gate.failures)

.github/workflows/music-data.yml

@@ -35,14 +35,14 @@ permissions:
 
 # One run at a time: a run replaces music-data.json and build.json.
 concurrency:
-  group: music-data
+  group: music-data-${{ vars.MUSIC_DATA_MASTERDATA_REGION || 'hk-tw-mo' }}
   cancel-in-progress: false
 
 env:
   # The story site's bucket (its repository variables) and the key prefix of the music data.
   STORY_S3_ENDPOINT: ${{ vars.STORY_S3_ENDPOINT || 'https://storage.bdon.moe' }}
   STORY_S3_BUCKET: ${{ vars.STORY_S3_BUCKET || 'moenotes' }}
-  MUSIC_DATA_S3_PREFIX: ${{ vars.MUSIC_DATA_S3_PREFIX || 'music-data' }}
+  MUSIC_DATA_S3_PREFIX: ${{ vars.MUSIC_DATA_S3_PREFIX || (vars.MUSIC_DATA_MASTERDATA_REGION == 'jp' && 'jp/music-data' || 'music-data') }}
   # The publishing switch: anything but 'true' makes every run a dry run.
   MUSIC_DATA_PUBLISH: ${{ vars.MUSIC_DATA_PUBLISH || 'false' }}
   # moenotes-masterdata-sync: decoded master data by region (index.json lists every file with its SHA-256).
@@ -53,12 +53,14 @@ env:
   PLAYER_REPOSITORY: ${{ vars.MUSIC_DATA_PLAYER_REPOSITORY || 'empty-sekai/ournotes-player' }}
   PLAYER_REF: ${{ vars.MUSIC_DATA_PLAYER_REF || '' }}
   PLAYFETCH_VERSION: ${{ vars.PLAYFETCH_VERSION || 'v0.92' }}
-  APK_PACKAGE: ${{ vars.STORY_APK_PACKAGE || 'com.bilibili.sirius' }}
+  APK_PACKAGE: ${{ vars.MUSIC_DATA_MASTERDATA_REGION == 'jp' && 'com.bushiroad.sirius' || vars.STORY_APK_PACKAGE || 'com.bilibili.sirius' }}
   PYTHON_VERSION: "3.13"
   WORK: ${{ github.workspace }}/../work
   # nnnotes settings (docs/configuration.md); the bundle key and the CDN come from secrets in the build step.
-  NNNOTES_CATALOG_REGION: tw
-  NNNOTES_CATALOG_LANGUAGE: zh-Hant
+  NNNOTES_CATALOG_REGION: ${{ vars.MUSIC_DATA_MASTERDATA_REGION == 'jp' && 'jp' || 'tw' }}
+  NNNOTES_CATALOG_LANGUAGE: ${{ vars.MUSIC_DATA_MASTERDATA_REGION == 'jp' && 'ja' || 'zh-Hant' }}
+  NNNOTES_SERVERS_JP_NAME: JP
+  NNNOTES_SERVERS_JP_LANGUAGES: ja
   NNNOTES_SERVERS_TW_NAME: TW/HK/MO
   NNNOTES_SERVERS_TW_LANGUAGES: zh-Hans,zh-Hant,en,ko,ja
 
@@ -115,7 +117,7 @@ jobs:
       - name: Gate self-test
         env:
           MUSIC_DATA_PAGE: ${{ env.WORK }}/page/examples/songs
-        run: python -m pytest -q -p no:cacheprovider .github/scripts/test_music_data.py
+        run: python -m pytest -q -p no:cacheprovider .github/scripts/test_music_data.py .github/scripts/test_jp_workflow.py
 
       - name: APK
         env:

.github/workflows/story-site.yml

@@ -34,14 +34,14 @@ permissions:
 
 # One run at a time: every run rewrites the site's indexes from the manifests it fetched.
 concurrency:
-  group: story-site
+  group: story-site-${{ vars.STORY_MASTERDATA_REGION || 'hk-tw-mo' }}
   cancel-in-progress: false
 
 env:
   # The bucket of the site (repository variables; the defaults are the StarMoe site).
   STORY_S3_ENDPOINT: ${{ vars.STORY_S3_ENDPOINT || 'https://storage.bdon.moe' }}
   STORY_S3_BUCKET: ${{ vars.STORY_S3_BUCKET || 'moenotes' }}
-  STORY_S3_PREFIX: ${{ vars.STORY_S3_PREFIX || '' }}
+  STORY_S3_PREFIX: ${{ vars.STORY_S3_PREFIX || (vars.STORY_MASTERDATA_REGION == 'jp' && 'jp' || '') }}
   # moenotes-masterdata-sync: decoded master data by region (index.json lists every table with its SHA-256).
   MASTERDATA_BASE_URL: ${{ vars.MASTERDATA_BASE_URL || 'https://metadata.bdon.moe' }}
   MASTERDATA_REGION: ${{ vars.STORY_MASTERDATA_REGION || 'hk-tw-mo' }}
@@ -51,12 +51,14 @@ env:
   PLAYER_REPOSITORY: ${{ vars.STORY_PLAYER_REPOSITORY || 'empty-sekai/ournotes-player' }}
   PLAYER_REF: ${{ vars.STORY_PLAYER_REF || '3774d8ac3987' }}
   PLAYFETCH_VERSION: ${{ vars.PLAYFETCH_VERSION || 'v0.92' }}
-  APK_PACKAGE: ${{ vars.STORY_APK_PACKAGE || 'com.bilibili.sirius' }}
+  APK_PACKAGE: ${{ vars.STORY_MASTERDATA_REGION == 'jp' && 'com.bushiroad.sirius' || vars.STORY_APK_PACKAGE || 'com.bilibili.sirius' }}
   PYTHON_VERSION: "3.13"
   WORK: ${{ github.workspace }}/../work
   # nnnotes settings (docs/configuration.md); the keys and the CDN come from secrets in the build step.
-  NNNOTES_CATALOG_REGION: tw
-  NNNOTES_CATALOG_LANGUAGE: zh-Hant
+  NNNOTES_CATALOG_REGION: ${{ vars.STORY_MASTERDATA_REGION == 'jp' && 'jp' || 'tw' }}
+  NNNOTES_CATALOG_LANGUAGE: ${{ vars.STORY_MASTERDATA_REGION == 'jp' && 'ja' || 'zh-Hant' }}
+  NNNOTES_SERVERS_JP_NAME: JP
+  NNNOTES_SERVERS_JP_LANGUAGES: ja
   NNNOTES_SERVERS_TW_NAME: TW/HK/MO
   NNNOTES_SERVERS_TW_LANGUAGES: zh-Hans,zh-Hant,en,ko,ja
 

README.en.md

@@ -1,5 +1,9 @@
 # nnnotes？！
 
+Japanese-release support includes anonymous version discovery, authenticated CDN downloads, gzip catalogs,
+snapshot-isolated caches and split APKs. See the [JP guide](https://github.com/StarMoe-org/nnnotes/blob/main/docs/jp.md)
+for configuration and validation limits.
+
 [简体中文](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.md) | [English](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.en.md)
 
 nnnotes is an offline data toolkit for the game files of BanG Dream! Our Notes: it reads Addressables catalogs,

README.md

@@ -52,6 +52,9 @@ nnnotes config check             # 每项设置的来源和格式是否有效，
 
 ## 完成度
 
+日服现已支持匿名版本查询、CDN 认证下载、gzip catalog、资源路径与缓存隔离，以及 Android 分包读取。
+配置、命令和日服验证范围见 [日服支持](docs/jp.md)。
+
 以下结果基于台服 1.0.1（zh-Hant）的全部数据：
 
 | 部分 | 状态 |

docs/configuration.md

@@ -1,5 +1,7 @@
 # Configuration
 
+For Japanese-release endpoints, catalog snapshots and split APKs, see [Japanese release](jp.md).
+
 nnnotes has no built-in keys, server addresses or data paths. Every setting comes from one of three sources, and a
 command that needs a setting nobody gave stops before it does any work.
 
@@ -87,6 +89,10 @@ exist when they are given.
 | `[catalog] region` | `NNNOTES_CATALOG_REGION` | `--region` | name of one `[servers.<region>]` table |
 | `[catalog] language` | `NNNOTES_CATALOG_LANGUAGE` | `--language` | catalog language: the `<language>` of `catalog_main_<language>.bin`: `ja`, `en`, `zh-Hant`, `zh-Hans` or `ko`; also the client language of `live`, `story` and `web` (the text field, fonts and line spacing of their UI) and of the model labels of `web --live2d` |
 | `[servers.<region>] name` | `NNNOTES_SERVERS_<REGION>_NAME` | — | label of the region in `browse` (default: the region name) |
+| `[servers.<region>] provider` | `NNNOTES_SERVERS_<REGION>_PROVIDER` | — | `international` or `jp`; empty selects `jp` for the region named jp, international otherwise |
+| `[servers.<region>] client_version` | `NNNOTES_SERVERS_<REGION>_CLIENT_VERSION` | — | per-region API client version; overrides `[client] version`, then falls back to the region's APK versionName |
+| `[servers.<region>] apk` | `NNNOTES_SERVERS_<REGION>_APK` | `--apk` | per-region APK, APKS/XAPK or directory with base.apk; adjacent splits of base.apk are read automatically; the flag overrides it |
+| `[servers.<region>] catalog` | `NNNOTES_SERVERS_<REGION>_CATALOG` | `--catalog` | per-region catalog file; JP needs the matching `<file>.source.json`; the flag overrides it |
 | `[servers.<region>] cdn` | `NNNOTES_SERVERS_<REGION>_CDN` | — | CDN base URL of the region (a trailing `/` is ignored) |
 | `[servers.<region>] languages` | `NNNOTES_SERVERS_<REGION>_LANGUAGES` | — | catalog languages `browse` lists: a TOML array of strings; comma-separated in the environment |
 | `[servers.<region>] api` | `NNNOTES_SERVERS_<REGION>_API` | — | API root of the region: `https://host[:port]` (TLS, port 443 by default), `host[:port]`, or `http://host[:port]` for a plain-text local server; no path |