diff --git a/README.en.md b/README.en.md index 6906a25..e6c39ee 100644 --- a/README.en.md +++ b/README.en.md @@ -1,5 +1,9 @@ # nnnotes？！ +Japanese-release support includes anonymous version discovery, authenticated CDN downloads, gzip catalogs, +snapshot-isolated caches and split APKs. See the [JP guide](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/jp.md) +for configuration and validation limits. + [简体中文](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.md) | [English](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.en.md) nnnotes is an offline data toolkit for the game files of BanG Dream! Our Notes: it reads Addressables catalogs, diff --git a/README.md b/README.md index 96b94dc..ab28b70 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,9 @@ nnnotes config check # 每项设置的来源和格式是否有效， ## 完成度 +日服现已支持匿名版本查询、CDN 认证下载、gzip catalog、资源路径与缓存隔离，以及 Android 分包读取。 +配置、命令和日服验证范围见 [日服支持](docs/jp.md)。 + 以下结果基于台服 1.0.1（zh-Hant）的全部数据： | 部分 | 状态 | diff --git a/docs/configuration.md b/docs/configuration.md index c98fbd7..af7515c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,5 +1,7 @@ # Configuration +For Japanese-release endpoints, catalog snapshots and split APKs, see [Japanese release](jp.md). + nnnotes has no built-in keys, server addresses or data paths. Every setting comes from one of three sources, and a command that needs a setting nobody gave stops before it does any work. @@ -87,6 +89,10 @@ exist when they are given. | `[catalog] region` | `NNNOTES_CATALOG_REGION` | `--region` | name of one `[servers.]` table | | `[catalog] language` | `NNNOTES_CATALOG_LANGUAGE` | `--language` | catalog language: the `` of `catalog_main_.bin`: `ja`, `en`, `zh-Hant`, `zh-Hans` or `ko`; also the client language of `live`, `story` and `web` (the text field, fonts and line spacing of their UI) and of the model labels of `web --live2d` | | `[servers.] name` | `NNNOTES_SERVERS__NAME` | — | label of the region in `browse` (default: the region name) | +| `[servers.] provider` | `NNNOTES_SERVERS__PROVIDER` | — | `international` or `jp`; empty selects `jp` for the region named jp, international otherwise | +| `[servers.] client_version` | `NNNOTES_SERVERS__CLIENT_VERSION` | — | per-region API client version; overrides `[client] version`, then falls back to the region's APK versionName | +| `[servers.] apk` | `NNNOTES_SERVERS__APK` | `--apk` | per-region APK, APKS/XAPK or directory with base.apk; adjacent splits of base.apk are read automatically; the flag overrides it | +| `[servers.] catalog` | `NNNOTES_SERVERS__CATALOG` | `--catalog` | per-region catalog file; JP needs the matching `.source.json`; the flag overrides it | | `[servers.] cdn` | `NNNOTES_SERVERS__CDN` | — | CDN base URL of the region (a trailing `/` is ignored) | | `[servers.] languages` | `NNNOTES_SERVERS__LANGUAGES` | — | catalog languages `browse` lists: a TOML array of strings; comma-separated in the environment | | `[servers.] api` | `NNNOTES_SERVERS__API` | — | API root of the region: `https://host[:port]` (TLS, port 443 by default), `host[:port]`, or `http://host[:port]` for a plain-text local server; no path | diff --git a/docs/jp.md b/docs/jp.md new file mode 100644 index 0000000..43a95d5 --- /dev/null +++ b/docs/jp.md @@ -0,0 +1,114 @@ +# Japanese release + +JP uses the same catalog, bundle and master parsers, with a separate version discovery and CDN download path. +The anonymous `MasterdataService/Version` response provides the master version, Android asset version/hash and +CDN authentication. No player account or login is needed. + +## Configuration + +Use your own client addresses and encryption settings. As elsewhere in nnnotes, neither addresses nor keys are +built into the package. Add a region table with these settings to your private configuration: + +```toml +[catalog] +region = "jp" +language = "ja" + +[servers.jp] +provider = "jp" +name = "JP" +api = "https://YOUR_API_HOST" +cdn = "https://YOUR_CDN_HOST" +client_version = "YOUR_CURRENT_CLIENT_VERSION" +languages = ["ja"] +apk = "/path/to/jp/base.apk" +master = "/path/to/jp/master" + +[paths] +cache = "/path/to/cache" +``` + +Keep `[bundle] key/nonce_seed` and `[master] key/iv` in your configuration. Master keys are unnecessary when +reading already decoded tables with `music-data --decoded-master`. + +`provider` defaults to `jp` for the region named `jp`, and to `international` otherwise. An alias such as +`servers.japan` needs `provider = "jp"`. `client_version` overrides `[client] version`; when neither is set, +nnnotes reads the APK versionName. Use the current accepted JP client version: an old extracted APK can still +contain readable local data even when its version is no longer accepted by the API. + +The `cdn` setting is also the allowed CDN origin for authentication. A different origin in Version fails before +any authenticated download. The JP transport uses HTTPS, follows no redirects, keeps authentication in memory, +and refreshes it once on 401/403. A 429 does not cause an authentication-refresh loop. Credentials are not written +to catalog metadata or task files. + +## APKs and split data + +`apk` accepts: + +- a normal APK; +- `base.apk` with adjacent `split_*.apk` / `config.*.apk` files; +- a directory containing `base.apk` and its splits; +- an `.apks` or `.xapk` archive containing `base.apk` and its splits. + +JP stores boot data in base.apk and the embedded catalog/master/bundles in the Unity data split. Both are read +through the same APK-set interface. `datapack.unity3d` is loaded alongside `data.unity3d` for external resources +such as materials and shaders. Native libraries are also read from the splits. No repacked APK is required. + +`[servers.] apk` and `catalog` override their `[paths]` equivalents; explicit `--apk` / `--catalog` flags +still win. For mixed-release chart builds, configure separate master directories and APK sets. JP stories and +models must use a separate site directory from international releases, since their model IDs may overlap. + +## Commands + +```sh +nnnotes --region jp master version +nnnotes --region jp master download --latest -o work/jp-master-bin +nnnotes master decode work/jp-master-bin -o work/jp-master +nnnotes --region jp catalog --prefix Live/MusicScore/ --limit 10 +nnnotes --region jp pull Live/MusicScore/0001/0001_00 +nnnotes --region jp catalogs fetch +nnnotes --region jp export --select key:Image/Jacket/jacket_temporary --views none -o out/jp-sample +nnnotes --region jp music-data --decoded-master --no-bgm -o out/jp-music-data.json +``` + +For `--decoded-master`, keep `MasterManifest.json` with the decoded tables (copy it from the download directory, +or use a snapshot from your masterdata service). `--no-bgm` omits song audio length inspection; omit that flag to +include it. `master version` reports `resourceHash` for JP as well as the two version fields. `music-data` records +the JP hash in `provenance.catalog.resourceHash`. + +JP catalogs are `catalog_main.bin` in a version/hash directory, without a language suffix. The HTTP payload can +be gzip; nnnotes keeps its original bytes and SHA-256, and bounds decompression before parsing. Remote bundle and +CRI locations use `{Fwk.Resource.RemoteAssetDir}`. These are indexed, resolved and downloaded through the selected +snapshot, including in `browse`, `catalogs`, `export`, `plan` and `run-stage`. + +The cache lives under `/jp////`. It cannot reuse the +international catalog cache. Embedded files are further isolated under `apk//`, so an APK +update cannot reuse old local bundles when the CDN snapshot stays unchanged. Each cached `catalog_main.bin` has a `catalog_main.bin.source.json` with its SHA-256 +and public source metadata. For an offline `--catalog` or `catalogs import`, copy the pair together. Credentials +are reacquired only when a missing file must be downloaded. + +Catalog database records include this source metadata, and JP catalog identity includes version/hash even when +the catalog bytes remain the same. Old international records retain their previous IDs. Cached historical files +can be read offline; downloading a missing historical file fails if Version now selects a different snapshot. +Refresh the catalog to use the new snapshot. Historical local bundles require the APK set matching the imported +APK catalog. A JP master download likewise rejects a different current master snapshot. + +## Validation and current limits + +The synthetic tests use local gRPC and HTTP servers, including gzip catalogs, split packages, authentication +rotation, redirects, 429, truncated responses, source isolation, APK-only and hash-only updates, offline replay +and invalid paths. + +Live validation on 2026-09-30 used JP client 1.0.4 and local JP Android 1.0.3 data: + +- 238 master tables downloaded with manifest hashes and decoded successfully; +- a 73-note chart and a 512×512 jacket read successfully; +- one Live2D runtime model and one complete story exported; the story's 22 FLAC files passed full FFmpeg decoding; +- selected `export` produced six files without failed tasks; +- `music-data --decoded-master --no-bgm` built 85 songs / 340 charts, including the pinned deck model's + statistics, with no unplayable chart; output passed JSON Schema validation. + +This does not establish full JP story/Live2D/movie export coverage, browser playback, every BGM, or native scoring +parity. JP's text tables retain five language columns, but most translated values are empty; story web builds +default to Japanese. Existing Bili chat skin view rules still refer to international `_iconAssetPath` fields; +JP rows with a different schema report no-value and are not represented as verified skin mappings. diff --git a/docs/schema/music-data.schema.json b/docs/schema/music-data.schema.json index dad6a10..3d9221b 100644 --- a/docs/schema/music-data.schema.json +++ b/docs/schema/music-data.schema.json @@ -71,6 +71,11 @@ ], "description": "the resource version recorded for the catalog in the catalog store, null when none is recorded" }, + "resourceHash": { + "type": "string", + "pattern": "^[0-9a-f]{32}$", + "description": "JP Android asset directory hash, when the catalog source is JP" + }, "sha256": { "$ref": "#/$defs/sha256", "description": "SHA-256 of the remote catalog file the charts were read with" diff --git a/docs/stages.md b/docs/stages.md index 9ac98ee..92b437f 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -13,7 +13,8 @@ The asset export is built from three layers: ## Stages Every stage is at version 1, except `unity.census` (version 2: it lists the scripts animation clip bindings name) -and `cri.movie` (version 2: every stream kind, codec and channel of a USM). +and `cri.movie` (version 2: every stream kind, codec and channel of a USM), and `catalog.index` (version 2: +gzip input and Japanese remote asset placeholders). | Stage | Subject | Inputs | Outputs | |---|---|---|---| diff --git a/src/nnnotes/addressables.py b/src/nnnotes/addressables.py index 23ca07a..19c9dbd 100644 --- a/src/nnnotes/addressables.py +++ b/src/nnnotes/addressables.py @@ -15,6 +15,8 @@ from __future__ import annotations import hashlib +import gzip +import io import struct from dataclasses import dataclass, field from html import escape @@ -32,6 +34,23 @@ CATALOG_MAGIC = 0x0DE38942 CATALOG_VERSION = 2 NONE = 0xFFFFFFFF +REMOTE_PREFIX = "{Fwk.Resource.RemoteAssetDir}/" +MAX_CATALOG = 128 * 1024 * 1024 + + +def catalog_bytes(data: bytes) -> bytes: + """Normalize HTTP gzip or raw binary catalog bytes with a bounded expanded size.""" + if len(data) > MAX_CATALOG: + raise ValueError("catalog exceeds the size limit") + if data.startswith(b"\x1f\x8b"): + try: + with gzip.GzipFile(fileobj=io.BytesIO(data)) as stream: + data = stream.read(MAX_CATALOG + 1) + except (OSError, EOFError): + raise ValueError("invalid gzip catalog") from None + if len(data) > MAX_CATALOG: + raise ValueError("expanded catalog exceeds the size limit") + return data @dataclass(frozen=True) @@ -56,6 +75,9 @@ def decrypt(data: bytes, filename: str, key: BundleKey) -> bytes: def remote_path(internal_id: str) -> str | None: """The path below the CDN base of a remote location (an absolute URL: everything after its host), else None.""" + if internal_id.startswith(REMOTE_PREFIX): + from .jp import relative_path + return "/" + relative_path(internal_id[len(REMOTE_PREFIX):]) scheme, sep, rest = internal_id.partition("://") if not sep or not scheme.isalpha(): return None @@ -65,6 +87,7 @@ def remote_path(internal_id: str) -> str | None: def parse(data: bytes) -> list[dict]: """Every location of a binary catalog: {offset, primary_key, internal_id, dependencies (location offsets)}.""" + data = catalog_bytes(data) def u32(offset): return struct.unpack_from(" dict: def parse_header(data: bytes) -> dict: """The header of a binary catalog: magic, version, keysOffset, locatorId, instanceProvider, sceneProvider ({id, assembly, type, data}), initObjects (the same, the providers the catalog initializes), buildResultHash.""" + data = catalog_bytes(data) if len(data) < HEADER.size: raise ValueError("unsupported catalog format") magic, version, keys, locator, instance, scene, init, build = HEADER.unpack_from(data, 0) @@ -244,6 +268,7 @@ def parse_header(data: bytes) -> dict: def parse_keys(data: bytes) -> list[dict]: """The key table of a binary catalog in stored order (ContentCatalogData.ResourceLocator.KeyData {u32 key object, u32 location set}): {"key": value, "type": the key's type name, "locations": location offsets}.""" + data = catalog_bytes(data) header = parse_header(data) buf = _Buffer(data) out = [] @@ -267,6 +292,7 @@ def parse_locations(data: bytes) -> list[dict]: offsets), dependency hash (i32), extra data (an ObjectTypeData) and resource type (a TypeSerializer.Data). extra_data of bundle and raw file locations is an AssetBundleRequestOptions: {type, hash, bundleName, crc, bundleSize, timeout, redirectLimit, retryCount, flags and the flag bits by name}.""" + data = catalog_bytes(data) header = parse_header(data) buf = _Buffer(data) offsets: set[int] = set() @@ -309,6 +335,7 @@ class Region: label: str cdn: str = field(repr=False) languages: list[str] + config: object = field(default=None, repr=False) class Handler(BaseHTTPRequestHandler): @@ -329,6 +356,12 @@ def listing(self, label: str, rows) -> None: def catalog(self, region: Region, language: str): key = (region.name, language) if key not in self.server.catalogs: + if region.config is not None and region.config.provider(region.name) == "jp": + from .jp import open_catalog + cat = open_catalog(region.config, region.name, bundle_key=self.server.bundle_key) + self.server.catalogs[key] = browse(cat.entries) + self.server.jp_catalogs[key] = cat + return self.server.catalogs[key] catalog = self.server.cache / "catalogs" / region.name / f"catalog_main_{language}.bin" if not catalog.exists(): with urlopen(region.cdn + f"/asset/Android/{catalog.name}", timeout=60) as response: @@ -358,7 +391,13 @@ def do_GET(self): if language not in region.languages: self.send_error(404) return - bundles, files = self.catalog(region, language) + from .gameapi import GameApiError + from .config import ConfigError + try: + bundles, files = self.catalog(region, language) + except (GameApiError, ConfigError, ValueError): + self.send_error(502, "Catalog could not be loaded") + return base = f"/{quote(region.name)}/{quote(language)}/" if len(parts) > 2 and parts[2] == "download": ident = parts[3] if len(parts) == 4 else "" @@ -367,8 +406,17 @@ def do_GET(self): return entry = bundles[int(ident)] name = entry["internal_id"].rsplit("/", 1)[1] - with urlopen(region.cdn + remote_path(entry["internal_id"]), timeout=60) as response: - data = decrypt(response.read(), name, self.server.bundle_key) + try: + cat = getattr(self.server, "jp_catalogs", {}).get((region.name, language)) + if cat is not None: + from .catalog import Bundle + data = cat.fetch(Bundle(int(ident), entry["internal_id"], name, True)).read_bytes() + else: + with urlopen(region.cdn + remote_path(entry["internal_id"]), timeout=60) as response: + data = decrypt(response.read(), name, self.server.bundle_key) + except (GameApiError, ConfigError, ValueError): + self.send_error(502, "Resource could not be downloaded") + return self.send_response(200) self.send_header("Content-Type", "application/octet-stream") self.send_header("Content-Disposition", "attachment; filename*=UTF-8''" + quote(name)) @@ -404,6 +452,7 @@ def serve(regions: list[Region], bundle_key: BundleKey, cache: Path, port: int, server.bundle_key = bundle_key server.cache = Path(cache) server.catalogs = {} + server.jp_catalogs = {} print(f"nnnotes browse: serving on {host}:{port}", flush=True) try: server.serve_forever() diff --git a/src/nnnotes/apkset.py b/src/nnnotes/apkset.py new file mode 100644 index 0000000..1852e98 --- /dev/null +++ b/src/nnnotes/apkset.py @@ -0,0 +1,77 @@ +"""Read a base APK and its splits, an APK directory, or an APKS/XAPK archive. + +The base manifest wins over split manifests. Resource members are resolved across the set; +no merged APK is produced. Nested APKs are spooled so large asset packs need not stay in RAM. +""" +from __future__ import annotations + +import shutil +import tempfile +import zipfile +from contextlib import ExitStack +from pathlib import Path + + +class ApkSet: + def __init__(self, source): + self.source = source + self._stack = ExitStack() + self._members = {} + + def __enter__(self): + try: + self._open() + return self + except BaseException: + self._stack.close() + raise + + def _add(self, source): + archive = self._stack.enter_context(zipfile.ZipFile(source)) + for info in archive.infolist(): + self._members.setdefault(info.filename, (archive, info)) + + def _open(self): + if hasattr(self.source, "read"): + self._add(self.source) + return + path = Path(self.source) + if path.is_dir(): + base = path / "base.apk" + if not base.is_file(): + raise FileNotFoundError("APK directory has no base.apk") + paths = [base] + sorted(p for p in path.glob("*.apk") if p != base) + elif path.suffix.lower() in (".apks", ".xapk"): + outer = self._stack.enter_context(zipfile.ZipFile(path)) + names = [n for n in outer.namelist() if n.lower().endswith(".apk")] + bases = [n for n in names if Path(n).name == "base.apk"] + if len(bases) != 1: + raise ValueError("APK archive must contain exactly one base.apk") + for name in bases + sorted(n for n in names if n not in bases): + tmp = self._stack.enter_context(tempfile.SpooledTemporaryFile(max_size=16 * 1024 * 1024)) + with outer.open(name) as stream: + shutil.copyfileobj(stream, tmp) + tmp.seek(0) + self._add(tmp) + return + else: + paths = [path] + if path.name == "base.apk": + paths += sorted(p for p in path.parent.glob("*.apk") + if p.name.startswith(("split_", "config."))) + for apk in paths: + self._add(apk) + + def namelist(self): + return list(self._members) + + def infolist(self): + return [info for _, info in self._members.values()] + + def read(self, name): + name = name.filename if isinstance(name, zipfile.ZipInfo) else name + archive, info = self._members[name] + return archive.read(info) + + def __exit__(self, *args): + return self._stack.__exit__(*args) diff --git a/src/nnnotes/catalog.py b/src/nnnotes/catalog.py index 82da771..73a9c45 100644 --- a/src/nnnotes/catalog.py +++ b/src/nnnotes/catalog.py @@ -7,15 +7,16 @@ from __future__ import annotations import http.client +import hashlib import sys import threading import urllib.parse import urllib.request -import zipfile from dataclasses import dataclass from pathlib import Path -from .addressables import BundleKey, decrypt, parse, parse_locations, remote_path +from .apkset import ApkSet +from .addressables import REMOTE_PREFIX, BundleKey, decrypt, parse, parse_locations, remote_path from .cache import write_atomic as _write_atomic from .config import ConfigError, apk_missing @@ -46,6 +47,10 @@ def location_kind(internal_id: str) -> str: def file_name(internal_id: str) -> str: """The name of a bundle or raw file location: its path below the platform directory (`Android/`), which for a bundle is its bare file name.""" + if internal_id.startswith(REMOTE_PREFIX): + from .jp import relative_path + path = relative_path(internal_id[len(REMOTE_PREFIX):]) + return path.rsplit("/", 1)[-1] if path.endswith(".bundle") else path rel = remote_path(internal_id) if rel is None: rel = internal_id[len(LOCAL_PREFIX):] if internal_id.startswith(LOCAL_PREFIX) else internal_id @@ -73,18 +78,26 @@ class Catalog: the first time it is needed (a ConfigError it raises is raised naming the file that needed the setting). """ - def __init__(self, catalog_bytes: bytes, cache_dir: Path, *, cdn=None, bundle_key=None, apk: Path | None = None): + def __init__(self, catalog_bytes: bytes, cache_dir: Path, *, cdn=None, bundle_key=None, apk: Path | None = None, + source=None, session=None): self._settings = {"cdn": cdn, "bundle_key": bundle_key} self.cache_dir = Path(cache_dir) self.cache_dir.mkdir(parents=True, exist_ok=True) self.apk = Path(apk) if apk else None + self.source, self.session = source, session self._sources = {"remote": catalog_bytes} self._locations: list[dict] | None = None self._parsed: tuple | None = None if self.apk is not None: - with zipfile.ZipFile(self.apk) as z: + with ApkSet(self.apk) as z: self._sources["apk"] = z.read(APK_CATALOG) + def local_cache_dir(self) -> Path: + """JP embedded files also depend on the APK catalog, independently of the CDN snapshot.""" + if self.source is not None and "apk" in self._sources: + return self.cache_dir / "apk" / hashlib.sha256(self._sources["apk"]).hexdigest() + return self.cache_dir + def _parse(self) -> tuple: """(entries, by offset, by primary key), parsed the first time a lookup needs them (a command that only wants the catalog files, sources(), does not pay for it).""" @@ -241,7 +254,7 @@ def resolve(self, key: str) -> list[Bundle]: # --- fetch ------------------------------------------------------------- def cached(self, b: Bundle) -> Path | None: """The file fetch(b) returns when the bundle is in the cache already, else None.""" - dst = self.cache_dir / "bundles" / b.name + dst = (self.cache_dir if b.remote else self.local_cache_dir()) / "bundles" / b.name return dst if dst.is_file() and dst.stat().st_size > 0 else None def cached_raw(self, e: dict) -> Path | None: @@ -252,31 +265,40 @@ def cached_raw(self, e: dict) -> Path | None: def fetch(self, b: Bundle) -> Path: """Local path to the decrypted bundle (CDN download or APK read).""" - dst = self.cache_dir / "bundles" / b.name + dst = (self.cache_dir if b.remote else self.local_cache_dir()) / "bundles" / b.name dst.parent.mkdir(parents=True, exist_ok=True) if dst.exists() and dst.stat().st_size > 0: return dst if b.remote: url = self._url(b.internal_id) key = self._setting("bundle_key", b.name) # before the download: a missing key fails first - data = download(url) + data = self._download(url) else: if self.apk is None: raise apk_missing(f"bundle {b.name}") rel = b.internal_id[len(LOCAL_PREFIX):].lstrip("/") - with zipfile.ZipFile(self.apk) as z: + with ApkSet(self.apk) as z: data = z.read(APK_AA_DIR + rel) key = self._setting("bundle_key", b.name) if data[:7] != b"UnityFS" else None _write_atomic(dst, _unityfs(data, b.name, key)) return dst def _url(self, internal_id: str) -> str: + if self.source is not None: + return self.source.url(internal_id) name = internal_id.rsplit('/', 1)[-1] cdn = self._setting("cdn", name) if not cdn: raise RuntimeError(f"{name} not cached and no CDN base given") return cdn.rstrip("/") + remote_path(internal_id) + def _download(self, url): + if self.source is not None: + if self.session is None: + raise ConfigError("JP downloads require a configured JP session") + return self.session.get(url, source=self.source) + return download(url) + def fetch_key(self, key: str) -> list[Path]: """Every bundle of the key's closure; APK-local ones only when an APK is set.""" return [self.fetch(b) for b in self.resolve(key) if b.remote or self.apk] @@ -290,21 +312,21 @@ def fetch_raw(self, e: dict) -> Path: dst = self.cache_dir / "raw" / rel.lstrip("/") dst.parent.mkdir(parents=True, exist_ok=True) if not (dst.exists() and dst.stat().st_size > 0): - _write_atomic(dst, download(self._url(iid))) + _write_atomic(dst, self._download(self._url(iid))) return dst def apk_bundle(self, name_contains: str) -> Path: """An APK-local addressable bundle whose file name contains the substring.""" if self.apk is None: raise apk_missing("an APK bundle") - with zipfile.ZipFile(self.apk) as z: + with ApkSet(self.apk) as z: names = [n for n in z.namelist() if n.startswith(APK_AA_DIR) and n.endswith(".bundle") and name_contains in n.rsplit("/", 1)[1]] if len(names) != 1: raise KeyError(f"{name_contains!r}: {len(names)} APK bundles match") name = names[0].rsplit("/", 1)[1] - dst = self.cache_dir / "bundles" / name + dst = self.local_cache_dir() / "bundles" / name if not (dst.exists() and dst.stat().st_size > 0): dst.parent.mkdir(parents=True, exist_ok=True) data = z.read(names[0]) diff --git a/src/nnnotes/catalogdb.py b/src/nnnotes/catalogdb.py index 8ff6709..2436f98 100644 --- a/src/nnnotes/catalogdb.py +++ b/src/nnnotes/catalogdb.py @@ -27,9 +27,9 @@ import re import urllib.error import urllib.request -import zipfile from pathlib import Path +from .apkset import ApkSet from . import contract from .addressables import parse_header, parse_keys, parse_locations, remote_path from .catalog import APK_CATALOG, Catalog, file_name, location_kind @@ -294,9 +294,13 @@ def format_diff(d: dict) -> str: # ---------------------------------------------------------------- the store of versions -def version_id(remote_sha: str, apk_sha: str | None) -> str: +def version_id(remote_sha: str, apk_sha: str | None, source: dict | None = None) -> str: """The id of a catalog version: the key hash of its two catalogs' content ids.""" - return contract.digest({"remote": remote_sha, "apk": apk_sha}) + identity = {"remote": remote_sha, "apk": apk_sha} + if source is not None: + from .jp import Source + identity["source"] = Source.from_dict(source).to_dict() + return contract.digest(identity) def default_label(remote_sha: str, resource_version: str | None = None) -> str: @@ -307,7 +311,7 @@ def default_label(remote_sha: str, resource_version: str | None = None) -> str: def apk_catalog(apk) -> bytes: """The local catalog inside an APK.""" - with zipfile.ZipFile(apk) as z: + with ApkSet(apk) as z: return z.read(APK_CATALOG) @@ -361,12 +365,12 @@ def _put(self, data: bytes) -> dict: def add(self, remote: bytes, apk: bytes | None = None, *, label: str | None = None, region: str | None = None, language: str | None = None, hash_text: str | None = None, resource_version: str | None = None, - apk_version_name: str | None = None) -> dict: + apk_version_name: str | None = None, source: dict | None = None) -> dict: """Import a catalog pair (idempotent: a known pair gets the label and the facts it did not have yet); the version. A label names one version per region and language.""" remote_rec = self._put(bytes(remote)) apk_rec = self._put(bytes(apk)) if apk is not None else None - vid = version_id(remote_rec["sha256"], apk_rec["sha256"] if apk_rec else None) + vid = version_id(remote_rec["sha256"], apk_rec["sha256"] if apk_rec else None, source) versions = self._load() v = next((x for x in versions if x["id"] == vid), None) if v is None: @@ -374,11 +378,14 @@ def add(self, remote: bytes, apk: bytes | None = None, *, label: str | None = No "region": region, "language": language, "remote": remote_rec, "apk": apk_rec, "hash": None, "resourceVersion": None, "apkVersionName": None} versions.append(v) + if source is not None: + v["source"] = dict(source) for k, given in (("region", region), ("language", language), ("hash", hash_text), ("resourceVersion", resource_version), ("apkVersionName", apk_version_name)): if given is not None and v[k] is None: v[k] = given - label = label or default_label(remote_rec["sha256"], resource_version) + label = label or (f"{source['version']}/{source['hash']}/{vid[:12]}" if source else + default_label(remote_rec["sha256"], resource_version)) for other in versions: if other is not v and label in other["labels"] and (other["region"], other["language"]) == ( v["region"], v["language"]): @@ -431,7 +438,7 @@ class IndexStage(Stage): """catalog.index: a catalog pair -> its index. Subjects and inputs come from the fact "catalogs": {subject: [Input "remote", Input "apk" (optional)]} (CatalogDB.inputs). Artifact "#index".""" name = "catalog.index" - version = 1 + version = 2 def subjects(self, env) -> list[str]: return sorted(env.fact("catalogs")) diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index b14ead1..4207a87 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -62,6 +62,7 @@ from .catalog import Catalog from .compress import DEFAULT_ENCODING, ENCODINGS from .config import DEFAULT_FILE, ENV_CONFIG, Config, ConfigError, config_files, find_file, use, user_file +from .gameapi import GameApiError from .jsonio import dumps, write_json from .webaudio import DEFAULT_AUDIO_FORMAT, WEB_AUDIO @@ -86,8 +87,11 @@ # ---------------------------------------------------------------- settings -> data def load_config(args) -> Config: overrides = {k: getattr(args, dest, None) for k, (dest, _) in FLAG_SETTINGS.items()} - return use(Config.load(getattr(args, "config", None), overrides=overrides, - flags={k: flag for k, (_, flag) in FLAG_SETTINGS.items()})) + cfg = Config.load(getattr(args, "config", None), overrides=overrides, + flags={k: flag for k, (_, flag) in FLAG_SETTINGS.items()}) + if cfg.provider() == "jp" and not cfg.has("catalog", "language"): + cfg = cfg.for_region(cfg.region()) + return use(cfg) def bundle_key(cfg: Config) -> BundleKey: @@ -96,6 +100,8 @@ def bundle_key(cfg: Config) -> BundleKey: def _existing(cfg: Config, section: str, key: str, kind: str = "file") -> Path | None: p = cfg.path(section, key) + if key == "apk" and p is not None and p.is_dir(): + return p if p is not None and not (p.is_file() if kind == "file" else p.is_dir()): raise ConfigError(f"setting {section}.{key}: {kind} {p} not found") return p @@ -107,9 +113,15 @@ def open_catalog(cfg: Config, bundles: bool = True, region: str | None = None) - serves them all. `bundles`: bundles will be fetched; else only the catalog is read. The region, its CDN base and the bundle key are read from the settings only when something must be downloaded (every file in the cache: none of them is needed), then a missing one is a ConfigError naming the setting.""" + if region: + cfg = cfg.for_region(region) cache = cfg.require_path("paths", "cache") catbin = _existing(cfg, "paths", "catalog") apk = _existing(cfg, "paths", "apk") + if cfg.provider() == "jp": + from .jp import open_catalog as jp_catalog + return jp_catalog(cfg, cfg.region(), catalog_file=catbin, + bundle_key=(lambda: bundle_key(cfg)) if bundles else None, apk=apk) language = cfg.require("catalog", "language") if catbin is None else None cdn = (lambda: cfg.cdn(region or cfg.region())) if bundles or catbin is None else None key = (lambda: bundle_key(cfg)) if bundles else None @@ -126,8 +138,10 @@ def master_dir(cfg: Config, region: str | None = None) -> Path: return _existing(cfg, section, key, "directory") -def player_data(cfg: Config): +def player_data(cfg: Config, region: str | None = None): from .player import PlayerData + if region: + cfg = cfg.for_region(region) cfg.require_path("paths", "apk") return PlayerData(_existing(cfg, "paths", "apk")) @@ -183,7 +197,7 @@ def cmd_browse(args, cfg): langs = cfg.get_list(f"servers.{r}", "languages") if not langs: raise cfg.missing(f"servers.{r}", "languages") - regions.append(Region(r, cfg.get(f"servers.{r}", "name") or r, cfg.cdn(r), langs)) + regions.append(Region(r, cfg.get(f"servers.{r}", "name") or r, cfg.cdn(r), langs, cfg)) serve(regions, bundle_key(cfg), cfg.require_path("paths", "cache"), args.port, args.host) @@ -368,7 +382,10 @@ def cmd_master_version(args, cfg): v = gameapi.master_version(cfg, region) except gameapi.GameApiError as e: sys.exit(f"nnnotes: {e}") - _print_json({"region": region, "masterVersion": v.version, "resourceVersion": v.resource_version}) + result = {"region": region, "masterVersion": v.version, "resourceVersion": v.resource_version} + if v.resource_hash is not None: + result["resourceHash"] = v.resource_hash + _print_json(result) def cmd_master_download(args, cfg): @@ -376,8 +393,16 @@ def cmd_master_download(args, cfg): region = cfg.region() cdn = cfg.cdn(region) try: - version = gameapi.master_version(cfg, region).version if args.latest else args.version - r = master.download(cdn, version, Path(args.out), workers=args.workers) + if cfg.provider(region) == "jp": + from .jp import Session, master_version + session = Session(cfg, region) + observation = session.observe() + version = observation.version.version if args.latest else master_version(args.version) + r = master.download(observation.cdn, version, Path(args.out), workers=args.workers, + get=lambda url: session.get(url, master=version), strict=True) + else: + version = gameapi.master_version(cfg, region).version if args.latest else args.version + r = master.download(cdn, version, Path(args.out), workers=args.workers) except (gameapi.GameApiError, master.DownloadError) as e: sys.exit(f"nnnotes: {e}") _print_json(r) @@ -590,6 +615,10 @@ def cmd_web(args, cfg): web.check_player(player) regions = (web.site_regions(cfg, args.web_regions, args.all_regions) if args.web_regions or args.all_regions else None) # None: the one [catalog] region + if (stories or models) and regions and len({cfg.provider(r) for r in regions}) > 1: + raise ConfigError("build JP stories/models in a separate site directory from international releases") + if (stories or models) and regions and cfg.provider(regions[0]) == "jp": + cfg = use(cfg.for_region(regions[0])) base = {"region": regions[0]} if regions else {} unknown = web.unknown_pairs(cfg, args.pair, regions) if args.pair else [] if unknown: @@ -967,6 +996,8 @@ def main(argv=None): except ConfigError as e: print(f"nnnotes: {e}", file=sys.stderr) sys.exit(2) + except GameApiError as e: + sys.exit(f"nnnotes: {e}") if __name__ == "__main__": diff --git a/src/nnnotes/cli_assets.py b/src/nnnotes/cli_assets.py index d89d910..11b89f4 100644 --- a/src/nnnotes/cli_assets.py +++ b/src/nnnotes/cli_assets.py @@ -49,6 +49,7 @@ import json import sys import threading +import zipfile from collections import Counter from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor @@ -57,6 +58,7 @@ from pathlib import Path from types import SimpleNamespace +from .apkset import ApkSet from . import contract from .config import ConfigError, describe as describe_setting, usable_cpus, use from .contract import Cost, IncompatibleTask, Input, Task @@ -289,8 +291,22 @@ def catalog(self, vid: str): raise FileNotFoundError(f"catalog version {vid[:12]} is not in the store") remote, apk = db.catalog_bytes(v) cfg, region = self.cfg, v.get("region") - cat = Catalog(remote, self.cache, cdn=lambda: cfg.cdn(region or cfg.region()), - bundle_key=lambda: _bundle_key(cfg), apk=cfg.path("paths", "apk")) + region = region or cfg.get("catalog", "region") + if region: + cfg = cfg.for_region(region) + source, session, cache = None, None, self.cache + if v.get("source") is not None: + from .jp import Source, Session + source = Source.from_dict(v["source"]) + session = Session(cfg, cfg.region()) + cache = source.cache_dir(cache) + cat = Catalog(remote, cache, cdn=lambda: cfg.cdn(cfg.region()), + bundle_key=lambda: _bundle_key(cfg), apk=cfg.path("paths", "apk"), + source=source, session=session) + if source is not None and apk is not None and cat.sources().get("apk") != apk: + raise ConfigError("JP historical catalog needs the APK set it was imported with") + # Replay the APK catalog imported with this version, not today's catalog offsets. + cat._sources = {"remote": remote, **({"apk": apk} if apk is not None else {})} hit = self._catalogs[vid] = (cat, catalogdb.by_id(catalogdb.index(remote, apk))) return hit @@ -305,24 +321,23 @@ def fetch_location(self, vid: str, lid: str) -> Path: if loc["kind"] == "bundle": return cat.fetch(Bundle(0, iid, file_name(iid), remote_path(iid) is not None)) if remote_path(iid) is None: - return self._apk_file(iid) + return self._apk_file(iid, apk=cat.apk, cache=cat.local_cache_dir()) return cat.fetch_raw({"internal_id": iid}) - def _apk_file(self, internal_id: str) -> Path: + def _apk_file(self, internal_id: str, *, apk=None, cache=None) -> Path: """A raw file of the APK as stored, through the cache (raw/, where a CDN file of that path would be); KeyError when the APK does not hold it.""" - import zipfile from .cache import write_atomic from .catalog import APK_AA_DIR if self.cache is None: raise self.cfg.missing("paths", "cache") - apk = self.cfg.path("paths", "apk") + apk = apk or self.cfg.path("paths", "apk") if apk is None: raise self.cfg.missing("paths", "apk") rel = apk_rel(internal_id) - dst = self.cache / "raw" / rel + dst = (cache or self.cache) / "raw" / rel if not (dst.is_file() and dst.stat().st_size > 0): - with zipfile.ZipFile(apk) as z: + with ApkSet(apk) as z: data = z.read(APK_AA_DIR + rel) dst.parent.mkdir(parents=True, exist_ok=True) write_atomic(dst, data) @@ -447,6 +462,9 @@ def __init__(self, args, cfg, common, log=_log): self.calibration = load_calibration(self.root) self.scheduled = calibrated(self.installed.stages, self.calibration) # the stages with calibrated costs self.version, self.remote, self.apk_catalog = self._catalog_version() + if self.version.get("region"): + self.cfg = cfg.for_region(self.version["region"]) + self.apk = self.cfg.path("paths", "apk") self._index = None self._problems: list = [] self._apk_names: set | None = None @@ -473,11 +491,13 @@ def _catalog_version(self): cat = self.common.open_catalog(self.cfg, bundles=False) src = cat.sources() remote, apk = src["remote"], src.get("apk") - vid = catalogdb.version_id(contract.sha256(remote), contract.sha256(apk) if apk is not None else None) + source = cat.source.to_dict() if getattr(cat, "source", None) else None + vid = catalogdb.version_id(contract.sha256(remote), contract.sha256(apk) if apk is not None else None, source) v = next((x for x in self.db.versions() if x["id"] == vid), None) if v is None: v = self.db.add(remote, apk, region=self.cfg.get("catalog", "region"), - language=self.cfg.get("catalog", "language"), apk_version_name=_apk_version(self.apk)) + language=self.cfg.get("catalog", "language"), apk_version_name=_apk_version(self.apk), + source=source, resource_version=source["version"] if source else None) return v, remote, apk def catalogs_fact(self) -> dict: @@ -661,17 +681,24 @@ def problems(self) -> list[dict]: def _cache_rel(self, kind: str, entry: dict, loc: dict) -> str: from .addressables import remote_path + prefix = "" + if self.version.get("source"): + from .jp import Source + prefix = Source.from_dict(self.version["source"]).cache_dir(Path(".")).as_posix() + "/" + if not entry.get("remote", True) and self.version.get("apk"): + prefix += "apk/" + self.version["apk"]["sha256"] + "/" if kind == "bundle": - return f"bundles/{entry['name']}" + return prefix + f"bundles/{entry['name']}" rel = remote_path(loc["internalId"]) if rel is None: # a raw file of the APK (CatalogFetcher._apk_file) rel = apk_rel(loc["internalId"]) - return "raw/" + rel.lstrip("/") + return prefix + "raw/" + rel.lstrip("/") def _inputs(self, kind: str, entries: list[dict], fetch: bool) -> dict: from .catalogdb import by_id locs = by_id(self.index()) memo_kind = "bundles" if kind == "bundle" else "raw" + memo_prefix = self.version["id"] + ":" if self.version.get("source") else "" out, todo = {}, [] for e in entries: loc = locs[e["location"]] @@ -682,7 +709,7 @@ def _inputs(self, kind: str, entries: list[dict], fetch: bool) -> dict: if path is not None and path.is_file() and path.stat().st_size > 0: todo.append((e, locators, path)) continue - known = self.store.named(memo_kind, e["name"]) if e["name"] != e["stable"] else None + known = self.store.named(memo_kind, memo_prefix + e["name"]) if e["name"] != e["stable"] else None if known is not None: # a hashed file name: its recorded identity holds out[e["stable"]] = Input(kind, known[0], known[1], e["name"], tuple(locators)) elif not e["remote"] and self.apk is None: @@ -701,7 +728,7 @@ def one(job): try: if path is None: path = self.fetcher.fetch_location(self.version["id"], e["location"]) - sha, size = self.store.identify(path, memo_kind, e["name"]) + sha, size = self.store.identify(path, memo_kind, memo_prefix + e["name"]) except ConfigError: raise except KeyError as x: # not in the APK @@ -726,9 +753,8 @@ def _in_apk(self, internal_id: str) -> bool: """Whether the APK holds the file of a location (read from its directory, not fetched).""" from .catalog import APK_AA_DIR if self._apk_names is None: - import zipfile try: - with zipfile.ZipFile(self.apk) as z: + with ApkSet(self.apk) as z: self._apk_names = set(z.namelist()) except (OSError, zipfile.BadZipFile): raise ConfigError(f"setting paths.apk: {self.apk} is not a readable APK") from None @@ -1309,7 +1335,7 @@ def _apk_catalog(args, cfg) -> bytes | None: apk = cfg.path("paths", "apk") if apk is None: return None - if not apk.is_file(): + if not apk.exists(): raise ConfigError(f"setting paths.apk: file {apk} not found") return apk_catalog(apk) @@ -1320,10 +1346,14 @@ def cmd_catalogs_import(args, cfg, common): except OSError as e: args.usage(f"{args.file}: {e.strerror or e}") try: + source = None + if cfg.provider() == "jp": + from .jp import read_source + source = read_source(args.file, remote).to_dict() v = _db(args, cfg).add(remote, _apk_catalog(args, cfg), label=args.label, region=cfg.get("catalog", "region"), language=cfg.get("catalog", "language"), resource_version=args.resource_version, - apk_version_name=_apk_version(cfg.path("paths", "apk"))) + apk_version_name=_apk_version(cfg.path("paths", "apk")), source=source) except ValueError as e: args.usage(str(e)) print(_version_line(v)) @@ -1332,6 +1362,16 @@ def cmd_catalogs_import(args, cfg, common): def cmd_catalogs_fetch(args, cfg, common): from . import catalogdb region = cfg.region() + if cfg.provider(region) == "jp": + from .jp import open_catalog + cat = open_catalog(cfg, region, apk=cfg.path("paths", "apk")) + src = cat.sources() + v = _db(args, cfg).add(src["remote"], src.get("apk"), label=args.label, region=region, + language=cfg.get("catalog", "language") or "ja", source=cat.source.to_dict(), + resource_version=cat.source.version, + apk_version_name=_apk_version(cfg.path("paths", "apk"))) + print(_version_line(v)) + return cdn = cfg.cdn(region) language = cfg.require("catalog", "language") try: diff --git a/src/nnnotes/config.py b/src/nnnotes/config.py index 50077bd..c0da951 100644 --- a/src/nnnotes/config.py +++ b/src/nnnotes/config.py @@ -202,6 +202,10 @@ def get_list(self, section: str, key: str) -> list[str]: return list(v) def path(self, section: str, key: str) -> Path | None: + if section == "paths" and key in ("apk", "catalog") and self.origin(section, key) != "flag": + region = self.get("catalog", "region") + if region and self.has(f"servers.{region}", key): + section = f"servers.{region}" v, origin = self._raw(section, key) if v is None: return None @@ -245,6 +249,19 @@ def regions(self) -> list[str]: def region(self) -> str: return self.require("catalog", "region") + def provider(self, region: str | None = None) -> str: + region = region or self.get("catalog", "region") + value = self.get(f"servers.{region}", "provider") or ("jp" if region == "jp" else "international") + if value not in ("jp", "international"): + raise ConfigError(f"setting servers.{region}.provider: must be jp or international") + return value + + def for_region(self, region: str) -> "Config": + overrides = {**self._over, ("catalog", "region"): region} + if self.provider(region) == "jp": + overrides[("catalog", "language")] = "ja" + return Config(self._data, self._base, self.source, self._env, overrides, self._flags) + def cdn(self, region: str) -> str: """CDN base of a region, without a trailing slash.""" return self.require(f"servers.{region}", "cdn").rstrip("/") diff --git a/src/nnnotes/configfile.py b/src/nnnotes/configfile.py index a879f55..2bcb8e2 100644 --- a/src/nnnotes/configfile.py +++ b/src/nnnotes/configfile.py @@ -28,6 +28,7 @@ LINKS = ("auto", "clone", "hard", "copy") FILES = {("paths", "catalog"), ("paths", "apk"), ("paths", "ffmpeg"), ("paths", "vgmstream"), ("paths", "node")} DIRS = {("paths", "master"), ("paths", "player"), (REGION, "master")} +REGION_PATHS = {"apk", "catalog"} CHECK = "nnnotes.config-check/1" PATHS = "nnnotes.config-path/1" PROBLEMS = ("invalid", "not found", "unknown") @@ -59,7 +60,8 @@ def _template_settings() -> list[Setting]: elif (m := _KEYVAL.match(line)) and section: sec = REGION if section == f"servers.{TEMPLATE_REGION}" else section key, value = m.group(2), m.group(3).strip() - kind = "list" if value.startswith("[") else "path" if sec == "paths" or (sec, key) in DIRS else "string" + kind = "list" if value.startswith("[") else "path" if (sec == "paths" or (sec, key) in DIRS + or sec == REGION and key in REGION_PATHS) else "string" out.append(Setting(sec, key, kind, (sec, key) in SECRETS, " ".join(comment))) comment = [] for k in FONT_KEYS: @@ -120,6 +122,8 @@ def validate(setting: Setting, value) -> None: u = urlsplit(value) if u.scheme not in ("https", "http") or not u.netloc: raise ValueError("must be an http(s):// URL") + elif k == (REGION, "provider") and value not in ("jp", "international"): + raise ValueError("must be jp or international") elif k in ((REGION, "api"), ("bootstrap", "api")): from .gameapi import channel_target channel_target(value) @@ -287,7 +291,10 @@ def _status(cfg: Config, s: Setting, section: str) -> tuple[str, str | None]: validate(s, cfg.get_list(section, s.key)) elif s.kind == "path": p = cfg.path(section, s.key) - if (s.section, s.key) in FILES or s.section == FONTS: + if s.key == "apk": + if not p.exists(): + return "not found", "no such APK or directory" + elif (s.section, s.key) in FILES or s.section == FONTS or (s.section, s.key) == (REGION, "catalog"): if not p.is_file(): return "not found", "no such file" elif (s.section, s.key) in DIRS and not p.is_dir(): diff --git a/src/nnnotes/cri.py b/src/nnnotes/cri.py index 0154d35..5926506 100644 --- a/src/nnnotes/cri.py +++ b/src/nnnotes/cri.py @@ -60,7 +60,8 @@ def hca_key(apk) -> int: """crikey.find_key of an APK, read once per process (keyed by the file's path, size and modification time).""" - k = cache.file_id(apk) + path = Path(apk) + k = cache.file_id(path / "base.apk" if path.is_dir() else path) with _keys_lock: if k in _keys: return _keys[k] diff --git a/src/nnnotes/crikey.py b/src/nnnotes/crikey.py index bff86c1..c03157b 100644 --- a/src/nnnotes/crikey.py +++ b/src/nnnotes/crikey.py @@ -13,17 +13,18 @@ import io import re import struct -import zipfile from pathlib import Path +from .apkset import ApkSet + DATA_IN_APK = "assets/bin/Data/data.unity3d" def _load_data(src: Path): import UnityPy # here, not at import: the modules that import this one seldom call it src = Path(src) - if src.suffix.lower() in (".apk", ".zip"): - with zipfile.ZipFile(src) as z: + if src.is_dir() or src.suffix.lower() in (".apk", ".zip", ".apks", ".xapk"): + with ApkSet(src) as z: return UnityPy.load(io.BytesIO(z.read(DATA_IN_APK))) return UnityPy.load(str(src)) diff --git a/src/nnnotes/crilips.py b/src/nnnotes/crilips.py index 15000ff..1516903 100644 --- a/src/nnnotes/crilips.py +++ b/src/nnnotes/crilips.py @@ -24,6 +24,7 @@ from dataclasses import dataclass, field from pathlib import Path +from .apkset import ApkSet from .jsonio import write_json LIB_ENTRY_SUFFIX = "lib/arm64-v8a/libcri_lips_unity.so" @@ -190,7 +191,7 @@ def _scan_const(rodata: bytes, c: Const) -> bytes: def _library_in(apk: Path) -> bytes | None: try: - with zipfile.ZipFile(apk) as z: + with ApkSet(apk) as z: for info in z.infolist(): if info.filename.endswith(LIB_ENTRY_SUFFIX): return z.read(info.filename) diff --git a/src/nnnotes/cristages.py b/src/nnnotes/cristages.py index c627f8c..1f8f1ef 100644 --- a/src/nnnotes/cristages.py +++ b/src/nnnotes/cristages.py @@ -37,10 +37,10 @@ import subprocess import tempfile import threading -import zipfile from collections import defaultdict from pathlib import Path +from .apkset import ApkSet from . import contract from .atoms import impl_id from .contract import Cost, IncompatibleTask, Input @@ -72,14 +72,19 @@ def boot_input(store, apk) -> Input | None: if apk is None: return None apk = Path(apk) - if apk.suffix.lower() not in (".apk", ".zip"): + if not apk.is_dir() and apk.suffix.lower() not in (".apk", ".zip", ".apks", ".xapk"): sha, size = store.identify(apk) return Input("boot", sha, size, apk.name, ({"kind": "file", "path": str(apk.resolve())},)) + if apk.is_dir(): + with ApkSet(apk) as z: + data = z.read(BOOT_IN_APK) + sha = store.put(data) + return Input("boot", sha, len(data), "data.unity3d", ({"kind": "store"},)) apk_sha, _ = store.identify(apk) name = f"boot:{apk_sha}" known = store.named("files", name) if known is None or not store.has(*known): - with zipfile.ZipFile(apk) as z: + with ApkSet(apk) as z: data = z.read(BOOT_IN_APK) known = store.identify(store.path(store.put(data)), "files", name) return Input("boot", known[0], known[1], "data.unity3d", ({"kind": "store"},)) diff --git a/src/nnnotes/deckdata.py b/src/nnnotes/deckdata.py index 4cd6905..180db4b 100644 --- a/src/nnnotes/deckdata.py +++ b/src/nnnotes/deckdata.py @@ -26,6 +26,8 @@ from collections import Counter from dataclasses import dataclass from pathlib import Path + +from .apkset import ApkSet from typing import Callable DECK_FORMAT = "nnnotes.deck-data/1" # the deck model's input format (ournotes-deck data::FORMAT) @@ -188,7 +190,7 @@ def apk_master(apk) -> MasterSource: """The master data files the APK ships (`assets/Master/`: MasterManifest.json + .bin).""" apk = Path(apk) try: - with zipfile.ZipFile(apk) as z: + with ApkSet(apk) as z: raw = z.read(APK_MASTER + MANIFEST) except KeyError: raise DeckDataError(f"{apk}: no {APK_MASTER}{MANIFEST}") from None @@ -197,7 +199,7 @@ def apk_master(apk) -> MasterSource: version, hashes = _manifest(raw, f"{apk} {APK_MASTER}") def read(names): - with zipfile.ZipFile(apk) as z: + with ApkSet(apk) as z: have = set(z.namelist()) missing = [n for n in names if APK_MASTER + n not in have] if missing: @@ -369,7 +371,7 @@ def apk_client(apk) -> dict: """{versionName, versionCode} of an APK's AndroidManifest.xml (None when absent).""" from .player import MANIFEST_IN_APK, manifest_version_code, manifest_version_name try: - with zipfile.ZipFile(apk) as z: + with ApkSet(apk) as z: data = z.read(MANIFEST_IN_APK) if MANIFEST_IN_APK in z.namelist() else None except (zipfile.BadZipFile, OSError): raise DeckDataError(f"{apk}: not a readable APK") from None @@ -381,7 +383,9 @@ def apk_client(apk) -> dict: def catalog_info(cat, store_root=None) -> dict: """{resourceVersion, sha256} of a catalog's remote catalog file (resource_version).""" sha = hashlib.sha256(cat.sources()["remote"]).hexdigest() - return {"resourceVersion": resource_version(store_root, sha), "sha256": sha} + source = getattr(cat, "source", None) + return {"resourceVersion": source.version if source else resource_version(store_root, sha), "sha256": sha, + **({"resourceHash": source.hash} if source else {})} def resource_version(store_root, remote_sha: str) -> str | None: diff --git a/src/nnnotes/gameapi.py b/src/nnnotes/gameapi.py index 45cc300..77b61f7 100644 --- a/src/nnnotes/gameapi.py +++ b/src/nnnotes/gameapi.py @@ -28,6 +28,7 @@ import grpc +from .apkset import ApkSet from .config import Config, ConfigError, describe VERSION_METHOD = "/app.masterdata.MasterdataService/Version" @@ -111,6 +112,7 @@ def strings(buf: bytes, names: dict[int, str]) -> dict[str, str]: class MasterVersion: version: str # master data version resource_version: str + resource_hash: str | None = None @dataclass(frozen=True) @@ -180,7 +182,7 @@ def channel_target(root: str) -> tuple[str, bool]: def call(root: str, method: str, request: bytes, client_version: str, *, timeout: float = TIMEOUT, - attempts: int = ATTEMPTS, setting: str = "the API root") -> bytes: + attempts: int = ATTEMPTS, setting: str = "the API root", response_metadata: dict | None = None) -> bytes: """One unary call with raw bytes; returns the response message's bytes. `setting` names the root in errors. Raises GameApiError.""" target, tls = channel_target(root) @@ -192,7 +194,11 @@ def call(root: str, method: str, request: bytes, client_version: str, *, timeout for i in range(attempts): last = i + 1 == attempts try: - body = stub(request, timeout=timeout, metadata=metadata(client_version)) + body, response = stub.with_call(request, timeout=timeout, metadata=metadata(client_version)) + if response_metadata is not None: + response_metadata.clear() + response_metadata.update(response.initial_metadata() or ()) + response_metadata.update(response.trailing_metadata() or ()) except grpc.RpcError as e: code = e.code() if code in RETRY_STATUS and not last: @@ -256,23 +262,26 @@ def apk_version_name(apk: Path) -> str | None: """`versionName` of the APK's AndroidManifest.xml (None when the APK has none).""" from .player import MANIFEST_IN_APK, manifest_version_name try: - with zipfile.ZipFile(apk) as z: + with ApkSet(apk) as z: data = z.read(MANIFEST_IN_APK) if MANIFEST_IN_APK in z.namelist() else None except (zipfile.BadZipFile, OSError): raise ConfigError(f"setting paths.apk: {apk} is not a readable APK") from None return manifest_version_name(data) if data is not None else None -def client_version(cfg: Config) -> str: +def client_version(cfg: Config, region: str | None = None) -> str: """The `x-client-version` of calls: `[client] version`, else the versionName of `[paths] apk`.""" - v = cfg.get("client", "version") + region = region or cfg.get("catalog", "region") + if region: + cfg = cfg.for_region(region) + v = cfg.get(f"servers.{region}", "client_version") or cfg.get("client", "version") what = "setting client.version" if v is None: apk = cfg.path("paths", "apk") if apk is None: raise ConfigError(f"the game API needs the client version: give it as {describe('client', 'version')}, " f"or give base.apk as {describe('paths', 'apk', '--apk')} to use its versionName") - if not apk.is_file(): + if not apk.exists(): raise ConfigError(f"setting paths.apk: file {apk} not found") v = apk_version_name(apk) if not v: @@ -288,8 +297,11 @@ def client_version(cfg: Config) -> str: def master_version(cfg: Config, region: str, *, timeout: float = TIMEOUT) -> MasterVersion: """The master data version region `region` serves now (its `[servers.] api`).""" section = f"servers.{region}" + if cfg.provider(region) == "jp": + from .jp import Session + return Session(cfg, region).observe(timeout=timeout).version root = api_root(cfg, section) - return fetch_master_version(root, client_version(cfg), timeout=timeout, setting=f"[{section}] api") + return fetch_master_version(root, client_version(cfg, region), timeout=timeout, setting=f"[{section}] api") def server_list(cfg: Config, *, timeout: float = TIMEOUT) -> list[Server]: diff --git a/src/nnnotes/jp.py b/src/nnnotes/jp.py new file mode 100644 index 0000000..842d93f --- /dev/null +++ b/src/nnnotes/jp.py @@ -0,0 +1,294 @@ +"""Japanese release: anonymous Version discovery and snapshot-bound CDN downloads. + +Addresses and client versions come from Config. Credentials remain in a Session, never in +source metadata, cache records, task descriptions or exception messages. +""" +from __future__ import annotations + +import base64 +import hashlib +import http.client +import json +import re +import threading +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from urllib.parse import quote, unquote, urlsplit + +from . import gameapi +from .config import ConfigError + +PLACEHOLDER = "{Fwk.Resource.RemoteAssetDir}/" +CATALOG_LIMIT = 64 * 1024 * 1024 +FILE_LIMIT = 1024 * 1024 * 1024 + + +def numeric_version(value): + if not isinstance(value, str) or not re.fullmatch(r"[0-9]+(?:\.[0-9]+){1,3}", value): + raise ValueError("invalid numeric version") + parts = tuple(map(int, value.split("."))) + if max(parts) > 2147483647: + raise ValueError("numeric version component is too large") + return parts + (0,) * (4 - len(parts)) + + +def master_version(value): + if not isinstance(value, str) or not re.fullmatch(r"[0-9]+(?:\.[0-9]+){1,3}/[0-9a-f]{32}", value): + raise ValueError("invalid JP master version") + return value + + +def select_asset(raw, client): + """Select the largest applicable live minimum; no top-level fallback when live has no match.""" + if not raw: + return None + try: + data = json.loads(raw) + if not isinstance(data, dict): + raise ValueError() + live = data.get("live") + if live: + if not isinstance(live, list): + raise ValueError() + current = numeric_version(client) + candidates = [] + for entry in live: + if not isinstance(entry, dict): + continue + try: + minimum = numeric_version(entry.get("minClientVersion")) + except ValueError: + continue + if minimum <= current: + candidates.append((minimum, entry)) + if not candidates: + return None + data = max(candidates, key=lambda x: x[0])[1] + version, digest = data.get("version"), data.get("Android") + numeric_version(version) + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{32}", digest): + raise ValueError() + return version, digest + except (ValueError, TypeError): + raise gameapi.GameApiError("JP Version: invalid asset metadata") from None + + +def origin(value): + try: + u = urlsplit(value) + if (not u.hostname or u.username or u.password or u.query or u.fragment + or u.path not in ("", "/") or u.scheme not in ("https", "http") + or u.scheme == "http" and u.hostname not in ("localhost", "127.0.0.1", "::1")): + raise ValueError() + port = u.port + host = f"[{u.hostname}]" if ":" in u.hostname else u.hostname + return f"{u.scheme}://{host}" + (f":{port}" if port and port != (443 if u.scheme == "https" else 80) else "") + except (ValueError, TypeError, AttributeError): + raise ConfigError("JP origin must be HTTPS with no path or credentials (HTTP loopback is allowed for tests)") from None + + +def relative_path(value): + decoded = unquote(value) + if (not value or value.startswith("/") or "\\" in decoded or "%" in decoded + or any(ord(c) < 32 for c in decoded) or any(p in ("", ".", "..") for p in decoded.split("/")) + or "?" in value or "#" in value or ":" in value): + raise ValueError("invalid JP resource path") + return quote(decoded, safe="/!$&'()+,;=@[]-._~") + + +@dataclass(frozen=True) +class Source: + provider: str + version: str + hash: str + cdn: str = field(repr=False) + platform: str = "Android" + + def __post_init__(self): + if self.provider != "jp" or self.platform != "Android": + raise ValueError("unsupported JP asset source") + numeric_version(self.version) + if not re.fullmatch(r"[0-9a-f]{32}", self.hash): + raise ValueError("invalid JP asset hash") + if origin(self.cdn) != self.cdn: + raise ValueError("JP source CDN is not a canonical origin") + + @property + def root(self): + return f"{self.cdn}/asset/{self.version}/Android/{self.hash}" + + @property + def catalog_url(self): + return self.root + "/catalog_main.bin" + + def url(self, internal_id): + if not internal_id.startswith(PLACEHOLDER): + raise ValueError("JP resource has no remote asset placeholder") + return self.root + "/" + relative_path(internal_id[len(PLACEHOLDER):]) + + def to_dict(self): + return asdict(self) + + @classmethod + def from_dict(cls, data): + if not isinstance(data, dict) or set(data) != {"provider", "version", "hash", "cdn", "platform"}: + raise ValueError("invalid JP source metadata") + return cls(**data) + + def cache_dir(self, cache): + site = hashlib.sha256(self.cdn.encode()).hexdigest()[:16] + return Path(cache) / "jp" / site / self.version / self.hash + + +@dataclass(frozen=True) +class Observation: + version: gameapi.MasterVersion + source: Source | None + cdn: str = field(repr=False) + authorization: str = field(repr=False) + user_agent: str = field(repr=False) + + +class Session: + def __init__(self, cfg, region): + self.cfg, self.region = cfg.for_region(region), region + self._observation = None + self._created = 0 + self._lock = threading.RLock() + + def observe(self, *, timeout=gameapi.TIMEOUT): + with self._lock: + section = f"servers.{self.region}" + api = gameapi.api_root(self.cfg, section) + origin(api) + allowed = origin(self.cfg.cdn(self.region)) + client = gameapi.client_version(self.cfg, self.region) + headers = {} + body = gameapi.call(api, gameapi.VERSION_METHOD, b"", client, timeout=timeout, + setting=f"[{section}] api", response_metadata=headers) + try: + version = master_version(gameapi.parse_version(body).version) + cdn = origin(headers.get("x-sirius-env", "")) + if cdn != allowed: + raise gameapi.GameApiError("JP Version: CDN differs from the configured CDN origin") + credential = headers.get("x-sirius-cred") + if not isinstance(credential, str) or not credential or len(credential) > 8192: + raise ValueError("missing CDN credential") + asset = select_asset(headers.get("x-asset-version"), client) + except (ValueError, ConfigError): + raise gameapi.GameApiError("JP Version: invalid master version or CDN metadata") from None + source = Source("jp", *asset, cdn) if asset else None + authorization = "Basic " + base64.b64encode(("sirius:" + credential).encode()).decode() + result = Observation(gameapi.MasterVersion(version, asset[0] if asset else "", asset[1] if asset else None), + source, cdn, authorization, "OurNotes/" + client) + self._observation, self._created = result, time.monotonic() + return result + + def _current(self): + with self._lock: + if self._observation is None or time.monotonic() - self._created > 300: + return self.observe() + return self._observation + + def get(self, url, *, source=None, master=None, limit=FILE_LIMIT): + """Download against one immutable source. Auth refresh never changes the requested snapshot.""" + for attempt in range(2): + observation = self._current() + if source is not None and observation.source != source: + raise gameapi.GameApiError("JP assets changed: refresh the catalog before downloading missing resources") + if master is not None and observation.version.version != master: + raise gameapi.GameApiError("JP master changed: start a new master download") + parsed = urlsplit(url) + base = origin(f"{parsed.scheme}://{parsed.netloc}") + prefix = source.root + "/" if source else observation.cdn + "/master/" + master_version(master) + "/" + if base != observation.cdn or not url.startswith(prefix): + raise gameapi.GameApiError("JP download URL is outside the selected source") + relative_path(url[len(prefix):]) + conn_type = http.client.HTTPSConnection if parsed.scheme == "https" else http.client.HTTPConnection + connection = conn_type(parsed.netloc, timeout=120) + try: + connection.request("GET", parsed.path, headers={"Authorization": observation.authorization, + "User-Agent": observation.user_agent}) + response = connection.getresponse() + status = response.status + if status == 200: + length = response.getheader("Content-Length") + if length is not None and not response.chunked: + if not length.isdecimal(): + raise gameapi.GameApiError("JP download has invalid Content-Length") + length = int(length) + if length > limit: + raise gameapi.GameApiError("JP download exceeds the size limit") + else: + length = None + data = response.read(limit + 1) + if len(data) > limit: + raise gameapi.GameApiError("JP download exceeds the size limit") + if length is not None and len(data) != length: + raise gameapi.GameApiError("JP download is truncated") + return data + except (OSError, http.client.HTTPException): + raise gameapi.GameApiError("JP CDN download failed: connection error") from None + finally: + connection.close() + if status in (401, 403) and attempt == 0: + with self._lock: + if self._observation is observation: + self.observe() + continue + raise gameapi.GameApiError(f"JP CDN download failed: HTTP {status}") + raise AssertionError("unreachable") + + +def sidecar(path): + return Path(str(path) + ".source.json") + + +def read_source(path, data): + try: + doc = json.loads(sidecar(path).read_bytes()) + if doc["catalog_sha256"] != hashlib.sha256(data).hexdigest(): + raise ValueError() + return Source.from_dict(doc["source"]) + except (OSError, ValueError, KeyError, TypeError): + raise ConfigError("JP catalog needs its matching .source.json sidecar (written by catalogs fetch)") from None + + +def write_source(path, data, source): + from .cache import write_atomic + doc = {"catalog_sha256": hashlib.sha256(data).hexdigest(), "source": source.to_dict()} + write_atomic(sidecar(path), (json.dumps(doc, sort_keys=True) + "\n").encode()) + + +def open_catalog(cfg, region, *, catalog_file=None, bundle_key=None, apk=None): + from .addressables import parse_header + from .cache import write_atomic + from .catalog import Catalog + session = Session(cfg, region) + if catalog_file is not None: + data = Path(catalog_file).read_bytes() + source = read_source(catalog_file, data) + else: + source = session.observe().source + if source is None: + raise gameapi.GameApiError("JP Version: no Android asset version applies to this client") + path = source.cache_dir(cfg.require_path("paths", "cache")) / "catalog_main.bin" + if path.exists(): + data = path.read_bytes() + if not sidecar(path).exists(): + # Interrupted between the two atomic writes: fetch the same snapshot again. + data = session.get(source.catalog_url, source=source, limit=CATALOG_LIMIT) + parse_header(data) + write_atomic(path, data) + write_source(path, data, source) + elif read_source(path, data) != source: + raise ConfigError("JP cached catalog source differs from the selected version") + else: + data = session.get(source.catalog_url, source=source, limit=CATALOG_LIMIT) + parse_header(data) + path.parent.mkdir(parents=True, exist_ok=True) + write_atomic(path, data) + write_source(path, data, source) + return Catalog(data, source.cache_dir(cfg.require_path("paths", "cache")), bundle_key=bundle_key, + apk=apk, source=source, session=session) diff --git a/src/nnnotes/liveaudio.py b/src/nnnotes/liveaudio.py index c0e08a6..8b0b754 100644 --- a/src/nnnotes/liveaudio.py +++ b/src/nnnotes/liveaudio.py @@ -25,9 +25,9 @@ from __future__ import annotations import struct -import zipfile from pathlib import Path +from .apkset import ApkSet from .acb import commands as _commands, tables as _tables, u16s as _u16s from .catalog import Catalog from .jsonio import write_json @@ -129,7 +129,7 @@ def cmd(table, index): def acf_info(apk: Path) -> dict: """Categories, buses and REACT entries of the game's ACF (`assets/Cri/Sound/Sirius.acf` in the APK).""" - with zipfile.ZipFile(apk) as z: + with ApkSet(apk) as z: top, T = _tables(z.read(ACF_IN_APK)) cat_names = {r["Index"]: r["Name"] for r in T["CategoryNameTable"]} cats = {} diff --git a/src/nnnotes/master.py b/src/nnnotes/master.py index 67e272b..bf412e1 100644 --- a/src/nnnotes/master.py +++ b/src/nnnotes/master.py @@ -18,6 +18,7 @@ import hashlib import json import os +import re import time import urllib.error import urllib.request @@ -218,25 +219,47 @@ def _get(url: str, retries: int = 3, timeout: int = 60) -> bytes: raise AssertionError("unreachable") -def download(cdn: str, version: str, out_dir: Path, workers: int = 16) -> dict: +def download(cdn: str, version: str, out_dir: Path, workers: int = 16, *, get=None, strict=False) -> dict: """Master data `version` from the CDN into / (the manifest and every listed file, SHA-256 checked; files already present with the right hash are kept).""" out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) base = f"{cdn.rstrip('/')}/master/{version}" - manifest_raw = _get(f"{base}/MasterManifest.json") + get = get or _get + manifest_raw = get(f"{base}/MasterManifest.json") manifest = json.loads(manifest_raw.decode("utf-8")) - (out_dir / "MasterManifest.json").write_bytes(manifest_raw) + if strict and (not isinstance(manifest, dict) or manifest.get("version") != version): + raise DownloadError("master manifest version differs from the requested version") files = manifest.get("files", []) + if strict: + if not isinstance(files, list) or not files: + raise DownloadError("JP master manifest has no file list") + seen = set() + for entry in files: + if not isinstance(entry, dict): + raise DownloadError("invalid JP master manifest entry") + name = entry.get("name") + if (not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+\.bin", name) + or name in seen or not re.fullmatch(r"[0-9a-fA-F]{64}", str(entry.get("hash", ""))) + or type(entry.get("size")) is not int or not 64 < entry["size"] <= 128 * 1024 * 1024): + raise DownloadError("invalid JP master manifest entry") + seen.add(name) + else: + # Preserve the original partial-download receipt even if a worker raises. + from .cache import write_atomic + write_atomic(out_dir / "MasterManifest.json", manifest_raw) def one(f: dict): name, sha = f["name"], (f.get("hash") or "").lower() if "/" in name or "\\" in name or name in ("", ".", ".."): return {"file": name, "error": "unexpected file name"} dst = out_dir / name - if sha and dst.is_file() and hashlib.sha256(dst.read_bytes()).hexdigest() == sha: + if (sha and dst.is_file() and (not strict or dst.stat().st_size == f["size"]) + and hashlib.sha256(dst.read_bytes()).hexdigest() == sha): return "kept" - data = _get(f"{base}/{name}") + data = get(f"{base}/{name}") + if strict and (len(data) != f.get("size") or not sha): + return {"file": name, "error": "size or hash missing/different from the manifest"} if sha and hashlib.sha256(data).hexdigest() != sha: return {"file": name, "error": "sha256 differs from the manifest"} tmp = dst.with_name(f"{name}.{os.getpid()}.part") @@ -246,6 +269,9 @@ def one(f: dict): with ThreadPoolExecutor(max(1, workers)) as ex: results = list(ex.map(one, files)) + if not strict or not any(isinstance(r, dict) for r in results): + from .cache import write_atomic + write_atomic(out_dir / "MasterManifest.json", manifest_raw) return {"version": manifest.get("version", version), "files": len(files), "downloaded": results.count("downloaded"), "kept": results.count("kept"), "failed": [r for r in results if isinstance(r, dict)], "out": str(out_dir)} diff --git a/src/nnnotes/musicdata.py b/src/nnnotes/musicdata.py index 24157dd..29fab44 100644 --- a/src/nnnotes/musicdata.py +++ b/src/nnnotes/musicdata.py @@ -496,7 +496,8 @@ def record(sid: int) -> dict: "provenance": { "region": region, "client": {"versionName": client.get("versionName"), "versionCode": client.get("versionCode")}, - "catalog": {"resourceVersion": catalog.get("resourceVersion"), "sha256": catalog.get("sha256")}, + "catalog": {"resourceVersion": catalog.get("resourceVersion"), "sha256": catalog.get("sha256"), + **({"resourceHash": catalog["resourceHash"]} if catalog.get("resourceHash") else {})}, "master": {"source": master_source, "version": master_version, "tables": {t: {"sha256": table_sha[t]} for t in read}}, "exporter": exporter, diff --git a/src/nnnotes/nnnotes.example.toml b/src/nnnotes/nnnotes.example.toml index 90fd280..f0b5c39 100644 --- a/src/nnnotes/nnnotes.example.toml +++ b/src/nnnotes/nnnotes.example.toml @@ -26,6 +26,14 @@ language = "" # One table per region; add more tables ([servers.]) for other regions. [servers.tw] +# protocol: international or jp (empty: jp for the region named jp, international otherwise) +provider = "" +# optional per-region client version; empty: [client] version, else this region's APK versionName +client_version = "" +# optional per-region APK, base.apk with adjacent splits, APKS/XAPK, or directory containing base.apk +apk = "" +# optional per-region catalog file; JP files use a .source.json sidecar written by catalogs fetch +catalog = "" # label of the region in the local catalog browser (`nnnotes browse`) name = "" # CDN base URL of the region: catalogs, bundles, raw audio data and master data are fetched below it @@ -46,7 +54,7 @@ api = "" version = "" [paths] -# catalog .bin file to read instead of downloading catalog_main_.bin into the cache (flag --catalog) +# catalog .bin file to read instead of downloading; JP source metadata is stored in .source.json catalog = "" # cache directory: downloaded catalogs, decrypted bundles, raw CDN files (flag --cache) cache = "" diff --git a/src/nnnotes/player.py b/src/nnnotes/player.py index d04365e..a05781f 100644 --- a/src/nnnotes/player.py +++ b/src/nnnotes/player.py @@ -18,13 +18,13 @@ import json import re import struct -import zipfile from importlib import resources from pathlib import Path import UnityPy from UnityPy.helpers.TypeTreeNode import TypeTreeNode +from .apkset import ApkSet from .config import ConfigError from .unity import DEFAULT_RESOURCES, deref, external_path, is_pptr @@ -149,8 +149,13 @@ def manifest_version_code(data: bytes) -> int | None: class PlayerData: def __init__(self, apk: Path): - with zipfile.ZipFile(apk) as z: + with ApkSet(apk) as z: self.env = UnityPy.load(io.BytesIO(z.read(DATA_IN_APK))) + # Split Unity builds keep resources.assets (materials, shaders and settings) in + # datapack.unity3d; level0 in data.unity3d refers to it by external file id. + datapack = "assets/bin/Data/datapack.unity3d" + if datapack in z.namelist(): + self.env.load_file(io.BytesIO(z.read(datapack)), name="datapack.unity3d") self.defaults = UnityPy.load(io.BytesIO(z.read(DEFAULT_RESOURCES_IN_APK))) self.game_version = (manifest_version_name(z.read(MANIFEST_IN_APK)) if MANIFEST_IN_APK in z.namelist() else None) diff --git a/src/nnnotes/storysite.py b/src/nnnotes/storysite.py index 7251578..2ed2114 100644 --- a/src/nnnotes/storysite.py +++ b/src/nnnotes/storysite.py @@ -658,6 +658,8 @@ def build(out_dir, adv_ids, cfg: Config, player_dir, audio_format: str = "aac", if fonts not in FONT_SOURCES: raise ValueError(f"fonts {fonts!r}: one of {', '.join(FONT_SOURCES)}") require_extra() + if story_languages is None and cfg.provider((regions or [cfg.get("catalog", "region")])[0]) == "jp": + story_languages = ["ja"] langs = check_languages(story_languages) base = languages.check(cfg.require("catalog", "language")) default = base if base in langs else langs[0] diff --git a/src/nnnotes/web.py b/src/nnnotes/web.py index 29bc010..5989669 100644 --- a/src/nnnotes/web.py +++ b/src/nnnotes/web.py @@ -437,7 +437,7 @@ def open_data(cfg: Config, region: str | None = None): """Catalog (fetching from the CDN of `region`), the master dir of `region` and PlayerData from the settings (as the command line opens them; `region` None: [catalog] region).""" from .cli import master_dir, open_catalog, player_data - return open_catalog(cfg, region=region), master_dir(cfg, region), player_data(cfg) + return open_catalog(cfg, region=region), master_dir(cfg, region), player_data(cfg, region) def _lock_fetches(cat, lock) -> None: @@ -723,6 +723,8 @@ def ingest(store: Store, site: Path, music_id: int, difficulty: str, live_dir: P def _worker_init(cfg: Config, lock, job: dict) -> None: + if job.get("region"): + cfg = cfg.for_region(job["region"]) use(cfg) configure_caches(job) cat, master, player = open_data(cfg, job.get("region")) @@ -1511,6 +1513,9 @@ def _build_group(site: Path, tmp_root: Path, pairs, cfg: Config, region: str, pr reads: "ReadSets | None" = None) -> tuple[list, list, int]: """The charts `pairs` of one region group into charts/.json (data of `region`); manifests that exist are skipped unless `force` and gain the group's `regions`. -> (results, skipped ids, workers).""" + cfg = cfg.for_region(region) + if cfg.provider(region) == "jp": + job = {**job, "language": "ja"} by_music: dict[int, list[str]] = {} skipped = [] for music_id, difficulty in pairs: @@ -1637,7 +1642,13 @@ def build(out_dir, pairs, cfg: Config, player_dir: Path, audio_format: str = DEF masters = region_masters(cfg, regions) for m in masters.values(): liveoptions.resolve(live_options, m) - groups = region_groups(masters) + # Equal master tables alone do not imply equal assets or player data across releases. + groups = [] + for group in region_groups(masters): + by_provider = {} + for region in group: + by_provider.setdefault(cfg.provider(region), []).append(region) + groups.extend(by_provider.values()) site = Path(out_dir).resolve() (site / "charts").mkdir(parents=True, exist_ok=True) site_store(site, encoding) diff --git a/src/nnnotes/webmodel.py b/src/nnnotes/webmodel.py index c2922a4..0711506 100644 --- a/src/nnnotes/webmodel.py +++ b/src/nnnotes/webmodel.py @@ -456,7 +456,7 @@ def write_models_index(site: Path) -> tuple[int, set[str]]: def _open(cfg: Config, region: str | None = None): from .cli import open_catalog, player_data - return open_catalog(cfg, region=region), player_data(cfg) + return open_catalog(cfg, region=region), player_data(cfg, region) def _worker_init(cfg: Config, lock, job: dict) -> None: diff --git a/tests/test_configfile.py b/tests/test_configfile.py index 1ab6373..131574f 100644 --- a/tests/test_configfile.py +++ b/tests/test_configfile.py @@ -73,7 +73,8 @@ def test_init_with_values(tmp_path, capsys): data = tomllib.loads(f.read_text(encoding="utf-8")) assert data["bundle"]["key"] == KEY and data["catalog"]["region"] == "en" assert data["servers"] == {"en": {"name": "", "cdn": "https://cdn.test/", "api": "", "languages": ["en", "ja"], - "master": ""}} # the example region table renamed + "master": "", "provider": "", "client_version": "", "apk": "", + "catalog": ""}} # the example region table renamed assert data["paths"]["cache"] == str((tmp_path / "cache").absolute()) assert "# CDN base URL of the region" in f.read_text(encoding="utf-8") # the comments stay if os.name != "nt": diff --git a/tests/test_jp.py b/tests/test_jp.py new file mode 100644 index 0000000..e67e3e6 --- /dev/null +++ b/tests/test_jp.py @@ -0,0 +1,353 @@ +"""JP transport, snapshot identity and split APK regressions; all servers and data are synthetic.""" +import base64 +import gzip +import io +import json +import threading +import zipfile +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from types import SimpleNamespace + +import pytest + +import synth +from test_gameapi import server as grpc_server, version_response +from nnnotes import addressables, catalogdb, cli, configfile, deckdata, gameapi, jp +from nnnotes.apkset import ApkSet +from nnnotes.catalog import Catalog, file_name, location_kind +from nnnotes.cli_assets import CatalogFetcher +from nnnotes.config import Config, ConfigError + +HASH = "a" * 32 +MASTER = "1.0.0.100/" + "b" * 32 +server = grpc_server + + +def cat_bytes(): + return synth.CatalogWriter().build([ + ("Live/MusicScore/0001/0001_00", "Assets/chart.asset", [1]), + ("chart.bundle", jp.PLACEHOLDER + "chart.bundle", []), + ("Cri/Sound/test", jp.PLACEHOLDER + "audio/test.acb", []), + ]) + + +@pytest.fixture +def upstream(server): + state = SimpleNamespace(hash=HASH, credential="synthetic-secret", asset=True, seen=[], status=200, + body=gzip.compress(cat_bytes()), cdn_override=None, content_length=None) + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + state.seen.append((self.path, dict(self.headers))) + expected = "Basic " + base64.b64encode(("sirius:" + state.credential).encode()).decode() + status = state.status if self.headers.get("Authorization") == expected else 401 + self.send_response(status) + if state.content_length is not None: + self.send_header("Content-Length", str(state.content_length)) + if status == 302: + self.send_header("Location", "http://127.0.0.1:1/leak") + self.end_headers() + if status == 200: + self.wfile.write(state.body) + + def log_message(self, *args): + pass + + http = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=http.serve_forever, daemon=True) + thread.start() + state.cdn = f"http://127.0.0.1:{http.server_port}" + + def version(req, ctx): + headers = [("x-sirius-env", state.cdn_override or state.cdn), ("x-sirius-cred", state.credential)] + if state.asset: + headers.append(("x-asset-version", json.dumps({"live": [ + {"minClientVersion": "1.0.4", "version": "1.0.0.300", "Android": state.hash}]}))) + ctx.send_initial_metadata(headers) + return version_response(MASTER) + + server.handlers[gameapi.VERSION_METHOD] = version + state.grpc = server + state.cfg = Config({"catalog": {"region": "jp", "language": "ja"}, + "client": {"version": "1.0.1"}, + "servers": {"jp": {"cdn": state.cdn, "api": server.root, "client_version": "1.0.4"}}}, + environ={}) + try: + yield state + finally: + http.shutdown() + http.server_close() + thread.join() + + +def test_version_headers_and_master_without_asset(upstream): + result = gameapi.master_version(upstream.cfg, "jp") + assert (result.version, result.resource_version, result.resource_hash) == (MASTER, "1.0.0.300", HASH) + assert upstream.grpc.seen[0].metadata["x-client-version"] == "1.0.4" + assert upstream.grpc.seen[0].method == gameapi.VERSION_METHOD + upstream.asset = False + result = gameapi.master_version(upstream.cfg, "jp") + assert result.version == MASTER and result.resource_version == "" and result.resource_hash is None + + +def test_live_selection_numeric_and_no_fallback(): + def entry(client, version): + return {"minClientVersion": client, "version": version, "Android": HASH} + raw = {"version": "9.9", "Android": HASH, + "live": [entry("1.0.10", "1.0.10"), entry("1.0.2", "1.0.2"), entry("8.0", "8.0")]} + assert jp.select_asset(json.dumps(raw), "1.0.12") == ("1.0.10", HASH) + assert jp.select_asset(json.dumps(raw), "1.0.1") is None + assert jp.select_asset(json.dumps({"version": "1.0", "Android": HASH, "live": []}), "1.0") == ("1.0", HASH) + with pytest.raises(gameapi.GameApiError, match="invalid asset"): + jp.select_asset('{"version":"../secret"}', "1.0") + + +def test_authenticated_download_rotation_and_redaction(upstream): + session = jp.Session(upstream.cfg, "jp") + obs = session.observe() + assert upstream.credential not in repr(obs) + upstream.credential = "synthetic-rotated" + assert session.get(obs.source.catalog_url, source=obs.source) == upstream.body + assert len(upstream.seen) == 2 and len(upstream.grpc.seen) == 2 + assert upstream.seen[-1][1]["User-Agent"] == "OurNotes/1.0.4" + assert "Authorization" not in json.dumps(obs.source.to_dict()) + + +def test_truncated_download_is_not_cached(upstream, tmp_path): + session = jp.Session(upstream.cfg, "jp") + source = session.observe().source + upstream.content_length = len(upstream.body) + 5 + cat = Catalog(cat_bytes(), tmp_path, source=source, session=session) + with pytest.raises(gameapi.GameApiError, match="truncated"): + cat.fetch_raw({"internal_id": jp.PLACEHOLDER + "audio/test.acb"}) + assert cat.cached_raw({"internal_id": jp.PLACEHOLDER + "audio/test.acb"}) is None + + +def test_declared_download_limit_is_checked(upstream): + session = jp.Session(upstream.cfg, "jp") + source = session.observe().source + upstream.content_length = 10000 + with pytest.raises(gameapi.GameApiError, match="size limit"): + session.get(source.catalog_url, source=source, limit=1000) + + +@pytest.mark.parametrize("status", [302, 429, 500, 403]) +def test_download_does_not_follow_redirects_or_loop(upstream, status): + session = jp.Session(upstream.cfg, "jp") + obs = session.observe() + upstream.status = status + with pytest.raises(gameapi.GameApiError, match=f"HTTP {status}") as error: + session.get(obs.source.catalog_url, source=obs.source) + expected = 2 if status == 403 else 1 + assert len(upstream.seen) == expected and len(upstream.grpc.seen) == expected + assert upstream.credential not in str(error.value) and upstream.cdn not in str(error.value) + + +def test_unapproved_origin_fails_before_download(upstream): + upstream.cdn_override = "https://elsewhere.invalid" + with pytest.raises(gameapi.GameApiError, match="CDN differs"): + jp.Session(upstream.cfg, "jp").observe() + assert not upstream.seen + + +def test_hash_only_change_does_not_rebind_old_catalog(upstream): + session = jp.Session(upstream.cfg, "jp") + old = session.observe().source + upstream.hash = "c" * 32 + session.observe() + with pytest.raises(gameapi.GameApiError, match="assets changed"): + session.get(old.catalog_url, source=old) + assert not upstream.seen + + +def test_gzip_catalog_classification_and_dependency_closure(tmp_path): + raw = cat_bytes() + compressed = gzip.compress(raw) + assert addressables.parse(raw) == addressables.parse(compressed) + assert addressables.parse_keys(raw) == addressables.parse_keys(compressed) + cat = Catalog(compressed, tmp_path) + assert len(cat.resolve("Live/MusicScore/0001/0001_00")) == 1 + assert cat.resolve("Live/MusicScore/0001/0001_00")[0].remote + assert len(cat.raw_files()) == 1 + assert catalogdb.summary(catalogdb.index(compressed))["remoteBundles"] == 1 + assert file_name(jp.PLACEHOLDER + "nested/chart.bundle") == "chart.bundle" + assert file_name(jp.PLACEHOLDER + "audio/test.acb") == "audio/test.acb" + + +def test_gzip_limit(monkeypatch): + monkeypatch.setattr(addressables, "MAX_CATALOG", 256) + with pytest.raises(ValueError, match="expanded catalog"): + addressables.parse(gzip.compress(b"x" * 257)) + + +@pytest.mark.parametrize("path", ["../secret", "%2e%2e/secret", "%252e%252e/secret", "/secret", "x\\y", "a?b", "a#b"]) +def test_placeholder_rejects_unsafe_paths(path): + with pytest.raises(ValueError): + location_kind(jp.PLACEHOLDER + path) + + +def test_catalog_cache_source_and_offline_replay(upstream, tmp_path, monkeypatch): + cfg = upstream.cfg + cfg._data["paths"] = {"cache": str(tmp_path / "cache")} + cat = cli.open_catalog(cfg, bundles=False) + path = cat.cache_dir / "catalog_main.bin" + assert path.read_bytes() == upstream.body + assert jp.read_source(path, upstream.body) == cat.source + assert cat.cache_dir != tmp_path / "cache" + assert cat.keys("Live/") == ["Live/MusicScore/0001/0001_00"] + record = catalogdb.CatalogDB(tmp_path / "store").add(upstream.body, source=cat.source.to_dict(), region="jp") + assert upstream.credential not in json.dumps(record) + cfg._data["paths"]["catalog"] = str(path) + monkeypatch.setattr(jp.Session, "observe", lambda *a, **k: pytest.fail("offline read made an API call")) + assert cli.open_catalog(cfg, bundles=False).keys() == cat.keys() + path.write_bytes(b"changed") + with pytest.raises(ConfigError, match="matching"): + cli.open_catalog(cfg, bundles=False) + + +def test_catalogdb_source_identity_and_fetcher(upstream, tmp_path): + cfg = upstream.cfg + cfg._data["paths"] = {"cache": str(tmp_path / "cache")} + session = jp.Session(cfg, "jp") + source = session.observe().source + db = catalogdb.CatalogDB(tmp_path / "store") + old = db.add(cat_bytes(), source=source.to_dict(), region="jp") + new_source = {**source.to_dict(), "hash": "c" * 32} + new = db.add(cat_bytes(), source=new_source, region="jp") + assert old["id"] != new["id"] and old["labels"] != new["labels"] + assert old["remote"] == new["remote"] + fetcher = CatalogFetcher(tmp_path / "store", tmp_path / "cache", cfg) + cat, _ = fetcher.catalog(old["id"]) + assert cat.source == source and cat.cache_dir == source.cache_dir(tmp_path / "cache") + + +def test_apk_update_with_same_asset_snapshot_does_not_reuse_embedded_cache(tmp_path): + from nnnotes.catalog import APK_CATALOG, APK_AA_DIR + source = jp.Source("jp", "1.0", HASH, "https://cdn.invalid") + apk = tmp_path / "base.apk" + cache = tmp_path / "cache" + db = catalogdb.CatalogDB(tmp_path / "store") + cfg = Config({"catalog": {"region": "jp"}, "paths": {"apk": str(apk)}}, environ={}) + from nnnotes.cli_assets import Workspace + for revision in (1, 2): + local = synth.CatalogWriter().build([("Local", "Assets/local", [1]), + ("local.bundle", synth.local("local.bundle"), [])], + build_hash=str(revision) * 32) + content = b"UnityFS\0" + bytes([revision]) + apk.write_bytes(zip_bytes({APK_CATALOG: local, APK_AA_DIR + "Android/local.bundle": content})) + cat = Catalog(cat_bytes(), source.cache_dir(cache), source=source, apk=apk) + v = db.add(cat_bytes(), local, source=source.to_dict(), region="jp") + workspace = Workspace.__new__(Workspace) + workspace.version = v + rel = workspace._cache_rel("bundle", {"name": "local.bundle", "remote": False}, {}) + assert cat.fetch(cat.resolve("Local")[0]).read_bytes() == content + assert (cache / rel).read_bytes() == content + fetcher = CatalogFetcher(tmp_path / "store", cache, cfg) + replay, _ = fetcher.catalog(v["id"]) + assert replay.fetch(replay.resolve("Local")[0]).read_bytes() == content + assert len(db.versions()) == 2 + + +def zip_bytes(files): + output = io.BytesIO() + with zipfile.ZipFile(output, "w") as z: + for name, data in files.items(): + z.writestr(name, data) + return output.getvalue() + + +@pytest.mark.parametrize("kind", ["base", "directory", "apks"]) +def test_split_apk_member_routing_and_manifest_precedence(tmp_path, kind): + base = zip_bytes({"AndroidManifest.xml": synth.axml(["versionName", "manifest", "1.0.4"], 2), + "assets/bin/Data/data.unity3d": b"boot"}) + assets = zip_bytes({"AndroidManifest.xml": b"split manifest", "assets/aa/catalog.bin": cat_bytes(), + "assets/Master/MasterManifest.json": b'{"version":"embedded", "files":[]}'}) + (tmp_path / "base.apk").write_bytes(base) + (tmp_path / "split_UnityDataAssetPack.apk").write_bytes(assets) + source = tmp_path / "base.apk" if kind == "base" else tmp_path + if kind == "apks": + source = tmp_path / "game.apks" + source.write_bytes(zip_bytes({"base.apk": base, "split_UnityDataAssetPack.apk": assets})) + with ApkSet(source) as archive: + assert archive.read("assets/bin/Data/data.unity3d") == b"boot" + assert archive.read("assets/aa/catalog.bin") == cat_bytes() + assert gameapi.apk_version_name(source) == "1.0.4" + assert deckdata.apk_master(source).version == "embedded" + assert catalogdb.apk_catalog(source) == cat_bytes() + + +def test_per_region_apk_and_client_settings(tmp_path): + cfg = Config({"catalog": {"region": "tw"}, "paths": {"apk": "global.apk"}, + "servers": {"jp": {"apk": "jp.apks", "client_version": "1.0.4"}}, + "client": {"version": "1.0.1"}}, base=tmp_path, environ={}) + assert cfg.path("paths", "apk") == tmp_path / "global.apk" + assert cfg.for_region("jp").path("paths", "apk") == tmp_path / "jp.apks" + assert gameapi.client_version(cfg, "jp") == "1.0.4" and gameapi.client_version(cfg, "tw") == "1.0.1" + assert configfile.resolve("servers.jp.apk")[0].kind == "path" + assert configfile.resolve("servers.jp.client_version")[0].kind == "string" + + +def test_datapack_is_loaded_in_the_boot_environment(tmp_path, monkeypatch): + from nnnotes import player + boot = SimpleNamespace(objects=[SimpleNamespace(type=SimpleNamespace(name="GraphicsSettings"), + assets_file=SimpleNamespace(unity_version="6000.3.12f1"))]) + loaded = [] + boot.load_file = lambda stream, **kw: loaded.append((stream.read(), kw["name"])) + monkeypatch.setattr(player.UnityPy, "load", lambda stream: boot if stream.read() == b"boot" else object()) + path = tmp_path / "base.apk" + path.write_bytes(zip_bytes({player.DATA_IN_APK: b"boot", player.DEFAULT_RESOURCES_IN_APK: b"defaults", + "assets/bin/Data/datapack.unity3d": b"resources"})) + player.PlayerData(path) + assert loaded == [(b"resources", "datapack.unity3d")] + + +def test_jp_master_manifest_checks_before_writing(tmp_path): + from nnnotes import master + manifest = {"version": MASTER, "files": [{"name": "../secret.bin", "hash": "a" * 64, "size": 128}]} + seen = [] + def get(url): + seen.append(url) + return json.dumps(manifest).encode() + with pytest.raises(master.DownloadError, match="manifest entry"): + master.download("https://cdn.invalid", MASTER, tmp_path, get=get, strict=True) + assert len(seen) == 1 and not (tmp_path / "MasterManifest.json").exists() + + +@pytest.mark.parametrize("files", [None, [], {}, [None]]) +def test_jp_master_rejects_missing_or_malformed_file_lists(tmp_path, files): + from nnnotes import master + with pytest.raises(master.DownloadError, match="manifest"): + master.download("https://cdn.invalid", MASTER, tmp_path, strict=True, + get=lambda _: json.dumps({"version": MASTER, "files": files}).encode()) + assert not (tmp_path / "MasterManifest.json").exists() + + +def test_export_cache_locator_uses_source_namespace(tmp_path): + from nnnotes.cli_assets import Workspace + source = jp.Source("jp", "1.0.0.300", HASH, "https://cdn.invalid") + workspace = Workspace.__new__(Workspace) + workspace.version = {"source": source.to_dict()} + rel = workspace._cache_rel("bundle", {"name": "same.bundle"}, {}) + assert tmp_path / rel == source.cache_dir(tmp_path) / "bundles/same.bundle" + rel = workspace._cache_rel("raw", {}, {"internalId": jp.PLACEHOLDER + "audio/sample.acb"}) + assert tmp_path / rel == source.cache_dir(tmp_path) / "raw/audio/sample.acb" + + +def test_music_data_preserves_asset_hash(tmp_path): + from test_musicdata import CHARTS, bgm, master_dir, KEY, PROV + from nnnotes import musicdata + output = tmp_path / "music.json" + musicdata.export(output, deckdata.master_files(master_dir(tmp_path)), KEY, CHARTS.__getitem__, bgm, + **{**PROV, "catalog": {**PROV["catalog"], "resourceHash": HASH}}) + doc = json.loads(output.read_bytes()) + assert doc["provenance"]["catalog"]["resourceHash"] == HASH + + +def test_directory_hca_key_tracks_replaced_base_apk(tmp_path, monkeypatch): + from nnnotes import cri + apk = tmp_path / "base.apk" + apk.write_bytes(b"first") + monkeypatch.setattr(cri.crikey, "find_key", lambda directory: len((directory / "base.apk").read_bytes())) + assert cri.hca_key(tmp_path) == 5 + apk.write_bytes(b"replacement") + assert cri.hca_key(tmp_path) == 11 diff --git a/tests/test_master.py b/tests/test_master.py index a236ced..9301e77 100644 --- a/tests/test_master.py +++ b/tests/test_master.py @@ -95,6 +95,17 @@ def test_download_rejects_path_names(tmp_path): assert not (tmp_path / "evil.bin").exists() +def test_international_manifest_survives_worker_exception(tmp_path): + manifest = json.dumps({"version": "v", "files": [{"name": "MasterA.bin", "hash": ""}]}).encode() + def get(url): + if url.endswith("MasterManifest.json"): + return manifest + raise master.DownloadError("connection failed") + with pytest.raises(master.DownloadError): + master.download("https://cdn.invalid", "v", tmp_path, get=get) + assert (tmp_path / "MasterManifest.json").read_bytes() == manifest + + # ---------------------------------------------------------------- the stored JSON def test_table_json_is_the_json_rule(): finite = {"_allData": [{"_id": 1, "_name": "テスト\n\"q\"", "_v": 0.1, "_w": -2.5e-12, "_big": 1e300, "_n": None,