From a0edbb0e5520ae99e091e37e4454d0372d5f27cc Mon Sep 17 00:00:00 2001 From: luoxiadesu <249937048+luoxiadesu@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:29:55 +0900 Subject: [PATCH] fix(catalog): select international catalogs by resource version --- docs/commands.md | 13 +- docs/configuration.md | 27 +++- docs/music-data.md | 2 +- src/nnnotes/addressables.py | 11 +- src/nnnotes/catalog.py | 31 +++-- src/nnnotes/catalogdb.py | 4 +- src/nnnotes/cli.py | 12 +- src/nnnotes/cli_assets.py | 22 ++- src/nnnotes/config.py | 25 ++++ src/nnnotes/configfile.py | 4 +- src/nnnotes/deckdata.py | 2 +- src/nnnotes/nnnotes.example.toml | 7 +- tests/test_catalog_versions.py | 227 +++++++++++++++++++++++++++++++ tests/test_configfile.py | 2 +- 14 files changed, 340 insertions(+), 49 deletions(-) create mode 100644 tests/test_catalog_versions.py diff --git a/docs/commands.md b/docs/commands.md index 6da9cbf..c8e9b58 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -4,7 +4,7 @@ nnnotes [global options] [command options] ``` -Global options (before the command): `--config`, `--region`, `--language`, `--catalog`, `--cache`, `--master`, +Global options (before the command): `--config`, `--region`, `--language`, `--catalog`, `--catalog-release`, `--cache`, `--master`, `--apk`, `--ffmpeg`, `--vgmstream`, `--node`, plus `--version` and `--help`. They set the settings described in [configuration.md](configuration.md), which also lists the settings each command needs. `nnnotes --help` prints the options of a command. The asset export commands (`export`, `plan`, @@ -42,16 +42,23 @@ setting's value. Details in [configuration.md](configuration.md#writing-the-conf `pull` and every extractor read bundles through the cache (`[paths] cache`): ``` -/catalog_main_.bin the catalog of the language, the same for every region (downloaded on first - use unless [paths] catalog is set) +/catalog_main_.bin the legacy main catalog (explicit main or no API/version pin) /bundles/ bundles of the dependency closures, decrypted (UnityFS) /raw/ raw CDN files stored as they are (e.g. CRI cue sheet data) /catalogs// catalogs downloaded by `browse` +/international/// + versioned catalog, bundles and raw files; isolated from old main caches ``` Files already in the cache are not downloaded again. When `[paths] apk` is set, the APK's own catalog is merged with the region's and bundles that ship inside the APK are read from it. +For international regions with a configured API, catalog-based commands discover `resource_version` first and +download `catalog__.bin`. `catalogs fetch` labels the bytes with the version used to +select that file, rather than querying a newer label after downloading main. Use `--catalog-release VERSION` or +the [version settings](configuration.md) to pin builds and use cached data offline. An explicit `--catalog FILE` +bypasses discovery. JP keeps its existing Version-driven path. + ## catalog ``` diff --git a/docs/configuration.md b/docs/configuration.md index af7515c..2532938 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -87,19 +87,21 @@ exist when they are given. | `[master] key` | `NNNOTES_MASTER_KEY` | — | Rijndael-256 key of the master data files: 32 bytes as 64 hex digits | | `[master] iv` | `NNNOTES_MASTER_IV` | — | Rijndael-256 CBC initialization vector: 32 bytes as 64 hex digits | | `[catalog] region` | `NNNOTES_CATALOG_REGION` | `--region` | name of one `[servers.]` table | -| `[catalog] language` | `NNNOTES_CATALOG_LANGUAGE` | `--language` | catalog language: the `` of `catalog_main_.bin`: `ja`, `en`, `zh-Hant`, `zh-Hans` or `ko`; also the client language of `live`, `story` and `web` (the text field, fonts and line spacing of their UI) and of the model labels of `web --live2d` | +| `[catalog] language` | `NNNOTES_CATALOG_LANGUAGE` | `--language` | catalog language: the `` of `catalog__.bin`: `ja`, `en`, `zh-Hant`, `zh-Hans` or `ko`; also the client language of `live`, `story` and `web` (the text field, fonts and line spacing of their UI) and of the model labels of `web --live2d` | +| `[catalog] version` | `NNNOTES_CATALOG_VERSION` | `--catalog-release` | optional international resource version; unset: query the configured region API, or use legacy `main` when no API is configured; JP uses its own discovery | | `[servers.] name` | `NNNOTES_SERVERS__NAME` | — | label of the region in `browse` (default: the region name) | | `[servers.] provider` | `NNNOTES_SERVERS__PROVIDER` | — | `international` or `jp`; empty selects `jp` for the region named jp, international otherwise | | `[servers.] client_version` | `NNNOTES_SERVERS__CLIENT_VERSION` | — | per-region API client version; overrides `[client] version`, then falls back to the region's APK versionName | | `[servers.] apk` | `NNNOTES_SERVERS__APK` | `--apk` | per-region APK, APKS/XAPK or directory with base.apk; adjacent splits of base.apk are read automatically; the flag overrides it | | `[servers.] catalog` | `NNNOTES_SERVERS__CATALOG` | `--catalog` | per-region catalog file; JP needs the matching `.source.json`; the flag overrides it | +| `[servers.] catalog_version` | `NNNOTES_SERVERS__CATALOG_VERSION` | `--catalog-release` | per-region international version pin; overrides `[catalog] version`, but not the command-line flag | | `[servers.] cdn` | `NNNOTES_SERVERS__CDN` | — | CDN base URL of the region (a trailing `/` is ignored) | | `[servers.] languages` | `NNNOTES_SERVERS__LANGUAGES` | — | catalog languages `browse` lists: a TOML array of strings; comma-separated in the environment | | `[servers.] api` | `NNNOTES_SERVERS__API` | — | API root of the region: `https://host[:port]` (TLS, port 443 by default), `host[:port]`, or `http://host[:port]` for a plain-text local server; no path | | `[servers.] master` | `NNNOTES_SERVERS__MASTER` | — | decoded master data directory of the region (default: `[paths] master`) | | `[bootstrap] api` | `NNNOTES_BOOTSTRAP_API` | — | API root that serves the server list (`nnnotes servers`), same format | | `[client] version` | `NNNOTES_CLIENT_VERSION` | — | client version sent to the game's API, e.g. `1.0.1`; unset: the `versionName` of `[paths] apk` | -| `[paths] catalog` | `NNNOTES_PATHS_CATALOG` | `--catalog` | a catalog `.bin` file to read instead of downloading `catalog_main_.bin` | +| `[paths] catalog` | `NNNOTES_PATHS_CATALOG` | `--catalog` | a catalog `.bin` file to read instead of discovering/downloading a catalog; explicit files stay usable offline | | `[paths] cache` | `NNNOTES_PATHS_CACHE` | `--cache` | cache directory (created when missing) | | `[paths] store` | `NNNOTES_PATHS_STORE` | `--store` (of `export`, `plan`, `run-stage`, `catalogs`, `store`) | store directory of the asset export ([assets.md](assets.md)); unset: `<[paths] cache>/store` | | `[paths] master` | `NNNOTES_PATHS_MASTER` | `--master` | decoded master data directory, one `.json` per table; the flag overrides `[servers.] master` | @@ -121,9 +123,24 @@ for several of them. The other commands use the one region named by `[catalog] r table per region. Master data per region: a command that reads master data for region `` takes the directory of the `--master` flag, -else `[servers.] master`, else `[paths] master`. The regions serve the same catalog for a language, so the -catalog settings and the cached `catalog_main_.bin` serve every region; bundles are fetched from the CDN of -the region in use. +else `[servers.] master`, else `[paths] master`. International catalog selection uses `resource_version`, not +the master version or a change to the CDN root. `catalog_main` is a separate catalog that may remain old. +When the region has an API root, catalog-based commands query Version before selecting a download; discovery +failure stops rather than treating main as current. Without an API or version pin, the old main/offline behavior +is retained. JP's version/hash directory and authentication remain unchanged. + +For repeatable builds, pin the resource version belonging to your master snapshot: + +```sh +nnnotes --region tw --catalog-release 1.0.0.201 web site --story 10948 +nnnotes --region tw --catalog-release 1.0.0.201 live2d MODEL_ID -o out/model +``` + +Alternatively set `NNNOTES_SERVERS_TW_CATALOG_VERSION` in the build environment; workers inherit this pin. +`--catalog-release` selects a remote resource release. The asset commands' existing `--catalog-version LABEL|SHA` +still selects an already imported store record. To inspect the historical main catalog, explicitly pin `main`. +Versioned caches are isolated by CDN root and resource version; a cached pin needs the CDN setting to identify +its directory but performs no API/catalog download. A missing versioned file never falls back to main. ## What each command needs diff --git a/docs/music-data.md b/docs/music-data.md index 357b6e7..24cec67 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -90,7 +90,7 @@ A **text** is an object with one string per language of `languages` (`{"ja": ... |---|---| | `region` | the region whose master data this is (a configured region name), or `embedded` for the APK's master data | | `client.versionName`, `client.versionCode` | the APK's version name and code (null without `[paths] apk`) | -| `catalog.resourceVersion` | the resource version recorded for the catalog in the catalog store (`nnnotes catalogs fetch` / `import`), null when none is recorded | +| `catalog.resourceVersion` | the resource version used to select the downloaded catalog, or recorded for an explicit file in the catalog store (`nnnotes catalogs fetch` / `import`); null when unknown | | `catalog.sha256` | SHA-256 of the remote catalog file the charts were read with | | `master.source` | `api` (`--master-files`, `--decoded-master`: the region's files) or `embedded` (`--apk-master`) | | `master.version` | the `version` of the master data manifest | diff --git a/src/nnnotes/addressables.py b/src/nnnotes/addressables.py index 19c9dbd..a04fc52 100644 --- a/src/nnnotes/addressables.py +++ b/src/nnnotes/addressables.py @@ -362,13 +362,10 @@ def catalog(self, region: Region, language: str): self.server.catalogs[key] = browse(cat.entries) self.server.jp_catalogs[key] = cat return self.server.catalogs[key] - catalog = self.server.cache / "catalogs" / region.name / f"catalog_main_{language}.bin" - if not catalog.exists(): - with urlopen(region.cdn + f"/asset/Android/{catalog.name}", timeout=60) as response: - raw = response.read() - catalog.parent.mkdir(parents=True, exist_ok=True) - catalog.write_bytes(raw) - self.server.catalogs[key] = browse(parse(catalog.read_bytes())) + from .catalog import Catalog + version = region.config.catalog_version(region.name) if region.config is not None else "main" + cat = Catalog.load(language, self.server.cache / "catalogs" / region.name, cdn=region.cdn, version=version) + self.server.catalogs[key] = browse(cat.entries) return self.server.catalogs[key] def do_GET(self): diff --git a/src/nnnotes/catalog.py b/src/nnnotes/catalog.py index 55e4c21..69877be 100644 --- a/src/nnnotes/catalog.py +++ b/src/nnnotes/catalog.py @@ -18,7 +18,7 @@ from .apkset import ApkSet from .addressables import REMOTE_PREFIX, BundleKey, decrypt, parse, parse_locations, remote_path from .cache import write_atomic as _write_atomic -from .config import ConfigError, apk_missing +from .config import ConfigError, apk_missing, check_catalog_version LOCAL_PREFIX = "{UnityEngine.AddressableAssets.Addressables.RuntimePath}" APK_AA_DIR = "assets/aa/" # + "Android/" @@ -80,12 +80,13 @@ class Catalog: """ def __init__(self, catalog_bytes: bytes, cache_dir: Path, *, cdn=None, bundle_key=None, apk: Path | None = None, - source=None, session=None, apk_catalog: bytes | None = None): + source=None, session=None, apk_catalog: bytes | None = None, resource_version: str | None = None): self._settings = {"cdn": cdn, "bundle_key": bundle_key} self.cache_dir = Path(cache_dir) self.cache_dir.mkdir(parents=True, exist_ok=True) self.apk = Path(apk) if apk else None self.source, self.session = source, session + self.resource_version = resource_version self._sources = {"remote": catalog_bytes} self._locations: list[dict] | None = None self._parsed: tuple | None = None @@ -142,18 +143,28 @@ def _by_key(self) -> dict[str, list[dict]]: # --- construction ------------------------------------------------------ @classmethod def load(cls, language: str, cache_dir: Path, *, cdn=None, bundle_key=None, - apk: Path | None = None) -> "Catalog": + apk: Path | None = None, version: str = "main") -> "Catalog": """The remote catalog of `language` from the cache, downloaded from the CDN on first use (`cdn` / - `bundle_key`: as for Catalog).""" + `bundle_key`: as for Catalog). A versioned catalog is isolated by CDN root and version; main retains + its legacy offline cache path. Missing versioned files never fall back to main.""" cache_dir = Path(cache_dir) - cat = cls.cache_file(language, cache_dir) + filename = cls.cache_file(language, Path("."), version=version).name + if version != "main": + cdn = _setting(cdn, filename) + if not cdn: + raise ConfigError("a versioned catalog needs its CDN root to identify the cache") + cache_dir = cache_dir / "international" / hashlib.sha256(cdn.rstrip("/").encode()).hexdigest() / version + cat = cache_dir / filename if not cat.exists(): cdn = _setting(cdn, cat.name) if not cdn: raise FileNotFoundError(f"{cat} not cached and no CDN base given") cat.parent.mkdir(parents=True, exist_ok=True) - _write_atomic(cat, download(cdn.rstrip("/") + f"/asset/Android/{cat.name}")) - return cls(cat.read_bytes(), cache_dir, cdn=cdn, bundle_key=bundle_key, apk=apk) + data = download(cdn.rstrip("/") + f"/asset/Android/{cat.name}") + parse(data) # Never install a corrupt/error response as a reusable catalog. + _write_atomic(cat, data) + return cls(cat.read_bytes(), cache_dir, cdn=cdn, bundle_key=bundle_key, apk=apk, + resource_version=version if version != "main" else None) def _setting(self, name: str, needed_by: str): """The `cdn` / `bundle_key` given to the catalog, its function called (once) now that `needed_by` needs it.""" @@ -167,9 +178,11 @@ def cdn(self) -> str | None: return None if callable(v) or not v else v.rstrip("/") @staticmethod - def cache_file(language: str, cache_dir: Path) -> Path: + def cache_file(language: str, cache_dir: Path, *, version: str = "main") -> Path: """Where the remote catalog of `language` is cached.""" - return Path(cache_dir) / f"catalog_main_{language}.bin" + check_catalog_version(version) + check_catalog_version(language) + return Path(cache_dir) / f"catalog_{version}_{language}.bin" # --- lookup ------------------------------------------------------------ def keys(self, prefix: str = "") -> list[str]: diff --git a/src/nnnotes/catalogdb.py b/src/nnnotes/catalogdb.py index 2436f98..f4c7e38 100644 --- a/src/nnnotes/catalogdb.py +++ b/src/nnnotes/catalogdb.py @@ -315,9 +315,9 @@ def apk_catalog(apk) -> bytes: return z.read(APK_CATALOG) -def fetch(cdn: str, language: str, timeout: float = 120) -> tuple[bytes, str | None]: +def fetch(cdn: str, language: str, timeout: float = 120, *, version: str = "main") -> tuple[bytes, str | None]: """The remote catalog of `language` from a CDN base, and the text of its `.hash` file (None when not served).""" - base = cdn.rstrip("/") + "/asset/Android/" + Catalog.cache_file(language, Path(".")).name + base = cdn.rstrip("/") + "/asset/Android/" + Catalog.cache_file(language, Path("."), version=version).name with urllib.request.urlopen(base, timeout=timeout) as r: data = r.read() try: diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index 1de3170..5bc0f2a 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -72,6 +72,7 @@ FLAG_SETTINGS = { ("catalog", "region"): ("region", "--region"), ("catalog", "language"): ("language", "--language"), + ("catalog", "version"): ("catalog_release", "--catalog-release"), ("paths", "catalog"): ("catalog", "--catalog"), ("paths", "cache"): ("cache", "--cache"), ("paths", "master"): ("master", "--master"), @@ -111,10 +112,10 @@ def _existing(cfg: Config, section: str, key: str, kind: str = "file") -> Path | def open_catalog(cfg: Config, bundles: bool = True, region: str | None = None) -> Catalog: """The catalog of [catalog] language (merged with the APK's when [paths] apk is set), fetching from the CDN of - `region` (default: [catalog] region); the regions serve the same catalog for a language, so one cached file - serves them all. `bundles`: bundles will be fetched; else only the catalog is read. The region, its CDN base - and the bundle key are read from the settings only when something must be downloaded (every file in the cache: - none of them is needed), then a missing one is a ConfigError naming the setting.""" + `region` (default: [catalog] region). International versioned catalogs are isolated by CDN root and resource + version. `bundles`: bundles will be fetched; else only the catalog is read. Explicit files bypass discovery. + Version pins bypass the API but need the CDN root to identify the cache; legacy main keeps lazy CDN lookup. + The bundle key is only read for a missing encrypted bundle.""" if region: cfg = cfg.for_region(region) cache = cfg.require_path("paths", "cache") @@ -129,7 +130,7 @@ def open_catalog(cfg: Config, bundles: bool = True, region: str | None = None) - key = (lambda: bundle_key(cfg)) if bundles else None if catbin is not None: return Catalog(catbin.read_bytes(), cache, cdn=cdn, bundle_key=key, apk=apk) - return Catalog.load(language, cache, cdn=cdn, bundle_key=key, apk=apk) + return Catalog.load(language, cache, cdn=cdn, bundle_key=key, apk=apk, version=cfg.catalog_version()) def master_dir(cfg: Config, region: str | None = None) -> Path: @@ -771,6 +772,7 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--language", help="catalog and client language: ja, en, zh-Hant, zh-Hans or ko " "([catalog] language)") p.add_argument("--catalog", help="catalog .bin file ([paths] catalog; else downloaded into the cache)") + p.add_argument("--catalog-release", help="pin an international resource version ([catalog] version; else API discovery when configured, otherwise main)") p.add_argument("--cache", help="cache directory ([paths] cache)") p.add_argument("--master", help="decoded master data directory ([paths] master)") p.add_argument("--apk", help="base.apk ([paths] apk)") diff --git a/src/nnnotes/cli_assets.py b/src/nnnotes/cli_assets.py index 4b66606..b027ce4 100644 --- a/src/nnnotes/cli_assets.py +++ b/src/nnnotes/cli_assets.py @@ -492,7 +492,7 @@ def _catalog_version(self): if v is None: v = self.db.add(remote, apk, region=self.cfg.get("catalog", "region"), language=self.cfg.get("catalog", "language"), apk_version_name=_apk_version(self.apk), - source=source, resource_version=source["version"] if source else None) + source=source, resource_version=source["version"] if source else getattr(cat, "resource_version", None)) return v, remote, apk def catalogs_fact(self) -> dict: @@ -1356,6 +1356,7 @@ def cmd_catalogs_import(args, cfg, common): def cmd_catalogs_fetch(args, cfg, common): from . import catalogdb + from .catalog import Catalog region = cfg.region() if cfg.provider(region) == "jp": from .jp import open_catalog @@ -1369,19 +1370,14 @@ def cmd_catalogs_fetch(args, cfg, common): return cdn = cfg.cdn(region) language = cfg.require("catalog", "language") + version = cfg.catalog_version(region) + filename = Catalog.cache_file(language, Path("."), version=version).name try: - data, hash_text = catalogdb.fetch(cdn, language) + data, hash_text = catalogdb.fetch(cdn, language, version=version) except OSError as e: # URLError, HTTPError, timeouts (the message has no URL) - sys.exit(f"nnnotes: catalog_main_{language}.bin could not be fetched: {type(e).__name__}: " + sys.exit(f"nnnotes: {filename} could not be fetched: {type(e).__name__}: " f"{getattr(e, 'reason', None) or getattr(e, 'code', None) or ''}".rstrip(": ")) - resource = None - if cfg.has(f"servers.{region}", "api"): - from . import gameapi - try: - resource = gameapi.master_version(cfg, region).resource_version - except gameapi.GameApiError as e: - print(f"nnnotes: no resource version (the label defaults to the catalog's sha256): {e}", - file=sys.stderr) + resource = version if version != "main" else None try: v = _db(args, cfg).add(data, _apk_catalog(args, cfg), label=args.label, region=region, language=language, hash_text=hash_text, resource_version=resource, @@ -1526,7 +1522,7 @@ def bind(fn): _store_arg(m) m.set_defaults(func=bind(cmd_catalogs_list), usage=m.error) m = csub.add_parser("import", help="import a catalog file (with the APK's catalog when [paths] apk is set)") - m.add_argument("file", help="catalog_main_.bin") + m.add_argument("file", help="catalog__.bin") m.add_argument("--label", help="label of the version (default: its resource version, else the sha256 prefix)") m.add_argument("--apk-catalog", help="the APK's catalog.bin (default: read from [paths] apk)") m.add_argument("--resource-version", help="the game's resource version of this catalog") @@ -1556,7 +1552,7 @@ def main(argv=None) -> None: """The asset commands alone, with the global flags of the command line (tests; `nnnotes` has them too).""" from .cli import load_config p = argparse.ArgumentParser(prog="nnnotes") - for flag in ("--config", "--region", "--language", "--catalog", "--cache", "--master", "--apk", "--ffmpeg", + for flag in ("--config", "--region", "--language", "--catalog", "--catalog-release", "--cache", "--master", "--apk", "--ffmpeg", "--vgmstream", "--node"): p.add_argument(flag) sub = p.add_subparsers(dest="cmd", required=True, metavar="") diff --git a/src/nnnotes/config.py b/src/nnnotes/config.py index c0da951..81c3bb0 100644 --- a/src/nnnotes/config.py +++ b/src/nnnotes/config.py @@ -31,6 +31,13 @@ class ConfigError(Exception): """A setting is missing or malformed (the message names the setting, never its value).""" +def check_catalog_version(value: str) -> str: + """One safe international catalog filename component, including the legacy main selector.""" + if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}", value) or ".." in value: + raise ValueError("must be a single catalog version without path separators") + return value + + def user_file(environ: dict[str, str] | None = None) -> Path | None: """The per-user config file: `%APPDATA%\\nnnotes\\nnnotes.toml` on Windows, else `$XDG_CONFIG_HOME/nnnotes/nnnotes.toml` (`~/.config` when XDG_CONFIG_HOME is unset); None when the environment @@ -266,6 +273,24 @@ def cdn(self, region: str) -> str: """CDN base of a region, without a trailing slash.""" return self.require(f"servers.{region}", "cdn").rstrip("/") + def catalog_version(self, region: str | None = None) -> str: + """International selector: flag, regional/global pin, API resource_version, else legacy main.""" + region = region or self.get("catalog", "region") + section, key = "catalog", "version" + if region and self.origin(section, key) != "flag" and self.has(f"servers.{region}", "catalog_version"): + section, key = f"servers.{region}", "catalog_version" + value = self.get(section, key) + if value is None and region and self.has(f"servers.{region}", "api"): + from .gameapi import master_version + value = master_version(self, region).resource_version + section, key = f"servers.{region}", "api" + elif value is None: + value = "main" + try: + return check_catalog_version(value) + except ValueError as error: + raise ConfigError(f"setting {section}.{key}: {error}") from None + def master(self, region: str | None) -> tuple[str, str]: """The setting that names the decoded master data of `region`: the --master flag, else `[servers.] master`, else `[paths] master` -> (section, key).""" diff --git a/src/nnnotes/configfile.py b/src/nnnotes/configfile.py index 2bcb8e2..ed5cdbd 100644 --- a/src/nnnotes/configfile.py +++ b/src/nnnotes/configfile.py @@ -17,7 +17,7 @@ from urllib.parse import urlsplit from . import languages -from .config import Config, ConfigError, env_name, template +from .config import Config, ConfigError, check_catalog_version, env_name, template SECRETS = {("bundle", "key"), ("bundle", "nonce_seed"), ("master", "key"), ("master", "iv")} HEX_BYTES = {("bundle", "key"): 16, ("bundle", "nonce_seed"): None, ("master", "key"): 32, ("master", "iv"): 32} @@ -124,6 +124,8 @@ def validate(setting: Setting, value) -> None: raise ValueError("must be an http(s):// URL") elif k == (REGION, "provider") and value not in ("jp", "international"): raise ValueError("must be jp or international") + elif k in (("catalog", "version"), (REGION, "catalog_version")): + check_catalog_version(value) elif k in ((REGION, "api"), ("bootstrap", "api")): from .gameapi import channel_target channel_target(value) diff --git a/src/nnnotes/deckdata.py b/src/nnnotes/deckdata.py index 180db4b..fe7f731 100644 --- a/src/nnnotes/deckdata.py +++ b/src/nnnotes/deckdata.py @@ -384,7 +384,7 @@ def catalog_info(cat, store_root=None) -> dict: """{resourceVersion, sha256} of a catalog's remote catalog file (resource_version).""" sha = hashlib.sha256(cat.sources()["remote"]).hexdigest() source = getattr(cat, "source", None) - return {"resourceVersion": source.version if source else resource_version(store_root, sha), "sha256": sha, + return {"resourceVersion": source.version if source else (getattr(cat, "resource_version", None) or resource_version(store_root, sha)), "sha256": sha, **({"resourceHash": source.hash} if source else {})} diff --git a/src/nnnotes/nnnotes.example.toml b/src/nnnotes/nnnotes.example.toml index f0b5c39..df5e951 100644 --- a/src/nnnotes/nnnotes.example.toml +++ b/src/nnnotes/nnnotes.example.toml @@ -20,9 +20,12 @@ iv = "" [catalog] # region the commands use: the name of one [servers.] table below (flag --region) region = "" -# catalog language: catalog_main_.bin, one of ja, en, zh-Hant, zh-Hans, ko; also the client language of the +# catalog language: catalog__.bin, one of ja, en, zh-Hant, zh-Hans, ko; also the client language of the # live start canvas, the story UI and the web model labels (flag --language) language = "" +# optional international resource version (flag --catalog-release); empty: API resource_version when configured, +# otherwise legacy main. Set main explicitly to inspect the old catalog; JP uses its own version/hash discovery. +version = "" # One table per region; add more tables ([servers.]) for other regions. [servers.tw] @@ -34,6 +37,8 @@ client_version = "" apk = "" # optional per-region catalog file; JP files use a .source.json sidecar written by catalogs fetch catalog = "" +# optional per-region international catalog version; overrides [catalog] version, but not --catalog-release +catalog_version = "" # label of the region in the local catalog browser (`nnnotes browse`) name = "" # CDN base URL of the region: catalogs, bundles, raw audio data and master data are fetched below it diff --git a/tests/test_catalog_versions.py b/tests/test_catalog_versions.py new file mode 100644 index 0000000..f54fa8e --- /dev/null +++ b/tests/test_catalog_versions.py @@ -0,0 +1,227 @@ +"""International resource versions select the catalog bytes, not just an import label.""" +import hashlib +import os +import subprocess +import sys +import urllib.error +from types import SimpleNamespace + +import pytest + +import synth +from nnnotes import addressables, catalog, catalogdb, cli, configfile, deckdata, gameapi +from nnnotes.config import Config, ConfigError + +MODEL = "Character/Live2D/test/new/model/new" +EPISODE = "Adv/Episode/event_new/event_new" + + +def binary(*keys): + return synth.CatalogWriter().build([(key, "Assets/" + key, []) for key in keys]) + + +def config(tmp_path, *, api=True, version=None, region="tw"): + return Config({"catalog": {"region": region, "language": "en", "version": version}, + "servers": {region: {"cdn": "https://cdn.example/prod/" + region, + **({"api": "https://api.example"} if api else {})}}, + "paths": {"cache": str(tmp_path / "cache")}}, environ={}) + + +def test_auto_discovery_ignores_stale_main_and_advances_resource_version(tmp_path, monkeypatch): + from nnnotes.webmodel import catalog_models + + cfg = config(tmp_path) + cache = cfg.require_path("paths", "cache") + cache.mkdir() + catalog.Catalog.cache_file("en", cache).write_bytes(binary("Old")) + publication = gameapi.MasterVersion("master-one", "1.0.0.201") + seen = [] + + def version(*args): + seen.append("version") + return publication + + def download(url): + seen.append(url) + assert url.endswith(f"catalog_{publication.resource_version}_en.bin") + return binary(MODEL, EPISODE, publication.version) + + monkeypatch.setattr(gameapi, "master_version", version) + monkeypatch.setattr(catalog, "download", download) + first = cli.open_catalog(cfg, bundles=False) + assert first.has(EPISODE) and catalog_models(first) == {"new": MODEL} + assert first.resource_version == "1.0.0.201" + assert deckdata.catalog_info(first)["resourceVersion"] == "1.0.0.201" + assert seen[0] == "version" and len(seen) == 2 + # An unchanged version reuses its catalog; a new version gets new bytes. + assert cli.open_catalog(cfg, bundles=False).has("master-one") + assert len(seen) == 3 + publication = gameapi.MasterVersion("master-two", "1.0.0.202") + second = cli.open_catalog(cfg, bundles=False) + assert second.has("master-two") and not second.has("master-one") + assert first.cache_dir != second.cache_dir + assert all("catalog_main" not in request for request in seen) + + +def test_explicit_version_cache_is_offline_and_cdn_scoped(tmp_path, monkeypatch): + cfg = config(tmp_path, version="1.0.0.201") + monkeypatch.setattr(gameapi, "master_version", lambda *args: pytest.fail("pin called Version")) + monkeypatch.setattr(catalog, "download", lambda url: binary("First")) + first = cli.open_catalog(cfg, bundles=False) + monkeypatch.setattr(catalog, "download", lambda url: pytest.fail("cached pin downloaded")) + assert cli.open_catalog(cfg, bundles=False).has("First") + monkeypatch.setattr(catalog, "download", lambda url: binary("Other region")) + other = cli.open_catalog(config(tmp_path, version="1.0.0.201", region="en"), bundles=False) + assert other.has("Other region") and first.cache_dir != other.cache_dir + + +def test_explicit_catalog_file_stays_offline(tmp_path, monkeypatch): + path = tmp_path / "saved.bin" + path.write_bytes(binary("Offline")) + cfg = Config({"catalog": {"region": "tw", "version": "1.0.0.201"}, + "servers": {"tw": {"api": "https://api.example", "catalog": str(path)}}, + "paths": {"cache": str(tmp_path / "cache")}}, environ={}) + monkeypatch.setattr(gameapi, "master_version", lambda *args: pytest.fail("explicit file called Version")) + assert cli.open_catalog(cfg, bundles=False).has("Offline") + + +def test_missing_versioned_catalog_does_not_fall_back_to_main(tmp_path, monkeypatch): + cfg = config(tmp_path, version="1.0.0.201") + cache = cfg.require_path("paths", "cache") + cache.mkdir() + catalog.Catalog.cache_file("en", cache).write_bytes(binary("Old")) + urls = [] + + def missing(url): + urls.append(url) + raise urllib.error.HTTPError(url, 404, "not found", None, None) + + monkeypatch.setattr(catalog, "download", missing) + with pytest.raises(urllib.error.HTTPError): + cli.open_catalog(cfg, bundles=False) + assert len(urls) == 1 and "catalog_1.0.0.201_en.bin" in urls[0] + assert not list((cache / "international").rglob("*.bin")) + + +def test_failed_discovery_does_not_claim_main_is_current(tmp_path, monkeypatch): + def fail(*args): + raise gameapi.GameApiError("unavailable") + + monkeypatch.setattr(gameapi, "master_version", fail) + monkeypatch.setattr(catalog, "download", lambda url: pytest.fail("fallback download")) + with pytest.raises(gameapi.GameApiError, match="unavailable"): + cli.open_catalog(config(tmp_path), bundles=False) + + +def test_configured_main_never_queries_api_or_invents_resource_version(tmp_path, monkeypatch): + cfg = config(tmp_path, version="main") + monkeypatch.setattr(gameapi, "master_version", lambda *args: pytest.fail("explicit main queried API")) + urls = [] + monkeypatch.setattr(catalog, "download", lambda url: urls.append(url) or binary("Old")) + cat = cli.open_catalog(cfg, bundles=False) + assert cat.has("Old") and cat.resource_version is None + assert urls[0].endswith("catalog_main_en.bin") + + +def test_version_precedence(tmp_path, monkeypatch): + monkeypatch.setattr(gameapi, "master_version", lambda *args: pytest.fail("pin queried API")) + data = {"catalog": {"region": "tw", "version": "1.0.0.1"}, + "servers": {"tw": {"catalog_version": "1.0.0.2"}}} + assert Config(data, environ={}).catalog_version() == "1.0.0.2" + cfg = Config(data, environ={}, overrides={("catalog", "version"): "1.0.0.3"}) + assert cfg.catalog_version() == "1.0.0.3" + assert config(tmp_path, api=False).catalog_version() == "main" + + +def test_build_workers_inherit_the_version_pin(tmp_path, monkeypatch): + monkeypatch.setenv("NNNOTES_CATALOG_REGION", "tw") + monkeypatch.setenv("NNNOTES_CATALOG_LANGUAGE", "en") + monkeypatch.setenv("NNNOTES_CATALOG_VERSION", "1.0.0.100") + monkeypatch.setenv("NNNOTES_SERVERS_TW_CATALOG_VERSION", "1.0.0.201") + monkeypatch.setenv("NNNOTES_SERVERS_TW_CDN", "https://cdn.example/prod/tw") + monkeypatch.setenv("NNNOTES_SERVERS_TW_API", "https://unavailable.example") + monkeypatch.setenv("NNNOTES_PATHS_CACHE", str(tmp_path / "cache")) + monkeypatch.setattr(catalog, "download", lambda url: binary(MODEL, EPISODE)) + assert cli.open_catalog(Config.load(), bundles=False).has(MODEL) + command = [sys.executable, "-c", "from nnnotes.config import Config; from nnnotes.cli import open_catalog; " + "from nnnotes.webmodel import catalog_models; c=open_catalog(Config.load(), bundles=False); " + "assert 'new' in catalog_models(c); print(c.resource_version)"] + child = subprocess.run(command, env=os.environ.copy(), text=True, capture_output=True, check=True) + assert child.stdout.strip() == "1.0.0.201" + + +@pytest.mark.parametrize("version", ["../bad", "/absolute", "1/2", "1\\2", "1%2f2", "1?x", "1#x", "v\n"]) +def test_invalid_version_cannot_be_used_as_a_path(tmp_path, version): + with pytest.raises(ConfigError): + config(tmp_path, version=version).catalog_version() + with pytest.raises(ValueError): + catalog.Catalog.cache_file("en", tmp_path, version=version) + with pytest.raises(ValueError): + configfile.validate(configfile.resolve("catalog.version")[0], version) + + +def test_empty_api_version_fails_before_download(tmp_path, monkeypatch): + monkeypatch.setattr(gameapi, "master_version", lambda *args: gameapi.MasterVersion("master", "")) + monkeypatch.setattr(catalog, "download", lambda url: pytest.fail("missing version downloaded")) + with pytest.raises(ConfigError, match="servers.tw.api"): + cli.open_catalog(config(tmp_path), bundles=False) + + +def test_bad_download_is_not_installed_in_cache(tmp_path, monkeypatch): + monkeypatch.setattr(catalog, "download", lambda url: b"not a catalog") + with pytest.raises(ValueError): + cli.open_catalog(config(tmp_path, version="1.0.0.201"), bundles=False) + assert not list((tmp_path / "cache").rglob("*.bin")) + + +def test_fetch_queries_version_before_downloading_and_labels_matching_bytes(tmp_path, monkeypatch, capsys): + monkeypatch.setenv("NNNOTES_SERVERS_TW_CDN", "https://cdn.example/prod/tw") + monkeypatch.setenv("NNNOTES_SERVERS_TW_API", "https://api.example") + events = [] + data = binary(MODEL, EPISODE) + + def observe(*args): + events.append("version") + return gameapi.MasterVersion("master", "1.0.0.201") + + def fetch(cdn, language, *, version): + events.append(version) + assert language == "en" and version == "1.0.0.201" + return data, "catalog-hash" + + monkeypatch.setattr(gameapi, "master_version", observe) + monkeypatch.setattr(catalogdb, "fetch", fetch) + cli.main(["--region", "tw", "--language", "en", "--cache", str(tmp_path / "cache"), "catalogs", "fetch"]) + assert events == ["version", "1.0.0.201"] + version = catalogdb.CatalogDB(tmp_path / "cache" / "store").versions()[0] + assert version["resourceVersion"] == "1.0.0.201" + assert version["remote"]["sha256"] == hashlib.sha256(data).hexdigest() + assert "1.0.0.201" in capsys.readouterr().out + + +def test_fetch_downloads_matching_bin_and_hash(tmp_path): + cdn = tmp_path / "cdn" / "asset" / "Android" + cdn.mkdir(parents=True) + data = binary("Versioned") + (cdn / "catalog_1.0.0.201_en.bin").write_bytes(data) + (cdn / "catalog_1.0.0.201_en.hash").write_text("new-hash\n") + (cdn / "catalog_main_en.bin").write_bytes(binary("Old")) + assert catalogdb.fetch((tmp_path / "cdn").as_uri(), "en", version="1.0.0.201") == (data, "new-hash") + + +def test_browser_uses_the_same_selector(tmp_path, monkeypatch): + cfg = config(tmp_path) + monkeypatch.setattr(gameapi, "master_version", lambda *args: gameapi.MasterVersion("master", "1.0.0.201")) + calls = [] + monkeypatch.setattr(catalog, "download", lambda url: calls.append(url) or binary(MODEL)) + handler = addressables.Handler.__new__(addressables.Handler) + handler.server = SimpleNamespace(catalogs={}, cache=tmp_path / "cache", bundle_key=None) + handler.catalog(addressables.Region("tw", "TW", cfg.cdn("tw"), ["en"], cfg), "en") + assert len(calls) == 1 and calls[0].endswith("catalog_1.0.0.201_en.bin") + + +def test_catalog_release_flag_is_distinct_from_stored_catalog_version(tmp_path): + args = cli.build_parser().parse_args(["--catalog-release", "1.0.0.201", "export", "--catalog-version", "stored-label", + "--select", "key:Example", "-o", str(tmp_path / "out")]) + assert args.catalog_release == "1.0.0.201" and args.catalog_version == "stored-label" + assert cli.load_config(args).catalog_version() == "1.0.0.201" diff --git a/tests/test_configfile.py b/tests/test_configfile.py index be276fc..65e3d85 100644 --- a/tests/test_configfile.py +++ b/tests/test_configfile.py @@ -74,7 +74,7 @@ def test_init_with_values(tmp_path, capsys): assert data["bundle"]["key"] == KEY and data["catalog"]["region"] == "en" assert data["servers"] == {"en": {"name": "", "cdn": "https://cdn.test/", "api": "", "languages": ["en", "ja"], "master": "", "provider": "", "client_version": "", "apk": "", - "catalog": ""}} # the example region table renamed + "catalog": "", "catalog_version": ""}} # the example region table renamed assert data["paths"]["cache"] == str((tmp_path / "cache").absolute()) assert "# CDN base URL of the region" in f.read_text(encoding="utf-8") # the comments stay if os.name != "nt":