Release 0.1.3: English multi-line text in stories

docs/story-data-format.md

@@ -387,7 +387,7 @@ ADV screen. Keys read:
 | `dotween` | The game's DOTween defaults. |
 | `shaders` | `{ index: "shaders/shaders.json", names }`: the UI shader directory, relative to `ui/`. |
 | `language` | `mode` (`LanguageMode`), `field` (the text field), `lineSpacing` (the line spacing localized texts get). |
-| `textStyle` | `language`, `units`, `roles`: the line metrics per font role of the game's fonts (informative; the layout uses the font assets of `ui/fonts.json`). |
+| `textStyle` | `language`, `units`, `roles`: the line metrics per font role of the game's fonts. The layout uses the font assets of `ui/fonts.json`; with open fonts it takes the line height, ascent and descent of their role from here ([Fonts](#fonts-uifontsjson)). |
 
 Node records beyond the uGUI component fields:
 
@@ -447,7 +447,11 @@ language: font assets holding exactly the characters the episode shows in that l
 materials and, per text node, the font asset and material it uses. The player lays text out with TextMeshPro's rules
 using these glyph metrics and draws it with the distance-field shader the material names (in `ui/shaders/`). The
 format is the same whether the glyphs were generated from a font file (`source` `open`) or taken from the game's font
-assets (`game`).
+assets (`game`). With open fonts, a font asset that the texts of a font role use (the role's `fontAsset` in
+`ui/languages.json`, or the font asset of a text node whose `textStyle.fontRole` names the role) is laid out with the
+line height, ascent and descent of the game's font of that role (`ui.json` `textStyle.roles`) in place of its own face
+info's. The language's line spacing is set for the game's fonts: English's -100 takes one em off the pitch of a font
+whose line height is 2 em, and would put the lines of a 1.448 em face 0.448 em apart.
 
 ```json
 { "format": "ournotes.story-fonts/1", "language": "en", "source": "open",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ournotes-player",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "Browser player for BanG Dream! Our Notes: live charts, story episodes and Live2D models with the game's own shaders (WebGL2 + WebAudio, custom elements)",
   "keywords": [
     "bang-dream",

src/story/simple/ui.js

@@ -4,6 +4,7 @@ import { ShaderLib } from "../../engine/glsl.js";
 import { GLTex } from "../../engine/texture.js";
 import { UI_CLIP_TARGET, UIDraw, UIError, UIImage, UILayout, UIMesh, UINode, UISprite, uiCanvasSize, uiColor32 } from "../../engine/ugui.js";
 import { TMPText, UIGradientMod } from "../../engine/uitext.js";
+import { StoryFontAssets } from "../ui-fonts.js";
 import { StoryText } from "../ui-ruby.js";
 
 // The uGUI documents of the simple player: the simple talk window (UISimpleAdvTalkWindow, ui/simple/ui.json with its
@@ -128,7 +129,7 @@ export class SimpleUIDoc {
       if (!t) throw new UIError(`sprite ${name}: texture ${s.texture} not in the data`);
       return [name, new UISprite(name, s, t)];
     }));
-    this._fonts = new Map();
+    this._fonts = this.fonts ? new StoryFontAssets(this.fonts, doc, this.language) : null;   // open fonts: the game's line metrics
     const clips = Object.fromEntries(Object.entries(doc.clips || {}).map(([k, c]) => [k, new SimpleUIClip(c)]));
     this.animators = [];
     this.unbound = [];
@@ -183,12 +184,9 @@ export class SimpleUIDoc {
 
   // ------------------------------------------------------------ text host (engine/uitext.js)
   fontAsset(name) {
-    if (!this._fonts.has(name)) {
-      const f = this.fonts.fonts[name];
-      if (!f) throw new UIError(`font asset ${name} not in the font data`);
-      this._fonts.set(name, { name, ...f, textureSize: this.fonts.textures, lineBreaking: this.fonts.lineBreaking || null });
-    }
-    return this._fonts.get(name);
+    const f = this._fonts ? this._fonts.get(name) : null;
+    if (!f) throw new UIError(`font asset ${name} not in the font data`);
+    return f;
   }
 
   material(name) { return this.fonts ? this.fonts.materials[name] || null : null; }

src/story/ui-fonts.js

@@ -0,0 +1,57 @@
+import { F } from "../engine/core.js";
+
+// The font asset records a story UI lays its texts out with (the text host's fontAsset, engine/uitext.js).
+//
+// Font data made from open fonts (ui/fonts.json `source` "open") has glyphs with the metrics of the open font, and
+// with them its face's line height, ascent and descent. The game lays text out with the line metrics of its own font
+// assets, and the language's line spacing (LocalizeManager.ApplyLanguageLineSpacing) is set for them: English's -100
+// (one em less) takes the line gap of the game's primary font (A-OTF-ShinGoPr6N: line height 2 em, ascent 0.88,
+// descent -0.12) out. An open face without that gap (Noto Sans CJK: 1.448 em with no gap) would put the lines 0.448 em
+// apart, over each other. So an open font asset standing in for a font role of the game gets the game's line height,
+// ascent and descent of that role (ui.json `textStyle.roles`, per role in em) in place of its own: the lines are as
+// far apart and sit where the game has them, in every language. The glyphs keep their own metrics (the open CJK
+// faces share the game's ideographic em box, 0.88 / -0.12).
+//
+// The role of a font asset: the fontRole of the text nodes whose bindings (fonts.texts[path].localized) use it, and
+// the font asset the language document names per role (ui/languages.json `roles`, which also covers the texts of the
+// chat window and the frames). Data from the game's fonts, or without textStyle roles, is taken as it is.
+export class StoryFontAssets {
+  // fonts = ui/fonts.json (or ui/simple/fonts.json); doc = the ui.json record document laid out with them;
+  // language = ui/languages.json (may be null)
+  constructor(fonts, doc, language) {
+    this.fonts = fonts;
+    this.cache = new Map();
+    this.roleOf = new Map();                          // font asset name -> the game's line metrics of its role
+    const roles = doc && doc.textStyle && doc.textStyle.roles;
+    if (!fonts || fonts.source !== "open" || !roles) return;
+    const add = (asset, role) => {
+      const m = roles[role];
+      if (typeof asset === "string" && m && !this.roleOf.has(asset)) this.roleOf.set(asset, m);
+    };
+    for (const [role, r] of Object.entries((language && language.roles) || {})) add(r && r.fontAsset, role);
+    for (const n of doc.nodes || []) {
+      const t = n.textStyle && fonts.texts ? fonts.texts[n.path] : null;
+      if (t && t.localized) add(t.localized.fontAsset, n.textStyle.fontRole);
+    }
+  }
+
+  // -> the font asset record of `name`, or null when ui/fonts.json has none
+  get(name) {
+    if (!this.cache.has(name)) {
+      const f = this.fonts.fonts[name];
+      if (!f) return null;
+      this.cache.set(name, { name, ...f, faceInfo: this.faceInfo(name, f.faceInfo), textureSize: this.fonts.textures,
+                             lineBreaking: this.fonts.lineBreaking || null });
+    }
+    return this.cache.get(name);
+  }
+
+  // the face info of the font asset `name`: with the line metrics of the game's font of its role, else its own
+  faceInfo(name, fi) {
+    const m = this.roleOf.get(name);
+    if (!m || !fi) return fi;
+    const em = F(fi.m_PointSize / (fi.m_Scale || 1));
+    return { ...fi, m_LineHeight: F(m.lineHeightEm * em), m_AscentLine: F(m.ascentEm * em),
+             m_DescentLine: F(m.descentEm * em) };
+  }
+}

src/story/ui.js

@@ -5,6 +5,7 @@ import { Tweens } from "../engine/tween.js";
 import { UIAnimator, UIClip, UIDraw, UIError, UIImage, UINode, UISprite, UITween, uiCanvasSize } from "../engine/ugui.js";
 import { TMPText, UIGradientMod, tmpSpriteAsset, tmpUnsupported } from "../engine/uitext.js";
 import { ADV_CANVAS_LAYER, StoryCommandError, createStoryUILayers } from "./interfaces.js";
+import { StoryFontAssets } from "./ui-fonts.js";
 import { StoryLayout } from "./ui-layout.js";
 import { StoryText, countedText, shownText } from "./ui-ruby.js";
 import { StoryTalkWindow, removeTagsWithRuby } from "./ui-talk.js";
@@ -103,7 +104,7 @@ export class StoryUI {
       if (!t) throw new UIError(`sprite ${n}: texture ${s.texture} not in the data`);
       return [n, new UISprite(n, s, t)];
     }));
-    this._fonts = new Map();
+    this._fonts = new StoryFontAssets(fonts, doc, language);   // open fonts: the game's line metrics
     this._clipCache = Object.fromEntries(Object.entries(doc.clips).map(([k, c]) => [k, new UIClip(c)]));
     this.animators = [];
     for (const n of byPath.values()) {
@@ -252,12 +253,9 @@ export class StoryUI {
 
   // ------------------------------------------------------------ text host (engine/uitext.js)
   fontAsset(name) {
-    if (!this._fonts.has(name)) {
-      const f = this.fonts.fonts[name];
-      if (!f) throw new UIError(`font asset ${name} not in ui/fonts.json`);
-      this._fonts.set(name, { name, ...f, textureSize: this.fonts.textures, lineBreaking: this.fonts.lineBreaking || null });
-    }
-    return this._fonts.get(name);
+    const f = this._fonts.get(name);
+    if (!f) throw new UIError(`font asset ${name} not in ui/fonts.json`);
+    return f;
   }
 
   material(name) { return this._materialInstances.get(name) || this.fonts.materials[name] || null; }

tests/data/story-ui-layout.test.mjs

@@ -67,21 +67,15 @@ test("story UI layout and timing", { skip: SKIP }, async () => {
   const locDone = locName !== null ? ui.showLocation(locName).then(() => { locDone.frame = loop.frameCount; }) : null;
   ui.showTalk();
   ui.setSpeakerName(speaker);
-  const typing = ui.setTalk(line);
+  const firstTyping = ui.setTalk(line);
   const plain = removeTagsWithRuby(line);
-  check("typewriter: totalLength", typing.totalLength, countRenderedCharacters(countedText(tt, P.talkText.storyText.b, plain)), 0, "tags removed");
-  check("typewriter: visible at call (frame 0)", tt.maxVisibleCharacters, 1, 0, "first character in the same frame");
-  const visibleAt = [0];
-  let bgFullAt = -1, typingDoneAt = -1;
-  const indicatorY = [], bgAlpha = [];
-  typing.finished.then(() => { typingDoneAt = loop.frameCount; });
+  let bgFullAt = -1;
+  const bgAlpha = [];
   for (let f = 1; f <= 90; f++) {
     await step();
-    while (visibleAt.length < tt.maxVisibleCharacters) visibleAt.push(f);
     const a = P.background.canvasGroup.alpha;
     if (f <= 6) bgAlpha.push(a);
     if (bgFullAt < 0 && a >= 1) bgFullAt = f;
-    if (P.nextIndicator.activeInHierarchy && indicatorY.length < 3) indicatorY.push([f, P.nextIndicator.anchoredPosition.y]);
     if (f === 30) {
       // --- layout at frame 30 (1.0 s): location and title in their hold phase --------------------------------------
       for (const [pw, ph] of [[2340, 1080], [2400, 1080]]) {
@@ -121,22 +115,6 @@ test("story UI layout and timing", { skip: SKIP }, async () => {
     check(`TalkBackground alpha frame ${i + 1}`, a, i + 1 >= 6 ? 1 : -k * (k - 2), 1e-6, "OutQuad");
   });
   check("TalkBackground alpha 1 at frame", bgFullAt, 6, 0, "0.2 s at 30 fps");
-  // --- typewriter: 1 WaitWhile tick, then Delay ticks until elapsed >= delay; ASCII letters 1 tick -----------------------
-  const delay = F(Math.trunc(F(F([1, 4].includes(language.mode) ? 0.015 : ui.talk.typingDelay) / 1) * 1000 + 0.5) / 1000);
-  let ticks = 0;
-  for (let e = 0; e < delay; e = F(e + F(1 / 30))) ticks++;
-  const perChar = (ch) => (/[A-Za-z]/.test(ch) ? 1 : 1 + ticks);
-  const letters = [...Array(typing.totalLength).keys()].map((i) => plain[i] || "");
-  const steps = visibleAt.slice(1).map((f, k) => f - visibleAt[k]);
-  check("typewriter: frames per char", steps.join(","), letters.slice(0, -1).map(perChar).join(","), 0, `1 + ${ticks} ticks, letters 1`);
-  const wantEnd = visibleAt[visibleAt.length - 1] + perChar(letters[letters.length - 1]);
-  check("typewriter: typing end frame", typingDoneAt, wantEnd, 0, `${typing.totalLength} chars`);
-  check("typewriter: isTyping after end", ui.isTyping, false, 0, "");
-  // --- NextIndicator Loop clip after activation at the typing end: y = 76 - 600 t^2 + 2000 t^3 -----------------------------
-  indicatorY.forEach(([f, y]) => {
-    const t = (f - typingDoneAt + 1) / 30;
-    check(`NextIndicator y frame ${f}`, y, 76 - 600 * t * t + 2000 * t * t * t, 1e-3, "clip (t = frames since enable)");
-  });
   if (locDone) {
     let seqPos = 0, seqSteps = 0;
     while (seqPos < F(2.5)) { seqPos = F(seqPos + F(1 / 30)); seqSteps++; }
@@ -233,6 +211,39 @@ test("story UI layout and timing", { skip: SKIP }, async () => {
   // MonoBehaviour Update, so it sees the end on frame 7
   check("fadeInLetterBox resolves at frame", lbDone, 7, 0, "WaitUntil in the UniTask Update runner");
 
+  // --- typewriter to its end, on a talk of its own (the longest line may outlast the scenario's 90 frames: 3 frames per
+  // character in Japanese): 1 WaitWhile tick, then Delay ticks until elapsed >= delay; ASCII letters 1 tick ------------
+  const delay = F(Math.trunc(F(F([1, 4].includes(language.mode) ? 0.015 : ui.talk.typingDelay) / 1) * 1000 + 0.5) / 1000);
+  let ticks = 0;
+  for (let e = 0; e < delay; e = F(e + F(1 / 30))) ticks++;
+  const perChar = (ch) => (/[A-Za-z]/.test(ch) ? 1 : 1 + ticks);
+  firstTyping.cancel();                                                 // Talk: typingCts.Cancel(), seen on the next tick
+  await step();
+  await firstTyping.finished;
+  const typing = ui.setTalk(line), t0 = loop.frameCount;
+  check("typewriter: totalLength", typing.totalLength, countRenderedCharacters(countedText(tt, P.talkText.storyText.b, plain)), 0, "tags removed");
+  check("typewriter: visible at call (frame 0)", tt.maxVisibleCharacters, 1, 0, "first character in the same frame");
+  const visibleAt = [0], indicatorY = [];
+  let typingDoneAt = -1;
+  typing.finished.then(() => { typingDoneAt = loop.frameCount - t0; });
+  const lastFrame = typing.totalLength * (1 + ticks) + 10;
+  for (let f = 1; f <= lastFrame && (typingDoneAt < 0 || indicatorY.length < 3); f++) {
+    await step();
+    while (visibleAt.length < tt.maxVisibleCharacters) visibleAt.push(f);
+    if (P.nextIndicator.activeInHierarchy && indicatorY.length < 3) indicatorY.push([f, P.nextIndicator.anchoredPosition.y]);
+  }
+  const letters = [...Array(typing.totalLength).keys()].map((i) => plain[i] || "");
+  const steps = visibleAt.slice(1).map((f, k) => f - visibleAt[k]);
+  check("typewriter: frames per char", steps.join(","), letters.slice(0, -1).map(perChar).join(","), 0, `1 + ${ticks} ticks, letters 1`);
+  const wantEnd = visibleAt[visibleAt.length - 1] + perChar(letters[letters.length - 1]);
+  check("typewriter: typing end frame", typingDoneAt, wantEnd, 0, `${typing.totalLength} chars`);
+  check("typewriter: isTyping after end", ui.isTyping, false, 0, "");
+  // --- NextIndicator Loop clip after activation at the typing end: y = 76 - 600 t^2 + 2000 t^3 -----------------------------
+  indicatorY.forEach(([f, y]) => {
+    const t = (f - typingDoneAt + 1) / 30;
+    check(`NextIndicator y frame ${f}`, y, 76 - 600 * t * t + 2000 * t * t * t, 1e-3, "clip (t = frames since enable)");
+  });
+
   // --- line breaks of every Talk line (optional dump) -------------------------------------------------------------------
   if (LINES) {
     const out = {};

tests/unit/story-ui-fonts.test.mjs

@@ -0,0 +1,43 @@
+// StoryFontAssets (src/story/ui-fonts.js): font data made from open fonts lays lines out with the line height, ascent
+// and descent of the game's font of each role (ui.json textStyle.roles), found through the language document's roles
+// and the text nodes' fontRole; game fonts, data without roles and fonts of no role keep their own face info.
+// Synthetic inputs only.
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { F } from "../../src/engine/core.js";
+import { StoryFontAssets } from "../../src/story/ui-fonts.js";
+
+// Noto Sans CJK at 40 px per em: line height 1.448 em, no line gap
+const face = (size = 40) => ({ m_PointSize: size, m_Scale: 1, m_LineHeight: F(1.448 * size), m_AscentLine: F(1.16 * size),
+                               m_DescentLine: F(-0.288 * size), m_CapLine: 29 });
+const fontsDoc = (source) => ({
+  source, textures: {}, fonts: { Talk: { faceInfo: face() }, Simple: { faceInfo: face(47) }, Digits: { faceInfo: face(80) } },
+  texts: { "W/TalkText": { localized: { fontAsset: "Simple" } } },
+});
+const uiDoc = { textStyle: { roles: { primary: { lineHeightEm: 2, ascentEm: 0.88, descentEm: -0.12 },
+                                      number: { lineHeightEm: 1.2, ascentEm: 0.83, descentEm: -0.17 } } },
+                nodes: [{ path: "W" }, { path: "W/TalkText", textStyle: { fontRole: "primary" } }] };
+const language = { lineSpacing: -100, roles: { primary: { fontAsset: "Talk" } } };
+
+test("open fonts: the game's line metrics of the role, by the language's roles and by the text nodes' fontRole", () => {
+  const a = new StoryFontAssets(fontsDoc("open"), uiDoc, language);
+  const t = a.get("Talk");
+  assert.deepEqual([t.faceInfo.m_LineHeight, t.faceInfo.m_AscentLine, t.faceInfo.m_DescentLine], [80, F(35.2), F(-4.8)]);
+  assert.equal(t.faceInfo.m_CapLine, 29);                               // the rest of the face as it is
+  assert.equal(t.name, "Talk");
+  assert.equal(a.get("Talk"), t);                                       // one record per asset
+  const s = a.get("Simple").faceInfo;                                   // the talk text's binding, role primary
+  assert.deepEqual([s.m_LineHeight, s.m_AscentLine, s.m_DescentLine], [94, F(F(0.88) * 47), F(F(-0.12) * 47)]);
+  assert.deepEqual(a.get("Digits").faceInfo, face(80));                // no text of the role: its own
+  assert.equal(a.get("Missing"), null);
+  // English line spacing -100 at 36 px: lineHeight x scale - 36 is one em, not 0.448 em
+  const pitch = F(t.faceInfo.m_LineHeight * F(36 / 40)) + language.lineSpacing * F(36 * 0.01);
+  assert.ok(Math.abs(pitch - 36) < 1e-4, String(pitch));
+});
+
+test("game fonts, and data without textStyle roles, keep the font assets' own face info", () => {
+  assert.deepEqual(new StoryFontAssets(fontsDoc("game"), uiDoc, language).get("Talk").faceInfo, face());
+  assert.deepEqual(new StoryFontAssets(fontsDoc("open"), { nodes: uiDoc.nodes }, language).get("Talk").faceInfo, face());
+  assert.deepEqual(new StoryFontAssets(fontsDoc("open"), uiDoc, null).get("Talk").faceInfo, face());   // by fontRole only
+  assert.equal(new StoryFontAssets(fontsDoc("open"), uiDoc, null).get("Simple").faceInfo.m_LineHeight, 94);
+});