perf(web): device models download at a third of their size

docs/internals/devices.md

@@ -156,3 +156,29 @@ hubs that serve it pay nothing, and switches on that 409, or when its MJPEG
 image fails in a browser that has `RTCPeerConnection`. Offers are not trickled:
 serve-sim takes one offer with every candidate. The video element only feeds
 the same canvas sink, so the panel and the mobile WebView keep one surface.
+
+## 3D device models
+
+The GLBs under `apps/web/src/components/device/models` are generated. The
+record is [`sources.json`](../../apps/web/src/components/device/models/sources.json)
+plus [`convert-device-model.py`](../../scripts/convert-device-model.py), whose
+header lists the gltf-transform steps that follow the Blender export.
+
+Apple's AR assets build glass and finishes from stacked shells. Where a shell
+is an exact copy of the layer beneath it, depth ties resolve the same way every
+frame. Where it crosses that layer, as the iPhone 16 and 16 Plus back glass
+does, WebGL draws a flickering patchwork, so `surfaceOffsets` pushes that shell
+0.0003 units outward. Quantization has the same failure mode: per-mesh grids
+round coincident layers apart, so meshopt runs on one scene-wide 16-bit grid.
+Do not simplify: even a 1e-5 error changes rim highlights and seams the Duo's
+inner displays.
+
+`hiddenNodes` lists parts that never change a pixel when hidden, rendered from
+26 directions with a logarithmic depth buffer. Parts that are only nearly
+hidden stay. The Duo has none listed because its hinge pose changes what is
+visible.
+
+Display meshes are quantized like everything else.
+[`modelScene.ts`](../../packages/client-runtime/src/device/modelScene.ts)
+bakes their node transforms back into float geometry at load, because screen
+UVs, touch projection, and the normalization checks work in model space.

packages/client-runtime/src/device/modelScene.test.ts

@@ -1,5 +1,6 @@
 import {
   BoxGeometry,
+  BufferAttribute,
   Group,
   Mesh,
   MeshBasicMaterial,
@@ -8,7 +9,11 @@ import {
   Texture,
 } from "three";
 import { expect, it, vi } from "vite-plus/test";
-import { createImportedPhoneScene, disposeDeviceModel } from "./modelScene.ts";
+import {
+  bakeDisplayTransforms,
+  createImportedPhoneScene,
+  disposeDeviceModel,
+} from "./modelScene.ts";
 import { phoneDisplayLayout } from "./phoneScene.ts";
 
 function asset() {
@@ -68,3 +73,23 @@ it("rejects missing and incorrectly normalized displays before changing their ma
   disposeDeviceModel(root);
   texture.dispose();
 });
+
+it("accepts a quantized display once loading bakes its node transform", () => {
+  const { root, screen } = asset();
+  // KHR_mesh_quantization stores normalized shorts and moves their scale and offset onto the node.
+  const position = screen.geometry.getAttribute("position");
+  const shorts = new Int16Array(position.count * 3);
+  for (let i = 0; i < position.count; i++)
+    for (let c = 0; c < 3; c++)
+      shorts[i * 3 + c] = Math.round(
+        ((position.getComponent(i, c) - (c === 2 ? 0.043 : 0)) / 1.1) * 32767,
+      );
+  screen.geometry.setAttribute("position", new BufferAttribute(shorts, 3, true));
+  screen.position.z = 0.043;
+  screen.scale.setScalar(1.1);
+  bakeDisplayTransforms(root);
+  expect(screen.geometry.getAttribute("position").array).toBeInstanceOf(Float32Array);
+  const phone = createImportedPhoneScene(root, new Texture(), phoneDisplayLayout(null, 1206, 2622));
+  expect(screen.geometry.getAttribute("uv").getX(0)).toBeCloseTo(0, 3);
+  phone.dispose();
+});

packages/client-runtime/src/device/modelScene.ts

@@ -1,9 +1,47 @@
 // @effect-diagnostics globalFetch:off - This browser-only renderer owns abortable asset requests without an Effect runtime, like the live stream.
-import { Box3, Group, Mesh, MeshBasicMaterial, Texture, type Material, type Object3D } from "three";
+import {
+  Box3,
+  BufferAttribute,
+  Group,
+  Mesh,
+  MeshBasicMaterial,
+  Texture,
+  type Material,
+  type Object3D,
+} from "three";
 import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js";
+import { MeshoptDecoder } from "three/addons/libs/meshopt_decoder.module.js";
 import type { DeviceAssetSource } from "./model.ts";
 import { createDisplayProjection, updateDisplayUv, type PhoneDisplayLayout } from "./phoneScene.ts";
 
+const DISPLAY_MESHES = new Set([
+  "device-screen",
+  "cover-display",
+  "inner-display-left",
+  "inner-display-right",
+]);
+
+// Quantized glTF keeps positions in a node-local box; the screen math needs model-space floats.
+export function bakeDisplayTransforms(root: Object3D) {
+  root.traverse((object) => {
+    if (!(object instanceof Mesh) || !DISPLAY_MESHES.has(object.name)) return;
+    const geometry = object.geometry;
+    for (const name of Object.keys(geometry.attributes)) {
+      const attribute = geometry.getAttribute(name);
+      const values = new Float32Array(attribute.count * attribute.itemSize);
+      for (let i = 0; i < attribute.count; i++)
+        for (let c = 0; c < attribute.itemSize; c++)
+          values[i * attribute.itemSize + c] = attribute.getComponent(i, c);
+      geometry.setAttribute(name, new BufferAttribute(values, attribute.itemSize));
+    }
+    object.updateMatrix();
+    geometry.applyMatrix4(object.matrix);
+    object.position.set(0, 0, 0);
+    object.quaternion.identity();
+    object.scale.set(1, 1, 1);
+  });
+}
+
 /** GLB resources belong to one viewer; the live framebuffer texture belongs to its stream presentation. */
 export function disposeDeviceModel(root: Object3D) {
   const geometries = new Set<Mesh["geometry"]>();
@@ -105,10 +143,10 @@ export async function loadDeviceModel(source: DeviceAssetSource, signal: AbortSi
   if (!response.ok) throw new Error(`Device model request failed: ${response.status}`);
   const data = await response.arrayBuffer();
   signal.throwIfAborted();
-  const gltf = await new GLTFLoader().parseAsync(
-    data,
-    new URL(".", new URL(source.url, globalThis.location.href)).href,
-  );
+  const gltf = await new GLTFLoader()
+    .setMeshoptDecoder(MeshoptDecoder)
+    .parseAsync(data, new URL(".", new URL(source.url, globalThis.location.href)).href);
+  bakeDisplayTransforms(gltf.scene);
   if (signal.aborted) {
     disposeDeviceModel(gltf.scene);
     signal.throwIfAborted();

scripts/convert-device-model.py

@@ -1,7 +1,9 @@
 # Run with Blender 4.5.9:
 # blender --background --python scripts/convert-device-model.py -- <model-id> <source.usdz> <output.glb>
 # Then optimize using @gltf-transform/cli 4.2.1 with --compress false --simplify false
-# --join false --palette false --flatten false --texture-size 1024 --texture-compress webp.
+# --join false --palette false --flatten false --texture-size 1024 --texture-compress webp,
+# then `weld` and `meshopt --quantization-volume scene --quantize-position 16`. Per-mesh grids
+# move coincident layers apart and z-fight; see docs/internals/devices.md#3d-device-models.
 import hashlib
 import json
 import sys
@@ -28,8 +30,9 @@
 objects = [o for o in body.children_recursive if o.type == "MESH"]
 if body.type == "MESH":
     objects.append(body)
-# Some sources stack every colour variant's back glass; keep one.
-objects = [o for o in objects if o.name not in model.get("excludeNodes", [])]
+# Some sources stack every colour variant's back glass; keep one. Hidden nodes never reach a pixel.
+skipped = set(model.get("excludeNodes", [])) | set(model.get("hiddenNodes", []))
+objects = [o for o in objects if o.name not in skipped]
 uv = screen.data.uv_layers.active
 points = []
 coords = []
@@ -63,6 +66,12 @@
     o.data.transform(matrix)
     # Blender is Z-up; glTF export converts to Y-up. Store canonical Y-up geometry as Blender Z-up.
     o.data.transform(Matrix.Rotation(np.pi / 2, 4, "X"))
+    # A shell that crosses the layer beneath it z-fights in WebGL; keep it in front.
+    offset = model.get("surfaceOffsets", {}).get(o.name)
+    if offset:
+        normals = [v.normal.copy() for v in o.data.vertices]
+        for v, n in zip(o.data.vertices, normals):
+            v.co += n * offset
     if o == screen:
         o.name = "device-screen"
         o.data.name = "device-screen"