diff --git a/apps/server/src/device/DeviceHubOperations.test.ts b/apps/server/src/device/DeviceHubOperations.test.ts new file mode 100644 index 0000000000000..7f6589febe748 --- /dev/null +++ b/apps/server/src/device/DeviceHubOperations.test.ts @@ -0,0 +1,73 @@ +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import { HttpClient, HttpClientResponse } from "effect/http"; +import { makeDeviceHubOperations } from "./DeviceHubOperations.ts"; + +const simulator = (id: string, name: string) => ({ + id, + name, + version: "iOS 27.0", + platform: "ios", + booted: true, + physical: false, +}); + +const operations = (simctl: { stdout: string; code: number }) => + makeDeviceHubOperations({ + hostId: "local", + origin: "http://hub", + http: HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + Response.json({ + simulators: [simulator("A", "iPhone (clean)"), simulator("B", "Unknown type")], + emulators: [], + }), + ), + ), + ), + run: (command, args) => + Effect.succeed( + command === "xcrun" && args.join(" ") === "simctl list -j" + ? { ...simctl, stderr: "" } + : { stdout: "", stderr: "", code: 1 }, + ), + androidAvailable: Effect.succeed(false), + }); + +it.effect("names a renamed simulator's hardware from its device type", () => + Effect.gen(function* () { + const { devices } = yield* operations({ + code: 0, + stdout: JSON.stringify({ + devicetypes: [ + { + identifier: "com.apple.CoreSimulator.SimDeviceType.iPhone-18-Pro", + name: "iPhone 18 Pro", + }, + ], + devices: { + "com.apple.CoreSimulator.SimRuntime.iOS-27-0": [ + { + udid: "A", + deviceTypeIdentifier: "com.apple.CoreSimulator.SimDeviceType.iPhone-18-Pro", + }, + { udid: "B", deviceTypeIdentifier: "com.apple.CoreSimulator.SimDeviceType.Gone" }, + ], + }, + }), + }).list; + expect(devices.map((device) => [device.name, device.model])).toEqual([ + ["iPhone (clean)", "iPhone 18 Pro"], + ["Unknown type", undefined], + ]); + }), +); + +it.effect("still lists simulators when simctl cannot describe them", () => + Effect.gen(function* () { + const { devices } = yield* operations({ code: 1, stdout: "" }).list; + expect(devices.map((device) => device.model)).toEqual([undefined, undefined]); + }), +); diff --git a/apps/server/src/device/DeviceHubOperations.ts b/apps/server/src/device/DeviceHubOperations.ts index 9371484f4c93f..181a3cce21574 100644 --- a/apps/server/src/device/DeviceHubOperations.ts +++ b/apps/server/src/device/DeviceHubOperations.ts @@ -39,6 +39,17 @@ const HubActionResult = Schema.Struct({ ), }); +const SimctlList = Schema.fromJsonString( + Schema.Struct({ + devicetypes: Schema.Array(Schema.Struct({ identifier: Schema.String, name: Schema.String })), + devices: Schema.Record( + Schema.String, + Schema.Array(Schema.Struct({ udid: Schema.String, deviceTypeIdentifier: Schema.String })), + ), + }), +); +const decodeSimctlList = Schema.decodeUnknownEffect(SimctlList); + const vendorPrefix = (platform: DevicePlatform) => platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; @@ -90,13 +101,38 @@ export const makeDeviceHubOperations = (options: { Effect.flatMap((request) => hubJson(request, HubActionResult, "attach stream", BOOT_TIMEOUT)), ); + // The hub reports only the user's simulator name; the device type survives renames. + const listSimulatorModels = Effect.fn("DeviceHubOperations.listSimulatorModels")( + function* () { + const models = new Map(); + const result = yield* run("xcrun", ["simctl", "list", "-j"]); + if (result.code !== 0) return models; + const { devicetypes, devices } = yield* decodeSimctlList(result.stdout); + const names = new Map(devicetypes.map((type) => [type.identifier, type.name])); + for (const device of Object.values(devices).flat()) { + const name = names.get(device.deviceTypeIdentifier); + if (name) models.set(device.udid, name); + } + return models; + }, + Effect.orElseSucceed(() => new Map()), + ); + const list: DeviceOperations["list"] = Effect.gen(function* () { const hubList = yield* hubJson( HttpClientRequest.get(`${origin}/api/devices`), HubDeviceList, "list", ); - const devices: Array = [...hubList.simulators, ...hubList.emulators]; + const simulatorModels = + hubList.simulators.length > 0 ? yield* listSimulatorModels() : new Map(); + const devices: Array = [ + ...hubList.simulators.map((device) => { + const model = simulatorModels.get(device.id); + return model ? { ...device, model } : device; + }), + ...hubList.emulators, + ]; const androidAvailable = yield* options.androidAvailable; const warnings = hubList.errors?.filter( (error) => diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx index 4b0b5c84ca5b9..88f99125f8636 100644 --- a/apps/web/src/components/device/DeviceStreamView.tsx +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -85,6 +85,8 @@ export function DeviceStreamView(props: { readonly platform: DevicePlatform; readonly deviceId: string; readonly deviceName?: string; + /** Hardware product name, which picks the 3D body when the simulator was renamed. */ + readonly deviceModel?: string | undefined; readonly deviceDescription?: string; readonly visible: boolean; readonly hostId: string; @@ -96,7 +98,8 @@ export function DeviceStreamView(props: { /** The host serves only serve-sim's stream and input socket (Simlock). */ readonly serveSimOnly?: boolean; }) { - const model = deviceModel(props.platform, props.deviceName ?? ""); + const hardwareName = props.deviceModel ?? props.deviceName ?? ""; + const model = deviceModel(props.platform, hardwareName); const isDuo = model?.id === "iphone-duo"; const [presentation, setPresentation] = useState<"phone" | "flat">("phone"); const [keyboardAttached, setKeyboardAttached] = useState(false); @@ -220,7 +223,7 @@ export function DeviceStreamView(props: { ? "Turn off accessibility frames to use 3D" : null; - const keyboardSource = deviceKeyboard(props.platform, props.deviceName ?? ""); + const keyboardSource = deviceKeyboard(props.platform, hardwareName); const resetView = () => { if (!isDuo) { const orientation = keyboardAttached ? "landscape_right" : "portrait"; @@ -230,7 +233,7 @@ export function DeviceStreamView(props: { }; const profile = resolveDeviceShape({ platform: props.platform, - name: props.deviceName ?? "", + name: hardwareName, portraitAspect: Math.min(aspect, 1 / aspect), }); @@ -416,7 +419,7 @@ export function DeviceStreamView(props: { ) : session && showPhone ? ( = { "iphone-duo": { id: "iphone-duo", url: iphoneDuo }, "iphone-18-pro": { id: "iphone-18-pro", url: iphone18Pro }, "iphone-18-pro-max": { id: "iphone-18-pro-max", url: iphone18ProMax }, + "iphone-17": { id: "iphone-17", url: iphone17 }, + "iphone-air": { id: "iphone-air", url: iphoneAir }, + "iphone-17e": { id: "iphone-17e", url: iphone17e }, + "iphone-17-pro": { id: "iphone-17-pro", url: iphone17Pro }, + "iphone-17-pro-max": { id: "iphone-17-pro-max", url: iphone17ProMax }, + "iphone-16-pro": { id: "iphone-16-pro", url: iphone16Pro }, + "iphone-16-pro-max": { id: "iphone-16-pro-max", url: iphone16ProMax }, + "iphone-16": { id: "iphone-16", url: iphone16 }, + "iphone-16-plus": { id: "iphone-16-plus", url: iphone16Plus }, + "iphone-15": { id: "iphone-15", url: iphone15 }, + "iphone-15-plus": { id: "iphone-15-plus", url: iphone15Plus }, + "iphone-14": { id: "iphone-14", url: iphone14 }, + "iphone-14-plus": { id: "iphone-14-plus", url: iphone14Plus }, "ipad-pro-13-m5": { id: "ipad-pro-13-m5", url: ipadPro13M5 }, }; diff --git a/apps/web/src/components/device/models/ipad-pro-13-m5-magic-keyboard.glb b/apps/web/src/components/device/models/ipad-pro-13-m5-magic-keyboard.glb index fd46333b4270a..6b114b61d0e94 100644 Binary files a/apps/web/src/components/device/models/ipad-pro-13-m5-magic-keyboard.glb and b/apps/web/src/components/device/models/ipad-pro-13-m5-magic-keyboard.glb differ diff --git a/apps/web/src/components/device/models/ipad-pro-13-m5.glb b/apps/web/src/components/device/models/ipad-pro-13-m5.glb index 0611ac7496071..141191ffededb 100644 Binary files a/apps/web/src/components/device/models/ipad-pro-13-m5.glb and b/apps/web/src/components/device/models/ipad-pro-13-m5.glb differ diff --git a/apps/web/src/components/device/models/iphone-14-plus.glb b/apps/web/src/components/device/models/iphone-14-plus.glb new file mode 100644 index 0000000000000..3584b22f6c9e2 Binary files /dev/null and b/apps/web/src/components/device/models/iphone-14-plus.glb differ diff --git a/apps/web/src/components/device/models/iphone-14.glb b/apps/web/src/components/device/models/iphone-14.glb new file mode 100644 index 0000000000000..6fb265c84e917 Binary files /dev/null and b/apps/web/src/components/device/models/iphone-14.glb differ diff --git a/apps/web/src/components/device/models/iphone-15-plus.glb b/apps/web/src/components/device/models/iphone-15-plus.glb new file mode 100644 index 0000000000000..bb8baa8814c22 Binary files /dev/null and b/apps/web/src/components/device/models/iphone-15-plus.glb differ diff --git a/apps/web/src/components/device/models/iphone-15.glb b/apps/web/src/components/device/models/iphone-15.glb new file mode 100644 index 0000000000000..7c6a80452218b Binary files /dev/null and b/apps/web/src/components/device/models/iphone-15.glb differ diff --git a/apps/web/src/components/device/models/iphone-16-plus.glb b/apps/web/src/components/device/models/iphone-16-plus.glb new file mode 100644 index 0000000000000..eacfc8f736c63 Binary files /dev/null and b/apps/web/src/components/device/models/iphone-16-plus.glb differ diff --git a/apps/web/src/components/device/models/iphone-16-pro-max.glb b/apps/web/src/components/device/models/iphone-16-pro-max.glb new file mode 100644 index 0000000000000..a25c7a1b530ce Binary files /dev/null and b/apps/web/src/components/device/models/iphone-16-pro-max.glb differ diff --git a/apps/web/src/components/device/models/iphone-16-pro.glb b/apps/web/src/components/device/models/iphone-16-pro.glb new file mode 100644 index 0000000000000..d8491d713ac17 Binary files /dev/null and b/apps/web/src/components/device/models/iphone-16-pro.glb differ diff --git a/apps/web/src/components/device/models/iphone-16.glb b/apps/web/src/components/device/models/iphone-16.glb new file mode 100644 index 0000000000000..c58ca24128097 Binary files /dev/null and b/apps/web/src/components/device/models/iphone-16.glb differ diff --git a/apps/web/src/components/device/models/iphone-17-pro-max.glb b/apps/web/src/components/device/models/iphone-17-pro-max.glb new file mode 100644 index 0000000000000..9ab6ec4bc236d Binary files /dev/null and b/apps/web/src/components/device/models/iphone-17-pro-max.glb differ diff --git a/apps/web/src/components/device/models/iphone-17-pro.glb b/apps/web/src/components/device/models/iphone-17-pro.glb new file mode 100644 index 0000000000000..02739b98c09dc Binary files /dev/null and b/apps/web/src/components/device/models/iphone-17-pro.glb differ diff --git a/apps/web/src/components/device/models/iphone-17.glb b/apps/web/src/components/device/models/iphone-17.glb new file mode 100644 index 0000000000000..5ef5dc4ab2199 Binary files /dev/null and b/apps/web/src/components/device/models/iphone-17.glb differ diff --git a/apps/web/src/components/device/models/iphone-17e.glb b/apps/web/src/components/device/models/iphone-17e.glb new file mode 100644 index 0000000000000..6848295c139fd Binary files /dev/null and b/apps/web/src/components/device/models/iphone-17e.glb differ diff --git a/apps/web/src/components/device/models/iphone-18-pro-max.glb b/apps/web/src/components/device/models/iphone-18-pro-max.glb index 52438e10f8d6e..f03137aca3fb1 100644 Binary files a/apps/web/src/components/device/models/iphone-18-pro-max.glb and b/apps/web/src/components/device/models/iphone-18-pro-max.glb differ diff --git a/apps/web/src/components/device/models/iphone-18-pro.glb b/apps/web/src/components/device/models/iphone-18-pro.glb index 2dae3e9beaa65..1efcdc8451dc9 100644 Binary files a/apps/web/src/components/device/models/iphone-18-pro.glb and b/apps/web/src/components/device/models/iphone-18-pro.glb differ diff --git a/apps/web/src/components/device/models/iphone-air.glb b/apps/web/src/components/device/models/iphone-air.glb new file mode 100644 index 0000000000000..2cd516faaa13b Binary files /dev/null and b/apps/web/src/components/device/models/iphone-air.glb differ diff --git a/apps/web/src/components/device/models/iphone-duo.glb b/apps/web/src/components/device/models/iphone-duo.glb index 0269a1dc40e03..ffd2cd217fda5 100644 Binary files a/apps/web/src/components/device/models/iphone-duo.glb and b/apps/web/src/components/device/models/iphone-duo.glb differ diff --git a/apps/web/src/components/device/models/sources.json b/apps/web/src/components/device/models/sources.json index 766b65a7a7597..e986312ca8a62 100644 --- a/apps/web/src/components/device/models/sources.json +++ b/apps/web/src/components/device/models/sources.json @@ -8,7 +8,9 @@ "Extracted device bodies and a separate Magic Keyboard accessory; excluded unrelated parts and presentation duplicates.", "Normalized conventional device portrait axes, centered their screen and uniformly scaled its height to 2.2 scene units. The Duo retains its upstream centimeter hinge rig.", "Replaced the baked marketing screen with a named placeholder for the live framebuffer.", - "Resized textures to at most 1024 pixels and encoded them as WebP; retained body geometry." + "Resized textures to at most 1024 pixels and encoded them as WebP; retained body geometry.", + "Removed parts that no camera direction reaches (hiddenNodes) and pushed the iPhone 16 and 16 Plus glass shells 0.0003 units outward (surfaceOffsets) so they no longer cross the base layer beneath them.", + "Welded vertices and compressed geometry with EXT_meshopt_compression on one scene-wide 16-bit quantization grid, which keeps coincident layers coincident." ] }, "models": [ @@ -18,7 +20,28 @@ "sourceUrl": "https://www.apple.com/105/media/us/iphone-18-pro/2026/591df885-5ee2-4173-86e6-401021249f7c/ar/iphone-18-pro-e-sim.usdz", "sourceSha256": "0f2dac354fbc6505e7917983de460f2cc68e253f96aabf52e9f0468a4803f317", "bodyNode": "UBGArkKGrAMRRnj", - "screenNode": "EHjrGDHNHzgNjtE.001" + "screenNode": "EHjrGDHNHzgNjtE.001", + "hiddenNodes": [ + "AeBMJWvYROmIznQ.001", + "CkWvqHNVFiGJvpB", + "DpHZPtCoplKzsmn.001", + "JLuWQNyAHkVtjBj", + "OKQXnqqXZBuUUit", + "SeCquheRYLOWcow.001", + "ZdjEdUeNrJlHfPr", + "aGDqlYEsfzCGKqA", + "dseYgUIxMhSrccR", + "eMqbNZEirboZmlk", + "gVjDdjCvpxUAQBO.001", + "jBMFTYJdkiLEILT.001", + "kVoChGBtLBcaCVZ.001", + "lrbmLhLqYBEujEb.001", + "pJlLctaTZcBoLFq", + "piknxdJPulwkZlx", + "vSUfuJuwHtQnJFM", + "whMdnAVdLqpcnOU", + "wnKKbrKJSeTpTMG" + ] }, { "id": "iphone-18-pro-max", @@ -26,7 +49,597 @@ "sourceUrl": "https://www.apple.com/105/media/us/iphone-18-pro/2026/591df885-5ee2-4173-86e6-401021249f7c/ar/iphone-18-pro-e-sim.usdz", "sourceSha256": "0f2dac354fbc6505e7917983de460f2cc68e253f96aabf52e9f0468a4803f317", "bodyNode": "wUOhcMgiBmCgGaw", - "screenNode": "EHjrGDHNHzgNjtE" + "screenNode": "EHjrGDHNHzgNjtE", + "hiddenNodes": [ + "AeBMJWvYROmIznQ", + "CkWvqHNVFiGJvpB.001", + "DpHZPtCoplKzsmn", + "JLuWQNyAHkVtjBj.001", + "NWcSDiJMCdsakCS.001", + "OKQXnqqXZBuUUit.001", + "SeCquheRYLOWcow", + "ZdjEdUeNrJlHfPr.001", + "aGDqlYEsfzCGKqA.001", + "dseYgUIxMhSrccR.001", + "eMqbNZEirboZmlk.001", + "gVjDdjCvpxUAQBO", + "jBMFTYJdkiLEILT", + "kVoChGBtLBcaCVZ", + "pJlLctaTZcBoLFq.001", + "piknxdJPulwkZlx.001", + "vSUfuJuwHtQnJFM.001", + "whMdnAVdLqpcnOU.001", + "wnKKbrKJSeTpTMG.001" + ] + }, + { + "id": "iphone-17", + "file": "iphone-17.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone-17/2025/b2c72de3-1cbc-4e24-b4d3-23c7abcec4ec/ar/iphone-17-e-sim.usdz", + "sourceSha256": "2eb04d020c595e12884e95e13b197dab706c13797450b45ca2fa8eefbfa9404b", + "bodyNode": "nVMdepcBMBmnVhZ", + "screenNode": "eXcdWRctfHqyWLU", + "blender": "5.2.2", + "hiddenNodes": [ + "BXAqDKUGaQijVoY", + "CQllheNMGoeRqKm", + "IAlyYUNXoAaMcKp", + "NKEgRFMpMROeZXo", + "PMSPfopQJyYltlK", + "PMjdrRSaQyYJhDP", + "QNuvwNZebttsjIy", + "ROSoyhXuNoyjsaC", + "SHHJlDfCWqBnYVe", + "YAEOwDzKQnyNWXH", + "bVwqhrJsSxSDneW", + "cPDpLFZpaBKrySQ", + "gnfLcUrSrhGfAQN", + "ljnYnfcDcSKbeoi", + "mBDzTBDexXbPzFJ", + "mTQpQGIPKSkyGlL", + "noxRzxFVClfmaMH", + "oNMpyZHDdamXimo", + "paEiHUvTyidQYhd", + "qyGBpiZpuMOaxzH", + "raflvAFabYAQBAS", + "sCxzcmUlHvqqZvX", + "sJcbKiJnntwsJsW", + "tcSskZZpztwdLjs", + "wTyOkkZyiEuMcnJ", + "wnlJMlAHaUPWnoh", + "xhqWHXMGxyLrmJS" + ] + }, + { + "id": "iphone-air", + "file": "iphone-air.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone-air/2025/731189b1-a606-493f-afa4-7c766a8fd08d/ar/iphone-air-e-sim.usdz", + "sourceSha256": "9b7587b91e945c5699a8fbb1c79a3cd07d4d9553d11f2e0b794fddc0fe32d5b6", + "bodyNode": "sePZlgtqCVXFQPm", + "screenNode": "tppzCQmnlKcHIpP", + "excludeNodes": ["avktKcFMCgBNWOI", "IJtRUXYEEXnnqxn"], + "blender": "5.2.2", + "hiddenNodes": [ + "BRwaUtFDoBhnbBz", + "FKcyzYmenRxNJOH", + "JSfOBlDCYhavkEO", + "LBiDPOAbTgirPmb", + "PMfhqlIlcIWpECc", + "UJnodNCUfbPYeLA", + "WHfCUeMXUgzNyib", + "WTfCiWsteDTZHVX", + "WjJwblsyptEpLou", + "XSvupwhOCAgZsvo", + "XgoHZcGVDkgRhQI", + "ZfXbLWjBEnIBiHS", + "ipEzeejaWYZqBeU", + "mHHMXipPTbcFzZc", + "pGosOatMBaTgKun", + "pfTTNPbKodrisVS", + "sFKWIqcsUHSWvYQ", + "sVvKRAEhdpLdUfD", + "zgWhqpHRajMuAlc" + ] + }, + { + "id": "iphone-17e", + "file": "iphone-17e.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone/shared/ar/2026/44fa1631-e6c3-4596-b59b-0d58f7e8735c/iphone-17e/iphone-17e-e-sim-soft-pink.usdz", + "sourceSha256": "e4efca18f6fb2b3c1547d75afec4ab4170838bfe7ccc616c4cd6cdc69a48dc01", + "bodyNode": "TGDVcCHLdxlIePk", + "screenNode": "saxrJLGXKkVuvan", + "blender": "5.2.2", + "hiddenNodes": [ + "AyTOLTKeVVKPQaA", + "FeZbjFZrdQHCQWS", + "IjvetuMImQvHyMS", + "NUfxunQyQNMkMLy", + "NVEnKpJLwrBAGvv", + "NYAlAaEywMxDBeT", + "PuOKtYjtMBPuIdg", + "QIKOrtIVbuxCIDj", + "QvkJsVIpZbNKzrT", + "WXiBrmKnHTYUxce", + "YRVcxEyYUXLHhCA", + "bLMHprzheXSyoLK", + "bbIzWcyqYUXaopV", + "blhkVlSiVMBpaes", + "dJyjKIAUQlaAYMi", + "dnQVZfkgpNitgVM", + "elbmjtSEspZXMRm", + "foxYHnChDpegekQ", + "gnlGsgCggetOerL", + "iZYiKARzCHOZlip", + "lmeCgGUUTrvGHor", + "nLnhXGtvGJhKdxP", + "oXHSEExlpswTXne", + "oahzylvTJzQKhrI", + "pKZJMpxsJwUFnYV", + "rTWCrfRfWKTsucI", + "rYXfMVCgjJIJmTy", + "tnPVVKoEIBvQmBu", + "uupnDToQfRaFWvJ", + "vUmlzFkjLRJXSLc", + "yXecDLjNUYsPNLj" + ] + }, + { + "id": "iphone-17-pro", + "file": "iphone-17-pro.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone-17-pro/2025/704d4474-8e63-4ce7-9917-bb47b1ca4ba0/ar/iphone-17-pro-e-sim.usdz", + "sourceSha256": "18e7f6fbb6f084d6394cedd3e93f9a920ab3f51740e660da6f84f5a7da5e029d", + "bodyNode": "uLzQKMZRvxmFuVz", + "screenNode": "HkNSnYzBPABcqwM", + "blender": "5.2.2", + "hiddenNodes": [ + "BwYqVlkQuMdREsr", + "CUXydfOmpZTOIwn", + "DUMhFmRQRtYyIsa", + "EPzOBwSqqcogiwK", + "FegzLCaANkYSjhw", + "HEpFadyGgDnvrsX", + "HyENgPuODCHFvcE", + "IvdeSiYDweqsnZm", + "KUfjtwUmEyDNVXZ", + "KwGbUIcVQHqwxOo", + "LLPSOlzuFgdAHGd", + "LvSiRAJgYXMPgYk", + "OVqomEMzWqyRLyn", + "PHWKwDZnTREMYuI", + "PjXKYqLWQECXPUd", + "QadnjIWVMhZXiTR", + "QrifvpLHKFTTEsa", + "RhBESHcBbtHIQyo", + "SRHCjsrOCyJnNJF", + "VQJvjrUDkUerjps", + "WOjLDhAMoKnWjRh", + "WaSQjvRMKxQhvQe", + "XMqdcXhDiFUsJct", + "XzQQIMeXTQrDuOE", + "ZfPoPnUTDmjoFQo", + "ZlrVlZDcyrymzYf", + "abBhkuHIWeWxdnv", + "bWNFUbSszjChSpE", + "cTzAYNMxyxGrXkR", + "eKXwwuvnxDrqUCv", + "fWLoYzYzArWJkEs", + "gucycQBzeMjlWnc", + "hdubzDKKYvrSSSW", + "hghphNGYoiYXKSv", + "iZDNdcKAFcuEfFc", + "lAVJNLotEOnEKjC", + "nkejtxZPZMUnCYr", + "nsOllonHmAArGzD", + "pPfvBmJoSbStZtX", + "qIpThkQkOPtihXV", + "sDrQkCDINonvXWK", + "yTmdRacfvebHTTS" + ] + }, + { + "id": "iphone-17-pro-max", + "file": "iphone-17-pro-max.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone-17-pro/2025/704d4474-8e63-4ce7-9917-bb47b1ca4ba0/ar/iphone-17-pro-e-sim.usdz", + "sourceSha256": "18e7f6fbb6f084d6394cedd3e93f9a920ab3f51740e660da6f84f5a7da5e029d", + "bodyNode": "hDqpVcSkjfqLgwJ", + "screenNode": "HkNSnYzBPABcqwM.001", + "blender": "5.2.2", + "hiddenNodes": [ + "BwYqVlkQuMdREsr.001", + "CUXydfOmpZTOIwn.001", + "DUMhFmRQRtYyIsa.001", + "EPzOBwSqqcogiwK.001", + "FegzLCaANkYSjhw.001", + "HEpFadyGgDnvrsX.001", + "HyENgPuODCHFvcE.001", + "IvdeSiYDweqsnZm.001", + "KUfjtwUmEyDNVXZ.001", + "KwGbUIcVQHqwxOo.001", + "LLPSOlzuFgdAHGd.001", + "LvSiRAJgYXMPgYk.001", + "OVqomEMzWqyRLyn.001", + "PHWKwDZnTREMYuI.001", + "PjXKYqLWQECXPUd.001", + "QadnjIWVMhZXiTR.001", + "QrifvpLHKFTTEsa.001", + "RhBESHcBbtHIQyo.001", + "SRHCjsrOCyJnNJF.001", + "VQJvjrUDkUerjps.001", + "WOjLDhAMoKnWjRh.001", + "WaSQjvRMKxQhvQe.001", + "XMqdcXhDiFUsJct.001", + "XzQQIMeXTQrDuOE.001", + "ZfPoPnUTDmjoFQo.001", + "ZlrVlZDcyrymzYf.001", + "abBhkuHIWeWxdnv.001", + "cTzAYNMxyxGrXkR.001", + "eKXwwuvnxDrqUCv.001", + "fWLoYzYzArWJkEs.001", + "gpNYXfBgCggjIbD.001", + "gucycQBzeMjlWnc.001", + "hdubzDKKYvrSSSW.001", + "hghphNGYoiYXKSv.001", + "iFmtcpQJRoqvVtP.001", + "iZDNdcKAFcuEfFc.001", + "lAVJNLotEOnEKjC.001", + "nkejtxZPZMUnCYr.001", + "nsOllonHmAArGzD.001", + "pPfvBmJoSbStZtX.001", + "qIpThkQkOPtihXV.001", + "sDrQkCDINonvXWK.001", + "yTmdRacfvebHTTS.001" + ] + }, + { + "id": "iphone-16-pro", + "file": "iphone-16-pro.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone/shared/ar/2024/1db11f29-1385-4ae5-bbfd-2c885f4463d4/iphone-16-pro/iphone-16-pro-no-sim-desert-titanium.usdz", + "sourceSha256": "bd0cb1e8da9e92ed8b66374ef8f18dab9744ac472d3c4767a2b97bd788f776d1", + "bodyNode": "ZwQgPfwRftWpxGi", + "screenNode": "afaAIHtsSEyJgev", + "blender": "5.2.2", + "hiddenNodes": [ + "BKRlgMrDitCBWeo", + "CNGHYtnKPIWFhHY", + "DIsFDFLTvlwomGm", + "DwbLecuoYyRtBsV", + "GWUtNpLhrifvEGk", + "HBSAZEWpNTEweqF", + "HqcpAscvBvJmcmB", + "KAyEbgGHUVutWFQ", + "MzIBMmsRMBRRWQD", + "NaAZDUdGawgQTCR", + "PAeIfWMBLFULpaw", + "QKMtGrLQHWbIWtU", + "RVGCpadiquVBKyf", + "SUZoZHTRwWdLDfK", + "TCprTPJFwpEPPtJ", + "TJhsNkAuRSiXJqf", + "VdlvjXtKWNQfpyD", + "WrTJCytvBCSdnbi", + "XOysTkZddpkoweQ", + "XSxRtBiMRRaoxLQ", + "YZdsBAkgGJgPEPW", + "YgxngJofPzeyLpf", + "YyficUnUMveyqPa", + "ZisuAWBUTJQsnKo", + "ZoonNbrPFgZgnqb", + "aAWistBepltpskP", + "eJffIOtGKQqcYmi", + "eXNokdwpuhWlKte", + "jOVgTBUXugVBxvo", + "kQYKCRBtkTOgKqJ", + "oIHONYyWwHeWZLQ", + "ovWZczgCEiqxIvF", + "svhJDLoIApkCSxS", + "uGpdljLAmFvrzGT", + "uPzoBLYKILpxSYV", + "wrYwNhCYYWxYMSk", + "yHcJJlPHxjaTgxw" + ] + }, + { + "id": "iphone-16-pro-max", + "file": "iphone-16-pro-max.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone/shared/ar/2024/1db11f29-1385-4ae5-bbfd-2c885f4463d4/iphone-16-pro/iphone-16-pro-max-no-sim-desert-titanium.usdz", + "sourceSha256": "f37cabb2b9aa942e2402d4bdc0daaf416ac32974acadfaaebe43de248bccfd6a", + "bodyNode": "MavDriqPQbzRgmo", + "screenNode": "EnNAOSEIVaSuWpJ", + "blender": "5.2.2", + "hiddenNodes": [ + "BJqnVxTPdSrumBa", + "BtQMQTDGdQUpAfV", + "CgYEYEEkhDTpxZQ", + "DWVddXPTZwJqxRH", + "DYVTLtqMHTsYIjj", + "KBFxhASIpyRODcN", + "LDqFywaUpScBDDA", + "McdqBcDMjwUVXjJ", + "MdAxmZRZiXvBTSO", + "NYJVgOOyDOurKLi", + "OHcIOEYzhYtjZfD", + "OvzFWNzQFhpmAJX", + "TxEgUNigqmCXRUo", + "UspOjqRKFOpussq", + "YZBECLUDGvviBZT", + "YonvKvLHVPxiPhx", + "ZRtlLQRpMvpEADx", + "ZSqPKeYlKPxNEWB", + "aeMcXvvBPXEDaTv", + "cfEKqTzNqGyraVw", + "ctvWVgnrlOtlGZG", + "dhMbwJTWbKBgATj", + "fosjWHinlvDIxtE", + "gnNJMtiCaZKKKEx", + "hrZaEjXDmCILobU", + "isWRMwSjKGqbTVD", + "kIZQXPmeDWRVqtz", + "navdSryPQGEvayo", + "nhhtQijpvnjtShG", + "pRAaKdtJHQYLcIP", + "rXKKjlYsSVqMAUK", + "vPMFwVtdHpyHdbb", + "wDBHeqowZtDZGth", + "yHbaUiTFREbQPQg", + "ylWRHNdlCRPqtsy", + "ztDfNEJyAxWfKpF" + ] + }, + { + "id": "iphone-16", + "file": "iphone-16.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone/shared/ar/2024/1db11f29-1385-4ae5-bbfd-2c885f4463d4/iphone-16/iphone-16-ultramarine-no-sim.usdz", + "sourceSha256": "4653ec827115fb0ab1f49c4b780dfad0b4bc5f07ac0b12851d0a75783f6abd19", + "bodyNode": "FDURRThpGfbMiMt", + "screenNode": "dgZLPrpEoFcPqfL", + "blender": "5.2.2", + "hiddenNodes": [ + "AelcPNzDHXMeLVB", + "DpOgrnwIUfhsbha", + "FdgWEbdiHcyQxnD", + "HucjLITnxhIEqHT", + "JKntyKNQUbpeArP", + "JlvteSBCcKLhDpE", + "MwOoPdXLtGBCdDg", + "NAZHeGhJSqfCgQD", + "NejWyEnZlSIXNEZ", + "OEXYuFSNeTrRNUh", + "PXbaxPhBOLrVtSu", + "PjPOWcVOZxuRXyy", + "QTwFlZwFjSoOcfg", + "QeQWDrevQWqKNSo", + "SHWoLYlgByMlISv", + "SdtrwaqmsLuwKSY", + "SiiUmoZxawgsScA", + "VzbYsWGgucYasFs", + "XgsYaVoTdqpRMXm", + "ZgHsYmuXMmQobxl", + "bwmrzkFHWvFhQMD", + "cvNrDEXvnpqTscR", + "dNdHTDEOmuwzFKI", + "gBQlljzzrEXGscU", + "invkkDmDIhQdjTN", + "kockrpfgHNquDIo", + "lFHmGyRxpomdpLa", + "myXyMrxTsBbqrPf", + "pDxdOfhISwHkTaO", + "pJyDbyiEqPkFezL", + "sOAObXLRufsAEqO", + "uPAkbBhFCtukrSS", + "wOJlkYhoGslVcFa" + ], + "surfaceOffsets": { + "SEyIWFCgLIGSTYP": 0.0003 + } + }, + { + "id": "iphone-16-plus", + "file": "iphone-16-plus.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone/shared/ar/2024/1db11f29-1385-4ae5-bbfd-2c885f4463d4/iphone-16/iphone-16-plus-ultramarine-no-sim.usdz", + "sourceSha256": "6ee9e29573ff1e34b3db9ffda5a9b03c79b4f67bb04d5d68e9ce15bb6ff97c3c", + "bodyNode": "XYhoKEVgwtvKSbJ", + "screenNode": "iRcWrnkfuSKriwn", + "blender": "5.2.2", + "hiddenNodes": [ + "BgfAInYsQNffpOe", + "CWZyezUZFcudkKV", + "CyUKggRzPZooytv", + "GjgPznjGDZHgVuD", + "GpyXtoWAoZhRBJC", + "IqMIWvfCYOsqSls", + "JbWMHsCHGmKfnZJ", + "MKFHImEAIrEsWYf", + "OjIZgpcUMUiqytf", + "PerzyjTYlZefcSB", + "PoykWJUmGzrpOHB", + "PscZFZpVmdsPujt", + "QSSUUCGXdWXvGug", + "SlOMajzxmcTEtgW", + "TIqnvnZZvQBtEFi", + "WfcFvAULKNJuvBM", + "XDmIfDTuKGwgaEW", + "dOqCQxcnRWWpfaG", + "dPmosJCyKPCGeqm", + "fJHvYbWLikDAAfs", + "fhJGLDzJAWelOvi", + "foVFQbooWUAgOqB", + "gKSktMHyZoDIZlP", + "kKUzkQhLjkUkGpa", + "kqVughEnjNUgqpu", + "mNEBtNDQTlIymMZ", + "nQYqmLCjYfMERag", + "qipzModrUtzTWSE", + "rwRdFlRZafiTKYO", + "tDTsNzquPSuDHha", + "uFsvfxZLABofEKl", + "wyvsNpZYmjHmIeq", + "xYSwNbSqAwNdKij" + ], + "surfaceOffsets": { + "rmvqhsGhuVJTHEe": 0.0003 + } + }, + { + "id": "iphone-15", + "file": "iphone-15.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone/shared/ar/2023/f6ca1075-bfa5-441b-b6c2-61e5ce4c15b2/iphone-15/iphone_15_pink_5G.usdz", + "sourceSha256": "9484def099291cd71f1262e01f6db77e7ecb392c50bd5f9cb76aafde05430ca9", + "bodyNode": "WzzPMsuuRaDhtYI", + "screenNode": "afCbrqyMGCECWJr", + "blender": "5.2.2", + "hiddenNodes": [ + "AAKezUdpphfnyyL", + "BaxtCxpdMRmHhGg", + "DhXrccZvNbGfmWw", + "DqnLyvDvAvIxeze", + "GHadUSZvABuphgc", + "KNcGxyCqervjYoY", + "KsyTJDidHqMIQoV", + "MmCDyBSWYfkHtqV", + "NnUPpYbsLPZqdOD", + "RePYTvpawgzGSjN", + "YbnPzAWtmhztfSI", + "brlAILUVKMBAtzg", + "cBKLZeSTnDgltoy", + "dUkqNHvwOkEYZGs", + "fXXQJwTBsyhwBzH", + "lpgeZZNMthIXxGs", + "pjOBZpnZzfFyQNb", + "qNXUYHQkPMzRMYR", + "qwpbyDPuuzuStII", + "tHFVHscQjgmelev", + "tTUIIzmtTKFyfbN", + "tpivdHyZQYerHxD", + "wpJYbYZnIVWmKAT", + "xBdrnzSnovjcqWk", + "xrlXcTfwpjPRksK" + ] + }, + { + "id": "iphone-15-plus", + "file": "iphone-15-plus.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone/shared/ar/2023/f6ca1075-bfa5-441b-b6c2-61e5ce4c15b2/iphone-15/iphone_15_plus_pink_5G.usdz", + "sourceSha256": "252a5e0a8da716b5db4dd4ead91be6d65510a0a66857775eb686229092b83789", + "bodyNode": "tvPjNAzuYItWhqV", + "screenNode": "dWfKekGlHmmeeFl", + "blender": "5.2.2", + "hiddenNodes": [ + "DfipmniQOnzFfID", + "FPQbWDKXeotryLu", + "GLWxoAEBpkHxikz", + "INbkNGjLOruCBjR", + "KyXdmQYHzAAEHsw", + "OORFnZYyMLoGLTD", + "PiAxdOxOesdlfCo", + "RFXEteoIaHctWJE", + "VbjACCvZyJPpjsm", + "XfRfUzxjlXFBKvp", + "XwUtKNawNxblnDU", + "baNWIxGEpxQJxeT", + "bfMKrnaGGNmIxVa", + "cjFtCmgeAFQUVan", + "fzsbWYyApHMXyWO", + "gAPJcRJbkPuPCiw", + "iTLlHgiBSLNcCWm", + "jTrHrcSlvwKXSMW", + "jbKHBmRYQoZTMSG", + "nPjAgNcSmzmDWTN", + "rhQtiJyumboOuEd", + "umtXCxeHrLTWUJv", + "wcjTxrNFajFuoFK", + "xNbVyPPDYNBWjlm", + "yVzdQLLOeiDOHvL" + ] + }, + { + "id": "iphone-14", + "file": "iphone-14.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone/shared/ar/2022/95e52aae-96c0-48d6-87f9-77b3fe97bff9/iphone-14/iphone_14_blue_5G.usdz", + "sourceSha256": "b3289a74dd8e28387b9bf891eff70a057d4a793aecbecc7ba47bd1a6236ab638", + "bodyNode": "NeFCAoxmPzCXcno", + "screenNode": "EWjThtniuhvXaBo", + "blender": "5.2.2", + "hiddenNodes": [ + "BTIcgvzKsLKMosd", + "BwxhSLFofHNVbss", + "EqmHbWlJKWMSaey", + "FvADlZNUEAwEuZR", + "GWOeQenkDWYYkLr", + "GshPkxiwgydWoLA", + "HzrDxmQENsVlvzW", + "IGFzFOQWlTYQsiB", + "MASqCasbplBiJIE", + "MyLGWfozYqgOHHp", + "OZVllIMBPluKfWo", + "VKtPrDosuqyfMnQ", + "VYDsWCPWujxOEdH", + "XByNRgGJnZxIaPP", + "XSAeSqSEcuGpOAi", + "YGqQEkJlppLxUUi", + "eFQzKObsUdpJpQC", + "gzFdxoYImTASnGh", + "hTsQyJAdQXgYXGD", + "hsxYyePyvCSJuhX", + "lWEbVVOLLGQogll", + "mPjzyQxiMNStNdr", + "nYzyUtPOHuewBUQ", + "pqFPKHPVWZZOJvJ", + "pthsXDHplKhubsb", + "sFwUfzbnYNUtjbC", + "sfRhnaLgPXjIaGT", + "tDUfslZfubZhiWH", + "wJIflIsbEqElscI", + "xsDXEApOdCRwZjQ", + "yChqqVaFtavuzPB", + "yrdrbsmijZMUwWW" + ] + }, + { + "id": "iphone-14-plus", + "file": "iphone-14-plus.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone/shared/ar/2022/95e52aae-96c0-48d6-87f9-77b3fe97bff9/iphone-14/iphone_14_plus_blue_5G.usdz", + "sourceSha256": "007561dc8a960ac8d2fbe0f24f2e7277642cb970bb901ac00678cb308570ed72", + "bodyNode": "fFXaBaQnFOdPBzq", + "screenNode": "uRcvOoIfAvSAPlM", + "blender": "5.2.2", + "hiddenNodes": [ + "DnRtqBJzFvxTXIx", + "FakJjUkMpCtrtdY", + "GiExdGgtpgPmcqk", + "IaKrkjNdaSyNAzd", + "JuwgIVSdYjLsAjX", + "MANYOcgqahYyxLG", + "MNdSNvsQrxnamPu", + "NgDBYxxHlWKJUtq", + "PNqPDALVWpmjIba", + "RDCpNyzRtFsrsDp", + "SmjDMSVEsPcivtn", + "SwrfvTAKyTIKFYQ", + "TjghSMwVYlYAPPq", + "VWbgsqziWJAYHzf", + "WUziwOmUgekiVyH", + "YWuMyedyfbSzJts", + "absnnGkzkksMgsA", + "asZVnOKRncBUfVj", + "cXQXmlQoAMgHeES", + "eRJzJGQcfehIURt", + "fFAOGOxMdrJyooh", + "flUWVfVaZfPgcZZ", + "fyvSvEYWQHUoXnb", + "gIwnAvBycCvNBaH", + "jKYufNEuYWGvlXC", + "jrfAEsBZpTkVaZd", + "mqcakHHnRxrOJoT", + "pkFlaKESqBILSkK", + "qHqUlAFcOxVLuUK", + "rKIAhcQyXXnWkgJ", + "rftVsroCwPACnNa", + "uWwcCjFiPTLSTJu", + "voHhULZOwcBrcjJ", + "xRLbxFVpaymptoK", + "xuXFJHShTBtUXcg", + "yUZEynNRKWmxSdL" + ] }, { "id": "ipad-pro-13-m5", @@ -34,7 +647,26 @@ "sourceUrl": "https://www.apple.com/105/media/us/ipad-pro/2025/adee90db-c01e-430d-b726-fe64c0063f08/ar/ipad-pro-space-black.usdz", "sourceSha256": "48098daf7c49117c81a4e4e1808899b93fad081faeba6c8b65124946b83b5ac6", "bodyNode": "zRrSLDpdYmKeRJQ", - "screenNode": "lsDiIbtoSGSmWWZ" + "screenNode": "lsDiIbtoSGSmWWZ", + "hiddenNodes": [ + "GZNddUnQRPabSxX", + "HOqFlitkFSEBwpB", + "HnpMSupcdGUIALF", + "IuGjRlDNiDKosQT", + "JdDdnFOwsHEqDKC", + "TPYVYyGxXgljGAJ", + "TYYjgzSzdQIBkkK", + "XptMSkTgejChrmH", + "ZnyZAcqoWZBGJim", + "cbyESvRKYknMBVA", + "eitnudLTYwnbaHa", + "fSBTwwPRvXpskFk", + "mriWlabCVLUskzX", + "pMtRErDuwchkoHE", + "sDgzpYIricXgwnM", + "uUyywRZMbjjlkpt", + "vZbnexKRTYsKnTd" + ] }, { "id": "ipad-pro-13-m5-magic-keyboard", @@ -43,7 +675,8 @@ "sourceSha256": "48098daf7c49117c81a4e4e1808899b93fad081faeba6c8b65124946b83b5ac6", "bodyNode": "zRrSLDpdYmKeRJQ", "screenNode": "lsDiIbtoSGSmWWZ", - "accessoryNode": "PoBqSMmyhhcJsBX" + "accessoryNode": "PoBqSMmyhhcJsBX", + "hiddenNodes": ["ANhlWSbARuEZHbf", "EmssyDjcNaAGWGe", "SLpVcmzCKSgySiu", "kNMnscxwbDEFIWr"] }, { "id": "iphone-duo", diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index 50b5ae47c449b..efcf8dab21e5d 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -401,6 +401,7 @@ function DeviceMiniPlayer({ deviceId={source.deviceId} hostId={source.hostId} deviceName={device?.name ?? source.name} + deviceModel={device?.model} deviceDescription={`${hostLabel} · ${device?.version ?? source.platform}`} visible viewId={viewId} diff --git a/docs/internals/devices.md b/docs/internals/devices.md index 5273f7942e9e0..29d467674470b 100644 --- a/docs/internals/devices.md +++ b/docs/internals/devices.md @@ -156,3 +156,29 @@ hubs that serve it pay nothing, and switches on that 409, or when its MJPEG image fails in a browser that has `RTCPeerConnection`. Offers are not trickled: serve-sim takes one offer with every candidate. The video element only feeds the same canvas sink, so the panel and the mobile WebView keep one surface. + +## 3D device models + +The GLBs under `apps/web/src/components/device/models` are generated. The +record is [`sources.json`](../../apps/web/src/components/device/models/sources.json) +plus [`convert-device-model.py`](../../scripts/convert-device-model.py), whose +header lists the gltf-transform steps that follow the Blender export. + +Apple's AR assets build glass and finishes from stacked shells. Where a shell +is an exact copy of the layer beneath it, depth ties resolve the same way every +frame. Where it crosses that layer, as the iPhone 16 and 16 Plus back glass +does, WebGL draws a flickering patchwork, so `surfaceOffsets` pushes that shell +0.0003 units outward. Quantization has the same failure mode: per-mesh grids +round coincident layers apart, so meshopt runs on one scene-wide 16-bit grid. +Do not simplify: even a 1e-5 error changes rim highlights and seams the Duo's +inner displays. + +`hiddenNodes` lists parts that never change a pixel when hidden, rendered from +26 directions with a logarithmic depth buffer. Parts that are only nearly +hidden stay. The Duo has none listed because its hinge pose changes what is +visible. + +Display meshes are quantized like everything else. +[`modelScene.ts`](../../packages/client-runtime/src/device/modelScene.ts) +bakes their node transforms back into float geometry at load, because screen +UVs, touch projection, and the normalization checks work in model space. diff --git a/docs/user/devices.md b/docs/user/devices.md index ce5a7d8d06d05..737c5474ece11 100644 --- a/docs/user/devices.md +++ b/docs/user/devices.md @@ -48,10 +48,12 @@ unless you power it off. Closed tabs stay closed after a reload. To watch the device again, choose it from **+ → Device**. Choose **3D view** to inspect supported devices while the live screen stays -interactive. On iPhone Duo, use the fold and stance controls to change its -physical pose, or pinch over the device to adjust the hinge. Turning the model -to the other screen switches the live display and touch input to that screen. -**Restore 3D view** returns the device to a screen-facing position. +interactive. The 3D body follows the simulator's device type, so a renamed +simulator still looks like its hardware. On iPhone Duo, use the fold and stance +controls to change its physical pose, or pinch over the device to adjust the +hinge. Turning the model to the other screen switches the live display and touch +input to that screen. **Restore 3D view** returns the device to a screen-facing +position. On supported Android foldables, use **Fold device** and **Unfold device** beside the screen to change its posture in either view. diff --git a/packages/client-runtime/src/device/model.test.ts b/packages/client-runtime/src/device/model.test.ts index dd01055e2edba..6cfa8f879ef0b 100644 --- a/packages/client-runtime/src/device/model.test.ts +++ b/packages/client-runtime/src/device/model.test.ts @@ -1,5 +1,5 @@ import { expect, it, vi } from "vite-plus/test"; -import { createDeviceModelSlot, type DeviceModelSource } from "./model.ts"; +import { createDeviceModelSlot, resolveDeviceModelId, type DeviceModelSource } from "./model.ts"; const pro: DeviceModelSource = { id: "iphone-18-pro", url: "/pro.glb" }; const max: DeviceModelSource = { id: "iphone-18-pro-max", url: "/max.glb" }; @@ -96,3 +96,17 @@ it("releases an invalid model and restores the fallback if scene preparation rej expect(install).toHaveBeenLastCalledWith(recovered); slot.dispose(); }); + +it.each([ + ["ios", "iPhone 18 Pro", "iphone-18-pro"], + ["ios", "iPhone 17 Pro Max", "iphone-17-pro-max"], + ["ios", "iPhone 15 Pro", "iphone-16-pro"], + ["ios", "iPhone 16e", "iphone-17e"], + ["ios", "iPad Pro 13-inch (M5) (16GB)", "ipad-pro-13-m5"], + ["ios", "iphone air", "iphone-air"], + ["ios", "iPhone 13 Pro", null], + ["ios", "iPhone (clean)", null], + ["android", "iPhone 18 Pro", null], +] as const)("maps %s %s to %s", (platform, name, id) => { + expect(resolveDeviceModelId(platform, name)).toBe(id); +}); diff --git a/packages/client-runtime/src/device/model.ts b/packages/client-runtime/src/device/model.ts index 4036c2d741abc..08bcb23be8e1a 100644 --- a/packages/client-runtime/src/device/model.ts +++ b/packages/client-runtime/src/device/model.ts @@ -1,6 +1,28 @@ import type { DevicePlatform } from "@t3tools/contracts"; -export type DeviceModelId = "iphone-18-pro" | "iphone-18-pro-max" | "ipad-pro-13-m5" | "iphone-duo"; +const MODEL_NAMES = { + "iphone-duo": ["iPhone Duo"], + "iphone-18-pro": ["iPhone 18 Pro"], + "iphone-18-pro-max": ["iPhone 18 Pro Max"], + "iphone-17-pro": ["iPhone 17 Pro"], + "iphone-17-pro-max": ["iPhone 17 Pro Max"], + "iphone-17": ["iPhone 17"], + "iphone-air": ["iPhone Air"], + // The 16e shares the 17e's body and display. + "iphone-17e": ["iPhone 17e", "iPhone 16e"], + // The 14 Pro and 15 Pro share the 16 Pro's square triple-camera back and display aspect. + "iphone-16-pro": ["iPhone 16 Pro", "iPhone 15 Pro", "iPhone 14 Pro"], + "iphone-16-pro-max": ["iPhone 16 Pro Max", "iPhone 15 Pro Max", "iPhone 14 Pro Max"], + "iphone-16": ["iPhone 16"], + "iphone-16-plus": ["iPhone 16 Plus"], + "iphone-15": ["iPhone 15"], + "iphone-15-plus": ["iPhone 15 Plus"], + "iphone-14": ["iPhone 14"], + "iphone-14-plus": ["iPhone 14 Plus"], + "ipad-pro-13-m5": ["iPad Pro 13-inch (M5)", "iPad Pro 13-inch (M4)"], +} as const; + +export type DeviceModelId = keyof typeof MODEL_NAMES; export interface DeviceModelSource { readonly id: DeviceModelId; @@ -15,13 +37,16 @@ export interface DeviceAccessorySource { export type DeviceAssetSource = DeviceModelSource | DeviceAccessorySource; +const modelByName = new Map( + Object.entries(MODEL_NAMES).flatMap(([id, names]) => + names.map((name) => [name.toLowerCase(), id as DeviceModelId] as const), + ), +); + +/** Matches a hardware name; simulator types can add a memory suffix such as " (16GB)". */ export function resolveDeviceModelId(platform: DevicePlatform, name: string): DeviceModelId | null { if (platform !== "ios") return null; - if (/^iPhone Duo$/i.test(name)) return "iphone-duo"; - if (/^iPhone 18 Pro Max$/i.test(name)) return "iphone-18-pro-max"; - if (/^iPhone 18 Pro$/i.test(name)) return "iphone-18-pro"; - if (/^iPad Pro 13-inch \(M5\)$/i.test(name)) return "ipad-pro-13-m5"; - return null; + return modelByName.get(name.replace(/ \(\d+GB\)$/i, "").toLowerCase()) ?? null; } /** Each request owns its result. A cancelled or superseded parse must still release its resources. */ diff --git a/packages/client-runtime/src/device/modelScene.test.ts b/packages/client-runtime/src/device/modelScene.test.ts index 40efa6e5a5472..775ce65d2ca71 100644 --- a/packages/client-runtime/src/device/modelScene.test.ts +++ b/packages/client-runtime/src/device/modelScene.test.ts @@ -1,5 +1,6 @@ import { BoxGeometry, + BufferAttribute, Group, Mesh, MeshBasicMaterial, @@ -8,7 +9,11 @@ import { Texture, } from "three"; import { expect, it, vi } from "vite-plus/test"; -import { createImportedPhoneScene, disposeDeviceModel } from "./modelScene.ts"; +import { + bakeDisplayTransforms, + createImportedPhoneScene, + disposeDeviceModel, +} from "./modelScene.ts"; import { phoneDisplayLayout } from "./phoneScene.ts"; function asset() { @@ -68,3 +73,23 @@ it("rejects missing and incorrectly normalized displays before changing their ma disposeDeviceModel(root); texture.dispose(); }); + +it("accepts a quantized display once loading bakes its node transform", () => { + const { root, screen } = asset(); + // KHR_mesh_quantization stores normalized shorts and moves their scale and offset onto the node. + const position = screen.geometry.getAttribute("position"); + const shorts = new Int16Array(position.count * 3); + for (let i = 0; i < position.count; i++) + for (let c = 0; c < 3; c++) + shorts[i * 3 + c] = Math.round( + ((position.getComponent(i, c) - (c === 2 ? 0.043 : 0)) / 1.1) * 32767, + ); + screen.geometry.setAttribute("position", new BufferAttribute(shorts, 3, true)); + screen.position.z = 0.043; + screen.scale.setScalar(1.1); + bakeDisplayTransforms(root); + expect(screen.geometry.getAttribute("position").array).toBeInstanceOf(Float32Array); + const phone = createImportedPhoneScene(root, new Texture(), phoneDisplayLayout(null, 1206, 2622)); + expect(screen.geometry.getAttribute("uv").getX(0)).toBeCloseTo(0, 3); + phone.dispose(); +}); diff --git a/packages/client-runtime/src/device/modelScene.ts b/packages/client-runtime/src/device/modelScene.ts index 3354cb1a31c6c..06a4a03e7e870 100644 --- a/packages/client-runtime/src/device/modelScene.ts +++ b/packages/client-runtime/src/device/modelScene.ts @@ -1,9 +1,47 @@ // @effect-diagnostics globalFetch:off - This browser-only renderer owns abortable asset requests without an Effect runtime, like the live stream. -import { Box3, Group, Mesh, MeshBasicMaterial, Texture, type Material, type Object3D } from "three"; +import { + Box3, + BufferAttribute, + Group, + Mesh, + MeshBasicMaterial, + Texture, + type Material, + type Object3D, +} from "three"; import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; +import { MeshoptDecoder } from "three/addons/libs/meshopt_decoder.module.js"; import type { DeviceAssetSource } from "./model.ts"; import { createDisplayProjection, updateDisplayUv, type PhoneDisplayLayout } from "./phoneScene.ts"; +const DISPLAY_MESHES = new Set([ + "device-screen", + "cover-display", + "inner-display-left", + "inner-display-right", +]); + +// Quantized glTF keeps positions in a node-local box; the screen math needs model-space floats. +export function bakeDisplayTransforms(root: Object3D) { + root.traverse((object) => { + if (!(object instanceof Mesh) || !DISPLAY_MESHES.has(object.name)) return; + const geometry = object.geometry; + for (const name of Object.keys(geometry.attributes)) { + const attribute = geometry.getAttribute(name); + const values = new Float32Array(attribute.count * attribute.itemSize); + for (let i = 0; i < attribute.count; i++) + for (let c = 0; c < attribute.itemSize; c++) + values[i * attribute.itemSize + c] = attribute.getComponent(i, c); + geometry.setAttribute(name, new BufferAttribute(values, attribute.itemSize)); + } + object.updateMatrix(); + geometry.applyMatrix4(object.matrix); + object.position.set(0, 0, 0); + object.quaternion.identity(); + object.scale.set(1, 1, 1); + }); +} + /** GLB resources belong to one viewer; the live framebuffer texture belongs to its stream presentation. */ export function disposeDeviceModel(root: Object3D) { const geometries = new Set(); @@ -105,10 +143,10 @@ export async function loadDeviceModel(source: DeviceAssetSource, signal: AbortSi if (!response.ok) throw new Error(`Device model request failed: ${response.status}`); const data = await response.arrayBuffer(); signal.throwIfAborted(); - const gltf = await new GLTFLoader().parseAsync( - data, - new URL(".", new URL(source.url, globalThis.location.href)).href, - ); + const gltf = await new GLTFLoader() + .setMeshoptDecoder(MeshoptDecoder) + .parseAsync(data, new URL(".", new URL(source.url, globalThis.location.href)).href); + bakeDisplayTransforms(gltf.scene); if (signal.aborted) { disposeDeviceModel(gltf.scene); signal.throwIfAborted(); diff --git a/packages/contracts/src/device.ts b/packages/contracts/src/device.ts index 24b72dbe26d7a..4353164846566 100644 --- a/packages/contracts/src/device.ts +++ b/packages/contracts/src/device.ts @@ -173,6 +173,8 @@ export const DeviceSummary = Schema.Struct({ id: DeviceId, platform: DevicePlatform, name: TrimmedNonEmptyString, + /** Hardware product name, such as "iPhone 18 Pro", which a renamed simulator keeps. */ + model: Schema.optional(Schema.String), version: Schema.String, booted: Schema.Boolean, physical: Schema.Boolean, diff --git a/scripts/convert-device-model.py b/scripts/convert-device-model.py index cba2d9c98ed43..e9afecd44ccc7 100644 --- a/scripts/convert-device-model.py +++ b/scripts/convert-device-model.py @@ -1,7 +1,9 @@ # Run with Blender 4.5.9: # blender --background --python scripts/convert-device-model.py -- # Then optimize using @gltf-transform/cli 4.2.1 with --compress false --simplify false -# --join false --palette false --flatten false --texture-size 1024 --texture-compress webp. +# --join false --palette false --flatten false --texture-size 1024 --texture-compress webp, +# then `weld` and `meshopt --quantization-volume scene --quantize-position 16`. Per-mesh grids +# move coincident layers apart and z-fight; see docs/internals/devices.md#3d-device-models. import hashlib import json import sys @@ -28,6 +30,9 @@ objects = [o for o in body.children_recursive if o.type == "MESH"] if body.type == "MESH": objects.append(body) +# Some sources stack every colour variant's back glass; keep one. Hidden nodes never reach a pixel. +skipped = set(model.get("excludeNodes", [])) | set(model.get("hiddenNodes", [])) +objects = [o for o in objects if o.name not in skipped] uv = screen.data.uv_layers.active points = [] coords = [] @@ -61,6 +66,12 @@ o.data.transform(matrix) # Blender is Z-up; glTF export converts to Y-up. Store canonical Y-up geometry as Blender Z-up. o.data.transform(Matrix.Rotation(np.pi / 2, 4, "X")) + # A shell that crosses the layer beneath it z-fights in WebGL; keep it in front. + offset = model.get("surfaceOffsets", {}).get(o.name) + if offset: + normals = [v.normal.copy() for v in o.data.vertices] + for v, n in zip(o.data.vertices, normals): + v.co += n * offset if o == screen: o.name = "device-screen" o.data.name = "device-screen"