From 9e473932e889866ca729420ecb3dd2576674e43b Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 27 Oct 2026 01:40:49 -0700 Subject: [PATCH 01/21] feat(evals): mobile design eval with scripted showcase runs Co-Authored-By: Claude Opus 5.5 --- evals/README.md | 1 + evals/mobile-design/README.md | 51 +++ evals/mobile-design/brief.md | 56 +++ evals/mobile-design/capture-desktop.ts | 127 +++++++ evals/mobile-design/capture-mobile.ts | 288 +++++++++++++++ evals/mobile-design/cases.ts | 366 +++++++++++++++++++ evals/mobile-design/ios.ts | 367 +++++++++++++++++++ evals/mobile-design/pack.ts | 250 +++++++++++++ evals/mobile-design/references/README.md | 21 ++ evals/mobile-design/run.test.ts | 51 +++ evals/mobile-design/run.ts | 197 ++++++++++ evals/mobile-design/scores.ts | 92 +++++ evals/package.json | 4 +- knip.jsonc | 2 + pnpm-lock.yaml | 3 + scripts/mobile-showcase-environment.ts | 406 +++++++++++++++++++++ scripts/mobile-showcase-provider.ts | 437 +++++++++++++++++++++++ scripts/mobile-showcase.ts | 58 ++- 18 files changed, 2757 insertions(+), 20 deletions(-) create mode 100644 evals/mobile-design/README.md create mode 100644 evals/mobile-design/brief.md create mode 100644 evals/mobile-design/capture-desktop.ts create mode 100644 evals/mobile-design/capture-mobile.ts create mode 100644 evals/mobile-design/cases.ts create mode 100644 evals/mobile-design/ios.ts create mode 100644 evals/mobile-design/pack.ts create mode 100644 evals/mobile-design/references/README.md create mode 100644 evals/mobile-design/run.test.ts create mode 100644 evals/mobile-design/run.ts create mode 100644 evals/mobile-design/scores.ts create mode 100644 scripts/mobile-showcase-provider.ts diff --git a/evals/README.md b/evals/README.md index 1aa5796b00378..839ed73d2121c 100644 --- a/evals/README.md +++ b/evals/README.md @@ -9,6 +9,7 @@ Write or run an eval when a change affects what a model sees or does: agent inst | [mcp-tool-use](mcp-tool-use/) | Whether agents use T3's MCP tools correctly, including device routing and recovery, in fake threads. | | [thread-titles](thread-titles/) | Whether generated thread titles name the right subject. | | [test-signal](test-signal/) | Whether a cheap model running the test-signal check flags low-signal tests. | +| [mobile-design](mobile-design/) | How a model council scores the iOS app's design. | ## Run an eval diff --git a/evals/mobile-design/README.md b/evals/mobile-design/README.md new file mode 100644 index 0000000000000..ae3cb6495bbc5 --- /dev/null +++ b/evals/mobile-design/README.md @@ -0,0 +1,51 @@ +# mobile-design + +Scores the design of the T3 Code iOS app on iPhone, iPhone Duo and iPad. Code cannot judge design, so a council of models scores screenshots. This eval makes those screenshots the same way every round, and writes the brief that every round asks. + +A run captures: + +- **Scenes**: every screen in `SCENES` (`cases.ts`), on each pose in `POSES`. The poses are iPhone 17 Pro dark and light, the largest accessibility text size and landscape; iPhone Duo closed and open; and iPad Pro 13 in landscape and portrait. Dark poses capture the full set; light poses capture a smaller set. A scene's `source` names the complaint it guards, when there is one. +- **Desktop pairs**: the same data in the desktop web app at 1440×900, next to the iPad and iPhone captures. +- **Flows**: common tasks walked step by step on the dark iPhone, with the taps and distinct screens each took. + +The data comes from the showcase fixture (`scripts/mobile-showcase-environment.ts`): three environments, seeded threads, and live threads run by a scripted Codex provider (`scripts/mobile-showcase-provider.ts`). That provider is how a capture shows a stream in progress, tool calls, an approval, a question and background work without a real model. + +## Run it + +Mobile capture needs macOS with Xcode, and a simulator runtime with the iPhone Duo device type (Xcode 27.1 or later). Desktop capture needs the `shot` skill. Run both on the same Mac: on Linux, `/etc/machine-info` names the machine in the desktop captures. + +1. From the repo root, run `vp i`. +2. From `evals/`, capture and build the pack: + + ```bash + pnpm mobile-design --out runs/ --capture mobile --capture desktop + ``` + + The first run builds the showcase app, which takes about 10 minutes. Add `--skip-build` to reuse it. The capture creates its own simulators, named `T3 Design …`, and deletes them when it finishes. + +3. Run the council. The run writes `runs//council.json` with the brief and the panel: Claude Opus 5.5 and Codex GPT-6.1 Sol, both at `xhigh`. Give each member the brief at `runs//pack/brief.md`, read-only, with one `delegate_task` each. Add `--panel opus,sol,muse` to add Muse Wonder Full for rounds about copy. +4. Save each reply as `runs//scores/.md`, then: + + ```bash + pnpm mobile-design --out runs/ --summarize --baseline runs/ + ``` + + `summary.md` has the median per axis and device class, each reviewer's scores, the change against the baseline, and each reviewer's top five fixes. + +Other flags: `--pose ` and `--case ` capture a subset, `--no-flows` skips the flows, `--pack` rebuilds the pack from earlier captures. `captures/captures.json` lists any scene that did not reach its ready state; look at those screenshots before you send the pack. + +## Blind review + +The pack holds only screenshots, the brief and the flow table. The commit under review is in `run.json`, outside the pack. Do not tell reviewers which build is new. + +## Add a scene or flow + +Add it to `SCENES` or `FLOWS` in `cases.ts`, with its `source`. Steps open deep links (`t3code://…`) or tap by accessibility label. When a scene needs a new state, add it to the fixture: a seeded thread in `SHOWCASE_THREADS`, or a live thread in `SHOWCASE_LIVE_THREADS` with the steps the scripted provider plays. + +## Reference screenshots + +`references/README.md` lists the Codex and Claude iOS screenshots the comparison axis needs. A maintainer captures them on a device; every image in that folder goes into the pack. + +## Time and cost + +A full capture on an M-series Mac takes about 45 minutes: about 15 minutes per device class, plus desktop pairs. A council round is two `xhigh` reviews of about 80 images. diff --git a/evals/mobile-design/brief.md b/evals/mobile-design/brief.md new file mode 100644 index 0000000000000..cdd86f81bcad6 --- /dev/null +++ b/evals/mobile-design/brief.md @@ -0,0 +1,56 @@ +# Design review: T3 Code for iPhone, iPhone Duo and iPad + +You are a senior product designer. Review the T3 Code mobile app from the screenshots in this pack. This is a read-only review. Do not edit files, start servers, or open browsers or simulators. + +T3 Code is a client for coding agents. People use the phone and tablet apps to watch agents work on their own machines, answer the agent's questions, approve commands, review diffs and start new work. The desktop app is the reference for the product's look. The mobile app should feel like the same product, adapted to touch and to each screen size. + +## What is in the pack + +Every image is in `{{PACK}}`. Read each one before you score. + +- `screens//.png`: one scene on one device shape. Each tile is labelled with its device, appearance and text size. +- `desktop/.png`: the same data in the desktop web app at 1440×900, next to the iPad and iPhone captures, for the "conformance with desktop" axis. +- `flows/.png`: a common task, one frame per step. The label above each frame names the action that led to it. + +The data is a fixture. Thread titles, projects and machine names are made up, and project paths point at temporary folders. Do not score the fixture's wording or paths. Do score how the app lays them out, for example how it truncates a long path. + +Screens: + +{{SCREENS}} + +Tasks, with the taps and distinct screens each one took: + +{{FLOWS}} + +References for the Codex and Claude iOS apps: + +{{REFERENCES}} + +## How to score + +Score each axis from 1 to 10 for each device class. 10 is the best mobile app you know for this kind of work. 7 is solid but clearly behind the best. + +Axes: + +{{AXES}} + +Device classes: + +{{DEVICES}} + +- **Visual design**: type, color, icons, spacing and hierarchy. +- **Layout quality**: use of the screen, safe areas, the keyboard, folds and rotation, large text. +- **Flow clarity**: how obvious each task is, from the flows and from what each screen offers next. +- **Polish**: alignment, truncation, clipped or overlapping content, empty space, state indicators. +- **Conformance with desktop**: whether this reads as the same product as the desktop app. +- **Quality versus Codex and Claude**: how this compares with the Codex and Claude iOS apps. + +Then list the five fixes that would raise the scores most, most important first. Each fix names the screen and device, what is wrong, and the change you would make. Be specific: "the question panel leaves a 120 pt scroll area on the closed Duo; let it take the space above the composer" is useful, "improve spacing" is not. + +## Reply format + +Write your review in plain prose and short lists. End the reply with exactly one fenced `json` block in this shape, with your numbers and your five fixes: + +```json +{{JSON}} +``` diff --git a/evals/mobile-design/capture-desktop.ts b/evals/mobile-design/capture-desktop.ts new file mode 100644 index 0000000000000..670afd022f59e --- /dev/null +++ b/evals/mobile-design/capture-desktop.ts @@ -0,0 +1,127 @@ +// @effect-diagnostics nodeBuiltinImport:off - Host-side capture uses Node subprocess and filesystem APIs directly. + +import type * as NodeChildProcess from "node:child_process"; +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import { + issuePairingCredential, + startShowcaseEnvironments, + stopProcess, +} from "../../scripts/mobile-showcase.ts"; +import { SHOWCASE_ENVIRONMENTS } from "../../scripts/mobile-showcase-environment.ts"; +import { SCENES, type SceneId } from "./cases.ts"; +import { expandPath, linkTargets, run } from "./ios.ts"; + +const REPO_ROOT = NodePath.resolve(import.meta.dirname, "../.."); +const WEB_ROOT = NodePath.join(REPO_ROOT, "apps/web"); +const VIEWPORT = "1440x900"; +// Dark mode is the primary look; the web app reads it from these keys before it renders. +const DARK_MODE = [ + ["t3code:theme", "dark"], + ["t3code:theme-appearance-mode", "dark"], + ["t3code:theme-follow-system", "false"], +] + .map(([key, value]) => `localStorage.setItem(${JSON.stringify(key)}, ${JSON.stringify(value)})`) + .join("; "); + +export interface DesktopShot { + readonly scene: SceneId; + readonly file: string; + readonly problem?: string; +} + +/** The shot skill's CLI; set SHOT or pass --shot when it lives elsewhere. */ +export async function findShot(explicit: string | undefined): Promise { + const home = NodeOS.homedir(); + const candidates = [ + explicit, + process.env.SHOT, + NodePath.join(home, ".claude/skills/shot/scripts/shot"), + NodePath.join(home, ".codex/skills/shot/scripts/shot"), + ].filter((value): value is string => Boolean(value)); + for (const candidate of candidates) { + if ( + await NodeFSP.access(candidate).then( + () => true, + () => false, + ) + ) + return candidate; + } + return null; +} + +/** + * Captures the desktop half of each "conforms to desktop" pair: the built web client, served by + * the same seeded environments the phone uses, at a desktop window size. + */ +export async function captureDesktop(input: { + readonly outDirectory: string; + readonly shot: string; + readonly scenes: ReadonlySet | null; + readonly skipBuild: boolean; + readonly log: (message: string) => void; +}): Promise> { + const built = await NodeFSP.access(NodePath.join(WEB_ROOT, "dist/index.html")).then( + () => true, + () => false, + ); + if (!input.skipBuild || !built) { + input.log("building the web client"); + await run("vp", ["build"], { cwd: WEB_ROOT, timeoutMs: 900_000 }); + } + const rootDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-design-web-")); + const servers: NodeChildProcess.ChildProcess[] = []; + const shots: DesktopShot[] = []; + try { + input.log("starting the fixture environments"); + const environments = await startShowcaseEnvironments( + rootDir, + (server) => servers.push(server), + SHOWCASE_ENVIRONMENTS, + { liveThreads: true }, + ); + const targets = linkTargets(environments); + for (const scene of SCENES) { + if (!("desktop" in scene) || (input.scenes && !input.scenes.has(scene.id))) continue; + // A thread opens on the environment that owns it; other pages use the first one. + const threadId = /\{thread:([^}]+)\}/.exec(scene.desktop.path)?.[1]; + const owner = threadId ? targets.get(threadId)?.environmentId : undefined; + const environment = + environments.find((candidate) => candidate.environmentId === owner) ?? environments[0]!; + const path = `/${expandPath(scene.desktop.path.slice(1), targets)}`; + const credential = await issuePairingCredential(environment.baseDir); + const file = NodePath.join("captures", "desktop", `${scene.id}.png`); + await NodeFSP.mkdir(NodePath.join(input.outDirectory, "captures", "desktop"), { + recursive: true, + }); + const result = await run( + input.shot, + [ + `http://127.0.0.1:${environment.port}/pair#token=${credential}`, + "--size", + VIEWPORT, + "--sleep", + "2500", + "--eval", + `${DARK_MODE}; location.assign(${JSON.stringify(path)})`, + "--sleep", + "4000", + ...("click" in scene.desktop ? ["--click", scene.desktop.click, "--sleep", "1500"] : []), + "--out", + NodePath.join(input.outDirectory, file), + ], + { allowFailure: true, timeoutMs: 120_000 }, + ); + const problem = result.code === 0 ? undefined : result.stderr.trim().split("\n")[0]; + if (problem) input.log(`desktop/${scene.id}: ${problem}`); + shots.push({ scene: scene.id, file, ...(problem ? { problem } : {}) }); + } + } finally { + await Promise.all(servers.map(stopProcess)); + await NodeFSP.rm(rootDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + return shots; +} diff --git a/evals/mobile-design/capture-mobile.ts b/evals/mobile-design/capture-mobile.ts new file mode 100644 index 0000000000000..171311f594f63 --- /dev/null +++ b/evals/mobile-design/capture-mobile.ts @@ -0,0 +1,288 @@ +// @effect-diagnostics nodeBuiltinImport:off globalTimers:off globalDate:off - Host-side simulator automation uses Node subprocess and timing APIs directly. + +import type * as NodeChildProcess from "node:child_process"; +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import sharp from "sharp"; + +import { + buildIos, + buildShowcasePairingUrl, + ensureIosFullScreenAppsMode, + IOS_APP_PATH, + issuePairingCredential, + reserveAvailablePort, + startShowcaseEnvironments, + startShowcaseMetro, + stopProcess, + waitForPort, +} from "../../scripts/mobile-showcase.ts"; +import { + SHOWCASE_ENVIRONMENTS, + SHOWCASE_PROJECTS, +} from "../../scripts/mobile-showcase-environment.ts"; +import { FLOWS, SCENES, type DeviceClassId, type Flow, type Pose, type SceneId } from "./cases.ts"; +import { delay, linkTargets, type LinkTargets, run, Simulator } from "./ios.ts"; + +/** One screenshot of the run, as `captures/mobile.json` records it. */ +export interface MobileShot { + readonly pose: string; + readonly scene: SceneId; + readonly file: string; + /** Set when the scene did not reach its ready state; the screenshot shows where it stopped. */ + readonly problem?: string; +} + +export interface FlowResult { + readonly flow: string; + readonly taps: number; + readonly screens: number; + readonly frames: ReadonlyArray<{ readonly file: string; readonly action: string }>; + readonly problem?: string; +} + +const THREAD_LIST = /^Plan offline mode/; +const LONG_PROMPT = `Rework the handoff from desktop to phone so it feels like one continuous session. + +1. When I pick up the phone, open the thread I last touched on the desktop, scrolled to the same message, with the composer draft carried over. +2. Keep the terminal attached: same buffer, same scroll position, and the cursor where I left it. +3. If the desktop is mid-turn, show the live stream instead of a spinner, and let me stop or steer from the phone. +4. Never lose a draft. If both devices edit the same draft, keep both and ask which one wins. + +Measure the time from unlock to a usable thread and keep it under 300ms on a warm start. Add a test for each rule.`; + +/** Pre-fills the long-prompt and attachment drafts before launch; the app reads them at start. */ +async function seedComposerDrafts(simulator: Simulator, targets: LinkTargets) { + const documents = NodePath.join(await simulator.dataContainer(), "Documents"); + const images = NodePath.join(documents, "design-eval"); + await NodeFSP.mkdir(images, { recursive: true }); + const attachments = await Promise.all( + SHOWCASE_PROJECTS.slice(0, 2).map(async (project, index) => { + const file = NodePath.join(images, `${project.id}-mock.png`); + const png = await sharp(Buffer.from(project.favicon)).resize(900, 900).png().toBuffer(); + await NodeFSP.writeFile(file, png); + return { + id: `design-eval-image-${index}`, + type: "image", + name: `${project.id}-mock.png`, + mimeType: "image/png", + sizeBytes: png.byteLength, + fileUri: `file://${file}`, + previewUri: `file://${file}`, + }; + }), + ); + const key = (id: string) => { + const thread = targets.get(id); + if (!thread) throw new Error(`Unknown draft thread ${id}.`); + return `${thread.environmentId}:${thread.threadId}`; + }; + await NodeFSP.mkdir(NodePath.join(documents, "composer-drafts"), { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(documents, "composer-drafts", "drafts.json"), + JSON.stringify({ + schemaVersion: 1, + drafts: { + [key("pocket-command-center")]: { text: LONG_PROMPT, attachments: [] }, + [key("handoff-haptics")]: { + text: "Match the haptic to these two frames of the handoff.", + attachments, + }, + }, + }), + ); +} + +async function captureScene( + simulator: Simulator, + pose: Pose, + sceneId: SceneId, + targets: LinkTargets, + outDirectory: string, +): Promise { + const scene = SCENES.find((candidate) => candidate.id === sceneId)!; + const file = NodePath.join("captures", pose.id, `${scene.id}.png`); + let problem: string | undefined; + try { + await simulator.home(THREAD_LIST); + for (const step of scene.steps) await simulator.perform(step, targets); + if ("ready" in scene) await simulator.waitFor(new RegExp(scene.ready), scene.title); + } catch (error) { + problem = error instanceof Error ? error.message : String(error); + } + await delay(1_500); + await simulator.screenshot(NodePath.join(outDirectory, file), pose); + return { pose: pose.id, scene: scene.id, file, ...(problem ? { problem } : {}) }; +} + +/** A screen is its title: the text in the top band, where the navigation bar or a sheet header sits. */ +async function screenSignature(simulator: Simulator): Promise { + const snapshot = await simulator.snapshot(); + const top = snapshot.viewport.height * 0.16; + return snapshot.nodes + .filter((node) => node.kind === "text" && node.label && (node.rect?.y ?? Infinity) < top) + .map((node) => node.label) + .join("|"); +} + +async function captureFlow( + simulator: Simulator, + pose: Pose, + flow: Flow, + targets: LinkTargets, + outDirectory: string, +): Promise { + const frames: Array<{ file: string; action: string }> = []; + const signatures = new Set(); + let problem: string | undefined; + const frame = async (action: string) => { + const file = NodePath.join("captures", "flows", flow.id, `${frames.length}.png`); + await delay(1_200); + await simulator.screenshot(NodePath.join(outDirectory, file), pose); + signatures.add(await screenSignature(simulator).catch(() => action)); + frames.push({ file, action }); + }; + try { + await simulator.home(THREAD_LIST); + await frame("Start"); + for (const step of flow.steps) { + await simulator.perform(step, targets); + const action = + "tap" in step + ? `Tap ${step.tap.replaceAll(/[\^$]/g, "")}` + : "type" in step + ? `Type "${step.type}"` + : "focus" in step + ? "Tap the composer" + : "link" in step + ? `Open ${step.link}` + : `Scroll ${step.scroll}`; + await frame(action); + } + } catch (error) { + problem = error instanceof Error ? error.message : String(error); + } + return { + flow: flow.id, + // Count what was done, so a flow that stopped early does not claim its planned taps. + taps: frames.filter((frame) => frame.action.startsWith("Tap")).length, + screens: signatures.size, + frames, + ...(problem ? { problem } : {}), + }; +} + +/** + * Captures every pose on macOS: boots the fixture environments with live threads, a Metro for the + * showcase build, and one dedicated simulator per device class, deleted afterwards. + */ +export async function captureMobile(input: { + readonly outDirectory: string; + readonly poses: ReadonlyArray; + readonly scenes: ReadonlySet | null; + readonly flows: boolean; + readonly skipBuild: boolean; + readonly log: (message: string) => void; +}): Promise<{ + readonly shots: ReadonlyArray; + readonly flows: ReadonlyArray; +}> { + // oxlint-disable-next-line t3code/no-global-process-runtime -- A host CLI with no Effect runtime. + if (process.platform !== "darwin") throw new Error("Mobile capture needs macOS with Xcode."); + const appPath = input.skipBuild ? IOS_APP_PATH : await buildIos(); + await NodeFSP.access(appPath).catch(() => { + throw new Error(`No showcase build at ${appPath}. Run without --skip-build.`); + }); + const runId = Date.now().toString(36); + const rootDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-design-")); + const servers: NodeChildProcess.ChildProcess[] = []; + let metro: NodeChildProcess.ChildProcess | null = null; + const shots: MobileShot[] = []; + const flows: FlowResult[] = []; + try { + input.log("starting the fixture environments"); + const environments = await startShowcaseEnvironments( + rootDir, + (server) => servers.push(server), + SHOWCASE_ENVIRONMENTS, + { liveThreads: true }, + ); + const targets = linkTargets(environments); + const metroPort = await reserveAvailablePort(); + metro = startShowcaseMetro(metroPort, { showcase: false }); + await waitForPort(metroPort, "Metro", 120_000); + input.log("bundling"); + // Warm the exact bundle the dev client asks for; a cold build can outlast its load timeout. + const manifest = await run("curl", [ + "-sf", + "-H", + "expo-platform: ios", + "-H", + "accept: application/expo+json,application/json", + `http://127.0.0.1:${metroPort}`, + ]); + const bundleUrl = (JSON.parse(manifest.stdout) as { launchAsset: { url: string } }).launchAsset + .url; + await run("curl", ["-sf", "-o", "/dev/null", bundleUrl], { timeoutMs: 900_000 }); + + const devices = [...new Set(input.poses.map((pose) => pose.device))] as DeviceClassId[]; + for (const device of devices) { + input.log(`${device}: creating the simulator`); + const simulator = await Simulator.create(device, runId); + try { + // iPadOS windowing keeps full-screen apps portrait unless it is off. + if (device === "ipad") await ensureIosFullScreenAppsMode(simulator.udid); + await simulator.install(appPath); + await seedComposerDrafts(simulator, targets); + input.log(`${device}: launching`); + await simulator.launch(`http://127.0.0.1:${metroPort}`); + const pairingUrls = await Promise.all( + environments.map(async (environment) => + buildShowcasePairingUrl( + "127.0.0.1", + environment.port, + await issuePairingCredential(environment.baseDir), + ), + ), + ); + await simulator.pair(pairingUrls, THREAD_LIST); + for (const pose of input.poses.filter((candidate) => candidate.device === device)) { + input.log(`${pose.id}: capturing`); + await simulator.applyPose(pose); + const scenes = (pose.scenes ?? SCENES.map((scene) => scene.id)).filter( + (scene) => input.scenes === null || input.scenes.has(scene), + ); + for (const scene of scenes) { + const shot = await captureScene(simulator, pose, scene, targets, input.outDirectory); + if (shot.problem) input.log(`${pose.id}/${scene}: ${shot.problem}`); + shots.push(shot); + } + } + const flowPose = input.poses.find((pose) => pose.id === "iphone-dark"); + if (input.flows && device === "iphone" && flowPose) { + await simulator.applyPose(flowPose); + for (const flow of FLOWS) { + input.log(`flow ${flow.id}`); + const result = await captureFlow( + simulator, + flowPose, + flow, + targets, + input.outDirectory, + ); + if (result.problem) input.log(`flow ${flow.id}: ${result.problem}`); + flows.push(result); + } + } + } finally { + await simulator.delete(); + } + } + } finally { + await Promise.all([...(metro ? [stopProcess(metro)] : []), ...servers.map(stopProcess)]); + await NodeFSP.rm(rootDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + return { shots, flows }; +} diff --git a/evals/mobile-design/cases.ts b/evals/mobile-design/cases.ts new file mode 100644 index 0000000000000..c0eb8cdd3ad81 --- /dev/null +++ b/evals/mobile-design/cases.ts @@ -0,0 +1,366 @@ +/** + * Steps: `link` opens `t3code://`, where `{thread:}` becomes `/` + * for a seeded or live fixture thread; `tap` presses the first element whose accessibility label + * matches the regex; `focus: "composer"` taps the unlabelled composer. + */ + +export const AXES = [ + { id: "visual", label: "Visual design" }, + { id: "layout", label: "Layout quality" }, + { id: "flow", label: "Flow clarity" }, + { id: "polish", label: "Polish" }, + { id: "desktop", label: "Conformance with desktop" }, + { id: "competitors", label: "Quality versus Codex and Claude" }, +] as const; +export type AxisId = (typeof AXES)[number]["id"]; + +export const DEVICE_CLASSES = [ + { id: "iphone", label: "iPhone 17 Pro" }, + { id: "duo", label: "iPhone Duo (closed and open)" }, + { id: "ipad", label: "iPad Pro 13" }, +] as const; +export type DeviceClassId = (typeof DEVICE_CLASSES)[number]["id"]; + +/** One simulator per device class; poses change its settings while the app runs. */ +export const SIMULATORS = { + iphone: { name: "T3 Design iPhone 17 Pro", type: "iPhone-17-Pro" }, + duo: { name: "T3 Design iPhone Duo", type: "iPhone-Duo" }, + ipad: { name: "T3 Design iPad Pro 13", type: "iPad-Pro-13-inch-M5-16GB" }, +} as const satisfies Record; + +export type StepAction = + | { readonly link: string } + | { readonly tap: string } + | { readonly type: string } + | { readonly scroll: "up" | "down" } + | { readonly focus: "composer" }; + +export interface Pose { + readonly id: string; + readonly device: DeviceClassId; + readonly label: string; + readonly appearance: "dark" | "light"; + readonly orientation?: "portrait" | "landscape"; + readonly fold?: "closed" | "open"; + readonly textSize?: string; + /** Scenes this pose captures; all scenes when omitted. */ + readonly scenes?: ReadonlyArray; + /** Why this pose exists, when it guards a known complaint. */ + readonly source?: string; +} + +export interface Scene { + readonly id: string; + readonly title: string; + /** Where the scene came from: the issue it guards, or the request that asked for it. */ + readonly source: string; + readonly steps: ReadonlyArray; + /** Wait for this label before the screenshot. */ + readonly ready?: string; + /** Desktop web path for the "conforms to desktop" pair, relative to the environment origin. */ + readonly desktop?: { readonly path: string; readonly click?: string }; +} + +const REQUEST = "Mobile design eval request"; +const thread = (id: string) => ({ link: `threads/{thread:${id}}` }); + +export const SCENES = [ + { + id: "threads", + title: "Thread list", + source: `${REQUEST}; #22543 (iPad sidebar hides the newest thread)`, + steps: [], + ready: "^Plan offline mode", + desktop: { path: "/" }, + }, + { + id: "thread-settled", + title: "Thread, settled", + source: REQUEST, + steps: [thread("remote-command-center")], + ready: "^Copy message$", + desktop: { path: "/{thread:remote-command-center}" }, + }, + { + id: "thread-tool-calls", + title: "Thread with tool calls and file changes, expanded", + source: `${REQUEST}; #24930 (conversation text under the header)`, + steps: [thread("tool-calls"), { tap: "^Worked for" }, { tap: "^Ran \\d+ commands" }], + ready: "^Copy message$", + desktop: { path: "/{thread:tool-calls}", click: "text=/^Worked for/" }, + }, + { + id: "thread-long-code", + title: "Thread with a long code block", + source: `${REQUEST}; #24930 (conversation text under the header)`, + steps: [thread("long-code"), { scroll: "up" }], + desktop: { path: "/{thread:long-code}" }, + }, + { + id: "thread-streaming", + title: "Thread while the agent streams", + source: REQUEST, + steps: [thread("streaming")], + ready: "^Stop agent$", + desktop: { path: "/{thread:streaming}" }, + }, + { + id: "composer-focused", + title: "Composer, empty and focused", + source: REQUEST, + steps: [thread("remote-command-center"), { focus: "composer" }], + }, + { + id: "composer-long-prompt", + title: "Composer with a long prompt", + source: REQUEST, + steps: [thread("pocket-command-center")], + desktop: { path: "/{thread:pocket-command-center}" }, + }, + { + id: "composer-attachments", + title: "Composer with image attachments", + source: REQUEST, + steps: [thread("handoff-haptics")], + }, + { + id: "composer-model-picker", + title: "Model picker open", + source: REQUEST, + steps: [thread("tool-calls"), { tap: "^Model and reasoning settings" }], + ready: "^Thread settings$", + desktop: { path: "/{thread:tool-calls}", click: "role=button[name=/GPT-5.4/]" }, + }, + { + id: "composer-slash", + title: "Slash suggestions while the agent runs", + source: `${REQUEST}; #16112 (running pill over slash suggestions)`, + steps: [thread("streaming"), { focus: "composer" }, { type: "/" }], + }, + { + id: "new-thread", + title: "New thread, project chosen", + source: REQUEST, + steps: [{ link: "new" }, { tap: "^T3 Code$" }], + desktop: { path: "/new" }, + }, + { + id: "question", + title: "Agent asks two questions", + source: `${REQUEST}; #24290 and #24291 (Duo question panel)`, + steps: [thread("question")], + ready: "^Submit answers$", + desktop: { path: "/{thread:question}" }, + }, + { + id: "approval", + title: "Agent asks to run a command", + source: REQUEST, + steps: [thread("approval")], + ready: "^Allow once$", + desktop: { path: "/{thread:approval}" }, + }, + { + id: "background", + title: "Background work after the turn", + source: REQUEST, + steps: [thread("background")], + ready: "^Stop background work$", + desktop: { path: "/{thread:background}" }, + }, + { + id: "background-sheet", + title: "Background work details", + source: REQUEST, + steps: [thread("background"), { tap: "^Running: " }], + }, + { + id: "terminal", + title: "Terminal", + source: REQUEST, + steps: [{ link: "threads/{thread:remote-command-center}/terminal?terminalId=term-1" }], + }, + { + id: "review", + title: "Review and diff", + source: REQUEST, + steps: [{ link: "threads/{thread:remote-command-center}/review" }], + ready: "^Changes vs", + }, + { + id: "settings", + title: "Settings", + source: REQUEST, + steps: [{ link: "settings" }], + desktop: { path: "/settings/general" }, + }, + { + id: "environments", + title: "Environments", + source: REQUEST, + steps: [{ link: "settings/environments" }], + desktop: { path: "/settings/connections" }, + }, +] as const satisfies ReadonlyArray; +export type SceneId = (typeof SCENES)[number]["id"]; + +/** Scenes where width and folding change the layout most. */ +const LAYOUT_SCENES: ReadonlyArray = [ + "threads", + "thread-tool-calls", + "thread-streaming", + "composer-model-picker", + "new-thread", + "question", + "approval", + "background", + "terminal", + "review", + "settings", +]; +const LIGHT_SCENES: ReadonlyArray = [ + "threads", + "thread-tool-calls", + "question", + "review", + "settings", +]; + +export const POSES = [ + { id: "iphone-dark", device: "iphone", label: "iPhone, dark", appearance: "dark" }, + { + id: "iphone-light", + device: "iphone", + label: "iPhone, light", + appearance: "light", + scenes: LIGHT_SCENES, + }, + { + id: "iphone-large-text", + device: "iphone", + label: "iPhone, largest accessibility text", + appearance: "dark", + textSize: "accessibility-extra-extra-extra-large", + scenes: ["threads", "thread-tool-calls", "question", "composer-model-picker", "settings"], + source: "#17912 (Dynamic Type)", + }, + { + id: "iphone-landscape", + device: "iphone", + label: "iPhone, landscape", + appearance: "dark", + orientation: "landscape", + scenes: ["threads", "thread-tool-calls"], + source: "#19709 and #16511 (landscape iPhone)", + }, + ...(["closed", "open"] as const).flatMap((fold) => + (["dark", "light"] as const).map((appearance): Pose => ({ + id: `duo-${fold}-${appearance}`, + device: "duo", + label: `iPhone Duo ${fold}, ${appearance}`, + appearance, + fold, + scenes: appearance === "dark" ? LAYOUT_SCENES : LIGHT_SCENES, + })), + ), + ...(["landscape", "portrait"] as const).flatMap((orientation) => + (["dark", "light"] as const).map((appearance): Pose => ({ + id: `ipad-${orientation}-${appearance}`, + device: "ipad", + label: `iPad ${orientation}, ${appearance}`, + appearance, + orientation, + scenes: appearance === "dark" ? [...LAYOUT_SCENES, "environments"] : LIGHT_SCENES, + })), + ), +] as const satisfies ReadonlyArray; + +export interface Flow { + readonly id: string; + readonly title: string; + readonly source: string; + /** Each step is one screenshot; taps are counted from `tap` steps. */ + readonly steps: ReadonlyArray; +} + +/** Common tasks, walked from the thread list on the dark iPhone pose, after every scene. */ +export const FLOWS = [ + { + id: "start-thread", + title: "Start a thread in a chosen project and machine", + source: REQUEST, + steps: [ + { tap: "^New chat$" }, + { tap: "^T3 Code$" }, + { tap: "^React$" }, + { tap: "^Choose environment" }, + { tap: "^Suspense Station" }, + { focus: "composer" }, + { type: "Add a loading skeleton to the profile page" }, + { tap: "^Send$" }, + ], + }, + { + id: "answer-question", + title: "Answer the agent's questions", + source: REQUEST, + steps: [ + { tap: "^Plan offline mode$" }, + { tap: "^Last 50 threads" }, + { scroll: "down" }, + { tap: "^Server state" }, + { tap: "^Submit answers$" }, + ], + }, + { + id: "review-diff", + title: "Review a thread's diff", + source: REQUEST, + steps: [ + { tap: "^Make remote coding feel local" }, + { tap: "^Git$" }, + { tap: "^Review changes" }, + ], + }, + { + id: "stop-background", + title: "Stop background work", + source: REQUEST, + steps: [{ tap: "^Check settings in the dev server$" }, { tap: "^Stop background work$" }], + }, + { + id: "find-old-thread", + title: "Find an old thread", + source: REQUEST, + steps: [{ tap: "^Search$" }, { type: "OOM" }, { tap: "OOM killer" }], + }, + { + id: "change-model", + title: "Change the model", + source: REQUEST, + steps: [ + { tap: "^Show a reconnect banner$" }, + { tap: "^Model and reasoning settings" }, + { tap: "^GPT-5.3 Codex" }, + ], + }, +] as const satisfies ReadonlyArray; + +/** The default council. `--panel` adds `muse` for copy-heavy rounds. */ +export const PANEL = { + opus: { + label: "Claude Opus 5.5", + provider: "claudeAgent", + model: "claude-opus-5-5", + effort: "xhigh", + }, + sol: { label: "Codex GPT-6.1 Sol", provider: "codex", model: "gpt-6.1-sol", effort: "xhigh" }, + muse: { + label: "Muse Wonder Full", + provider: "muse", + model: "muse-proxy-wonder-full", + effort: "xhigh", + }, +} as const; +export type PanelMember = keyof typeof PANEL; +export const DEFAULT_PANEL: ReadonlyArray = ["opus", "sol"]; diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts new file mode 100644 index 0000000000000..0ca98fc91c8c6 --- /dev/null +++ b/evals/mobile-design/ios.ts @@ -0,0 +1,367 @@ +// @effect-diagnostics nodeBuiltinImport:off globalTimers:off globalDate:off - Host-side simulator automation uses Node subprocess and timing APIs directly. + +import * as NodeChildProcess from "node:child_process"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; + +import { + normalizeIosSimulator, + readPngDimensions, + suppressIosSystemFollowUps, + type ShowcaseEnvironment, +} from "../../scripts/mobile-showcase.ts"; +import { + SHOWCASE_ENVIRONMENTS, + SHOWCASE_THREADS, +} from "../../scripts/mobile-showcase-environment.ts"; +import { SIMULATORS, type DeviceClassId, type Pose, type StepAction } from "./cases.ts"; + +const BUNDLE_ID = "com.t3tools.t3code"; +const REPO_ROOT = NodePath.resolve(import.meta.dirname, "../.."); +const AGENT_DEVICE = NodePath.join( + REPO_ROOT, + "scripts/node_modules/agent-device/bin/agent-device.mjs", +); + +export const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export function run( + command: string, + args: ReadonlyArray, + options: { + readonly allowFailure?: boolean; + readonly timeoutMs?: number; + readonly cwd?: string; + } = {}, +): Promise<{ readonly code: number; readonly stdout: string; readonly stderr: string }> { + return new Promise((resolve, reject) => { + const child = NodeChildProcess.spawn(command, args, { + cwd: options.cwd ?? REPO_ROOT, + stdio: ["ignore", "pipe", "pipe"], + timeout: options.timeoutMs ?? 120_000, + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("error", reject); + child.on("close", (code) => { + if (code !== 0 && !options.allowFailure) + reject(new Error(`${command} ${args.join(" ")} failed (${code}): ${stderr || stdout}`)); + else resolve({ code: code ?? 1, stdout, stderr }); + }); + }); +} + +const simctl = (...args: ReadonlyArray) => run("xcrun", ["simctl", ...args]); + +export interface SnapshotNode { + readonly ref?: string; + /** agent-device's role name, such as `button`, `text`, `text-view` or `navigation-bar`. */ + readonly kind?: string; + readonly label?: string; + readonly rect?: { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; + }; + readonly hittable?: boolean; +} +export interface Snapshot { + readonly nodes: ReadonlyArray; + readonly viewport: { readonly width: number; readonly height: number }; +} + +/** Where each `{thread:}` lives: seeded and live fixture threads, by environment. */ +export type LinkTargets = ReadonlyMap< + string, + { readonly environmentId: string; readonly threadId: string } +>; + +export function linkTargets(environments: ReadonlyArray): LinkTargets { + const threads = new Map(); + for (const [index, environment] of SHOWCASE_ENVIRONMENTS.entries()) { + const started = environments[index]; + if (!started) throw new Error(`Environment ${environment.id} did not start.`); + for (const thread of SHOWCASE_THREADS) { + if ((environment.projectIds as ReadonlyArray).includes(thread.projectId)) + threads.set(thread.id, { environmentId: started.environmentId, threadId: thread.id }); + } + for (const [id, threadId] of started.liveThreadIds) + threads.set(id, { environmentId: started.environmentId, threadId }); + } + return threads; +} + +export function expandPath(path: string, targets: LinkTargets): string { + return path.replaceAll(/\{thread:([^}]+)\}/g, (_, id: string) => { + const thread = targets.get(id); + if (!thread) throw new Error(`Unknown thread '${id}'.`); + return `${thread.environmentId}/${encodeURIComponent(thread.threadId)}`; + }); +} + +/** One simulator this run created, driven with simctl for system state and agent-device for UI. */ +export class Simulator { + readonly device: DeviceClassId; + readonly udid: string; + private readonly session: string; + + private constructor(device: DeviceClassId, udid: string, session: string) { + this.device = device; + this.udid = udid; + this.session = session; + } + + /** Creates a dedicated simulator on the newest runtime, so runs never touch shared devices. */ + static async create(device: DeviceClassId, runId: string): Promise { + const spec = SIMULATORS[device]; + const udid = ( + await simctl( + "create", + `${spec.name} ${runId}`, + `com.apple.CoreSimulator.SimDeviceType.${spec.type}`, + ) + ).stdout.trim(); + const simulator = new Simulator(device, udid, `t3-design-${device}-${runId}`); + await suppressIosSystemFollowUps(udid); + await simctl("boot", udid); + await simctl("bootstatus", udid, "-b"); + return simulator; + } + + async delete() { + await this.agent(["close"]).catch(() => undefined); + await simctl("shutdown", this.udid).catch(() => undefined); + await simctl("delete", this.udid).catch(() => undefined); + } + + async install(appPath: string) { + await normalizeIosSimulator("dark", this.udid); + await simctl("install", this.udid, appPath); + } + + async dataContainer(): Promise { + return (await simctl("get_app_container", this.udid, BUNDLE_ID, "data")).stdout.trim(); + } + + /** + * Loads the JS straight from Metro. The dev menu's preferences hide its tools button and + * launch sheet; a dev-launcher link would set them too, but iOS asks before opening it. + */ + async launch(metroUrl: string) { + for (const [key, value] of [ + ["EXDevMenuShowFloatingActionButton", "NO"], + ["EXDevMenuShowsAtLaunch", "NO"], + ["EXDevMenuIsOnboardingFinished", "YES"], + ] as const) { + await simctl("spawn", this.udid, "defaults", "write", BUNDLE_ID, key, "-bool", value); + } + // Attaches the agent-device session, which every later command needs. + await this.agent(["open", BUNDLE_ID], 600_000); + const loaded = /^(Chats|Add environment|New chat|Search threads)$/; + for (let attempt = 0; attempt < 3; attempt += 1) { + await simctl( + "launch", + "--terminate-running-process", + this.udid, + BUNDLE_ID, + "--initialUrl", + metroUrl, + ); + const snapshot = await this.waitFor( + new RegExp(`${loaded.source}|^Reload$`), + "the app to load", + 180_000, + ).catch(() => null); + if (snapshot && this.find(snapshot, loaded)) return; + } + throw new Error("The app did not load from Metro."); + } + + async pair(pairingUrls: ReadonlyArray, firstThreadTitle: RegExp) { + for (const pairingUrl of pairingUrls) { + const query = new URLSearchParams({ pairingUrl, autoConnect: "1" }); + await this.openUrl(`t3code://connections/new?${query}`); + await delay(4_000); + } + await this.waitFor(firstThreadTitle, "the paired thread list", 90_000); + } + + /** iOS asks before a link opens the app; accept the prompt once it shows. */ + async openUrl(url: string) { + await simctl("openurl", this.udid, url); + // A busy host can take seconds per XCTest call, and the prompt itself can be late. + const deadline = Date.now() + 60_000; + while (Date.now() < deadline) { + const accepted = await this.agent(["alert", "accept"]).then( + () => true, + () => false, + ); + if (accepted) return; + await delay(500); + } + } + + async agent(args: ReadonlyArray, timeoutMs = 180_000): Promise> { + const { stdout, stderr } = await run( + process.execPath, + [ + AGENT_DEVICE, + ...args, + "--platform", + "ios", + "--udid", + this.udid, + "--session", + this.session, + "--json", + ], + { allowFailure: true, timeoutMs }, + ); + let parsed: { success?: boolean; data?: Record; error?: { message?: string } }; + try { + parsed = JSON.parse(stdout); + } catch { + throw new Error(`agent-device ${args[0]}: ${(stderr || stdout).slice(-400)}`); + } + if (!parsed.success) + throw new Error(`agent-device ${args.join(" ")}: ${parsed.error?.message}`); + return parsed.data ?? {}; + } + + async snapshot(): Promise { + // The first command of a session attaches it to the app. + const data = await this.agent(["snapshot"]).catch(async () => { + await this.agent(["open", BUNDLE_ID]); + return this.agent(["snapshot"]); + }); + return data as unknown as Snapshot; + } + + find(snapshot: Snapshot, label: RegExp): SnapshotNode | undefined { + const onScreen = (node: SnapshotNode) => + node.rect !== undefined && + node.rect.y + node.rect.height > 0 && + node.rect.y < snapshot.viewport.height; + const matches = snapshot.nodes.filter((node) => label.test(node.label ?? "") && onScreen(node)); + return matches.find((node) => node.hittable) ?? matches[0]; + } + + async waitFor(label: RegExp, what: string, timeoutMs = 20_000): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + const snapshot = await this.snapshot().catch(() => null); + if (snapshot && this.find(snapshot, label)) return snapshot; + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what} (${label})`); + await delay(750); + } + } + + async press(node: SnapshotNode) { + if (!node.rect) throw new Error(`${node.label ?? "element"} has no frame`); + const { x, y, width, height } = node.rect; + // Refs can be refused when agent-device thinks the element is covered; the centre still takes a tap. + await this.agent(["press", `@${node.ref}`]).catch(() => + this.agent(["press", String(Math.round(x + width / 2)), String(Math.round(y + height / 2))]), + ); + } + + async tap(label: RegExp) { + const node = this.find(await this.waitFor(label, `'${label.source}'`), label); + if (!node) throw new Error(`nothing labelled ${label}`); + await this.press(node); + } + + /** The composer is a native text view with no label: the lowest editable field on screen. */ + async focusComposer() { + const snapshot = await this.snapshot(); + const composer = snapshot.nodes + .filter((node) => (node.kind === "text-view" || node.kind === "text-field") && node.rect) + .toSorted((a, b) => b.rect!.y - a.rect!.y)[0]; + if (!composer) throw new Error("no composer on screen"); + await this.press(composer); + } + + async perform(step: StepAction, targets: LinkTargets) { + if ("link" in step) await this.openUrl(`t3code://${expandPath(step.link, targets)}`); + else if ("tap" in step) await this.tap(new RegExp(step.tap)); + else if ("type" in step) await this.agent(["type", step.type]); + else if ("scroll" in step) await this.agent(["scroll", step.scroll]); + else await this.focusComposer(); + await delay(1_200); + } + + /** Backs out of sheets, pushed screens and the keyboard until the thread list shows. */ + async home(threadListLabel: RegExp) { + for (let attempt = 0; attempt < 8; attempt += 1) { + const snapshot = await this.snapshot(); + // A thread's back button is labelled with the tab it returns to, so only count it up top. + const back = snapshot.nodes.find( + (node) => + node.kind === "button" && + (/^(Back|Close|Done|Cancel)$/.test(node.label ?? "") || + (node.label === "Chats" && (node.rect?.y ?? Infinity) < snapshot.viewport.height / 4)), + ); + if (!back && this.find(snapshot, threadListLabel)) { + await this.agent(["keyboard", "dismiss"]).catch(() => undefined); + // Start every scene from the top of the list. + await this.agent(["scroll", "up"]).catch(() => undefined); + await delay(600); + return; + } + // Another tab, such as Settings, shows no back button; its tab bar returns to Chats. + const chatsTab = snapshot.nodes.find( + (node) => + node.kind === "button" && + node.label === "Chats" && + (node.rect?.y ?? 0) > snapshot.viewport.height * 0.75, + ); + if (back) await this.press(back); + else if (chatsTab) await this.press(chatsTab); + else await this.agent(["back"]).catch(() => undefined); + await delay(1_000); + } + throw new Error("could not return to the thread list"); + } + + /** Applies the pose's system settings to the running app. */ + async applyPose(pose: Pose) { + await simctl("ui", this.udid, "appearance", pose.appearance); + await simctl("ui", this.udid, "content_size", pose.textSize ?? "large"); + if (pose.fold) { + await this.agent(["fold", pose.fold]); + await delay(2_500); + } + // A fold sets the Duo's shape itself. + if (!pose.fold) await this.orient(pose.orientation ?? "portrait"); + } + + /** agent-device names rotations from the natural orientation, so try each until the shape fits. */ + private async orient(shape: "portrait" | "landscape") { + for (const name of ["portrait", "landscape-left", "landscape-right"]) { + const { width, height } = (await this.snapshot()).viewport; + if (width > height === (shape === "landscape")) return; + await this.agent(["orientation", name]); + await delay(2_000); + } + } + + async screenshot(destination: string, pose: Pose) { + await NodeFSP.mkdir(NodePath.dirname(destination), { recursive: true }); + await simctl( + "io", + this.udid, + "screenshot", + ...(pose.fold ? ["--display", pose.fold === "closed" ? "1" : "3"] : []), + destination, + ); + // Headless displays stay portrait when the device rotates. + const landscape = pose.orientation === "landscape" || pose.fold === "open"; + const { width, height } = readPngDimensions(await NodeFSP.readFile(destination)); + if (landscape && height > width) { + await run("sips", ["--rotate", pose.fold ? "90" : "270", destination]); + } + } +} diff --git a/evals/mobile-design/pack.ts b/evals/mobile-design/pack.ts new file mode 100644 index 0000000000000..1edf2dc15f904 --- /dev/null +++ b/evals/mobile-design/pack.ts @@ -0,0 +1,250 @@ +// @effect-diagnostics nodeBuiltinImport:off - Builds image files with sharp and Node filesystem APIs. + +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; + +import sharp from "sharp"; + +import type { DesktopShot } from "./capture-desktop.ts"; +import type { FlowResult, MobileShot } from "./capture-mobile.ts"; +import { + AXES, + DEVICE_CLASSES, + FLOWS, + PANEL, + POSES, + SCENES, + type PanelMember, + type Pose, +} from "./cases.ts"; + +const MAX_WIDTH = 2400; +const MAX_TILE_HEIGHT = 1400; +const LABEL_HEIGHT = 44; +const GAP = 24; + +interface Tile { + readonly file: string; + readonly label: string; +} + +const escapeXml = (text: string) => + text.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">"); + +/** Lays tiles out in one row, scaled to a shared height, each with a label above it. */ +async function contactSheet(tiles: ReadonlyArray, destination: string): Promise { + const images = await Promise.all( + tiles.map(async (tile) => ({ ...tile, meta: await sharp(tile.file).metadata() })), + ); + const ratios = images.map((image) => (image.meta.width ?? 1) / (image.meta.height ?? 1)); + const widthBudget = MAX_WIDTH - GAP * (images.length + 1); + const height = Math.floor( + Math.min(MAX_TILE_HEIGHT, widthBudget / ratios.reduce((sum, ratio) => sum + ratio, 0)), + ); + const widths = ratios.map((ratio) => Math.round(ratio * height)); + const sheetWidth = widths.reduce((sum, width) => sum + width, 0) + GAP * (images.length + 1); + const sheetHeight = height + LABEL_HEIGHT + GAP * 2; + let x = GAP; + const layers = await Promise.all( + images.map(async (image, index) => { + const left = x; + x += widths[index]! + GAP; + const input = await sharp(image.file).resize(widths[index], height).png().toBuffer(); + const label = Buffer.from( + `${escapeXml(image.label)}`, + ); + return [ + { input: label, left, top: GAP }, + { input, left, top: GAP + LABEL_HEIGHT }, + ]; + }), + ); + await NodeFSP.mkdir(NodePath.dirname(destination), { recursive: true }); + await sharp({ + create: { width: sheetWidth, height: sheetHeight, channels: 3, background: "#1c1c1e" }, + }) + .composite(layers.flat()) + .png() + .toFile(destination); +} + +/** Poses whose screenshots share a shape, so they sit well side by side. */ +function shapeGroup(pose: Pose): string { + if (pose.device === "iphone") + return pose.orientation === "landscape" ? "iphone-landscape" : "iphone"; + if (pose.device === "duo") return `duo-${pose.fold}`; + return `ipad-${pose.orientation}`; +} + +export interface PackEntry { + readonly file: string; + readonly scene: string; + readonly view: string; +} + +/** + * Writes the reviewer pack: contact sheets per scene and device shape, desktop pairs, flow + * filmstrips, and the brief. Nothing in it says which build it shows. + */ +export async function buildPack(input: { + readonly outDirectory: string; + readonly shots: ReadonlyArray; + readonly flows: ReadonlyArray; + readonly desktop: ReadonlyArray; + readonly referencesDirectory: string; + readonly panel: ReadonlyArray; +}): Promise<{ readonly brief: string }> { + const pack = NodePath.join(input.outDirectory, "pack"); + await NodeFSP.rm(pack, { recursive: true, force: true }); + const at = (file: string) => NodePath.join(input.outDirectory, file); + const entries: PackEntry[] = []; + + for (const scene of SCENES) { + const shots = input.shots.filter((shot) => shot.scene === scene.id); + const groups = new Map(); + for (const shot of shots) { + const pose = POSES.find((candidate) => candidate.id === shot.pose); + if (!pose) continue; + const group = shapeGroup(pose); + groups.set(group, [...(groups.get(group) ?? []), { file: at(shot.file), label: pose.label }]); + } + for (const [group, tiles] of groups) { + const file = NodePath.join("screens", group, `${scene.id}.png`); + await contactSheet(tiles, NodePath.join(pack, file)); + entries.push({ file, scene: scene.title, view: group }); + } + const desktop = input.desktop.find((shot) => shot.scene === scene.id && !shot.problem); + if (desktop) { + const phone = shots.find((shot) => shot.pose === "iphone-dark"); + const tablet = shots.find((shot) => shot.pose === "ipad-landscape-dark"); + const file = NodePath.join("desktop", `${scene.id}.png`); + await contactSheet( + [ + { file: at(desktop.file), label: "Desktop web, 1440×900" }, + ...(tablet ? [{ file: at(tablet.file), label: "iPad landscape" }] : []), + ...(phone ? [{ file: at(phone.file), label: "iPhone" }] : []), + ], + NodePath.join(pack, file), + ); + entries.push({ file, scene: scene.title, view: "desktop pair" }); + } + } + + const flowRows: string[] = []; + for (const result of input.flows) { + const flow = FLOWS.find((candidate) => candidate.id === result.flow); + if (!flow || result.frames.length === 0) continue; + const file = NodePath.join("flows", `${flow.id}.png`); + // Rows of five keep each phone frame large enough to read. + const rows = await Promise.all( + Array.from({ length: Math.ceil(result.frames.length / 5) }, async (_, row) => { + const rowFile = NodePath.join(pack, "flows", `${flow.id}-${row}.png`); + await contactSheet( + result.frames.slice(row * 5, row * 5 + 5).map((frame, index) => ({ + file: at(frame.file), + label: `${row * 5 + index}. ${frame.action}`, + })), + rowFile, + ); + return rowFile; + }), + ); + const metas = await Promise.all(rows.map((row) => sharp(row).metadata())); + let top = 0; + await sharp({ + create: { + width: Math.max(...metas.map((meta) => meta.width ?? 0)), + height: metas.reduce((sum, meta) => sum + (meta.height ?? 0), 0), + channels: 3, + background: "#1c1c1e", + }, + }) + .composite( + rows.map((row, index) => { + const layer = { input: row, left: 0, top }; + top += metas[index]!.height ?? 0; + return layer; + }), + ) + .png() + .toFile(NodePath.join(pack, file)); + await Promise.all(rows.map((row) => NodeFSP.rm(row))); + entries.push({ file, scene: flow.title, view: "flow" }); + flowRows.push( + `| ${flow.title} | ${result.taps} | ${result.screens} | \`${file}\`${result.problem ? ` (stopped early: ${result.problem.split("\n")[0]})` : ""} |`, + ); + } + + const references = await NodeFSP.readdir(input.referencesDirectory).catch(() => []); + const referenceImages = references.filter((name) => /\.(png|jpe?g)$/i.test(name)); + if (referenceImages.length > 0) { + await NodeFSP.mkdir(NodePath.join(pack, "references"), { recursive: true }); + for (const name of referenceImages) { + await NodeFSP.copyFile( + NodePath.join(input.referencesDirectory, name), + NodePath.join(pack, "references", name), + ); + } + } + + const template = await NodeFSP.readFile(NodePath.join(import.meta.dirname, "brief.md"), "utf8"); + const brief = template + .replaceAll("{{PACK}}", pack) + .replace( + "{{SCREENS}}", + entries + .filter((entry) => entry.view !== "flow") + .map((entry) => `- \`${entry.file}\`: ${entry.scene} (${entry.view})`) + .join("\n"), + ) + .replace( + "{{FLOWS}}", + flowRows.length > 0 + ? ["| Task | Taps | Screens | Filmstrip |", "| --- | --- | --- | --- |", ...flowRows].join( + "\n", + ) + : "No flows were captured this round.", + ) + .replace( + "{{REFERENCES}}", + referenceImages.length > 0 + ? referenceImages.map((name) => `- \`references/${name}\``).join("\n") + : "No reference screenshots of the Codex or Claude iOS apps are in this pack. Judge that axis from what you know of those apps, and say so.", + ) + .replace("{{AXES}}", AXES.map((axis) => `- \`${axis.id}\`: ${axis.label}`).join("\n")) + .replace( + "{{DEVICES}}", + DEVICE_CLASSES.map((device) => `- \`${device.id}\`: ${device.label}`).join("\n"), + ) + .replace( + "{{JSON}}", + JSON.stringify( + { + scores: Object.fromEntries( + DEVICE_CLASSES.map((device) => [ + device.id, + Object.fromEntries(AXES.map((axis) => [axis.id, 0])), + ]), + ), + fixes: ["…", "…", "…", "…", "…"], + }, + null, + 2, + ), + ); + const briefFile = NodePath.join(pack, "brief.md"); + await NodeFSP.writeFile(briefFile, brief); + await NodeFSP.writeFile( + NodePath.join(input.outDirectory, "council.json"), + `${JSON.stringify( + { + brief: briefFile, + scores: NodePath.join(input.outDirectory, "scores"), + panel: input.panel.map((member) => ({ member, ...PANEL[member] })), + }, + null, + 2, + )}\n`, + ); + return { brief: briefFile }; +} diff --git a/evals/mobile-design/references/README.md b/evals/mobile-design/references/README.md new file mode 100644 index 0000000000000..a93c249855945 --- /dev/null +++ b/evals/mobile-design/references/README.md @@ -0,0 +1,21 @@ +# Reference screenshots + +The "quality versus Codex and Claude" axis compares T3 Code with the Codex and Claude iOS apps. Their App Store builds do not run on a simulator, so a maintainer captures these on a real device and saves them here. Every PNG or JPEG in this folder goes into the next pack, under `references/`. + +Do not scrape or commit third-party marketing images. Use your own screenshots of the apps, with your own test account and no personal data in view. + +Name each file `--.png`, for example `codex-iphone-thread.png`. Capture each app in dark mode, at the default text size: + +| Screen | iPhone | iPad | +| --------------------------------------------------------- | ------------------------------- | ------------------------ | +| Conversation list | `-iphone-threads.png` | `-ipad-threads.png` | +| A conversation with tool calls or code | `-iphone-thread.png` | `-ipad-thread.png` | +| The agent still writing its reply | `-iphone-streaming.png` | | +| Composer with the keyboard up | `-iphone-composer.png` | | +| Model picker open | `-iphone-model-picker.png` | | +| New conversation or task | `-iphone-new.png` | `-ipad-new.png` | +| A question or approval from the agent, if the app has one | `-iphone-question.png` | | +| A diff or code review view, if the app has one | `-iphone-diff.png` | `-ipad-diff.png` | +| Settings | `-iphone-settings.png` | | + +`` is `codex` or `claude`. Skip a screen the app does not have. diff --git a/evals/mobile-design/run.test.ts b/evals/mobile-design/run.test.ts new file mode 100644 index 0000000000000..d6541733606ed --- /dev/null +++ b/evals/mobile-design/run.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { AXES, DEVICE_CLASSES } from "./cases.ts"; +import { median, parseReviewerReply, renderSummary, summarizeRound } from "./scores.ts"; + +const uniform = (value: number) => + Object.fromEntries( + DEVICE_CLASSES.map((device) => [ + device.id, + Object.fromEntries(AXES.map((axis) => [axis.id, value])), + ]), + ); + +const reply = (value: number, fixes = ["Give the question panel the space above the composer."]) => + `Prose first.\n\n\`\`\`json\n{"example": true}\n\`\`\`\n\nFinal:\n\n\`\`\`json\n${JSON.stringify({ scores: uniform(value), fixes })}\n\`\`\`\n`; + +describe("parseReviewerReply", () => { + it("reads the last json block", () => { + expect(parseReviewerReply(reply(7)).scores.duo.polish).toBe(7); + }); + + it("rejects a reply without scores for every device and axis", () => { + const partial = JSON.stringify({ scores: { iphone: uniform(7).iphone }, fixes: ["x"] }); + expect(() => parseReviewerReply(`\`\`\`json\n${partial}\n\`\`\``)).toThrow(); + }); + + it("rejects scores outside 1 to 10", () => { + expect(() => parseReviewerReply(reply(11))).toThrow(); + }); +}); + +describe("summarizeRound", () => { + it("takes the median per device and axis", () => { + expect(median([9, 6, 7])).toBe(7); + expect(median([6, 9])).toBe(7.5); + const round = summarizeRound({ + opus: parseReviewerReply(reply(6)), + sol: parseReviewerReply(reply(9)), + }); + expect(round.medians.ipad.visual).toBe(7.5); + }); + + it("shows deltas against the baseline medians", () => { + const baseline = summarizeRound({ opus: parseReviewerReply(reply(6)) }).medians; + const markdown = renderSummary( + summarizeRound({ opus: parseReviewerReply(reply(8)) }), + baseline, + ); + expect(markdown).toContain("| iPhone 17 Pro | Visual design | 8.0 | 8.0 | +2.0 |"); + }); +}); diff --git a/evals/mobile-design/run.ts b/evals/mobile-design/run.ts new file mode 100644 index 0000000000000..241c8ee013ac9 --- /dev/null +++ b/evals/mobile-design/run.ts @@ -0,0 +1,197 @@ +#!/usr/bin/env node +// @effect-diagnostics nodeBuiltinImport:off globalDate:off - A host-side CLI that drives captures and writes files. +import * as NodeChildProcess from "node:child_process"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeUtil from "node:util"; + +import { captureDesktop, findShot, type DesktopShot } from "./capture-desktop.ts"; +import { captureMobile, type FlowResult, type MobileShot } from "./capture-mobile.ts"; +import { DEFAULT_PANEL, PANEL, POSES, SCENES, type PanelMember, type SceneId } from "./cases.ts"; +import { buildPack } from "./pack.ts"; +import { parseReviewerReply, renderSummary, summarizeRound, type Medians } from "./scores.ts"; + +const { values } = NodeUtil.parseArgs({ + options: { + out: { type: "string" }, + capture: { type: "string", multiple: true }, + pack: { type: "boolean", default: false }, + summarize: { type: "boolean", default: false }, + baseline: { type: "string" }, + pose: { type: "string", multiple: true }, + case: { type: "string", multiple: true }, + "no-flows": { type: "boolean", default: false }, + "skip-build": { type: "boolean", default: false }, + panel: { type: "string" }, + shot: { type: "string" }, + }, +}); + +if (!values.out) { + throw new Error( + "Use --out . Add --capture mobile|desktop, --pack, or --summarize [--baseline ].", + ); +} +const out = NodePath.resolve(values.out); +const log = (message: string) => process.stderr.write(`[mobile-design] ${message}\n`); +const capturesFile = NodePath.join(out, "captures", "captures.json"); + +const unknownCases = (values.case ?? []).filter((id) => !SCENES.some((scene) => scene.id === id)); +if (unknownCases.length > 0) throw new Error(`Unknown case ${unknownCases.join(", ")}.`); +const scenes = values.case ? new Set(values.case as ReadonlyArray) : null; +const poses = POSES.filter((pose) => !values.pose || values.pose.includes(pose.id)); +if (poses.length === 0) + throw new Error(`No pose matches. Poses: ${POSES.map((pose) => pose.id).join(", ")}.`); +const panel = (values.panel?.split(",") ?? DEFAULT_PANEL).map((member) => { + if (!(member in PANEL)) + throw new Error(`Unknown panel member ${member}. Use ${Object.keys(PANEL).join(", ")}.`); + return member as PanelMember; +}); + +interface Captures { + readonly shots: ReadonlyArray; + readonly flows: ReadonlyArray; + readonly desktop: ReadonlyArray; +} + +async function readCaptures(): Promise { + const text = await NodeFSP.readFile(capturesFile, "utf8").catch(() => null); + return text ? (JSON.parse(text) as Captures) : { shots: [], flows: [], desktop: [] }; +} + +/** Merges new captures over earlier ones, so mobile and desktop can run on different hosts. */ +async function saveCaptures(update: Partial) { + const current = await readCaptures(); + const replaced = ( + earlier: ReadonlyArray, + next: ReadonlyArray | undefined, + ) => + next + ? [...earlier.filter((item) => !next.some((n) => n.file === item.file)), ...next] + : earlier; + const merged: Captures = { + shots: replaced(current.shots, update.shots), + flows: update.flows + ? [ + ...current.flows.filter((flow) => !update.flows!.some((n) => n.flow === flow.flow)), + ...update.flows, + ] + : current.flows, + desktop: replaced(current.desktop, update.desktop), + }; + await NodeFSP.mkdir(NodePath.dirname(capturesFile), { recursive: true }); + await NodeFSP.writeFile(capturesFile, `${JSON.stringify(merged, null, 2)}\n`); +} + +/** The commit under review and how long each capture took. Kept outside the pack so reviewers stay blind. */ +async function writeRunInfo(capture: string, startedAt: number) { + const commit = NodeChildProcess.execFileSync("git", ["rev-parse", "HEAD"], { + encoding: "utf8", + }).trim(); + const file = NodePath.join(out, "run.json"); + const earlier: { readonly minutes?: Record } = await NodeFSP.readFile( + file, + "utf8", + ).then( + (text) => JSON.parse(text), + () => ({}), + ); + const minutes = { + ...earlier.minutes, + [capture]: Math.round((Date.now() - startedAt) / 6_000) / 10, + }; + await NodeFSP.writeFile(file, `${JSON.stringify({ commit, minutes }, null, 2)}\n`); +} + +async function summarize() { + const directory = NodePath.join(out, "scores"); + const files = (await NodeFSP.readdir(directory).catch(() => [])).filter((name) => + name.endsWith(".md"), + ); + if (files.length === 0) + throw new Error(`Save each reviewer's reply as ${directory}/.md first.`); + const reviewers = Object.fromEntries( + await Promise.all( + files.map(async (name) => { + const reply = await NodeFSP.readFile(NodePath.join(directory, name), "utf8"); + try { + return [name.replace(/\.md$/, ""), parseReviewerReply(reply)] as const; + } catch (error) { + throw new Error(`${name}: ${error instanceof Error ? error.message : String(error)}`, { + cause: error, + }); + } + }), + ), + ); + const baseline = values.baseline + ? ( + JSON.parse( + await NodeFSP.readFile( + NodePath.join(NodePath.resolve(values.baseline), "summary.json"), + "utf8", + ), + ) as { medians: Medians } + ).medians + : null; + const round = summarizeRound(reviewers); + await NodeFSP.writeFile( + NodePath.join(out, "summary.json"), + `${JSON.stringify(round, null, 2)}\n`, + ); + await NodeFSP.writeFile(NodePath.join(out, "summary.md"), renderSummary(round, baseline)); + log(`wrote ${NodePath.join(out, "summary.md")}`); +} + +const steps = new Set(values.capture ?? []); +for (const step of steps) { + if (step !== "mobile" && step !== "desktop") + throw new Error(`Unknown capture ${step}. Use mobile or desktop.`); +} +if (steps.has("mobile")) { + const startedAt = Date.now(); + const result = await captureMobile({ + outDirectory: out, + poses, + scenes, + flows: !values["no-flows"], + skipBuild: values["skip-build"], + log, + }); + await saveCaptures(result); + await writeRunInfo("mobile", startedAt); +} +if (steps.has("desktop")) { + const startedAt = Date.now(); + const shot = await findShot(values.shot); + if (!shot) + throw new Error("Desktop capture uses the shot skill. Install it, or pass --shot ."); + await saveCaptures({ + desktop: await captureDesktop({ + outDirectory: out, + shot, + scenes, + skipBuild: values["skip-build"], + log, + }), + }); + await writeRunInfo("desktop", startedAt); +} +if (values.pack || steps.size > 0) { + const captures = await readCaptures(); + const { brief } = await buildPack({ + outDirectory: out, + ...captures, + referencesDirectory: NodePath.join(import.meta.dirname, "references"), + panel, + }); + const problems = [ + ...captures.shots.filter((shot) => shot.problem).map((shot) => `${shot.pose}/${shot.scene}`), + ...captures.flows.filter((flow) => flow.problem).map((flow) => `flow ${flow.flow}`), + ...captures.desktop.filter((shot) => shot.problem).map((shot) => `desktop/${shot.scene}`), + ]; + log(`pack ready: ${brief}`); + if (problems.length > 0) + log(`check these captures, they did not reach their ready state: ${problems.join(", ")}`); +} +if (values.summarize) await summarize(); diff --git a/evals/mobile-design/scores.ts b/evals/mobile-design/scores.ts new file mode 100644 index 0000000000000..9d5a4509eb99c --- /dev/null +++ b/evals/mobile-design/scores.ts @@ -0,0 +1,92 @@ +import * as Schema from "effect/Schema"; + +import { AXES, DEVICE_CLASSES, type AxisId, type DeviceClassId } from "./cases.ts"; + +const Score = Schema.Number.check(Schema.isBetween({ minimum: 1, maximum: 10 })); +const DeviceScores = Schema.Struct( + Object.fromEntries(AXES.map((axis) => [axis.id, Score])) as Record, +); + +/** The JSON block every reviewer ends its reply with (see brief.md). */ +export const ReviewerScores = Schema.Struct({ + scores: Schema.Struct( + Object.fromEntries(DEVICE_CLASSES.map((device) => [device.id, DeviceScores])) as Record< + DeviceClassId, + typeof DeviceScores + >, + ), + fixes: Schema.Array(Schema.String).check(Schema.isMinLength(1)), +}); +export type ReviewerScores = typeof ReviewerScores.Type; + +const decodeScores = Schema.decodeUnknownSync(Schema.fromJsonString(ReviewerScores)); + +/** Reads the last fenced JSON block of a reviewer's reply. */ +export function parseReviewerReply(reply: string): ReviewerScores { + const blocks = [...reply.matchAll(/```json\s*\n([\s\S]*?)```/g)]; + const last = blocks.at(-1)?.[1]; + if (!last) throw new Error("The reply has no ```json block."); + return decodeScores(last); +} + +export function median(values: ReadonlyArray): number { + if (values.length === 0) return Number.NaN; + const sorted = values.toSorted((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2; +} + +export type Medians = Record>; + +export interface RoundSummary { + readonly reviewers: Readonly>; + readonly medians: Medians; +} + +export function summarizeRound(reviewers: Readonly>): RoundSummary { + const all = Object.values(reviewers); + const medians = Object.fromEntries( + DEVICE_CLASSES.map((device) => [ + device.id, + Object.fromEntries( + AXES.map((axis) => [ + axis.id, + median(all.map((review) => review.scores[device.id][axis.id])), + ]), + ), + ]), + ) as Medians; + return { reviewers, medians }; +} + +const formatScore = (value: number) => (Number.isNaN(value) ? "–" : value.toFixed(1)); +const formatDelta = (value: number) => + Number.isNaN(value) ? "–" : value === 0 ? "0.0" : `${value > 0 ? "+" : ""}${value.toFixed(1)}`; + +/** The summary table: one row per device and axis, with deltas when a baseline is given. */ +export function renderSummary(round: RoundSummary, baseline: Medians | null): string { + const names = Object.keys(round.reviewers); + const header = ["Device", "Axis", "Median", ...names, ...(baseline ? ["Δ baseline"] : [])]; + const rows = DEVICE_CLASSES.flatMap((device) => + AXES.map((axis) => { + const value = round.medians[device.id][axis.id]; + return [ + device.label, + axis.label, + formatScore(value), + ...names.map((name) => formatScore(round.reviewers[name]!.scores[device.id][axis.id])), + ...(baseline ? [formatDelta(value - baseline[device.id][axis.id])] : []), + ]; + }), + ); + const table = [header, header.map(() => "---"), ...rows] + .map((row) => `| ${row.join(" | ")} |`) + .join("\n"); + const fixes = names + .map( + (name) => + `### ${name}\n\n${round.reviewers[name]!.fixes.map((fix, index) => `${index + 1}. ${fix}`).join("\n")}`, + ) + .join("\n\n"); + return `# Mobile design scores\n\n${table}\n\n## Top fixes\n\n${fixes}\n`; +} diff --git a/evals/package.json b/evals/package.json index 681da5963142b..26f2aeb0cd1ed 100644 --- a/evals/package.json +++ b/evals/package.json @@ -5,6 +5,7 @@ "scripts": { "mcp-tool-use": "node --env-file-if-exists=../.env mcp-tool-use/run.ts", "mcp-tool-use:trim-capture": "node mcp-tool-use/trim-capture.ts", + "mobile-design": "node --env-file-if-exists=../.env mobile-design/run.ts", "thread-titles": "node --env-file-if-exists=../.env thread-titles/run.ts", "test-signal": "node --env-file-if-exists=../.env test-signal/run.ts", "typecheck": "tsc --noEmit", @@ -15,7 +16,8 @@ "@t3tools/contracts": "workspace:*", "@t3tools/provider-core": "workspace:*", "@t3tools/shared": "workspace:*", - "effect": "catalog:" + "effect": "catalog:", + "sharp": "0.35.4" }, "devDependencies": { "@effect/vitest": "catalog:", diff --git a/knip.jsonc b/knip.jsonc index c2093737fd942..4ecdb65d24535 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -30,6 +30,8 @@ "knip-schemas.ts", "mobile-behavior/run.ts", "mobile-native-client.ts", + // Showcase servers launch it as their Codex binary, through a shell shim. + "mobile-showcase-provider.ts", "setup-worktree.ts", "sign-macos.ts", "smoke-desktop-app.ts", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f2eabadf0a4f9..bca74d5de0d89 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1277,6 +1277,9 @@ importers: effect: specifier: 4.0.5 version: 4.0.5 + sharp: + specifier: 0.35.4 + version: 0.35.4(@types/node@24.12.4) devDependencies: '@effect/vitest': specifier: 4.0.5 diff --git a/scripts/mobile-showcase-environment.ts b/scripts/mobile-showcase-environment.ts index e7e2c76b6a2f2..036b935e83e36 100644 --- a/scripts/mobile-showcase-environment.ts +++ b/scripts/mobile-showcase-environment.ts @@ -248,6 +248,412 @@ export const SHOWCASE_THREADS = [ }, ] as const; +export type ShowcaseLiveStep = + | { + readonly type: "message"; + readonly text: string; + readonly phase?: "commentary" | "final_answer"; + /** Stop partway through and keep the turn running, for a streaming capture. */ + readonly hold?: boolean; + } + | { + readonly type: "command"; + readonly command: string; + readonly output: string; + readonly exitCode?: number; + readonly durationMs?: number; + /** Keep running after the turn ends, which the app shows as background work. */ + readonly background?: boolean; + } + | { + readonly type: "fileChange"; + readonly changes: ReadonlyArray<{ + readonly path: string; + readonly kind: "add" | "update"; + readonly diff: string; + }>; + } + | { + readonly type: "approval"; + readonly command: string; + readonly reason: string; + readonly output: string; + } + | { + readonly type: "question"; + readonly questions: ReadonlyArray<{ + readonly id: string; + readonly header: string; + readonly question: string; + readonly options: ReadonlyArray<{ readonly label: string; readonly description: string }>; + }>; + }; + +const RECONNECT_BANNER_DIFF = `@@ -0,0 +1,24 @@ ++import { Text, View } from "react-native"; ++ ++import { useReconnectState } from "./useReconnectState"; ++ ++export function ReconnectBanner(props: { readonly environmentId: string }) { ++ const state = useReconnectState(props.environmentId); ++ if (state.status === "connected") return null; ++ return ( ++ ++ ++ {state.status === "retrying" ++ ? \`Reconnecting in \${state.secondsLeft}s\` ++ : "Connection lost"} ++ ++ ++ ); ++} +`; + +const RECONNECT_STATE_DIFF = `@@ -12,9 +12,17 @@ export function useReconnectState(environmentId: string) { + const connection = useEnvironmentConnection(environmentId); +- const [attempt, setAttempt] = useState(0); ++ const [attempt, setAttempt] = useState(0); ++ const delayMs = Math.min(30_000, 1_000 * 2 ** attempt); + useEffect(() => { +- if (connection.status !== "lost") return; +- const timer = setTimeout(() => connection.retry(), 1_000); ++ if (connection.status !== "lost") { ++ setAttempt(0); ++ return; ++ } ++ const timer = setTimeout(() => { ++ setAttempt((value) => value + 1); ++ connection.retry(); ++ }, delayMs); + return () => clearTimeout(timer); +- }, [connection]); ++ }, [connection, delayMs]); +`; + +const LONG_CODE_ANSWER = `Here is the retry helper. It keeps one timer per environment, backs off to 30 seconds, and resets after a clean reconnect. + +\`\`\`ts +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Schedule from "effect/Schedule"; + +export interface ReconnectOptions { + readonly environmentId: string; + readonly maxDelay?: Duration.Duration; + readonly onAttempt?: (attempt: number, delay: Duration.Duration) => void; +} + +const DEFAULT_MAX_DELAY = Duration.seconds(30); + +/** Exponential backoff with jitter, capped, that never gives up on a paired environment. */ +export function reconnectSchedule(options: ReconnectOptions) { + const maxDelay = options.maxDelay ?? DEFAULT_MAX_DELAY; + return Schedule.exponential(Duration.seconds(1), 2).pipe( + Schedule.jittered, + Schedule.modifyDelay((_, delay) => Duration.min(delay, maxDelay)), + Schedule.tapOutput((delay) => + Effect.sync(() => options.onAttempt?.(Schedule.CurrentIterationMetadata.attempt, delay)), + ), + ); +} + +export const reconnect = Effect.fn("Environment.reconnect")(function* ( + options: ReconnectOptions, + connect: Effect.Effect, +) { + yield* Effect.logInfo("Reconnecting", { environmentId: options.environmentId }); + yield* connect.pipe(Effect.retry(reconnectSchedule(options))); + yield* Effect.logInfo("Reconnected", { environmentId: options.environmentId }); +}); + +export class ConnectionError extends Error { + readonly _tag = "ConnectionError"; + constructor( + readonly environmentId: string, + readonly reason: "timeout" | "refused" | "auth", + ) { + super(\`\${environmentId} could not connect: \${reason}\`); + } +} +\`\`\` + +Wire it into \`useEnvironmentConnection\` so the banner reads \`attempt\` and \`delay\` from the same schedule.`; + +/** + * Threads the scripted provider (mobile-showcase-provider.ts) runs live, so captures show real + * runs: tool calls, a diff, a long code block, a stream in progress, an approval, a question and + * background work. Each one starts with its prompt; the provider plays the steps that match. + */ +export const SHOWCASE_LIVE_THREADS = [ + { + id: "tool-calls", + projectId: "t3code", + title: "Show a reconnect banner", + prompt: "Show a banner when an environment drops, and retry with backoff.", + steps: [ + { + type: "message", + phase: "commentary", + text: "I'll find where connection state lives, then add the banner and backoff.", + }, + { + type: "command", + command: 'rg -n "useEnvironmentConnection" apps/mobile/src', + output: + "apps/mobile/src/features/home/HomeRouteScreen.tsx:41: const connection = useEnvironmentConnection(environmentId);\napps/mobile/src/features/connection/useReconnectState.ts:12: const connection = useEnvironmentConnection(environmentId);\n", + durationMs: 180, + }, + { + type: "fileChange", + changes: [ + { + path: "apps/mobile/src/features/connection/ReconnectBanner.tsx", + kind: "add", + diff: RECONNECT_BANNER_DIFF, + }, + { + path: "apps/mobile/src/features/connection/useReconnectState.ts", + kind: "update", + diff: RECONNECT_STATE_DIFF, + }, + ], + }, + { + type: "command", + command: "vp test run apps/mobile/src/features/connection", + output: + " ✓ useReconnectState.test.ts (9 tests) 41ms\n ✓ ReconnectBanner.test.ts (4 tests) 18ms\n\n Test Files 2 passed (2)\n Tests 13 passed (13)\n", + durationMs: 3_400, + }, + { + type: "message", + text: "The banner now appears as soon as an environment drops and counts down to the next retry.\n\n- Backoff doubles from 1s and caps at 30s\n- A clean reconnect resets the counter\n- 13 tests pass in `features/connection`", + }, + ], + }, + { + id: "long-code", + projectId: "t3code", + title: "Write the reconnect schedule", + prompt: "Write the reconnect schedule as an Effect helper with jittered backoff.", + steps: [{ type: "message", text: LONG_CODE_ANSWER }], + }, + { + id: "streaming", + projectId: "t3code", + title: "Cut thread list render cost", + prompt: "Profile the thread list and cut the cost of each row render.", + steps: [ + { + type: "message", + phase: "commentary", + text: "Measuring the row renders first, so the change has a baseline.", + }, + { + type: "command", + command: "vp run render-metrics --surface thread-list", + output: + "thread-list rows: 240 mean: 3.8ms p95: 9.6ms commits: 612\n ThreadRow 2.9ms (memo miss: relative time label)\n ProjectFavicon 0.6ms\n", + durationMs: 8_200, + }, + { + type: "message", + hold: true, + text: "Most of the cost is the relative time label. Every row re-renders once a minute because each label owns its own timer.\n\n1. **Share one clock.** A single minute ticker at the list level, read through context, so rows only re-render when their own label text changes.\n2. **Memo the row on its shell.** `ThreadRow` gets the thread shell and the label string, not the whole entity.\n3. **Keep favicons out of the row commit.**", + }, + ], + }, + { + id: "approval", + projectId: "t3code", + title: "Reset the Metro cache", + prompt: "Clear the stale Metro cache and restart the bundler.", + steps: [ + { + type: "message", + phase: "commentary", + text: "The bundle hash is stale. Clearing the cache needs a delete outside the source tree.", + }, + { + type: "approval", + command: "rm -rf apps/mobile/.expo apps/mobile/node_modules/.cache", + reason: "Delete cached Metro bundles so the next start rebuilds them.", + output: "", + }, + { + type: "message", + text: "Cache cleared. The next `vp run dev:client` rebuilds the bundle from scratch.", + }, + ], + }, + { + id: "question", + projectId: "react", + title: "Plan offline mode", + prompt: "Add an offline mode for the thread list.", + steps: [ + { + type: "message", + phase: "commentary", + text: "Two choices change the design, so I'll ask before writing code.", + }, + { + type: "question", + questions: [ + { + id: "cache_scope", + header: "Cache", + question: "How much history should the phone keep offline?", + options: [ + { + label: "Last 50 threads (Recommended)", + description: "Fast to load and small on disk. Older threads load when online.", + }, + { label: "Pinned threads only", description: "The smallest cache." }, + { label: "Everything", description: "Full history, at the cost of storage." }, + ], + }, + { + id: "conflicts", + header: "Sync", + question: "When the connection returns, which version wins?", + options: [ + { + label: "Server state (Recommended)", + description: "Local drafts stay, everything else follows the server.", + }, + { label: "Ask me each time", description: "Show a prompt for each conflict." }, + ], + }, + ], + }, + { + type: "message", + text: "Thanks. I'll cache the last 50 threads and let the server win on reconnect.", + }, + ], + }, + { + id: "background", + projectId: "t3code", + title: "Check settings in the dev server", + prompt: "Start the web dev server and keep it running while you check the settings page.", + steps: [ + { + type: "command", + command: "vp run dev --filter web", + background: true, + output: + " VITE v7.1.1 ready in 1.24s\n\n ➜ Local: http://127.0.0.1:5173/\n ➜ press h + enter to show help\n", + }, + { + type: "message", + text: "The dev server is running in the background on port 5173. Settings renders without console errors; I left the server up so you can open it.", + }, + ], + }, +] as const satisfies ReadonlyArray<{ + readonly id: string; + readonly projectId: string; + readonly title: string; + readonly prompt: string; + readonly steps: ReadonlyArray; +}>; + +export type ShowcaseLiveThreadId = (typeof SHOWCASE_LIVE_THREADS)[number]["id"]; + +/** + * Points the environment's only provider at the scripted Codex peer, so the model picker and + * every run are the same on every machine. Call before the server starts. + */ +export async function writeShowcaseProviderSettings(baseDir: string): Promise { + const binDirectory = NodePath.join(baseDir, "showcase-bin"); + const codexPath = NodePath.join(binDirectory, "codex"); + const providerScript = NodePath.join(import.meta.dirname, "mobile-showcase-provider.ts"); + await NodeFSP.mkdir(binDirectory, { recursive: true }); + await NodeFSP.writeFile( + codexPath, + `#!/bin/sh\nexec ${JSON.stringify(process.execPath)} ${JSON.stringify(providerScript)} "$@"\n`, + { mode: 0o755 }, + ); + const disabled = ["claudeAgent", "cursor", "grok", "opencode", "antigravity", "pi"]; + await NodeFSP.mkdir(NodePath.join(baseDir, "userdata"), { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(baseDir, "userdata", "settings.json"), + `${JSON.stringify( + { + providerInstances: { + codex: { driver: "codex", enabled: true, config: { binaryPath: codexPath } }, + ...Object.fromEntries( + disabled.map((instanceId) => [instanceId, { driver: instanceId, enabled: false }]), + ), + }, + }, + null, + 2, + )}\n`, + ); +} + +/** The server lists itself for the CLI a moment after it starts listening; retry until then. */ +async function launchThreadWhenServerIsUp( + launch: () => Promise<{ readonly stdout: string }>, +): Promise { + for (let attempt = 1; ; attempt += 1) { + try { + return (await launch()).stdout; + } catch (error) { + const notListed = String((error as { stderr?: unknown }).stderr ?? "").includes( + "No running T3 Code server", + ); + if (!notListed || attempt >= 40) throw error; + await new Promise((resolve) => setTimeout(resolve, 500)); + } + } +} + +/** Starts each live thread whose project this environment owns; returns their thread ids. */ +export async function startShowcaseLiveThreads(input: { + readonly baseDir: string; + readonly projectIds: ReadonlyArray; + readonly repoRoot: string; +}): Promise> { + const threads = SHOWCASE_LIVE_THREADS.filter((thread) => + input.projectIds.includes(thread.projectId), + ); + const launched = await Promise.all( + threads.map(async (thread) => { + const stdout = await launchThreadWhenServerIsUp(() => { + const launch = execFile( + process.execPath, + [ + "apps/server/src/bin.ts", + "thread", + "start", + thread.projectId, + thread.prompt, + "--title", + thread.title, + "--base-dir", + input.baseDir, + "--json", + ], + { cwd: input.repoRoot, env: { ...process.env, NO_COLOR: "1" } }, + ); + // The CLI reads a piped stdin to its end, so close it. + launch.child.stdin?.end(); + return launch; + }); + const { threadId } = JSON.parse( + stdout.slice(stdout.indexOf("{"), stdout.lastIndexOf("}") + 1), + ) as { readonly threadId?: string }; + if (!threadId) throw new Error(`Live thread '${thread.id}' did not start.`); + return [thread.id, threadId] as const; + }), + ); + return new Map(launched); +} + function minutesBefore(now: number, minutes: number): string { return new Date(now - minutes * 60_000).toISOString(); } diff --git a/scripts/mobile-showcase-provider.ts b/scripts/mobile-showcase-provider.ts new file mode 100644 index 0000000000000..7c7c61b2a8668 --- /dev/null +++ b/scripts/mobile-showcase-provider.ts @@ -0,0 +1,437 @@ +#!/usr/bin/env node +// @effect-diagnostics nodeBuiltinImport:off globalTimers:off globalDate:off - A standalone stdio peer with no Effect runtime. + +/** + * A scripted `codex app-server`, so showcase fixtures get real runs from the real server: each + * turn plays the SHOWCASE_LIVE_THREADS steps whose prompt matches. Frame shapes follow the + * recorded transcripts in apps/server/src/orchestration-v2/testkit/fixtures. + */ + +import * as NodeCrypto from "node:crypto"; +import * as NodeReadline from "node:readline"; + +import { SHOWCASE_LIVE_THREADS, type ShowcaseLiveStep } from "./mobile-showcase-environment.ts"; + +const VERSION = "0.157.0"; +const MODELS = [ + { id: "gpt-5.4", displayName: "GPT-5.4", description: "Flagship model for complex coding work." }, + { + id: "gpt-5.4-mini", + displayName: "GPT-5.4 Mini", + description: "Fast model for everyday edits.", + }, + { + id: "gpt-5.3-codex", + displayName: "GPT-5.3 Codex", + description: "Tuned for long agentic runs.", + }, +] as const; + +type Json = + | null + | boolean + | number + | string + | ReadonlyArray + | { readonly [key: string]: Json }; +type Frame = { readonly [key: string]: Json }; + +const write = (frame: Frame) => process.stdout.write(`${JSON.stringify(frame)}\n`); +const notify = (method: string, params: Frame) => write({ method, params }); +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const nowSeconds = () => Math.floor(Date.now() / 1000); +const newId = (prefix: string) => `${prefix}-${NodeCrypto.randomUUID()}`; + +let nextServerRequestId = 0; +let nextProcessId = 4_100; +const pendingServerRequests = new Map void>(); +const threadCwds = new Map(); +/** Background commands still running, by process id, so terminate can finish them. */ +const backgroundCommands = new Map(); +const interruptedTurns = new Set(); + +function serverRequest(method: string, params: Frame): Promise { + const id = nextServerRequestId++; + write({ method, id, params }); + return new Promise((resolve) => pendingServerRequests.set(id, resolve)); +} + +function threadPayload(threadId: string, cwd: string, model: string): Frame { + const now = nowSeconds(); + return { + id: threadId, + environments: [{ environmentId: "local", cwd, runtimeWorkspaceRoots: [cwd] }], + extra: null, + sessionId: threadId, + forkedFromId: null, + parentThreadId: null, + preview: "", + ephemeral: false, + section: null, + sectionEnteredAt: null, + projectId: null, + historyMode: "paginated", + modelProvider: "openai", + model, + reasoningEffort: null, + createdAt: now, + updatedAt: now, + recencyAt: now, + status: { type: "idle" }, + path: null, + cwd, + cliVersion: VERSION, + originator: "t3code_desktop", + source: "vscode", + canAcceptDirectInput: true, + threadSource: null, + agentNickname: null, + agentRole: null, + gitInfo: null, + name: null, + daybreakEnabled: null, + turns: [], + }; +} + +function turnPayload(turnId: string, status: string, startedAt: number | null): Frame { + return { + id: turnId, + items: [], + itemsView: status === "inProgress" ? "notLoaded" : "summary", + status, + error: null, + startedAt, + completedAt: status === "inProgress" ? null : nowSeconds(), + durationMs: null, + }; +} + +function commandItem( + id: string, + command: string, + cwd: string, + options: { readonly processId: string | null; readonly background: boolean }, +) { + return { + type: "commandExecution", + id, + pluginId: null, + scriptPath: null, + command: `/bin/zsh -lc ${JSON.stringify(command)}`, + cwd, + processId: options.processId, + source: options.background ? "unifiedExecStartup" : "agent", + status: "inProgress", + commandActions: [{ type: "unknown", command }], + aggregatedOutput: null, + exitCode: null, + durationMs: null, + }; +} + +/** Plays one scripted turn. Returns when the turn completes, or never for a held stream. */ +async function playTurn(threadId: string, turnId: string, prompt: string, cwd: string) { + const startedAt = nowSeconds(); + const turn = { threadId, turnId }; + const stopped = () => interruptedTurns.has(turnId); + notify("thread/status/changed", { threadId, status: { type: "active", activeFlags: [] } }); + notify("turn/started", { threadId, turn: turnPayload(turnId, "inProgress", startedAt) }); + const userItem = { + type: "userMessage", + id: newId("user"), + clientId: null, + content: [{ type: "text", text: prompt, text_elements: [] }], + }; + notify("item/started", { ...turn, item: userItem, startedAtMs: Date.now() }); + notify("item/completed", { ...turn, item: userItem, completedAtMs: Date.now() }); + + const script = SHOWCASE_LIVE_THREADS.find((thread) => thread.prompt === prompt.trim())?.steps ?? [ + { type: "message", text: "On it. I will report back here when the change is ready." } as const, + ]; + for (const step of script) { + if (stopped()) return; + await playStep(step, turn, cwd); + } + if (stopped()) return; + notify("thread/status/changed", { threadId, status: { type: "idle" } }); + notify("turn/completed", { threadId, turn: turnPayload(turnId, "completed", startedAt) }); +} + +async function playStep( + step: ShowcaseLiveStep, + turn: { readonly threadId: string; readonly turnId: string }, + cwd: string, +) { + switch (step.type) { + case "message": { + const item = { + type: "agentMessage", + id: newId("msg"), + text: "", + phase: step.phase ?? "final_answer", + memoryCitation: null, + delivery: null, + questions: null, + }; + notify("item/started", { ...turn, item, startedAtMs: Date.now() }); + // Whole lines keep markdown blocks intact while the text streams. + const chunks = step.text.match(/[^\n]*\n|[^\n]+$/g) ?? []; + const shown = step.hold ? chunks.slice(0, Math.ceil(chunks.length * 0.6)) : chunks; + for (const delta of shown) { + notify("item/agentMessage/delta", { ...turn, itemId: item.id, delta }); + await sleep(25); + } + // A held message keeps streaming until the turn is interrupted. + if (step.hold) await new Promise(() => undefined); + notify("item/completed", { + ...turn, + item: { ...item, text: step.text }, + completedAtMs: Date.now(), + }); + return; + } + case "command": { + const processId = step.background ? String(nextProcessId++) : null; + const item = commandItem(newId("exec"), step.command, cwd, { + processId, + background: step.background === true, + }); + notify("item/started", { ...turn, item, startedAtMs: Date.now() }); + notify("item/commandExecution/outputDelta", { ...turn, itemId: item.id, delta: step.output }); + if (step.background && processId) { + backgroundCommands.set(processId, { + ...turn, + item: { ...item, aggregatedOutput: step.output }, + }); + return; + } + // Real time, capped, so "Worked for" reads like a real run without slowing fixtures down. + await sleep(Math.min(step.durationMs ?? 1_200, 2_500)); + notify("item/completed", { + ...turn, + item: { + ...item, + status: "completed", + aggregatedOutput: step.output, + exitCode: step.exitCode ?? 0, + durationMs: step.durationMs ?? 1_200, + }, + completedAtMs: Date.now(), + }); + return; + } + case "fileChange": { + const item = { + type: "fileChange", + id: newId("patch"), + changes: step.changes.map((change) => ({ + path: `${cwd}/${change.path}`, + kind: change.kind === "add" ? { type: "add" } : { type: "update", move_path: null }, + diff: change.diff, + })), + status: "inProgress", + }; + notify("item/started", { ...turn, item, startedAtMs: Date.now() }); + await sleep(40); + notify("item/completed", { + ...turn, + item: { ...item, status: "completed" }, + completedAtMs: Date.now(), + }); + return; + } + case "approval": { + const item = commandItem(newId("exec"), step.command, cwd, { + processId: null, + background: false, + }); + notify("item/started", { ...turn, item, startedAtMs: Date.now() }); + const requestId = nextServerRequestId; + const result = await serverRequest("item/commandExecution/requestApproval", { + kind: "command", + ...turn, + itemId: item.id, + startedAtMs: Date.now(), + environmentId: "local", + reason: step.reason, + command: item.command, + cwd, + commandActions: item.commandActions, + proposedExecpolicyAmendment: null, + availableDecisions: ["accept", "acceptForSession", "decline"], + }); + notify("serverRequest/resolved", { threadId: turn.threadId, requestId }); + const accepted = result.decision === "accept" || result.decision === "acceptForSession"; + notify("item/completed", { + ...turn, + item: accepted + ? { + ...item, + status: "completed", + aggregatedOutput: step.output, + exitCode: 0, + durationMs: 900, + } + : { ...item, status: "declined" }, + completedAtMs: Date.now(), + }); + return; + } + case "question": { + const requestId = nextServerRequestId; + await serverRequest("item/tool/requestUserInput", { + ...turn, + itemId: newId("call"), + questions: step.questions.map((question) => ({ + ...question, + isOther: false, + isSecret: false, + })), + isBlocking: true, + autoResolutionMs: null, + }); + notify("serverRequest/resolved", { threadId: turn.threadId, requestId }); + return; + } + } +} + +function respond(id: Json, result: Json) { + write({ id, result }); +} + +function handleRequest(id: Json, method: string, params: Frame) { + switch (method) { + case "initialize": + return respond(id, { + userAgent: `T3 Showcase/${VERSION} (Mac OS 26.0.0; arm64)`, + codexHome: process.env.CODEX_HOME ?? process.cwd(), + platformFamily: "unix", + platformOs: "macos", + }); + case "account/read": + return respond(id, { account: { type: "apiKey" }, requiresOpenaiAuth: false }); + case "model/list": + return respond(id, { + data: MODELS.map((model, index) => ({ + ...model, + model: model.id, + hidden: false, + isDefault: index === 0, + defaultReasoningEffort: "medium", + supportedReasoningEfforts: ["low", "medium", "high", "xhigh"].map((effort) => ({ + reasoningEffort: effort, + description: `${effort[0]?.toUpperCase()}${effort.slice(1)} reasoning`, + })), + inputModalities: ["text", "image"], + })), + nextCursor: null, + }); + case "skills/list": + return respond(id, { data: [] }); + case "config/read": + return respond(id, { config: {}, origins: {} }); + case "thread/start": + case "thread/resume": { + const threadId = typeof params.threadId === "string" ? params.threadId : newId("thread"); + const cwd = typeof params.cwd === "string" ? params.cwd : process.cwd(); + const model = typeof params.model === "string" ? params.model : MODELS[0].id; + threadCwds.set(threadId, cwd); + const thread = threadPayload(threadId, cwd, model); + respond(id, { + thread, + model, + modelProvider: "openai", + serviceTier: null, + disabledPluginIds: [], + cwd, + runtimeWorkspaceRoots: [cwd], + instructionSources: [], + approvalPolicy: "on-request", + approvalsReviewer: "user", + sandbox: { + type: "workspaceWrite", + writableRoots: [], + networkAccess: false, + excludeTmpdirEnvVar: false, + excludeSlashTmp: false, + }, + activePermissionProfile: { id: ":workspace", extends: null }, + reasoningEffort: null, + multiAgentMode: "explicitRequestOnly", + }); + return notify("thread/started", { thread }); + } + case "turn/start": { + const threadId = String(params.threadId); + const turnId = newId("turn"); + const input = Array.isArray(params.input) ? params.input : []; + const prompt = input + .map((entry) => + entry && typeof entry === "object" && "text" in entry ? String(entry.text) : "", + ) + .join("\n"); + respond(id, { turn: turnPayload(turnId, "inProgress", null) }); + void playTurn(threadId, turnId, prompt, threadCwds.get(threadId) ?? process.cwd()); + return; + } + case "turn/interrupt": { + const threadId = String(params.threadId); + const turnId = String(params.turnId); + interruptedTurns.add(turnId); + respond(id, {}); + notify("thread/status/changed", { threadId, status: { type: "idle" } }); + return notify("turn/completed", { threadId, turn: turnPayload(turnId, "interrupted", null) }); + } + case "thread/backgroundTerminals/list": + return respond(id, { + data: [...backgroundCommands.keys()].map((processId) => ({ processId })), + nextCursor: null, + }); + case "thread/backgroundTerminals/terminate": { + const processId = String(params.processId); + const running = backgroundCommands.get(processId); + backgroundCommands.delete(processId); + respond(id, { terminated: true }); + if (running) { + notify("item/completed", { + threadId: running.threadId, + turnId: running.turnId, + item: { ...running.item, status: "completed", exitCode: 143, durationMs: 60_000 }, + completedAtMs: Date.now(), + }); + } + return; + } + default: + return write({ id, error: { code: -32601, message: `${method} is not scripted.` } }); + } +} + +function main() { + const args = process.argv.slice(2); + if (args.includes("--version")) { + process.stdout.write(`codex-cli ${VERSION}\n`); + return; + } + if (args[0] !== "app-server") process.exit(1); + const lines = NodeReadline.createInterface({ input: process.stdin }); + lines.on("line", (line) => { + if (!line.trim()) return; + const frame = JSON.parse(line) as Frame; + const { id, method } = frame; + if (typeof method === "string") { + if (id !== undefined) handleRequest(id, method, (frame.params ?? {}) as Frame); + return; + } + // A response to one of this peer's requests. + if (typeof id === "number") { + pendingServerRequests.get(id)?.((frame.result ?? {}) as Frame); + pendingServerRequests.delete(id); + } + }); + lines.on("close", () => process.exit(0)); +} + +main(); diff --git a/scripts/mobile-showcase.ts b/scripts/mobile-showcase.ts index 17a609b989472..8adcfbc38ad67 100644 --- a/scripts/mobile-showcase.ts +++ b/scripts/mobile-showcase.ts @@ -27,7 +27,10 @@ import { SHOWCASE_ENVIRONMENTS, SHOWCASE_PROJECTS, SHOWCASE_THREADS, + type ShowcaseLiveThreadId, seedShowcaseEnvironment, + startShowcaseLiveThreads, + writeShowcaseProviderSettings, } from "./mobile-showcase-environment.ts"; const REPO_ROOT = NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), ".."); @@ -39,7 +42,7 @@ const DEV_CLIENT_LAUNCH_FLAGS = "disableOnboarding=1&disableFab=1&disableAutoLau const IOS_READY_FILENAME = "T3ShowcaseReadyScene"; const SERVER_HOST = "0.0.0.0"; const IOS_SIMULATOR_ARCH = NodeProcess.arch === "arm64" ? "arm64" : "x86_64"; -const IOS_APP_PATH = NodePath.join( +export const IOS_APP_PATH = NodePath.join( MOBILE_ROOT, ".showcase/ios-derived-data/Build/Products/Debug-iphonesimulator/T3Code.app", ); @@ -644,13 +647,19 @@ export interface ShowcaseEnvironment { readonly environmentId: string; readonly label: string; readonly port: number; + /** Server thread ids of the live threads this environment runs, when started with them. */ + readonly liveThreadIds: ReadonlyMap; } -/** Boot one seeded fixture server per showcase environment; `track` receives every spawned server. */ +/** + * Boot one seeded fixture server per showcase environment; `track` receives every spawned server. + * `liveThreads` swaps Codex for the scripted provider and starts SHOWCASE_LIVE_THREADS on it. + */ export async function startShowcaseEnvironments( rootDir: string, track: (server: NodeChildProcess.ChildProcess) => void, environments: ReadonlyArray<(typeof SHOWCASE_ENVIRONMENTS)[number]> = SHOWCASE_ENVIRONMENTS, + options: { readonly liveThreads?: boolean } = {}, ): Promise { const started: ShowcaseEnvironment[] = []; for (const environment of environments) { @@ -664,6 +673,7 @@ export async function startShowcaseEnvironments( await NodeFSP.mkdir(workspaceRoot, { recursive: true }); const shellPath = await createShowcaseShell(baseDir); const labelProbeDirectory = await createShowcaseLabelProbe(baseDir, environment.label); + if (options.liveThreads) await writeShowcaseProviderSettings(baseDir); const server = startShowcaseServer( baseDir, workspaceRoot, @@ -691,7 +701,14 @@ export async function startShowcaseEnvironments( NodePath.join(baseDir, "userdata", "environment-id"), `${environment.label} environment id`, ); - started.push({ baseDir, environmentId, label: environment.label, port }); + const liveThreadIds = options.liveThreads + ? await startShowcaseLiveThreads({ + baseDir, + projectIds: environment.projectIds, + repoRoot: REPO_ROOT, + }) + : new Map(); + started.push({ baseDir, environmentId, label: environment.label, port, liveThreadIds }); } return started; } @@ -730,18 +747,18 @@ export function encodeAndroidPairingUrls(pairingUrls: ReadonlyArray): st return `json-uri:${encodeURIComponent(JSON.stringify(pairingUrls))}`; } -function startMetro(config: ShowcaseConfig): NodeChildProcess.ChildProcess { - return spawnProcess( - "pnpm", - ["exec", "expo", "start", "--dev-client", "--port", String(config.metroPort)], - { - cwd: MOBILE_ROOT, - env: { - ...MOBILE_BUILD_ENV, - EXPO_PUBLIC_SHOWCASE: "1", - }, +/** Metro for the capture build; `showcase` turns on the in-app capture coordinator. */ +export function startShowcaseMetro( + port: number, + options: { readonly showcase: boolean }, +): NodeChildProcess.ChildProcess { + return spawnProcess("pnpm", ["exec", "expo", "start", "--dev-client", "--port", String(port)], { + cwd: MOBILE_ROOT, + env: { + ...MOBILE_BUILD_ENV, + ...(options.showcase ? { EXPO_PUBLIC_SHOWCASE: "1" } : {}), }, - ); + }); } async function warmMetroBundle( @@ -753,7 +770,7 @@ async function warmMetroBundle( await runCommand("curl", ["--fail", "--silent", "--show-error", "--output", "/dev/null", url]); } -async function buildIos(): Promise { +export async function buildIos(): Promise { const derivedData = NodePath.join(MOBILE_ROOT, ".showcase/ios-derived-data"); await runCommand("pnpm", ["exec", "expo", "prebuild", "--clean", "--platform", "ios"], { cwd: MOBILE_ROOT, @@ -868,7 +885,7 @@ async function iosSimulatorDataPath(udid: string): Promise { } // Apple Intelligence posts persistent first-boot banners; mark readiness and clear queued CoreFollowUp notices. -async function suppressIosSystemFollowUps(udid: string): Promise { +export async function suppressIosSystemFollowUps(udid: string): Promise { const dataPath = await iosSimulatorDataPath(udid); const preferences = NodePath.join(dataPath, "Library/Preferences"); await NodeFSP.mkdir(preferences, { recursive: true }); @@ -890,7 +907,10 @@ async function suppressIosSystemFollowUps(udid: string): Promise { }); } -async function normalizeIosSimulator(appearance: ShowcaseAppearance, udid: string): Promise { +export async function normalizeIosSimulator( + appearance: ShowcaseAppearance, + udid: string, +): Promise { await runCommand("xcrun", ["simctl", "ui", udid, "appearance", appearance]); // Disable always-on dimming so lock-screen wake can distinguish off from fully lit. await runCommand("xcrun", [ @@ -923,7 +943,7 @@ async function normalizeIosSimulator(appearance: ShowcaseAppearance, udid: strin } // iPadOS 26 windowing forces fullscreen apps into portrait, defeating capture-build rotation. -async function ensureIosFullScreenAppsMode(udid: string): Promise { +export async function ensureIosFullScreenAppsMode(udid: string): Promise { const current = await commandOutput("xcrun", [ "simctl", "spawn", @@ -1567,7 +1587,7 @@ async function main(): Promise { ); if (!options.skipMetro) { - metro = startMetro(showcaseConfig); + metro = startShowcaseMetro(showcaseConfig.metroPort, { showcase: true }); await waitForPort(showcaseConfig.metroPort, "Metro"); await Promise.all([ hasIos ? warmMetroBundle("ios", metroHost, showcaseConfig) : Promise.resolve(), From e7926d4ddc55f70a5699355ebe6e6b9a9edf0abe Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 27 Oct 2026 02:14:02 -0700 Subject: [PATCH 02/21] fix(evals): council round 1 fixes for the mobile design eval Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/brief.md | 15 ++---- evals/mobile-design/capture-mobile.ts | 10 ++-- evals/mobile-design/cases.ts | 75 ++++++++++++-------------- evals/mobile-design/ios.ts | 30 +++++++---- evals/mobile-design/pack.ts | 9 ++-- evals/mobile-design/run.test.ts | 12 ++++- evals/mobile-design/run.ts | 20 ++++--- evals/mobile-design/scores.ts | 12 +++-- scripts/mobile-showcase-environment.ts | 6 +-- scripts/mobile-showcase-provider.ts | 43 ++++++++------- 10 files changed, 127 insertions(+), 105 deletions(-) diff --git a/evals/mobile-design/brief.md b/evals/mobile-design/brief.md index cdd86f81bcad6..aa3dbf666cade 100644 --- a/evals/mobile-design/brief.md +++ b/evals/mobile-design/brief.md @@ -6,10 +6,10 @@ T3 Code is a client for coding agents. People use the phone and tablet apps to w ## What is in the pack -Every image is in `{{PACK}}`. Read each one before you score. +Image paths are relative to the folder that holds this brief. Read each image before you score. - `screens//.png`: one scene on one device shape. Each tile is labelled with its device, appearance and text size. -- `desktop/.png`: the same data in the desktop web app at 1440×900, next to the iPad and iPhone captures, for the "conformance with desktop" axis. +- `desktop/.png`: the same data in the desktop web app at 1440×900, next to the iPad and iPhone captures, for the "conformance with desktop" axis. The desktop app is connected to one machine, so its lists are shorter than the phone's. - `flows/.png`: a common task, one frame per step. The label above each frame names the action that led to it. The data is a fixture. Thread titles, projects and machine names are made up, and project paths point at temporary folders. Do not score the fixture's wording or paths. Do score how the app lays them out, for example how it truncates a long path. @@ -30,21 +30,14 @@ References for the Codex and Claude iOS apps: Score each axis from 1 to 10 for each device class. 10 is the best mobile app you know for this kind of work. 7 is solid but clearly behind the best. -Axes: +Axes, by id: {{AXES}} -Device classes: +Device classes, by id: {{DEVICES}} -- **Visual design**: type, color, icons, spacing and hierarchy. -- **Layout quality**: use of the screen, safe areas, the keyboard, folds and rotation, large text. -- **Flow clarity**: how obvious each task is, from the flows and from what each screen offers next. -- **Polish**: alignment, truncation, clipped or overlapping content, empty space, state indicators. -- **Conformance with desktop**: whether this reads as the same product as the desktop app. -- **Quality versus Codex and Claude**: how this compares with the Codex and Claude iOS apps. - Then list the five fixes that would raise the scores most, most important first. Each fix names the screen and device, what is wrong, and the change you would make. Be specific: "the question panel leaves a 120 pt scroll area on the closed Duo; let it take the space above the composer" is useful, "improve spacing" is not. ## Reply format diff --git a/evals/mobile-design/capture-mobile.ts b/evals/mobile-design/capture-mobile.ts index 171311f594f63..3d4d4e8a4758a 100644 --- a/evals/mobile-design/capture-mobile.ts +++ b/evals/mobile-design/capture-mobile.ts @@ -23,10 +23,10 @@ import { SHOWCASE_ENVIRONMENTS, SHOWCASE_PROJECTS, } from "../../scripts/mobile-showcase-environment.ts"; -import { FLOWS, SCENES, type DeviceClassId, type Flow, type Pose, type SceneId } from "./cases.ts"; +import { FLOWS, SCENES, type Flow, type Pose, type SceneId } from "./cases.ts"; import { delay, linkTargets, type LinkTargets, run, Simulator } from "./ios.ts"; -/** One screenshot of the run, as `captures/mobile.json` records it. */ +/** One screenshot of the run, as `captures/captures.json` records it. */ export interface MobileShot { readonly pose: string; readonly scene: SceneId; @@ -227,7 +227,11 @@ export async function captureMobile(input: { .url; await run("curl", ["-sf", "-o", "/dev/null", bundleUrl], { timeoutMs: 900_000 }); - const devices = [...new Set(input.poses.map((pose) => pose.device))] as DeviceClassId[]; + // The flows answer the question and stop background work on the shared servers, so the + // iPhone, which runs them, goes last. + const devices = [...new Set(input.poses.map((pose) => pose.device))].toSorted( + (a, b) => Number(a === "iphone") - Number(b === "iphone"), + ); for (const device of devices) { input.log(`${device}: creating the simulator`); const simulator = await Simulator.create(device, runId); diff --git a/evals/mobile-design/cases.ts b/evals/mobile-design/cases.ts index c0eb8cdd3ad81..617fd55125833 100644 --- a/evals/mobile-design/cases.ts +++ b/evals/mobile-design/cases.ts @@ -5,12 +5,37 @@ */ export const AXES = [ - { id: "visual", label: "Visual design" }, - { id: "layout", label: "Layout quality" }, - { id: "flow", label: "Flow clarity" }, - { id: "polish", label: "Polish" }, - { id: "desktop", label: "Conformance with desktop" }, - { id: "competitors", label: "Quality versus Codex and Claude" }, + { + id: "visual", + label: "Visual design", + description: "type, color, icons, spacing and hierarchy", + }, + { + id: "layout", + label: "Layout quality", + description: "use of the screen, safe areas, the keyboard, folds and rotation, large text", + }, + { + id: "flow", + label: "Flow clarity", + description: "how obvious each task is, from the flows and from what each screen offers next", + }, + { + id: "polish", + label: "Polish", + description: + "alignment, truncation, clipped or overlapping content, empty space, state indicators", + }, + { + id: "desktop", + label: "Conformance with desktop", + description: "whether this reads as the same product as the desktop app", + }, + { + id: "competitors", + label: "Quality versus Codex and Claude", + description: "how this compares with the Codex and Claude iOS apps", + }, ] as const; export type AxisId = (typeof AXES)[number]["id"]; @@ -115,7 +140,6 @@ export const SCENES = [ title: "Composer with a long prompt", source: REQUEST, steps: [thread("pocket-command-center")], - desktop: { path: "/{thread:pocket-command-center}" }, }, { id: "composer-attachments", @@ -204,37 +228,9 @@ export const SCENES = [ ] as const satisfies ReadonlyArray; export type SceneId = (typeof SCENES)[number]["id"]; -/** Scenes where width and folding change the layout most. */ -const LAYOUT_SCENES: ReadonlyArray = [ - "threads", - "thread-tool-calls", - "thread-streaming", - "composer-model-picker", - "new-thread", - "question", - "approval", - "background", - "terminal", - "review", - "settings", -]; -const LIGHT_SCENES: ReadonlyArray = [ - "threads", - "thread-tool-calls", - "question", - "review", - "settings", -]; - export const POSES = [ { id: "iphone-dark", device: "iphone", label: "iPhone, dark", appearance: "dark" }, - { - id: "iphone-light", - device: "iphone", - label: "iPhone, light", - appearance: "light", - scenes: LIGHT_SCENES, - }, + { id: "iphone-light", device: "iphone", label: "iPhone, light", appearance: "light" }, { id: "iphone-large-text", device: "iphone", @@ -260,7 +256,6 @@ export const POSES = [ label: `iPhone Duo ${fold}, ${appearance}`, appearance, fold, - scenes: appearance === "dark" ? LAYOUT_SCENES : LIGHT_SCENES, })), ), ...(["landscape", "portrait"] as const).flatMap((orientation) => @@ -270,7 +265,6 @@ export const POSES = [ label: `iPad ${orientation}, ${appearance}`, appearance, orientation, - scenes: appearance === "dark" ? [...LAYOUT_SCENES, "environments"] : LIGHT_SCENES, })), ), ] as const satisfies ReadonlyArray; @@ -279,7 +273,7 @@ export interface Flow { readonly id: string; readonly title: string; readonly source: string; - /** Each step is one screenshot; taps are counted from `tap` steps. */ + /** Each step is one screenshot; `tap` and `focus` steps count as taps. */ readonly steps: ReadonlyArray; } @@ -294,7 +288,8 @@ export const FLOWS = [ { tap: "^T3 Code$" }, { tap: "^React$" }, { tap: "^Choose environment" }, - { tap: "^Suspense Station" }, + // Pick whichever machine the picker offers; when it is not the project's, the frames show it. + { tap: "^(Suspense Station|Moonbase Terminal|Kernel Cabin)" }, { focus: "composer" }, { type: "Add a loading skeleton to the profile page" }, { tap: "^Send$" }, diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index 0ca98fc91c8c6..55fe64ae4a207 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -148,7 +148,7 @@ export class Simulator { /** * Loads the JS straight from Metro. The dev menu's preferences hide its tools button and - * launch sheet; a dev-launcher link would set them too, but iOS asks before opening it. + * launch sheet; the same flags on --initialUrl do not reach them. */ async launch(metroUrl: string) { for (const [key, value] of [ @@ -158,24 +158,26 @@ export class Simulator { ] as const) { await simctl("spawn", this.udid, "defaults", "write", BUNDLE_ID, key, "-bool", value); } - // Attaches the agent-device session, which every later command needs. - await this.agent(["open", BUNDLE_ID], 600_000); const loaded = /^(Chats|Add environment|New chat|Search threads)$/; for (let attempt = 0; attempt < 3; attempt += 1) { + // Launch arguments only reach a fresh process, so start it before agent-device attaches. await simctl( "launch", - "--terminate-running-process", + ...(attempt > 0 ? ["--terminate-running-process"] : []), this.udid, BUNDLE_ID, "--initialUrl", metroUrl, ); + // The first snapshot attaches the agent-device session, which can take minutes. + await this.agent(["open", BUNDLE_ID], 600_000); const snapshot = await this.waitFor( new RegExp(`${loaded.source}|^Reload$`), "the app to load", 180_000, ).catch(() => null); if (snapshot && this.find(snapshot, loaded)) return; + if (snapshot) await this.tap(/^Reload$/).catch(() => undefined); } throw new Error("The app did not load from Metro."); } @@ -183,17 +185,18 @@ export class Simulator { async pair(pairingUrls: ReadonlyArray, firstThreadTitle: RegExp) { for (const pairingUrl of pairingUrls) { const query = new URLSearchParams({ pairingUrl, autoConnect: "1" }); - await this.openUrl(`t3code://connections/new?${query}`); + // Pairing runs once per device, so it can wait out a slow host. + await this.openUrl(`t3code://connections/new?${query}`, 180_000); await delay(4_000); } - await this.waitFor(firstThreadTitle, "the paired thread list", 90_000); + await this.waitFor(firstThreadTitle, "the paired thread list", 180_000); } /** iOS asks before a link opens the app; accept the prompt once it shows. */ - async openUrl(url: string) { + async openUrl(url: string, timeoutMs = 60_000) { await simctl("openurl", this.udid, url); // A busy host can take seconds per XCTest call, and the prompt itself can be late. - const deadline = Date.now() + 60_000; + const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { const accepted = await this.agent(["alert", "accept"]).then( () => true, @@ -340,12 +343,19 @@ export class Simulator { /** agent-device names rotations from the natural orientation, so try each until the shape fits. */ private async orient(shape: "portrait" | "landscape") { - for (const name of ["portrait", "landscape-left", "landscape-right"]) { + const fits = async () => { const { width, height } = (await this.snapshot()).viewport; - if (width > height === (shape === "landscape")) return; + return width > height === (shape === "landscape"); + }; + if (await fits()) return; + for (const name of shape === "landscape" + ? ["landscape-left", "landscape-right"] + : ["portrait"]) { await this.agent(["orientation", name]); await delay(2_000); + if (await fits()) return; } + throw new Error(`could not rotate to ${shape}`); } async screenshot(destination: string, pose: Pose) { diff --git a/evals/mobile-design/pack.ts b/evals/mobile-design/pack.ts index 1edf2dc15f904..1eecdcccf0da4 100644 --- a/evals/mobile-design/pack.ts +++ b/evals/mobile-design/pack.ts @@ -100,7 +100,8 @@ export async function buildPack(input: { const entries: PackEntry[] = []; for (const scene of SCENES) { - const shots = input.shots.filter((shot) => shot.scene === scene.id); + // A capture that missed its ready state shows the harness, not the design; run.ts lists it. + const shots = input.shots.filter((shot) => shot.scene === scene.id && !shot.problem); const groups = new Map(); for (const shot of shots) { const pose = POSES.find((candidate) => candidate.id === shot.pose); @@ -189,7 +190,6 @@ export async function buildPack(input: { const template = await NodeFSP.readFile(NodePath.join(import.meta.dirname, "brief.md"), "utf8"); const brief = template - .replaceAll("{{PACK}}", pack) .replace( "{{SCREENS}}", entries @@ -211,7 +211,10 @@ export async function buildPack(input: { ? referenceImages.map((name) => `- \`references/${name}\``).join("\n") : "No reference screenshots of the Codex or Claude iOS apps are in this pack. Judge that axis from what you know of those apps, and say so.", ) - .replace("{{AXES}}", AXES.map((axis) => `- \`${axis.id}\`: ${axis.label}`).join("\n")) + .replace( + "{{AXES}}", + AXES.map((axis) => `- \`${axis.id}\`: **${axis.label}**: ${axis.description}.`).join("\n"), + ) .replace( "{{DEVICES}}", DEVICE_CLASSES.map((device) => `- \`${device.id}\`: ${device.label}`).join("\n"), diff --git a/evals/mobile-design/run.test.ts b/evals/mobile-design/run.test.ts index d6541733606ed..71d8f2e4e4bc2 100644 --- a/evals/mobile-design/run.test.ts +++ b/evals/mobile-design/run.test.ts @@ -41,11 +41,21 @@ describe("summarizeRound", () => { }); it("shows deltas against the baseline medians", () => { - const baseline = summarizeRound({ opus: parseReviewerReply(reply(6)) }).medians; + const baseline = summarizeRound({ opus: parseReviewerReply(reply(6)) }); const markdown = renderSummary( summarizeRound({ opus: parseReviewerReply(reply(8)) }), baseline, ); expect(markdown).toContain("| iPhone 17 Pro | Visual design | 8.0 | 8.0 | +2.0 |"); + expect(markdown).not.toContain("baseline panel"); + }); + + it("flags a baseline scored by a different panel", () => { + const baseline = summarizeRound({ opus: parseReviewerReply(reply(6)) }); + const round = summarizeRound({ + opus: parseReviewerReply(reply(8)), + muse: parseReviewerReply(reply(8)), + }); + expect(renderSummary(round, baseline)).toContain("The baseline panel was opus"); }); }); diff --git a/evals/mobile-design/run.ts b/evals/mobile-design/run.ts index 241c8ee013ac9..8fb4163410838 100644 --- a/evals/mobile-design/run.ts +++ b/evals/mobile-design/run.ts @@ -9,7 +9,7 @@ import { captureDesktop, findShot, type DesktopShot } from "./capture-desktop.ts import { captureMobile, type FlowResult, type MobileShot } from "./capture-mobile.ts"; import { DEFAULT_PANEL, PANEL, POSES, SCENES, type PanelMember, type SceneId } from "./cases.ts"; import { buildPack } from "./pack.ts"; -import { parseReviewerReply, renderSummary, summarizeRound, type Medians } from "./scores.ts"; +import { parseReviewerReply, renderSummary, summarizeRound, type RoundSummary } from "./scores.ts"; const { values } = NodeUtil.parseArgs({ options: { @@ -59,7 +59,7 @@ async function readCaptures(): Promise { return text ? (JSON.parse(text) as Captures) : { shots: [], flows: [], desktop: [] }; } -/** Merges new captures over earlier ones, so mobile and desktop can run on different hosts. */ +/** Merges new captures over earlier ones, so one run can add desktop pairs or redo a pose. */ async function saveCaptures(update: Partial) { const current = await readCaptures(); const replaced = ( @@ -124,15 +124,13 @@ async function summarize() { }), ), ); - const baseline = values.baseline - ? ( - JSON.parse( - await NodeFSP.readFile( - NodePath.join(NodePath.resolve(values.baseline), "summary.json"), - "utf8", - ), - ) as { medians: Medians } - ).medians + const baseline: RoundSummary | null = values.baseline + ? JSON.parse( + await NodeFSP.readFile( + NodePath.join(NodePath.resolve(values.baseline), "summary.json"), + "utf8", + ), + ) : null; const round = summarizeRound(reviewers); await NodeFSP.writeFile( diff --git a/evals/mobile-design/scores.ts b/evals/mobile-design/scores.ts index 9d5a4509eb99c..73ad3d1cc927f 100644 --- a/evals/mobile-design/scores.ts +++ b/evals/mobile-design/scores.ts @@ -64,8 +64,14 @@ const formatDelta = (value: number) => Number.isNaN(value) ? "–" : value === 0 ? "0.0" : `${value > 0 ? "+" : ""}${value.toFixed(1)}`; /** The summary table: one row per device and axis, with deltas when a baseline is given. */ -export function renderSummary(round: RoundSummary, baseline: Medians | null): string { +export function renderSummary(round: RoundSummary, baseline: RoundSummary | null): string { const names = Object.keys(round.reviewers); + const baselineNames = baseline ? Object.keys(baseline.reviewers) : names; + // A different panel moves the median by itself, so say so next to the deltas. + const panelNote = + baselineNames.toSorted().join() === names.toSorted().join() + ? "" + : `\nThe baseline panel was ${baselineNames.join(", ")}; this round's is ${names.join(", ")}. Read the deltas with that in mind.\n`; const header = ["Device", "Axis", "Median", ...names, ...(baseline ? ["Δ baseline"] : [])]; const rows = DEVICE_CLASSES.flatMap((device) => AXES.map((axis) => { @@ -75,7 +81,7 @@ export function renderSummary(round: RoundSummary, baseline: Medians | null): st axis.label, formatScore(value), ...names.map((name) => formatScore(round.reviewers[name]!.scores[device.id][axis.id])), - ...(baseline ? [formatDelta(value - baseline[device.id][axis.id])] : []), + ...(baseline ? [formatDelta(value - baseline.medians[device.id][axis.id])] : []), ]; }), ); @@ -88,5 +94,5 @@ export function renderSummary(round: RoundSummary, baseline: Medians | null): st `### ${name}\n\n${round.reviewers[name]!.fixes.map((fix, index) => `${index + 1}. ${fix}`).join("\n")}`, ) .join("\n\n"); - return `# Mobile design scores\n\n${table}\n\n## Top fixes\n\n${fixes}\n`; + return `# Mobile design scores\n${panelNote}\n${table}\n\n## Top fixes\n\n${fixes}\n`; } diff --git a/scripts/mobile-showcase-environment.ts b/scripts/mobile-showcase-environment.ts index 036b935e83e36..a8a1535e03eba 100644 --- a/scripts/mobile-showcase-environment.ts +++ b/scripts/mobile-showcase-environment.ts @@ -576,7 +576,6 @@ export async function writeShowcaseProviderSettings(baseDir: string): Promise [instanceId, { driver: instanceId, enabled: false }]), - ), + // The only other provider on by default. + claudeAgent: { driver: "claudeAgent", enabled: false }, }, }, null, diff --git a/scripts/mobile-showcase-provider.ts b/scripts/mobile-showcase-provider.ts index 7c7c61b2a8668..02ff2a5c0ea01 100644 --- a/scripts/mobile-showcase-provider.ts +++ b/scripts/mobile-showcase-provider.ts @@ -50,10 +50,13 @@ const threadCwds = new Map(); const backgroundCommands = new Map(); const interruptedTurns = new Set(); -function serverRequest(method: string, params: Frame): Promise { +/** Sends a request to the server and resolves with its id and the server's answer. */ +function serverRequest(method: string, params: Frame): Promise<{ id: number; result: Frame }> { const id = nextServerRequestId++; write({ method, id, params }); - return new Promise((resolve) => pendingServerRequests.set(id, resolve)); + return new Promise((resolve) => + pendingServerRequests.set(id, (result) => resolve({ id, result })), + ); } function threadPayload(threadId: string, cwd: string, model: string): Frame { @@ -247,20 +250,22 @@ async function playStep( background: false, }); notify("item/started", { ...turn, item, startedAtMs: Date.now() }); - const requestId = nextServerRequestId; - const result = await serverRequest("item/commandExecution/requestApproval", { - kind: "command", - ...turn, - itemId: item.id, - startedAtMs: Date.now(), - environmentId: "local", - reason: step.reason, - command: item.command, - cwd, - commandActions: item.commandActions, - proposedExecpolicyAmendment: null, - availableDecisions: ["accept", "acceptForSession", "decline"], - }); + const { id: requestId, result } = await serverRequest( + "item/commandExecution/requestApproval", + { + kind: "command", + ...turn, + itemId: item.id, + startedAtMs: Date.now(), + environmentId: "local", + reason: step.reason, + command: item.command, + cwd, + commandActions: item.commandActions, + proposedExecpolicyAmendment: null, + availableDecisions: ["accept", "acceptForSession", "decline"], + }, + ); notify("serverRequest/resolved", { threadId: turn.threadId, requestId }); const accepted = result.decision === "accept" || result.decision === "acceptForSession"; notify("item/completed", { @@ -279,8 +284,7 @@ async function playStep( return; } case "question": { - const requestId = nextServerRequestId; - await serverRequest("item/tool/requestUserInput", { + const { id: requestId } = await serverRequest("item/tool/requestUserInput", { ...turn, itemId: newId("call"), questions: step.questions.map((question) => ({ @@ -415,7 +419,8 @@ function main() { process.stdout.write(`codex-cli ${VERSION}\n`); return; } - if (args[0] !== "app-server") process.exit(1); + if (args[0] !== "app-server" || args.includes("--help")) + process.exit(args.includes("--help") ? 0 : 1); const lines = NodeReadline.createInterface({ input: process.stdin }); lines.on("line", (line) => { if (!line.trim()) return; From 2bd0ad46f2b066c988ffe411f7bc8618788a9de1 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 27 Oct 2026 04:35:41 -0700 Subject: [PATCH 03/21] fix(evals): make mobile design captures reliable on busy hosts Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/capture-mobile.ts | 27 +++++++++----- evals/mobile-design/cases.ts | 15 +++++--- evals/mobile-design/ios.ts | 51 +++++++++++++++++---------- evals/mobile-design/pack.ts | 1 + 4 files changed, 62 insertions(+), 32 deletions(-) diff --git a/evals/mobile-design/capture-mobile.ts b/evals/mobile-design/capture-mobile.ts index 3d4d4e8a4758a..3987d4e6e6964 100644 --- a/evals/mobile-design/capture-mobile.ts +++ b/evals/mobile-design/capture-mobile.ts @@ -114,6 +114,9 @@ async function captureScene( } await delay(1_500); await simulator.screenshot(NodePath.join(outDirectory, file), pose); + if ("reset" in scene) { + for (const step of scene.reset) await simulator.perform(step, targets).catch(() => undefined); + } return { pose: pose.id, scene: scene.id, file, ...(problem ? { problem } : {}) }; } @@ -151,14 +154,16 @@ async function captureFlow( await simulator.perform(step, targets); const action = "tap" in step - ? `Tap ${step.tap.replaceAll(/[\^$]/g, "")}` + ? `Tap ${step.name ?? step.tap.replaceAll(/[\^$]/g, "")}` : "type" in step ? `Type "${step.type}"` : "focus" in step ? "Tap the composer" - : "link" in step - ? `Open ${step.link}` - : `Scroll ${step.scroll}`; + : "clear" in step + ? "Clear the composer" + : "link" in step + ? `Open ${step.link}` + : `Scroll ${step.scroll}`; await frame(action); } } catch (error) { @@ -242,16 +247,17 @@ export async function captureMobile(input: { await seedComposerDrafts(simulator, targets); input.log(`${device}: launching`); await simulator.launch(`http://127.0.0.1:${metroPort}`); - const pairingUrls = await Promise.all( - environments.map(async (environment) => - buildShowcasePairingUrl( + const pairings = await Promise.all( + environments.map(async (environment) => ({ + url: buildShowcasePairingUrl( "127.0.0.1", environment.port, await issuePairingCredential(environment.baseDir), ), - ), + label: environment.label, + })), ); - await simulator.pair(pairingUrls, THREAD_LIST); + await simulator.pair(pairings); for (const pose of input.poses.filter((candidate) => candidate.device === device)) { input.log(`${pose.id}: capturing`); await simulator.applyPose(pose); @@ -280,6 +286,9 @@ export async function captureMobile(input: { flows.push(result); } } + } catch (error) { + // One device failing should not cost the others their captures. + input.log(`${device}: stopped: ${error instanceof Error ? error.message : String(error)}`); } finally { await simulator.delete(); } diff --git a/evals/mobile-design/cases.ts b/evals/mobile-design/cases.ts index 617fd55125833..c61084feacfbb 100644 --- a/evals/mobile-design/cases.ts +++ b/evals/mobile-design/cases.ts @@ -1,7 +1,8 @@ /** * Steps: `link` opens `t3code://`, where `{thread:}` becomes `/` * for a seeded or live fixture thread; `tap` presses the first element whose accessibility label - * matches the regex; `focus: "composer"` taps the unlabelled composer. + * matches the regex, shown in flows as its `name` when it has one; `focus: "composer"` taps the + * unlabelled composer. */ export const AXES = [ @@ -55,10 +56,11 @@ export const SIMULATORS = { export type StepAction = | { readonly link: string } - | { readonly tap: string } + | { readonly tap: string; readonly name?: string } | { readonly type: string } | { readonly scroll: "up" | "down" } - | { readonly focus: "composer" }; + | { readonly focus: "composer" } + | { readonly clear: "composer" }; export interface Pose { readonly id: string; @@ -84,6 +86,8 @@ export interface Scene { readonly ready?: string; /** Desktop web path for the "conforms to desktop" pair, relative to the environment origin. */ readonly desktop?: { readonly path: string; readonly click?: string }; + /** Runs after the screenshot, to undo state later scenes would see, such as a typed draft. */ + readonly reset?: ReadonlyArray; } const REQUEST = "Mobile design eval request"; @@ -160,6 +164,7 @@ export const SCENES = [ title: "Slash suggestions while the agent runs", source: `${REQUEST}; #16112 (running pill over slash suggestions)`, steps: [thread("streaming"), { focus: "composer" }, { type: "/" }], + reset: [{ clear: "composer" }], }, { id: "new-thread", @@ -243,7 +248,7 @@ export const POSES = [ { id: "iphone-landscape", device: "iphone", - label: "iPhone, landscape", + label: "iPhone held in landscape", appearance: "dark", orientation: "landscape", scenes: ["threads", "thread-tool-calls"], @@ -289,7 +294,7 @@ export const FLOWS = [ { tap: "^React$" }, { tap: "^Choose environment" }, // Pick whichever machine the picker offers; when it is not the project's, the frames show it. - { tap: "^(Suspense Station|Moonbase Terminal|Kernel Cabin)" }, + { tap: "^(Suspense Station|Moonbase Terminal|Kernel Cabin)", name: "the offered machine" }, { focus: "composer" }, { type: "Add a loading skeleton to the profile page" }, { tap: "^Send$" }, diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index 55fe64ae4a207..3f05c38a583c1 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -182,14 +182,19 @@ export class Simulator { throw new Error("The app did not load from Metro."); } - async pair(pairingUrls: ReadonlyArray, firstThreadTitle: RegExp) { - for (const pairingUrl of pairingUrls) { - const query = new URLSearchParams({ pairingUrl, autoConnect: "1" }); + /** Pairs each environment, waiting until its threads list it before the next, so links never overlap. */ + async pair(pairings: ReadonlyArray<{ readonly url: string; readonly label: string }>) { + for (const pairing of pairings) { + const query = new URLSearchParams({ pairingUrl: pairing.url, autoConnect: "1" }); // Pairing runs once per device, so it can wait out a slow host. await this.openUrl(`t3code://connections/new?${query}`, 180_000); - await delay(4_000); + await this.waitFor( + new RegExp(`^${RegExp.escape(pairing.label)}$`), + `${pairing.label} after pairing`, + 180_000, + true, + ); } - await this.waitFor(firstThreadTitle, "the paired thread list", 180_000); } /** iOS asks before a link opens the app; accept the prompt once it shows. */ @@ -243,20 +248,27 @@ export class Simulator { return data as unknown as Snapshot; } - find(snapshot: Snapshot, label: RegExp): SnapshotNode | undefined { + /** The first element labelled `label`; `offScreen` also accepts rows scrolled out of view. */ + find(snapshot: Snapshot, label: RegExp, offScreen = false): SnapshotNode | undefined { const onScreen = (node: SnapshotNode) => - node.rect !== undefined && - node.rect.y + node.rect.height > 0 && - node.rect.y < snapshot.viewport.height; + offScreen || + (node.rect !== undefined && + node.rect.y + node.rect.height > 0 && + node.rect.y < snapshot.viewport.height); const matches = snapshot.nodes.filter((node) => label.test(node.label ?? "") && onScreen(node)); return matches.find((node) => node.hittable) ?? matches[0]; } - async waitFor(label: RegExp, what: string, timeoutMs = 20_000): Promise { + async waitFor( + label: RegExp, + what: string, + timeoutMs = 20_000, + offScreen = false, + ): Promise { const deadline = Date.now() + timeoutMs; for (;;) { const snapshot = await this.snapshot().catch(() => null); - if (snapshot && this.find(snapshot, label)) return snapshot; + if (snapshot && this.find(snapshot, label, offScreen)) return snapshot; if (Date.now() > deadline) throw new Error(`timed out waiting for ${what} (${label})`); await delay(750); } @@ -278,13 +290,13 @@ export class Simulator { } /** The composer is a native text view with no label: the lowest editable field on screen. */ - async focusComposer() { + private async composer(): Promise { const snapshot = await this.snapshot(); const composer = snapshot.nodes .filter((node) => (node.kind === "text-view" || node.kind === "text-field") && node.rect) .toSorted((a, b) => b.rect!.y - a.rect!.y)[0]; if (!composer) throw new Error("no composer on screen"); - await this.press(composer); + return composer; } async perform(step: StepAction, targets: LinkTargets) { @@ -292,7 +304,8 @@ export class Simulator { else if ("tap" in step) await this.tap(new RegExp(step.tap)); else if ("type" in step) await this.agent(["type", step.type]); else if ("scroll" in step) await this.agent(["scroll", step.scroll]); - else await this.focusComposer(); + else if ("focus" in step) await this.press(await this.composer()); + else await this.agent(["fill", `@${(await this.composer()).ref}`, ""]); await delay(1_200); } @@ -314,12 +327,12 @@ export class Simulator { await delay(600); return; } - // Another tab, such as Settings, shows no back button; its tab bar returns to Chats. + // Another tab, such as Settings, shows no back button; the tab bar or side rail returns to Chats. const chatsTab = snapshot.nodes.find( (node) => node.kind === "button" && node.label === "Chats" && - (node.rect?.y ?? 0) > snapshot.viewport.height * 0.75, + (node.rect?.y ?? 0) >= snapshot.viewport.height / 4, ); if (back) await this.press(back); else if (chatsTab) await this.press(chatsTab); @@ -334,7 +347,9 @@ export class Simulator { await simctl("ui", this.udid, "appearance", pose.appearance); await simctl("ui", this.udid, "content_size", pose.textSize ?? "large"); if (pose.fold) { - await this.agent(["fold", pose.fold]); + // agent-device verifies a fold by a hinge-angle read, which fails when the Duo is already in + // that pose; the screenshot of that pose's display shows whether it took. + await this.agent(["fold", pose.fold]).catch(() => undefined); await delay(2_500); } // A fold sets the Duo's shape itself. @@ -355,7 +370,7 @@ export class Simulator { await delay(2_000); if (await fits()) return; } - throw new Error(`could not rotate to ${shape}`); + // A portrait-locked app keeps its shape when the device turns; the capture shows that. } async screenshot(destination: string, pose: Pose) { diff --git a/evals/mobile-design/pack.ts b/evals/mobile-design/pack.ts index 1eecdcccf0da4..8add37a69992e 100644 --- a/evals/mobile-design/pack.ts +++ b/evals/mobile-design/pack.ts @@ -96,6 +96,7 @@ export async function buildPack(input: { }): Promise<{ readonly brief: string }> { const pack = NodePath.join(input.outDirectory, "pack"); await NodeFSP.rm(pack, { recursive: true, force: true }); + await NodeFSP.mkdir(pack, { recursive: true }); const at = (file: string) => NodePath.join(input.outDirectory, file); const entries: PackEntry[] = []; From 4f468d222166de4a3bfdd942755c0ae9893d8192 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 27 Oct 2026 04:40:50 -0700 Subject: [PATCH 04/21] fix(evals): council round 2 fixes for the mobile design eval Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/capture-mobile.ts | 5 ++- evals/mobile-design/ios.ts | 33 +++++++++------ evals/mobile-design/pack.ts | 59 ++++++++++++++------------- 3 files changed, 54 insertions(+), 43 deletions(-) diff --git a/evals/mobile-design/capture-mobile.ts b/evals/mobile-design/capture-mobile.ts index 3987d4e6e6964..fbb4460adc1b4 100644 --- a/evals/mobile-design/capture-mobile.ts +++ b/evals/mobile-design/capture-mobile.ts @@ -239,8 +239,9 @@ export async function captureMobile(input: { ); for (const device of devices) { input.log(`${device}: creating the simulator`); - const simulator = await Simulator.create(device, runId); + let simulator: Simulator | null = null; try { + simulator = await Simulator.create(device, runId); // iPadOS windowing keeps full-screen apps portrait unless it is off. if (device === "ipad") await ensureIosFullScreenAppsMode(simulator.udid); await simulator.install(appPath); @@ -290,7 +291,7 @@ export async function captureMobile(input: { // One device failing should not cost the others their captures. input.log(`${device}: stopped: ${error instanceof Error ? error.message : String(error)}`); } finally { - await simulator.delete(); + await simulator?.delete(); } } } finally { diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index 3f05c38a583c1..dc30fc9107497 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -107,6 +107,7 @@ export class Simulator { readonly device: DeviceClassId; readonly udid: string; private readonly session: string; + private appLandscape = false; private constructor(device: DeviceClassId, udid: string, session: string) { this.device = device; @@ -125,9 +126,14 @@ export class Simulator { ) ).stdout.trim(); const simulator = new Simulator(device, udid, `t3-design-${device}-${runId}`); - await suppressIosSystemFollowUps(udid); - await simctl("boot", udid); - await simctl("bootstatus", udid, "-b"); + try { + await suppressIosSystemFollowUps(udid); + await simctl("boot", udid); + await simctl("bootstatus", udid, "-b"); + } catch (error) { + await simulator.delete(); + throw error; + } return simulator; } @@ -353,24 +359,28 @@ export class Simulator { await delay(2_500); } // A fold sets the Duo's shape itself. - if (!pose.fold) await this.orient(pose.orientation ?? "portrait"); + this.appLandscape = pose.fold + ? pose.fold === "open" + : await this.orient(pose.orientation ?? "portrait"); } /** agent-device names rotations from the natural orientation, so try each until the shape fits. */ - private async orient(shape: "portrait" | "landscape") { - const fits = async () => { + /** Returns whether the app now lays out in landscape. */ + private async orient(shape: "portrait" | "landscape"): Promise { + const isLandscape = async () => { const { width, height } = (await this.snapshot()).viewport; - return width > height === (shape === "landscape"); + return width > height; }; - if (await fits()) return; + if ((await isLandscape()) === (shape === "landscape")) return shape === "landscape"; for (const name of shape === "landscape" ? ["landscape-left", "landscape-right"] : ["portrait"]) { await this.agent(["orientation", name]); await delay(2_000); - if (await fits()) return; + if ((await isLandscape()) === (shape === "landscape")) return shape === "landscape"; } // A portrait-locked app keeps its shape when the device turns; the capture shows that. + return isLandscape(); } async screenshot(destination: string, pose: Pose) { @@ -382,10 +392,9 @@ export class Simulator { ...(pose.fold ? ["--display", pose.fold === "closed" ? "1" : "3"] : []), destination, ); - // Headless displays stay portrait when the device rotates. - const landscape = pose.orientation === "landscape" || pose.fold === "open"; + // Headless displays stay portrait when the app rotates. const { width, height } = readPngDimensions(await NodeFSP.readFile(destination)); - if (landscape && height > width) { + if (this.appLandscape && height > width) { await run("sips", ["--rotate", pose.fold ? "90" : "270", destination]); } } diff --git a/evals/mobile-design/pack.ts b/evals/mobile-design/pack.ts index 8add37a69992e..0387e1e28946f 100644 --- a/evals/mobile-design/pack.ts +++ b/evals/mobile-design/pack.ts @@ -32,7 +32,9 @@ const escapeXml = (text: string) => text.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">"); /** Lays tiles out in one row, scaled to a shared height, each with a label above it. */ -async function contactSheet(tiles: ReadonlyArray, destination: string): Promise { +async function contactSheet( + tiles: ReadonlyArray, +): Promise<{ readonly data: Buffer; readonly width: number; readonly height: number }> { const images = await Promise.all( tiles.map(async (tile) => ({ ...tile, meta: await sharp(tile.file).metadata() })), ); @@ -59,13 +61,18 @@ async function contactSheet(tiles: ReadonlyArray, destination: string): Pr ]; }), ); - await NodeFSP.mkdir(NodePath.dirname(destination), { recursive: true }); - await sharp({ + const data = await sharp({ create: { width: sheetWidth, height: sheetHeight, channels: 3, background: "#1c1c1e" }, }) .composite(layers.flat()) .png() - .toFile(destination); + .toBuffer(); + return { data, width: sheetWidth, height: sheetHeight }; +} + +async function writeImage(data: Buffer, destination: string) { + await NodeFSP.mkdir(NodePath.dirname(destination), { recursive: true }); + await NodeFSP.writeFile(destination, data); } /** Poses whose screenshots share a shape, so they sit well side by side. */ @@ -112,7 +119,7 @@ export async function buildPack(input: { } for (const [group, tiles] of groups) { const file = NodePath.join("screens", group, `${scene.id}.png`); - await contactSheet(tiles, NodePath.join(pack, file)); + await writeImage((await contactSheet(tiles)).data, NodePath.join(pack, file)); entries.push({ file, scene: scene.title, view: group }); } const desktop = input.desktop.find((shot) => shot.scene === scene.id && !shot.problem); @@ -120,14 +127,12 @@ export async function buildPack(input: { const phone = shots.find((shot) => shot.pose === "iphone-dark"); const tablet = shots.find((shot) => shot.pose === "ipad-landscape-dark"); const file = NodePath.join("desktop", `${scene.id}.png`); - await contactSheet( - [ - { file: at(desktop.file), label: "Desktop web, 1440×900" }, - ...(tablet ? [{ file: at(tablet.file), label: "iPad landscape" }] : []), - ...(phone ? [{ file: at(phone.file), label: "iPhone" }] : []), - ], - NodePath.join(pack, file), - ); + const pair = await contactSheet([ + { file: at(desktop.file), label: "Desktop web, 1440×900" }, + ...(tablet ? [{ file: at(tablet.file), label: "iPad landscape" }] : []), + ...(phone ? [{ file: at(phone.file), label: "iPhone" }] : []), + ]); + await writeImage(pair.data, NodePath.join(pack, file)); entries.push({ file, scene: scene.title, view: "desktop pair" }); } } @@ -139,38 +144,34 @@ export async function buildPack(input: { const file = NodePath.join("flows", `${flow.id}.png`); // Rows of five keep each phone frame large enough to read. const rows = await Promise.all( - Array.from({ length: Math.ceil(result.frames.length / 5) }, async (_, row) => { - const rowFile = NodePath.join(pack, "flows", `${flow.id}-${row}.png`); - await contactSheet( + Array.from({ length: Math.ceil(result.frames.length / 5) }, (_, row) => + contactSheet( result.frames.slice(row * 5, row * 5 + 5).map((frame, index) => ({ file: at(frame.file), label: `${row * 5 + index}. ${frame.action}`, })), - rowFile, - ); - return rowFile; - }), + ), + ), ); - const metas = await Promise.all(rows.map((row) => sharp(row).metadata())); let top = 0; - await sharp({ + const filmstrip = await sharp({ create: { - width: Math.max(...metas.map((meta) => meta.width ?? 0)), - height: metas.reduce((sum, meta) => sum + (meta.height ?? 0), 0), + width: Math.max(...rows.map((row) => row.width)), + height: rows.reduce((sum, row) => sum + row.height, 0), channels: 3, background: "#1c1c1e", }, }) .composite( - rows.map((row, index) => { - const layer = { input: row, left: 0, top }; - top += metas[index]!.height ?? 0; + rows.map((row) => { + const layer = { input: row.data, left: 0, top }; + top += row.height; return layer; }), ) .png() - .toFile(NodePath.join(pack, file)); - await Promise.all(rows.map((row) => NodeFSP.rm(row))); + .toBuffer(); + await writeImage(filmstrip, NodePath.join(pack, file)); entries.push({ file, scene: flow.title, view: "flow" }); flowRows.push( `| ${flow.title} | ${result.taps} | ${result.screens} | \`${file}\`${result.problem ? ` (stopped early: ${result.problem.split("\n")[0]})` : ""} |`, From b8fb37ebca772e17927fbeb57bf133f75899abbf Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 27 Oct 2026 04:42:14 -0700 Subject: [PATCH 05/21] fix(evals): surface failed poses and screenshots in the mobile design eval Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/README.md | 6 ++-- evals/mobile-design/capture-mobile.ts | 26 +++++++++++++---- evals/mobile-design/ios.ts | 40 ++++++++++++++++----------- 3 files changed, 48 insertions(+), 24 deletions(-) diff --git a/evals/mobile-design/README.md b/evals/mobile-design/README.md index ae3cb6495bbc5..a8ba5ba6485b3 100644 --- a/evals/mobile-design/README.md +++ b/evals/mobile-design/README.md @@ -4,7 +4,7 @@ Scores the design of the T3 Code iOS app on iPhone, iPhone Duo and iPad. Code ca A run captures: -- **Scenes**: every screen in `SCENES` (`cases.ts`), on each pose in `POSES`. The poses are iPhone 17 Pro dark and light, the largest accessibility text size and landscape; iPhone Duo closed and open; and iPad Pro 13 in landscape and portrait. Dark poses capture the full set; light poses capture a smaller set. A scene's `source` names the complaint it guards, when there is one. +- **Scenes**: every screen in `SCENES` (`cases.ts`), on each pose in `POSES`. The poses are iPhone 17 Pro dark and light, the largest accessibility text size and landscape; iPhone Duo closed and open; and iPad Pro 13 in landscape and portrait. Every pose captures every scene, except the large-text and landscape passes, which capture a few. A scene's `source` names the complaint it guards, when there is one. - **Desktop pairs**: the same data in the desktop web app at 1440×900, next to the iPad and iPhone captures. - **Flows**: common tasks walked step by step on the dark iPhone, with the taps and distinct screens each took. @@ -32,6 +32,8 @@ Mobile capture needs macOS with Xcode, and a simulator runtime with the iPhone D `summary.md` has the median per axis and device class, each reviewer's scores, the change against the baseline, and each reviewer's top five fixes. +To split the work across Macs, for example when only one has the Duo runtime, run each with its own `--pose` list, then copy the `captures/` folders into one run, join the `shots` lists in their `captures.json`, and build the pack with `--pack`. + Other flags: `--pose ` and `--case ` capture a subset, `--no-flows` skips the flows, `--pack` rebuilds the pack from earlier captures. `captures/captures.json` lists any scene that did not reach its ready state; look at those screenshots before you send the pack. ## Blind review @@ -48,4 +50,4 @@ Add it to `SCENES` or `FLOWS` in `cases.ts`, with its `source`. Steps open deep ## Time and cost -A full capture on an M-series Mac takes about 45 minutes: about 15 minutes per device class, plus desktop pairs. A council round is two `xhigh` reviews of about 80 images. +A full capture on an idle M-series Mac takes about 90 minutes: about 30 minutes per device class, plus 5 for desktop pairs. On a busy host it can take twice that. A council round is two `xhigh` reviews of about 115 images. diff --git a/evals/mobile-design/capture-mobile.ts b/evals/mobile-design/capture-mobile.ts index fbb4460adc1b4..1dd0ac80d789a 100644 --- a/evals/mobile-design/capture-mobile.ts +++ b/evals/mobile-design/capture-mobile.ts @@ -101,19 +101,23 @@ async function captureScene( sceneId: SceneId, targets: LinkTargets, outDirectory: string, + poseProblem: string | undefined, ): Promise { const scene = SCENES.find((candidate) => candidate.id === sceneId)!; const file = NodePath.join("captures", pose.id, `${scene.id}.png`); - let problem: string | undefined; + let problem = poseProblem; try { await simulator.home(THREAD_LIST); for (const step of scene.steps) await simulator.perform(step, targets); if ("ready" in scene) await simulator.waitFor(new RegExp(scene.ready), scene.title); } catch (error) { - problem = error instanceof Error ? error.message : String(error); + problem ??= error instanceof Error ? error.message : String(error); } + // Screenshot a failed scene too: where it stopped is the diagnosis. await delay(1_500); - await simulator.screenshot(NodePath.join(outDirectory, file), pose); + await simulator.screenshot(NodePath.join(outDirectory, file), pose).catch((error: unknown) => { + problem ??= `screenshot failed: ${error instanceof Error ? error.message : String(error)}`; + }); if ("reset" in scene) { for (const step of scene.reset) await simulator.perform(step, targets).catch(() => undefined); } @@ -144,7 +148,9 @@ async function captureFlow( const file = NodePath.join("captures", "flows", flow.id, `${frames.length}.png`); await delay(1_200); await simulator.screenshot(NodePath.join(outDirectory, file), pose); - signatures.add(await screenSignature(simulator).catch(() => action)); + // A failed snapshot says nothing about the screen, so it adds no signature. + const signature = await screenSignature(simulator).catch(() => null); + if (signature !== null) signatures.add(signature); frames.push({ file, action }); }; try { @@ -261,12 +267,20 @@ export async function captureMobile(input: { await simulator.pair(pairings); for (const pose of input.poses.filter((candidate) => candidate.device === device)) { input.log(`${pose.id}: capturing`); - await simulator.applyPose(pose); + const poseProblem = await simulator.applyPose(pose); + if (poseProblem) input.log(`${pose.id}: ${poseProblem}`); const scenes = (pose.scenes ?? SCENES.map((scene) => scene.id)).filter( (scene) => input.scenes === null || input.scenes.has(scene), ); for (const scene of scenes) { - const shot = await captureScene(simulator, pose, scene, targets, input.outDirectory); + const shot = await captureScene( + simulator, + pose, + scene, + targets, + input.outDirectory, + poseProblem, + ); if (shot.problem) input.log(`${pose.id}/${scene}: ${shot.problem}`); shots.push(shot); } diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index dc30fc9107497..4f69f721cef45 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -129,7 +129,7 @@ export class Simulator { try { await suppressIosSystemFollowUps(udid); await simctl("boot", udid); - await simctl("bootstatus", udid, "-b"); + await run("xcrun", ["simctl", "bootstatus", udid, "-b"], { timeoutMs: 600_000 }); } catch (error) { await simulator.delete(); throw error; @@ -176,11 +176,11 @@ export class Simulator { metroUrl, ); // The first snapshot attaches the agent-device session, which can take minutes. - await this.agent(["open", BUNDLE_ID], 600_000); + await this.agent(["open", BUNDLE_ID], 600_000).catch(() => undefined); const snapshot = await this.waitFor( new RegExp(`${loaded.source}|^Reload$`), "the app to load", - 180_000, + { timeoutMs: 180_000 }, ).catch(() => null); if (snapshot && this.find(snapshot, loaded)) return; if (snapshot) await this.tap(/^Reload$/).catch(() => undefined); @@ -197,8 +197,7 @@ export class Simulator { await this.waitFor( new RegExp(`^${RegExp.escape(pairing.label)}$`), `${pairing.label} after pairing`, - 180_000, - true, + { timeoutMs: 180_000, offScreen: true }, ); } } @@ -268,9 +267,9 @@ export class Simulator { async waitFor( label: RegExp, what: string, - timeoutMs = 20_000, - offScreen = false, + options: { readonly timeoutMs?: number; readonly offScreen?: boolean } = {}, ): Promise { + const { timeoutMs = 20_000, offScreen = false } = options; const deadline = Date.now() + timeoutMs; for (;;) { const snapshot = await this.snapshot().catch(() => null); @@ -348,23 +347,32 @@ export class Simulator { throw new Error("could not return to the thread list"); } - /** Applies the pose's system settings to the running app. */ - async applyPose(pose: Pose) { + /** + * Applies the pose's system settings to the running app. Returns why the app is not in the + * pose's shape, when it is not, so its shots stay out of the pack. + */ + async applyPose(pose: Pose): Promise { await simctl("ui", this.udid, "appearance", pose.appearance); await simctl("ui", this.udid, "content_size", pose.textSize ?? "large"); if (pose.fold) { - // agent-device verifies a fold by a hinge-angle read, which fails when the Duo is already in - // that pose; the screenshot of that pose's display shows whether it took. + // agent-device verifies a fold by a hinge-angle read, which fails when the Duo is already + // in that pose, so check the app's shape instead. await this.agent(["fold", pose.fold]).catch(() => undefined); await delay(2_500); + const { width, height } = (await this.snapshot()).viewport; + this.appLandscape = width > height; + if (this.appLandscape !== (pose.fold === "open")) return `the Duo did not ${pose.fold}`; + return undefined; + } + const shape = pose.orientation ?? "portrait"; + this.appLandscape = await this.orient(shape); + // A portrait-locked iPhone is the finding; anything else that keeps its shape is a failure. + if (this.appLandscape !== (shape === "landscape") && this.device !== "iphone") { + return `the ${this.device} did not rotate to ${shape}`; } - // A fold sets the Duo's shape itself. - this.appLandscape = pose.fold - ? pose.fold === "open" - : await this.orient(pose.orientation ?? "portrait"); + return undefined; } - /** agent-device names rotations from the natural orientation, so try each until the shape fits. */ /** Returns whether the app now lays out in landscape. */ private async orient(shape: "portrait" | "landscape"): Promise { const isLandscape = async () => { From 30d770d0a78ba976e35f9de3dbb37ea3159a0723 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 27 Oct 2026 04:43:01 -0700 Subject: [PATCH 06/21] fix(evals): pair each machine before the next on slow hosts Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/capture-mobile.ts | 26 ++++++++++++++++++-------- evals/mobile-design/ios.ts | 10 ++++++---- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/evals/mobile-design/capture-mobile.ts b/evals/mobile-design/capture-mobile.ts index 1dd0ac80d789a..11b3f47590b22 100644 --- a/evals/mobile-design/capture-mobile.ts +++ b/evals/mobile-design/capture-mobile.ts @@ -22,6 +22,7 @@ import { import { SHOWCASE_ENVIRONMENTS, SHOWCASE_PROJECTS, + SHOWCASE_THREADS, } from "../../scripts/mobile-showcase-environment.ts"; import { FLOWS, SCENES, type Flow, type Pose, type SceneId } from "./cases.ts"; import { delay, linkTargets, type LinkTargets, run, Simulator } from "./ios.ts"; @@ -255,14 +256,23 @@ export async function captureMobile(input: { input.log(`${device}: launching`); await simulator.launch(`http://127.0.0.1:${metroPort}`); const pairings = await Promise.all( - environments.map(async (environment) => ({ - url: buildShowcasePairingUrl( - "127.0.0.1", - environment.port, - await issuePairingCredential(environment.baseDir), - ), - label: environment.label, - })), + environments.map(async (environment, index) => { + const projectIds: ReadonlyArray = SHOWCASE_ENVIRONMENTS[index]!.projectIds; + return { + url: buildShowcasePairingUrl( + "127.0.0.1", + environment.port, + await issuePairingCredential(environment.baseDir), + ), + // Rows name their machine only once two are paired, so its threads count too. + labels: [ + environment.label, + ...SHOWCASE_THREADS.filter((thread) => projectIds.includes(thread.projectId)).map( + (thread) => thread.title, + ), + ], + }; + }), ); await simulator.pair(pairings); for (const pose of input.poses.filter((candidate) => candidate.device === device)) { diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index 4f69f721cef45..e4753a196ea6f 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -188,15 +188,17 @@ export class Simulator { throw new Error("The app did not load from Metro."); } - /** Pairs each environment, waiting until its threads list it before the next, so links never overlap. */ - async pair(pairings: ReadonlyArray<{ readonly url: string; readonly label: string }>) { + /** Pairs each environment, waiting until the list shows it before the next, so links never overlap. */ + async pair( + pairings: ReadonlyArray<{ readonly url: string; readonly labels: ReadonlyArray }>, + ) { for (const pairing of pairings) { const query = new URLSearchParams({ pairingUrl: pairing.url, autoConnect: "1" }); // Pairing runs once per device, so it can wait out a slow host. await this.openUrl(`t3code://connections/new?${query}`, 180_000); await this.waitFor( - new RegExp(`^${RegExp.escape(pairing.label)}$`), - `${pairing.label} after pairing`, + new RegExp(`^(${pairing.labels.map((label) => RegExp.escape(label)).join("|")})`), + `${pairing.labels[0]} after pairing`, { timeoutMs: 180_000, offScreen: true }, ); } From 1e55174bab3182a4310efdcfba9022edcf9b68a8 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 27 Oct 2026 06:33:22 -0700 Subject: [PATCH 07/21] fix(evals): score only the device classes a run captured Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/brief.md | 2 +- evals/mobile-design/capture-mobile.ts | 29 ++++++----------- evals/mobile-design/ios.ts | 45 +++++++++++++++++++-------- evals/mobile-design/pack.ts | 11 +++++-- evals/mobile-design/run.test.ts | 18 ++++++++--- evals/mobile-design/scores.ts | 42 ++++++++++++------------- 6 files changed, 86 insertions(+), 61 deletions(-) diff --git a/evals/mobile-design/brief.md b/evals/mobile-design/brief.md index aa3dbf666cade..4bdd55e1e7f21 100644 --- a/evals/mobile-design/brief.md +++ b/evals/mobile-design/brief.md @@ -1,4 +1,4 @@ -# Design review: T3 Code for iPhone, iPhone Duo and iPad +# Design review: the T3 Code mobile app You are a senior product designer. Review the T3 Code mobile app from the screenshots in this pack. This is a read-only review. Do not edit files, start servers, or open browsers or simulators. diff --git a/evals/mobile-design/capture-mobile.ts b/evals/mobile-design/capture-mobile.ts index 11b3f47590b22..3a9d5b1aad558 100644 --- a/evals/mobile-design/capture-mobile.ts +++ b/evals/mobile-design/capture-mobile.ts @@ -22,7 +22,6 @@ import { import { SHOWCASE_ENVIRONMENTS, SHOWCASE_PROJECTS, - SHOWCASE_THREADS, } from "../../scripts/mobile-showcase-environment.ts"; import { FLOWS, SCENES, type Flow, type Pose, type SceneId } from "./cases.ts"; import { delay, linkTargets, type LinkTargets, run, Simulator } from "./ios.ts"; @@ -255,26 +254,16 @@ export async function captureMobile(input: { await seedComposerDrafts(simulator, targets); input.log(`${device}: launching`); await simulator.launch(`http://127.0.0.1:${metroPort}`); - const pairings = await Promise.all( - environments.map(async (environment, index) => { - const projectIds: ReadonlyArray = SHOWCASE_ENVIRONMENTS[index]!.projectIds; - return { - url: buildShowcasePairingUrl( - "127.0.0.1", - environment.port, - await issuePairingCredential(environment.baseDir), - ), - // Rows name their machine only once two are paired, so its threads count too. - labels: [ - environment.label, - ...SHOWCASE_THREADS.filter((thread) => projectIds.includes(thread.projectId)).map( - (thread) => thread.title, - ), - ], - }; - }), + const pairingUrls = await Promise.all( + environments.map(async (environment) => + buildShowcasePairingUrl( + "127.0.0.1", + environment.port, + await issuePairingCredential(environment.baseDir), + ), + ), ); - await simulator.pair(pairings); + await simulator.pair(pairingUrls, THREAD_LIST); for (const pose of input.poses.filter((candidate) => candidate.device === device)) { input.log(`${pose.id}: capturing`); const poseProblem = await simulator.applyPose(pose); diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index e4753a196ea6f..d68ea887d8c78 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -188,20 +188,18 @@ export class Simulator { throw new Error("The app did not load from Metro."); } - /** Pairs each environment, waiting until the list shows it before the next, so links never overlap. */ - async pair( - pairings: ReadonlyArray<{ readonly url: string; readonly labels: ReadonlyArray }>, - ) { - for (const pairing of pairings) { - const query = new URLSearchParams({ pairingUrl: pairing.url, autoConnect: "1" }); + /** Pairs every environment, then waits for the list to show `threadList`. */ + async pair(pairingUrls: ReadonlyArray, threadList: RegExp) { + for (const pairingUrl of pairingUrls) { + const query = new URLSearchParams({ pairingUrl, autoConnect: "1" }); // Pairing runs once per device, so it can wait out a slow host. await this.openUrl(`t3code://connections/new?${query}`, 180_000); - await this.waitFor( - new RegExp(`^(${pairing.labels.map((label) => RegExp.escape(label)).join("|")})`), - `${pairing.labels[0]} after pairing`, - { timeoutMs: 180_000, offScreen: true }, - ); + await delay(4_000); } + await this.waitFor(threadList, "the paired thread list", { + timeoutMs: 180_000, + offScreen: true, + }); } /** iOS asks before a link opens the app; accept the prompt once it shows. */ @@ -215,6 +213,15 @@ export class Simulator { () => false, ); if (accepted) return; + // The Duo's runner can see the prompt without answering it; its Open button still takes a tap. + const snapshot = await this.snapshot().catch(() => null); + const open = snapshot?.nodes.some((node) => /^Open in /.test(node.label ?? "")) + ? this.find(snapshot, /^Open$/) + : undefined; + if (open) { + await this.press(open); + return; + } await delay(500); } } @@ -273,10 +280,22 @@ export class Simulator { ): Promise { const { timeoutMs = 20_000, offScreen = false } = options; const deadline = Date.now() + timeoutMs; + let lastError = ""; for (;;) { - const snapshot = await this.snapshot().catch(() => null); + const snapshot = await this.snapshot().catch((error: unknown) => { + lastError = error instanceof Error ? error.message : String(error); + return null; + }); if (snapshot && this.find(snapshot, label, offScreen)) return snapshot; - if (Date.now() > deadline) throw new Error(`timed out waiting for ${what} (${label})`); + if (Date.now() > deadline) { + const seen = snapshot + ? `saw: ${snapshot.nodes + .flatMap((node) => (node.label ? [node.label] : [])) + .slice(0, 8) + .join(" | ")}` + : `snapshot failed: ${lastError.slice(0, 300)}`; + throw new Error(`timed out waiting for ${what} (${seen})`); + } await delay(750); } } diff --git a/evals/mobile-design/pack.ts b/evals/mobile-design/pack.ts index 0387e1e28946f..d6a3d3586cc45 100644 --- a/evals/mobile-design/pack.ts +++ b/evals/mobile-design/pack.ts @@ -190,6 +190,13 @@ export async function buildPack(input: { } } + // Reviewers score only the device classes this run captured. + const devices = DEVICE_CLASSES.filter((device) => + input.shots.some( + (shot) => + !shot.problem && POSES.some((pose) => pose.id === shot.pose && pose.device === device.id), + ), + ); const template = await NodeFSP.readFile(NodePath.join(import.meta.dirname, "brief.md"), "utf8"); const brief = template .replace( @@ -219,14 +226,14 @@ export async function buildPack(input: { ) .replace( "{{DEVICES}}", - DEVICE_CLASSES.map((device) => `- \`${device.id}\`: ${device.label}`).join("\n"), + devices.map((device) => `- \`${device.id}\`: ${device.label}`).join("\n"), ) .replace( "{{JSON}}", JSON.stringify( { scores: Object.fromEntries( - DEVICE_CLASSES.map((device) => [ + devices.map((device) => [ device.id, Object.fromEntries(AXES.map((axis) => [axis.id, 0])), ]), diff --git a/evals/mobile-design/run.test.ts b/evals/mobile-design/run.test.ts index 71d8f2e4e4bc2..12140ef52629f 100644 --- a/evals/mobile-design/run.test.ts +++ b/evals/mobile-design/run.test.ts @@ -16,11 +16,11 @@ const reply = (value: number, fixes = ["Give the question panel the space above describe("parseReviewerReply", () => { it("reads the last json block", () => { - expect(parseReviewerReply(reply(7)).scores.duo.polish).toBe(7); + expect(parseReviewerReply(reply(7)).scores.duo?.polish).toBe(7); }); - it("rejects a reply without scores for every device and axis", () => { - const partial = JSON.stringify({ scores: { iphone: uniform(7).iphone }, fixes: ["x"] }); + it("rejects a device scored on only some axes", () => { + const partial = JSON.stringify({ scores: { iphone: { visual: 7 } }, fixes: ["x"] }); expect(() => parseReviewerReply(`\`\`\`json\n${partial}\n\`\`\``)).toThrow(); }); @@ -37,7 +37,17 @@ describe("summarizeRound", () => { opus: parseReviewerReply(reply(6)), sol: parseReviewerReply(reply(9)), }); - expect(round.medians.ipad.visual).toBe(7.5); + expect(round.medians.ipad?.visual).toBe(7.5); + }); + + it("leaves out devices nobody scored", () => { + const iphoneOnly = JSON.stringify({ scores: { iphone: uniform(7).iphone }, fixes: ["x"] }); + const round = summarizeRound({ + opus: parseReviewerReply(`\`\`\`json\n${iphoneOnly}\n\`\`\``), + }); + expect(round.medians.iphone?.polish).toBe(7); + expect(round.medians.duo).toBeUndefined(); + expect(renderSummary(round, null)).not.toContain("iPhone Duo"); }); it("shows deltas against the baseline medians", () => { diff --git a/evals/mobile-design/scores.ts b/evals/mobile-design/scores.ts index 73ad3d1cc927f..2a02286ca54a0 100644 --- a/evals/mobile-design/scores.ts +++ b/evals/mobile-design/scores.ts @@ -7,13 +7,12 @@ const DeviceScores = Schema.Struct( Object.fromEntries(AXES.map((axis) => [axis.id, Score])) as Record, ); -/** The JSON block every reviewer ends its reply with (see brief.md). */ +/** The JSON block every reviewer ends its reply with (see brief.md); it scores the captured devices. */ export const ReviewerScores = Schema.Struct({ scores: Schema.Struct( - Object.fromEntries(DEVICE_CLASSES.map((device) => [device.id, DeviceScores])) as Record< - DeviceClassId, - typeof DeviceScores - >, + Object.fromEntries( + DEVICE_CLASSES.map((device) => [device.id, Schema.optionalKey(DeviceScores)]), + ) as Record>, ), fixes: Schema.Array(Schema.String).check(Schema.isMinLength(1)), }); @@ -36,7 +35,7 @@ export function median(values: ReadonlyArray): number { return sorted.length % 2 === 1 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2; } -export type Medians = Record>; +export type Medians = Partial>>; export interface RoundSummary { readonly reviewers: Readonly>; @@ -45,17 +44,14 @@ export interface RoundSummary { export function summarizeRound(reviewers: Readonly>): RoundSummary { const all = Object.values(reviewers); - const medians = Object.fromEntries( - DEVICE_CLASSES.map((device) => [ - device.id, - Object.fromEntries( - AXES.map((axis) => [ - axis.id, - median(all.map((review) => review.scores[device.id][axis.id])), - ]), - ), - ]), - ) as Medians; + const medians: Medians = {}; + for (const device of DEVICE_CLASSES) { + const scored = all.flatMap((review) => review.scores[device.id] ?? []); + if (scored.length === 0) continue; + medians[device.id] = Object.fromEntries( + AXES.map((axis) => [axis.id, median(scored.map((scores) => scores[axis.id]))]), + ) as Record; + } return { reviewers, medians }; } @@ -73,15 +69,19 @@ export function renderSummary(round: RoundSummary, baseline: RoundSummary | null ? "" : `\nThe baseline panel was ${baselineNames.join(", ")}; this round's is ${names.join(", ")}. Read the deltas with that in mind.\n`; const header = ["Device", "Axis", "Median", ...names, ...(baseline ? ["Δ baseline"] : [])]; - const rows = DEVICE_CLASSES.flatMap((device) => + const rows = DEVICE_CLASSES.filter((device) => round.medians[device.id]).flatMap((device) => AXES.map((axis) => { - const value = round.medians[device.id][axis.id]; + const value = round.medians[device.id]![axis.id]; return [ device.label, axis.label, formatScore(value), - ...names.map((name) => formatScore(round.reviewers[name]!.scores[device.id][axis.id])), - ...(baseline ? [formatDelta(value - baseline.medians[device.id][axis.id])] : []), + ...names.map((name) => + formatScore(round.reviewers[name]!.scores[device.id]?.[axis.id] ?? Number.NaN), + ), + ...(baseline + ? [formatDelta(value - (baseline.medians[device.id]?.[axis.id] ?? Number.NaN))] + : []), ]; }), ); From d351ad2fe970571c4d15be47003cc0df155d6890 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 27 Oct 2026 06:37:30 -0700 Subject: [PATCH 08/21] fix(evals): pair the mobile new-thread scene with the web home page Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/cases.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evals/mobile-design/cases.ts b/evals/mobile-design/cases.ts index c61084feacfbb..f6ddfc4717c8f 100644 --- a/evals/mobile-design/cases.ts +++ b/evals/mobile-design/cases.ts @@ -171,7 +171,8 @@ export const SCENES = [ title: "New thread, project chosen", source: REQUEST, steps: [{ link: "new" }, { tap: "^T3 Code$" }], - desktop: { path: "/new" }, + // The web app has no new-thread route; its home page is the new-thread screen. + desktop: { path: "/" }, }, { id: "question", From f8b99103a7ada941b074f367ca7a011b618a0435 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 27 Oct 2026 07:07:15 -0700 Subject: [PATCH 09/21] fix(evals): council round 3 and review bot fixes for the mobile design eval Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/capture-desktop.ts | 5 +- evals/mobile-design/capture-mobile.ts | 48 +++++++++++++----- evals/mobile-design/cases.ts | 1 + evals/mobile-design/ios.ts | 48 +++++++++--------- evals/mobile-design/pack.ts | 70 ++++++++++++-------------- evals/mobile-design/run.test.ts | 10 ++++ evals/mobile-design/run.ts | 22 +++++++- 7 files changed, 129 insertions(+), 75 deletions(-) diff --git a/evals/mobile-design/capture-desktop.ts b/evals/mobile-design/capture-desktop.ts index 670afd022f59e..d21d4f6170726 100644 --- a/evals/mobile-design/capture-desktop.ts +++ b/evals/mobile-design/capture-desktop.ts @@ -115,7 +115,10 @@ export async function captureDesktop(input: { ], { allowFailure: true, timeoutMs: 120_000 }, ); - const problem = result.code === 0 ? undefined : result.stderr.trim().split("\n")[0]; + const problem = + result.code === 0 + ? undefined + : result.stderr.trim().split("\n")[0] || `shot exited with code ${result.code}`; if (problem) input.log(`desktop/${scene.id}: ${problem}`); shots.push({ scene: scene.id, file, ...(problem ? { problem } : {}) }); } diff --git a/evals/mobile-design/capture-mobile.ts b/evals/mobile-design/capture-mobile.ts index 3a9d5b1aad558..4988516b06936 100644 --- a/evals/mobile-design/capture-mobile.ts +++ b/evals/mobile-design/capture-mobile.ts @@ -124,6 +124,12 @@ async function captureScene( return { pose: pose.id, scene: scene.id, file, ...(problem ? { problem } : {}) }; } +function scenesFor(pose: Pose, only: ReadonlySet | null): ReadonlyArray { + return (pose.scenes ?? SCENES.map((scene) => scene.id)).filter( + (scene) => only === null || only.has(scene), + ); +} + /** A screen is its title: the text in the top band, where the navigation bar or a sheet header sits. */ async function screenSignature(simulator: Simulator): Promise { const snapshot = await simulator.snapshot(); @@ -254,23 +260,25 @@ export async function captureMobile(input: { await seedComposerDrafts(simulator, targets); input.log(`${device}: launching`); await simulator.launch(`http://127.0.0.1:${metroPort}`); - const pairingUrls = await Promise.all( - environments.map(async (environment) => - buildShowcasePairingUrl( + const pairings = await Promise.all( + environments.map(async (environment) => ({ + url: buildShowcasePairingUrl( "127.0.0.1", environment.port, await issuePairingCredential(environment.baseDir), ), - ), + label: environment.label, + })), ); - await simulator.pair(pairingUrls, THREAD_LIST); + await simulator.pair(pairings); for (const pose of input.poses.filter((candidate) => candidate.device === device)) { input.log(`${pose.id}: capturing`); - const poseProblem = await simulator.applyPose(pose); + // One failed XCTest call marks this pose's shots; it should not cost the other poses. + const poseProblem = await simulator + .applyPose(pose) + .catch((error: unknown) => (error instanceof Error ? error.message : String(error))); if (poseProblem) input.log(`${pose.id}: ${poseProblem}`); - const scenes = (pose.scenes ?? SCENES.map((scene) => scene.id)).filter( - (scene) => input.scenes === null || input.scenes.has(scene), - ); + const scenes = scenesFor(pose, input.scenes); for (const scene of scenes) { const shot = await captureScene( simulator, @@ -286,7 +294,10 @@ export async function captureMobile(input: { } const flowPose = input.poses.find((pose) => pose.id === "iphone-dark"); if (input.flows && device === "iphone" && flowPose) { - await simulator.applyPose(flowPose); + const flowProblem = await simulator + .applyPose(flowPose) + .catch((error: unknown) => (error instanceof Error ? error.message : String(error))); + if (flowProblem) input.log(`flows: ${flowProblem}`); for (const flow of FLOWS) { input.log(`flow ${flow.id}`); const result = await captureFlow( @@ -301,8 +312,21 @@ export async function captureMobile(input: { } } } catch (error) { - // One device failing should not cost the others their captures. - input.log(`${device}: stopped: ${error instanceof Error ? error.message : String(error)}`); + // One device failing should not cost the others their captures. Its missing scenes are + // recorded as failed, so a reused run directory cannot pack their older screenshots. + const problem = `${device} stopped: ${error instanceof Error ? error.message : String(error)}`; + input.log(problem); + for (const pose of input.poses.filter((candidate) => candidate.device === device)) { + for (const scene of scenesFor(pose, input.scenes)) { + if (shots.some((shot) => shot.pose === pose.id && shot.scene === scene)) continue; + shots.push({ + pose: pose.id, + scene, + file: NodePath.join("captures", pose.id, `${scene}.png`), + problem, + }); + } + } } finally { await simulator?.delete(); } diff --git a/evals/mobile-design/cases.ts b/evals/mobile-design/cases.ts index f6ddfc4717c8f..1cdef85e5f57c 100644 --- a/evals/mobile-design/cases.ts +++ b/evals/mobile-design/cases.ts @@ -343,6 +343,7 @@ export const FLOWS = [ { tap: "^Show a reconnect banner$" }, { tap: "^Model and reasoning settings" }, { tap: "^GPT-5.3 Codex" }, + { tap: "^Save thread settings$", name: "Save" }, ], }, ] as const satisfies ReadonlyArray; diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index d68ea887d8c78..36dd096848db4 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -188,42 +188,48 @@ export class Simulator { throw new Error("The app did not load from Metro."); } - /** Pairs every environment, then waits for the list to show `threadList`. */ - async pair(pairingUrls: ReadonlyArray, threadList: RegExp) { - for (const pairingUrl of pairingUrls) { - const query = new URLSearchParams({ pairingUrl, autoConnect: "1" }); + /** Pairs every environment, then checks that each one shows as paired. */ + async pair(pairings: ReadonlyArray<{ readonly url: string; readonly label: string }>) { + for (const pairing of pairings) { + const query = new URLSearchParams({ pairingUrl: pairing.url, autoConnect: "1" }); // Pairing runs once per device, so it can wait out a slow host. await this.openUrl(`t3code://connections/new?${query}`, 180_000); await delay(4_000); } - await this.waitFor(threadList, "the paired thread list", { - timeoutMs: 180_000, - offScreen: true, - }); + // The environments page lists every paired machine, however long the thread list is. + await this.openUrl("t3code://settings/environments"); + for (const pairing of pairings) { + await this.waitFor(new RegExp(`^${RegExp.escape(pairing.label)}`), pairing.label, { + timeoutMs: 180_000, + }); + } } - /** iOS asks before a link opens the app; accept the prompt once it shows. */ + /** + * Opens a link. Some runtimes ask "Open in …?" first: accept that prompt, or tap its Open + * button when the runner sees it but cannot answer. Without a prompt the link opened already. + */ async openUrl(url: string, timeoutMs = 60_000) { await simctl("openurl", this.udid, url); - // A busy host can take seconds per XCTest call, and the prompt itself can be late. - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { + const started = Date.now(); + while (Date.now() < started + timeoutMs) { const accepted = await this.agent(["alert", "accept"]).then( () => true, () => false, ); if (accepted) return; - // The Duo's runner can see the prompt without answering it; its Open button still takes a tap. const snapshot = await this.snapshot().catch(() => null); - const open = snapshot?.nodes.some((node) => /^Open in /.test(node.label ?? "")) - ? this.find(snapshot, /^Open$/) - : undefined; + const prompted = snapshot?.nodes.some((node) => /^Open in /.test(node.label ?? "")); + const open = snapshot && prompted ? this.find(snapshot, /^Open$/) : undefined; if (open) { await this.press(open); return; } + // A prompt shows within a second or two; a clean screen after that means none is coming. + if (snapshot && !prompted && Date.now() - started > 4_000) return; await delay(500); } + throw new Error(`no one answered the prompt to open ${url.split("?")[0]}`); } async agent(args: ReadonlyArray, timeoutMs = 180_000): Promise> { @@ -401,13 +407,9 @@ export class Simulator { return width > height; }; if ((await isLandscape()) === (shape === "landscape")) return shape === "landscape"; - for (const name of shape === "landscape" - ? ["landscape-left", "landscape-right"] - : ["portrait"]) { - await this.agent(["orientation", name]); - await delay(2_000); - if ((await isLandscape()) === (shape === "landscape")) return shape === "landscape"; - } + // screenshot() turns a portrait frame by 270°, which matches landscape-left only. + await this.agent(["orientation", shape === "landscape" ? "landscape-left" : "portrait"]); + await delay(2_000); // A portrait-locked app keeps its shape when the device turns; the capture shows that. return isLandscape(); } diff --git a/evals/mobile-design/pack.ts b/evals/mobile-design/pack.ts index d6a3d3586cc45..901005292a9ba 100644 --- a/evals/mobile-design/pack.ts +++ b/evals/mobile-design/pack.ts @@ -83,6 +83,11 @@ function shapeGroup(pose: Pose): string { return `ipad-${pose.orientation}`; } +/** Fills `{{KEY}}` placeholders. A replacer function inserts values literally, so `$'` stays text. */ +export function fillTemplate(template: string, values: Readonly>): string { + return template.replaceAll(/\{\{(\w+)\}\}/g, (match, key: string) => values[key] ?? match); +} + export interface PackEntry { readonly file: string; readonly scene: string; @@ -198,52 +203,40 @@ export async function buildPack(input: { ), ); const template = await NodeFSP.readFile(NodePath.join(import.meta.dirname, "brief.md"), "utf8"); - const brief = template - .replace( - "{{SCREENS}}", - entries - .filter((entry) => entry.view !== "flow") - .map((entry) => `- \`${entry.file}\`: ${entry.scene} (${entry.view})`) - .join("\n"), - ) - .replace( - "{{FLOWS}}", + const values: Record = { + SCREENS: entries + .filter((entry) => entry.view !== "flow") + .map((entry) => `- \`${entry.file}\`: ${entry.scene} (${entry.view})`) + .join("\n"), + FLOWS: flowRows.length > 0 ? ["| Task | Taps | Screens | Filmstrip |", "| --- | --- | --- | --- |", ...flowRows].join( "\n", ) : "No flows were captured this round.", - ) - .replace( - "{{REFERENCES}}", + REFERENCES: referenceImages.length > 0 ? referenceImages.map((name) => `- \`references/${name}\``).join("\n") : "No reference screenshots of the Codex or Claude iOS apps are in this pack. Judge that axis from what you know of those apps, and say so.", - ) - .replace( - "{{AXES}}", - AXES.map((axis) => `- \`${axis.id}\`: **${axis.label}**: ${axis.description}.`).join("\n"), - ) - .replace( - "{{DEVICES}}", - devices.map((device) => `- \`${device.id}\`: ${device.label}`).join("\n"), - ) - .replace( - "{{JSON}}", - JSON.stringify( - { - scores: Object.fromEntries( - devices.map((device) => [ - device.id, - Object.fromEntries(AXES.map((axis) => [axis.id, 0])), - ]), - ), - fixes: ["…", "…", "…", "…", "…"], - }, - null, - 2, - ), - ); + AXES: AXES.map((axis) => `- \`${axis.id}\`: **${axis.label}**: ${axis.description}.`).join( + "\n", + ), + DEVICES: devices.map((device) => `- \`${device.id}\`: ${device.label}`).join("\n"), + JSON: JSON.stringify( + { + scores: Object.fromEntries( + devices.map((device) => [ + device.id, + Object.fromEntries(AXES.map((axis) => [axis.id, 0])), + ]), + ), + fixes: ["…", "…", "…", "…", "…"], + }, + null, + 2, + ), + }; + const brief = fillTemplate(template, values); const briefFile = NodePath.join(pack, "brief.md"); await NodeFSP.writeFile(briefFile, brief); await NodeFSP.writeFile( @@ -252,6 +245,7 @@ export async function buildPack(input: { { brief: briefFile, scores: NodePath.join(input.outDirectory, "scores"), + devices: devices.map((device) => device.id), panel: input.panel.map((member) => ({ member, ...PANEL[member] })), }, null, diff --git a/evals/mobile-design/run.test.ts b/evals/mobile-design/run.test.ts index 12140ef52629f..d885d3a888cd2 100644 --- a/evals/mobile-design/run.test.ts +++ b/evals/mobile-design/run.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vite-plus/test"; import { AXES, DEVICE_CLASSES } from "./cases.ts"; +import { fillTemplate } from "./pack.ts"; import { median, parseReviewerReply, renderSummary, summarizeRound } from "./scores.ts"; const uniform = (value: number) => @@ -69,3 +70,12 @@ describe("summarizeRound", () => { expect(renderSummary(round, baseline)).toContain("The baseline panel was opus"); }); }); + +describe("fillTemplate", () => { + it("inserts values literally, even replacement patterns from a stopped flow", () => { + const flows = "stopped: timed out waiting for '^Plan offline mode$' (saw: Chats)"; + expect(fillTemplate("{{FLOWS}}\n{{AXES}}", { FLOWS: flows, AXES: "axes" })).toBe( + `${flows}\naxes`, + ); + }); +}); diff --git a/evals/mobile-design/run.ts b/evals/mobile-design/run.ts index 8fb4163410838..4f6e7f09a8b52 100644 --- a/evals/mobile-design/run.ts +++ b/evals/mobile-design/run.ts @@ -7,7 +7,15 @@ import * as NodeUtil from "node:util"; import { captureDesktop, findShot, type DesktopShot } from "./capture-desktop.ts"; import { captureMobile, type FlowResult, type MobileShot } from "./capture-mobile.ts"; -import { DEFAULT_PANEL, PANEL, POSES, SCENES, type PanelMember, type SceneId } from "./cases.ts"; +import { + DEFAULT_PANEL, + PANEL, + POSES, + SCENES, + type DeviceClassId, + type PanelMember, + type SceneId, +} from "./cases.ts"; import { buildPack } from "./pack.ts"; import { parseReviewerReply, renderSummary, summarizeRound, type RoundSummary } from "./scores.ts"; @@ -132,6 +140,18 @@ async function summarize() { ), ) : null; + // A reply that skips a device the pack shows would quietly thin that device's median. + const council: { readonly devices?: ReadonlyArray } = await NodeFSP.readFile( + NodePath.join(out, "council.json"), + "utf8", + ).then( + (text) => JSON.parse(text), + () => ({}), + ); + for (const [name, review] of Object.entries(reviewers)) { + const missing = (council.devices ?? []).filter((device) => !review.scores[device]); + if (missing.length > 0) throw new Error(`${name}.md does not score ${missing.join(", ")}.`); + } const round = summarizeRound(reviewers); await NodeFSP.writeFile( NodePath.join(out, "summary.json"), From 9c9041bdc33a0a3ec1601d7bce024734c1e578db Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 27 Oct 2026 07:13:03 -0700 Subject: [PATCH 10/21] fix(evals): mark flows that did not run or ran in the wrong pose Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/capture-mobile.ts | 12 ++++++++++-- evals/mobile-design/ios.ts | 4 +++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/evals/mobile-design/capture-mobile.ts b/evals/mobile-design/capture-mobile.ts index 4988516b06936..b532861216f0f 100644 --- a/evals/mobile-design/capture-mobile.ts +++ b/evals/mobile-design/capture-mobile.ts @@ -146,10 +146,11 @@ async function captureFlow( flow: Flow, targets: LinkTargets, outDirectory: string, + poseProblem: string | undefined, ): Promise { const frames: Array<{ file: string; action: string }> = []; const signatures = new Set(); - let problem: string | undefined; + let problem = poseProblem; const frame = async (action: string) => { const file = NodePath.join("captures", "flows", flow.id, `${frames.length}.png`); await delay(1_200); @@ -179,7 +180,7 @@ async function captureFlow( await frame(action); } } catch (error) { - problem = error instanceof Error ? error.message : String(error); + problem ??= error instanceof Error ? error.message : String(error); } return { flow: flow.id, @@ -306,6 +307,7 @@ export async function captureMobile(input: { flow, targets, input.outDirectory, + flowProblem, ); if (result.problem) input.log(`flow ${flow.id}: ${result.problem}`); flows.push(result); @@ -327,6 +329,12 @@ export async function captureMobile(input: { }); } } + if (input.flows && device === "iphone") { + for (const flow of FLOWS) { + if (flows.some((result) => result.flow === flow.id)) continue; + flows.push({ flow: flow.id, taps: 0, screens: 0, frames: [], problem }); + } + } } finally { await simulator?.delete(); } diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index 36dd096848db4..8337918eea509 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -229,7 +229,9 @@ export class Simulator { if (snapshot && !prompted && Date.now() - started > 4_000) return; await delay(500); } - throw new Error(`no one answered the prompt to open ${url.split("?")[0]}`); + throw new Error( + `the prompt to open ${url.split("?")[0]} was not answered, or the screen could not be read`, + ); } async agent(args: ReadonlyArray, timeoutMs = 180_000): Promise> { From 45c0a577ca081f31d9c9ca452931c511eaf870a8 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 27 Oct 2026 07:24:20 -0700 Subject: [PATCH 11/21] fix(evals): keep every cause when a capture fails twice Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/capture-mobile.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/evals/mobile-design/capture-mobile.ts b/evals/mobile-design/capture-mobile.ts index b532861216f0f..f7210bb2e6b49 100644 --- a/evals/mobile-design/capture-mobile.ts +++ b/evals/mobile-design/capture-mobile.ts @@ -95,6 +95,10 @@ async function seedComposerDrafts(simulator: Simulator, targets: LinkTargets) { ); } +/** Keeps every cause when a capture fails for more than one reason. */ +const joinProblems = (earlier: string | undefined, next: string) => + earlier ? `${earlier}; ${next}` : next; + async function captureScene( simulator: Simulator, pose: Pose, @@ -111,12 +115,15 @@ async function captureScene( for (const step of scene.steps) await simulator.perform(step, targets); if ("ready" in scene) await simulator.waitFor(new RegExp(scene.ready), scene.title); } catch (error) { - problem ??= error instanceof Error ? error.message : String(error); + problem = joinProblems(problem, error instanceof Error ? error.message : String(error)); } // Screenshot a failed scene too: where it stopped is the diagnosis. await delay(1_500); await simulator.screenshot(NodePath.join(outDirectory, file), pose).catch((error: unknown) => { - problem ??= `screenshot failed: ${error instanceof Error ? error.message : String(error)}`; + problem = joinProblems( + problem, + `screenshot failed: ${error instanceof Error ? error.message : String(error)}`, + ); }); if ("reset" in scene) { for (const step of scene.reset) await simulator.perform(step, targets).catch(() => undefined); @@ -180,7 +187,7 @@ async function captureFlow( await frame(action); } } catch (error) { - problem ??= error instanceof Error ? error.message : String(error); + problem = joinProblems(problem, error instanceof Error ? error.message : String(error)); } return { flow: flow.id, From aab5b099a320e3ef467bbf1569675dca3c4490c7 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Wed, 28 Oct 2026 00:39:54 -0700 Subject: [PATCH 12/21] fix(evals): mobile design steps accept the redesign's labels Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/brief.md | 2 ++ evals/mobile-design/cases.ts | 12 ++++++++---- evals/mobile-design/ios.ts | 10 +++++++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/evals/mobile-design/brief.md b/evals/mobile-design/brief.md index 4bdd55e1e7f21..b139074a12730 100644 --- a/evals/mobile-design/brief.md +++ b/evals/mobile-design/brief.md @@ -38,6 +38,8 @@ Device classes, by id: {{DEVICES}} +Then say what feels worse, blander or less clear than it should, and what works well and must stay as it is. Name the screen and device for each. + Then list the five fixes that would raise the scores most, most important first. Each fix names the screen and device, what is wrong, and the change you would make. Be specific: "the question panel leaves a 120 pt scroll area on the closed Duo; let it take the space above the composer" is useful, "improve spacing" is not. ## Reply format diff --git a/evals/mobile-design/cases.ts b/evals/mobile-design/cases.ts index 1cdef85e5f57c..e4b2e71032544 100644 --- a/evals/mobile-design/cases.ts +++ b/evals/mobile-design/cases.ts @@ -290,7 +290,7 @@ export const FLOWS = [ title: "Start a thread in a chosen project and machine", source: REQUEST, steps: [ - { tap: "^New chat$" }, + { tap: "^(New chat|New thread)$", name: "New thread" }, { tap: "^T3 Code$" }, { tap: "^React$" }, { tap: "^Choose environment" }, @@ -319,8 +319,8 @@ export const FLOWS = [ source: REQUEST, steps: [ { tap: "^Make remote coding feel local" }, - { tap: "^Git$" }, - { tap: "^Review changes" }, + { tap: "^(Git|Toggle panel)$", name: "Git or Panel" }, + { tap: "^(Review changes|Diff$)", name: "Review changes or Diff" }, ], }, { @@ -333,7 +333,11 @@ export const FLOWS = [ id: "find-old-thread", title: "Find an old thread", source: REQUEST, - steps: [{ tap: "^Search$" }, { type: "OOM" }, { tap: "OOM killer" }], + steps: [ + { tap: "^(Search|Search threads)$", name: "Search" }, + { type: "OOM" }, + { tap: "OOM killer" }, + ], }, { id: "change-model", diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index 8337918eea509..eb6a106edf120 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -164,7 +164,7 @@ export class Simulator { ] as const) { await simctl("spawn", this.udid, "defaults", "write", BUNDLE_ID, key, "-bool", value); } - const loaded = /^(Chats|Add environment|New chat|Search threads)$/; + const loaded = /^(Chats|Threads|Add environment|New chat|New thread|Search threads)$/; for (let attempt = 0; attempt < 3; attempt += 1) { // Launch arguments only reach a fresh process, so start it before agent-device attaches. await simctl( @@ -351,8 +351,12 @@ export class Simulator { const back = snapshot.nodes.find( (node) => node.kind === "button" && - (/^(Back|Close|Done|Cancel)$/.test(node.label ?? "") || - (node.label === "Chats" && (node.rect?.y ?? Infinity) < snapshot.viewport.height / 4)), + (/^(Back|Close|Done|Cancel|Go to threads list|Back to chat|Return to chat)$/.test( + node.label ?? "", + ) || + // A pushed page's back button is named after the list it returns to. + (/^(Chats|Threads)$/.test(node.label ?? "") && + (node.rect?.y ?? Infinity) < snapshot.viewport.height / 4)), ); if (!back && this.find(snapshot, threadListLabel)) { await this.agent(["keyboard", "dismiss"]).catch(() => undefined); From 4427d3c06977aebbbffc080efc430c9a15156a00 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Wed, 28 Oct 2026 01:00:59 -0700 Subject: [PATCH 13/21] fix(evals): return to the thread list from an iPad sidebar section Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/ios.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index eb6a106edf120..cf3474bf89348 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -348,9 +348,10 @@ export class Simulator { for (let attempt = 0; attempt < 8; attempt += 1) { const snapshot = await this.snapshot(); // A thread's back button is labelled with the tab it returns to, so only count it up top. + // The iPad sidebar's section footer "Back" is not a button, so accept it by label alone. const back = snapshot.nodes.find( (node) => - node.kind === "button" && + (node.kind === "button" || node.label === "Back") && (/^(Back|Close|Done|Cancel|Go to threads list|Back to chat|Return to chat)$/.test( node.label ?? "", ) || From 675823e44961e31ef6a14d57fb8ee3d6e1a90836 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Wed, 28 Oct 2026 01:54:14 -0700 Subject: [PATCH 14/21] fix(evals): relaunch when no way back works, and tap iPad landscape scenes in portrait Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/capture-mobile.ts | 4 +++- evals/mobile-design/cases.ts | 2 +- evals/mobile-design/ios.ts | 28 +++++++++++++++++++++++---- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/evals/mobile-design/capture-mobile.ts b/evals/mobile-design/capture-mobile.ts index f7210bb2e6b49..0aca36af28238 100644 --- a/evals/mobile-design/capture-mobile.ts +++ b/evals/mobile-design/capture-mobile.ts @@ -112,7 +112,9 @@ async function captureScene( let problem = poseProblem; try { await simulator.home(THREAD_LIST); - for (const step of scene.steps) await simulator.perform(step, targets); + await simulator.withUsableTaps(async () => { + for (const step of scene.steps) await simulator.perform(step, targets); + }); if ("ready" in scene) await simulator.waitFor(new RegExp(scene.ready), scene.title); } catch (error) { problem = joinProblems(problem, error instanceof Error ? error.message : String(error)); diff --git a/evals/mobile-design/cases.ts b/evals/mobile-design/cases.ts index e4b2e71032544..38458ee76de04 100644 --- a/evals/mobile-design/cases.ts +++ b/evals/mobile-design/cases.ts @@ -319,7 +319,7 @@ export const FLOWS = [ source: REQUEST, steps: [ { tap: "^Make remote coding feel local" }, - { tap: "^(Git|Toggle panel)$", name: "Git or Panel" }, + { tap: "^(Git|Panel|Toggle panel)$", name: "Git or Panel" }, { tap: "^(Review changes|Diff$)", name: "Review changes or Diff" }, ], }, diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index cf3474bf89348..08aadb372327a 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -108,6 +108,7 @@ export class Simulator { readonly udid: string; private readonly session: string; private appLandscape = false; + private metroUrl: string | null = null; private constructor(device: DeviceClassId, udid: string, session: string) { this.device = device; @@ -156,7 +157,8 @@ export class Simulator { * Loads the JS straight from Metro. The dev menu's preferences hide its tools button and * launch sheet; the same flags on --initialUrl do not reach them. */ - async launch(metroUrl: string) { + async launch(metroUrl: string, options: { readonly restart?: boolean } = {}) { + this.metroUrl = metroUrl; for (const [key, value] of [ ["EXDevMenuShowFloatingActionButton", "NO"], ["EXDevMenuShowsAtLaunch", "NO"], @@ -169,7 +171,7 @@ export class Simulator { // Launch arguments only reach a fresh process, so start it before agent-device attaches. await simctl( "launch", - ...(attempt > 0 ? ["--terminate-running-process"] : []), + ...(attempt > 0 || options.restart ? ["--terminate-running-process"] : []), this.udid, BUNDLE_ID, "--initialUrl", @@ -345,7 +347,7 @@ export class Simulator { /** Backs out of sheets, pushed screens and the keyboard until the thread list shows. */ async home(threadListLabel: RegExp) { - for (let attempt = 0; attempt < 8; attempt += 1) { + for (let attempt = 0; attempt < 5; attempt += 1) { const snapshot = await this.snapshot(); // A thread's back button is labelled with the tab it returns to, so only count it up top. // The iPad sidebar's section footer "Back" is not a button, so accept it by label alone. @@ -378,7 +380,11 @@ export class Simulator { else await this.agent(["back"]).catch(() => undefined); await delay(1_000); } - throw new Error("could not return to the thread list"); + // Some screens offer no way back that works, such as an iPad sidebar section opened by a + // link; a fresh launch always opens on the thread list. + if (!this.metroUrl) throw new Error("could not return to the thread list"); + await this.launch(this.metroUrl, { restart: true }); + await this.waitFor(threadListLabel, "the thread list after a relaunch", { offScreen: true }); } /** @@ -407,6 +413,20 @@ export class Simulator { return undefined; } + /** + * Runs `steps` in portrait when the iPad is in landscape, then turns back. XCTest sends taps to + * a landscape iPad in portrait coordinates, so they land on the wrong element. + */ + async withUsableTaps(steps: () => Promise) { + if (this.device !== "ipad" || !this.appLandscape) return steps(); + await this.orient("portrait"); + try { + await steps(); + } finally { + this.appLandscape = await this.orient("landscape"); + } + } + /** Returns whether the app now lays out in landscape. */ private async orient(shape: "portrait" | "landscape"): Promise { const isLandscape = async () => { From 321ade00a5a68d82d3fd421646550e2d967c4171 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Wed, 28 Oct 2026 02:09:48 -0700 Subject: [PATCH 15/21] fix(evals): flow frames name the label a build shows, so packs stay blind Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/capture-mobile.ts | 5 +++-- evals/mobile-design/cases.ts | 21 ++++++++------------- evals/mobile-design/ios.ts | 11 ++++++++--- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/evals/mobile-design/capture-mobile.ts b/evals/mobile-design/capture-mobile.ts index 0aca36af28238..a925b29c6a038 100644 --- a/evals/mobile-design/capture-mobile.ts +++ b/evals/mobile-design/capture-mobile.ts @@ -173,10 +173,11 @@ async function captureFlow( await simulator.home(THREAD_LIST); await frame("Start"); for (const step of flow.steps) { - await simulator.perform(step, targets); + const tapped = await simulator.perform(step, targets); + // The label actually tapped, so the filmstrip names what this build shows. const action = "tap" in step - ? `Tap ${step.name ?? step.tap.replaceAll(/[\^$]/g, "")}` + ? `Tap ${tapped}` : "type" in step ? `Type "${step.type}"` : "focus" in step diff --git a/evals/mobile-design/cases.ts b/evals/mobile-design/cases.ts index 38458ee76de04..6cccede46d223 100644 --- a/evals/mobile-design/cases.ts +++ b/evals/mobile-design/cases.ts @@ -1,8 +1,7 @@ /** * Steps: `link` opens `t3code://`, where `{thread:}` becomes `/` * for a seeded or live fixture thread; `tap` presses the first element whose accessibility label - * matches the regex, shown in flows as its `name` when it has one; `focus: "composer"` taps the - * unlabelled composer. + * matches the regex; `focus: "composer"` taps the unlabelled composer. */ export const AXES = [ @@ -56,7 +55,7 @@ export const SIMULATORS = { export type StepAction = | { readonly link: string } - | { readonly tap: string; readonly name?: string } + | { readonly tap: string } | { readonly type: string } | { readonly scroll: "up" | "down" } | { readonly focus: "composer" } @@ -290,12 +289,12 @@ export const FLOWS = [ title: "Start a thread in a chosen project and machine", source: REQUEST, steps: [ - { tap: "^(New chat|New thread)$", name: "New thread" }, + { tap: "^(New chat|New thread)$" }, { tap: "^T3 Code$" }, { tap: "^React$" }, { tap: "^Choose environment" }, // Pick whichever machine the picker offers; when it is not the project's, the frames show it. - { tap: "^(Suspense Station|Moonbase Terminal|Kernel Cabin)", name: "the offered machine" }, + { tap: "^(Suspense Station|Moonbase Terminal|Kernel Cabin)" }, { focus: "composer" }, { type: "Add a loading skeleton to the profile page" }, { tap: "^Send$" }, @@ -319,8 +318,8 @@ export const FLOWS = [ source: REQUEST, steps: [ { tap: "^Make remote coding feel local" }, - { tap: "^(Git|Panel|Toggle panel)$", name: "Git or Panel" }, - { tap: "^(Review changes|Diff$)", name: "Review changes or Diff" }, + { tap: "^(Git|Panel|Toggle panel)$" }, + { tap: "^(Review changes|Diff$)" }, ], }, { @@ -333,11 +332,7 @@ export const FLOWS = [ id: "find-old-thread", title: "Find an old thread", source: REQUEST, - steps: [ - { tap: "^(Search|Search threads)$", name: "Search" }, - { type: "OOM" }, - { tap: "OOM killer" }, - ], + steps: [{ tap: "^(Search|Search threads)$" }, { type: "OOM" }, { tap: "OOM killer" }], }, { id: "change-model", @@ -347,7 +342,7 @@ export const FLOWS = [ { tap: "^Show a reconnect banner$" }, { tap: "^Model and reasoning settings" }, { tap: "^GPT-5.3 Codex" }, - { tap: "^Save thread settings$", name: "Save" }, + { tap: "^Save thread settings$" }, ], }, ] as const satisfies ReadonlyArray; diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index 08aadb372327a..a134597b9b13f 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -319,10 +319,12 @@ export class Simulator { ); } - async tap(label: RegExp) { + /** Taps the first element labelled `label`, and returns the label it tapped. */ + async tap(label: RegExp): Promise { const node = this.find(await this.waitFor(label, `'${label.source}'`), label); if (!node) throw new Error(`nothing labelled ${label}`); await this.press(node); + return (node.label ?? "").split(",")[0]!.slice(0, 40); } /** The composer is a native text view with no label: the lowest editable field on screen. */ @@ -335,14 +337,17 @@ export class Simulator { return composer; } - async perform(step: StepAction, targets: LinkTargets) { + /** Performs one step; returns the tapped label for a `tap` step. */ + async perform(step: StepAction, targets: LinkTargets): Promise { + let tapped: string | undefined; if ("link" in step) await this.openUrl(`t3code://${expandPath(step.link, targets)}`); - else if ("tap" in step) await this.tap(new RegExp(step.tap)); + else if ("tap" in step) tapped = await this.tap(new RegExp(step.tap)); else if ("type" in step) await this.agent(["type", step.type]); else if ("scroll" in step) await this.agent(["scroll", step.scroll]); else if ("focus" in step) await this.press(await this.composer()); else await this.agent(["fill", `@${(await this.composer()).ref}`, ""]); await delay(1_200); + return tapped; } /** Backs out of sheets, pushed screens and the keyboard until the thread list shows. */ From 447b50cf499a6793cc40fa1b993c4b03da0aa071 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Wed, 28 Oct 2026 03:10:49 -0700 Subject: [PATCH 16/21] fix(evals): mobile design flows take the shortest control a build offers Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/capture-mobile.ts | 1 + evals/mobile-design/cases.ts | 19 ++++++++----- evals/mobile-design/ios.ts | 39 ++++++++++++++++++++------- 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/evals/mobile-design/capture-mobile.ts b/evals/mobile-design/capture-mobile.ts index a925b29c6a038..e52f3a431688f 100644 --- a/evals/mobile-design/capture-mobile.ts +++ b/evals/mobile-design/capture-mobile.ts @@ -174,6 +174,7 @@ async function captureFlow( await frame("Start"); for (const step of flow.steps) { const tapped = await simulator.perform(step, targets); + if (tapped === null) continue; // The label actually tapped, so the filmstrip names what this build shows. const action = "tap" in step diff --git a/evals/mobile-design/cases.ts b/evals/mobile-design/cases.ts index 6cccede46d223..fb016d0920786 100644 --- a/evals/mobile-design/cases.ts +++ b/evals/mobile-design/cases.ts @@ -55,7 +55,14 @@ export const SIMULATORS = { export type StepAction = | { readonly link: string } - | { readonly tap: string } + | { + /** One pattern, or several in order of preference; the first one on screen is tapped. */ + readonly tap: string | ReadonlyArray; + /** Skip the step when nothing matches within a few seconds. */ + readonly optional?: true; + /** Skip the step when this label already shows. */ + readonly skipIf?: string; + } | { readonly type: string } | { readonly scroll: "up" | "down" } | { readonly focus: "composer" } @@ -178,7 +185,7 @@ export const SCENES = [ title: "Agent asks two questions", source: `${REQUEST}; #24290 and #24291 (Duo question panel)`, steps: [thread("question")], - ready: "^Submit answers$", + ready: "^(Submit answers|Next question)$", desktop: { path: "/{thread:question}" }, }, { @@ -307,7 +314,7 @@ export const FLOWS = [ steps: [ { tap: "^Plan offline mode$" }, { tap: "^Last 50 threads" }, - { scroll: "down" }, + { tap: "^Next question$", optional: true }, { tap: "^Server state" }, { tap: "^Submit answers$" }, ], @@ -318,8 +325,8 @@ export const FLOWS = [ source: REQUEST, steps: [ { tap: "^Make remote coding feel local" }, - { tap: "^(Git|Panel|Toggle panel)$" }, - { tap: "^(Review changes|Diff$)" }, + { tap: ["^Changes [+]", "^(Git|Panel|Toggle panel)$"] }, + { tap: "^(Review changes|Diff$)", skipIf: "^Select diff$" }, ], }, { @@ -342,7 +349,7 @@ export const FLOWS = [ { tap: "^Show a reconnect banner$" }, { tap: "^Model and reasoning settings" }, { tap: "^GPT-5.3 Codex" }, - { tap: "^Save thread settings$" }, + { tap: "^Save thread settings$", optional: true }, ], }, ] as const satisfies ReadonlyArray; diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index a134597b9b13f..d0671ad9118df 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -185,7 +185,7 @@ export class Simulator { { timeoutMs: 180_000 }, ).catch(() => null); if (snapshot && this.find(snapshot, loaded)) return; - if (snapshot) await this.tap(/^Reload$/).catch(() => undefined); + if (snapshot) await this.tap([/^Reload$/]).catch(() => undefined); } throw new Error("The app did not load from Metro."); } @@ -319,10 +319,14 @@ export class Simulator { ); } - /** Taps the first element labelled `label`, and returns the label it tapped. */ - async tap(label: RegExp): Promise { - const node = this.find(await this.waitFor(label, `'${label.source}'`), label); - if (!node) throw new Error(`nothing labelled ${label}`); + /** Taps the first element matching the first pattern that shows, and returns its label. */ + async tap(patterns: ReadonlyArray, timeoutMs?: number): Promise { + const any = new RegExp(patterns.map((pattern) => `(?:${pattern.source})`).join("|")); + const snapshot = await this.waitFor(any, `'${any.source}'`, timeoutMs ? { timeoutMs } : {}); + const node = patterns + .map((pattern) => this.find(snapshot, pattern)) + .find((candidate) => candidate !== undefined); + if (!node) throw new Error(`nothing labelled ${any}`); await this.press(node); return (node.label ?? "").split(",")[0]!.slice(0, 40); } @@ -337,12 +341,22 @@ export class Simulator { return composer; } - /** Performs one step; returns the tapped label for a `tap` step. */ - async perform(step: StepAction, targets: LinkTargets): Promise { + /** Performs one step; returns the tapped label for a `tap` step, or null when it was skipped. */ + async perform(step: StepAction, targets: LinkTargets): Promise { let tapped: string | undefined; if ("link" in step) await this.openUrl(`t3code://${expandPath(step.link, targets)}`); - else if ("tap" in step) tapped = await this.tap(new RegExp(step.tap)); - else if ("type" in step) await this.agent(["type", step.type]); + else if ("tap" in step) { + if (step.skipIf && this.find(await this.snapshot(), new RegExp(step.skipIf))) return null; + const patterns = (typeof step.tap === "string" ? [step.tap] : step.tap).map( + (pattern) => new RegExp(pattern), + ); + try { + tapped = await this.tap(patterns, step.optional ? 5_000 : undefined); + } catch (error) { + if (step.optional) return null; + throw error; + } + } else if ("type" in step) await this.agent(["type", step.type]); else if ("scroll" in step) await this.agent(["scroll", step.scroll]); else if ("focus" in step) await this.press(await this.composer()); else await this.agent(["fill", `@${(await this.composer()).ref}`, ""]); @@ -388,6 +402,13 @@ export class Simulator { // Some screens offer no way back that works, such as an iPad sidebar section opened by a // link; a fresh launch always opens on the thread list. if (!this.metroUrl) throw new Error("could not return to the thread list"); + const stuck = await this.snapshot().catch(() => null); + process.stderr.write( + `[mobile-design] ${this.device}: no way back to the thread list worked, relaunching (screen: ${stuck?.nodes + .flatMap((node) => (node.label ? [`${node.kind}:${node.label.slice(0, 30)}`] : [])) + .slice(0, 14) + .join(" | ")})\n`, + ); await this.launch(this.metroUrl, { restart: true }); await this.waitFor(threadListLabel, "the thread list after a relaunch", { offScreen: true }); } From c9e70b1ac6bac371c262ac19f045bbf1a54110d8 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Wed, 28 Oct 2026 03:22:54 -0700 Subject: [PATCH 17/21] fix(evals): press back controls in portrait on a landscape iPad Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/ios.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index d0671ad9118df..f4393c00ffcb7 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -369,11 +369,12 @@ export class Simulator { for (let attempt = 0; attempt < 5; attempt += 1) { const snapshot = await this.snapshot(); // A thread's back button is labelled with the tab it returns to, so only count it up top. - // The iPad sidebar's section footer "Back" is not a button, so accept it by label alone. + // The iPad sidebar footer's "Back" and a sheet's dimmed backdrop are not buttons, so accept + // them by label alone. const back = snapshot.nodes.find( (node) => - (node.kind === "button" || node.label === "Back") && - (/^(Back|Close|Done|Cancel|Go to threads list|Back to chat|Return to chat)$/.test( + (node.kind === "button" || /^(Back|dismiss popup)$/.test(node.label ?? "")) && + (/^(Back|Close|Done|Cancel|dismiss popup|Go to threads list|Back to chat|Return to chat)$/.test( node.label ?? "", ) || // A pushed page's back button is named after the list it returns to. @@ -394,9 +395,14 @@ export class Simulator { node.label === "Chats" && (node.rect?.y ?? 0) >= snapshot.viewport.height / 4, ); - if (back) await this.press(back); - else if (chatsTab) await this.press(chatsTab); - else await this.agent(["back"]).catch(() => undefined); + const target = back ?? chatsTab; + if (target) { + await this.withUsableTaps(async () => { + // A turn to portrait moves the element, so find it again by its label. + const label = new RegExp(`^${RegExp.escape(target.label ?? "")}$`); + await this.press(this.find(await this.snapshot(), label) ?? target); + }); + } else await this.agent(["back"]).catch(() => undefined); await delay(1_000); } // Some screens offer no way back that works, such as an iPad sidebar section opened by a From f1b42c69bdedbb693dcb57eeb8bc451cfe844496 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Wed, 28 Oct 2026 04:23:52 -0700 Subject: [PATCH 18/21] fix(evals): mobile design flows go through a folded header menu and an already open project picker Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/cases.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/evals/mobile-design/cases.ts b/evals/mobile-design/cases.ts index fb016d0920786..ed303a4d74396 100644 --- a/evals/mobile-design/cases.ts +++ b/evals/mobile-design/cases.ts @@ -297,7 +297,8 @@ export const FLOWS = [ source: REQUEST, steps: [ { tap: "^(New chat|New thread)$" }, - { tap: "^T3 Code$" }, + // New thread can open straight onto the project picker. + { tap: "^T3 Code$", skipIf: "^Choose project$" }, { tap: "^React$" }, { tap: "^Choose environment" }, // Pick whichever machine the picker offers; when it is not the project's, the frames show it. @@ -325,8 +326,13 @@ export const FLOWS = [ source: REQUEST, steps: [ { tap: "^Make remote coding feel local" }, - { tap: ["^Changes [+]", "^(Git|Panel|Toggle panel)$"] }, - { tap: "^(Review changes|Diff$)", skipIf: "^Select diff$" }, + // A crowded header folds its buttons into a More menu. + { tap: ["^Changes [+]", "^(Git|Panel|Toggle panel)$", "^More$"] }, + { + tap: ["^Changes [+]", "^(Review changes|Diff$)", "^(Panel|Toggle panel)$"], + skipIf: "^Select diff$", + }, + { tap: "^Diff$", optional: true, skipIf: "^Select diff$" }, ], }, { From e3f7b733a0c061fbb600afd5ed029e2402dc3c14 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Wed, 28 Oct 2026 06:11:44 -0700 Subject: [PATCH 19/21] fix(evals): mobile design flows take a draft's single project chip Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/cases.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/evals/mobile-design/cases.ts b/evals/mobile-design/cases.ts index ed303a4d74396..090a8b40157e8 100644 --- a/evals/mobile-design/cases.ts +++ b/evals/mobile-design/cases.ts @@ -176,7 +176,7 @@ export const SCENES = [ id: "new-thread", title: "New thread, project chosen", source: REQUEST, - steps: [{ link: "new" }, { tap: "^T3 Code$" }], + steps: [{ link: "new" }, { tap: "^T3 Code$", skipIf: "^Project: " }], // The web app has no new-thread route; its home page is the new-thread screen. desktop: { path: "/" }, }, @@ -297,12 +297,13 @@ export const FLOWS = [ source: REQUEST, steps: [ { tap: "^(New chat|New thread)$" }, - // New thread can open straight onto the project picker. - { tap: "^T3 Code$", skipIf: "^Choose project$" }, + // New thread can open straight onto the project picker, or a draft with one project chip. + { tap: ["^Project: ", "^T3 Code$"], skipIf: "^Choose project$" }, { tap: "^React$" }, - { tap: "^Choose environment" }, + // A build whose project chip also names the machine has no separate machine step. + { tap: "^Choose environment", skipIf: "^Project: " }, // Pick whichever machine the picker offers; when it is not the project's, the frames show it. - { tap: "^(Suspense Station|Moonbase Terminal|Kernel Cabin)" }, + { tap: "^(Suspense Station|Moonbase Terminal|Kernel Cabin)", skipIf: "^Project: " }, { focus: "composer" }, { type: "Add a loading skeleton to the profile page" }, { tap: "^Send$" }, @@ -330,9 +331,9 @@ export const FLOWS = [ { tap: ["^Changes [+]", "^(Git|Panel|Toggle panel)$", "^More$"] }, { tap: ["^Changes [+]", "^(Review changes|Diff$)", "^(Panel|Toggle panel)$"], - skipIf: "^Select diff$", + skipIf: "^(Select diff|Open full review)$", }, - { tap: "^Diff$", optional: true, skipIf: "^Select diff$" }, + { tap: "^Diff$", optional: true, skipIf: "^(Select diff|Open full review)$" }, ], }, { From e2d92d950c7394fd2fc4f7c857520dcf1a360fa4 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Wed, 28 Oct 2026 07:28:27 -0700 Subject: [PATCH 20/21] fix(evals): mobile design accepts the new review header and change chip, and opens the thread list fresh Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/capture-mobile.ts | 2 +- evals/mobile-design/cases.ts | 12 ++++++++---- evals/mobile-design/ios.ts | 6 ++++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/evals/mobile-design/capture-mobile.ts b/evals/mobile-design/capture-mobile.ts index e52f3a431688f..3d3d4f0d1b94f 100644 --- a/evals/mobile-design/capture-mobile.ts +++ b/evals/mobile-design/capture-mobile.ts @@ -111,7 +111,7 @@ async function captureScene( const file = NodePath.join("captures", pose.id, `${scene.id}.png`); let problem = poseProblem; try { - await simulator.home(THREAD_LIST); + await ("fresh" in scene ? simulator.relaunch(THREAD_LIST) : simulator.home(THREAD_LIST)); await simulator.withUsableTaps(async () => { for (const step of scene.steps) await simulator.perform(step, targets); }); diff --git a/evals/mobile-design/cases.ts b/evals/mobile-design/cases.ts index 090a8b40157e8..bb6ad406d29f7 100644 --- a/evals/mobile-design/cases.ts +++ b/evals/mobile-design/cases.ts @@ -90,6 +90,8 @@ export interface Scene { readonly steps: ReadonlyArray; /** Wait for this label before the screenshot. */ readonly ready?: string; + /** Relaunch first: on iPad, a thread an earlier scene opened would fill the detail column. */ + readonly fresh?: true; /** Desktop web path for the "conforms to desktop" pair, relative to the environment origin. */ readonly desktop?: { readonly path: string; readonly click?: string }; /** Runs after the screenshot, to undo state later scenes would see, such as a typed draft. */ @@ -105,6 +107,7 @@ export const SCENES = [ title: "Thread list", source: `${REQUEST}; #22543 (iPad sidebar hides the newest thread)`, steps: [], + fresh: true, ready: "^Plan offline mode", desktop: { path: "/" }, }, @@ -221,7 +224,8 @@ export const SCENES = [ title: "Review and diff", source: REQUEST, steps: [{ link: "threads/{thread:remote-command-center}/review" }], - ready: "^Changes vs", + // Older builds title it "Changes vs main"; newer ones put "vs main" on a second line. + ready: "^(Changes )?vs main", }, { id: "settings", @@ -299,7 +303,7 @@ export const FLOWS = [ { tap: "^(New chat|New thread)$" }, // New thread can open straight onto the project picker, or a draft with one project chip. { tap: ["^Project: ", "^T3 Code$"], skipIf: "^Choose project$" }, - { tap: "^React$" }, + { tap: "^React(,|$)" }, // A build whose project chip also names the machine has no separate machine step. { tap: "^Choose environment", skipIf: "^Project: " }, // Pick whichever machine the picker offers; when it is not the project's, the frames show it. @@ -328,9 +332,9 @@ export const FLOWS = [ steps: [ { tap: "^Make remote coding feel local" }, // A crowded header folds its buttons into a More menu. - { tap: ["^Changes [+]", "^(Git|Panel|Toggle panel)$", "^More$"] }, + { tap: ["^(Changes )?[+]\\d+ [-−]\\d+", "^(Git|Panel|Toggle panel)$", "^More$"] }, { - tap: ["^Changes [+]", "^(Review changes|Diff$)", "^(Panel|Toggle panel)$"], + tap: ["^(Changes )?[+]\\d+ [-−]\\d+", "^(Review changes|Diff$)", "^(Panel|Toggle panel)$"], skipIf: "^(Select diff|Open full review)$", }, { tap: "^Diff$", optional: true, skipIf: "^(Select diff|Open full review)$" }, diff --git a/evals/mobile-design/ios.ts b/evals/mobile-design/ios.ts index f4393c00ffcb7..f92b73559d2d2 100644 --- a/evals/mobile-design/ios.ts +++ b/evals/mobile-design/ios.ts @@ -415,6 +415,12 @@ export class Simulator { .slice(0, 14) .join(" | ")})\n`, ); + await this.relaunch(threadListLabel); + } + + /** Starts the app again; it opens on the thread list with nothing selected. */ + async relaunch(threadListLabel: RegExp) { + if (!this.metroUrl) throw new Error("the app was never launched"); await this.launch(this.metroUrl, { restart: true }); await this.waitFor(threadListLabel, "the thread list after a relaunch", { offScreen: true }); } From 810cc2dbcccab94766db2f40528081efd03821f8 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Wed, 28 Oct 2026 18:14:46 -0700 Subject: [PATCH 21/21] fix(evals): mobile design packs fit a reviewer's image limit Co-Authored-By: Claude Opus 5.5 --- evals/mobile-design/pack.ts | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/evals/mobile-design/pack.ts b/evals/mobile-design/pack.ts index 901005292a9ba..51487dd127d35 100644 --- a/evals/mobile-design/pack.ts +++ b/evals/mobile-design/pack.ts @@ -22,6 +22,9 @@ const MAX_WIDTH = 2400; const MAX_TILE_HEIGHT = 1400; const LABEL_HEIGHT = 44; const GAP = 24; +// Claude reads images at most 1568 px on the long edge, and one request holds about 32 MB of +// images, so a larger file only pushes the pack's later images out of a review. +const MAX_IMAGE_EDGE = 1568; interface Tile { readonly file: string; @@ -70,9 +73,16 @@ async function contactSheet( return { data, width: sheetWidth, height: sheetHeight }; } -async function writeImage(data: Buffer, destination: string) { +/** Writes an image no larger than a reviewer reads it; a PNG gets a palette to cut its size. */ +async function writeImage(input: Buffer | string, destination: string) { await NodeFSP.mkdir(NodePath.dirname(destination), { recursive: true }); - await NodeFSP.writeFile(destination, data); + const image = sharp(input).resize({ + width: MAX_IMAGE_EDGE, + height: MAX_IMAGE_EDGE, + fit: "inside", + withoutEnlargement: true, + }); + await (destination.endsWith(".png") ? image.png({ palette: true }) : image).toFile(destination); } /** Poses whose screenshots share a shape, so they sit well side by side. */ @@ -185,14 +195,11 @@ export async function buildPack(input: { const references = await NodeFSP.readdir(input.referencesDirectory).catch(() => []); const referenceImages = references.filter((name) => /\.(png|jpe?g)$/i.test(name)); - if (referenceImages.length > 0) { - await NodeFSP.mkdir(NodePath.join(pack, "references"), { recursive: true }); - for (const name of referenceImages) { - await NodeFSP.copyFile( - NodePath.join(input.referencesDirectory, name), - NodePath.join(pack, "references", name), - ); - } + for (const name of referenceImages) { + await writeImage( + NodePath.join(input.referencesDirectory, name), + NodePath.join(pack, "references", name), + ); } // Reviewers score only the device classes this run captured.