From b48a228e8e7e9c0d6ccc6123a90594fec0fdc373 Mon Sep 17 00:00:00 2001 From: Brent Vatne Date: Sat, 24 Oct 2026 18:05:44 -0700 Subject: [PATCH] Viewer: a Flows view for compare:flows --report, each flow a filmstrip of its checkpoints compare:flows --report DIR writes a report that opens on the Viewer's new Flows view: the after run's report with the comparison beside it (exploration.json flowReview) and both builds' checkpoint screenshots. Each checkpoint carries its verdict, changed regions, and the text its screen showed (labels and values from the run's accessibility artifacts). A flows run now records each flow's steps (actions, described steps, checkpoints) and the step it failed at, so the review can show the steps that led to a checkpoint. The Flows view lists the flows with what happened to each (Changed / All, Changed keeping every run that stopped), and shows one flow as a filmstrip of its checkpoints, opened at its first problem. Above the middle checkpoint are the steps that led to it, its name and status, and, for a change too small to read in the picture, the text that changed, or why the run stopped. Before | After (X) flips every picture in place; the arrows step the checkpoints, J and K the changed areas. Nothing is saved. The filmstrip is Screens' own, moved to components/shared/filmstrip.tsx; Screens wraps it. Flows takes P and Ctrl+4, and appears only in a report with a review. --- README.md | 8 + skills/explore-binary/SKILL.md | 2 +- src/binary/flows.ts | 11 + src/binary/run.ts | 6 + src/binary/strategy.ts | 5 +- src/cli/args.ts | 2 +- src/cli/binary-commands.ts | 11 +- src/cli/help.ts | 5 +- src/contracts/flow-comparison.ts | 30 +- src/contracts/standalone-report.ts | 10 + src/review/flow-comparison.ts | 14 +- src/review/flow-review.ts | 83 +++++ test/node/binary-flows.test.mjs | 13 + test/node/canvas-view-transition.test.mjs | 3 +- test/node/flow-comparison.test.mjs | 55 +++- test/node/flows-model.test.mjs | 82 +++++ test/node/navigation.test.mjs | 10 +- viewer/src/app.tsx | 26 +- viewer/src/components/canvas/toolbar.tsx | 8 +- .../components/compare/shortcuts-dialog.tsx | 19 +- viewer/src/components/phone/phone-chrome.tsx | 8 +- viewer/src/components/shared/filmstrip.tsx | 285 ++++++++++++++++ viewer/src/components/views/flows.css | 113 +++++++ viewer/src/components/views/flows.tsx | 266 +++++++++++++++ .../components/views/screens-filmstrip.tsx | 310 +++--------------- viewer/src/utils/canvas-view-transition.ts | 2 +- viewer/src/utils/commands.ts | 6 +- viewer/src/utils/embedded-report.ts | 5 +- viewer/src/utils/flows-model.ts | 156 +++++++++ viewer/src/utils/navigation.ts | 15 +- viewer/src/utils/shortcuts.ts | 15 +- 31 files changed, 1267 insertions(+), 317 deletions(-) create mode 100644 src/review/flow-review.ts create mode 100644 test/node/flows-model.test.mjs create mode 100644 viewer/src/components/shared/filmstrip.tsx create mode 100644 viewer/src/components/views/flows.css create mode 100644 viewer/src/components/views/flows.tsx create mode 100644 viewer/src/utils/flows-model.ts diff --git a/README.md b/README.md index 531c873..8202ad7 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,14 @@ npx @expo/explore compare:flows --before explore-app-1 --after explore-app-2 --o ``` Each checkpoint comes back `unchanged`, `changed` (with where the screenshots differ, the status bar and home indicator set aside, as in a review), `missing` (no picture on one side, with why), `different-size` (another device or orientation, so not compared), or `added` / `removed` (in only one run's flows file). The result also lists how the two runs were captured differently (the app, device, OS version, locale, orientation, or appearance), since a change can come from those. + +Add `--report DIRECTORY` to write a report that opens on the Viewer's Flows view, and host it with `report:upload DIRECTORY`. It is the after run's report with the comparison beside it and both builds' checkpoint screenshots: + +- The list shows each flow with what happened to it ("Stopped · 2 not reached", "3 changed"). Changed keeps every flow with a change and every run that stopped. +- A flow is a filmstrip of its checkpoints, as in Screens, and opens at its first problem. Above the checkpoint in the middle are the steps that led to it, and, for a change too small to read in the picture, the text that changed. ← and → step through the checkpoints, J and K through the changed areas of one. +- Before | After (X) flips the pictures to the other build in place. + +A run records each flow's steps for the review; runs made before Explore recorded them show none. - A `{{secret:NAME}}` or a described step limited by `platforms` to another platform needs no value, and no model, on this one. diff --git a/skills/explore-binary/SKILL.md b/skills/explore-binary/SKILL.md index 1572fab..aee4172 100644 --- a/skills/explore-binary/SKILL.md +++ b/skills/explore-binary/SKILL.md @@ -42,7 +42,7 @@ To compare runs, or an iPhone with an Android phone, screen for screen, pass `-- When the user describes a flow, or a set of flows, to look at ("sign in, then rename a pattern"), write them as a flows file and pass `--flows flows.json`, not `--replay`. The format is in the README ("Follow named flows, step by step"): `{ "flows": [{ "name": "Sign in", "steps": [{ "checkpoint": "Welcome" }, { "do": "Sign in with {{secret:TEST_EMAIL}}" }, { "checkpoint": "Home" }] }] }`. - Each step is an exact action (the `--replay` format), a step described in words (`do`) that a model carries out, or a named `checkpoint`. Use exact actions where you know the labels, and `do` where you don't. - Each flow starts from a relaunched app, and its checkpoints are photographed under their names. `report/exploration.json` lists each flow under `exploration.flows`, with its status and its checkpoints in order, so two runs pair up by flow and checkpoint name. A checkpoint on a screen that withholds its picture (a secret on it) is listed with why it has none, and its flow goes on. -- To compare two runs of the same flows file, run `npx @expo/explore compare:flows --before DIR --after DIR` with the two output directories. Each checkpoint is `unchanged`, `changed` (with regions), `missing`, `different-size`, `added` or `removed`, and `conditions` lists what differed between the runs (device, OS, locale, appearance). +- To compare two runs of the same flows file, run `npx @expo/explore compare:flows --before DIR --after DIR` with the two output directories. Each checkpoint is `unchanged`, `changed` (with regions), `missing`, `different-size`, `added` or `removed`, and `conditions` lists what differed between the runs (device, OS, locale, appearance). Add `--report DIR` to write a report that opens on the Viewer's Flows view (each flow's checkpoints, the steps between them and the text that changed), and host it with `report:upload DIR`. - The file is checked before a device starts (`FLOWS_PLAN_INVALID`, `FLOWS_PLAN_EMPTY`, `FLOWS_PLAN_OVER_BUDGET`). `--provider`, `--model` and `--hints` steer the model that carries out `do` steps, and are refused when there are none (`FLOWS_OPTIONS_IGNORED`). `--flows` and `--replay` cannot be combined. ## Sign-in and other app state diff --git a/src/binary/flows.ts b/src/binary/flows.ts index 50a67e6..34f922f 100644 --- a/src/binary/flows.ts +++ b/src/binary/flows.ts @@ -3,6 +3,7 @@ import { CiError } from "../shared/ci-errors.js"; import type { CiAgentSession, CiAgentSessionRunRequest, CiAgentSessionRunResult, CiAgentToolResult } from "../capture/agent/session.js"; import type { CiAgentTool } from "../capture/agent/provider.js"; import type { CaptureSetupAction } from "../config/explore-config.js"; +import type { ExplorationFlowStep } from "../contracts/standalone-report.js"; import { explorationSecretNames } from "../config/exploration-inputs.js"; import { TOOL_LIMITS } from "./explorer.js"; import { actInput, describeAction, listed, parseReplayStep, type ReplayAction, type ReplayPlatform } from "./replay.js"; @@ -22,6 +23,8 @@ export type FlowSession = { budgetNote(): string | undefined; /** Each flow's checkpoints the run has not attempted, in order: the explorer records them as missing. */ notReached(): Array<{ flow: string; checkpoints: string[] }>; + /** Each flow's steps on the platform, and the one it failed at, for the report. */ + steps(): Array<{ flow: string; steps: ExplorationFlowStep[]; failedStep?: number }>; }; const MAX_FLOWS = 50; @@ -228,11 +231,16 @@ export function createFlowSession(plan: FlowPlan, platform: ReplayPlatform, opti const checkpoints = steps.flatMap(({ step }) => "checkpoint" in step && !attempted.get(flow.name)?.has(step.checkpoint) ? [step.checkpoint] : []); return checkpoints.length ? [{ flow: flow.name, checkpoints }] : []; }); + /** The step each failed flow stopped at, counted from 1 among its steps on the platform. */ + const failedAt = new Map(); return { provider: "flows", model: "flows file", async preflight() {}, notReached, + steps: () => flows.map(({ flow, steps }) => ({ flow: flow.name, + steps: steps.map(({ step }): ExplorationFlowStep => "checkpoint" in step ? { checkpoint: step.checkpoint } : "do" in step ? { do: step.do } : { action: describeAction(step) }), + ...(failedAt.has(flow.name) ? { failedStep: failedAt.get(flow.name)! } : {}) })), budgetNote() { const names = notReached().flatMap(({ flow, checkpoints }) => checkpoints.map(name => `${flow} › ${name}`)); return names.length ? `Not reached: ${listed(names, TOOL_LIMITS.reason, "checkpoints")}.` : undefined; @@ -273,6 +281,7 @@ export function createFlowSession(plan: FlowPlan, platform: ReplayPlatform, opti continue; } failure = `Checkpoint "${step.checkpoint}" could not be taken: ${why(taken.value)}`; + failedAt.set(flow.name, index + 1); break; } if ("do" in step) { @@ -284,12 +293,14 @@ export function createFlowSession(plan: FlowPlan, platform: ReplayPlatform, opti usage.inputTokens += goal.usage.inputTokens; usage.outputTokens += goal.usage.outputTokens; if (goal.reached) continue; failure = `Step ${number} ("${step.do.slice(0, 120)}") was not done: ${goal.reason}`; + failedAt.set(flow.name, index + 1); break; } const acted = await call("act", actInput(step)); if (!acted.failed) continue; if (step.optional) { skipped.push(`${flow.name} step ${number} (${describeAction(step)})`); continue; } failure = `Step ${number} (${describeAction(step)}) failed: ${why(acted.value)}`; + failedAt.set(flow.name, index + 1); break; } await call("end_flow", failure ? { status: "failed", reason: failure.slice(0, TOOL_LIMITS.reason) } : { status: "completed" }); diff --git a/src/binary/run.ts b/src/binary/run.ts index 5cac424..3ba7e15 100644 --- a/src/binary/run.ts +++ b/src/binary/run.ts @@ -470,6 +470,12 @@ export async function exploreConnectedSession(ctx: ConnectedSessionExploration): throw new Error(failureText(error)); } } }); + for (const planned of strategy.flowSteps?.() ?? []) { + const record = exploration.flows?.find(flow => flow.name.toLowerCase() === planned.flow.toLowerCase()); + if (!record) continue; + record.steps = planned.steps; + if (planned.failedStep !== undefined && record.status === "failed") record.failedStep = planned.failedStep; + } // "complete" is reserved for a run with no pending frontier or open gaps; a model that finishes early is "partial". // The map schema allows stopReason only on partial/failed runs. map.run.status = exploration.stopReason === "model-error" ? "failed" diff --git a/src/binary/strategy.ts b/src/binary/strategy.ts index 0db9523..4e8689c 100644 --- a/src/binary/strategy.ts +++ b/src/binary/strategy.ts @@ -2,6 +2,7 @@ import { createCiAgentSession, type CiAgentSession } from "../capture/agent/sess import { createLocalCiAgentSession } from "../capture/agent/session-local.js"; import type { CiAgentProviderName } from "../capture/agent/provider.js"; import { createExplorationInputs } from "../config/exploration-inputs.js"; +import type { ExplorationFlowStep } from "../contracts/standalone-report.js"; import { CiError } from "../shared/ci-errors.js"; import type { Platform } from "../types.js"; import { assertAppUrl } from "./app-url.js"; @@ -33,6 +34,8 @@ export type ExplorationStrategy = { * and names each flow's checkpoints it never attempted, for the explorer to record as missing. */ flows?: () => Array<{ flow: string; checkpoints: string[] }>; + /** Each flow's steps and the one it failed at, which the report keeps beside the flow. */ + flowSteps?: () => Array<{ flow: string; steps: ExplorationFlowStep[]; failedStep?: number }>; }; export const DEFAULT_MODELS: Record = { anthropic: "claude-sonnet-4-5", openai: "gpt-6.1-sol" }; @@ -127,7 +130,7 @@ export async function startStrategy(choice: StrategyChoice, context: { platform: assertFlowsFit(choice.plan, platform, { ...limits, maxTurns: turnBudget(limits) }); const goals = flowsNeedModel(choice.plan, platform) ? await preflightModelSession(options, environment, say) : undefined; const session = createFlowSession(choice.plan, platform, { app, ...(goals ? { model: goals.session } : {}), ...(options.hints ? { hints: options.hints } : {}) }); - return { session, budgetNote: session.budgetNote, flows: session.notReached, provider: "flows", model: goals ? `${goals.provider} ${goals.model}` : session.model, auth: goals?.auth ?? "none" }; + return { session, budgetNote: session.budgetNote, flows: session.notReached, flowSteps: session.steps, provider: "flows", model: goals ? `${goals.provider} ${goals.model}` : session.model, auth: goals?.auth ?? "none" }; } return preflightModelSession(options, environment, say); } diff --git a/src/cli/args.ts b/src/cli/args.ts index 3625b34..8963502 100644 --- a/src/cli/args.ts +++ b/src/cli/args.ts @@ -105,7 +105,7 @@ const flagKinds: Record> = { "ci:find-baseline": { sha:"value", target:"value", platform:"value" }, "ci:compare-refs": { project:"value", base:"value", head:"value", title:"value", output:"value" }, "ci:after-capture": { output:"value", checkout:"value" }, - "compare:flows": { before: "value", after: "value", output: "value" }, + "compare:flows": { before: "value", after: "value", output: "value", report: "value" }, compare: { baseline: "value", current: "value", diff --git a/src/cli/binary-commands.ts b/src/cli/binary-commands.ts index ebf18c8..11bdf96 100644 --- a/src/cli/binary-commands.ts +++ b/src/cli/binary-commands.ts @@ -11,6 +11,7 @@ import { readReplayPlan } from "../binary/replay.js"; import { readFlowPlan } from "../binary/flows.js"; import { EXPLORATION_REPORT_KIND, type ExplorationFlow, type ExplorationReport } from "../contracts/standalone-report.js"; import { compareFlowRuns, type FlowRun } from "../review/flow-comparison.js"; +import { writeFlowReviewReport } from "../review/flow-review.js"; import { diffScreenshots } from "../review/visual-evidence.js"; import { agentDeviceLauncher, ensureController } from "../shared/controller-cache.js"; import { CiError } from "../shared/ci-errors.js"; @@ -177,7 +178,10 @@ async function flowRun(path: string, name: "before" | "after"): Promise return refuse(`No exploration.json was found in ${resolve(path)} or its report folder.`); } -/** `compare:flows --before DIR --after DIR [--output FILE]`: two runs of one flows file, checkpoint by checkpoint. */ +/** + * `compare:flows --before DIR --after DIR [--output FILE] [--report DIR]`: two runs of one flows file, checkpoint by + * checkpoint. `--report` writes a report the Viewer opens on its Flows view. + */ export async function compareFlows(): Promise { const [before, after] = await Promise.all([flowRun(required("before"), "before"), flowRun(required("after"), "after")]); let command: string | undefined; @@ -185,7 +189,8 @@ export async function compareFlows(): Promise { if (!command) { await ensureController("agent-device", { environment: process.env }); command = agentDeviceLauncher(process.env); } return diffScreenshots({ baselinePath: beforePath, currentPath: afterPath, command }); }); - const path = flag("output"); + const path = flag("output"), reportPath = flag("report"); if (path) await writeFile(resolve(path), JSON.stringify(comparison, null, 2)); - output({ ok: true, ...(path ? { output: resolve(path) } : {}), comparison }); + const report = reportPath ? await writeFlowReviewReport(comparison, before, after, reportPath) : undefined; + output({ ok: true, ...(path ? { output: resolve(path) } : {}), ...(report ? { report } : {}), comparison }); } diff --git a/src/cli/help.ts b/src/cli/help.ts index c115206..f55f6e8 100644 --- a/src/cli/help.ts +++ b/src/cli/help.ts @@ -84,8 +84,9 @@ const CI = `CI (the installed workflows run these; they manage remote EAS Simula capture:baseline --project DIRECTORY --sha COMMIT --output TEMP_DIRECTORY [--config FILE] [--target ID | --platform ios|android] [--controller agent-device|argent] [--simulator eas|local] [--repack eas|local] source:verify --base-project DIRECTORY --head-project DIRECTORY --base-sha COMMIT --head-sha COMMIT --output DIRECTORY [--config FILE] [--scope all|changes] [--target ID | --platform ios|android] - compare:flows --before DIRECTORY --after DIRECTORY [--output FILE] - (two runs of one --flows file, checkpoint by checkpoint: unchanged, changed with where, or why not compared) + compare:flows --before DIRECTORY --after DIRECTORY [--output FILE] [--report DIRECTORY] + (two runs of one --flows file, checkpoint by checkpoint: unchanged, changed with where, or why not compared; + --report writes a report the Viewer opens on its Flows view) compare --baseline DIRECTORY --current DIRECTORY --output FILE [--review-output FILE] [--catalog-output FILE] [--catalog-name NAME] [--base-surface-coverage FILE] [--head-surface-coverage FILE] diff --git a/src/contracts/flow-comparison.ts b/src/contracts/flow-comparison.ts index 252e561..d981aab 100644 --- a/src/contracts/flow-comparison.ts +++ b/src/contracts/flow-comparison.ts @@ -1,7 +1,35 @@ -import type { ExplorationReport } from "./standalone-report.js"; +import type { ExplorationFlow, ExplorationFlowStep, ExplorationReport } from "./standalone-report.js"; export const FLOW_COMPARISON_KIND = "explore.flow-comparison" as const; +/** + * `changed` and `unchanged` compare the two screenshots; `different-size` could not (another device or orientation); + * `missing` has no screenshot on one side; `added` and `removed` are in only one run's flows file. + */ +export type CheckpointVerdict = "unchanged" | "changed" | "different-size" | "missing" | "added" | "removed"; +/** Where two screenshots differ, in percent of the image (0–100). */ +export type ChangedRegion = { x: number; y: number; width: number; height: number }; + +/** One build's side of a checkpoint in a flow review: its screenshot (an asset of the review's report) and the text its screen showed, or why it has none. */ +export type FlowReviewSide = { screenshot: string; text?: string[] } | { missing: string }; +export type FlowReviewCheckpoint = { name: string; verdict: CheckpointVerdict; before?: FlowReviewSide; after?: FlowReviewSide; regions?: ChangedRegion[] }; +/** How one build's run took the flow. */ +export type FlowReviewRun = Pick & { steps?: ExplorationFlowStep[] }; +export type FlowReviewFlow = { name: string; before?: FlowReviewRun; after?: FlowReviewRun; checkpoints: FlowReviewCheckpoint[] }; + +/** + * Two runs of one flows file, compared checkpoint by checkpoint, as the Viewer shows them: carried in the after run's + * exploration report, whose assets hold both builds' checkpoint screenshots. + */ +export type FlowReview = { + before: RunConditions; + after: RunConditions; + /** How the runs were captured differently; a checkpoint that changed may owe its change to these. */ + conditions: RunConditionDifference[]; + notes: string[]; + flows: FlowReviewFlow[]; +}; + /** One run's side of a checkpoint: its screenshot (a path inside that run's report directory), or why it has none. */ export type CheckpointSide = { observationId: string; screenshot: string } | { missing: string }; diff --git a/src/contracts/standalone-report.ts b/src/contracts/standalone-report.ts index 6af94a3..c4dadd2 100644 --- a/src/contracts/standalone-report.ts +++ b/src/contracts/standalone-report.ts @@ -1,4 +1,5 @@ import type { PublicReview } from "./capture.js"; +import type { FlowReview } from "./flow-comparison.js"; import type { BuildWorkspacePayload, Coverage, ExploreMap } from "../types.js"; export const STANDALONE_REPORT_CONTRACT = "expo.explore.standalone-report.v1" as const; @@ -24,8 +25,15 @@ export type ExplorationFlow = { /** The map's journeys for the flow: its walk from the screen it started on, in parts where the app moved without a transition. */ journeyIds?: string[]; checkpoints: Array<{ name: string; observationId: string; missing?: never } | { name: string; missing: string; observationId?: never }>; + /** The flow's steps on this run's platform, as the flows file names them. */ + steps?: ExplorationFlowStep[]; + /** The step the flow failed at, counted from 1 in `steps`. */ + failedStep?: number; }; +/** An exact action (`tap "Settings"`), a step described in words for a model, or a checkpoint. */ +export type ExplorationFlowStep = { action: string } | { do: string } | { checkpoint: string }; + /** What `explore:binary` learned about one prebuilt app: provenance plus the map it explored, with no review or pull request. */ export type ExplorationReport = { kind: typeof EXPLORATION_REPORT_KIND; @@ -47,6 +55,8 @@ export type ExplorationReport = { exploreVersion: string; }; assets: Array<{ sha256: string; path: string }>; + /** Present when compare:flows wrote this report: the run compared with an earlier run of the same flows file. */ + flowReview?: FlowReview; }; export type EmbeddedReport = { diff --git a/src/review/flow-comparison.ts b/src/review/flow-comparison.ts index 5135e50..267433e 100644 --- a/src/review/flow-comparison.ts +++ b/src/review/flow-comparison.ts @@ -1,22 +1,20 @@ import { open } from "node:fs/promises"; import { resolve } from "node:path"; import { isFoldable, systemRegions } from "../contracts/capture.js"; -import { FLOW_COMPARISON_KIND, conditionDifferences, pairCheckpoints, runConditions, type CheckpointPair, type RunConditionDifference, type RunConditions } from "../contracts/flow-comparison.js"; +import { + FLOW_COMPARISON_KIND, conditionDifferences, pairCheckpoints, runConditions, + type ChangedRegion, type CheckpointPair, type CheckpointVerdict, type RunConditionDifference, type RunConditions, +} from "../contracts/flow-comparison.js"; import type { ExplorationReport } from "../contracts/standalone-report.js"; import { ignoreSystemRegions, systemBandFrameLoader } from "./evidence-assessment.js"; import type { ScreenshotDiffMeasurement } from "./visual-evidence.js"; -/** - * `changed` and `unchanged` compare the two screenshots; `different-size` could not (another device or orientation); - * `missing` has no screenshot on one side; `added` and `removed` are in only one run's flows file. - */ -export type CheckpointVerdict = "unchanged" | "changed" | "different-size" | "missing" | "added" | "removed"; +export type { CheckpointVerdict } from "../contracts/flow-comparison.js"; export type ComparedCheckpoint = CheckpointPair & { verdict: CheckpointVerdict; /** Share of the screenshot that differs once the status bar and home indicator are set aside. */ mismatchPercentage?: number; - /** Where the screenshots differ, in percent of the image (0–100). */ - regions?: Array<{ x: number; y: number; width: number; height: number }>; + regions?: ChangedRegion[]; }; export type FlowComparison = { diff --git a/src/review/flow-review.ts b/src/review/flow-review.ts new file mode 100644 index 0000000..956fad2 --- /dev/null +++ b/src/review/flow-review.ts @@ -0,0 +1,83 @@ +import { copyFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import { basename, dirname, join, resolve } from "node:path"; +import type { CheckpointSide, FlowReview, FlowReviewRun, FlowReviewSide } from "../contracts/flow-comparison.js"; +import type { ExplorationFlow, ExplorationReport } from "../contracts/standalone-report.js"; +import type { FlowComparison, FlowRun } from "./flow-comparison.js"; + +const keyOf = (name: string) => name.trim().toLowerCase(); + +/** + * The text a screen showed, in tree order and once each: every label and value its accessibility tree kept. A run's + * artifacts sit beside its report directory; a report copied on its own has none, and its checkpoints carry no text. + */ +async function screenText(run: FlowRun, observationId: string): Promise { + const observation = run.report.map.observations.find(item => item.id === observationId); + const artifact = observation?.accessibilityArtifactId ? run.report.map.artifacts.find(item => item.id === observation.accessibilityArtifactId) : undefined; + if (!artifact) return undefined; + const roots = basename(run.directory) === "report" ? [dirname(run.directory), run.directory] : [run.directory]; + for (const root of roots) { + let tree: { nodes?: Array<{ label?: unknown; value?: unknown }> }; + try { tree = JSON.parse(await readFile(resolve(root, artifact.path), "utf8")) as typeof tree; } catch { continue; } + const text = new Set(); + for (const node of tree.nodes ?? []) for (const value of [node.label, node.value]) { + if (typeof value === "string" && value.trim()) text.add(value.trim()); + } + return [...text]; + } + return undefined; +} + +async function reviewSide(run: FlowRun, side: CheckpointSide | undefined): Promise { + if (!side) return undefined; + if ("missing" in side) return { missing: side.missing }; + const text = await screenText(run, side.observationId); + return { screenshot: side.screenshot, ...(text ? { text } : {}) }; +} + +function reviewRun(report: ExplorationReport, name: string): FlowReviewRun | undefined { + const flow: ExplorationFlow | undefined = report.exploration.flows?.find(item => keyOf(item.name) === keyOf(name)); + if (!flow) return undefined; + return { status: flow.status, ...(flow.reason ? { reason: flow.reason } : {}), ...(flow.steps ? { steps: flow.steps } : {}), + ...(flow.failedStep !== undefined ? { failedStep: flow.failedStep } : {}) }; +} + +/** The comparison as the Viewer reads it: flows in the comparison's order, each with both runs' outcome and steps. */ +export async function flowReview(comparison: FlowComparison, before: FlowRun, after: FlowRun): Promise { + const flows: FlowReview["flows"] = []; + for (const item of comparison.checkpoints) { + let flow = flows.at(-1); + if (!flow || flow.name !== item.flow) { + const was = reviewRun(before.report, item.flow), now = reviewRun(after.report, item.flow); + flows.push(flow = { name: item.flow, ...(was ? { before: was } : {}), ...(now ? { after: now } : {}), checkpoints: [] }); + } + const [was, now] = await Promise.all([reviewSide(before, item.before), reviewSide(after, item.after)]); + flow.checkpoints.push({ name: item.checkpoint, verdict: item.verdict, ...(was ? { before: was } : {}), ...(now ? { after: now } : {}), + ...(item.regions ? { regions: item.regions } : {}) }); + } + const { report: _before, ...was } = comparison.before, { report: _after, ...now } = comparison.after; + return { before: was, after: now, conditions: comparison.conditions, notes: comparison.notes, flows }; +} + +/** + * Writes a report the Viewer opens: the after run's exploration report with the review beside it, and every asset it + * names, the before run's checkpoint screenshots included. Assets are named by their SHA-256, so the two runs share one + * folder without clashing. + */ +export async function writeFlowReviewReport(comparison: FlowComparison, before: FlowRun, after: FlowRun, directory: string): Promise { + const target = resolve(directory); + if ([before.directory, after.directory].some(source => resolve(source) === target)) throw new Error("The review report needs its own folder, not one of the runs' reports."); + const review = await flowReview(comparison, before, after); + await mkdir(join(target, "assets"), { recursive: true }); + const assets = [...after.report.assets]; + for (const asset of after.report.assets) await copyFile(resolve(after.directory, asset.path), join(target, asset.path)); + const known = new Set(assets.map(asset => asset.path)); + for (const asset of before.report.assets) { + const used = comparison.checkpoints.some(item => item.before && "screenshot" in item.before && item.before.screenshot === asset.path); + if (!used || known.has(asset.path)) continue; + await copyFile(resolve(before.directory, asset.path), join(target, asset.path)); + assets.push(asset); known.add(asset.path); + } + const report: ExplorationReport = { ...after.report, assets, flowReview: review }; + await writeFile(join(target, "exploration.json"), JSON.stringify(report)); + return target; +} diff --git a/test/node/binary-flows.test.mjs b/test/node/binary-flows.test.mjs index fce8f86..ca7e946 100644 --- a/test/node/binary-flows.test.mjs +++ b/test/node/binary-flows.test.mjs @@ -113,6 +113,19 @@ test("a failed step ends its flow with the reason and the checkpoints it did not assert.deepEqual(await validateMap(map, store), []); }); +test("the session records each flow's steps as the flows file names them, and the step a failed flow stopped at", async () => { + const plan = parseFlowPlan({ flows: [ + { name: "Broken", steps: [{ checkpoint: "Home" }, { action: "tap", label: "Nowhere" }, { checkpoint: "Never" }] }, + { name: "Settings", steps: [{ action: "tap", label: "Settings" }, { checkpoint: "Settings" }] }, + ] }); + const session = createFlowSession(plan, "ios", { app: { name: "Example" } }); + await explore(plan, { session }); + assert.deepEqual(session.steps(), [ + { flow: "Broken", steps: [{ checkpoint: "Home" }, { action: 'tap "Nowhere"' }, { checkpoint: "Never" }], failedStep: 2 }, + { flow: "Settings", steps: [{ action: 'tap "Settings"' }, { checkpoint: "Settings" }] }, + ]); +}); + test("a destructive tap in a flow is refused by the explorer, which fails the flow", async () => { const { result, acted } = await explore(parseFlowPlan({ flows: [{ name: "Danger", steps: [{ action: "tap", label: "Delete account" }, { checkpoint: "After" }] }] })); assert.ok(!acted.some(action => action.label === "Delete account")); diff --git a/test/node/canvas-view-transition.test.mjs b/test/node/canvas-view-transition.test.mjs index c7fd09f..a70bd79 100644 --- a/test/node/canvas-view-transition.test.mjs +++ b/test/node/canvas-view-transition.test.mjs @@ -8,7 +8,8 @@ const transition = await tsImport( ); test("canvas view direction follows the toolbar order", () => { - assert.deepEqual(transition.CANVAS_VIEW_ORDER, ["map", "screens", "compare"]); + assert.deepEqual(transition.CANVAS_VIEW_ORDER, ["map", "screens", "compare", "flows"]); + assert.equal(transition.canvasViewTransitionDirection("compare", "flows"), "forward"); assert.equal(transition.canvasViewTransitionDirection("screens", "compare"), "forward"); assert.equal(transition.canvasViewTransitionDirection("map", "screens"), "forward"); assert.equal(transition.canvasViewTransitionDirection("compare", "screens"), "backward"); diff --git a/test/node/flow-comparison.test.mjs b/test/node/flow-comparison.test.mjs index 9ee7750..d7652cd 100644 --- a/test/node/flow-comparison.test.mjs +++ b/test/node/flow-comparison.test.mjs @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; -import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { access, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; import { createRequire } from "node:module"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -8,6 +9,7 @@ import { tsImport } from "tsx/esm/api"; const { pairCheckpoints, conditionDifferences, runConditions } = await tsImport("../../src/contracts/flow-comparison.ts", import.meta.url); const { compareFlowRuns } = await tsImport("../../src/review/flow-comparison.ts", import.meta.url); +const { writeFlowReviewReport } = await tsImport("../../src/review/flow-review.ts", import.meta.url); const { PNG } = createRequire(import.meta.url)("pngjs"); const png = (width, height) => PNG.sync.write(new PNG({ width, height })); @@ -24,14 +26,25 @@ async function run(flows, options = {}) { exploration: { app: { name: "Example", bundleId: options.bundleId ?? "dev.example.app", platform: options.platform ?? "ios" }, device: { model: options.device ?? "iPhone 17 Pro", osVersion: options.osVersion ?? "26.0" }, flows: [] } }; let index = 0; for (const [name, checkpoints] of Object.entries(flows)) { - const flow = { name, status: "completed", checkpoints: [] }; + const flow = { name, status: "completed", checkpoints: [], ...options.flow?.[name] }; for (const [checkpoint, shot] of checkpoints) { if (shot.missing) { flow.checkpoints.push({ name: checkpoint, missing: shot.missing }); continue; } - const sha = String(++index).padStart(64, "0"), id = `obs_${index}`; + // Named by a digest, as a run names its assets: two runs never share a name for different pictures. + const sha = createHash("sha256").update(`${directory}:${++index}`).digest("hex"), id = `obs_${index}`; await writeFile(join(directory, "assets", `${sha}.png`), png(...shot)); report.assets.push({ sha256: sha, path: `assets/${sha}.png` }); report.map.artifacts.push({ id: `art_${index}`, kind: "screenshot", sha256: sha }); - report.map.observations.push({ id, screenshotArtifactId: `art_${index}` }); + const observation = { id, screenshotArtifactId: `art_${index}` }; + const text = options.text?.[checkpoint]; + if (text) { + // The accessibility tree a run keeps beside its report, as the explorer writes it. + const path = `artifacts/accessibility/${index}.json`; + await mkdir(join(directory, "artifacts", "accessibility"), { recursive: true }); + await writeFile(join(directory, path), JSON.stringify({ nodes: text.map(label => ({ label })) })); + report.map.artifacts.push({ id: `acc_${index}`, kind: "accessibility", path }); + observation.accessibilityArtifactId = `acc_${index}`; + } + report.map.observations.push(observation); flow.checkpoints.push({ name: checkpoint, observationId: id }); } report.exploration.flows.push(flow); @@ -52,7 +65,7 @@ test("checkpoints pair by flow and checkpoint name, ignoring case, in the after assert.deepEqual(pairs.map(pair => [pair.flow, pair.checkpoint, !!pair.before, !!pair.after]), [ ["sign in", "home", true, true], ["sign in", "welcome", true, true], ["Profile", "Profile", false, true], ["Settings", "Settings", true, false]]); assert.deepEqual(pairs[1].after, { missing: "The screenshot was blocked." }); - assert.match(pairs[0].before.screenshot, /^assets\/0+2\.png$/); + assert.equal(pairs[0].before.screenshot, before.report.assets[1].path, "Home is the before run's second picture"); }); test("a run's conditions that differ are listed, not refused", async () => { @@ -113,3 +126,35 @@ test("compare:flows reads two output directories, needs no diff when no pair has assert.notEqual(refused.status, 0); assert.match(JSON.parse(refused.stderr).error, /did not follow a flows file/); }); + +test("a review report is the after run's report with the review beside it: both builds' pictures and text, each flow's outcome and steps", async () => { + const steps = [{ checkpoint: "Home" }, { action: 'tap "Continue"' }, { checkpoint: "Done" }]; + const before = await run({ Flow: [["Home", [4, 8]], ["Done", [4, 8]]] }, { text: { Home: ["Welcome", "Sign in"], Done: ["All set"] }, flow: { Flow: { steps } } }); + const after = await run({ Flow: [["Home", [4, 8]], ["Done", { missing: "The flow did not reach it." }]] }, + { text: { Home: ["Welcome", "Log in"] }, flow: { Flow: { status: "failed", reason: "Step 2 (tap \"Continue\") failed: no such element", steps, failedStep: 2 } } }); + const comparison = await compareFlowRuns(before, after, measuring(() => [{ x: 10, y: 40, width: 20, height: 10 }])); + const target = await mkdtemp(join(tmpdir(), "explore-flow-review-")); + await writeFlowReviewReport(comparison, before, after, target); + const written = JSON.parse(await readFile(join(target, "exploration.json"), "utf8")); + assert.equal(written.kind, "explore.exploration-report"); + assert.deepEqual(written.exploration, after.report.exploration, "the after run's report is kept as it was"); + const [flow] = written.flowReview.flows; + assert.deepEqual(flow.after, { status: "failed", reason: 'Step 2 (tap "Continue") failed: no such element', steps, failedStep: 2 }); + assert.deepEqual(flow.before, { status: "completed", steps }); + assert.deepEqual(flow.checkpoints.map(item => [item.name, item.verdict]), [["Home", "changed"], ["Done", "missing"]]); + assert.deepEqual(flow.checkpoints[0].before.text, ["Welcome", "Sign in"]); + assert.deepEqual(flow.checkpoints[0].after.text, ["Welcome", "Log in"]); + assert.deepEqual(flow.checkpoints[0].regions, [{ x: 10, y: 40, width: 20, height: 10 }]); + assert.deepEqual(flow.checkpoints[1].after, { missing: "The flow did not reach it." }); + // Every picture the review names is in the report's folder, the before run's included. + for (const checkpoint of flow.checkpoints) for (const side of [checkpoint.before, checkpoint.after]) { + if (side?.screenshot) await access(join(target, side.screenshot)); + } + assert.equal(written.assets.length, 3); +}); + +test("a review report is not written over one of the runs' own reports", async () => { + const before = await run({ Flow: [["Home", [4, 8]]] }), after = await run({ Flow: [["Home", [4, 8]]] }); + const comparison = await compareFlowRuns(before, after, measuring()); + await assert.rejects(writeFlowReviewReport(comparison, before, after, after.directory), /its own folder/); +}); diff --git a/test/node/flows-model.test.mjs b/test/node/flows-model.test.mjs new file mode 100644 index 0000000..54b7858 --- /dev/null +++ b/test/node/flows-model.test.mjs @@ -0,0 +1,82 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { tsImport } from "tsx/esm/api"; + +const model = await tsImport("../../viewer/src/utils/flows-model.ts", import.meta.url); + +const shot = (sha, text) => ({ screenshot: `assets/${sha.padStart(64, "0")}.png`, ...(text ? { text } : {}) }); +const steps = [{ action: 'tap "Settings"' }, { checkpoint: "Settings" }, { do: "Open the output picker" }, { checkpoint: "Output" }, { do: "Close it" }, { do: "Open the input picker" }, { checkpoint: "Input" }]; + +const completed = { name: "Connect", before: { status: "completed", steps }, after: { status: "completed", steps }, checkpoints: [ + { name: "Settings", verdict: "unchanged", before: shot("1"), after: shot("2") }, + { name: "Output", verdict: "changed", before: shot("3", ["Output", "No outputs over USB-C."]), after: shot("4", ["Output", "No outputs over USB-C or Bluetooth."]), regions: [{ x: 1, y: 2, width: 3, height: 4 }] }, + { name: "Input", verdict: "unchanged", before: shot("5"), after: shot("6") }, +] }; +const stopped = { name: "Share", before: { status: "completed", steps }, after: { status: "failed", reason: "The model used its 40 tool calls without finishing the step.", steps, failedStep: 3 }, checkpoints: [ + { name: "Settings", verdict: "changed", before: shot("7"), after: shot("8") }, + { name: "Output", verdict: "missing", before: shot("9"), after: { missing: "The flow did not reach it." } }, + { name: "Input", verdict: "missing", before: shot("a"), after: { missing: "The flow did not reach it." } }, +] }; +const quiet = { name: "Edit", before: { status: "completed" }, after: { status: "completed" }, checkpoints: [{ name: "Home", verdict: "unchanged", before: shot("b"), after: shot("c") }] }; +const review = { before: {}, after: {}, conditions: [], notes: [], flows: [completed, stopped, quiet] }; + +test("Changed keeps every flow with a checkpoint that is not unchanged, and every run that stopped; All keeps every flow", () => { + assert.deepEqual(model.visibleFlows(review, "changed").map(flow => flow.name), ["Connect", "Share"]); + assert.deepEqual(model.visibleFlows(review, "all").map(flow => flow.name), ["Connect", "Share", "Edit"]); + const stoppedWithoutChanges = { ...quiet, after: { status: "stopped" } }; + assert.equal(model.flowNeedsReview(stoppedWithoutChanges), true, "a stopped run is never hidden by the filter"); +}); + +test("a flow's mark and status line say whether it stopped, then how many checkpoints of each kind", () => { + assert.equal(model.flowMark(completed), "changed"); + assert.equal(model.flowStatusLine(completed), "1 changed"); + assert.equal(model.flowMark(stopped), "stopped"); + assert.equal(model.flowStatusLine(stopped), "Stopped · 2 not reached · 1 changed"); + assert.equal(model.flowMark(quiet), "unchanged"); + assert.equal(model.flowStatusLine(quiet), "Unchanged"); + assert.equal(model.flowMark({ ...quiet, before: undefined }), "added"); + assert.equal(model.flowMark({ ...quiet, after: undefined }), "removed"); +}); + +test("a flow opens at its first problem", () => { + assert.equal(model.firstProblem(completed), 1); + assert.deepEqual(model.problemIndexes(stopped), [0, 1, 2]); + assert.equal(model.firstProblem(quiet), 0, "a flow with nothing changed opens at its first checkpoint"); +}); + +test("a missing picture says which build has none", () => { + assert.equal(model.checkpointLabel(stopped.checkpoints[1]), "Not reached"); + const reachedOnlyAfter = { name: "Output", verdict: "missing", before: { missing: "The flow did not reach it." }, after: shot("d") }; + assert.equal(model.checkpointLabel(reachedOnlyAfter), "Not reached before"); + assert.equal(model.checkpointMark({ name: "Rotated", verdict: "different-size" }), "not-comparable"); +}); + +test("the steps before a checkpoint are the ones since the previous checkpoint, with the failed one and those never run marked", () => { + assert.deepEqual(model.stepsBefore(completed, 0), [{ step: { action: 'tap "Settings"' }, failed: false, ran: true }]); + assert.deepEqual(model.stepsBefore(completed, 2).map(item => item.step), [{ do: "Close it" }, { do: "Open the input picker" }]); + assert.deepEqual(model.stepsBefore(stopped, 1), [{ step: { do: "Open the output picker" }, failed: true, ran: true }]); + assert.deepEqual(model.stepsBefore(stopped, 2).map(item => item.ran), [false, false]); + assert.deepEqual(model.stepsBefore(quiet, 0), [], "a run from before steps were recorded has none to show"); +}); + +test("text changes are the lines one build's screen showed and the other's did not", () => { + assert.deepEqual(model.textChanges(completed.checkpoints[1]), { removed: ["No outputs over USB-C."], added: ["No outputs over USB-C or Bluetooth."] }); + assert.equal(model.textChanges(completed.checkpoints[0]), undefined, "without the text of both screens there is nothing to compare"); + assert.equal(model.textChanges(stopped.checkpoints[1]), undefined); +}); + +test("a picture is found by the SHA-256 its asset is named with", () => { + assert.deepEqual(model.sidePicture(shot("4")), { sha256: "4".padStart(64, "0") }); + assert.deepEqual(model.sidePicture({ missing: "Blocked." }), { missing: "Blocked." }); + assert.equal(model.sidePicture(undefined), undefined); +}); + +test("a text-sized change is spelled out in one line; a larger one is read off the picture", () => { + const changed = (before, after, height = 2) => ({ name: "Output", verdict: "changed", before: shot("3", before), after: shot("4", after), regions: [{ x: 0, y: 40, width: 50, height }] }); + assert.equal(model.textChangeLine(changed(["Output", "No outputs."], ["Output", "No outputs over USB-C."])), "“No outputs over USB-C.”, was “No outputs.”"); + assert.equal(model.textChangeLine(changed(["A"], ["A", "B", "C"])), "Adds “B” and 1 more"); + assert.equal(model.textChangeLine(changed(["A", "B"], ["A"])), "Drops “B”"); + assert.equal(model.textChangeLine(changed(["A"], ["B"], 12)), undefined, "a change taller than a line or two of text"); + assert.equal(model.textChangeLine(changed(["A"], ["A"])), undefined, "the same text, so the change is not in it"); + assert.equal(model.textChangeLine(stopped.checkpoints[0]), undefined, "without the text of both screens there is nothing to say"); +}); diff --git a/test/node/navigation.test.mjs b/test/node/navigation.test.mjs index c966991..803fc56 100644 --- a/test/node/navigation.test.mjs +++ b/test/node/navigation.test.mjs @@ -193,10 +193,18 @@ test("a screen without a route, or one the map does not know yet, travels as an assert.equal(nav.formatRoute(pending, ctx), "/screens/(tabs)/later"); }); +test("a flow review opens on Flows, and a link that names another view still opens it", () => { + const flows = { ...ctx, flows: true }; + assert.equal(nav.parseLocation({ pathname: "/", search: "", hash: "" }, flows).canvas, "flows"); + assert.equal(nav.parseLocation({ pathname: "/flows", search: "", hash: "" }, flows).canvas, "flows"); + assert.equal(nav.parseLocation({ pathname: "/screens", search: "", hash: "" }, flows).canvas, "screens"); + assert.equal(nav.formatRoute(nav.parseLocation({ pathname: "/flows", search: "", hash: "" }, flows), flows), "/flows"); +}); + test("legacy hashes still parse", () => { assert.deepEqual(parse("/#workspace"), base); assert.deepEqual(parse("/#workspace/screens"), { ...base, canvas: "screens" }); - // Flows was retired; its links land on the map. + // Flows is a view only of a flow review: elsewhere its links, the retired Flows canvas's among them, land on the map. assert.deepEqual(parse("/#workspace/flows"), base); assert.deepEqual(parse("/flows?flow=derived-apps"), base); assert.deepEqual(parse("/#workspace/graph"), base); diff --git a/viewer/src/app.tsx b/viewer/src/app.tsx index 470fc0f..63c2be4 100644 --- a/viewer/src/app.tsx +++ b/viewer/src/app.tsx @@ -21,6 +21,7 @@ import { MapView } from "./components/views/semantic-map"; import { RightPanel } from "./components/panels/right"; import { SearchCommand } from "./components/search-command"; import { FIND_KEY, ScreensView } from "./components/views/screens"; +import { FlowsView } from "./components/views/flows"; import { publicReviewProjection, publicReviewToWorkspace } from "./public/public-workspace"; import type { PublicReview } from "../../src/contracts/capture"; import { @@ -81,7 +82,7 @@ import { presentScreensByPath, screenAliasesForPresentation, } from "./utils/screen-presentation"; -import { canvasViewTransitionDirection } from "./utils/canvas-view-transition"; +import { CANVAS_VIEW_ORDER, canvasViewTransitionDirection } from "./utils/canvas-view-transition"; import { STATIC_VIEWER_SOURCE, withStaticSourceParameters, @@ -304,6 +305,11 @@ const ViewerApp = memo(function ViewerApp({ reportKey }: { reportKey: number }) const [workspace, setWorkspace] = useState(); const [loadedPublicReview, setLoadedPublicReview] = useState(); const [error, setError] = useState(); + // A report compare:flows wrote carries its flow review; the Flows view exists only then. + const flowReview = REPORT_SOURCE.data?.exploration?.flowReview; + const flowReviewRef = useRef(flowReview); + flowReviewRef.current = flowReview; + const views: readonly CanvasView[] = flowReview ? CANVAS_VIEW_ORDER : CANVAS_VIEW_ORDER.filter((view) => view !== "flows"); // The URL is the one record of where the viewer is; `route` is it parsed, and everything a // reader would call navigation — canvas, rail, selection, inspector tabs, baseline, focus — // is read off it rather than held on its own. @@ -402,7 +408,7 @@ const ViewerApp = memo(function ViewerApp({ reportKey }: { reportKey: number }) setLoadedPublicReview(undefined); setWorkspace(undefined); setPayload({ map, coverage }); - const nextContext = routeContextForMap(map); + const nextContext: RouteContext = { ...routeContextForMap(map), ...(REPORT_SOURCE.data.exploration.flowReview ? { flows: true } : {}) }; const nextRoute = viewerRoute(parseLocation(viewerLocation(), nextContext)); contextRef.current = nextContext; routeRef.current = nextRoute; @@ -969,6 +975,7 @@ const ViewerApp = memo(function ViewerApp({ reportKey }: { reportKey: number }) }, [applyCanvasPanelDefaults, navigate, requestCanvasFit]); const selectCanvas = useCallback((canvas: CanvasView) => { + if (canvas === "flows" && !flowReviewRef.current) return; const direction = canvasViewTransitionDirection(routeRef.current.canvas, canvas); if (!direction) return; if (typeof document.startViewTransition !== "function") { @@ -1353,6 +1360,7 @@ const ViewerApp = memo(function ViewerApp({ reportKey }: { reportKey: number }) const routeContext = useMemo( () => (map ? { ...buildRouteContext, + ...(flowReview ? { flows: true } : {}), screenRoutes: screenRoutesOf(map.screens), screenVariants: screenVariantsOf(map.screens), ...(screenAliases ? { @@ -1361,7 +1369,7 @@ const ViewerApp = memo(function ViewerApp({ reportKey }: { reportKey: number }) screenRouteAliases: screenAliases.screenRouteAliases, } : {}), } : { ...buildRouteContext, screenRoutes: new Map() }), - [buildRouteContext, map, screenAliases], + [buildRouteContext, map, screenAliases, flowReview], ); contextRef.current = routeContext; const cameraDocumentId = currentBuild?.id ?? map?.run.id ?? payload?.map.run.id ?? "loading"; @@ -1617,11 +1625,12 @@ const ViewerApp = memo(function ViewerApp({ reportKey }: { reportKey: number }) changed: compareOrder, selectedScreenId, find: !phone, + flows: Boolean(flowReview), handoff: Boolean(handoffInput), pullRequest: review?.pullRequest, theme: themePreference, appearances: appearances.length > 1 ? appearances.map((option) => ({ id: option, label: appearanceLabel(option), current: option === appearance })) : [], - inspector: phone || comparingScreens ? undefined : { open: !rightCollapsed }, + inspector: phone || comparingScreens || route.canvas === "flows" ? undefined : { open: !rightCollapsed }, fullscreen: !phone && fullscreenSupported() ? fullscreen : undefined, targets: (() => { const run = runTargets(map.run); @@ -1696,6 +1705,7 @@ const ViewerApp = memo(function ViewerApp({ reportKey }: { reportKey: number }) onCloseInspector={closeInspector} /> ) : null} + {route.canvas === "flows" && flowReview ? : null} {comparing && phone && diff && selectedScreenId && comparePicker ? ( ) : setShortcutsOpen(true)} />} onSelectCanvas={selectCanvas} @@ -1770,7 +1782,7 @@ const ViewerApp = memo(function ViewerApp({ reportKey }: { reportKey: number }) {shortcutsOpen ? ( 0, handoff: Boolean(handoffInput), appearances: appearances.length > 1, + context={{ canvas: route.canvas, review: reviewingInSidebar, hasChanges: compareOrder.length > 0, flows: Boolean(flowReview), handoff: Boolean(handoffInput), appearances: appearances.length > 1, fullscreen: !phone && fullscreenSupported(), ...(switchTargets ? { targets: switchTargets } : {}) }} onClose={() => setShortcutsOpen(false)} @@ -1787,7 +1799,7 @@ const ViewerApp = memo(function ViewerApp({ reportKey }: { reportKey: number }) onClose={closeSearch} onSelectScreen={selectSearchScreen} /> - {comparingScreens ? null : = { map: , screens: , compare: , + flows: , }; const TOOLBAR_MINIMUM_FRAME_WIDTH = 220; type CanvasToolbarProps = { canvas: CanvasView; + /** The views this report has, in the toolbar's order; Flows only with a flow review. */ + views: readonly CanvasView[]; searchOpen: boolean; /** Page-wide actions beside Search (Hand off to AI). */ actions?: React.ReactNode; @@ -76,7 +78,7 @@ export function CanvasToolbar(props: CanvasToolbarProps) { "--canvas-visible-bottom-inset": `calc(100% - ${visibleFrame.y + visibleFrame.height}px)`, "--canvas-toolbar-shift": `${shift}px`, } as CSSProperties) : undefined; - const views = CANVAS_VIEW_ORDER; + const views = props.views; return (
diff --git a/viewer/src/components/compare/shortcuts-dialog.tsx b/viewer/src/components/compare/shortcuts-dialog.tsx index 672f94a..1f561aa 100644 --- a/viewer/src/components/compare/shortcuts-dialog.tsx +++ b/viewer/src/components/compare/shortcuts-dialog.tsx @@ -1,8 +1,8 @@ import { useEffect } from "react"; import { createPortal } from "react-dom"; import { X } from "lucide-react"; -import { CANVAS_NUMBER_KEYS, formatShortcut, SHORTCUTS, TARGET_NUMBER_KEYS, type Shortcut } from "../../utils/shortcuts"; -import type { CanvasView } from "../../utils/navigation"; +import { CANVAS_LABELS, type CanvasView } from "../../utils/navigation"; +import { CANVAS_NUMBER_KEYS, FLOWS_KEYS, formatShortcut, SHORTCUTS, TARGET_NUMBER_KEYS, type Shortcut } from "../../utils/shortcuts"; import { useModalDialog } from "../../hooks/use-modal-dialog"; type Row = { label: string; keys: (Shortcut | string)[] }; @@ -15,6 +15,8 @@ export type ShortcutContext = { review: boolean; /** The report has changed screens, so Compare and the changes menu exist. */ hasChanges: boolean; + /** The report compares two runs of a flows file, so it has a Flows view. */ + flows?: boolean; /** The report can be handed off to an agent (a hosted preview). */ handoff?: boolean; /** The report was captured in both light and dark, so the app's appearance can be switched. */ @@ -72,12 +74,12 @@ const CAMERA: Group = { /** The groups for one context, in two columns: what this section does, then the app-wide keys. */ export function shortcutGroups(context: ShortcutContext): Group[][] { - const sections: CanvasView[] = context.hasChanges ? ["map", "screens", "compare"] : ["map", "screens"]; + const sections: CanvasView[] = ["map", "screens", ...(context.hasChanges ? ["compare" as const] : []), ...(context.flows ? ["flows" as const] : [])]; const goTo: Group = { title: "Go to", rows: [ { - label: context.hasChanges ? "Map, Screens, Compare" : "Map, Screens", + label: sections.map((section) => CANVAS_LABELS[section]).join(", "), keys: sections.map((section) => SHORTCUTS.canvas[section]), }, ...(CANVAS_NUMBER_KEYS @@ -116,8 +118,17 @@ export function shortcutGroups(context: ShortcutContext): Group[][] { { label: "Close the inspector, then leave the filmstrip or deselect", keys: ["Esc"] }, ], }; + const flowsGroup: Group = { + title: "Flows", + rows: [ + { label: "Previous or next checkpoint", keys: [FLOWS_KEYS.previous, FLOWS_KEYS.next] }, + { label: "Next or previous changed area", keys: [FLOWS_KEYS.nextArea, FLOWS_KEYS.previousArea] }, + { label: "Flip between Before and After", keys: [FLOWS_KEYS.flip] }, + ], + }; const here: Group[] = context.canvas === "compare" ? [reviewGroup(context)] + : context.canvas === "flows" ? [flowsGroup] : [...(context.review ? [reviewGroup(context)] : []), ...(context.canvas === "map" ? [mapGroup] : []), ...(context.canvas === "screens" ? [screensGroup] : []), CAMERA]; return [here, [goTo, view]]; diff --git a/viewer/src/components/phone/phone-chrome.tsx b/viewer/src/components/phone/phone-chrome.tsx index 8c862aa..ad64d92 100644 --- a/viewer/src/components/phone/phone-chrome.tsx +++ b/viewer/src/components/phone/phone-chrome.tsx @@ -1,14 +1,14 @@ import { useEffect, useRef, type PointerEvent as ReactPointerEvent, type ReactNode } from "react"; import { createPortal } from "react-dom"; -import { GitCompareArrows, LayoutGrid, Map as MapIcon, X } from "lucide-react"; +import { GitCompareArrows, LayoutGrid, Map as MapIcon, Waypoints, X } from "lucide-react"; import { CANVAS_LABELS, type CanvasView } from "../../utils/navigation"; -import { CANVAS_VIEW_ORDER } from "../../utils/canvas-view-transition"; import "./phone.css"; const VIEW_ICONS: Record = { map: , screens: , compare: , + flows: , }; /** @@ -17,12 +17,14 @@ const VIEW_ICONS: Record = { */ export function PhoneTabBar(props: { canvas: CanvasView; + /** The views this report has, in the toolbar's order; Flows only with a flow review. */ + views: readonly CanvasView[]; compareCount: number; onSelectCanvas: (canvas: CanvasView) => void; }) { return (
+ ); + }) : null} + + + ); +} diff --git a/viewer/src/components/views/flows.css b/viewer/src/components/views/flows.css new file mode 100644 index 0000000..14bfc5d --- /dev/null +++ b/viewer/src/components/views/flows.css @@ -0,0 +1,113 @@ +/* The Flows view: the Screens sidebar on the left, one flow on the stage, the flow bar where Compare keeps its bar. */ +.flows-page { + --flows-right-inset: 0px; + --flows-mono: "JetBrains Mono", ui-monospace, SFMono-Regular, Menlo, monospace; + position: absolute; + inset: 0; + overflow: hidden; + padding: calc(var(--chrome-top) + var(--chrome-height) + 20px) calc(var(--flows-right-inset) + 28px) 96px var(--canvas-chrome-inline-inset); + color: var(--text); + background: var(--screen); +} +.flows-body { display: flex; height: 100%; align-items: flex-start; gap: 32px; } +.flows-side { max-height: 100%; } +.flows-find { margin: 0 0 8px; } +.flows-mark { flex: none; } + +/* ── The list: each flow with its status line; the open flow's checkpoints under it ── */ +.flows-list, .flows-checkpoints { margin: 0; padding: 0; list-style: none; } +.flows-list { display: flex; flex-direction: column; gap: 1px; } +.flows-row { + display: flex; + width: 100%; + align-items: flex-start; + gap: 8px; + padding: 6px 8px; + border-radius: 6px; + color: var(--text); + background: transparent; + text-align: left; +} +.flows-row > .flows-mark { margin-top: 2px; } +.flows-row:hover { background: var(--panel-hover); } +.flows-row-text { display: flex; min-width: 0; flex: 1; flex-direction: column; gap: 1px; } +.flows-row-name { overflow: hidden; font-size: 13px; text-overflow: ellipsis; white-space: nowrap; } +.flows-row.open .flows-row-name { font-weight: 600; } +.flows-row-status { color: var(--secondary); font-size: 12px; } +.flows-row-status.stopped { color: var(--warning); } +.flows-checkpoints { padding: 2px 0 6px; } +.flows-checkpoint { padding-left: 30px; color: var(--text); } + +/* ── The flow bar: the flow's name and Before | After ── */ +.flows-top { z-index: 6; } +.flows-bar { width: auto; gap: 10px; padding-left: 14px; } +.flows-bar-name { flex: none; font-size: 14px; font-weight: 600; white-space: nowrap; } +.flows-bar .toggle-group { width: auto; flex: none; } +.flows-bar .toggle-row { width: auto; } +.flows-bar .toggle-row .tooltip-anchor { flex: none; } +.flows-bar .toggle-button { padding: 0 12px; white-space: nowrap; } +.flows-bar .toggle-label { overflow-wrap: normal; } + +/* ── The strip: Screens' filmstrip, a checkpoint to a frame ── */ +/* A sideways swipe steps the strip; it must not also take the browser back a page. */ +html:has(.flows-page) { overscroll-behavior-x: none; } +.flows-body > .screens-strip { height: 100%; } +/* The middle checkpoint opens nothing: it is already as large as the page shows it. */ +.screens-strip.static-middle .screens-strip-frame.focused .screens-strip-target { cursor: default; } +/* As on Screens, the middle frame keeps its screenshot's own colours when the arrows focus it. */ +.flows-page .screens-strip-target:focus-visible { filter: none; } +.flows-strip-head { display: flex; width: 100%; flex: none; flex-direction: column; align-items: center; gap: 2px; } +.flows-step.exact .screens-strip-crumb-label { font-family: var(--flows-mono); font-size: 12px; } +.flows-step.failed { color: var(--text); font-weight: 500; } +.flows-step.not-run { opacity: 0.6; } +.flows-word { flex: none; color: var(--secondary); font-size: 13px; font-weight: 600; } +.flows-word.changed { color: var(--modified); } +.flows-word.added { color: var(--success); } +.flows-word.removed { color: var(--danger); } +.flows-word.not-comparable, .flows-word.stopped, .flows-word.not-reached { color: var(--warning); } +.flows-strip-note { + max-width: 100%; + margin: 0; + overflow: hidden; + color: var(--secondary); + font-size: 13px; + text-overflow: ellipsis; + white-space: nowrap; +} +.flows-strip-note.stopped { color: var(--warning); } + +.flows-shot { position: relative; display: block; width: 100%; height: 100%; } +.flows-shot img { display: block; width: 100%; height: 100%; object-fit: contain; } +.flows-shot.missing { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 8px; + padding: 16px; + color: var(--secondary); + font-size: 13px; + font-weight: 500; + text-align: center; +} +.flows-area { + position: absolute; + margin: -3px; + padding: 3px; + border-radius: 4px; + outline: 2px solid var(--modified); + background: color-mix(in srgb, var(--modified) 9%, transparent); + box-sizing: content-box; + pointer-events: none; +} +.flows-area.current { outline-width: 3px; background: color-mix(in srgb, var(--modified) 18%, transparent); } + +.flows-page.phone { padding: calc(var(--chrome-top) + var(--chrome-height) + 12px) 12px 96px; overflow-y: auto; } +/* On a phone the bar is the page's first row, under the pill, with the flow's name and Before | After. */ +.flows-page.phone .flows-top { position: static; max-width: none; margin: 0 0 12px; transform: none; } +.flows-page.phone .flows-bar { width: 100%; zoom: 1; } +.flows-page.phone .flows-bar-name { min-width: 0; flex: 1; overflow: hidden; text-overflow: ellipsis; } +.flows-page.phone .flows-body { flex-direction: column; height: auto; } +.flows-page.phone .flows-side { width: 100%; } +/* On a phone the strip comes first, as tall as most of the screen, and the list of flows follows it. */ +.flows-page.phone .flows-body > .screens-strip { order: -1; width: 100%; height: 72svh; flex: none; } diff --git a/viewer/src/components/views/flows.tsx b/viewer/src/components/views/flows.tsx new file mode 100644 index 0000000..7d06f77 --- /dev/null +++ b/viewer/src/components/views/flows.tsx @@ -0,0 +1,266 @@ +import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; +import { ChevronRight } from "lucide-react"; +import type { FlowReview, FlowReviewCheckpoint, FlowReviewFlow } from "../../../../src/contracts/flow-comparison"; +import { embeddedArtifactUrl } from "../../utils/embedded-report"; +import { + checkpointLabel, checkpointMark, firstProblem, flowMark, flowNeedsReview, flowStatusLine, sidePicture, stepsBefore, stoppedEarly, + textChangeLine, visibleFlows, type FlowFilter, type FlowMark, type FlowSide, +} from "../../utils/flows-model"; +import { normalizeSearchText } from "../../utils/search"; +import { FLOWS_KEYS, formatShortcut, matchesShortcut, shortcutsLive } from "../../utils/shortcuts"; +import { useCanvasVisibleFrame } from "../canvas/visible-frame-provider"; +import { Filmstrip } from "../shared/filmstrip"; +import { PanelSearch } from "../shared/panel-search"; +import { DEFAULT_SCREENSHOT_RATIO, ScreenshotImage } from "../shared/screenshot-image"; +import { ToggleGroup } from "../shared/toggle-group"; +import "./screens.css"; +import "./flows.css"; + +/** A neighbour's outline in the strip, in its status colour, as Screens' filmstrip outlines a changed neighbour. */ +const FRAME_CLASS: Partial> = { changed: "diff-modified", added: "diff-added", removed: "diff-removed" }; + +/** A status by the shape of its icon as well as its colour, so no state depends on colour alone. */ +export function FlowMarkIcon(props: { mark: FlowMark; size?: number }) { + const size = props.size ?? 14, c = size / 2, r = size / 2 - 1.5; + const body: Record = { + changed: <>, + unchanged: <>, + "not-reached": , + stopped: <>, + added: <>, + removed: <>, + "not-comparable": <>, + }; + return ; +} + +/** One checkpoint's picture on one side, with its changed areas outlined, or why it has none. */ +function CheckpointPicture(props: { checkpoint: FlowReviewCheckpoint; side: FlowSide; areas: boolean; area?: number | undefined; eager: boolean }) { + const picture = sidePicture(props.side === "after" ? props.checkpoint.after : props.checkpoint.before); + if (!picture || "missing" in picture) { + const why = !picture ? (props.side === "after" ? "Not in this build's flows file" : "Not in the earlier build's flows file") : "Not reached"; + return ( + + + {why} + + ); + } + return ( + + + {props.areas ? (props.checkpoint.regions ?? []).map((region, place) => ( + + ); +} + +/** + * Above the strip: the steps that led to the checkpoint in the middle, then its name and status, as the Screens + * filmstrip shows the path to its screen. One line under it says what the picture cannot: the text a small change + * changed, or why the run stopped. + */ +function StripHeader(props: { flow: FlowReviewFlow; index: number }) { + const checkpoint = props.flow.checkpoints[props.index]!; + const mark = checkpointMark(checkpoint); + const steps = stepsBefore(props.flow, props.index); + const failed = steps.some(step => step.failed); + const note = failed && props.flow.after?.reason ? props.flow.after.reason : textChangeLine(checkpoint); + return ( +
+
+ {steps.map(({ step, failed: stoppedHere, ran }, place) => { + const text = "action" in step ? step.action : step.do; + return ( + + {stoppedHere ? : null} + {text} + + ); + })} + + + {checkpoint.name} + {checkpoint.verdict === "unchanged" ? null : {checkpointLabel(checkpoint)}} + +
+ {note ?

{note}

: null} +
+ ); +} + +function matches(flow: FlowReviewFlow, query: string): boolean { + if (!query) return true; + const haystack = [flow.name, ...flow.checkpoints.map((checkpoint) => checkpoint.name)]; + return haystack.some((text) => normalizeSearchText(text).includes(query)); +} + +/** + * The Flows view: a report that compares two runs of a flows file, one flow at a time, as a filmstrip of its + * checkpoints in order. A flow opens at its first problem. Before | After (X) flips every picture to the other build in + * place. Nothing is saved: the filter, the flow and the checkpoint are this page's state. + */ +export function FlowsView(props: { review: FlowReview; phone: boolean }) { + const { review } = props; + const anyChanged = review.flows.some(flowNeedsReview); + const [filter, setFilter] = useState(anyChanged ? "changed" : "all"); + const [query, setQuery] = useState(""); + const shown = useMemo(() => { + const needle = normalizeSearchText(query.trim()); + return visibleFlows(review, filter).filter((flow) => matches(flow, needle)); + }, [review, filter, query]); + const [flowName, setFlowName] = useState(() => visibleFlows(review, anyChanged ? "changed" : "all")[0]?.name ?? review.flows[0]?.name); + const flow = review.flows.find((item) => item.name === flowName) ?? shown[0] ?? review.flows[0]; + const [index, setIndex] = useState(() => (flow ? firstProblem(flow) : 0)); + const [side, setSide] = useState("after"); + const [area, setArea] = useState(undefined); + const { visibleFrame } = useCanvasVisibleFrame(); + + const openFlow = (next: FlowReviewFlow) => { + setFlowName(next.name); + setIndex(firstProblem(next)); + setArea(undefined); + }; + const openCheckpoint = (place: number) => { + setIndex(place); + setArea(undefined); + }; + + // The arrows step the strip, J and K walk the changed areas of the checkpoint in the middle, X flips. + const state = useRef({ flow, index, area }); + state.current = { flow, index, area }; + useEffect(() => { + const onKeyDown = (event: KeyboardEvent) => { + if (!shortcutsLive(event)) return; + const { flow: current, index: place, area: shownArea } = state.current; + if (!current) return; + const regions = current.checkpoints[place]?.regions?.length ?? 0; + if (matchesShortcut(event, FLOWS_KEYS.flip)) setSide((value) => (value === "after" ? "before" : "after")); + else if (matchesShortcut(event, FLOWS_KEYS.next) || matchesShortcut(event, FLOWS_KEYS.previous)) { + const next = place + (matchesShortcut(event, FLOWS_KEYS.next) ? 1 : -1); + if (next < 0 || next >= current.checkpoints.length) return; + setIndex(next); setArea(undefined); + } else if (matchesShortcut(event, FLOWS_KEYS.nextArea) || matchesShortcut(event, FLOWS_KEYS.previousArea)) { + if (!regions) return; + const forward = matchesShortcut(event, FLOWS_KEYS.nextArea); + setArea(shownArea === undefined ? (forward ? 0 : regions - 1) : (shownArea + (forward ? 1 : regions - 1)) % regions); + } else return; + event.preventDefault(); + }; + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, []); + + if (!flow) { + return ( +
+

No flows

The runs compared have no flows in common.

+
+ ); + } + const rightInset = visibleFrame ? `calc(100% - ${visibleFrame.x + visibleFrame.width}px)` : "0px"; + + const sidebar = ( + + ); + + // The flow's name and the build its pictures show. + const flowBar = ( +
+
+ + {flow.name} + + ariaLabel="Build" + value={side} + arrowKeys={false} + onChange={setSide} + options={[ + { id: "before", label: "Before", shortcut: formatShortcut(FLOWS_KEYS.flip) }, + { id: "after", label: "After", shortcut: formatShortcut(FLOWS_KEYS.flip) }, + ]} + /> +
+
+ ); + + const checkpoints = flow.checkpoints; + const strip = ( + ({ id: String(place), aspect: DEFAULT_SCREENSHOT_RATIO }))} + focusedId={String(Math.min(index, checkpoints.length - 1))} + header={} + ariaLabel={`${flow.name}, in order`} + phone={props.phone} + frameClass={(id) => FRAME_CLASS[checkpointMark(checkpoints[Number(id)]!)]} + label={(id) => checkpoints[Number(id)]!.name} + onPick={(id) => openCheckpoint(Number(id))} + content={(id, { middle, distance }) => ( + + )} + /> + ); + + return ( +
+ {flowBar} +
+ {sidebar} + {strip} +
+
+ ); +} diff --git a/viewer/src/components/views/screens-filmstrip.tsx b/viewer/src/components/views/screens-filmstrip.tsx index 2de7094..06ff1ed 100644 --- a/viewer/src/components/views/screens-filmstrip.tsx +++ b/viewer/src/components/views/screens-filmstrip.tsx @@ -1,35 +1,13 @@ -import { useEffect, useLayoutEffect, useMemo, useRef, useState, type CSSProperties, type PointerEvent, type WheelEvent } from "react"; +import { useMemo } from "react"; import { ChevronRight } from "lucide-react"; import { artifactUrl, isLaunchScreen, type MapIndex, type Screen } from "../../utils/model"; import { primaryScreenshotForScreen, screenshotSize, type ScreenshotSize } from "../../utils/screen-images"; import { diffClass, type BuildDiff } from "../../utils/build-diff"; import { screenLabel } from "../../utils/screen-path"; import { treeNodeKey, type ScreenTree, type ScreenTreeIndex, type ScreenTreeNode } from "../../utils/screen-tree"; +import { Filmstrip } from "../shared/filmstrip"; import { ScreenshotImage } from "../shared/screenshot-image"; -/** - * A neighbour's size and opacity by its distance from the screen in the middle. The last, invisible, place is where a - * screen waits to come in, so a swipe or a step fades it in rather than popping it. - */ -const SCALES = [1, 0.74, 0.54, 0.4, 0.3]; -const OPACITIES = [1, 0.9, 0.6, 0.3, 0]; -const GAP = 32; -/** A swipe has ended once its wheel events (and the trackpad's momentum after it) stop for this long. */ -const SWIPE_END_MS = 100; -/** - * At the end of a swipe the strip settles on the next screen in its direction once it has moved this far towards it, - * so one wheel notch steps a screen and a twitch springs back. - */ -const SWIPE_COMMIT = 0.2; -/** How far a finger moves before a touch is a swipe rather than a tap. */ -const TAP_SLOP_PX = 8; -/** How far ahead a flick's speed carries the strip when the finger lifts: its velocity times this. */ -const FLICK_MS = 160; -/** - * How long the strip takes to settle after a gesture: longer the further it has left to go, so that a short way home - * is not crossed at the speed of a whole step and a long one does not snap. - */ -const settleMs = (distance: number) => Math.round(260 + 260 * Math.min(1, distance)); /** An iPhone's portrait shape, for a run with no screenshot sizes at all. */ const FALLBACK_SIZE: ScreenshotSize = { imageWidth: 390, imageHeight: 844 }; @@ -61,50 +39,12 @@ function labelOf(screen: Screen, index: MapIndex): string { return isLaunchScreen(screen) ? "Cold start" : screenLabel(screen, index); } -/** The stage's size, which the frames are fitted to. */ -function useSize(): [(element: HTMLDivElement | null) => void, { width: number; height: number }] { - const [element, setElement] = useState(null); - const [size, setSize] = useState({ width: 0, height: 0 }); - useLayoutEffect(() => { - if (!element) return; - const measure = () => setSize({ width: element.clientWidth, height: element.clientHeight }); - measure(); - if (typeof ResizeObserver === "undefined") return; - const observer = new ResizeObserver(measure); - observer.observe(element); - return () => observer.disconnect(); - }, [element]); - return [setElement, size]; -} - /** - * Screens one at a time: the focused screen large in the middle, the screens before and after it smaller and fainter - * the further they are, in the tree's order. Above it, the rows of the tree it was opened from, ending in its name with - * its status dot. The neighbours, whose names are not shown, are outlined in their status colour instead. + * Screens one at a time, in the tree's order. Above them, the rows of the tree the middle screen was opened from, + * ending in its name with its status dot. The neighbours, whose names are not shown, are outlined in their status colour + * instead. */ export function ScreensFilmstrip(props: ScreensFilmstripProps) { - const [stageRef, stage] = useSize(); - const at = props.order.indexOf(props.focusedId); - // While a swipe lasts, the strip sits at a fractional position in `order`, following the gesture. - const [swipe, setSwipe] = useState<{ position: number; direction: number } | null>(null); - // Once the gesture lets go, the strip eases the rest of the way on a timing of its own (settleMs). - const [settling, setSettling] = useState(null); - const settled = useRef>(undefined); - // The swipe's position as of the last wheel event, which can arrive before the render that shows the one before it. - const live = useRef(null); - // A finger on the strip: where it went down, the strip's position then, and its latest speed in px/ms. - const touch = useRef<{ id: number; x: number; from: number; lastX: number; lastAt: number; velocity: number; moved: boolean } | null>(null); - // A swipe ends with the finger lifting over a screen; the click that follows is not a tap on it. - const swallowClick = useRef(false); - const swipeEnd = useRef>(undefined); - useEffect(() => () => { clearTimeout(swipeEnd.current); clearTimeout(settled.current); }, []); - // The gesture under way: the screens it has put in the middle, the last of them, and whether it has added its one - // history entry. A screen put in the middle by anything else (an arrow key, the tree, a click) ends it. - const gesture = useRef<{ shown: string; ids: Set; pushed: boolean; start: number } | null>(null); - const focusedNow = useRef(props.focusedId); - useEffect(() => { focusedNow.current = props.focusedId; }); - const focused = props.index.screens.get(props.focusedId); - // A row of the tree by its fold key, as the path above the frame names it. const rowLabels = useMemo(() => { const labels = new Map(); @@ -121,214 +61,54 @@ export function ScreensFilmstrip(props: ScreensFilmstripProps) { walk(props.tree.tree); return labels; }, [props.index, props.tree]); + const focused = props.index.screens.get(props.focusedId); + if (!focused) return null; - if (!focused || at < 0) return null; - - const sizeOf = (screen: Screen) => screenshotSize(primaryScreenshotForScreen(screen, props.index)) ?? props.typicalSize ?? FALLBACK_SIZE; - const aspect = (screen: Screen) => { - const size = sizeOf(screen); - return size.imageWidth / size.imageHeight; - }; - // Every frame is laid out as the middle one would be, filling the stage's height short of growing wider than half - // the stage (on a phone, most of it, with the neighbours peeking in at the sides), so a landscape capture beside portrait ones is as wide as it may be rather than as tall; then it is scaled - // down around its centre, and the centres are spaced so the scaled frames sit a gap apart, the gap shrinking with them. - const frameSize = (screen: Screen) => { - const height = Math.max(0, Math.min(stage.height - 16, (stage.width * (props.phone ? 0.7 : 0.5)) / aspect(screen))); - return { width: height * aspect(screen), height }; - }; - // Where each screen sits with the screen at `centre` in the middle: its distance and its centre's offset. - const layout = (centre: number) => { - const placed = new Map(); - const middle = props.index.screens.get(props.order[centre] ?? ""); - if (!middle) return placed; - placed.set(props.order[centre]!, { distance: 0, x: 0 }); - for (const direction of [-1, 1]) { - let edge = frameSize(middle).width / 2; - for (let distance = 1; distance < SCALES.length; distance++) { - const id = props.order[centre + direction * distance]; - const screen = id ? props.index.screens.get(id) : undefined; - if (!id || !screen) break; - const scaled = frameSize(screen).width * SCALES[distance]!; - const x = edge + GAP * SCALES[distance]! + scaled / 2; - placed.set(id, { distance, x: direction * x }); - edge = x + scaled / 2; - } - } - return placed; - }; - // A swipe someone else's step overtook (an arrow key, a click) no longer places the strip. - const swiping = swipe !== null && Math.round(swipe.position) === at; - const position = swiping ? swipe.position : at; - const base = Math.floor(position), fraction = position - base; - const from = layout(base), to = fraction > 0 ? layout(base + 1) : from; - const lerp = (a: number, b: number) => a + (b - a) * fraction; - const last = SCALES.length - 1; - // In the strip's order, so the frames' elements keep their order in the page as the middle changes: React moves an - // element that changes place among its siblings, and a moved element jumps instead of easing to where it goes. - const placeOf = new Map(props.order.map((id, place) => [id, place])); - const frames = [...new Set([...from.keys(), ...to.keys()])].sort((a, b) => placeOf.get(a)! - placeOf.get(b)!).flatMap(id => { + const items = props.order.flatMap(id => { const screen = props.index.screens.get(id); if (!screen) return []; - const a = from.get(id), b = to.get(id); - // A screen in only one of the two places is coming in or going out at the invisible end. - const distance = lerp(a?.distance ?? last, b?.distance ?? last); - const x = lerp(a?.x ?? b!.x, b?.x ?? a!.x); - const scale = lerp(SCALES[a?.distance ?? last]!, SCALES[b?.distance ?? last]!); - const opacity = lerp(OPACITIES[a?.distance ?? last]!, OPACITIES[b?.distance ?? last]!); - return [{ id, screen, distance, x, scale, opacity, ...frameSize(screen) }]; + const size = screenshotSize(primaryScreenshotForScreen(screen, props.index)) ?? props.typicalSize ?? FALLBACK_SIZE; + return [{ id, aspect: size.imageWidth / size.imageHeight }]; }); - // The frames' gap at rest, which one screen of swipe travels. - const pitch = Math.abs(layout(at).get(props.order[at + 1] ?? props.order[at - 1] ?? "")?.x ?? frameSize(focused).width) || 1; - - const clamp = (value: number) => Math.max(0, Math.min(props.order.length - 1, value)); - const abandon = () => { clearTimeout(swipeEnd.current); gesture.current = null; live.current = null; }; - const overtaken = () => !gesture.current || !gesture.current.ids.has(focusedNow.current); - // Where a gesture starts: where the one under way has the strip, or the screen in the middle for a new one. - const begin = (): number => { - if (!overtaken() && live.current !== null) return live.current; - abandon(); - const index = props.order.indexOf(focusedNow.current); - const start = index < 0 ? at : index; - gesture.current = { shown: focusedNow.current, ids: new Set([focusedNow.current]), pushed: false, start }; - return start; - }; - // Puts a screen in the middle for the gesture: its first pick is one step back in the history, the rest replace it. - const show = (id: string) => { - const current = gesture.current; - if (!current || id === current.shown) return; - current.shown = id; current.ids.add(id); - props.onPick(id, current.pushed ? { replace: true } : undefined); - current.pushed = true; - }; - // Puts the strip at `next` while a gesture lasts; the middle screen, and with it the path above, follows the swipe as - // it passes halfway. - const moveTo = (next: number, direction: number) => { - live.current = next; - setSwipe({ position: next, direction }); - show(props.order[Math.round(next)]!); - }; - // When the gesture ends, the strip settles on the next screen in its direction once it is SWIPE_COMMIT of the way - // there; `next` may be ahead of where the strip is, carried on by a flick. - const settle = (next: number, direction: number) => { - if (overtaken()) { abandon(); setSwipe(null); return; } - const index = clamp(direction > 0 ? Math.floor(next + 1 - SWIPE_COMMIT) : direction < 0 ? Math.ceil(next - 1 + SWIPE_COMMIT) : Math.round(next)); - show(props.order[index]!); - gesture.current = null; - const ms = settleMs(Math.abs(index - (live.current ?? next))); - clearTimeout(settled.current); - setSettling(ms); - settled.current = setTimeout(() => setSettling(null), ms); - live.current = null; - setSwipe(null); - }; - - // A swipe along either axis moves the strip with it, so a mouse wheel works as well as a trackpad; it has ended when - // its events stop. - const onWheel = (event: WheelEvent) => { - // A pinch arrives as a wheel with Control held; it zooms the page, not the strip. - if (event.ctrlKey) return; - const delta = Math.abs(event.deltaX) > Math.abs(event.deltaY) ? event.deltaX : event.deltaY; - const pixels = event.deltaMode === 1 ? delta * 16 : event.deltaMode === 2 ? delta * stage.width : delta; - if (!pixels) return; - const next = clamp(begin() + pixels / pitch); - // The way the whole swipe went, not its last event: momentum's last 1px back must not turn a swipe around. - const direction = Math.sign(next - gesture.current!.start) || Math.sign(pixels); - moveTo(next, direction); - clearTimeout(swipeEnd.current); - swipeEnd.current = setTimeout(() => settle(next, direction), SWIPE_END_MS); - }; - - // A finger drags the strip under it; on lifting, a flick carries it on by its speed before it settles. A mouse - // keeps its clicks and wheel. - const onPointerDown = (event: PointerEvent) => { - // A browser that sent no click after the last swipe must not have this press's click swallowed, by finger or mouse. - swallowClick.current = false; - // A touch whose end never reached the strip (it slid off before it moved enough to be held) gives way to this one. - if (event.pointerType === "mouse" || touch.current?.id === event.pointerId) return; - touch.current = { id: event.pointerId, x: event.clientX, from: begin(), lastX: event.clientX, lastAt: event.timeStamp, velocity: 0, moved: false }; - }; - const onPointerMove = (event: PointerEvent) => { - const current = touch.current; - if (!current || current.id !== event.pointerId) return; - const dx = event.clientX - current.x; - if (!current.moved && Math.abs(dx) < TAP_SLOP_PX) return; - // A wheel swipe's settle waits until the finger takes over; a tap leaves it to settle. - if (!current.moved) { current.moved = true; clearTimeout(swipeEnd.current); event.currentTarget.setPointerCapture(event.pointerId); } - const elapsed = event.timeStamp - current.lastAt; - if (elapsed > 0) current.velocity = (event.clientX - current.lastX) / elapsed; - current.lastX = event.clientX; current.lastAt = event.timeStamp; - moveTo(clamp(current.from - dx / pitch), -Math.sign(dx)); - }; - const onPointerEnd = (event: PointerEvent) => { - const current = touch.current; - if (!current || current.id !== event.pointerId) return; - touch.current = null; - if (!current.moved) return; - swallowClick.current = true; - // A finger that stopped before lifting carries nothing on. - const velocity = event.timeStamp - current.lastAt > 80 ? 0 : current.velocity; - const next = clamp((live.current ?? position) - (velocity * FLICK_MS) / pitch); - settle(next, velocity ? -Math.sign(velocity) : -Math.sign(event.clientX - current.x)); - }; - const path = (props.shape.ancestors.get(props.focusedId) ?? []).flatMap(key => rowLabels.get(key) ?? []); const plain = props.index.plainTitles || isLaunchScreen(focused); const dot = props.dot(props.focusedId); - - return ( -
{ if (swallowClick.current) { swallowClick.current = false; event.preventDefault(); event.stopPropagation(); } }}> -
- {path.map((label, position) => ( - - {label} - - ))} - - {dot ?
+ ); -
- {stage.width > 0 ? frames.map(({ id, screen, distance, x, scale, opacity, width, height }) => { - const artifact = primaryScreenshotForScreen(screen, props.index); - const url = artifactUrl(artifact); - const size = screenshotSize(artifact); - const placeholder =
-
+ return ( + { + const frameDiff = props.diff?.screens.get(id); + return [frameDiff ? diffClass(frameDiff) : "", props.dimmed(id) ? "dimmed" : ""].filter(Boolean).join(" ") || undefined; + }} + label={(id) => { const screen = props.index.screens.get(id); return screen ? labelOf(screen, props.index) : id; }} + selected={(id) => props.selectedId === id} + onPick={props.onPick} onOpen={props.onOpen} + content={(id, { distance }) => { + const screen = props.index.screens.get(id)!; + const artifact = primaryScreenshotForScreen(screen, props.index); + const url = artifactUrl(artifact); + const size = screenshotSize(artifact); + const placeholder =