From d6b97331b3e77d39a70b8d76f3001afc76fc5dc7 Mon Sep 17 00:00:00 2001 From: Brent Vatne Date: Sat, 24 Oct 2026 14:00:39 -0700 Subject: [PATCH] Flows: compare two runs of a flows file checkpoint by checkpoint (compare:flows) compare:flows --before DIR --after DIR [--output FILE] pairs the checkpoints of two runs by flow and checkpoint name and diffs each pair with the review's screenshot diff, setting aside the status bar and home indicator as a review does. Each checkpoint is unchanged, changed (with the regions that differ), missing (no picture on one side, with why), different-size (another device or orientation, so not diffed), added or removed. The runs' conditions that differ (app, platform, device, OS version, locale, orientation, appearance) are listed rather than refused, so an iPhone run and an Android run still line up. On a foldable the status bar is not set aside, because a run does not record which panel was in use; the result says so. The pairing is a pure function in contracts/flow-comparison.ts; the diff is in review/flow-comparison.ts. cli now imports review, which ARCHITECTURE lists. --- ARCHITECTURE.md | 2 +- README.md | 8 ++ skills/explore-binary/SKILL.md | 1 + src/cli.ts | 3 +- src/cli/args.ts | 2 + src/cli/binary-commands.ts | 39 +++++++++- src/cli/help.ts | 2 + src/contracts/capture.ts | 2 +- src/contracts/flow-comparison.ts | 58 +++++++++++++++ src/review/flow-comparison.ts | 79 ++++++++++++++++++++ test/node/flow-comparison.test.mjs | 115 +++++++++++++++++++++++++++++ 11 files changed, 305 insertions(+), 6 deletions(-) create mode 100644 src/contracts/flow-comparison.ts create mode 100644 src/review/flow-comparison.ts create mode 100644 test/node/flow-comparison.test.mjs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a1398a5..4718c60 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -46,7 +46,7 @@ Each layer imports only from the layers below it. 4. `capture` imports `capture/agent`, `config` and `map`. `capture/agent` imports `capture` and `config`. 5. `review` imports `capture` and `map`. 6. `ci` imports `config`, `source`, `capture`, `capture/agent`, `review` and `report`. `binary` imports `config`, `capture`, `capture/agent`, `map` and `report`. -7. `cli` imports `ci`, `binary`, `map`, `config` and `report`. `src/cli.ts` imports `cli`, `map` and `shared`. +7. `cli` imports `ci`, `binary`, `map`, `config`, `report` and `review`. `src/cli.ts` imports `cli`, `map` and `shared`. Every folder may import `contracts/`, `shared/` and `src/types.ts`. The exception is `contracts/`: its value imports stay inside `contracts/` or go to `src/types.ts`. diff --git a/README.md b/README.md index e160184..531c873 100644 --- a/README.md +++ b/README.md @@ -267,6 +267,14 @@ npx @expo/explore explore:binary ./my-app.app --flows flows.json - The report lists every flow and checkpoint of the file, in order, even those a run never reached: each one without a picture says why, so two runs still pair up step by step. - Each flow is also a journey of the map: the path its steps took, from the screen it started on. When the app moved without a tap the map could record (it left the app, or changed on its own), the path goes on as a second journey of the flow. - Relaunching ends the app's process, and the app keeps its data, so a flow that signs in leaves the next flow signed in. On `explore:session`, the app is relaunched before the first flow too. + +To compare two runs of the same flows file, such as two builds, pass both output directories: + +```sh +npx @expo/explore compare:flows --before explore-app-1 --after explore-app-2 --output flows.json +``` + +Each checkpoint comes back `unchanged`, `changed` (with where the screenshots differ, the status bar and home indicator set aside, as in a review), `missing` (no picture on one side, with why), `different-size` (another device or orientation, so not compared), or `added` / `removed` (in only one run's flows file). The result also lists how the two runs were captured differently (the app, device, OS version, locale, orientation, or appearance), since a change can come from those. - A `{{secret:NAME}}` or a described step limited by `platforms` to another platform needs no value, and no model, on this one. diff --git a/skills/explore-binary/SKILL.md b/skills/explore-binary/SKILL.md index a2939c5..1572fab 100644 --- a/skills/explore-binary/SKILL.md +++ b/skills/explore-binary/SKILL.md @@ -42,6 +42,7 @@ To compare runs, or an iPhone with an Android phone, screen for screen, pass `-- When the user describes a flow, or a set of flows, to look at ("sign in, then rename a pattern"), write them as a flows file and pass `--flows flows.json`, not `--replay`. The format is in the README ("Follow named flows, step by step"): `{ "flows": [{ "name": "Sign in", "steps": [{ "checkpoint": "Welcome" }, { "do": "Sign in with {{secret:TEST_EMAIL}}" }, { "checkpoint": "Home" }] }] }`. - Each step is an exact action (the `--replay` format), a step described in words (`do`) that a model carries out, or a named `checkpoint`. Use exact actions where you know the labels, and `do` where you don't. - Each flow starts from a relaunched app, and its checkpoints are photographed under their names. `report/exploration.json` lists each flow under `exploration.flows`, with its status and its checkpoints in order, so two runs pair up by flow and checkpoint name. A checkpoint on a screen that withholds its picture (a secret on it) is listed with why it has none, and its flow goes on. +- To compare two runs of the same flows file, run `npx @expo/explore compare:flows --before DIR --after DIR` with the two output directories. Each checkpoint is `unchanged`, `changed` (with regions), `missing`, `different-size`, `added` or `removed`, and `conditions` lists what differed between the runs (device, OS, locale, appearance). - The file is checked before a device starts (`FLOWS_PLAN_INVALID`, `FLOWS_PLAN_EMPTY`, `FLOWS_PLAN_OVER_BUDGET`). `--provider`, `--model` and `--hints` steer the model that carries out `do` steps, and are refused when there are none (`FLOWS_OPTIONS_IGNORED`). `--flows` and `--replay` cannot be combined. ## Sign-in and other app state diff --git a/src/cli.ts b/src/cli.ts index 41de0f2..312d76e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -11,7 +11,7 @@ import { help } from "./cli/help.js"; import { listScreens, getScreen, listElements, listObservations, getObservation, nextFrontier, listAttempts, getAttempt, listTransitions, getTransition, neighbors, pathQuery, listJourneys, getJourney, listFrontier, listGaps, summary, search, status, validate, exportArtifacts } from "./cli/map-commands.js"; import { compareCommand, exportWeb, view } from "./cli/catalog-commands.js"; import { ciReport, ciPublish, ciFailure, ciDeploy, ciPullRequest, ciFindBaseline, ciCompareRefs, ciAfterCapture, setup, doctor, sourceVerify, captureReview, captureBaseline, configPlatforms, configTargetList } from "./cli/ci-commands.js"; -import { exploreBinary, exploreSession } from "./cli/binary-commands.js"; +import { compareFlows, exploreBinary, exploreSession } from "./cli/binary-commands.js"; import { reports, reportUpload } from "./cli/report-commands.js"; const handlers: Record Promise | void> = { @@ -57,6 +57,7 @@ const handlers: Record Promise | void> = { "config:targets": configTargetList, doctor, compare: compareCommand, + "compare:flows": compareFlows, view, help, "--help": help, diff --git a/src/cli/args.ts b/src/cli/args.ts index 2c9e628..3625b34 100644 --- a/src/cli/args.ts +++ b/src/cli/args.ts @@ -19,6 +19,7 @@ const aliases: Record = { "frontier list": "frontier:list", "gap list": "gap:list", "export web": "export:web", + "compare flows": "compare:flows", "ci report": "ci:report", "ci publish": "ci:publish", "ci failure": "ci:failure", @@ -104,6 +105,7 @@ const flagKinds: Record> = { "ci:find-baseline": { sha:"value", target:"value", platform:"value" }, "ci:compare-refs": { project:"value", base:"value", head:"value", title:"value", output:"value" }, "ci:after-capture": { output:"value", checkout:"value" }, + "compare:flows": { before: "value", after: "value", output: "value" }, compare: { baseline: "value", current: "value", diff --git a/src/cli/binary-commands.ts b/src/cli/binary-commands.ts index a98c374..ebf18c8 100644 --- a/src/cli/binary-commands.ts +++ b/src/cli/binary-commands.ts @@ -1,4 +1,4 @@ -import { stat } from "node:fs/promises"; +import { readFile, stat, writeFile } from "node:fs/promises"; import { basename, join, resolve } from "node:path"; import { DEFAULT_BINARY_PROJECT, runBinaryExploration, runSessionExploration } from "../binary/run.js"; import { openViewerInBrowser, startViewerServer, waitForViewerShutdown } from "../map/viewer-server.js"; @@ -6,10 +6,14 @@ import { writeExplorationReport } from "../binary/report.js"; import { assertHostingExport, deployExplorationReport, zipReportDirectory } from "../report/preview-deploy.js"; import { keepReports } from "./report-commands.js"; import { warnIgnoredExpiry } from "../config/explore-config.js"; -import { argumentError, flag, flags, has, nonNegativeInteger, oneOf, output, positional, positiveInteger } from "./args.js"; +import { argumentError, flag, flags, has, nonNegativeInteger, oneOf, output, positional, positiveInteger, required } from "./args.js"; import { readReplayPlan } from "../binary/replay.js"; import { readFlowPlan } from "../binary/flows.js"; -import type { ExplorationFlow } from "../contracts/standalone-report.js"; +import { EXPLORATION_REPORT_KIND, type ExplorationFlow, type ExplorationReport } from "../contracts/standalone-report.js"; +import { compareFlowRuns, type FlowRun } from "../review/flow-comparison.js"; +import { diffScreenshots } from "../review/visual-evidence.js"; +import { agentDeviceLauncher, ensureController } from "../shared/controller-cache.js"; +import { CiError } from "../shared/ci-errors.js"; /** `--alias NAME` plus a fixed token from EXPLORE_PREVIEW_TOKEN: the hosted link stays the same from one run to the next (validated by the deploy). */ @@ -156,3 +160,32 @@ export async function exploreSession(): Promise { openViewerInBrowser(viewer.url); await waitForViewerShutdown(viewer); } + +/** A flows run's report: the report directory explore:binary or explore:session wrote, or the output directory holding it. */ +async function flowRun(path: string, name: "before" | "after"): Promise { + const refuse = (message: string) => { throw new CiError({ code: "FLOW_COMPARE_INPUT_INVALID", stage: "preflight", message, + nextAction: `Pass --${name} the output directory of an explore:binary or explore:session run that used --flows (or its report directory).` }); }; + for (const directory of [resolve(path), resolve(path, "report")]) { + let text: string; + try { text = await readFile(join(directory, "exploration.json"), "utf8"); } catch { continue; } + let report: ExplorationReport; + try { report = JSON.parse(text) as ExplorationReport; } catch { return refuse(`${join(directory, "exploration.json")} is not JSON.`); } + if (report?.kind !== EXPLORATION_REPORT_KIND) return refuse(`${join(directory, "exploration.json")} is not an exploration report.`); + if (!Array.isArray(report.exploration?.flows)) return refuse(`The run in ${directory} did not follow a flows file, so it has no checkpoints to compare.`); + return { report, directory }; + } + return refuse(`No exploration.json was found in ${resolve(path)} or its report folder.`); +} + +/** `compare:flows --before DIR --after DIR [--output FILE]`: two runs of one flows file, checkpoint by checkpoint. */ +export async function compareFlows(): Promise { + const [before, after] = await Promise.all([flowRun(required("before"), "before"), flowRun(required("after"), "after")]); + let command: string | undefined; + const comparison = await compareFlowRuns(before, after, async (beforePath, afterPath) => { + if (!command) { await ensureController("agent-device", { environment: process.env }); command = agentDeviceLauncher(process.env); } + return diffScreenshots({ baselinePath: beforePath, currentPath: afterPath, command }); + }); + const path = flag("output"); + if (path) await writeFile(resolve(path), JSON.stringify(comparison, null, 2)); + output({ ok: true, ...(path ? { output: resolve(path) } : {}), comparison }); +} diff --git a/src/cli/help.ts b/src/cli/help.ts index 461e369..c115206 100644 --- a/src/cli/help.ts +++ b/src/cli/help.ts @@ -84,6 +84,8 @@ const CI = `CI (the installed workflows run these; they manage remote EAS Simula capture:baseline --project DIRECTORY --sha COMMIT --output TEMP_DIRECTORY [--config FILE] [--target ID | --platform ios|android] [--controller agent-device|argent] [--simulator eas|local] [--repack eas|local] source:verify --base-project DIRECTORY --head-project DIRECTORY --base-sha COMMIT --head-sha COMMIT --output DIRECTORY [--config FILE] [--scope all|changes] [--target ID | --platform ios|android] + compare:flows --before DIRECTORY --after DIRECTORY [--output FILE] + (two runs of one --flows file, checkpoint by checkpoint: unchanged, changed with where, or why not compared) compare --baseline DIRECTORY --current DIRECTORY --output FILE [--review-output FILE] [--catalog-output FILE] [--catalog-name NAME] [--base-surface-coverage FILE] [--head-surface-coverage FILE] diff --git a/src/contracts/capture.ts b/src/contracts/capture.ts index 1181642..43aebf8 100644 --- a/src/contracts/capture.ts +++ b/src/contracts/capture.ts @@ -189,7 +189,7 @@ export const DUO_PANELS = { inner: { width: 951, height: 669, statusBar: { x: 883, y: 34, width: 40, height: 64 } }, } as const; const DUO_STATUS_BAR_PADDING_POINTS = 4; -const isFoldable = (platform: "ios" | "android" | undefined, device: string | undefined) => platform !== "android" && /\bduo\b/i.test(device ?? ""); +export const isFoldable = (platform: "ios" | "android" | undefined, device: string | undefined) => platform !== "android" && /\bduo\b/i.test(device ?? ""); /** * The areas the verdict leaves out for a capture: the platform's status-bar band and, on iOS, its home-indicator diff --git a/src/contracts/flow-comparison.ts b/src/contracts/flow-comparison.ts new file mode 100644 index 0000000..252e561 --- /dev/null +++ b/src/contracts/flow-comparison.ts @@ -0,0 +1,58 @@ +import type { ExplorationReport } from "./standalone-report.js"; + +export const FLOW_COMPARISON_KIND = "explore.flow-comparison" as const; + +/** One run's side of a checkpoint: its screenshot (a path inside that run's report directory), or why it has none. */ +export type CheckpointSide = { observationId: string; screenshot: string } | { missing: string }; + +/** + * A checkpoint of one flow in two runs, paired by flow name and checkpoint name. `before` or `after` is absent when + * that run's flows file has no such flow or checkpoint. + */ +export type CheckpointPair = { flow: string; checkpoint: string; before?: CheckpointSide; after?: CheckpointSide }; + +/** What a run was captured on: two runs that differ here can still be compared, but a difference may come from it. */ +export type RunConditions = { app: string; platform: string; device: string; osVersion?: string; locale?: string; orientation?: string; colorScheme?: string }; +export type RunConditionDifference = { field: keyof RunConditions; before: string; after: string }; + +export function runConditions(report: ExplorationReport): RunConditions { + const environment = report.map.run.environment ?? {}; + return { app: report.exploration.app.bundleId, platform: report.exploration.app.platform, device: report.exploration.device.model, + ...(report.exploration.device.osVersion ? { osVersion: report.exploration.device.osVersion } : {}), + ...(environment.locale ? { locale: environment.locale } : {}), ...(environment.orientation ? { orientation: environment.orientation } : {}), + ...(environment.colorScheme ? { colorScheme: environment.colorScheme } : {}) }; +} + +export function conditionDifferences(before: RunConditions, after: RunConditions): RunConditionDifference[] { + const fields = ["app", "platform", "device", "osVersion", "locale", "orientation", "colorScheme"] as const; + return fields.flatMap(field => (before[field] ?? "unknown") === (after[field] ?? "unknown") ? [] + : [{ field, before: before[field] ?? "unknown", after: after[field] ?? "unknown" }]); +} + +const keyOf = (name: string) => name.trim().toLowerCase(); + +function side(report: ExplorationReport, checkpoint: NonNullable[number]["checkpoints"][number]): CheckpointSide { + if (checkpoint.observationId === undefined) return { missing: checkpoint.missing }; + const observation = report.map.observations.find(item => item.id === checkpoint.observationId); + const artifact = observation?.screenshotArtifactId ? report.map.artifacts.find(item => item.id === observation.screenshotArtifactId) : undefined; + const asset = artifact ? report.assets.find(item => item.sha256 === artifact.sha256) : undefined; + return asset ? { observationId: checkpoint.observationId, screenshot: asset.path } : { missing: "The report has no screenshot for this checkpoint." }; +} + +/** + * Pairs the checkpoints of two runs of a flows file: the after run's flows and checkpoints in its order, then those only + * the before run has. Names match as the flows file compares them, ignoring case and surrounding space. + */ +export function pairCheckpoints(before: ExplorationReport, after: ExplorationReport): CheckpointPair[] { + const beforeFlows = before.exploration.flows ?? [], afterFlows = after.exploration.flows ?? []; + const flowNames = [...afterFlows, ...beforeFlows.filter(flow => !afterFlows.some(other => keyOf(other.name) === keyOf(flow.name)))].map(flow => flow.name); + return flowNames.flatMap(name => { + const was = beforeFlows.find(flow => keyOf(flow.name) === keyOf(name)), now = afterFlows.find(flow => keyOf(flow.name) === keyOf(name)); + const wasPoints = was?.checkpoints ?? [], nowPoints = now?.checkpoints ?? []; + const names = [...nowPoints, ...wasPoints.filter(point => !nowPoints.some(other => keyOf(other.name) === keyOf(point.name)))].map(point => point.name); + return names.map(checkpoint => { + const old = wasPoints.find(point => keyOf(point.name) === keyOf(checkpoint)), current = nowPoints.find(point => keyOf(point.name) === keyOf(checkpoint)); + return { flow: name, checkpoint, ...(old ? { before: side(before, old) } : {}), ...(current ? { after: side(after, current) } : {}) }; + }); + }); +} diff --git a/src/review/flow-comparison.ts b/src/review/flow-comparison.ts new file mode 100644 index 0000000..5135e50 --- /dev/null +++ b/src/review/flow-comparison.ts @@ -0,0 +1,79 @@ +import { open } from "node:fs/promises"; +import { resolve } from "node:path"; +import { isFoldable, systemRegions } from "../contracts/capture.js"; +import { FLOW_COMPARISON_KIND, conditionDifferences, pairCheckpoints, runConditions, type CheckpointPair, type RunConditionDifference, type RunConditions } from "../contracts/flow-comparison.js"; +import type { ExplorationReport } from "../contracts/standalone-report.js"; +import { ignoreSystemRegions, systemBandFrameLoader } from "./evidence-assessment.js"; +import type { ScreenshotDiffMeasurement } from "./visual-evidence.js"; + +/** + * `changed` and `unchanged` compare the two screenshots; `different-size` could not (another device or orientation); + * `missing` has no screenshot on one side; `added` and `removed` are in only one run's flows file. + */ +export type CheckpointVerdict = "unchanged" | "changed" | "different-size" | "missing" | "added" | "removed"; +export type ComparedCheckpoint = CheckpointPair & { + verdict: CheckpointVerdict; + /** Share of the screenshot that differs once the status bar and home indicator are set aside. */ + mismatchPercentage?: number; + /** Where the screenshots differ, in percent of the image (0–100). */ + regions?: Array<{ x: number; y: number; width: number; height: number }>; +}; + +export type FlowComparison = { + kind: typeof FLOW_COMPARISON_KIND; + before: RunConditions & { report: string }; + after: RunConditions & { report: string }; + /** How the runs were captured differently; a checkpoint that changed may owe its change to these. */ + conditions: RunConditionDifference[]; + notes: string[]; + counts: Record; + checkpoints: ComparedCheckpoint[]; +}; + +export type FlowRun = { report: ExplorationReport; directory: string }; +export type MeasureScreenshots = (beforePath: string, afterPath: string) => Promise; + +/** A PNG's pixel size, from its header. */ +async function pngSize(path: string): Promise<{ width: number; height: number }> { + const file = await open(path, "r"); + try { + const header = Buffer.alloc(24); + await file.read(header, 0, 24, 0); + return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) }; + } finally { await file.close(); } +} + +/** + * Compares two runs of the same flows file checkpoint by checkpoint, with the review's screenshot diff and the + * status-bar and home-indicator regions it sets aside. `measure` is only called for pairs with a screenshot on both + * sides of the same size. + */ +export async function compareFlowRuns(before: FlowRun, after: FlowRun, measure: MeasureScreenshots, threshold = 0.1): Promise { + const was = runConditions(before.report), now = runConditions(after.report); + const notes: string[] = []; + // A Duo's status bar sits on whichever panel was in use, and a run does not record the pose. + const foldable = isFoldable(before.report.exploration.app.platform, was.device); + if (foldable) notes.push("The status bar is not set aside: the run does not record which panel of the foldable was in use."); + const system = foldable ? [] : systemRegions({ platform: before.report.exploration.app.platform, device: was.device }); + const checkpoints: ComparedCheckpoint[] = []; + for (const pair of pairCheckpoints(before.report, after.report)) { + if (!pair.before) { checkpoints.push({ ...pair, verdict: "added" }); continue; } + if (!pair.after) { checkpoints.push({ ...pair, verdict: "removed" }); continue; } + if (!("screenshot" in pair.before) || !("screenshot" in pair.after)) { checkpoints.push({ ...pair, verdict: "missing" }); continue; } + const beforePath = resolve(before.directory, pair.before.screenshot), afterPath = resolve(after.directory, pair.after.screenshot); + const [a, b] = await Promise.all([pngSize(beforePath), pngSize(afterPath)]); + if (a.width !== b.width || a.height !== b.height) { checkpoints.push({ ...pair, verdict: "different-size" }); continue; } + const raw = await measure(beforePath, afterPath); + const { measurement } = ignoreSystemRegions(raw, system, systemBandFrameLoader(raw, threshold)); + const different = measurement.regions.reduce((sum, region) => sum + region.differentPixels, 0); + const mismatchPercentage = raw.totalPixels ? Math.round((different / raw.totalPixels) * 10000) / 100 : 0; + // As in a review, a mismatch that rounds to 0.00% is rendering noise. + const changed = measurement.regions.length > 0 && mismatchPercentage > 0; + checkpoints.push({ ...pair, verdict: changed ? "changed" : "unchanged", mismatchPercentage, + ...(changed ? { regions: measurement.regions.map(region => region.normalizedRect) } : {}) }); + } + const counts = { unchanged: 0, changed: 0, "different-size": 0, missing: 0, added: 0, removed: 0 }; + for (const item of checkpoints) counts[item.verdict] += 1; + return { kind: FLOW_COMPARISON_KIND, before: { ...was, report: before.directory }, after: { ...now, report: after.directory }, + conditions: conditionDifferences(was, now), notes, counts, checkpoints }; +} diff --git a/test/node/flow-comparison.test.mjs b/test/node/flow-comparison.test.mjs new file mode 100644 index 0000000..9ee7750 --- /dev/null +++ b/test/node/flow-comparison.test.mjs @@ -0,0 +1,115 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { tsImport } from "tsx/esm/api"; + +const { pairCheckpoints, conditionDifferences, runConditions } = await tsImport("../../src/contracts/flow-comparison.ts", import.meta.url); +const { compareFlowRuns } = await tsImport("../../src/review/flow-comparison.ts", import.meta.url); +const { PNG } = createRequire(import.meta.url)("pngjs"); + +const png = (width, height) => PNG.sync.write(new PNG({ width, height })); + +/** + * A minimal exploration report of a flows run, written to a directory. `flows` maps each flow name to its checkpoints, + * each `[name, size]` for a picture of that size or `[name, { missing }]` for one without. + */ +async function run(flows, options = {}) { + const directory = await mkdtemp(join(tmpdir(), "explore-flow-compare-")); + await mkdir(join(directory, "assets")); + const report = { kind: "explore.exploration-report", assets: [], + map: { run: { environment: { deviceModel: options.device ?? "iPhone 17 Pro", locale: options.locale ?? "en_US", orientation: "portrait", colorScheme: options.colorScheme ?? "light" } }, observations: [], artifacts: [] }, + exploration: { app: { name: "Example", bundleId: options.bundleId ?? "dev.example.app", platform: options.platform ?? "ios" }, device: { model: options.device ?? "iPhone 17 Pro", osVersion: options.osVersion ?? "26.0" }, flows: [] } }; + let index = 0; + for (const [name, checkpoints] of Object.entries(flows)) { + const flow = { name, status: "completed", checkpoints: [] }; + for (const [checkpoint, shot] of checkpoints) { + if (shot.missing) { flow.checkpoints.push({ name: checkpoint, missing: shot.missing }); continue; } + const sha = String(++index).padStart(64, "0"), id = `obs_${index}`; + await writeFile(join(directory, "assets", `${sha}.png`), png(...shot)); + report.assets.push({ sha256: sha, path: `assets/${sha}.png` }); + report.map.artifacts.push({ id: `art_${index}`, kind: "screenshot", sha256: sha }); + report.map.observations.push({ id, screenshotArtifactId: `art_${index}` }); + flow.checkpoints.push({ name: checkpoint, observationId: id }); + } + report.exploration.flows.push(flow); + } + return { report, directory }; +} + +/** A diff that finds the given regions (in percent), each 1% of the pixels. */ +const measuring = (regionsFor = () => []) => async (baselinePath, currentPath) => { + const regions = regionsFor(baselinePath, currentPath).map((normalizedRect, index) => ({ index, normalizedRect, rect: normalizedRect, differentPixels: 100, shareOfDiffPercentage: 0, densityPercentage: 0, shape: "", size: "" })); + return { baselinePath, currentPath, totalPixels: 10_000, differentPixels: regions.length * 100, mismatchPercentage: regions.length, match: regions.length === 0, regions }; +}; + +test("checkpoints pair by flow and checkpoint name, ignoring case, in the after run's order", async () => { + const before = await run({ "Sign in": [["Welcome", [4, 8]], ["Home", [4, 8]]], Settings: [["Settings", [4, 8]]] }); + const after = await run({ "sign in": [["home", [4, 8]], ["welcome", { missing: "The screenshot was blocked." }]], Profile: [["Profile", [4, 8]]] }); + const pairs = pairCheckpoints(before.report, after.report); + assert.deepEqual(pairs.map(pair => [pair.flow, pair.checkpoint, !!pair.before, !!pair.after]), [ + ["sign in", "home", true, true], ["sign in", "welcome", true, true], ["Profile", "Profile", false, true], ["Settings", "Settings", true, false]]); + assert.deepEqual(pairs[1].after, { missing: "The screenshot was blocked." }); + assert.match(pairs[0].before.screenshot, /^assets\/0+2\.png$/); +}); + +test("a run's conditions that differ are listed, not refused", async () => { + const before = await run({ A: [["Home", [4, 8]]] }), after = await run({ A: [["Home", [4, 8]]] }, { osVersion: "27.0", colorScheme: "dark" }); + assert.deepEqual(conditionDifferences(runConditions(before.report), runConditions(after.report)), + [{ field: "osVersion", before: "26.0", after: "27.0" }, { field: "colorScheme", before: "light", after: "dark" }]); +}); + +test("each pair gets a verdict: unchanged, changed with where, missing, different-size, added or removed", async () => { + const before = await run({ Flow: [["Same", [4, 8]], ["Moved", [4, 8]], ["Gone", [4, 8]], ["Blocked", [4, 8]], ["Rotated", [4, 8]]] }); + const after = await run({ Flow: [["Same", [4, 8]], ["Moved", [4, 8]], ["New", [4, 8]], ["Blocked", { missing: "A secret was on screen." }], ["Rotated", [8, 4]]] }); + const measured = []; + const result = await compareFlowRuns(before, after, measuring((baseline, current) => { + measured.push([baseline, current]); + return baseline.endsWith(before.report.assets[1].path) ? [{ x: 10, y: 40, width: 20, height: 10 }] : []; + })); + assert.deepEqual(result.checkpoints.map(item => [item.checkpoint, item.verdict]), + [["Same", "unchanged"], ["Moved", "changed"], ["New", "added"], ["Blocked", "missing"], ["Rotated", "different-size"], ["Gone", "removed"]]); + assert.equal(measured.length, 2, "only pairs with a picture of the same size on both sides are diffed"); + assert.deepEqual(result.checkpoints[1].regions, [{ x: 10, y: 40, width: 20, height: 10 }]); + assert.equal(result.checkpoints[1].mismatchPercentage, 1); + assert.deepEqual(result.counts, { unchanged: 1, changed: 1, "different-size": 1, missing: 1, added: 1, removed: 1 }); + assert.deepEqual(result.conditions, []); +}); + +test("a difference only in the status bar is set aside, as in a review", async () => { + const before = await run({ Flow: [["Home", [4, 8]]] }), after = await run({ Flow: [["Home", [4, 8]]] }); + const result = await compareFlowRuns(before, after, measuring(() => [{ x: 80, y: 1, width: 10, height: 2 }])); + assert.equal(result.checkpoints[0].verdict, "unchanged"); +}); + +test("on a foldable, the status bar is not set aside and the result says why", async () => { + const before = await run({ Flow: [["Home", [4, 8]]] }, { device: "iPhone Duo" }), after = await run({ Flow: [["Home", [4, 8]]] }, { device: "iPhone Duo" }); + // Where the folded panel's status bar is: guessing that panel would hide a change the open panel shows there. + const result = await compareFlowRuns(before, after, measuring(() => [{ x: 86, y: 13, width: 4, height: 4 }])); + assert.equal(result.checkpoints[0].verdict, "changed"); + assert.match(result.notes[0], /foldable/); +}); + +test("compare:flows reads two output directories, needs no diff when no pair has two pictures, and refuses a run without flows", async () => { + const { spawnSync } = await import("node:child_process"); + const root = join(import.meta.dirname, "../.."); + const cli = join(root, "dist", "cli.js"); + const before = await run({ Flow: [["Home", { missing: "Not reached." }]] }), after = await run({ Flow: [["Home", [4, 8]]] }); + // explore:binary leaves the report in /report. + const output = await mkdtemp(join(tmpdir(), "explore-flow-compare-out-")); + await mkdir(join(output, "report", "assets"), { recursive: true }); + await writeFile(join(output, "report", "exploration.json"), JSON.stringify(after.report)); + await writeFile(join(before.directory, "exploration.json"), JSON.stringify(before.report)); + const compared = spawnSync(process.execPath, [cli, "compare:flows", "--before", before.directory, "--after", output], { cwd: root, encoding: "utf8" }); + assert.equal(compared.status, 0, compared.stderr); + const result = JSON.parse(compared.stdout); + assert.deepEqual(result.comparison.checkpoints.map(item => [item.checkpoint, item.verdict]), [["Home", "missing"]]); + const plain = await run({}); + delete plain.report.exploration.flows; + await writeFile(join(plain.directory, "exploration.json"), JSON.stringify(plain.report)); + const refused = spawnSync(process.execPath, [cli, "compare:flows", "--before", plain.directory, "--after", output], { cwd: root, encoding: "utf8" }); + assert.notEqual(refused.status, 0); + assert.match(JSON.parse(refused.stderr).error, /did not follow a flows file/); +});