diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a1398a5..4718c60 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -46,7 +46,7 @@ Each layer imports only from the layers below it. 4. `capture` imports `capture/agent`, `config` and `map`. `capture/agent` imports `capture` and `config`. 5. `review` imports `capture` and `map`. 6. `ci` imports `config`, `source`, `capture`, `capture/agent`, `review` and `report`. `binary` imports `config`, `capture`, `capture/agent`, `map` and `report`. -7. `cli` imports `ci`, `binary`, `map`, `config` and `report`. `src/cli.ts` imports `cli`, `map` and `shared`. +7. `cli` imports `ci`, `binary`, `map`, `config`, `report` and `review`. `src/cli.ts` imports `cli`, `map` and `shared`. Every folder may import `contracts/`, `shared/` and `src/types.ts`. The exception is `contracts/`: its value imports stay inside `contracts/` or go to `src/types.ts`. diff --git a/README.md b/README.md index e160184..531c873 100644 --- a/README.md +++ b/README.md @@ -267,6 +267,14 @@ npx @expo/explore explore:binary ./my-app.app --flows flows.json - The report lists every flow and checkpoint of the file, in order, even those a run never reached: each one without a picture says why, so two runs still pair up step by step. - Each flow is also a journey of the map: the path its steps took, from the screen it started on. When the app moved without a tap the map could record (it left the app, or changed on its own), the path goes on as a second journey of the flow. - Relaunching ends the app's process, and the app keeps its data, so a flow that signs in leaves the next flow signed in. On `explore:session`, the app is relaunched before the first flow too. + +To compare two runs of the same flows file, such as two builds, pass both output directories: + +```sh +npx @expo/explore compare:flows --before explore-app-1 --after explore-app-2 --output flows.json +``` + +Each checkpoint comes back `unchanged`, `changed` (with where the screenshots differ, the status bar and home indicator set aside, as in a review), `missing` (no picture on one side, with why), `different-size` (another device or orientation, so not compared), or `added` / `removed` (in only one run's flows file). The result also lists how the two runs were captured differently (the app, device, OS version, locale, orientation, or appearance), since a change can come from those. - A `{{secret:NAME}}` or a described step limited by `platforms` to another platform needs no value, and no model, on this one. diff --git a/skills/explore-binary/SKILL.md b/skills/explore-binary/SKILL.md index a2939c5..1572fab 100644 --- a/skills/explore-binary/SKILL.md +++ b/skills/explore-binary/SKILL.md @@ -42,6 +42,7 @@ To compare runs, or an iPhone with an Android phone, screen for screen, pass `-- When the user describes a flow, or a set of flows, to look at ("sign in, then rename a pattern"), write them as a flows file and pass `--flows flows.json`, not `--replay`. The format is in the README ("Follow named flows, step by step"): `{ "flows": [{ "name": "Sign in", "steps": [{ "checkpoint": "Welcome" }, { "do": "Sign in with {{secret:TEST_EMAIL}}" }, { "checkpoint": "Home" }] }] }`. - Each step is an exact action (the `--replay` format), a step described in words (`do`) that a model carries out, or a named `checkpoint`. Use exact actions where you know the labels, and `do` where you don't. - Each flow starts from a relaunched app, and its checkpoints are photographed under their names. `report/exploration.json` lists each flow under `exploration.flows`, with its status and its checkpoints in order, so two runs pair up by flow and checkpoint name. A checkpoint on a screen that withholds its picture (a secret on it) is listed with why it has none, and its flow goes on. +- To compare two runs of the same flows file, run `npx @expo/explore compare:flows --before DIR --after DIR` with the two output directories. Each checkpoint is `unchanged`, `changed` (with regions), `missing`, `different-size`, `added` or `removed`, and `conditions` lists what differed between the runs (device, OS, locale, appearance). - The file is checked before a device starts (`FLOWS_PLAN_INVALID`, `FLOWS_PLAN_EMPTY`, `FLOWS_PLAN_OVER_BUDGET`). `--provider`, `--model` and `--hints` steer the model that carries out `do` steps, and are refused when there are none (`FLOWS_OPTIONS_IGNORED`). `--flows` and `--replay` cannot be combined. ## Sign-in and other app state diff --git a/src/cli.ts b/src/cli.ts index 41de0f2..312d76e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -11,7 +11,7 @@ import { help } from "./cli/help.js"; import { listScreens, getScreen, listElements, listObservations, getObservation, nextFrontier, listAttempts, getAttempt, listTransitions, getTransition, neighbors, pathQuery, listJourneys, getJourney, listFrontier, listGaps, summary, search, status, validate, exportArtifacts } from "./cli/map-commands.js"; import { compareCommand, exportWeb, view } from "./cli/catalog-commands.js"; import { ciReport, ciPublish, ciFailure, ciDeploy, ciPullRequest, ciFindBaseline, ciCompareRefs, ciAfterCapture, setup, doctor, sourceVerify, captureReview, captureBaseline, configPlatforms, configTargetList } from "./cli/ci-commands.js"; -import { exploreBinary, exploreSession } from "./cli/binary-commands.js"; +import { compareFlows, exploreBinary, exploreSession } from "./cli/binary-commands.js"; import { reports, reportUpload } from "./cli/report-commands.js"; const handlers: Record Promise | void> = { @@ -57,6 +57,7 @@ const handlers: Record Promise | void> = { "config:targets": configTargetList, doctor, compare: compareCommand, + "compare:flows": compareFlows, view, help, "--help": help, diff --git a/src/cli/args.ts b/src/cli/args.ts index 2c9e628..3625b34 100644 --- a/src/cli/args.ts +++ b/src/cli/args.ts @@ -19,6 +19,7 @@ const aliases: Record = { "frontier list": "frontier:list", "gap list": "gap:list", "export web": "export:web", + "compare flows": "compare:flows", "ci report": "ci:report", "ci publish": "ci:publish", "ci failure": "ci:failure", @@ -104,6 +105,7 @@ const flagKinds: Record> = { "ci:find-baseline": { sha:"value", target:"value", platform:"value" }, "ci:compare-refs": { project:"value", base:"value", head:"value", title:"value", output:"value" }, "ci:after-capture": { output:"value", checkout:"value" }, + "compare:flows": { before: "value", after: "value", output: "value" }, compare: { baseline: "value", current: "value", diff --git a/src/cli/binary-commands.ts b/src/cli/binary-commands.ts index a98c374..ebf18c8 100644 --- a/src/cli/binary-commands.ts +++ b/src/cli/binary-commands.ts @@ -1,4 +1,4 @@ -import { stat } from "node:fs/promises"; +import { readFile, stat, writeFile } from "node:fs/promises"; import { basename, join, resolve } from "node:path"; import { DEFAULT_BINARY_PROJECT, runBinaryExploration, runSessionExploration } from "../binary/run.js"; import { openViewerInBrowser, startViewerServer, waitForViewerShutdown } from "../map/viewer-server.js"; @@ -6,10 +6,14 @@ import { writeExplorationReport } from "../binary/report.js"; import { assertHostingExport, deployExplorationReport, zipReportDirectory } from "../report/preview-deploy.js"; import { keepReports } from "./report-commands.js"; import { warnIgnoredExpiry } from "../config/explore-config.js"; -import { argumentError, flag, flags, has, nonNegativeInteger, oneOf, output, positional, positiveInteger } from "./args.js"; +import { argumentError, flag, flags, has, nonNegativeInteger, oneOf, output, positional, positiveInteger, required } from "./args.js"; import { readReplayPlan } from "../binary/replay.js"; import { readFlowPlan } from "../binary/flows.js"; -import type { ExplorationFlow } from "../contracts/standalone-report.js"; +import { EXPLORATION_REPORT_KIND, type ExplorationFlow, type ExplorationReport } from "../contracts/standalone-report.js"; +import { compareFlowRuns, type FlowRun } from "../review/flow-comparison.js"; +import { diffScreenshots } from "../review/visual-evidence.js"; +import { agentDeviceLauncher, ensureController } from "../shared/controller-cache.js"; +import { CiError } from "../shared/ci-errors.js"; /** `--alias NAME` plus a fixed token from EXPLORE_PREVIEW_TOKEN: the hosted link stays the same from one run to the next (validated by the deploy). */ @@ -156,3 +160,32 @@ export async function exploreSession(): Promise { openViewerInBrowser(viewer.url); await waitForViewerShutdown(viewer); } + +/** A flows run's report: the report directory explore:binary or explore:session wrote, or the output directory holding it. */ +async function flowRun(path: string, name: "before" | "after"): Promise { + const refuse = (message: string) => { throw new CiError({ code: "FLOW_COMPARE_INPUT_INVALID", stage: "preflight", message, + nextAction: `Pass --${name} the output directory of an explore:binary or explore:session run that used --flows (or its report directory).` }); }; + for (const directory of [resolve(path), resolve(path, "report")]) { + let text: string; + try { text = await readFile(join(directory, "exploration.json"), "utf8"); } catch { continue; } + let report: ExplorationReport; + try { report = JSON.parse(text) as ExplorationReport; } catch { return refuse(`${join(directory, "exploration.json")} is not JSON.`); } + if (report?.kind !== EXPLORATION_REPORT_KIND) return refuse(`${join(directory, "exploration.json")} is not an exploration report.`); + if (!Array.isArray(report.exploration?.flows)) return refuse(`The run in ${directory} did not follow a flows file, so it has no checkpoints to compare.`); + return { report, directory }; + } + return refuse(`No exploration.json was found in ${resolve(path)} or its report folder.`); +} + +/** `compare:flows --before DIR --after DIR [--output FILE]`: two runs of one flows file, checkpoint by checkpoint. */ +export async function compareFlows(): Promise { + const [before, after] = await Promise.all([flowRun(required("before"), "before"), flowRun(required("after"), "after")]); + let command: string | undefined; + const comparison = await compareFlowRuns(before, after, async (beforePath, afterPath) => { + if (!command) { await ensureController("agent-device", { environment: process.env }); command = agentDeviceLauncher(process.env); } + return diffScreenshots({ baselinePath: beforePath, currentPath: afterPath, command }); + }); + const path = flag("output"); + if (path) await writeFile(resolve(path), JSON.stringify(comparison, null, 2)); + output({ ok: true, ...(path ? { output: resolve(path) } : {}), comparison }); +} diff --git a/src/cli/help.ts b/src/cli/help.ts index 461e369..c115206 100644 --- a/src/cli/help.ts +++ b/src/cli/help.ts @@ -84,6 +84,8 @@ const CI = `CI (the installed workflows run these; they manage remote EAS Simula capture:baseline --project DIRECTORY --sha COMMIT --output TEMP_DIRECTORY [--config FILE] [--target ID | --platform ios|android] [--controller agent-device|argent] [--simulator eas|local] [--repack eas|local] source:verify --base-project DIRECTORY --head-project DIRECTORY --base-sha COMMIT --head-sha COMMIT --output DIRECTORY [--config FILE] [--scope all|changes] [--target ID | --platform ios|android] + compare:flows --before DIRECTORY --after DIRECTORY [--output FILE] + (two runs of one --flows file, checkpoint by checkpoint: unchanged, changed with where, or why not compared) compare --baseline DIRECTORY --current DIRECTORY --output FILE [--review-output FILE] [--catalog-output FILE] [--catalog-name NAME] [--base-surface-coverage FILE] [--head-surface-coverage FILE] diff --git a/src/contracts/capture.ts b/src/contracts/capture.ts index 1181642..43aebf8 100644 --- a/src/contracts/capture.ts +++ b/src/contracts/capture.ts @@ -189,7 +189,7 @@ export const DUO_PANELS = { inner: { width: 951, height: 669, statusBar: { x: 883, y: 34, width: 40, height: 64 } }, } as const; const DUO_STATUS_BAR_PADDING_POINTS = 4; -const isFoldable = (platform: "ios" | "android" | undefined, device: string | undefined) => platform !== "android" && /\bduo\b/i.test(device ?? ""); +export const isFoldable = (platform: "ios" | "android" | undefined, device: string | undefined) => platform !== "android" && /\bduo\b/i.test(device ?? ""); /** * The areas the verdict leaves out for a capture: the platform's status-bar band and, on iOS, its home-indicator diff --git a/src/contracts/flow-comparison.ts b/src/contracts/flow-comparison.ts new file mode 100644 index 0000000..252e561 --- /dev/null +++ b/src/contracts/flow-comparison.ts @@ -0,0 +1,58 @@ +import type { ExplorationReport } from "./standalone-report.js"; + +export const FLOW_COMPARISON_KIND = "explore.flow-comparison" as const; + +/** One run's side of a checkpoint: its screenshot (a path inside that run's report directory), or why it has none. */ +export type CheckpointSide = { observationId: string; screenshot: string } | { missing: string }; + +/** + * A checkpoint of one flow in two runs, paired by flow name and checkpoint name. `before` or `after` is absent when + * that run's flows file has no such flow or checkpoint. + */ +export type CheckpointPair = { flow: string; checkpoint: string; before?: CheckpointSide; after?: CheckpointSide }; + +/** What a run was captured on: two runs that differ here can still be compared, but a difference may come from it. */ +export type RunConditions = { app: string; platform: string; device: string; osVersion?: string; locale?: string; orientation?: string; colorScheme?: string }; +export type RunConditionDifference = { field: keyof RunConditions; before: string; after: string }; + +export function runConditions(report: ExplorationReport): RunConditions { + const environment = report.map.run.environment ?? {}; + return { app: report.exploration.app.bundleId, platform: report.exploration.app.platform, device: report.exploration.device.model, + ...(report.exploration.device.osVersion ? { osVersion: report.exploration.device.osVersion } : {}), + ...(environment.locale ? { locale: environment.locale } : {}), ...(environment.orientation ? { orientation: environment.orientation } : {}), + ...(environment.colorScheme ? { colorScheme: environment.colorScheme } : {}) }; +} + +export function conditionDifferences(before: RunConditions, after: RunConditions): RunConditionDifference[] { + const fields = ["app", "platform", "device", "osVersion", "locale", "orientation", "colorScheme"] as const; + return fields.flatMap(field => (before[field] ?? "unknown") === (after[field] ?? "unknown") ? [] + : [{ field, before: before[field] ?? "unknown", after: after[field] ?? "unknown" }]); +} + +const keyOf = (name: string) => name.trim().toLowerCase(); + +function side(report: ExplorationReport, checkpoint: NonNullable[number]["checkpoints"][number]): CheckpointSide { + if (checkpoint.observationId === undefined) return { missing: checkpoint.missing }; + const observation = report.map.observations.find(item => item.id === checkpoint.observationId); + const artifact = observation?.screenshotArtifactId ? report.map.artifacts.find(item => item.id === observation.screenshotArtifactId) : undefined; + const asset = artifact ? report.assets.find(item => item.sha256 === artifact.sha256) : undefined; + return asset ? { observationId: checkpoint.observationId, screenshot: asset.path } : { missing: "The report has no screenshot for this checkpoint." }; +} + +/** + * Pairs the checkpoints of two runs of a flows file: the after run's flows and checkpoints in its order, then those only + * the before run has. Names match as the flows file compares them, ignoring case and surrounding space. + */ +export function pairCheckpoints(before: ExplorationReport, after: ExplorationReport): CheckpointPair[] { + const beforeFlows = before.exploration.flows ?? [], afterFlows = after.exploration.flows ?? []; + const flowNames = [...afterFlows, ...beforeFlows.filter(flow => !afterFlows.some(other => keyOf(other.name) === keyOf(flow.name)))].map(flow => flow.name); + return flowNames.flatMap(name => { + const was = beforeFlows.find(flow => keyOf(flow.name) === keyOf(name)), now = afterFlows.find(flow => keyOf(flow.name) === keyOf(name)); + const wasPoints = was?.checkpoints ?? [], nowPoints = now?.checkpoints ?? []; + const names = [...nowPoints, ...wasPoints.filter(point => !nowPoints.some(other => keyOf(other.name) === keyOf(point.name)))].map(point => point.name); + return names.map(checkpoint => { + const old = wasPoints.find(point => keyOf(point.name) === keyOf(checkpoint)), current = nowPoints.find(point => keyOf(point.name) === keyOf(checkpoint)); + return { flow: name, checkpoint, ...(old ? { before: side(before, old) } : {}), ...(current ? { after: side(after, current) } : {}) }; + }); + }); +} diff --git a/src/review/flow-comparison.ts b/src/review/flow-comparison.ts new file mode 100644 index 0000000..5135e50 --- /dev/null +++ b/src/review/flow-comparison.ts @@ -0,0 +1,79 @@ +import { open } from "node:fs/promises"; +import { resolve } from "node:path"; +import { isFoldable, systemRegions } from "../contracts/capture.js"; +import { FLOW_COMPARISON_KIND, conditionDifferences, pairCheckpoints, runConditions, type CheckpointPair, type RunConditionDifference, type RunConditions } from "../contracts/flow-comparison.js"; +import type { ExplorationReport } from "../contracts/standalone-report.js"; +import { ignoreSystemRegions, systemBandFrameLoader } from "./evidence-assessment.js"; +import type { ScreenshotDiffMeasurement } from "./visual-evidence.js"; + +/** + * `changed` and `unchanged` compare the two screenshots; `different-size` could not (another device or orientation); + * `missing` has no screenshot on one side; `added` and `removed` are in only one run's flows file. + */ +export type CheckpointVerdict = "unchanged" | "changed" | "different-size" | "missing" | "added" | "removed"; +export type ComparedCheckpoint = CheckpointPair & { + verdict: CheckpointVerdict; + /** Share of the screenshot that differs once the status bar and home indicator are set aside. */ + mismatchPercentage?: number; + /** Where the screenshots differ, in percent of the image (0–100). */ + regions?: Array<{ x: number; y: number; width: number; height: number }>; +}; + +export type FlowComparison = { + kind: typeof FLOW_COMPARISON_KIND; + before: RunConditions & { report: string }; + after: RunConditions & { report: string }; + /** How the runs were captured differently; a checkpoint that changed may owe its change to these. */ + conditions: RunConditionDifference[]; + notes: string[]; + counts: Record; + checkpoints: ComparedCheckpoint[]; +}; + +export type FlowRun = { report: ExplorationReport; directory: string }; +export type MeasureScreenshots = (beforePath: string, afterPath: string) => Promise; + +/** A PNG's pixel size, from its header. */ +async function pngSize(path: string): Promise<{ width: number; height: number }> { + const file = await open(path, "r"); + try { + const header = Buffer.alloc(24); + await file.read(header, 0, 24, 0); + return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) }; + } finally { await file.close(); } +} + +/** + * Compares two runs of the same flows file checkpoint by checkpoint, with the review's screenshot diff and the + * status-bar and home-indicator regions it sets aside. `measure` is only called for pairs with a screenshot on both + * sides of the same size. + */ +export async function compareFlowRuns(before: FlowRun, after: FlowRun, measure: MeasureScreenshots, threshold = 0.1): Promise { + const was = runConditions(before.report), now = runConditions(after.report); + const notes: string[] = []; + // A Duo's status bar sits on whichever panel was in use, and a run does not record the pose. + const foldable = isFoldable(before.report.exploration.app.platform, was.device); + if (foldable) notes.push("The status bar is not set aside: the run does not record which panel of the foldable was in use."); + const system = foldable ? [] : systemRegions({ platform: before.report.exploration.app.platform, device: was.device }); + const checkpoints: ComparedCheckpoint[] = []; + for (const pair of pairCheckpoints(before.report, after.report)) { + if (!pair.before) { checkpoints.push({ ...pair, verdict: "added" }); continue; } + if (!pair.after) { checkpoints.push({ ...pair, verdict: "removed" }); continue; } + if (!("screenshot" in pair.before) || !("screenshot" in pair.after)) { checkpoints.push({ ...pair, verdict: "missing" }); continue; } + const beforePath = resolve(before.directory, pair.before.screenshot), afterPath = resolve(after.directory, pair.after.screenshot); + const [a, b] = await Promise.all([pngSize(beforePath), pngSize(afterPath)]); + if (a.width !== b.width || a.height !== b.height) { checkpoints.push({ ...pair, verdict: "different-size" }); continue; } + const raw = await measure(beforePath, afterPath); + const { measurement } = ignoreSystemRegions(raw, system, systemBandFrameLoader(raw, threshold)); + const different = measurement.regions.reduce((sum, region) => sum + region.differentPixels, 0); + const mismatchPercentage = raw.totalPixels ? Math.round((different / raw.totalPixels) * 10000) / 100 : 0; + // As in a review, a mismatch that rounds to 0.00% is rendering noise. + const changed = measurement.regions.length > 0 && mismatchPercentage > 0; + checkpoints.push({ ...pair, verdict: changed ? "changed" : "unchanged", mismatchPercentage, + ...(changed ? { regions: measurement.regions.map(region => region.normalizedRect) } : {}) }); + } + const counts = { unchanged: 0, changed: 0, "different-size": 0, missing: 0, added: 0, removed: 0 }; + for (const item of checkpoints) counts[item.verdict] += 1; + return { kind: FLOW_COMPARISON_KIND, before: { ...was, report: before.directory }, after: { ...now, report: after.directory }, + conditions: conditionDifferences(was, now), notes, counts, checkpoints }; +} diff --git a/test/node/flow-comparison.test.mjs b/test/node/flow-comparison.test.mjs new file mode 100644 index 0000000..9ee7750 --- /dev/null +++ b/test/node/flow-comparison.test.mjs @@ -0,0 +1,115 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { tsImport } from "tsx/esm/api"; + +const { pairCheckpoints, conditionDifferences, runConditions } = await tsImport("../../src/contracts/flow-comparison.ts", import.meta.url); +const { compareFlowRuns } = await tsImport("../../src/review/flow-comparison.ts", import.meta.url); +const { PNG } = createRequire(import.meta.url)("pngjs"); + +const png = (width, height) => PNG.sync.write(new PNG({ width, height })); + +/** + * A minimal exploration report of a flows run, written to a directory. `flows` maps each flow name to its checkpoints, + * each `[name, size]` for a picture of that size or `[name, { missing }]` for one without. + */ +async function run(flows, options = {}) { + const directory = await mkdtemp(join(tmpdir(), "explore-flow-compare-")); + await mkdir(join(directory, "assets")); + const report = { kind: "explore.exploration-report", assets: [], + map: { run: { environment: { deviceModel: options.device ?? "iPhone 17 Pro", locale: options.locale ?? "en_US", orientation: "portrait", colorScheme: options.colorScheme ?? "light" } }, observations: [], artifacts: [] }, + exploration: { app: { name: "Example", bundleId: options.bundleId ?? "dev.example.app", platform: options.platform ?? "ios" }, device: { model: options.device ?? "iPhone 17 Pro", osVersion: options.osVersion ?? "26.0" }, flows: [] } }; + let index = 0; + for (const [name, checkpoints] of Object.entries(flows)) { + const flow = { name, status: "completed", checkpoints: [] }; + for (const [checkpoint, shot] of checkpoints) { + if (shot.missing) { flow.checkpoints.push({ name: checkpoint, missing: shot.missing }); continue; } + const sha = String(++index).padStart(64, "0"), id = `obs_${index}`; + await writeFile(join(directory, "assets", `${sha}.png`), png(...shot)); + report.assets.push({ sha256: sha, path: `assets/${sha}.png` }); + report.map.artifacts.push({ id: `art_${index}`, kind: "screenshot", sha256: sha }); + report.map.observations.push({ id, screenshotArtifactId: `art_${index}` }); + flow.checkpoints.push({ name: checkpoint, observationId: id }); + } + report.exploration.flows.push(flow); + } + return { report, directory }; +} + +/** A diff that finds the given regions (in percent), each 1% of the pixels. */ +const measuring = (regionsFor = () => []) => async (baselinePath, currentPath) => { + const regions = regionsFor(baselinePath, currentPath).map((normalizedRect, index) => ({ index, normalizedRect, rect: normalizedRect, differentPixels: 100, shareOfDiffPercentage: 0, densityPercentage: 0, shape: "", size: "" })); + return { baselinePath, currentPath, totalPixels: 10_000, differentPixels: regions.length * 100, mismatchPercentage: regions.length, match: regions.length === 0, regions }; +}; + +test("checkpoints pair by flow and checkpoint name, ignoring case, in the after run's order", async () => { + const before = await run({ "Sign in": [["Welcome", [4, 8]], ["Home", [4, 8]]], Settings: [["Settings", [4, 8]]] }); + const after = await run({ "sign in": [["home", [4, 8]], ["welcome", { missing: "The screenshot was blocked." }]], Profile: [["Profile", [4, 8]]] }); + const pairs = pairCheckpoints(before.report, after.report); + assert.deepEqual(pairs.map(pair => [pair.flow, pair.checkpoint, !!pair.before, !!pair.after]), [ + ["sign in", "home", true, true], ["sign in", "welcome", true, true], ["Profile", "Profile", false, true], ["Settings", "Settings", true, false]]); + assert.deepEqual(pairs[1].after, { missing: "The screenshot was blocked." }); + assert.match(pairs[0].before.screenshot, /^assets\/0+2\.png$/); +}); + +test("a run's conditions that differ are listed, not refused", async () => { + const before = await run({ A: [["Home", [4, 8]]] }), after = await run({ A: [["Home", [4, 8]]] }, { osVersion: "27.0", colorScheme: "dark" }); + assert.deepEqual(conditionDifferences(runConditions(before.report), runConditions(after.report)), + [{ field: "osVersion", before: "26.0", after: "27.0" }, { field: "colorScheme", before: "light", after: "dark" }]); +}); + +test("each pair gets a verdict: unchanged, changed with where, missing, different-size, added or removed", async () => { + const before = await run({ Flow: [["Same", [4, 8]], ["Moved", [4, 8]], ["Gone", [4, 8]], ["Blocked", [4, 8]], ["Rotated", [4, 8]]] }); + const after = await run({ Flow: [["Same", [4, 8]], ["Moved", [4, 8]], ["New", [4, 8]], ["Blocked", { missing: "A secret was on screen." }], ["Rotated", [8, 4]]] }); + const measured = []; + const result = await compareFlowRuns(before, after, measuring((baseline, current) => { + measured.push([baseline, current]); + return baseline.endsWith(before.report.assets[1].path) ? [{ x: 10, y: 40, width: 20, height: 10 }] : []; + })); + assert.deepEqual(result.checkpoints.map(item => [item.checkpoint, item.verdict]), + [["Same", "unchanged"], ["Moved", "changed"], ["New", "added"], ["Blocked", "missing"], ["Rotated", "different-size"], ["Gone", "removed"]]); + assert.equal(measured.length, 2, "only pairs with a picture of the same size on both sides are diffed"); + assert.deepEqual(result.checkpoints[1].regions, [{ x: 10, y: 40, width: 20, height: 10 }]); + assert.equal(result.checkpoints[1].mismatchPercentage, 1); + assert.deepEqual(result.counts, { unchanged: 1, changed: 1, "different-size": 1, missing: 1, added: 1, removed: 1 }); + assert.deepEqual(result.conditions, []); +}); + +test("a difference only in the status bar is set aside, as in a review", async () => { + const before = await run({ Flow: [["Home", [4, 8]]] }), after = await run({ Flow: [["Home", [4, 8]]] }); + const result = await compareFlowRuns(before, after, measuring(() => [{ x: 80, y: 1, width: 10, height: 2 }])); + assert.equal(result.checkpoints[0].verdict, "unchanged"); +}); + +test("on a foldable, the status bar is not set aside and the result says why", async () => { + const before = await run({ Flow: [["Home", [4, 8]]] }, { device: "iPhone Duo" }), after = await run({ Flow: [["Home", [4, 8]]] }, { device: "iPhone Duo" }); + // Where the folded panel's status bar is: guessing that panel would hide a change the open panel shows there. + const result = await compareFlowRuns(before, after, measuring(() => [{ x: 86, y: 13, width: 4, height: 4 }])); + assert.equal(result.checkpoints[0].verdict, "changed"); + assert.match(result.notes[0], /foldable/); +}); + +test("compare:flows reads two output directories, needs no diff when no pair has two pictures, and refuses a run without flows", async () => { + const { spawnSync } = await import("node:child_process"); + const root = join(import.meta.dirname, "../.."); + const cli = join(root, "dist", "cli.js"); + const before = await run({ Flow: [["Home", { missing: "Not reached." }]] }), after = await run({ Flow: [["Home", [4, 8]]] }); + // explore:binary leaves the report in /report. + const output = await mkdtemp(join(tmpdir(), "explore-flow-compare-out-")); + await mkdir(join(output, "report", "assets"), { recursive: true }); + await writeFile(join(output, "report", "exploration.json"), JSON.stringify(after.report)); + await writeFile(join(before.directory, "exploration.json"), JSON.stringify(before.report)); + const compared = spawnSync(process.execPath, [cli, "compare:flows", "--before", before.directory, "--after", output], { cwd: root, encoding: "utf8" }); + assert.equal(compared.status, 0, compared.stderr); + const result = JSON.parse(compared.stdout); + assert.deepEqual(result.comparison.checkpoints.map(item => [item.checkpoint, item.verdict]), [["Home", "missing"]]); + const plain = await run({}); + delete plain.report.exploration.flows; + await writeFile(join(plain.directory, "exploration.json"), JSON.stringify(plain.report)); + const refused = spawnSync(process.execPath, [cli, "compare:flows", "--before", plain.directory, "--after", output], { cwd: root, encoding: "utf8" }); + assert.notEqual(refused.status, 0); + assert.match(JSON.parse(refused.stderr).error, /did not follow a flows file/); +});