Flows: compare two runs of a flows file checkpoint by checkpoint (compare:flows)

ARCHITECTURE.md

@@ -46,7 +46,7 @@ Each layer imports only from the layers below it.
 4. `capture` imports `capture/agent`, `config` and `map`. `capture/agent` imports `capture` and `config`.
 5. `review` imports `capture` and `map`.
 6. `ci` imports `config`, `source`, `capture`, `capture/agent`, `review` and `report`. `binary` imports `config`, `capture`, `capture/agent`, `map` and `report`.
-7. `cli` imports `ci`, `binary`, `map`, `config` and `report`. `src/cli.ts` imports `cli`, `map` and `shared`.
+7. `cli` imports `ci`, `binary`, `map`, `config`, `report` and `review`. `src/cli.ts` imports `cli`, `map` and `shared`.
 
 Every folder may import `contracts/`, `shared/` and `src/types.ts`.
 The exception is `contracts/`: its value imports stay inside `contracts/` or go to `src/types.ts`.

README.md

@@ -267,6 +267,14 @@ npx @expo/explore explore:binary ./my-app.app --flows flows.json
 - The report lists every flow and checkpoint of the file, in order, even those a run never reached: each one without a picture says why, so two runs still pair up step by step.
 - Each flow is also a journey of the map: the path its steps took, from the screen it started on. When the app moved without a tap the map could record (it left the app, or changed on its own), the path goes on as a second journey of the flow.
 - Relaunching ends the app's process, and the app keeps its data, so a flow that signs in leaves the next flow signed in. On `explore:session`, the app is relaunched before the first flow too.
+
+To compare two runs of the same flows file, such as two builds, pass both output directories:
+
+```sh
+npx @expo/explore compare:flows --before explore-app-1 --after explore-app-2 --output flows.json
+```
+
+Each checkpoint comes back `unchanged`, `changed` (with where the screenshots differ, the status bar and home indicator set aside, as in a review), `missing` (no picture on one side, with why), `different-size` (another device or orientation, so not compared), or `added` / `removed` (in only one run's flows file). The result also lists how the two runs were captured differently (the app, device, OS version, locale, orientation, or appearance), since a change can come from those.
 - A `{{secret:NAME}}` or a described step limited by `platforms` to another platform needs no value, and no model, on this one.
 
 </details>

skills/explore-binary/SKILL.md

@@ -42,6 +42,7 @@ To compare runs, or an iPhone with an Android phone, screen for screen, pass `--
 When the user describes a flow, or a set of flows, to look at ("sign in, then rename a pattern"), write them as a flows file and pass `--flows flows.json`, not `--replay`. The format is in the README ("Follow named flows, step by step"): `{ "flows": [{ "name": "Sign in", "steps": [{ "checkpoint": "Welcome" }, { "do": "Sign in with {{secret:TEST_EMAIL}}" }, { "checkpoint": "Home" }] }] }`.
 - Each step is an exact action (the `--replay` format), a step described in words (`do`) that a model carries out, or a named `checkpoint`. Use exact actions where you know the labels, and `do` where you don't.
 - Each flow starts from a relaunched app, and its checkpoints are photographed under their names. `report/exploration.json` lists each flow under `exploration.flows`, with its status and its checkpoints in order, so two runs pair up by flow and checkpoint name. A checkpoint on a screen that withholds its picture (a secret on it) is listed with why it has none, and its flow goes on.
+- To compare two runs of the same flows file, run `npx @expo/explore compare:flows --before DIR --after DIR` with the two output directories. Each checkpoint is `unchanged`, `changed` (with regions), `missing`, `different-size`, `added` or `removed`, and `conditions` lists what differed between the runs (device, OS, locale, appearance).
 - The file is checked before a device starts (`FLOWS_PLAN_INVALID`, `FLOWS_PLAN_EMPTY`, `FLOWS_PLAN_OVER_BUDGET`). `--provider`, `--model` and `--hints` steer the model that carries out `do` steps, and are refused when there are none (`FLOWS_OPTIONS_IGNORED`). `--flows` and `--replay` cannot be combined.
 
 ## Sign-in and other app state

src/cli.ts

@@ -11,7 +11,7 @@ import { help } from "./cli/help.js";
 import { listScreens, getScreen, listElements, listObservations, getObservation, nextFrontier, listAttempts, getAttempt, listTransitions, getTransition, neighbors, pathQuery, listJourneys, getJourney, listFrontier, listGaps, summary, search, status, validate, exportArtifacts } from "./cli/map-commands.js";
 import { compareCommand, exportWeb, view } from "./cli/catalog-commands.js";
 import { ciReport, ciPublish, ciFailure, ciDeploy, ciPullRequest, ciFindBaseline, ciCompareRefs, ciAfterCapture, setup, doctor, sourceVerify, captureReview, captureBaseline, configPlatforms, configTargetList } from "./cli/ci-commands.js";
-import { exploreBinary, exploreSession } from "./cli/binary-commands.js";
+import { compareFlows, exploreBinary, exploreSession } from "./cli/binary-commands.js";
 import { reports, reportUpload } from "./cli/report-commands.js";
 
 const handlers: Record<string, () => Promise<void> | void> = {
@@ -57,6 +57,7 @@ const handlers: Record<string, () => Promise<void> | void> = {
   "config:targets": configTargetList,
   doctor,
   compare: compareCommand,
+  "compare:flows": compareFlows,
   view,
   help,
   "--help": help,

src/cli/args.ts

@@ -19,6 +19,7 @@ const aliases: Record<string, string> = {
   "frontier list": "frontier:list",
   "gap list": "gap:list",
   "export web": "export:web",
+  "compare flows": "compare:flows",
   "ci report": "ci:report",
   "ci publish": "ci:publish",
   "ci failure": "ci:failure",
@@ -104,6 +105,7 @@ const flagKinds: Record<string, Record<string, "value" | "boolean">> = {
   "ci:find-baseline": { sha:"value", target:"value", platform:"value" },
   "ci:compare-refs": { project:"value", base:"value", head:"value", title:"value", output:"value" },
   "ci:after-capture": { output:"value", checkout:"value" },
+  "compare:flows": { before: "value", after: "value", output: "value" },
   compare: {
     baseline: "value",
     current: "value",

src/cli/binary-commands.ts

@@ -1,15 +1,19 @@
-import { stat } from "node:fs/promises";
+import { readFile, stat, writeFile } from "node:fs/promises";
 import { basename, join, resolve } from "node:path";
 import { DEFAULT_BINARY_PROJECT, runBinaryExploration, runSessionExploration } from "../binary/run.js";
 import { openViewerInBrowser, startViewerServer, waitForViewerShutdown } from "../map/viewer-server.js";
 import { writeExplorationReport } from "../binary/report.js";
 import { assertHostingExport, deployExplorationReport, zipReportDirectory } from "../report/preview-deploy.js";
 import { keepReports } from "./report-commands.js";
 import { warnIgnoredExpiry } from "../config/explore-config.js";
-import { argumentError, flag, flags, has, nonNegativeInteger, oneOf, output, positional, positiveInteger } from "./args.js";
+import { argumentError, flag, flags, has, nonNegativeInteger, oneOf, output, positional, positiveInteger, required } from "./args.js";
 import { readReplayPlan } from "../binary/replay.js";
 import { readFlowPlan } from "../binary/flows.js";
-import type { ExplorationFlow } from "../contracts/standalone-report.js";
+import { EXPLORATION_REPORT_KIND, type ExplorationFlow, type ExplorationReport } from "../contracts/standalone-report.js";
+import { compareFlowRuns, type FlowRun } from "../review/flow-comparison.js";
+import { diffScreenshots } from "../review/visual-evidence.js";
+import { agentDeviceLauncher, ensureController } from "../shared/controller-cache.js";
+import { CiError } from "../shared/ci-errors.js";
 
 
 /** `--alias NAME` plus a fixed token from EXPLORE_PREVIEW_TOKEN: the hosted link stays the same from one run to the next (validated by the deploy). */
@@ -156,3 +160,32 @@ export async function exploreSession(): Promise<void> {
   openViewerInBrowser(viewer.url);
   await waitForViewerShutdown(viewer);
 }
+
+/** A flows run's report: the report directory explore:binary or explore:session wrote, or the output directory holding it. */
+async function flowRun(path: string, name: "before" | "after"): Promise<FlowRun> {
+  const refuse = (message: string) => { throw new CiError({ code: "FLOW_COMPARE_INPUT_INVALID", stage: "preflight", message,
+    nextAction: `Pass --${name} the output directory of an explore:binary or explore:session run that used --flows (or its report directory).` }); };
+  for (const directory of [resolve(path), resolve(path, "report")]) {
+    let text: string;
+    try { text = await readFile(join(directory, "exploration.json"), "utf8"); } catch { continue; }
+    let report: ExplorationReport;
+    try { report = JSON.parse(text) as ExplorationReport; } catch { return refuse(`${join(directory, "exploration.json")} is not JSON.`); }
+    if (report?.kind !== EXPLORATION_REPORT_KIND) return refuse(`${join(directory, "exploration.json")} is not an exploration report.`);
+    if (!Array.isArray(report.exploration?.flows)) return refuse(`The run in ${directory} did not follow a flows file, so it has no checkpoints to compare.`);
+    return { report, directory };
+  }
+  return refuse(`No exploration.json was found in ${resolve(path)} or its report folder.`);
+}
+
+/** `compare:flows --before DIR --after DIR [--output FILE]`: two runs of one flows file, checkpoint by checkpoint. */
+export async function compareFlows(): Promise<void> {
+  const [before, after] = await Promise.all([flowRun(required("before"), "before"), flowRun(required("after"), "after")]);
+  let command: string | undefined;
+  const comparison = await compareFlowRuns(before, after, async (beforePath, afterPath) => {
+    if (!command) { await ensureController("agent-device", { environment: process.env }); command = agentDeviceLauncher(process.env); }
+    return diffScreenshots({ baselinePath: beforePath, currentPath: afterPath, command });
+  });
+  const path = flag("output");
+  if (path) await writeFile(resolve(path), JSON.stringify(comparison, null, 2));
+  output({ ok: true, ...(path ? { output: resolve(path) } : {}), comparison });
+}

src/cli/help.ts

@@ -84,6 +84,8 @@ const CI = `CI (the installed workflows run these; they manage remote EAS Simula
   capture:baseline --project DIRECTORY --sha COMMIT --output TEMP_DIRECTORY [--config FILE] [--target ID | --platform ios|android] [--controller agent-device|argent] [--simulator eas|local] [--repack eas|local]
   source:verify --base-project DIRECTORY --head-project DIRECTORY
        --base-sha COMMIT --head-sha COMMIT --output DIRECTORY [--config FILE] [--scope all|changes] [--target ID | --platform ios|android]
+  compare:flows --before DIRECTORY --after DIRECTORY [--output FILE]
+       (two runs of one --flows file, checkpoint by checkpoint: unchanged, changed with where, or why not compared)
   compare --baseline DIRECTORY --current DIRECTORY --output FILE
        [--review-output FILE] [--catalog-output FILE] [--catalog-name NAME]
        [--base-surface-coverage FILE] [--head-surface-coverage FILE]

src/contracts/capture.ts

@@ -189,7 +189,7 @@ export const DUO_PANELS = {
   inner: { width: 951, height: 669, statusBar: { x: 883, y: 34, width: 40, height: 64 } },
 } as const;
 const DUO_STATUS_BAR_PADDING_POINTS = 4;
-const isFoldable = (platform: "ios" | "android" | undefined, device: string | undefined) => platform !== "android" && /\bduo\b/i.test(device ?? "");
+export const isFoldable = (platform: "ios" | "android" | undefined, device: string | undefined) => platform !== "android" && /\bduo\b/i.test(device ?? "");
 
 /**
  * The areas the verdict leaves out for a capture: the platform's status-bar band and, on iOS, its home-indicator

src/contracts/flow-comparison.ts

@@ -0,0 +1,58 @@
+import type { ExplorationReport } from "./standalone-report.js";
+
+export const FLOW_COMPARISON_KIND = "explore.flow-comparison" as const;
+
+/** One run's side of a checkpoint: its screenshot (a path inside that run's report directory), or why it has none. */
+export type CheckpointSide = { observationId: string; screenshot: string } | { missing: string };
+
+/**
+ * A checkpoint of one flow in two runs, paired by flow name and checkpoint name. `before` or `after` is absent when
+ * that run's flows file has no such flow or checkpoint.
+ */
+export type CheckpointPair = { flow: string; checkpoint: string; before?: CheckpointSide; after?: CheckpointSide };
+
+/** What a run was captured on: two runs that differ here can still be compared, but a difference may come from it. */
+export type RunConditions = { app: string; platform: string; device: string; osVersion?: string; locale?: string; orientation?: string; colorScheme?: string };
+export type RunConditionDifference = { field: keyof RunConditions; before: string; after: string };
+
+export function runConditions(report: ExplorationReport): RunConditions {
+  const environment = report.map.run.environment ?? {};
+  return { app: report.exploration.app.bundleId, platform: report.exploration.app.platform, device: report.exploration.device.model,
+    ...(report.exploration.device.osVersion ? { osVersion: report.exploration.device.osVersion } : {}),
+    ...(environment.locale ? { locale: environment.locale } : {}), ...(environment.orientation ? { orientation: environment.orientation } : {}),
+    ...(environment.colorScheme ? { colorScheme: environment.colorScheme } : {}) };
+}
+
+export function conditionDifferences(before: RunConditions, after: RunConditions): RunConditionDifference[] {
+  const fields = ["app", "platform", "device", "osVersion", "locale", "orientation", "colorScheme"] as const;
+  return fields.flatMap(field => (before[field] ?? "unknown") === (after[field] ?? "unknown") ? []
+    : [{ field, before: before[field] ?? "unknown", after: after[field] ?? "unknown" }]);
+}
+
+const keyOf = (name: string) => name.trim().toLowerCase();
+
+function side(report: ExplorationReport, checkpoint: NonNullable<ExplorationReport["exploration"]["flows"]>[number]["checkpoints"][number]): CheckpointSide {
+  if (checkpoint.observationId === undefined) return { missing: checkpoint.missing };
+  const observation = report.map.observations.find(item => item.id === checkpoint.observationId);
+  const artifact = observation?.screenshotArtifactId ? report.map.artifacts.find(item => item.id === observation.screenshotArtifactId) : undefined;
+  const asset = artifact ? report.assets.find(item => item.sha256 === artifact.sha256) : undefined;
+  return asset ? { observationId: checkpoint.observationId, screenshot: asset.path } : { missing: "The report has no screenshot for this checkpoint." };
+}
+
+/**
+ * Pairs the checkpoints of two runs of a flows file: the after run's flows and checkpoints in its order, then those only
+ * the before run has. Names match as the flows file compares them, ignoring case and surrounding space.
+ */
+export function pairCheckpoints(before: ExplorationReport, after: ExplorationReport): CheckpointPair[] {
+  const beforeFlows = before.exploration.flows ?? [], afterFlows = after.exploration.flows ?? [];
+  const flowNames = [...afterFlows, ...beforeFlows.filter(flow => !afterFlows.some(other => keyOf(other.name) === keyOf(flow.name)))].map(flow => flow.name);
+  return flowNames.flatMap(name => {
+    const was = beforeFlows.find(flow => keyOf(flow.name) === keyOf(name)), now = afterFlows.find(flow => keyOf(flow.name) === keyOf(name));
+    const wasPoints = was?.checkpoints ?? [], nowPoints = now?.checkpoints ?? [];
+    const names = [...nowPoints, ...wasPoints.filter(point => !nowPoints.some(other => keyOf(other.name) === keyOf(point.name)))].map(point => point.name);
+    return names.map(checkpoint => {
+      const old = wasPoints.find(point => keyOf(point.name) === keyOf(checkpoint)), current = nowPoints.find(point => keyOf(point.name) === keyOf(checkpoint));
+      return { flow: name, checkpoint, ...(old ? { before: side(before, old) } : {}), ...(current ? { after: side(after, current) } : {}) };
+    });
+  });
+}