diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ef5c8ef..a1398a5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -29,10 +29,11 @@ This file describes the source layout and the rules that keep it in order. | `review` | Compares base and head evidence, and prepares the local report assets. `ci` calls `evidence-assessment.ts` and `capture-review.ts`. | `evidence-assessment.ts`, `capture-review.ts`, `visual-evidence.ts`, `runtime-evidence.ts`, `evidence-comparator.ts`, `reviewer-changes.ts` | | `report` | The report artifact, the pull-request comment, and the Viewer links. | `artifact.ts`, `pr-comment*.ts`, `viewer-links.ts` | | `ci` | The pipeline behind `source:verify`, `capture:review` and `capture:baseline`. | `pipeline.ts`, `verified-plan.ts`, `artifact-cache.ts` | +| `binary` | Explores an app with no source (`explore:binary`, `explore:session`): the archive, the device session, the explorer, and the strategies that drive it. | `run.ts` (entry), `explorer.ts`, `strategy.ts`, `replay.ts`, `flows.ts`, `archive.ts`, `report.ts` | | `cli` | Argument parsing, help, and the command handlers. | `args.ts`, `help.ts`, `*-commands.ts` | | `map` | The local map store, build catalogs and comparisons, and the server for `expo-explore view`. | `store.ts`, `build-workspace.ts`, `compare.ts`, `viewer-server.ts` | | `contracts` | The modules that `viewer/src` imports from `src/`, by value or by type: data shapes, validators and projections that the CLI and the Viewer share. `capture.ts` also defines the review types that the Viewer reads, such as `PublicReview`, and `review-validation.ts` validates them. | `capture.ts`, `review.ts`, `review-validation.ts`, `source-validation.ts` | -| `shared` | Small Node utilities that several folders use, and the pinned `agent-device` version that capture and review both check. | `ci-errors.ts`, `canonical-json.ts`, `bounded-bytes.ts`, `png-evidence.ts`, `agent-device-version.ts`, `timings.ts` (a capture job's timings and its summary, CI-FLOW §4.0) | +| `shared` | Small Node utilities that several folders use, and the pinned `agent-device` version that capture and review both check. | `ci-errors.ts`, `canonical-json.ts`, `bounded-bytes.ts`, `png-evidence.ts`, `archive-entries.ts` (tar.gz and zip readers), `agent-device-version.ts`, `timings.ts` (a capture job's timings and its summary, CI-FLOW §4.0) | ## Dependency rule @@ -44,14 +45,24 @@ Each layer imports only from the layers below it. 3. `source` imports `source/route-providers`. 4. `capture` imports `capture/agent`, `config` and `map`. `capture/agent` imports `capture` and `config`. 5. `review` imports `capture` and `map`. -6. `ci` imports `config`, `source`, `capture`, `capture/agent`, `review` and `report`. -7. `cli` imports `ci`, `map`, `config` and `report`. `src/cli.ts` imports `cli`, `map` and `shared`. +6. `ci` imports `config`, `source`, `capture`, `capture/agent`, `review` and `report`. `binary` imports `config`, `capture`, `capture/agent`, `map` and `report`. +7. `cli` imports `ci`, `binary`, `map`, `config` and `report`. `src/cli.ts` imports `cli`, `map` and `shared`. Every folder may import `contracts/`, `shared/` and `src/types.ts`. The exception is `contracts/`: its value imports stay inside `contracts/` or go to `src/types.ts`. The only cycle is `capture` and `capture/agent`, which are one feature. Do not add a new import that points up a layer. +## Capture strategies + +A capture decides three things: what to look at, how to get there, and what to record. + +- **What to look at.** For a review, `source` plans the routes from the app's source (`capture-plan.ts`). An exploration has no source; its strategy decides as it goes. +- **How to get there.** In a review, `capture/route-capture.ts` opens each planned route by deep link, or, in agent mode, a model navigates to it (`capture/agent/runtime.ts`). In an exploration, a strategy drives the explorer's tools: a model exploring on its own, a replay plan (`binary/replay.ts`), or named flows (`binary/flows.ts`). +- **What to record.** The host records the evidence, never the strategy: `capture/route-capture.ts` in a review, `binary/explorer.ts` in an exploration. The host also owns the device, the budgets, refusing destructive and app-leaving actions, and redacting secrets. Both write an `ExploreMap` (`src/types.ts`, `schema/screen-map.v1.schema.json`). A review adds the capture bundle (`contracts/capture.ts`); an exploration adds the report in `contracts/standalone-report.ts`. + +To add an exploration strategy, add a case to `StrategyChoice` in `binary/strategy.ts`: what it checks before any device starts, and the session it runs. The session is given the explorer's tools and calls them, as the model sessions, `createReplaySession` and `createFlowSession` do. Reviews have no such seam yet: the deep-link loop and agent mode are two branches of `capturePlannedRoutes`. + ## The contracts boundary `viewer/src` imports `src/` only through `src/contracts/` and `src/types.ts`. diff --git a/src/binary/app-url.ts b/src/binary/app-url.ts new file mode 100644 index 0000000..7e3d7c0 --- /dev/null +++ b/src/binary/app-url.ts @@ -0,0 +1,17 @@ +import { CiError } from "../shared/ci-errors.js"; + +/** Schemes the OS routes to a browser, the store, Settings, or another app: opening one hands the session away from the app under test. */ +const SYSTEM_URL_SCHEMES = /^(https?|ftp|file|javascript|data|blob|mailto|tel|telprompt|facetime(-audio)?|sms|mms|imessage|itms(-apps|-services|-beta)?|macappstore|app-settings|prefs|shortcuts|maps|music|videos|photos-redirect|x-apple-[a-z-]+|market|intent|content|android-app|geo|package):$/i; + +const urlError = (code: string, message: string, nextAction: string) => new CiError({ code, stage: "session", message, nextAction }); + +/** Only the app's own URLs may be opened; an https page or a store link would hand the session to another app. */ +export function assertAppUrl(url: string, appSchemes: readonly string[] = []): void { + let parsed: URL; + try { parsed = new URL(url); } catch { throw urlError("AGENT_URL_INVALID", `Not a URL: ${url}`, "Pass a custom-scheme app URL."); } + if (SYSTEM_URL_SCHEMES.test(parsed.protocol)) throw urlError("AGENT_URL_NOT_ALLOWED", `${parsed.protocol} URLs leave the app; only the app's own URL schemes are allowed.`, "Use the app's own URL scheme."); + const scheme = parsed.protocol.slice(0, -1).toLowerCase(); + if (appSchemes.length && !appSchemes.includes(scheme)) { + throw urlError("AGENT_URL_NOT_ALLOWED", `${parsed.protocol} is not a URL scheme this app registers (${appSchemes.join(", ")}).`, "Use one of the app's own URL schemes."); + } +} diff --git a/src/binary/archive.ts b/src/binary/archive.ts index 771e252..dbd1d1a 100644 --- a/src/binary/archive.ts +++ b/src/binary/archive.ts @@ -2,9 +2,7 @@ import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; import { lstat, mkdir, open, readdir, readFile, realpath, writeFile, chmod } from "node:fs/promises"; import { basename, join, relative, resolve, sep } from "node:path"; -import { createGunzip } from "node:zlib"; -import { Reader, Uint8ArrayWriter, ZipReader, configure } from "@zip.js/zip.js"; -import { CiError } from "../shared/ci-errors.js"; +import { MAX_ARCHIVE_ENTRIES, MAX_METADATA_BYTES, archiveError, readZipEntry, safeRelativePath, walkTar, zipEntries } from "../shared/archive-entries.js"; import { APP_ICON_SOURCE_LIMIT, isWebp, shrinkAppIcon, webpSize } from "../capture/app-icon.js"; import { resolveResourceStrings } from "./android-resources.js"; import type { Platform } from "../types.js"; @@ -38,14 +36,8 @@ export type InspectedArchive = { }; export const DEFAULT_MAX_ARCHIVE_BYTES = 1024 * 1024 * 1024; -const MAX_METADATA_BYTES = 4 * 1024 * 1024; -const MAX_ARCHIVE_ENTRIES = 200_000; const MAX_RESOURCE_TABLE_BYTES = 32 * 1024 * 1024; -export function archiveError(code: string, message: string, nextAction: string, details?: Record): CiError { - return new CiError({ code, stage: "archive", message, nextAction, ...(details ? { details } : {}) }); -} - /** The archive kind from its name; the content check follows and may still reject it. */ export function archiveKind(path: string): ArchiveKind { const name = basename(path).toLowerCase(); @@ -231,63 +223,6 @@ async function hashDirectory(root: string, maxBytes: number): Promise<{ sha256: return { sha256: hash.digest("hex"), bytes }; } -// ---- tar.gz ------------------------------------------------------------------------------------- - -type TarVisitor = (entry: { name: string; type: string; size: number; mode: number }, contents: () => Promise, skip: () => Promise) => Promise; - -/** Walk a gzip tarball once; `visit` returns true to stop. Long GNU names are honoured; nothing is written. */ -export async function walkTar(path: string, visit: TarVisitor): Promise { - const stream = createReadStream(path).pipe(createGunzip()); - const iterator = stream[Symbol.asyncIterator](); - let buffer = Buffer.alloc(0); - let done = false; - const fill = async (needed: number) => { - while (buffer.length < needed && !done) { - const next = await iterator.next(); - if (next.done) { done = true; break; } - buffer = Buffer.concat([buffer, next.value as Buffer]); - } - }; - const take = async (count: number): Promise => { - await fill(count); - if (buffer.length < count) throw archiveError("ARCHIVE_INVALID", "The tarball ended unexpectedly.", "Pass a complete .tar.gz archive."); - const out = buffer.subarray(0, count); buffer = buffer.subarray(count); return Buffer.from(out); - }; - const drop = async (count: number) => { - let remaining = count; - while (remaining > 0) { - await fill(Math.min(remaining, 1 << 20)); - const step = Math.min(remaining, buffer.length); - if (!step) throw archiveError("ARCHIVE_INVALID", "The tarball ended unexpectedly.", "Pass a complete .tar.gz archive."); - buffer = buffer.subarray(step); remaining -= step; - } - }; - let longName: string | undefined; - let entries = 0; - try { - for (;;) { - await fill(512); - if (buffer.length < 512 || buffer.subarray(0, 512).every(byte => byte === 0)) return; - const header = await take(512); - const field = (offset: number, length: number) => header.subarray(offset, offset + length).toString("utf8").replace(/\0.*$/s, ""); - const size = parseInt(field(124, 12).trim() || "0", 8); - const type = field(156, 1) || "0"; - const prefix = field(345, 155); - const name = longName ?? (prefix ? `${prefix}/${field(0, 100)}` : field(0, 100)); - longName = undefined; - const padded = Math.ceil(size / 512) * 512; - if (type === "L") { longName = (await take(padded)).subarray(0, size).toString("utf8").replace(/\0+$/, ""); continue; } - if (++entries > MAX_ARCHIVE_ENTRIES) throw archiveError("ARCHIVE_TOO_LARGE", "The tarball has too many entries.", "Pass a smaller archive."); - let consumed = false; - const stop = await visit({ name, type, size, mode: parseInt(field(100, 8).trim() || "0", 8) }, - async () => { consumed = true; return (await take(padded)).subarray(0, size); }, - async () => { consumed = true; await drop(padded); }); - if (!consumed) await drop(padded); - if (stop) return; - } - } finally { await iterator.return?.(undefined); } -} - async function tarEntry(path: string, match: (name: string) => boolean): Promise<{ name: string; contents: Buffer } | undefined> { let found: { name: string; contents: Buffer } | undefined; await walkTar(path, async (entry, contents) => { @@ -299,12 +234,6 @@ async function tarEntry(path: string, match: (name: string) => boolean): Promise return found; } -export function safeRelativePath(name: string): string | undefined { - const parts = name.replace(/^\.\//, "").split("/").filter(part => part.length); - if (!parts.length || parts.some(part => part === "." || part === ".." || part.includes("\\") || part.includes("\0"))) return undefined; - return parts.join("/"); -} - /** * Unpack the iOS bundle out of a .tar.gz or simulator .ipa into `destination` for installation. * Paths are confined to the destination; symbolic and hard links are rejected (an app bundle has none). @@ -347,60 +276,6 @@ export async function extractIosBundle(archive: InspectedArchive, destination: s return appDirectory; } -// ---- zip ---------------------------------------------------------------------------------------- - -type ZipEntry = Awaited["getEntries"]>>[number]; - -/** zip.js reads the central directory and entries by range, so a 200 MB APK is never loaded whole. */ -class FileRangeReader extends Reader { - constructor(private readonly path: string) { super(path); } - override async init(): Promise { - this.size = (await lstat(this.path)).size; - } - /** Each range opens its own descriptor: nothing stays open after the entries are read, so no handle leaks when a caller stops early. */ - override async readUint8Array(index: number, length: number): Promise { - const out = new Uint8Array(length); - const handle = await open(this.path, "r"); - try { - let offset = 0; - while (offset < length) { - const { bytesRead } = await handle.read(out, offset, length - offset, index + offset); - if (bytesRead === 0) break; - offset += bytesRead; - } - return offset === length ? out : out.subarray(0, offset); - } finally { await handle.close(); } - } -} - -const readers = new WeakMap(); - -/** The ABIs an APK ships native libraries for (`lib//`), without reading its manifest. */ -export async function apkAbis(path: string): Promise { - return [...new Set((await zipEntries(path)).map(entry => /^lib\/([^/]+)\//.exec(entry.filename)?.[1]).filter((abi): abi is string => !!abi))].sort(); -} - -async function zipEntries(path: string): Promise { - configure({ useWebWorkers: false }); - const reader = new FileRangeReader(path); - const zip = new ZipReader(reader, { useWebWorkers: false }); - try { - const entries = await zip.getEntries(); - if (entries.length > MAX_ARCHIVE_ENTRIES) throw archiveError("ARCHIVE_TOO_LARGE", "The archive has too many entries.", "Pass a smaller archive."); - for (const entry of entries) readers.set(entry, reader); - return entries; - } catch (error) { - if (error instanceof CiError) throw error; - throw archiveError("ARCHIVE_INVALID", `${basename(path)} is not a readable ZIP archive: ${error instanceof Error ? error.message : String(error)}`, "Pass a valid .ipa or .apk."); - } -} - -async function readZipEntry(entry: ZipEntry, maxBytes = MAX_METADATA_BYTES): Promise { - if (entry.directory || !entry.getData) throw archiveError("ARCHIVE_INVALID", `${entry.filename} is not a file entry.`, "Pass a valid archive."); - if (entry.uncompressedSize > maxBytes) throw archiveError("ARCHIVE_INVALID", `${entry.filename} is too large.`, "Pass a valid archive."); - return entry.getData(new Uint8ArrayWriter(), { checkSignature: true, useWebWorkers: false }); -} - // ---- plist -------------------------------------------------------------------------------------- /** Property list reader for Info.plist: XML (Xcode default) or binary (`bplist00`). */ diff --git a/src/binary/run.ts b/src/binary/run.ts index 3d550bc..5cac424 100644 --- a/src/binary/run.ts +++ b/src/binary/run.ts @@ -2,11 +2,12 @@ import { createHash, randomUUID } from "node:crypto"; import { lstat, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { createCiAgentSession, type CiAgentSession, type CiAgentSessionConfig } from "../capture/agent/session.js"; -import { createLocalCiAgentSession } from "../capture/agent/session-local.js"; +import type { CiAgentSession, CiAgentSessionConfig } from "../capture/agent/session.js"; import type { CiAgentProviderName } from "../capture/agent/provider.js"; -import { assertReplayFits, createReplaySession, parseReplayPlan, replaySetup, type ReplayPlan } from "./replay.js"; -import { assertFlowsFit, createFlowSession, flowGoals, flowSetup, flowsNeedModel, parseFlowPlan, type FlowPlan } from "./flows.js"; +import type { ReplayPlan } from "./replay.js"; +import type { FlowPlan } from "./flows.js"; +import { assertAppUrl } from "./app-url.js"; +import { chooseStrategy, startStrategy, strategyInputs, turnBudget, type ExplorationStrategy, type ModelAuth } from "./strategy.js"; import { CONTROLLER_SESSION, baseEnvironment, isRecord } from "../capture/common.js"; import { controllerData, defaultRunner, openedSimulatorDevice } from "../capture/controller.js"; import type { CommandRequest, CommandResult } from "../capture/run.js"; @@ -22,7 +23,7 @@ import { screenSpace } from "../capture/route-capture.js"; import { approveSessionUrlSchemes, simulatorControlActions, type EasCommand } from "../capture/simulator-control.js"; import { DEFAULT_ANDROID_DEVICE } from "../config/explore-config.js"; import { DUO_PANELS } from "../contracts/capture.js"; -import { createExplorationInputs } from "../config/exploration-inputs.js"; +import type { createExplorationInputs } from "../config/exploration-inputs.js"; import { ExploreStore, getCoverage } from "../map/store.js"; import { whoamiAccounts } from "../report/preview-deploy.js"; import { CiError, serializeCiError, toCiError } from "../shared/ci-errors.js"; @@ -34,7 +35,6 @@ import { runBlackBoxExploration, type BlackBoxExplorerResult, type BlackBoxFrame export const DEFAULT_IOS_DEVICE = "iPhone 17 Pro"; export const DEFAULT_BINARY_PROJECT = "expo-explore"; -export const DEFAULT_MODELS: Record = { anthropic: "claude-sonnet-4-5", openai: "gpt-6.1-sol" }; export const BINARY_LIMITS = { maxDurationMinutes: 15, maxScreens: 40, maxActions: 150 } as const; export type BinaryExploreOptions = { @@ -76,7 +76,7 @@ export type BinaryExploreResult = { archive: Omit & { path: string }; session: { id: string; platform: Platform; device: string; owner: string; project: string }; runtime: ActiveSimulatorRuntime; - model: { provider: CiAgentProviderName | "replay" | "flows"; model: string; auth: "local-login" | "api-key" | "none" }; + model: Pick; exploration: BlackBoxExplorerResult; durationMs: number; }; @@ -166,23 +166,6 @@ export function resolveSessionApp(runner: typeof defaultRunner, connection: Simu "Pass --bundle-id with the app's bundle identifier (iOS) or package name (Android).", { installed }); } -/** - * App-test inputs: each {{secret:NAME}} in the hints names an environment variable whose value Explore types for the - * model. Read before any session starts, so a missing one costs nothing. - */ -export function hintInputs(hints: string | undefined, environment: NodeJS.ProcessEnv, replay?: ReplayPlan, flows?: FlowPlan, platform?: Platform): ReturnType { - const setup = replay ? replaySetup(replay) : flows ? flowSetup(flows, platform) : undefined; - try { return createExplorationInputs({ instructions: [hints ?? "", ...(flows ? flowGoals(flows, platform) : [])].join("\n"), ...(setup ? { setup } : {}), environment }); } - catch (error) { - if (!(error instanceof CiError) || !error.code.startsWith("EXPLORATION_SECRET_")) throw error; - const name = typeof error.details?.secretName === "string" ? error.details.secretName : "NAME"; - throw new CiError({ code: error.code, stage: "preflight", message: error.message, ...(error.details ? { details: error.details } : {}), - nextAction: error.code === "EXPLORATION_SECRET_REFERENCE_INVALID" - ? "Write each input in --hints as {{secret:NAME}} with an uppercase environment variable name that is not a provider, Expo, or CI credential." - : `Set ${name} in the environment that runs Explore (export ${name}=…) to the single-line test value, then rerun.` }); - } -} - function validatedLimits(options: Pick) { const limits = { maxDurationMinutes: options.maxDurationMinutes ?? BINARY_LIMITS.maxDurationMinutes, maxScreens: options.maxScreens ?? BINARY_LIMITS.maxScreens, maxActions: options.maxActions ?? BINARY_LIMITS.maxActions }; for (const [name, value] of Object.entries(limits)) if (!Number.isSafeInteger(value) || value < 1 || value > 1000) throw binaryError("EXPLORE_LIMIT_INVALID", `${name} must be an integer between 1 and 1000.`, "Correct the budget flag and rerun."); @@ -193,108 +176,6 @@ function binaryError(code: string, message: string, nextAction: string, details? return new CiError({ code, stage: "session", message, nextAction, ...(details ? { details } : {}) }); } -type ModelAuth = "local-login" | "api-key"; -type ModelSessionOptions = Pick; - -function modelSession(provider: CiAgentProviderName, auth: ModelAuth, options: ModelSessionOptions, environment: NodeJS.ProcessEnv) { - const config = { provider, model: options.model ?? DEFAULT_MODELS[provider] }; - const session = options.createSession ? options.createSession(config, { env: environment, auth }) - : auth === "api-key" ? createCiAgentSession(config, { env: environment }) : createLocalCiAgentSession(config, { env: environment }); - return { session, provider, model: config.model, auth }; -} - -/** The turns the model gets: three for each action and two for each screen, plus a few to look around and finish. */ -const turnBudget = (limits: { maxScreens: number; maxActions: number }): number => limits.maxActions * 3 + limits.maxScreens * 2 + 20; - -/** - * A replay plan is checked here as well as by the CLI, for callers that pass one directly. It takes the model's place, so - * the options that only steer a model would do nothing; the plan's own fill and type steps name the secrets it types. - */ -function replayPlan(options: Pick): ReplayPlan | undefined { - if (!options.replay) return undefined; - const ignored = (["provider", "model", "hints"] as const).filter(name => options[name] !== undefined); - if (ignored.length) { - throw binaryError("REPLAY_OPTIONS_IGNORED", `--replay takes the model's place, so ${ignored.map(name => `--${name}`).join(" and ")} would do nothing.`, - `Drop ${ignored.map(name => `--${name}`).join(" and ")}. A {{secret:NAME}} in the plan's fill and type steps is read from the environment, as one in --hints is.`); - } - const plan = parseReplayPlan(options.replay); - for (const [index, step] of plan.steps.entries()) { - if (!("action" in step) || step.action !== "open") continue; - try { assertAppUrl(step.url); } - catch (error) { throw new CiError({ code: "REPLAY_PLAN_INVALID", stage: "exploration", message: `Step ${index + 1}: ${error instanceof Error ? error.message : String(error)}`, nextAction: "Open only the app's own URLs in the replay plan." }); } - } - return plan; -} - -/** A replay plan takes the model's place: no model access is checked or used. */ -function replayModel(plan: ReplayPlan, platform: Platform, limits: { maxScreens: number; maxActions: number }) { - assertReplayFits(plan, platform, { ...limits, maxTurns: turnBudget(limits) }); - const session = createReplaySession(plan, platform); - return { session, budgetNote: session.budgetNote, provider: "replay" as const, model: "replay plan", auth: "none" as const }; -} - -/** - * A flows file is checked here as well as by the CLI. It decides every step, as a replay plan does, but a step it - * describes in words is carried out by a model, so --provider, --model and --hints apply only when it has one. - */ -function flowsPlan(options: Pick): FlowPlan | undefined { - if (!options.flows) return undefined; - if (options.replay) throw binaryError("FLOWS_WITH_REPLAY", "--flows and --replay each decide every step, so only one of them can run.", "Drop --replay, or move its steps into a flow."); - const plan = parseFlowPlan(options.flows); - const ignored = flowsNeedModel(plan) ? [] : (["provider", "model", "hints"] as const).filter(name => options[name] !== undefined); - if (ignored.length) { - throw binaryError("FLOWS_OPTIONS_IGNORED", `Every step of the flows file is exact, so ${ignored.map(name => `--${name}`).join(" and ")} would do nothing.`, - `Drop ${ignored.map(name => `--${name}`).join(" and ")}, or describe a step in words ({"do": "…"}) for a model to carry out.`); - } - for (const [index, flow] of plan.flows.entries()) for (const [stepIndex, step] of flow.steps.entries()) { - if (!("action" in step) || step.action !== "open") continue; - try { assertAppUrl(step.url); } - catch (error) { throw new CiError({ code: "FLOWS_PLAN_INVALID", stage: "exploration", message: `Flow ${index + 1} (${flow.name}), step ${stepIndex + 1}: ${error instanceof Error ? error.message : String(error)}`, nextAction: "Open only the app's own URLs in the flows file." }); } - } - return plan; -} - -/** The flows run in place of a model's exploration; a model is checked, and used, only when a step is described in words. */ -async function flowsModel(plan: FlowPlan, platform: Platform, limits: { maxScreens: number; maxActions: number }, options: ModelSessionOptions & Pick, - environment: NodeJS.ProcessEnv, say: (message: string) => void, app: { name: string }) { - assertFlowsFit(plan, platform, { ...limits, maxTurns: turnBudget(limits) }); - const goals = flowsNeedModel(plan, platform) ? await preflightModelSession(options, environment, say) : undefined; - const session = createFlowSession(plan, platform, { app, ...(goals ? { model: goals.session } : {}), ...(options.hints ? { hints: options.hints } : {}) }); - return { session, budgetNote: session.budgetNote, flows: session.notReached, provider: "flows" as const, model: goals ? `${goals.provider} ${goals.model}` : session.model, auth: goals?.auth ?? "none" as const }; -} - -/** Local `claude`/`codex` login unless an API key is present; preflight proves access before a simulator is paid for. */ -export function selectModelSession(options: ModelSessionOptions, environment: NodeJS.ProcessEnv) { - const provider: CiAgentProviderName = options.provider ?? (environment.OPENAI_API_KEY?.trim() && !environment.ANTHROPIC_API_KEY?.trim() ? "openai" : "anthropic"); - const keyName = provider === "anthropic" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY"; - return modelSession(provider, environment[keyName]?.trim() ? "api-key" : "local-login", options, environment); -} - -/** - * With --provider or --model, only that choice is tried. Otherwise every model access this shell has is tried - * in turn (API keys, then the local claude and codex logins) and the first that passes preflight runs. - */ -export async function preflightModelSession(options: ModelSessionOptions, environment: NodeJS.ProcessEnv, say: (message: string) => void) { - const candidates = options.provider || options.model ? [selectModelSession(options, environment)] : [ - ...(["anthropic", "openai"] as const).filter(provider => environment[provider === "anthropic" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY"]?.trim()) - .map(provider => modelSession(provider, "api-key", options, environment)), - modelSession("anthropic", "local-login", options, environment), - modelSession("openai", "local-login", options, environment), - ]; - let firstError: unknown; - for (const [index, candidate] of candidates.entries()) { - say(`checking ${candidate.provider} ${candidate.model} (${candidate.auth === "api-key" ? "API key" : "local login"})`); - try { - await candidate.session.preflight(); - return candidate; - } catch (error) { - firstError ??= error; - if (index < candidates.length - 1) say(`${candidate.provider} ${candidate.auth === "api-key" ? "API key" : "local login"} unavailable (${error instanceof Error ? error.message : String(error)}); trying the next`); - } - } - throw firstError; -} - export type SessionExploreOptions = Omit & { /** App to explore; defaults to the app in the foreground, else the only user-installed app. */ bundleId?: string; @@ -329,8 +210,7 @@ export type ConnectedSessionExploration = { limits: { maxDurationMinutes: number; maxScreens: number; maxActions: number }; store: ExploreStore; map: ExploreMap; - /** `flows`: the session takes the explorer's flow tools, and names the checkpoints it never attempted. */ - model: { session: Pick; budgetNote?: () => string | undefined; flows?: () => Array<{ flow: string; checkpoints: string[] }> }; + strategy: ExplorationStrategy; /** `eas simulator:get --json` for the attached session; descriptive (OS version) only. */ sessionJson: () => string; /** agent-device session name on the daemon; explore:session reuses the caller's session so the device is not "in use" by another. */ @@ -353,7 +233,7 @@ export type ConnectedSessionExploration = { /** Drives an agent-device daemon we already hold a connection to: optional install, open, then the model-led exploration. */ export async function exploreConnectedSession(ctx: ConnectedSessionExploration): Promise<{ runtime: ActiveSimulatorRuntime; exploration: BlackBoxExplorerResult }> { - const { connection, platform, device, bundleId, app, install, work, runner, environment, secrets, inputs, say, now, limits, store, map, model, sessionJson, launchUrl, hints, allowDestructive, local } = ctx; + const { connection, platform, device, bundleId, app, install, work, runner, environment, secrets, inputs, say, now, limits, store, map, strategy, sessionJson, launchUrl, hints, allowDestructive, local } = ctx; const controllerSession = ctx.controllerSession ?? CONTROLLER_SESSION; const controllerKind = connectionController(connection); const controllerName = controllerKind === "argent" ? "Argent" : "agent-device"; @@ -392,7 +272,7 @@ export async function exploreConnectedSession(ctx: ConnectedSessionExploration): ctx.afterInstall?.(); // On a local device, the first open also builds agent-device's iOS runner. // Flows start from the app's first screen: an app already running on an attached session is relaunched, not brought forward. - const opened = controller(["open", bundleId, ...(model.flows ? ["--relaunch"] : []), "--level", "full", "--json"], local ? 15 * 60_000 : 2 * 60_000); + const opened = controller(["open", bundleId, ...(strategy.flows ? ["--relaunch"] : []), "--level", "full", "--json"], local ? 15 * 60_000 : 2 * 60_000); const openedDevice = openedSimulatorDevice(opened.stdout, bundleId, "initial app open", platform, controllerSession); const data = controllerData(opened.stdout, "initial app open"); const deviceData = data.device && typeof data.device === "object" ? data.device as Record : {}; @@ -539,12 +419,12 @@ export async function exploreConnectedSession(ctx: ConnectedSessionExploration): map.gaps.push({ id: `gap_${randomUUID().slice(0, 8)}`, kind: "coverage", status: "open", message: reason, createdAt: new Date(now()).toISOString() }); say("exploration blocked: the app does not stay running"); blockedAtLaunch = { stopReason: "blocked", summary: reason, screens: 0, observations: 0, actions: 0, turns: 0, - ...(model.flows ? { flows: model.flows().map(({ flow, checkpoints }) => ({ name: flow, status: "failed" as const, reason, checkpoints: checkpoints.map(name => ({ name, missing: "The app did not stay open." })) })) } : {}) }; + ...(strategy.flows ? { flows: strategy.flows().map(({ flow, checkpoints }) => ({ name: flow, status: "failed" as const, reason, checkpoints: checkpoints.map(name => ({ name, missing: "The app did not stay open." })) })) } : {}) }; } } - const exploration = blockedAtLaunch ?? await runBlackBoxExploration({ session: model.session, store, map, app: { name: app.name, bundleId, platform, ...(app.version ? { version: app.version } : {}) }, + const exploration = blockedAtLaunch ?? await runBlackBoxExploration({ session: strategy.session, store, map, app: { name: app.name, bundleId, platform, ...(app.version ? { version: app.version } : {}) }, inputs, ...(hints ? { hints: hints } : {}), ...(launchUrl ? { launchUrl: launchUrl } : {}), ...(allowDestructive ? { allowDestructive: allowDestructive } : {}), ...(ctx.appIcon ? { appIcon: ctx.appIcon } : {}), - ...(model.budgetNote ? { budgetNote: model.budgetNote } : {}), ...(model.flows ? { flows: { restart, notReached: model.flows } } : {}), + ...(strategy.budgetNote ? { budgetNote: strategy.budgetNote } : {}), ...(strategy.flows ? { flows: { restart, notReached: strategy.flows } } : {}), deadlineMs, maxScreens: limits.maxScreens, maxActions: limits.maxActions, maxTurns: turnBudget(limits), now, observe, progress: say, act: async action => { const perform = async () => { @@ -609,11 +489,10 @@ export async function runSessionExploration(options: SessionExploreOptions): Pro const environment = options.environment ?? process.env; const runner = options.runner ?? defaultRunner; const limits = validatedLimits(options); - const replay = replayPlan(options); - const flows = flowsPlan(options); + const choice = chooseStrategy(options); // A flows run without --platform checks its secrets once the session reports its platform, before any device work, so a // secret that only another platform's steps type is not required. - let inputs = flows && !options.platform ? undefined : hintInputs(options.hints, environment, replay, flows, options.platform); + let inputs = choice.kind === "flows" && !options.platform ? undefined : strategyInputs(choice, options.hints, environment, options.platform); if (options.bundleId !== undefined && !appIdPattern.test(options.bundleId)) throw binaryError("BUNDLE_ID_INVALID", `${options.bundleId} is not a bundle or package id.`, "Pass --bundle-id with the app's bundle identifier (iOS) or package name (Android)."); let connection: SimulatorConnection; try { connection = await resolveAttachedSimulatorSession(environment, options.envFile, options.session); } @@ -659,7 +538,7 @@ export async function runSessionExploration(options: SessionExploreOptions): Pro } platform = info.platform; if (!inputs) { - inputs = hintInputs(options.hints, environment, replay, flows, platform); + inputs = strategyInputs(choice, options.hints, environment, platform); secrets.push(...inputs.values); } const candidates = daemonSessions(runner, connection, work, platform, environment).filter(item => !device || item.device === device); @@ -681,16 +560,15 @@ export async function runSessionExploration(options: SessionExploreOptions): Pro map.run.environment = { deviceModel: device }; session = { id: connection.sessionId, platform, device, name: info.name, ...(info.deviceRunSessionUrl ? { url: info.deviceRunSessionUrl } : {}) }; - const model = replay ? replayModel(replay, platform, limits) : flows ? await flowsModel(flows, platform, limits, options, environment, say, app) - : await preflightModelSession(options, environment, say); + const strategy = await startStrategy(choice, { platform, limits, options, environment, say, app }); const outputDirectory = await realpath(await mkdir(options.output, { recursive: true }).then(() => options.output)); const store = new ExploreStore(outputDirectory); if (await store.exists()) throw binaryError("OUTPUT_NOT_EMPTY", `${outputDirectory} already holds an Explore map.`, "Pass an empty --output directory."); - ({ runtime, exploration } = await exploreConnectedSession({ connection, platform, device, bundleId: app.bundleId, app, work, runner, environment, secrets, inputs, say, now, limits, store, map, model, sessionJson, + ({ runtime, exploration } = await exploreConnectedSession({ connection, platform, device, bundleId: app.bundleId, app, work, runner, environment, secrets, inputs, say, now, limits, store, map, strategy, sessionJson, ...(existing ? { controllerSession: existing.name } : {}), ...(options.launchUrl ? { launchUrl: options.launchUrl } : {}), ...(options.hints ? { hints: options.hints } : {}), ...(options.allowDestructive ? { allowDestructive: options.allowDestructive } : {}) })); - return { outputDirectory, mapPath: store.mapPath, app: { ...app, platform }, session, runtime, model: { provider: model.provider, model: model.model, auth: model.auth }, exploration, durationMs: Date.now() - startedAt }; + return { outputDirectory, mapPath: store.mapPath, app: { ...app, platform }, session, runtime, model: { provider: strategy.provider, model: strategy.model, auth: strategy.auth }, exploration, durationMs: Date.now() - startedAt }; } catch (error) { primaryError = toCiError(error, { code: "EXPLORE_SESSION_FAILED", stage: "exploration", nextAction: "Inspect the diagnostic, correct the failing prerequisite, and rerun explore:session. The session was left running." }); throw primaryError; @@ -775,11 +653,10 @@ export async function runBinaryExploration(options: BinaryExploreOptions): Promi const environment = options.environment ?? process.env; const runner = options.runner ?? defaultRunner; const limits = validatedLimits(options); - const replay = replayPlan(options); - const flows = flowsPlan(options); + const choice = chooseStrategy(options); // The archive's name gives its platform before it is read (inspectArchive confirms it), so a secret only another platform's steps type is not required. const namedPlatform = options.platform ?? (() => { try { return archiveKind(options.archive) === "android-apk" ? "android" as const : "ios" as const; } catch { return undefined; } })(); - const inputs = hintInputs(options.hints, environment, replay, flows, namedPlatform); + const inputs = strategyInputs(choice, options.hints, environment, namedPlatform); const local = options.simulator === "local"; // A local emulator runs this Mac's ABI (arm64-v8a on Apple silicon); EAS's runs x86_64. const emulatorAbi = local ? hostEmulatorAbi() : "x86_64"; @@ -796,8 +673,7 @@ export async function runBinaryExploration(options: BinaryExploreOptions): Promi say(`${archive.appName} ${archive.version ?? ""} (${bundleId}, ${platform}, ${(archive.bytes / 1048576).toFixed(0)} MB, sha256 ${archive.sha256.slice(0, 12)})`); if (archive.missingLibs?.length && emulatorAbi === "x86_64") say(`warning: lib/x86_64 lacks ${archive.missingLibs.slice(0, 4).join(", ")}${archive.missingLibs.length > 4 ? ` and ${archive.missingLibs.length - 4} more` : ""} that another ABI ships; the app may stop with UnsatisfiedLinkError on the emulator`); - const model = replay ? replayModel(replay, platform, limits) : flows ? await flowsModel(flows, platform, limits, options, environment, say, { name: archive.appName }) - : await preflightModelSession(options, environment, say); + const strategy = await startStrategy(choice, { platform, limits, options, environment, say, app: { name: archive.appName } }); const outputDirectory = await realpath(await mkdir(options.output, { recursive: true }).then(() => options.output)); const store = new ExploreStore(outputDirectory); @@ -837,7 +713,7 @@ export async function runBinaryExploration(options: BinaryExploreOptions): Promi ...(appIcon ? { appIcon } : {}), app: { name: archive.appName, ...(archive.version ? { version: archive.version } : {}) }, // A local device has the app installed by simctl or adb before the controller starts. ...(localDevice ? { local: localDevice } : { install: { path: installPath, kind: archive.kind } }), - work, runner, environment, secrets: [environment.EXPO_TOKEN ?? "", connection.daemonAuthToken, ...inputs.values].filter(Boolean), inputs, say, now, limits, store, map, model, + work, runner, environment, secrets: [environment.EXPO_TOKEN ?? "", connection.daemonAuthToken, ...inputs.values].filter(Boolean), inputs, say, now, limits, store, map, strategy, urlSchemes: archive.urlSchemes, sessionJson, ...(options.launchUrl ? { launchUrl: options.launchUrl } : {}), ...(options.hints ? { hints: options.hints } : {}), ...(options.allowDestructive ? { allowDestructive: options.allowDestructive } : {}) }); if (local) { @@ -920,7 +796,7 @@ export async function runBinaryExploration(options: BinaryExploreOptions): Promi } if (primaryError) throw primaryError; return { outputDirectory, mapPath: store.mapPath, archive, session: { id: sessionId!, platform, device, owner, project }, runtime: runtime!, - model: { provider: model.provider, model: model.model, auth: model.auth }, exploration: exploration!, durationMs: Date.now() - startedAt }; + model: { provider: strategy.provider, model: strategy.model, auth: strategy.auth }, exploration: exploration!, durationMs: Date.now() - startedAt }; } function lastLine(text: string): string { return text.trim().split(/\r?\n/).filter(Boolean).at(-1) ?? ""; } @@ -935,10 +811,6 @@ async function easLoggedIn(runner: typeof defaultRunner, environment: NodeJS.Pro } catch { return false; } } -/** Schemes the OS routes to a browser, the store, Settings, or another app: opening one hands the session away from the app under test. */ -const SYSTEM_URL_SCHEMES = /^(https?|ftp|file|javascript|data|blob|mailto|tel|telprompt|facetime(-audio)?|sms|mms|imessage|itms(-apps|-services|-beta)?|macappstore|app-settings|prefs|shortcuts|maps|music|videos|photos-redirect|x-apple-[a-z-]+|market|intent|content|android-app|geo|package):$/i; - -/** The model may only open app-owned URLs; an https page or a store link would hand the session to another app. */ const RUNNER_BUSY = /still finishing a previous command/i; const sleepSync = (ms: number): void => { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); }; @@ -957,16 +829,6 @@ export function retryWhileBusy(run: () => T, pause: (ms: number) => void, bud } } -export function assertAppUrl(url: string, appSchemes: readonly string[] = []): void { - let parsed: URL; - try { parsed = new URL(url); } catch { throw binaryError("AGENT_URL_INVALID", `Not a URL: ${url}`, "Pass a custom-scheme app URL."); } - if (SYSTEM_URL_SCHEMES.test(parsed.protocol)) throw binaryError("AGENT_URL_NOT_ALLOWED", `${parsed.protocol} URLs leave the app; only the app's own URL schemes are allowed.`, "Use the app's own URL scheme."); - const scheme = parsed.protocol.slice(0, -1).toLowerCase(); - if (appSchemes.length && !appSchemes.includes(scheme)) { - throw binaryError("AGENT_URL_NOT_ALLOWED", `${parsed.protocol} is not a URL scheme this app registers (${appSchemes.join(", ")}).`, "Use one of the app's own URL schemes."); - } -} - export const BLANK_SCREENSHOT_MESSAGE = "The app blocks screen capture on this screen (Android FLAG_SECURE), so its screenshot came back blank."; export { solidColour }; diff --git a/src/binary/strategy.ts b/src/binary/strategy.ts new file mode 100644 index 0000000..0db9523 --- /dev/null +++ b/src/binary/strategy.ts @@ -0,0 +1,172 @@ +import { createCiAgentSession, type CiAgentSession } from "../capture/agent/session.js"; +import { createLocalCiAgentSession } from "../capture/agent/session-local.js"; +import type { CiAgentProviderName } from "../capture/agent/provider.js"; +import { createExplorationInputs } from "../config/exploration-inputs.js"; +import { CiError } from "../shared/ci-errors.js"; +import type { Platform } from "../types.js"; +import { assertAppUrl } from "./app-url.js"; +import { assertFlowsFit, createFlowSession, flowGoals, flowSetup, flowsNeedModel, parseFlowPlan, type FlowPlan } from "./flows.js"; +import { assertReplayFits, createReplaySession, parseReplayPlan, replaySetup, type ReplayPlan } from "./replay.js"; +import type { BinaryExploreOptions } from "./run.js"; + +/** + * How an exploration decides what to do on the device. The explorer (`runBlackBoxExploration`) owns the device and the + * tools that drive it, the budgets, what is refused, and what is recorded; a strategy only calls those tools, through + * `session.run`. A model explores on its own, a replay plan takes fixed steps, and a flows file takes named flows. + * A new strategy is one more case here: what `chooseStrategy` checks before any device starts, and the session + * `startStrategy` runs. + */ +export type StrategyChoice = { kind: "model" } | { kind: "replay"; plan: ReplayPlan } | { kind: "flows"; plan: FlowPlan }; + +export type ModelAuth = "local-login" | "api-key"; + +/** A strategy ready to run, and how the report names it. */ +export type ExplorationStrategy = { + session: Pick; + provider: CiAgentProviderName | "replay" | "flows"; + model: string; + auth: ModelAuth | "none"; + /** A sentence a budget stop's gap adds to its own: the planned screens or checkpoints the run did not reach. */ + budgetNote?: () => string | undefined; + /** + * Takes the explorer's flow tools (start_flow, checkpoint, end_flow), with a relaunch before each flow after the first, + * and names each flow's checkpoints it never attempted, for the explorer to record as missing. + */ + flows?: () => Array<{ flow: string; checkpoints: string[] }>; +}; + +export const DEFAULT_MODELS: Record = { anthropic: "claude-sonnet-4-5", openai: "gpt-6.1-sol" }; + +type StrategyOptions = Pick; +type ModelSessionOptions = Pick; + +function strategyError(code: string, message: string, nextAction: string): CiError { + return new CiError({ code, stage: "session", message, nextAction }); +} + +/** The turns the model gets: three for each action and two for each screen, plus a few to look around and finish. */ +export const turnBudget = (limits: { maxScreens: number; maxActions: number }): number => limits.maxActions * 3 + limits.maxScreens * 2 + 20; + +/** + * The strategy the options ask for, checked before any device starts. A plan file is checked here as well as by the + * CLI, for callers that pass one directly. + */ +export function chooseStrategy(options: StrategyOptions): StrategyChoice { + if (options.replay && options.flows) throw strategyError("FLOWS_WITH_REPLAY", "--flows and --replay each decide every step, so only one of them can run.", "Drop --replay, or move its steps into a flow."); + if (options.replay) return { kind: "replay", plan: replayPlan(options.replay, options) }; + if (options.flows) return { kind: "flows", plan: flowsPlan(options.flows, options) }; + return { kind: "model" }; +} + +/** + * A replay plan takes the model's place, so the options that only steer a model would do nothing; the plan's own fill + * and type steps name the secrets it types. + */ +function replayPlan(value: ReplayPlan, options: StrategyOptions): ReplayPlan { + const ignored = (["provider", "model", "hints"] as const).filter(name => options[name] !== undefined); + if (ignored.length) { + throw strategyError("REPLAY_OPTIONS_IGNORED", `--replay takes the model's place, so ${ignored.map(name => `--${name}`).join(" and ")} would do nothing.`, + `Drop ${ignored.map(name => `--${name}`).join(" and ")}. A {{secret:NAME}} in the plan's fill and type steps is read from the environment, as one in --hints is.`); + } + const plan = parseReplayPlan(value); + for (const [index, step] of plan.steps.entries()) { + if (!("action" in step) || step.action !== "open") continue; + try { assertAppUrl(step.url); } + catch (error) { throw new CiError({ code: "REPLAY_PLAN_INVALID", stage: "exploration", message: `Step ${index + 1}: ${error instanceof Error ? error.message : String(error)}`, nextAction: "Open only the app's own URLs in the replay plan." }); } + } + return plan; +} + +/** + * A flows file decides every step, as a replay plan does, but a step it describes in words is carried out by a model, + * so --provider, --model and --hints apply only when it has one. + */ +function flowsPlan(value: FlowPlan, options: StrategyOptions): FlowPlan { + const plan = parseFlowPlan(value); + const ignored = flowsNeedModel(plan) ? [] : (["provider", "model", "hints"] as const).filter(name => options[name] !== undefined); + if (ignored.length) { + throw strategyError("FLOWS_OPTIONS_IGNORED", `Every step of the flows file is exact, so ${ignored.map(name => `--${name}`).join(" and ")} would do nothing.`, + `Drop ${ignored.map(name => `--${name}`).join(" and ")}, or describe a step in words ({"do": "…"}) for a model to carry out.`); + } + for (const [index, flow] of plan.flows.entries()) for (const [stepIndex, step] of flow.steps.entries()) { + if (!("action" in step) || step.action !== "open") continue; + try { assertAppUrl(step.url); } + catch (error) { throw new CiError({ code: "FLOWS_PLAN_INVALID", stage: "exploration", message: `Flow ${index + 1} (${flow.name}), step ${stepIndex + 1}: ${error instanceof Error ? error.message : String(error)}`, nextAction: "Open only the app's own URLs in the flows file." }); } + } + return plan; +} + +/** + * App-test inputs: each {{secret:NAME}} in the hints, and in the strategy's own fill and type steps and described steps, + * names an environment variable whose value Explore types. Read before any session starts, so a missing one costs + * nothing; a step limited to another platform than `platform`, when it is known, needs none. + */ +export function strategyInputs(choice: StrategyChoice, hints: string | undefined, environment: NodeJS.ProcessEnv, platform?: Platform): ReturnType { + const setup = choice.kind === "replay" ? replaySetup(choice.plan) : choice.kind === "flows" ? flowSetup(choice.plan, platform) : undefined; + try { return createExplorationInputs({ instructions: [hints ?? "", ...(choice.kind === "flows" ? flowGoals(choice.plan, platform) : [])].join("\n"), ...(setup ? { setup } : {}), environment }); } + catch (error) { + if (!(error instanceof CiError) || !error.code.startsWith("EXPLORATION_SECRET_")) throw error; + const name = typeof error.details?.secretName === "string" ? error.details.secretName : "NAME"; + throw new CiError({ code: error.code, stage: "preflight", message: error.message, ...(error.details ? { details: error.details } : {}), + nextAction: error.code === "EXPLORATION_SECRET_REFERENCE_INVALID" + ? "Write each input in --hints as {{secret:NAME}} with an uppercase environment variable name that is not a provider, Expo, or CI credential." + : `Set ${name} in the environment that runs Explore (export ${name}=…) to the single-line test value, then rerun.` }); + } +} + +/** The session the chosen strategy runs. Model access is checked here, before a simulator is paid for, and only when the strategy uses a model. */ +export async function startStrategy(choice: StrategyChoice, context: { platform: Platform; limits: { maxScreens: number; maxActions: number }; options: StrategyOptions; + environment: NodeJS.ProcessEnv; say: (message: string) => void; app: { name: string } }): Promise { + const { platform, limits, options, environment, say, app } = context; + if (choice.kind === "replay") { + assertReplayFits(choice.plan, platform, { ...limits, maxTurns: turnBudget(limits) }); + const session = createReplaySession(choice.plan, platform); + return { session, budgetNote: session.budgetNote, provider: "replay", model: "replay plan", auth: "none" }; + } + if (choice.kind === "flows") { + assertFlowsFit(choice.plan, platform, { ...limits, maxTurns: turnBudget(limits) }); + const goals = flowsNeedModel(choice.plan, platform) ? await preflightModelSession(options, environment, say) : undefined; + const session = createFlowSession(choice.plan, platform, { app, ...(goals ? { model: goals.session } : {}), ...(options.hints ? { hints: options.hints } : {}) }); + return { session, budgetNote: session.budgetNote, flows: session.notReached, provider: "flows", model: goals ? `${goals.provider} ${goals.model}` : session.model, auth: goals?.auth ?? "none" }; + } + return preflightModelSession(options, environment, say); +} + +function modelSession(provider: CiAgentProviderName, auth: ModelAuth, options: ModelSessionOptions, environment: NodeJS.ProcessEnv) { + const config = { provider, model: options.model ?? DEFAULT_MODELS[provider] }; + const session = options.createSession ? options.createSession(config, { env: environment, auth }) + : auth === "api-key" ? createCiAgentSession(config, { env: environment }) : createLocalCiAgentSession(config, { env: environment }); + return { session, provider, model: config.model, auth }; +} + +/** Local `claude`/`codex` login unless an API key is present; preflight proves access before a simulator is paid for. */ +export function selectModelSession(options: ModelSessionOptions, environment: NodeJS.ProcessEnv) { + const provider: CiAgentProviderName = options.provider ?? (environment.OPENAI_API_KEY?.trim() && !environment.ANTHROPIC_API_KEY?.trim() ? "openai" : "anthropic"); + const keyName = provider === "anthropic" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY"; + return modelSession(provider, environment[keyName]?.trim() ? "api-key" : "local-login", options, environment); +} + +/** + * With --provider or --model, only that choice is tried. Otherwise every model access this shell has is tried + * in turn (API keys, then the local claude and codex logins) and the first that passes preflight runs. + */ +export async function preflightModelSession(options: ModelSessionOptions, environment: NodeJS.ProcessEnv, say: (message: string) => void) { + const candidates = options.provider || options.model ? [selectModelSession(options, environment)] : [ + ...(["anthropic", "openai"] as const).filter(provider => environment[provider === "anthropic" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY"]?.trim()) + .map(provider => modelSession(provider, "api-key", options, environment)), + modelSession("anthropic", "local-login", options, environment), + modelSession("openai", "local-login", options, environment), + ]; + let firstError: unknown; + for (const [index, candidate] of candidates.entries()) { + say(`checking ${candidate.provider} ${candidate.model} (${candidate.auth === "api-key" ? "API key" : "local login"})`); + try { + await candidate.session.preflight(); + return candidate; + } catch (error) { + firstError ??= error; + if (index < candidates.length - 1) say(`${candidate.provider} ${candidate.auth === "api-key" ? "API key" : "local login"} unavailable (${error instanceof Error ? error.message : String(error)}); trying the next`); + } + } + throw firstError; +} diff --git a/src/capture/local-simulator.ts b/src/capture/local-simulator.ts index 6020c13..be7da7f 100644 --- a/src/capture/local-simulator.ts +++ b/src/capture/local-simulator.ts @@ -3,7 +3,7 @@ import { existsSync } from "node:fs"; import { mkdir, readdir, readFile, rm, stat, symlink, writeFile } from "node:fs/promises"; import { homedir } from "node:os"; import { join } from "node:path"; -import { apkAbis } from "../binary/archive.js"; +import { apkAbis } from "../shared/archive-entries.js"; import { exploreCacheRoot } from "../shared/cache-root.js"; import { CiError } from "../shared/ci-errors.js"; import { isRecord, parseJson } from "./common.js"; diff --git a/src/ci/baseline-wait.ts b/src/ci/baseline-wait.ts index 5bd7f9d..ac180b1 100644 --- a/src/ci/baseline-wait.ts +++ b/src/ci/baseline-wait.ts @@ -7,7 +7,7 @@ import { CiError, environmentSecrets } from "../shared/ci-errors.js"; import { silentProgress, type Progress } from "../shared/progress.js"; import { githubRunLink, safeText, silentTimings, type Timings } from "../shared/timings.js"; import { retryGitHubLookup } from "../report/github-lookup.js"; -import { safeRelativePath } from "../binary/archive.js"; +import { safeRelativePath } from "../shared/archive-entries.js"; import { CAPTURE_CACHE_MAX_BYTES, CAPTURE_CACHE_MAX_FILES } from "./artifact-cache.js"; import { baselineLookup, githubContext, trustedBaselineRun, type GithubArtifact, type GithubRun } from "./workflow-steps.js"; diff --git a/src/shared/archive-entries.ts b/src/shared/archive-entries.ts new file mode 100644 index 0000000..cf60d4b --- /dev/null +++ b/src/shared/archive-entries.ts @@ -0,0 +1,135 @@ +import { createReadStream } from "node:fs"; +import { lstat, open } from "node:fs/promises"; +import { basename } from "node:path"; +import { createGunzip } from "node:zlib"; +import { Reader, Uint8ArrayWriter, ZipReader, configure } from "@zip.js/zip.js"; +import { CiError } from "./ci-errors.js"; + +/** + * Reads the entries of tar.gz and zip archives without unpacking them: a prebuilt app, a downloaded controller, a + * baseline artifact. Nothing here writes a file; callers place entries themselves, through safeRelativePath. + */ + +export const MAX_METADATA_BYTES = 4 * 1024 * 1024; +export const MAX_ARCHIVE_ENTRIES = 200_000; + +export function archiveError(code: string, message: string, nextAction: string, details?: Record): CiError { + return new CiError({ code, stage: "archive", message, nextAction, ...(details ? { details } : {}) }); +} + +// ---- tar.gz ------------------------------------------------------------------------------------- + +type TarVisitor = (entry: { name: string; type: string; size: number; mode: number }, contents: () => Promise, skip: () => Promise) => Promise; + +/** Walk a gzip tarball once; `visit` returns true to stop. Long GNU names are honoured; nothing is written. */ +export async function walkTar(path: string, visit: TarVisitor): Promise { + const stream = createReadStream(path).pipe(createGunzip()); + const iterator = stream[Symbol.asyncIterator](); + let buffer = Buffer.alloc(0); + let done = false; + const fill = async (needed: number) => { + while (buffer.length < needed && !done) { + const next = await iterator.next(); + if (next.done) { done = true; break; } + buffer = Buffer.concat([buffer, next.value as Buffer]); + } + }; + const take = async (count: number): Promise => { + await fill(count); + if (buffer.length < count) throw archiveError("ARCHIVE_INVALID", "The tarball ended unexpectedly.", "Pass a complete .tar.gz archive."); + const out = buffer.subarray(0, count); buffer = buffer.subarray(count); return Buffer.from(out); + }; + const drop = async (count: number) => { + let remaining = count; + while (remaining > 0) { + await fill(Math.min(remaining, 1 << 20)); + const step = Math.min(remaining, buffer.length); + if (!step) throw archiveError("ARCHIVE_INVALID", "The tarball ended unexpectedly.", "Pass a complete .tar.gz archive."); + buffer = buffer.subarray(step); remaining -= step; + } + }; + let longName: string | undefined; + let entries = 0; + try { + for (;;) { + await fill(512); + if (buffer.length < 512 || buffer.subarray(0, 512).every(byte => byte === 0)) return; + const header = await take(512); + const field = (offset: number, length: number) => header.subarray(offset, offset + length).toString("utf8").replace(/\0.*$/s, ""); + const size = parseInt(field(124, 12).trim() || "0", 8); + const type = field(156, 1) || "0"; + const prefix = field(345, 155); + const name = longName ?? (prefix ? `${prefix}/${field(0, 100)}` : field(0, 100)); + longName = undefined; + const padded = Math.ceil(size / 512) * 512; + if (type === "L") { longName = (await take(padded)).subarray(0, size).toString("utf8").replace(/\0+$/, ""); continue; } + if (++entries > MAX_ARCHIVE_ENTRIES) throw archiveError("ARCHIVE_TOO_LARGE", "The tarball has too many entries.", "Pass a smaller archive."); + let consumed = false; + const stop = await visit({ name, type, size, mode: parseInt(field(100, 8).trim() || "0", 8) }, + async () => { consumed = true; return (await take(padded)).subarray(0, size); }, + async () => { consumed = true; await drop(padded); }); + if (!consumed) await drop(padded); + if (stop) return; + } + } finally { await iterator.return?.(undefined); } +} + +export function safeRelativePath(name: string): string | undefined { + const parts = name.replace(/^\.\//, "").split("/").filter(part => part.length); + if (!parts.length || parts.some(part => part === "." || part === ".." || part.includes("\\") || part.includes("\0"))) return undefined; + return parts.join("/"); +} + +// ---- zip ---------------------------------------------------------------------------------------- + +export type ZipEntry = Awaited["getEntries"]>>[number]; + +/** zip.js reads the central directory and entries by range, so a 200 MB APK is never loaded whole. */ +class FileRangeReader extends Reader { + constructor(private readonly path: string) { super(path); } + override async init(): Promise { + this.size = (await lstat(this.path)).size; + } + /** Each range opens its own descriptor: nothing stays open after the entries are read, so no handle leaks when a caller stops early. */ + override async readUint8Array(index: number, length: number): Promise { + const out = new Uint8Array(length); + const handle = await open(this.path, "r"); + try { + let offset = 0; + while (offset < length) { + const { bytesRead } = await handle.read(out, offset, length - offset, index + offset); + if (bytesRead === 0) break; + offset += bytesRead; + } + return offset === length ? out : out.subarray(0, offset); + } finally { await handle.close(); } + } +} + +const readers = new WeakMap(); + +/** The ABIs an APK ships native libraries for (`lib//`), without reading its manifest. */ +export async function apkAbis(path: string): Promise { + return [...new Set((await zipEntries(path)).map(entry => /^lib\/([^/]+)\//.exec(entry.filename)?.[1]).filter((abi): abi is string => !!abi))].sort(); +} + +export async function zipEntries(path: string): Promise { + configure({ useWebWorkers: false }); + const reader = new FileRangeReader(path); + const zip = new ZipReader(reader, { useWebWorkers: false }); + try { + const entries = await zip.getEntries(); + if (entries.length > MAX_ARCHIVE_ENTRIES) throw archiveError("ARCHIVE_TOO_LARGE", "The archive has too many entries.", "Pass a smaller archive."); + for (const entry of entries) readers.set(entry, reader); + return entries; + } catch (error) { + if (error instanceof CiError) throw error; + throw archiveError("ARCHIVE_INVALID", `${basename(path)} is not a readable ZIP archive: ${error instanceof Error ? error.message : String(error)}`, "Pass a valid .ipa or .apk."); + } +} + +export async function readZipEntry(entry: ZipEntry, maxBytes = MAX_METADATA_BYTES): Promise { + if (entry.directory || !entry.getData) throw archiveError("ARCHIVE_INVALID", `${entry.filename} is not a file entry.`, "Pass a valid archive."); + if (entry.uncompressedSize > maxBytes) throw archiveError("ARCHIVE_INVALID", `${entry.filename} is too large.`, "Pass a valid archive."); + return entry.getData(new Uint8ArrayWriter(), { checkSignature: true, useWebWorkers: false }); +} diff --git a/src/shared/controller-cache.ts b/src/shared/controller-cache.ts index 2db6e35..3b90c40 100644 --- a/src/shared/controller-cache.ts +++ b/src/shared/controller-cache.ts @@ -11,7 +11,7 @@ import { chmod, mkdir, open, readdir, rename, rm, stat, writeFile } from "node:f import { homedir } from "node:os"; import { dirname, join, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; -import { safeRelativePath, walkTar } from "../binary/archive.js"; +import { safeRelativePath, walkTar } from "./archive-entries.js"; import { CiError } from "./ci-errors.js"; import { AGENT_DEVICE_VERSION } from "./agent-device-version.js"; import { ARGENT_VERSION } from "./argent-version.js"; diff --git a/test/helpers/flows-host.ts b/test/helpers/flows-host.ts index 0ed5826..b1f1162 100644 --- a/test/helpers/flows-host.ts +++ b/test/helpers/flows-host.ts @@ -2,7 +2,7 @@ // instance), and every tsImport call loads its own copy of each module. export { runBlackBoxExploration } from "../../src/binary/explorer.js"; export { assertFlowsFit, createFlowSession, flowsNeedModel, parseFlowPlan } from "../../src/binary/flows.js"; -export { hintInputs } from "../../src/binary/run.js"; +export { chooseStrategy, startStrategy, strategyInputs } from "../../src/binary/strategy.js"; export { runBoundedSession } from "../../src/capture/agent/session-shared.js"; export { ExploreStore, validateMap } from "../../src/map/store.js"; export { CiError } from "../../src/shared/ci-errors.js"; diff --git a/test/node/binary-explorer.test.mjs b/test/node/binary-explorer.test.mjs index 9a6a416..160afe4 100644 --- a/test/node/binary-explorer.test.mjs +++ b/test/node/binary-explorer.test.mjs @@ -9,7 +9,8 @@ const { runBlackBoxExploration, screenEvidenceKey, elementsOf, fieldName, system const { ExploreStore, validateMap } = await tsImport("../../src/map/store.ts", import.meta.url); const { CONTRACT } = await tsImport("../../src/types.ts", import.meta.url); const tsRun = await tsImport("../../src/binary/run.ts", import.meta.url); -const { assertAppUrl, describeForeground, foreignNode, guardPositionals, leftForeground, retryWhileBusy } = tsRun; +const { describeForeground, foreignNode, guardPositionals, leftForeground, retryWhileBusy } = tsRun; +const { assertAppUrl } = await tsImport("../../src/binary/app-url.ts", import.meta.url); const PNG = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", "base64"); const SCREENS = { @@ -246,7 +247,8 @@ test("a {{secret:NAME}} from the hints is typed on the device, while the model, }); test("hint inputs are read before a session starts and name what to set", async () => { - const { hintInputs } = await tsImport("../../src/binary/run.ts", import.meta.url); + const { strategyInputs } = await tsImport("../../src/binary/strategy.ts", import.meta.url); + const hintInputs = (hints, environment) => strategyInputs({ kind: "model" }, hints, environment); assert.deepEqual(hintInputs("Explore the settings", {}).names, []); assert.deepEqual(hintInputs("Use {{secret:APP_TEST_EMAIL}} and {{secret:APP_TEST_PASSWORD}}", { APP_TEST_EMAIL: "a@example.com", APP_TEST_PASSWORD: "pw" }).names, ["APP_TEST_EMAIL", "APP_TEST_PASSWORD"]); assert.throws(() => hintInputs("Use {{secret:APP_TEST_PASSWORD}}", {}), error => error.code === "EXPLORATION_SECRET_MISSING" && /export APP_TEST_PASSWORD=/.test(error.nextAction)); @@ -854,7 +856,7 @@ test("a coordinate tap with no description on an icon-font button is named by wh }); test("with no --provider, model access falls through API keys and local logins to the first that passes preflight", async () => { - const { preflightModelSession } = await tsImport("../../src/binary/run.ts", import.meta.url); + const { preflightModelSession } = await tsImport("../../src/binary/strategy.ts", import.meta.url); const tried = []; const working = "anthropic/local-login"; const createSession = (config, { auth }) => ({ async preflight() { @@ -870,7 +872,7 @@ test("with no --provider, model access falls through API keys and local logins t }); test("an explicit --provider is the only model access tried, and its preflight error is the one reported", async () => { - const { preflightModelSession } = await tsImport("../../src/binary/run.ts", import.meta.url); + const { preflightModelSession } = await tsImport("../../src/binary/strategy.ts", import.meta.url); const tried = []; const createSession = (config, { auth }) => ({ async preflight() { tried.push(`${config.provider}/${auth}`); throw new Error("refused"); } }); await assert.rejects(preflightModelSession({ provider: "openai", createSession }, { ANTHROPIC_API_KEY: "key" }, () => {}), /refused/); @@ -878,7 +880,7 @@ test("an explicit --provider is the only model access tried, and its preflight e }); test("when nothing passes preflight, the first failure is reported", async () => { - const { preflightModelSession } = await tsImport("../../src/binary/run.ts", import.meta.url); + const { preflightModelSession } = await tsImport("../../src/binary/strategy.ts", import.meta.url); const createSession = (config, { auth }) => ({ async preflight() { throw new Error(`${config.provider} ${auth} refused`); } }); await assert.rejects(preflightModelSession({ createSession }, {}, () => {}), /anthropic local-login refused/); }); diff --git a/test/node/binary-flows.test.mjs b/test/node/binary-flows.test.mjs index 3b8ce14..fce8f86 100644 --- a/test/node/binary-flows.test.mjs +++ b/test/node/binary-flows.test.mjs @@ -5,7 +5,7 @@ import { join } from "node:path"; import test from "node:test"; import { tsImport } from "tsx/esm/api"; -const { runBlackBoxExploration, assertFlowsFit, createFlowSession, flowsNeedModel, parseFlowPlan, hintInputs, runBoundedSession, ExploreStore, validateMap, CiError, CONTRACT } = +const { runBlackBoxExploration, assertFlowsFit, createFlowSession, flowsNeedModel, parseFlowPlan, chooseStrategy, startStrategy, strategyInputs, runBoundedSession, ExploreStore, validateMap, CiError, CONTRACT } = await tsImport("../helpers/flows-host.ts", import.meta.url); const PNG = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", "base64"); @@ -185,8 +185,21 @@ test("a flows file is checked before any device starts", () => { test("a {{secret:NAME}} in a described step must be set before any device starts, as one in the hints must", () => { const plan = parseFlowPlan({ flows: [{ name: "Sign in", steps: [{ do: "Sign in with {{secret:TEST_PASSWORD}}" }, { checkpoint: "Home" }] }] }); - assert.throws(() => hintInputs(undefined, {}, undefined, plan), error => error.code === "EXPLORATION_SECRET_MISSING" && /TEST_PASSWORD/.test(error.nextAction)); - assert.deepEqual(hintInputs(undefined, { TEST_PASSWORD: "hunter2" }, undefined, plan).names, ["TEST_PASSWORD"]); + assert.throws(() => strategyInputs({ kind: "flows", plan }, undefined, {}), error => error.code === "EXPLORATION_SECRET_MISSING" && /TEST_PASSWORD/.test(error.nextAction)); + assert.deepEqual(strategyInputs({ kind: "flows", plan }, undefined, { TEST_PASSWORD: "hunter2" }).names, ["TEST_PASSWORD"]); +}); + +test("the strategy is chosen before any device starts, and options that would do nothing with it are refused", () => { + const exact = { flows: [{ name: "A", steps: [{ action: "tap", label: "Settings" }, { checkpoint: "Settings" }] }] }; + const described = { flows: [{ name: "A", steps: [{ do: "Open the settings" }, { checkpoint: "Settings" }] }] }; + assert.deepEqual(chooseStrategy({}), { kind: "model" }); + assert.equal(chooseStrategy({ flows: exact }).kind, "flows"); + assert.equal(chooseStrategy({ flows: described, provider: "openai", hints: "Use the test account" }).kind, "flows"); + assert.throws(() => chooseStrategy({ flows: exact, replay: { steps: [{ capture: "Home" }] } }), { code: "FLOWS_WITH_REPLAY" }); + assert.throws(() => chooseStrategy({ flows: exact, model: "gpt-6.1-sol" }), error => error.code === "FLOWS_OPTIONS_IGNORED" && /--model would do nothing/.test(error.message)); + assert.throws(() => chooseStrategy({ replay: { steps: [{ capture: "Home" }] }, hints: "x" }), { code: "REPLAY_OPTIONS_IGNORED" }); + assert.throws(() => chooseStrategy({ flows: { flows: [{ name: "A", steps: [{ action: "open", url: "https://example.com" }, { checkpoint: "Web" }] }] } }), + error => error.code === "FLOWS_PLAN_INVALID" && /Flow 1 \(A\), step 1: https: URLs leave the app/.test(error.message)); }); test("a checkpoint after a relaunch takes a new picture, even of the screen the previous flow ended on untouched", async () => { @@ -264,8 +277,8 @@ test("a described step limited to another platform needs no model, and its secre assert.equal(flowsNeedModel(plan, "ios"), false); assert.equal(flowsNeedModel(plan, "android"), true); assert.doesNotThrow(() => createFlowSession(plan, "ios", { app: { name: "Example" } })); - assert.deepEqual(hintInputs(undefined, {}, undefined, plan, "ios").names, []); - assert.throws(() => hintInputs(undefined, {}, undefined, plan), error => error.code === "EXPLORATION_SECRET_MISSING", "without a platform, every step's secrets are needed"); + assert.deepEqual(strategyInputs({ kind: "flows", plan }, undefined, {}, "ios").names, []); + assert.throws(() => strategyInputs({ kind: "flows", plan }, undefined, {}), error => error.code === "EXPLORATION_SECRET_MISSING", "without a platform, every step's secrets are needed"); }); test("flows whose tool calls cannot fit the turn budget are refused before any device starts", () => { @@ -275,6 +288,14 @@ test("flows whose tool calls cannot fit the turn budget are refused before any d assert.doesNotThrow(() => assertFlowsFit(flows(41), "ios", limits), "41 flows take 124 calls"); }); +test("flows whose described steps run only on another platform start with no model at all", async () => { + const plan = parseFlowPlan({ flows: [{ name: "A", steps: [{ do: "Sign in", platforms: ["android"] }, { action: "tap", label: "Settings" }, { checkpoint: "Settings" }] }] }); + const createSession = () => { throw new Error("no model may be started"); }; + const strategy = await startStrategy({ kind: "flows", plan }, { platform: "ios", limits: { maxScreens: 10, maxActions: 10 }, options: { createSession }, environment: {}, say() {}, app: { name: "Example" } }); + assert.deepEqual([strategy.provider, strategy.model, strategy.auth], ["flows", "flows file", "none"]); + assert.deepEqual(strategy.flows(), [{ flow: "A", checkpoints: ["Settings"] }]); +}); + test("a checkpoint whose screen withholds its picture is recorded as missing, its flow goes on, and the summary names it", async () => { const plan = parseFlowPlan({ flows: [{ name: "Home", steps: [{ checkpoint: "Home" }, { action: "wait", milliseconds: 1 }, { checkpoint: "Still home" }, { action: "tap", label: "Settings" }, { checkpoint: "Settings" }] }] }); const { result, map } = await explore(plan, { device: { withholdPicture: [2] } }); @@ -330,7 +351,7 @@ test("a flow with no checkpoint on the platform is skipped, and its steps need n { name: "Settings", steps: [{ action: "tap", label: "Settings" }, { checkpoint: "Settings" }] }, ] }); assert.equal(flowsNeedModel(plan, "ios"), false); - assert.deepEqual(hintInputs(undefined, {}, undefined, plan, "ios").names, []); + assert.deepEqual(strategyInputs({ kind: "flows", plan }, undefined, {}, "ios").names, []); const { result } = await explore(plan); assert.deepEqual(result.flows.map(flow => flow.name), ["Settings"]); assert.match(result.summary, /^Ran 1 flow on ios/);