From c919c42cac5403340bd6bf7239211ec554fd1411 Mon Sep 17 00:00:00 2001 From: Brent Vatne Date: Sat, 24 Oct 2026 14:00:32 -0700 Subject: [PATCH] Explorer: judge a tap by the control under it and by its identifier, and name --allow-destructive in a refusal - riskOf splits identifiers into words before matching, so btn_delete, deleteAccount and sign-out read as destructive; \b does not split them. - A coordinate tap is judged by the smallest hittable control under the point, by its own label and identifier, as well as by the target the model wrote. A tap over no control is still allowed and recorded as unknown. - A control allowed by its label stays allowed however it is targeted, even when its identifier reads as risky. - A refusal tells whoever runs Explore that --allow-destructive allows a safe control; it reaches them through flow and replay results. - The README and the explore-binary skill say that a secret is recognised in the accessibility tree only, so one drawn without an accessible label is not. --- README.md | 4 +-- skills/explore-binary/SKILL.md | 4 +-- src/binary/explorer.ts | 41 ++++++++++++++++++------- test/node/binary-explorer.test.mjs | 49 ++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 269f742..47b088e 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,7 @@ Pointed to a build, the [quick start](#quick-start) maps every screen of it, wit - A simulator build. A device `.ipa`, an `.aab`, an unsigned APK, or an APK without native libraries for the emulator (x86_64 on EAS Simulator, `arm64-v8a` locally on Apple silicon; a universal APK has both) is rejected before a device starts. - Platform, bundle id, name, and version are read from the build; you pass nothing else. - A run stops after 15 minutes, 40 screens, or 150 actions (`--max-duration-minutes`, `--max-screens`, `--max-actions`). -- Buttons that delete, pay, sign out, send, or leave the app are never tapped. `--allow-destructive "Reset filters"` allows one by its exact label. +- Buttons that delete, pay, sign out, send, or leave the app are never tapped. Explore judges a control by its label and its identifier (`btn_delete`, `deleteAccount`), including one tapped by position. `--allow-destructive "Reset filters"` allows one by its exact label. - `--hints "…"` tells the model what to focus on or how to sign in ([details](#sign-in-and-other-app-state)). - On EAS Simulator the report is hosted, and the link lasts as long as its EAS Hosting deployment (see [report access](docs/private-review.md#hosted-previews)); `--no-upload` keeps it local. On a local simulator it opens in the local Viewer (`--no-open` or `--json` skips that; `view --dir ` opens it later), and `--upload` hosts it, which needs an EAS login. `--json` prints the result for scripts. - For a link that stays the same from run to run, pass `--alias NAME` and set `EXPLORE_PREVIEW_TOKEN` to a fixed token of 32 characters from a–z and 2–7 (`LC_ALL=C tr -dc a-z2-7 ): boolean { return traits.some(trait => /button|link|tab|adjustable/i.test(trait)); } +const allowed = (label: string, allow: readonly string[]) => allow.some(item => item.toLowerCase() === label.toLowerCase()); + export function riskOf(label: string, allow: readonly string[]): ActionRisk { - if (allow.some(allowed => allowed.toLowerCase() === label.toLowerCase())) return "safe"; - if (DESTRUCTIVE.test(label)) return "destructive"; - if (EXTERNAL.test(label)) return "external"; + if (allowed(label, allow)) return "safe"; + // Identifiers join their words (btn_delete, deleteAccount, sign-out), which \b does not split. + const words = label.replace(/(\p{Ll}|\d)(\p{Lu})/gu, "$1 $2").replace(/[_.-]+/g, " "); + if (DESTRUCTIVE.test(words)) return "destructive"; + if (EXTERNAL.test(words)) return "external"; return "safe"; } +const RISK_ORDER: readonly ActionRisk[] = ["safe", "unknown", "external", "destructive"]; +const worstRisk = (...risks: ActionRisk[]): ActionRisk => risks.reduce((worst, risk) => RISK_ORDER.indexOf(risk) > RISK_ORDER.indexOf(worst) ? risk : worst, "safe"); + /** Structure plus stable text, with values and numerals masked, so a clock tick or a counter does not create a new screen. */ /** * A text field's label is whatever it shows: its placeholder while empty (Android sets `hintShowing`), then the typed text. @@ -660,7 +669,11 @@ export async function runBlackBoxExploration(options: BlackBoxExplorerOptions): if (input.frameId !== undefined) { const point = { action: "tap" as const, frameId: text(input.frameId, "frameId", 256), x: Number(input.x), y: Number(input.y) }; const target = input.target !== undefined ? cleanLabel(text(input.target, "target", 80)) : ""; - return { action: point, kind: "press", label: target || placeUnder(point) || `tap ${point.x.toFixed(2)},${point.y.toFixed(2)}`, risk: target ? riskOf(target, allow) : "unknown" }; + // The control under the point is judged by its own label and identifier too, not only by the model's description of it. + const under = nodeUnder(point), underLabel = str(under?.label); + const risk = underLabel && allowed(underLabel, allow) ? "safe" + : worstRisk(target ? riskOf(target, allow) : "unknown", ...[underLabel, str(under?.identifier)].flatMap(name => name ? [riskOf(name, allow)] : [])); + return { action: point, kind: "press", label: target || placeUnder(point) || `tap ${point.x.toFixed(2)},${point.y.toFixed(2)}`, risk }; } if (!selector) throw failure("AGENT_ACTION_INVALID", "tap needs a label, an identifier, or frame coordinates."); return { action: { action: "tap", ...selector } as BlackBoxAction, kind: "press", label, risk }; @@ -714,13 +727,12 @@ export async function runBlackBoxExploration(options: BlackBoxExplorerOptions): return { action: "tap", frameId: latest.frameId ?? "current", x: (bounds.x + bounds.width / 2) / extent.width, y: (bounds.y + bounds.height / 2) / extent.height }; }; - /** A coordinate tap the model did not describe, on a node the map has no name for (an icon-font button in a header), is - * named by what it is and where it sits: "icon button, top right" says more than the coordinates did. */ - const placeUnder = (point: { x: number; y: number }): string | undefined => { + /** The smallest hittable control at a frame point in the latest tree, whether or not the map has recorded it. */ + const nodeUnder = (point: { x: number; y: number }): Record | undefined => { const extent = latest ? frameExtent(latest.nodes) : undefined; if (!extent) return undefined; const x = point.x * extent.width, y = point.y * extent.height; - const under = latest!.nodes + return latest!.nodes .filter(node => { const rect = node.rect as { x: number; y: number; width: number; height: number } | undefined; return !!rect && node.hittable === true && x >= rect.x && x <= rect.x + rect.width && y >= rect.y && y <= rect.y + rect.height @@ -730,6 +742,12 @@ export async function runBlackBoxExploration(options: BlackBoxExplorerOptions): const ra = a.rect as { width: number; height: number }, rb = b.rect as { width: number; height: number }; return ra.width * ra.height - rb.width * rb.height; })[0]; + }; + + /** A coordinate tap the model did not describe, on a node the map has no name for (an icon-font button in a header), is + * named by what it is and where it sits: "icon button, top right" says more than the coordinates did. */ + const placeUnder = (point: { x: number; y: number }): string | undefined => { + const under = nodeUnder(point); if (!under) return undefined; const role = roleOf(under); const label = str(under.label) ?? ""; @@ -805,7 +823,8 @@ export async function runBlackBoxExploration(options: BlackBoxExplorerOptions): : elementUnder(chosen.action, fromScreenId); const { action, kind } = chosen; const label = named ? named.label : chosen.label; - const risk = named && chosen.risk !== "destructive" && chosen.risk !== "external" ? riskOf(named.label, allow) : chosen.risk; + // A control allowed by its label is allowed however it is targeted, even when its identifier reads as risky. + const risk = named && allowed(named.label, allow) ? "safe" : named && chosen.risk !== "destructive" && chosen.risk !== "external" ? riskOf(named.label, allow) : chosen.risk; const startedAt = timestamp(); const attemptId = `att_${randomUUID().slice(0, 12)}`; const matched = map.frontier.find(item => item.status === "pending" && item.screenId === fromScreenId && item.actionKind === kind @@ -817,8 +836,8 @@ export async function runBlackBoxExploration(options: BlackBoxExplorerOptions): startedAt, completedAt: timestamp(), reason }); await persist(); return toolResult({ ok: false, blocked: true, reason: risk === "destructive" - ? `"${label}" looks destructive (delete/reset/sign out/purchase/send). Explore does not activate such controls. Choose another control.` - : `"${label}" looks like it opens another app (browser/store/share/mail/maps). Explore does not activate such controls. Choose another control.` }); + ? `"${label}" looks destructive (delete/reset/sign out/purchase/send). Explore does not activate such controls. Choose another control. ${ALLOW_HINT}` + : `"${label}" looks like it opens another app (browser/store/share/mail/maps). Explore does not activate such controls. Choose another control. ${ALLOW_HINT}` }); } // The map keeps the reference the model typed; only the device gets the value. const typed = action.action === "fill" || action.action === "type" ? { ...action, text: inputs.resolve(action.text) } : action; diff --git a/test/node/binary-explorer.test.mjs b/test/node/binary-explorer.test.mjs index 9a6a416..7ec1ddc 100644 --- a/test/node/binary-explorer.test.mjs +++ b/test/node/binary-explorer.test.mjs @@ -1207,3 +1207,52 @@ test("a replay plan's {{secret:NAME}} inputs are read before any device starts a assert.equal(map.actionAttempts[0].action.input.value, "{{secret:APP_TEST_PASSWORD}}"); assert.ok(!JSON.stringify(map).includes("hunter2-test")); }); + +test("risk words are found inside identifiers that join them", async () => { + const { riskOf } = await tsImport("../../src/binary/explorer.ts", import.meta.url); + for (const label of ["btn_delete", "deleteAccount", "sign-out", "logOutButton", "settings.reset"]) assert.equal(riskOf(label, []), "destructive", label); + assert.equal(riskOf("openInSafari", []), "external"); + for (const label of ["postsTab", "settings_button", "Profile", "MapView"]) assert.equal(riskOf(label, []), "safe", label); + assert.equal(riskOf("resetFilters", ["resetFilters"]), "safe", "an allowed identifier"); +}); + +/** + * A screen whose only control is a hittable custom view the map does not record as a control (type Other, as many + * custom pressables are): its risk is in its own label and identifier. + */ +function iconScreen(icon) { + return [{ type: "Window", rect: { x: 0, y: 0, width: 400, height: 800 } }, { type: "StaticText", label: "Inbox" }, + { type: "Other", hittable: true, rect: { x: 340, y: 40, width: 44, height: 44 }, ...icon }]; +} + +test("a coordinate tap is judged by the control under the point, whatever the model calls it", async () => { + for (const icon of [{ identifier: "btn_delete" }, { label: "Delete conversation" }]) { + const { calls, acted, map } = await harness([["observe", { includeImage: true }], ["act", { action: "tap", frameId: "frame-0", x: 0.9, y: 0.075, target: "Profile" }], ["finish", { summary: "x" }]], + { observe: async includeImage => ({ nodes: iconScreen(icon), capturedAt: new Date().toISOString(), ...(includeImage ? { frameId: "frame-0", png: PNG, imageSha256: "d".repeat(64) } : {}) }) }); + assert.equal(acted.length, 0, JSON.stringify(icon)); + assert.equal(calls[1].result.blocked, true); + assert.match(calls[1].result.reason, /looks destructive.*--allow-destructive/); + assert.equal(map.actionAttempts.at(-1).risk, "destructive"); + } +}); + +test("a coordinate tap on a control allowed by its label goes through, even when its identifier reads as risky", async () => { + const { acted } = await harness([["observe", { includeImage: true }], ["act", { action: "tap", frameId: "frame-0", x: 0.9, y: 0.075 }], ["finish", { summary: "x" }]], + { allowDestructive: ["Reset filters"], + observe: async includeImage => ({ nodes: iconScreen({ label: "Reset filters", identifier: "resetFilters" }), capturedAt: new Date().toISOString(), ...(includeImage ? { frameId: "frame-0", png: PNG, imageSha256: "d".repeat(64) } : {}) }) }); + assert.equal(acted.length, 1); +}); + +test("a coordinate tap with no control under the point is still recorded as unknown, not refused", async () => { + const { acted, map } = await harness([["observe", { includeImage: true }], ["act", { action: "tap", frameId: "frame-0", x: 0.5, y: 0.5 }], ["finish", { summary: "x" }]], + { observe: async includeImage => ({ nodes: iconScreen({ identifier: "btn_delete" }), capturedAt: new Date().toISOString(), ...(includeImage ? { frameId: "frame-0", png: PNG, imageSha256: "d".repeat(64) } : {}) }) }); + assert.equal(acted.length, 1); + assert.equal(map.actionAttempts.at(-1).risk, "unknown"); +}); + +test("a tap by identifier on a control allowed by its label goes through", async () => { + const screen = [{ type: "StaticText", label: "Filters" }, { type: "Button", label: "Reset filters", identifier: "resetFilters" }]; + const { acted } = await harness([["observe", { includeImage: true }], ["act", { action: "tap", identifier: "resetFilters" }], ["finish", { summary: "x" }]], + { allowDestructive: ["Reset filters"], observe: async includeImage => ({ nodes: screen, capturedAt: new Date().toISOString(), ...(includeImage ? { frameId: "frame-0", png: PNG, imageSha256: "e".repeat(64) } : {}) }) }); + assert.equal(acted.length, 1); +});