diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b9be9e1..2faf4a83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +- **Semantic page map.** `snapshotTab` reads a tab from `listTabs` without the + debugger and without changing focus. Each entry has a role, an accessible + name, text, state, visibility, and a stable ref. `actOnRef` clicks or types + that ref with the same untrusted path as `softClick` and `softType`, then + reports what changed: the URL, new console errors, and new failed network + requests. - **Non-scriptable pages are reported the same way everywhere.** `chrome://`, DevTools, `view-source:`, the Chrome Web Store, extension pages, and the built-in PDF viewer come back as metadata only (`access: "metadata-only"`, diff --git a/README.md b/README.md index 6160e276..251bfcb6 100644 --- a/README.md +++ b/README.md @@ -101,9 +101,11 @@ setup and a silent one look the same. | `listBrowserTabs` | Tabs with DevTools open, addressed by a DevTools-connected `tabId` | | `listTabs` | Tabs `chrome.tabs` returns (not every open tab), addressed by `chromeTabId` | | `readTab` | Headings, excerpt, and a snapshot of a `chromeTabId`, without the debugger | +| `snapshotTab` | Semantic page map of a `chromeTabId`: role, name, text, state, visibility, and a stable ref | | `navigateBrowser` | Load a URL in a DevTools-connected tab, without focusing it | | `navigateTab` | Load a URL in the background from `listTabs`, visible window only | | `softClick` / `softType` | Untrusted click or type. Result is `worked` or `needs a real gesture` | +| `actOnRef` | Click or type a `snapshotTab` ref, then report URL, console, and failed-request changes | | `takeScreenshot` | Screenshot returned **as an image**, plus its file path | | `refreshBrowser` | Reloads the inspected tab | | `getBrowserStorage` | localStorage, sessionStorage and cookies (values gated) | @@ -123,13 +125,15 @@ clients can auto-approve them safely. ### Tabs without DevTools -`listTabs`, `readTab`, `navigateTab`, `softClick`, and `softType` use `chrome.tabs` and `chrome.scripting`. They do not attach the debugger, so they do not show the debugging banner, and they do not activate a tab or focus a window. +`listTabs`, `readTab`, `snapshotTab`, `navigateTab`, `softClick`, `softType`, and `actOnRef` use `chrome.tabs` and `chrome.scripting`. They do not attach the debugger, so they do not show the debugging banner, and they do not activate a tab or focus a window. `chromeTabId` comes from `listTabs`. `tabId` on the other tools comes from `listBrowserTabs`. They are different id spaces. Pass each id only to the tool that issued it. `listTabs` reports the tabs `chrome.tabs` returns in the windows it returns. Other Arc Spaces, Little Arc, pinned favorites, split-view partners, and slept tabs may be missing. Incognito is skipped. This is not every open tab. Background navigation stays in the visible window until that is measured on Arc. -The popup switch has to be on. Navigation and soft input also need `--allow-input` on `browser-tools-mcp`, or `--allow-actions` on the `browser-tools` CLI. The same origin rules as other input apply. An asleep tab is reported and not reloaded. A soft action says `worked` or `needs a real gesture` and does not claim a failed action worked. +The popup switch has to be on. Navigation, soft input, and `actOnRef` also need `--allow-input` on `browser-tools-mcp`, or `--allow-actions` on the `browser-tools` CLI. The same origin rules as other input apply. An asleep tab is reported and not reloaded. A soft action says `worked` or `needs a real gesture` and does not claim a failed action worked. + +`snapshotTab` returns a semantic page map for one `chromeTabId`: role, accessible name, text, state, visibility, and a stable ref. `actOnRef` uses a ref from that map and the same untrusted click or type path as `softClick` and `softType`. After the action it reports a short delta: whether the URL changed, console errors that appeared, and network requests that failed. Neither call attaches the debugger or changes which tab is active. ### Several tabs at once diff --git a/SECURITY.md b/SECURITY.md index 6c8ea2b0..824e23e1 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -127,8 +127,9 @@ be, both are worth reporting. - Chrome's "started debugging this browser" banner is a browser safety control. It is intentional, and this project never hides it, suppresses it, or documents a setup step that does. Features that can run without the - debugger do so. `listTabs`, `readTab`, `navigateTab`, `softClick`, and - `softType` use ordinary extension APIs and do not call `chrome.debugger`, + debugger do so. `listTabs`, `readTab`, `snapshotTab`, `navigateTab`, + `softClick`, `softType`, and `actOnRef` use ordinary extension APIs and do + not call `chrome.debugger`, `tabs.update({active:true})`, or `windows.update({focused:true})`. When a feature attaches the debugger, the banner stays visible for that attachment, and dismissing it stops the attachment. diff --git a/browser-tools-mcp/src/connector/api-policy.ts b/browser-tools-mcp/src/connector/api-policy.ts index 2f44182e..afa635fb 100644 --- a/browser-tools-mcp/src/connector/api-policy.ts +++ b/browser-tools-mcp/src/connector/api-policy.ts @@ -36,9 +36,11 @@ const AUDIT_TOOL: Record = { export const API_TOOL_NAMES: readonly string[] = [ "listTabs", "readTab", + "snapshotTab", "navigateTab", "softClick", "softType", + "actOnRef", "listBrowserTabs", "getConsoleLogs", "getConsoleErrors", @@ -104,9 +106,11 @@ function toolsForRequest(method: string, path: string, query: Record; + snapshotChromeTab(options: { chromeTabId: number }): Promise; + actOnChromeTabRef(options: { + chromeTabId: number; + ref: string; + op: "click" | "type"; + text?: string; + submit?: boolean; + }): Promise; close(): Promise; } @@ -622,6 +634,19 @@ export async function createConnector(config: ConnectorConfig = {}): Promise { + try { + const chromeTabId = req.body?.chromeTabId; + if (typeof chromeTabId !== "number") { + res.status(400).json({ error: "chromeTabId is required" }); + return; + } + res.json(await snapshotChromeTab({ chromeTabId })); + } catch (error) { + respondWithError(res, error); + } + }); + api.post("/chrome-tabs/navigate", async (req: Request, res: Response) => { try { const url = typeof req.body?.url === "string" ? req.body.url.trim() : ""; @@ -677,6 +702,37 @@ export async function createConnector(config: ConnectorConfig = {}): Promise { + try { + const chromeTabId = req.body?.chromeTabId; + const ref = req.body?.ref; + const op = req.body?.op; + const text = req.body?.text; + if (typeof chromeTabId !== "number" || typeof ref !== "string") { + res.status(400).json({ error: "chromeTabId and ref are required" }); + return; + } + if (op !== "click" && op !== "type") { + res.status(400).json({ error: "op must be click or type" }); + return; + } + if (op === "type" && typeof text !== "string") { + res.status(400).json({ error: "text is required" }); + return; + } + res.json( + await actOnChromeTabRef({ + chromeTabId, + ref, + op, + ...(op === "type" && typeof text === "string" ? { text, submit: req.body?.submit === true } : {}), + }) + ); + } catch (error) { + respondWithError(res, error); + } + }); + api.post("/navigate", async (req: Request, res: Response) => { try { const url = typeof req.body?.url === "string" ? req.body.url.trim() : ""; @@ -1225,7 +1281,9 @@ export async function createConnector(config: ConnectorConfig = {}): Promise { + const gate = await checkCapability("readTab", { tabId: options.chromeTabId }); + if (!gate.allowed) throw new ExtensionRequestError(gate.reason); + const response = await requestFromExtension(requireWorker(), "snapshot-chrome-tab", { + chromeTabId: options.chromeTabId, + approvedUrl: gate.liveUrl, + }); + assertTabActionSafe(response); + return parseSemanticSnapshot(options.chromeTabId, response["snapshot"]); + } + async function navigateChromeTab(options: { url: string; chromeTabId?: number; @@ -1821,6 +1890,27 @@ export async function createConnector(config: ConnectorConfig = {}): Promise { + const gate = await checkCapability(options.op === "type" ? "softType" : "softClick", { + tabId: options.chromeTabId, + }); + if (!gate.allowed) throw new ExtensionRequestError(gate.reason); + const response = await requestFromExtension(requireWorker(), "act-chrome-tab", { + op: options.op, + chromeTabId: options.chromeTabId, + ref: options.ref, + approvedUrl: gate.liveUrl, + ...(options.op === "type" ? { text: options.text ?? "", submit: options.submit === true } : {}), + }); + return parseActOnRef(options.chromeTabId, options.ref, response); + } + async function close(): Promise { clearInterval(heartbeat); for (const { timer, reject } of pending.values()) { @@ -1875,6 +1965,8 @@ export async function createConnector(config: ConnectorConfig = {}): Promise { const onError = (error: NodeJS.ErrnoException) => { + server.removeListener("listening", onListening); if (error.code === "EADDRINUSE" && remaining > 1 && startPort !== 0) { remaining -= 1; port += 1; - setImmediate(tryListen); + server.close(() => tryListen()); return; } reject(error); }; - server.once("error", onError); - server.listen(port, host, () => { - server.off("error", onError); + const onListening = () => { + server.removeListener("error", onError); const address = server.address(); resolve(typeof address === "object" && address ? address.port : port); - }); + }; + + server.once("error", onError); + server.once("listening", onListening); + server.listen(port, host); }; tryListen(); diff --git a/browser-tools-mcp/src/mcp/client.ts b/browser-tools-mcp/src/mcp/client.ts index fdf68b12..2808de6b 100644 --- a/browser-tools-mcp/src/mcp/client.ts +++ b/browser-tools-mcp/src/mcp/client.ts @@ -19,9 +19,11 @@ import { type RestrictedPageKind, } from "../restricted-page.js"; import type { + ActOnRefResult, ChromeNavigateResult, ChromeTabListing, ChromeTabRead, + SemanticSnapshot, SoftInputResult, } from "../tabs/contract.js"; @@ -103,6 +105,14 @@ export interface ConnectorClient { text: string; submit?: boolean; }): Promise; + snapshotChromeTab(options: { chromeTabId: number }): Promise; + actOnChromeTabRef(options: { + chromeTabId: number; + ref: string; + op: "click" | "type"; + text?: string; + submit?: boolean; + }): Promise; storage(kinds: string[], options?: { tabId?: TabId }): Promise>; audit( category: AuditCategory, @@ -213,6 +223,20 @@ export class InProcessConnectorClient implements ConnectorClient { return this.#connector.softTypeChromeTab(options); } + snapshotChromeTab(options: { chromeTabId: number }): Promise { + return this.#connector.snapshotChromeTab(options); + } + + actOnChromeTabRef(options: { + chromeTabId: number; + ref: string; + op: "click" | "type"; + text?: string; + submit?: boolean; + }): Promise { + return this.#connector.actOnChromeTabRef(options); + } + async storage( kinds: string[], options: { tabId?: TabId } = {} @@ -366,6 +390,20 @@ export class HttpConnectorClient implements ConnectorClient { return this.#request("/api/chrome-tabs/soft-type", { method: "POST", body: options }); } + snapshotChromeTab(options: { chromeTabId: number }): Promise { + return this.#request("/api/chrome-tabs/snapshot", { method: "POST", body: options }); + } + + actOnChromeTabRef(options: { + chromeTabId: number; + ref: string; + op: "click" | "type"; + text?: string; + submit?: boolean; + }): Promise { + return this.#request("/api/chrome-tabs/act", { method: "POST", body: options }); + } + async storage( kinds: string[], options: { tabId?: TabId } = {} diff --git a/browser-tools-mcp/src/mcp/tool-invoke.ts b/browser-tools-mcp/src/mcp/tool-invoke.ts index e9eb3441..ddca9199 100644 --- a/browser-tools-mcp/src/mcp/tool-invoke.ts +++ b/browser-tools-mcp/src/mcp/tool-invoke.ts @@ -324,8 +324,10 @@ export const INVOKABLE_TOOL_NAMES: readonly string[] = [ "listTabs", "readTab", "navigateTab", + "snapshotTab", "softClick", "softType", + "actOnRef", "listBrowserTabs", "getConsoleLogs", "getConsoleErrors", diff --git a/browser-tools-mcp/src/runtime.ts b/browser-tools-mcp/src/runtime.ts index 3dc5d68f..b8b65b7e 100644 --- a/browser-tools-mcp/src/runtime.ts +++ b/browser-tools-mcp/src/runtime.ts @@ -87,6 +87,12 @@ class UnavailableConnectorClient implements ConnectorClient { softTypeChromeTab(): Promise { return this.#fail(); } + snapshotChromeTab(): Promise { + return this.#fail(); + } + actOnChromeTabRef(): Promise { + return this.#fail(); + } storage(): Promise { return this.#fail(); } diff --git a/browser-tools-mcp/src/tabs/contract.ts b/browser-tools-mcp/src/tabs/contract.ts index 7b1353be..5254965e 100644 --- a/browser-tools-mcp/src/tabs/contract.ts +++ b/browser-tools-mcp/src/tabs/contract.ts @@ -10,7 +10,7 @@ import { RESTRICTED_PAGE_MESSAGE } from "../restricted-page.js"; * connected to the connector. Pass it as `tabId` to the existing tools. * - A chrome.tabs id comes from listTabs. It is chrome.tabs.Tab.id for a tab * the browser returned. Pass it as `chromeTabId` to listTabs, readTab, - * navigateTab, softClick, and softType. + * snapshotTab, navigateTab, softClick, softType, and actOnRef. * * Chrome often uses the same integer for both when DevTools is open on that * tab. The tools still do not accept each other's ids: a DevTools-connected @@ -96,6 +96,26 @@ export const SOFT_TYPE_DESCRIPTION = "The result is outcome \"worked\" or \"needs a real gesture\" with a reason. A failed action is never reported as worked. " + "Gated like input: popup switch, --allow-input (browser-tools CLI: --allow-actions), and the same origin rules."; +export const SNAPSHOT_TAB_DESCRIPTION = + "Builds a semantic page map of a tab from listTabs. Each entry has a role, an accessible name, text, state, visibility, and a stable ref. " + + "chromeTabId is a chrome.tabs id from listTabs, not a DevTools-connected tabId from listBrowserTabs. " + + "Does not attach the debugger and does not change focus. At most 80 elements. " + + "The same element keeps its ref until the document is replaced. " + + "Private details are stripped in the browser before the result leaves the page. " + + "An asleep (discarded) tab is reported as asleep and is not reloaded. A frozen tab is skipped. " + + "chrome:// pages, the Chrome Web Store, and the PDF viewer cannot be mapped. " + + "Needs the popup switch and host access for that site."; + +export const ACT_ON_REF_DESCRIPTION = + "Clicks or types on a ref from snapshotTab, using the same untrusted click and type path as softClick and softType. " + + "Does not attach the debugger and does not change focus. " + + "chromeTabId is a chrome.tabs id from listTabs, not a DevTools-connected tabId from listBrowserTabs. " + + "op is click or type. A type uses the native value setter or execCommand('insertText'), and submit true uses form.requestSubmit(). " + + "The result is outcome \"worked\" or \"needs a real gesture\" with a reason, plus a short delta: whether the URL changed, " + + "console errors that appeared after the action, and network requests that failed after the action. " + + "A failed action is never reported as worked. " + + "Gated like input: popup switch, --allow-input (browser-tools CLI: --allow-actions), and the same origin rules."; + export const SOFT_OUTCOMES = ["worked", "needs a real gesture"] as const; export type SoftOutcome = (typeof SOFT_OUTCOMES)[number]; @@ -201,4 +221,69 @@ export interface SoftInputResult { trusted: false; } -export const TAB_TOOL_NAMES = ["listTabs", "readTab", "navigateTab", "softClick", "softType"] as const; +export interface SemanticState { + disabled?: boolean; + checked?: boolean | "mixed"; + expanded?: boolean; + pressed?: boolean | "mixed"; + selected?: boolean; + required?: boolean; + invalid?: boolean; + readonly?: boolean; +} + +export interface SemanticNode { + ref: string; + role: string; + name: string; + text: string; + state: SemanticState; + visible: boolean; + depth: number; +} + +export interface SemanticSnapshot { + chromeTabId: number; + url: string; + title: string; + site: string; + asleep: boolean; + frozen: boolean; + skipped: string | null; + hostAccess: boolean; + truncated: boolean; + nodes: SemanticNode[]; +} + +export interface FailedRequestDelta { + url: string; + status: number | null; +} + +export interface PageChangeDelta { + urlChanged: boolean; + previousUrl: string; + url: string; + consoleErrors: string[]; + failedRequests: FailedRequestDelta[]; +} + +export interface ActOnRefResult { + chromeTabId: number; + ref: string; + outcome: SoftOutcome; + reason: GestureReason | null; + detail: string; + trusted: false; + delta: PageChangeDelta; +} + +export const TAB_TOOL_NAMES = [ + "listTabs", + "readTab", + "snapshotTab", + "navigateTab", + "softClick", + "softType", + "actOnRef", +] as const; diff --git a/browser-tools-mcp/src/tabs/listing.ts b/browser-tools-mcp/src/tabs/listing.ts index 773a9cd9..32abefb0 100644 --- a/browser-tools-mcp/src/tabs/listing.ts +++ b/browser-tools-mcp/src/tabs/listing.ts @@ -7,9 +7,15 @@ import { type ChromeNavigateResult, type ChromeTabHeading, type ChromeTabListing, + type ActOnRefResult, type ChromeTabRead, type ChromeTabSnapshotNode, type ChromeTabView, + type FailedRequestDelta, + type PageChangeDelta, + type SemanticNode, + type SemanticSnapshot, + type SemanticState, type ChromeWindowView, type GestureReason, type OpenedTab, @@ -180,6 +186,118 @@ export function parseChromeNavigate(requestedUrl: string, payload: Record= 0 && depth <= 40 ? depth : 0, + }); + } + } + const skipped = typeof read["skipped"] === "string" && read["skipped"] ? read["skipped"].slice(0, 120) : null; + return { + chromeTabId, + url, + title: text(read["title"], 300), + site: siteOf(url), + asleep: read["asleep"] === true, + frozen: read["frozen"] === true, + skipped, + hostAccess: read["hostAccess"] === true, + truncated: read["truncated"] === true, + nodes, + }; +} + +function parseDelta(value: unknown): PageChangeDelta { + const delta = asRecord(value); + if (!delta) throw new Error("The action did not report what changed."); + const previousUrl = text(delta["previousUrl"], 2000); + const url = text(delta["url"], 2000); + const consoleErrors: string[] = []; + if (Array.isArray(delta["consoleErrors"])) { + for (const item of delta["consoleErrors"].slice(0, 8)) { + const line = text(item, 240); + if (line) consoleErrors.push(line); + } + } + const failedRequests: FailedRequestDelta[] = []; + if (Array.isArray(delta["failedRequests"])) { + for (const item of delta["failedRequests"].slice(0, 8)) { + const request = asRecord(item); + if (!request) continue; + const requestUrl = text(request["url"], 300); + const status = finiteNumber(request["status"]); + failedRequests.push({ + url: requestUrl, + status: status !== null && status >= 100 && status <= 599 ? Math.trunc(status) : null, + }); + } + } + return { + urlChanged: url !== previousUrl, + previousUrl, + url, + consoleErrors, + failedRequests, + }; +} + +export function parseActOnRef(chromeTabId: number, ref: string, payload: Record): ActOnRefResult { + assertTabActionSafe(payload); + if (!REF_PATTERN.test(ref)) throw new Error("ref must be a ref from snapshotTab"); + const outcome = payload["outcome"]; + if (outcome !== "worked" && outcome !== "needs a real gesture") { + throw new Error("The action did not report worked or needs a real gesture."); + } + const reasonRaw = payload["reason"]; + const reason = GESTURE_REASONS.find((entry) => entry === reasonRaw) ?? null; + if (outcome === "needs a real gesture" && !reason) { + throw new Error("The action refused without a reason."); + } + return { + chromeTabId, + ref, + outcome, + reason: outcome === "worked" ? null : reason, + detail: text(payload["detail"], 300), + trusted: false, + delta: parseDelta(payload["delta"]), + }; +} + export function parseSoftInput(chromeTabId: number, payload: Record): SoftInputResult { assertTabActionSafe(payload); const outcome = payload["outcome"]; diff --git a/browser-tools-mcp/src/tabs/tools.ts b/browser-tools-mcp/src/tabs/tools.ts index 6b204f6d..208c7a38 100644 --- a/browser-tools-mcp/src/tabs/tools.ts +++ b/browser-tools-mcp/src/tabs/tools.ts @@ -3,10 +3,12 @@ import { z } from "zod"; import type { ConnectorClient } from "../mcp/client.js"; import { + ACT_ON_REF_DESCRIPTION, CHROME_TAB_ID, LIST_TABS_DESCRIPTION, NAVIGATE_TAB_DESCRIPTION, READ_TAB_DESCRIPTION, + SNAPSHOT_TAB_DESCRIPTION, SOFT_CLICK_DESCRIPTION, SOFT_TYPE_DESCRIPTION, } from "./contract.js"; @@ -39,6 +41,48 @@ const selectorInput = z .max(500) .describe("CSS selector for the element in that tab. Not a DevTools-connected tab id."); +const refInput = z + .string() + .regex(/^e[1-9][0-9]{0,4}$/) + .describe("Stable ref from snapshotTab, such as e1. Not a chromeTabId or a DevTools-connected tabId."); + +const semanticStateShape = z.object({ + disabled: z.boolean().optional(), + checked: z.union([z.boolean(), z.literal("mixed")]).optional(), + expanded: z.boolean().optional(), + pressed: z.union([z.boolean(), z.literal("mixed")]).optional(), + selected: z.boolean().optional(), + required: z.boolean().optional(), + invalid: z.boolean().optional(), + readonly: z.boolean().optional(), +}); + +const semanticNodeShape = { + ref: z.string(), + role: z.string(), + name: z.string(), + text: z.string(), + state: semanticStateShape, + visible: z.boolean(), + depth: z.number().int(), +}; + +const pageDeltaShape = { + urlChanged: z.boolean(), + previousUrl: z.string(), + url: z.string(), + consoleErrors: z.array(z.string()), + failedRequests: z.array(z.object({ url: z.string(), status: z.number().int().nullable() })), +}; + +const softOutcomeShape = { + chromeTabId: z.number().int(), + outcome: z.enum(["worked", "needs a real gesture"]), + reason: z.string().nullable(), + detail: z.string(), + trusted: z.literal(false), +}; + const safetyShape = { debuggerCalls: z.number().int(), focusChanges: z.number().int(), @@ -96,6 +140,11 @@ export async function invokeTabTool( if (chromeTabId === undefined) return fail(new Error("chromeTabId is required")); return ok((await client.readChromeTab({ chromeTabId })) as unknown as Record); } + case "snapshotTab": { + const chromeTabId = chromeTabIdFrom(args); + if (chromeTabId === undefined) return fail(new Error("chromeTabId is required")); + return ok((await client.snapshotChromeTab({ chromeTabId })) as unknown as Record); + } case "navigateTab": { const url = args["url"]; if (typeof url !== "string" || !url.trim()) return fail(new Error("url is required")); @@ -133,6 +182,25 @@ export async function invokeTabTool( })) as unknown as Record ); } + case "actOnRef": { + const chromeTabId = chromeTabIdFrom(args); + const ref = args["ref"]; + const op = args["op"]; + if (chromeTabId === undefined || typeof ref !== "string") { + return fail(new Error("chromeTabId and ref are required")); + } + if (op !== "click" && op !== "type") return fail(new Error("op must be click or type")); + const text = args["text"]; + if (op === "type" && typeof text !== "string") return fail(new Error("text is required")); + return ok( + (await client.actOnChromeTabRef({ + chromeTabId, + ref, + op, + ...(op === "type" && typeof text === "string" ? { text, submit: args["submit"] === true } : {}), + })) as unknown as Record + ); + } default: return null; } @@ -253,13 +321,7 @@ export const TAB_TOOLS: TabToolDefinition[] = [ title: "Soft click", description: SOFT_CLICK_DESCRIPTION, inputSchema: { chromeTabId: chromeTabIdInput, selector: selectorInput }, - outputSchema: { - chromeTabId: z.number().int(), - outcome: z.enum(["worked", "needs a real gesture"]), - reason: z.string().nullable(), - detail: z.string(), - trusted: z.literal(false), - }, + outputSchema: softOutcomeShape, annotations: { title: "Soft click", readOnlyHint: false, @@ -289,22 +351,84 @@ export const TAB_TOOLS: TabToolDefinition[] = [ .optional() .describe("After a successful type, submit the surrounding form with requestSubmit()."), }, + outputSchema: softOutcomeShape, + annotations: { + title: "Soft type", + readOnlyHint: false, + destructiveHint: false, + idempotentHint: false, + openWorldHint: true, + }, + }, + async (args) => invokeTabTool("softType", client, args) as Promise + ); + }, + }, + { + name: "snapshotTab", + register(server, client) { + server.registerTool( + "snapshotTab", + { + title: "Semantic page map", + description: SNAPSHOT_TAB_DESCRIPTION, + inputSchema: { chromeTabId: chromeTabIdInput }, outputSchema: { chromeTabId: z.number().int(), - outcome: z.enum(["worked", "needs a real gesture"]), - reason: z.string().nullable(), - detail: z.string(), - trusted: z.literal(false), + url: z.string(), + title: z.string(), + site: z.string(), + asleep: z.boolean(), + frozen: z.boolean(), + skipped: z.string().nullable(), + hostAccess: z.boolean(), + truncated: z.boolean(), + nodes: z.array(z.object(semanticNodeShape)), }, annotations: { - title: "Soft type", + title: "Semantic page map", + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: true, + }, + }, + async (args) => invokeTabTool("snapshotTab", client, args) as Promise + ); + }, + }, + { + name: "actOnRef", + register(server, client) { + server.registerTool( + "actOnRef", + { + title: "Act on a page ref", + description: ACT_ON_REF_DESCRIPTION, + inputSchema: { + chromeTabId: chromeTabIdInput, + ref: refInput, + op: z.enum(["click", "type"]).describe("click or type. Type reuses the soft type path."), + text: z.string().max(20000).optional().describe("Text to insert when op is type. Password fields are refused."), + submit: z + .boolean() + .optional() + .describe("After a successful type, submit the surrounding form with requestSubmit()."), + }, + outputSchema: { + ...softOutcomeShape, + ref: z.string(), + delta: z.object(pageDeltaShape), + }, + annotations: { + title: "Act on a page ref", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true, }, }, - async (args) => invokeTabTool("softType", client, args) as Promise + async (args) => invokeTabTool("actOnRef", client, args) as Promise ); }, }, diff --git a/browser-tools-mcp/src/tools-cli.ts b/browser-tools-mcp/src/tools-cli.ts index 03c5d347..9f0425b7 100644 --- a/browser-tools-mcp/src/tools-cli.ts +++ b/browser-tools-mcp/src/tools-cli.ts @@ -97,6 +97,8 @@ Examples: browser-tools getConnectionStatus browser-tools getConsoleLogs '{"limit":20}' browser-tools listTabs + browser-tools snapshotTab '{"chromeTabId":4}' + browser-tools --allow-actions actOnRef '{"chromeTabId":4,"ref":"e1","op":"click"}' browser-tools --allow-actions navigateBrowser '{"url":"http://localhost:3000/","tabId":1}' browser-tools --allow-actions navigateTab '{"url":"http://127.0.0.1:3000/","chromeTabId":4}' `; diff --git a/browser-tools-mcp/test/e2e/tab-awareness.e2e.test.ts b/browser-tools-mcp/test/e2e/tab-awareness.e2e.test.ts index 0890631b..9313f7ed 100644 --- a/browser-tools-mcp/test/e2e/tab-awareness.e2e.test.ts +++ b/browser-tools-mcp/test/e2e/tab-awareness.e2e.test.ts @@ -218,4 +218,58 @@ describe("tab awareness without DevTools", () => { }, 180_000 ); + + it.skipIf(!browserSupport.usable)( + "maps a background tab and reports what changed after acting on a ref", + async () => { + const sw = await extensionWorker(); + await sw.evaluate(async () => { + const chromeApi = (globalThis as unknown as { chrome: any }).chrome; + await chromeApi.storage.session.set({ btmcpPopupSwitch: true }); + }); + const before = await browserState(); + const opened = await callTool("navigateTab", { url: `${fixture.url}map` }); + const chromeTabId = opened["chromeTabId"] as number; + + let map: Record | undefined; + const deadline = Date.now() + 20_000; + while (Date.now() < deadline) { + map = await callTool("snapshotTab", { chromeTabId }); + if (JSON.stringify(map["nodes"]).includes("Run check")) break; + await new Promise((resolve) => setTimeout(resolve, 200)); + } + const nodes = map?.["nodes"] as { ref: string; role: string; name: string; visible: boolean }[]; + const button = nodes.find((node) => node.name === "Run check"); + const ghost = nodes.find((node) => node.name === "Ghost"); + expect(button?.role).toBe("button"); + expect(button?.ref).toMatch(/^e[1-9][0-9]*$/); + expect(ghost?.visible).toBe(false); + expect(JSON.stringify(nodes)).not.toContain("Secret control"); + + const again = await callTool("snapshotTab", { chromeTabId }); + const againButton = (again["nodes"] as { ref: string; name: string }[]).find((node) => node.name === "Run check"); + expect(againButton?.ref).toBe(button?.ref); + + const acted = await callTool("actOnRef", { chromeTabId, ref: button?.ref, op: "click" }); + expect(acted).toMatchObject({ outcome: "worked", trusted: false, ref: button?.ref }); + const delta = acted["delta"] as { + urlChanged: boolean; + url: string; + consoleErrors: string[]; + failedRequests: { url: string; status: number | null }[]; + }; + expect(delta.urlChanged).toBe(true); + expect(delta.url).toContain("changed=1"); + expect(delta.consoleErrors.join("\n")).toContain("MARKER-MAP-ERROR"); + expect(delta.failedRequests.some((request) => request.status === 500 && request.url.includes("/api/fail"))).toBe(true); + + const after = await browserState(); + expect(after.activeId).toBe(before.activeId); + expect(after.activeUrl).toBe(before.activeUrl); + expect(after.safety).toEqual(before.safety); + expect(page.url()).toBe(fixture.url); + expect(await extensionDebuggerAttached(chromeTabId)).toBe(false); + }, + 180_000 + ); }); diff --git a/browser-tools-mcp/test/fixtures/server.ts b/browser-tools-mcp/test/fixtures/server.ts index b7fa75d7..7096bff1 100644 --- a/browser-tools-mcp/test/fixtures/server.ts +++ b/browser-tools-mcp/test/fixtures/server.ts @@ -118,6 +118,27 @@ export async function startFixtureServer(): Promise { return; } + if (url === "/map" || url.startsWith("/map?")) { + res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); + res.end(` + Page map + +

Page map

+ + + + + + `); + return; + } + if (url === "/form" || url.startsWith("/form?")) { res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); res.end(` diff --git a/browser-tools-mcp/test/helpers/fake-extension.ts b/browser-tools-mcp/test/helpers/fake-extension.ts index 372b0ff7..6aca483f 100644 --- a/browser-tools-mcp/test/helpers/fake-extension.ts +++ b/browser-tools-mcp/test/helpers/fake-extension.ts @@ -19,6 +19,8 @@ export interface FakeExtensionOptions { onReadChromeTab?: (msg: any) => Promise | unknown; onNavigateChromeTab?: (msg: any) => Promise | unknown; onSoftInput?: (msg: any) => Promise | unknown; + onSnapshotChromeTab?: (msg: any) => Promise | unknown; + onActChromeTab?: (msg: any) => Promise | unknown; onStorage?: (msg: any) => Promise | unknown; /** `worker` answers capability probes and is not a DevTools tab. */ role?: "devtools" | "worker"; @@ -39,12 +41,25 @@ export class FakeExtension { } async connect(): Promise { + try { + await this.#connectOnce(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!/timed out|ECONNRESET|ECONNREFUSED|socket hang up/i.test(message)) throw error; + this.#ws?.terminate(); + this.#ws = null; + await this.#connectOnce(); + } + } + + async #connectOnce(): Promise { const { port, origin = EXAMPLE_CHROME_EXTENSION_ORIGIN, token: given } = this.#options; let token = given; if (!token) { const paired = await fetch(`http://127.0.0.1:${port}/.pair`, { method: "POST", headers: { Origin: origin }, + signal: AbortSignal.timeout(3_000), }); if (!paired.ok) { throw new Error(`Extension pairing failed (${paired.status})`); @@ -66,9 +81,17 @@ export class FakeExtension { ws.on("message", (raw) => void this.#handle(raw)); await new Promise((resolve, reject) => { - const onError = (err: Error) => reject(err); + const timer = setTimeout(() => { + ws.terminate(); + reject(new Error("Timed out opening the extension socket")); + }, 3_000); + const onError = (err: Error) => { + clearTimeout(timer); + reject(err); + }; ws.once("error", onError); ws.once("open", () => { + clearTimeout(timer); ws.off("error", onError); resolve(); }); @@ -181,6 +204,51 @@ export class FakeExtension { this.send({ type: "soft-input-result", requestId: msg.requestId, ...(payload as object) }); break; } + case "snapshot-chrome-tab": { + const custom = this.#options.onSnapshotChromeTab; + const payload = custom + ? await custom(msg) + : { + ok: true, + safety: { debuggerCalls: 0, focusChanges: 0 }, + snapshot: { + url: "http://127.0.0.1/", + title: "", + asleep: false, + frozen: false, + skipped: null, + hostAccess: true, + truncated: false, + nodes: [], + }, + }; + this.send({ type: "chrome-tab-snapshot-result", requestId: msg.requestId, ...(payload as object) }); + break; + } + case "act-chrome-tab": { + const custom = this.#options.onActChromeTab; + const payload = custom + ? await custom(msg) + : { + ok: true, + chromeTabId: msg.chromeTabId, + ref: msg.ref, + outcome: "worked", + reason: null, + detail: "ok", + trusted: false, + delta: { + urlChanged: false, + previousUrl: "http://127.0.0.1/", + url: "http://127.0.0.1/", + consoleErrors: [], + failedRequests: [], + }, + safety: { debuggerCalls: 0, focusChanges: 0 }, + }; + this.send({ type: "act-ref-result", requestId: msg.requestId, ...(payload as object) }); + break; + } case "get-storage": { const custom = this.#options.onStorage; const payload = custom diff --git a/browser-tools-mcp/test/helpers/port.ts b/browser-tools-mcp/test/helpers/port.ts index dc8cf634..e7a15e03 100644 --- a/browser-tools-mcp/test/helpers/port.ts +++ b/browser-tools-mcp/test/helpers/port.ts @@ -68,14 +68,21 @@ function portOf(server: net.Server): number { function listenOn(port: number): Promise { return new Promise((resolve, reject) => { const server = net.createServer(); - const onError = (error: Error) => { - server.close(() => reject(error)); + let settled = false; + const finish = (error?: Error, listening = false) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (!error && listening) { + server.removeAllListeners("error"); + resolve(server); + return; + } + server.close(() => reject(error ?? new Error(`Timed out listening on ${port}`))); }; - server.once("error", onError); - server.listen({ port, host: "127.0.0.1", exclusive: true }, () => { - server.off("error", onError); - resolve(server); - }); + const timer = setTimeout(() => finish(new Error(`Timed out listening on ${port}`)), 1_000); + server.once("error", (error) => finish(error)); + server.listen({ port, host: "127.0.0.1", exclusive: true }, () => finish(undefined, true)); }); } diff --git a/browser-tools-mcp/test/integration/connector-flow.test.ts b/browser-tools-mcp/test/integration/connector-flow.test.ts index a16a7fde..eb3ade20 100644 --- a/browser-tools-mcp/test/integration/connector-flow.test.ts +++ b/browser-tools-mcp/test/integration/connector-flow.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import request from "supertest"; +import net from "node:net"; import path from "node:path"; import os from "node:os"; import fs from "node:fs"; @@ -39,6 +40,22 @@ async function connectExtension( return ext; } +describe("port binding", () => { + it("uses the next port when the requested one is taken and does not keep the failed listen", async () => { + const held = net.createServer(); + await new Promise((resolve) => { + held.listen(0, "127.0.0.1", () => resolve()); + }); + const taken = (held.address() as net.AddressInfo).port; + const moved = await createConnector({ port: taken, screenshotDir }); + expect(moved.port).toBe(taken + 1); + expect(moved.server.listenerCount("listening")).toBeLessThanOrEqual(1); + expect(moved.server.listenerCount("error")).toBe(1); + await moved.close(); + await new Promise((resolve) => held.close(() => resolve())); + }); +}); + describe("extension handshake", () => { it("welcomes a connecting extension and reports it as connected", async () => { expect(connector.hasExtension()).toBe(false); diff --git a/browser-tools-mcp/test/integration/mcp-server.test.ts b/browser-tools-mcp/test/integration/mcp-server.test.ts index de18dc93..01c16c86 100644 --- a/browser-tools-mcp/test/integration/mcp-server.test.ts +++ b/browser-tools-mcp/test/integration/mcp-server.test.ts @@ -75,8 +75,10 @@ describe("tool registration", () => { "navigateBrowser", "navigateTab", "readTab", + "snapshotTab", "softClick", "softType", + "actOnRef", "refreshBrowser", "runAccessibilityAudit", "runBestPracticesAudit", @@ -110,6 +112,7 @@ describe("tool registration", () => { "getConnectionStatus", "listTabs", "readTab", + "snapshotTab", ]) { expect(byName.get(name)?.annotations?.readOnlyHint, name).toBe(true); } @@ -122,6 +125,7 @@ describe("tool registration", () => { expect(byName.get("navigateTab")?.annotations?.readOnlyHint).toBe(false); expect(byName.get("softClick")?.annotations?.readOnlyHint).toBe(false); expect(byName.get("softType")?.annotations?.readOnlyHint).toBe(false); + expect(byName.get("actOnRef")?.annotations?.readOnlyHint).toBe(false); expect(byName.get("wipeLogs")?.annotations?.readOnlyHint).toBe(false); expect(byName.get("wipeLogs")?.annotations?.destructiveHint).toBe(true); expect(byName.get("refreshBrowser")?.annotations?.readOnlyHint).toBe(false); diff --git a/browser-tools-mcp/test/integration/mcp-stdio.test.ts b/browser-tools-mcp/test/integration/mcp-stdio.test.ts index ae4b32cf..59bcbf68 100644 --- a/browser-tools-mcp/test/integration/mcp-stdio.test.ts +++ b/browser-tools-mcp/test/integration/mcp-stdio.test.ts @@ -192,7 +192,7 @@ describe("stdio transport contract", () => { } finally { await held.release(); } - }); + }, 45_000); }); describe("command line interface", () => { diff --git a/browser-tools-mcp/test/integration/tab-awareness.test.ts b/browser-tools-mcp/test/integration/tab-awareness.test.ts index 747e3cf4..87bb7f6f 100644 --- a/browser-tools-mcp/test/integration/tab-awareness.test.ts +++ b/browser-tools-mcp/test/integration/tab-awareness.test.ts @@ -103,6 +103,46 @@ describe("chrome.tabs tools", () => { trusted: false, safety: { debuggerCalls: 0, focusChanges: 0 }, }), + onSnapshotChromeTab: () => ({ + ok: true, + safety: { debuggerCalls: 0, focusChanges: 0 }, + snapshot: { + url: "http://127.0.0.1:3000/app", + title: "Fixture", + asleep: false, + frozen: false, + skipped: null, + hostAccess: true, + truncated: false, + nodes: [ + { + ref: "e1", + role: "button", + name: "Run check", + text: "Run check", + state: {}, + visible: true, + depth: 1, + }, + ], + }, + }), + onActChromeTab: (msg) => ({ + ok: true, + chromeTabId: msg.chromeTabId, + ref: msg.ref, + outcome: "worked", + reason: null, + detail: "Untrusted click was delivered to the element.", + trusted: false, + delta: { + previousUrl: "http://127.0.0.1:3000/app", + url: "http://127.0.0.1:3000/app?changed=1", + consoleErrors: ["MARKER-MAP-ERROR"], + failedRequests: [{ url: "http://127.0.0.1:3000/api/fail", status: 500 }], + }, + safety: { debuggerCalls: 0, focusChanges: 0 }, + }), }); await worker.connect(); worker.send({ type: "capability-state", popupEnabled: true }); @@ -225,7 +265,72 @@ describe("chrome.tabs tools", () => { expect(navigate?.description).toContain("visible window"); const called = await mcp.callTool({ name: "softType", arguments: { chromeTabId: 4, selector: "#name", text: "Ada" } }); expect(called.isError).toBeUndefined(); + const mapTool = listed.tools.find((tool) => tool.name === "snapshotTab"); + const actTool = listed.tools.find((tool) => tool.name === "actOnRef"); + expect(mapTool?.description).toContain("stable ref"); + expect(mapTool?.description).not.toMatch(/computer[- ]use/i); + expect(actTool?.description).toContain("console errors"); + expect(actTool?.description).not.toMatch(/computer[- ]use/i); await mcp.close(); await server.close(); }); + + it("maps a tab without input permission and refuses an action until input is allowed", async () => { + await start(false); + const snapshot = await connector!.snapshotChromeTab({ chromeTabId: 4 }); + expect(snapshot.nodes[0]).toMatchObject({ ref: "e1", role: "button", name: "Run check", visible: true }); + await expect(connector!.actOnChromeTabRef({ chromeTabId: 4, ref: "e1", op: "click" })).rejects.toThrow(/input is not allowed/); + expect(worker!.received.some((message) => message.type === "act-chrome-tab")).toBe(false); + }); + + it("returns a what-changed delta after acting on a ref and refuses a focus change", async () => { + await start(true); + const client = new InProcessConnectorClient(connector!); + const acted = await invokeTool("actOnRef", client, { chromeTabId: 4, ref: "e1", op: "click" }); + expect(acted.isError).toBeUndefined(); + expect(acted.structuredContent).toMatchObject({ + ref: "e1", + outcome: "worked", + trusted: false, + delta: { + urlChanged: true, + consoleErrors: ["MARKER-MAP-ERROR"], + failedRequests: [{ url: "http://127.0.0.1:3000/api/fail", status: 500 }], + }, + }); + const sent = worker!.received.filter((message) => message.type === "act-chrome-tab"); + expect(sent.at(-1)).toMatchObject({ ref: "e1", op: "click", chromeTabId: 4 }); + + await worker!.close(); + worker = new FakeExtension({ + port: connector!.port, + role: "worker", + onCapabilityProbe: () => ({ + popupEnabled: true, + panelEnabled: false, + liveUrl: "http://127.0.0.1:3000/app", + excludeOrigins: [], + }), + onActChromeTab: () => ({ + ok: true, + outcome: "worked", + reason: null, + detail: "clicked", + trusted: false, + delta: { + previousUrl: "http://127.0.0.1:3000/app", + url: "http://127.0.0.1:3000/app", + consoleErrors: [], + failedRequests: [], + }, + safety: { debuggerCalls: 0, focusChanges: 1 }, + }), + }); + await worker.connect(); + worker.send({ type: "capability-state", popupEnabled: true }); + await expect + .poll(async () => (await connector!.checkCapability("softClick", { tabId: 4 })).allowed) + .toBe(true); + await expect(connector!.actOnChromeTabRef({ chromeTabId: 4, ref: "e1", op: "click" })).rejects.toThrow(/change focus/); + }); }); diff --git a/browser-tools-mcp/test/unit/api-policy.test.ts b/browser-tools-mcp/test/unit/api-policy.test.ts index f4e86593..94397cdd 100644 --- a/browser-tools-mcp/test/unit/api-policy.test.ts +++ b/browser-tools-mcp/test/unit/api-policy.test.ts @@ -86,6 +86,37 @@ describe("decideApiAccess", () => { ).toBe("TOOL_DISABLED"); }); + it("treats a page-map read as a tool and an action on a ref as state-changing", () => { + expect( + decideApiAccess({ + method: "POST", + path: "/chrome-tabs/snapshot", + query: {}, + enabled, + actionsAllowed: false, + }).ok + ).toBe(true); + expect( + decideApiAccess({ + method: "POST", + path: "/chrome-tabs/act", + query: {}, + enabled, + actionsAllowed: false, + }).code + ).toBe("ACTION_NOT_CONFIRMED"); + const withoutAct = new Set([...enabled].filter((name) => name !== "actOnRef")); + expect( + decideApiAccess({ + method: "POST", + path: "/chrome-tabs/act", + query: {}, + enabled: withoutAct, + actionsAllowed: true, + }).code + ).toBe("TOOL_DISABLED"); + }); + it("leaves settings reads alone and gates settings writes", () => { expect( decideApiAccess({ diff --git a/browser-tools-mcp/test/unit/extension-gates.test.ts b/browser-tools-mcp/test/unit/extension-gates.test.ts index 5df781c4..d3b3a20d 100644 --- a/browser-tools-mcp/test/unit/extension-gates.test.ts +++ b/browser-tools-mcp/test/unit/extension-gates.test.ts @@ -135,6 +135,8 @@ describe("worker probe", () => { expect(worker).toContain("tabs.get"); expect(worker).toContain("navigate-tab"); expect(worker).toContain("list-chrome-tabs"); + expect(worker).toContain("snapshot-chrome-tab"); + expect(worker).toContain("act-chrome-tab"); expect(worker).not.toMatch(/chrome\.debugger|api\.debugger|\.attach\(/); expect(worker).not.toContain("windows.update"); expect(worker).not.toContain("executeScript"); @@ -153,6 +155,10 @@ describe("worker probe", () => { expect(actions).toContain("active: false"); expect(actions).toContain("visible window"); expect(page).toContain("requestSubmit"); + expect(page).toContain("data-btmcp-ref"); + expect(page).toContain("btmcpSemanticSnapshot"); + expect(page).toContain("btmcpActOnRef"); + expect(page).not.toMatch(/computer[- ]use/i); expect(page).toContain("insertText"); expect(page).toContain("needs a real gesture"); expect(safety).toContain("Refusing to change focus."); diff --git a/browser-tools-mcp/test/unit/tab-contract.test.ts b/browser-tools-mcp/test/unit/tab-contract.test.ts index 8021a740..c0e5d178 100644 --- a/browser-tools-mcp/test/unit/tab-contract.test.ts +++ b/browser-tools-mcp/test/unit/tab-contract.test.ts @@ -7,6 +7,8 @@ import { LIST_TABS_DESCRIPTION, NAVIGATE_TAB_DESCRIPTION, READ_TAB_DESCRIPTION, + ACT_ON_REF_DESCRIPTION, + SNAPSHOT_TAB_DESCRIPTION, SOFT_CLICK_DESCRIPTION, SOFT_TYPE_DESCRIPTION, TAB_COVERAGE, @@ -51,7 +53,14 @@ describe("chrome.tabs tool descriptions", () => { expect(LIST_TABS_DESCRIPTION.startsWith(TAB_COVERAGE)).toBe(true); expect(LIST_TABS_DESCRIPTION).toMatch(/This is not every open tab/); expect(LIST_TABS_DESCRIPTION).not.toMatch(/every tab you have|all open tabs|see every/i); - for (const text of [READ_TAB_DESCRIPTION, NAVIGATE_TAB_DESCRIPTION, SOFT_CLICK_DESCRIPTION, SOFT_TYPE_DESCRIPTION]) { + for (const text of [ + READ_TAB_DESCRIPTION, + SNAPSHOT_TAB_DESCRIPTION, + NAVIGATE_TAB_DESCRIPTION, + SOFT_CLICK_DESCRIPTION, + SOFT_TYPE_DESCRIPTION, + ACT_ON_REF_DESCRIPTION, + ]) { expect(text).toMatch(/chromeTabId/); expect(text).toMatch(/not a DevTools-connected tabId/); expect(text).not.toMatch(/every open tab/i); @@ -77,8 +86,32 @@ describe("chrome.tabs tool descriptions", () => { } }); + it("describes the page map and the what-changed delta without calling it computer use", () => { + expect(SNAPSHOT_TAB_DESCRIPTION).toMatch(/accessible name/); + expect(SNAPSHOT_TAB_DESCRIPTION).toMatch(/stable ref/); + expect(SNAPSHOT_TAB_DESCRIPTION).toMatch(/visibility/); + expect(SNAPSHOT_TAB_DESCRIPTION).toMatch(/does not change focus/i); + expect(ACT_ON_REF_DESCRIPTION).toMatch(/snapshotTab/); + expect(ACT_ON_REF_DESCRIPTION).toMatch(/softClick/); + expect(ACT_ON_REF_DESCRIPTION).toMatch(/console errors/); + expect(ACT_ON_REF_DESCRIPTION).toMatch(/failed after the action/); + expect(ACT_ON_REF_DESCRIPTION).toMatch(/--allow-input/); + for (const text of [SNAPSHOT_TAB_DESCRIPTION, ACT_ON_REF_DESCRIPTION]) { + expect(text).not.toMatch(/computer[- ]use/i); + expect(text).not.toMatch(/chrome\.debugger/); + } + }); + it("lists the chrome.tabs tools separately from listBrowserTabs", () => { - expect([...TAB_TOOL_NAMES]).toEqual(["listTabs", "readTab", "navigateTab", "softClick", "softType"]); + expect([...TAB_TOOL_NAMES]).toEqual([ + "listTabs", + "readTab", + "snapshotTab", + "navigateTab", + "softClick", + "softType", + "actOnRef", + ]); expect(TAB_TOOL_NAMES).not.toContain("listBrowserTabs"); expect(TAB_TOOL_NAMES).not.toContain("navigateBrowser"); }); diff --git a/browser-tools-mcp/test/unit/tab-extension.test.ts b/browser-tools-mcp/test/unit/tab-extension.test.ts index 0d5b47c1..a70338e7 100644 --- a/browser-tools-mcp/test/unit/tab-extension.test.ts +++ b/browser-tools-mcp/test/unit/tab-extension.test.ts @@ -41,6 +41,7 @@ function loadExtension(chromeApi: Record, documentMock?: Record NodeFilter: { SHOW_TEXT: 4, FILTER_REJECT: 2, FILTER_ACCEPT: 1 }, }; context["globalThis"] = context; + context["window"] = context; vm.createContext(context); for (const file of ["gates.js", "tab-safety.js", "tab-page.js", "tab-actions.js"]) { vm.runInContext(fs.readFileSync(path.join(extensionDir, file), "utf8"), context); @@ -55,7 +56,9 @@ function loadExtension(chromeApi: Record, documentMock?: Record list: (api: unknown, message: unknown, respond: (id: string, payload: Record) => void) => Promise; navigateChrome: (api: unknown, message: unknown, respond: (id: string, payload: Record) => void) => Promise; read: (api: unknown, message: unknown, respond: (id: string, payload: Record) => void) => Promise; + snapshot: (api: unknown, message: unknown, respond: (id: string, payload: Record) => void) => Promise; soft: (api: unknown, message: unknown, respond: (id: string, payload: Record) => void) => Promise; + act: (api: unknown, message: unknown, respond: (id: string, payload: Record) => void) => Promise; }; btmcpScrubText: (value: string) => string; btmcpSoftAct: (command: Record) => { outcome: string; reason: string | null; trusted: boolean }; @@ -340,3 +343,391 @@ describe("soft input in the page", () => { expect(stored).toBe("Ada"); }); }); + +type FakeNode = { + tagName: string; + id: string; + children: FakeNode[]; + parent: FakeNode | null; + attrs: Record; + textContent: string; + innerText: string; + hidden: boolean; + style: { display?: string; visibility?: string; opacity?: string }; + value?: string; + checked?: boolean; + disabled?: boolean; + type?: string; + labels?: FakeNode[]; + htmlFor?: string; + focusCalls: number; + listeners: Record void>; + getAttribute(name: string): string | null; + setAttribute(name: string, value: string): void; + hasAttribute(name: string): boolean; + getBoundingClientRect(): { width: number; height: number; left: number; top: number }; + closest(selector: string): FakeNode | null; + contains(other: FakeNode): boolean; + addEventListener(type: string, fn: () => void): void; + removeEventListener(type: string): void; + dispatchEvent(event: { type: string }): void; + focus(): void; +}; + +function makeNode( + tag: string, + options: { + id?: string; + text?: string; + children?: FakeNode[]; + attrs?: Record; + hidden?: boolean; + style?: FakeNode["style"]; + value?: string; + checked?: boolean; + type?: string; + htmlFor?: string; + rect?: { width: number; height: number; left: number; top: number }; + } = {} +): FakeNode { + const node = { + tagName: tag, + id: options.id ?? "", + children: options.children ?? [], + parent: null, + attrs: { ...(options.attrs ?? {}) }, + textContent: options.text ?? "", + innerText: options.text ?? "", + hidden: options.hidden === true, + style: options.style ?? {}, + value: options.value, + checked: options.checked, + disabled: false, + type: options.type, + htmlFor: options.htmlFor, + focusCalls: 0, + listeners: {}, + } as FakeNode; + node.getAttribute = (name: string) => { + if (name === "type" && node.type) return node.type; + return Object.prototype.hasOwnProperty.call(node.attrs, name) ? node.attrs[name]! : null; + }; + node.setAttribute = (name: string, value: string) => { + node.attrs[name] = value; + }; + node.hasAttribute = (name: string) => name === "hidden" && node.hidden ? true : Object.prototype.hasOwnProperty.call(node.attrs, name); + node.getBoundingClientRect = () => options.rect ?? { width: 20, height: 10, left: 0, top: 0 }; + node.closest = (selector: string) => { + if (selector !== "label") return null; + let current = node.parent; + while (current) { + if (current.tagName === "LABEL") return current; + current = current.parent; + } + return null; + }; + node.contains = (other: FakeNode) => other === node || node.children.some((child) => child.contains(other)); + node.addEventListener = (type: string, fn: () => void) => { + node.listeners[type] = fn; + }; + node.removeEventListener = (type: string) => { + delete node.listeners[type]; + }; + node.dispatchEvent = (event: { type: string }) => { + node.listeners[event.type]?.(); + }; + node.focus = () => { + node.focusCalls += 1; + }; + for (const child of node.children) child.parent = node; + return node; +} + +function flatten(node: FakeNode, into: FakeNode[] = []): FakeNode[] { + into.push(node); + for (const child of node.children) flatten(child, into); + return into; +} + +describe("semantic page map", () => { + function mapContext() { + const button = makeNode("BUTTON", { id: "go", text: "Run check" }); + const hiddenChild = makeNode("BUTTON", { text: "Hidden action" }); + const hiddenWrap = makeNode("DIV", { style: { display: "none" }, children: [hiddenChild] }); + const ghost = makeNode("BUTTON", { text: "Ghost", style: { visibility: "hidden" } }); + const email = makeNode("BUTTON", { text: "user@example.com" }); + const label = makeNode("LABEL", { id: "name-label", text: "Name", htmlFor: "name" }); + const input = makeNode("INPUT", { id: "name", type: "text", value: "" }); + input.labels = [label]; + const password = makeNode("INPUT", { id: "pw", type: "password", value: "hunter2" }); + const box = makeNode("INPUT", { + id: "agree", + type: "checkbox", + checked: true, + attrs: { "aria-label": "Agree" }, + }); + const labelled = makeNode("BUTTON", { attrs: { "aria-labelledby": "name-label" }, text: "ignored" }); + const body = makeNode("BODY", { + children: [button, hiddenWrap, ghost, email, label, input, password, box, labelled], + }); + const nodes = flatten(body); + const rootAttrs: Record = {}; + const document = { + title: "Map", + body, + documentElement: { + getAttribute: (name: string) => rootAttrs[name] ?? null, + setAttribute: (name: string, value: string) => { + rootAttrs[name] = value; + }, + }, + getElementById: (id: string) => nodes.find((node) => node.id === id) ?? null, + getElementsByTagName: (tag: string) => nodes.filter((node) => node.tagName === tag.toUpperCase()), + querySelector: (selector: string) => { + const match = /\[data-btmcp-ref="([^"]+)"\]/.exec(selector); + if (!match) return null; + return nodes.find((node) => node.attrs["data-btmcp-ref"] === match[1]) ?? null; + }, + elementFromPoint: () => null as FakeNode | null, + }; + const loaded = loadExtension({}, document) as ReturnType & { + btmcpSemanticSnapshot: () => { + nodes: Array<{ + ref: string; + role: string; + name: string; + text: string; + visible: boolean; + state: Record; + }>; + truncated: boolean; + }; + btmcpActOnRef: (command: Record) => Promise<{ + outcome: string; + reason: string | null; + trusted: boolean; + delta: { + urlChanged: boolean; + previousUrl: string; + url: string; + consoleErrors: string[]; + failedRequests: { url: string; status: number | null }[]; + }; + }>; + btmcpEnsureWatch: () => void; + BTMCP_DELTA_WAIT_MS: number; + console: { error: (...args: unknown[]) => void }; + fetch: (input: string) => Promise<{ status: number; url: string }>; + location: { href: string; hostname: string }; + document: typeof document; + getComputedStyle: (element: FakeNode) => { display: string; visibility: string; opacity: string }; + HTMLInputElement: { prototype: object }; + }; + loaded.getComputedStyle = (element: FakeNode) => ({ + display: element.style.display || "block", + visibility: element.style.visibility || "visible", + opacity: element.style.opacity || "1", + }); + loaded.document.elementFromPoint = () => loaded.document.querySelector("[data-btmcp-ref]") as FakeNode | null; + const originalQuery = loaded.document.querySelector; + loaded.document.querySelector = (selector: string) => { + const found = originalQuery(selector); + if (found) loaded.document.elementFromPoint = () => found; + return found; + }; + loaded.BTMCP_DELTA_WAIT_MS = 0; + loaded.location.href = "http://127.0.0.1/map"; + return { loaded, button, input, nodes }; + } + + it("maps role, name, text, state, and visibility, and keeps refs stable", () => { + const { loaded, button } = mapContext(); + const first = loaded.btmcpSemanticSnapshot(); + const again = loaded.btmcpSemanticSnapshot(); + const names = first.nodes.map((node) => `${node.role}:${node.name}:${node.visible}`); + expect(names).toContain("button:Run check:true"); + expect(names).toContain("button:Ghost:false"); + expect(names).toContain("button:[redacted]:true"); + expect(names).toContain("textbox:Name:true"); + expect(names).toContain("checkbox:Agree:true"); + expect(JSON.stringify(first)).not.toContain("user@example.com"); + expect(JSON.stringify(first)).not.toContain("hunter2"); + expect(JSON.stringify(first)).not.toContain("Hidden action"); + const agree = first.nodes.find((node) => node.name === "Agree"); + expect(agree?.state).toMatchObject({ checked: true }); + expect(first.nodes.map((node) => node.ref)).toEqual(again.nodes.map((node) => node.ref)); + expect(button.attrs["data-btmcp-ref"]).toBe(first.nodes[0]?.ref); + expect(button.focusCalls).toBe(0); + expect(first.truncated).toBe(false); + }); + + it("acts on a ref with the soft gates and reports only new console and network failures", async () => { + const { loaded, input } = mapContext(); + let stored = ""; + class HTMLInputElement {} + Object.defineProperty(HTMLInputElement.prototype, "value", { + configurable: true, + set(value: string) { + stored = value; + }, + }); + loaded.HTMLInputElement = HTMLInputElement; + Object.defineProperty(input, "value", { + get: () => stored, + set: (value: string) => { + stored = value; + }, + }); + const seen: string[] = []; + loaded.console = { + error: (...args: unknown[]) => { + seen.push(args.map(String).join(" ")); + }, + }; + const fetches: string[] = []; + loaded.fetch = async (url: string) => { + fetches.push(url); + return { status: 500, url: "http://127.0.0.1/api/fail" }; + }; + input.dispatchEvent = () => { + loaded.console.error("before-should-not-leak"); + }; + loaded.btmcpEnsureWatch(); + loaded.console.error("already there"); + input.dispatchEvent = (event: { type: string }) => { + if (event.type !== "input") return; + loaded.console.error("MARKER-MAP-ERROR"); + void loaded.fetch("/api/fail"); + loaded.location.href = "http://127.0.0.1/map?changed=1"; + }; + + const map = loaded.btmcpSemanticSnapshot(); + const ref = map.nodes.find((node) => node.role === "textbox" && node.name === "Name")?.ref; + expect(ref).toMatch(/^e[1-9]/); + const result = await loaded.btmcpActOnRef({ op: "type", ref, text: "Ada" }); + expect(result).toMatchObject({ outcome: "worked", reason: null, trusted: false }); + expect(stored).toBe("Ada"); + expect(result.delta.urlChanged).toBe(true); + expect(result.delta.url).toBe("http://127.0.0.1/map?changed=1"); + expect(result.delta.consoleErrors).toEqual(["MARKER-MAP-ERROR"]); + expect(result.delta.failedRequests).toEqual([{ url: "http://127.0.0.1/api/fail", status: 500 }]); + expect(fetches).toEqual(["/api/fail"]); + expect(seen.join(" ")).toContain("MARKER-MAP-ERROR"); + expect(input.focusCalls).toBe(0); + }); +}); + +describe("snapshot and act extension actions", () => { + function apiFor(tabs: Record>) { + const calls: { method: string; props?: unknown }[] = []; + const scripts: { world?: string; files?: string[] }[] = []; + const api = { + storage: { + session: storageArea({ btmcpPopupSwitch: true }), + local: storageArea({ btmcpExcludeOrigins: [] }), + }, + tabs: { + get: async (id: number) => tabs[id], + update: async () => { + calls.push({ method: "tabs.update" }); + }, + }, + windows: { update: async () => calls.push({ method: "windows.update" }) }, + permissions: { contains: async () => true }, + scripting: { + executeScript: async (details: { world?: string; files?: string[] }) => { + scripts.push(details); + if (details.files) return [{ result: null }]; + if (details.world === "ISOLATED") { + return [ + { + result: { + title: "Map", + truncated: false, + nodes: [{ ref: "e1", role: "button", name: "Run check", text: "Run check", state: {}, visible: true, depth: 1 }], + }, + }, + ]; + } + return [ + { + result: { + outcome: "worked", + reason: null, + detail: "Untrusted click was delivered to the element.", + trusted: false, + delta: { + previousUrl: "http://127.0.0.1:3000/map", + url: "http://127.0.0.1:3000/map?changed=1", + consoleErrors: ["MARKER-MAP-ERROR"], + failedRequests: [{ url: "http://127.0.0.1:3000/api/fail", status: 500 }], + }, + }, + }, + ]; + }, + }, + debugger: { attach() { calls.push({ method: "debugger.attach" }); } }, + }; + return { api, calls, scripts }; + } + + it("snapshots in the isolated world and does not attach the debugger or change focus", async () => { + const { api, calls, scripts } = apiFor({ + 7: { id: 7, url: "http://127.0.0.1:3000/map", title: "Map", discarded: false, frozen: false }, + }); + const loaded = loadExtension(api); + loaded.BTMCP_SAFETY.install(api); + let result: Record = {}; + await loaded.BTMCP_TABS.snapshot(api, { requestId: "s", chromeTabId: 7, approvedUrl: "http://127.0.0.1:3000/map" }, (_id, payload) => { + result = payload; + }); + expect(result["ok"]).toBe(true); + const snapshot = result["snapshot"] as { nodes: { ref: string }[]; hostAccess: boolean }; + expect(snapshot.hostAccess).toBe(true); + expect(snapshot.nodes[0]?.ref).toBe("e1"); + expect(scripts[0]).toMatchObject({ world: "ISOLATED", files: ["tab-page.js"] }); + expect(calls).toEqual([]); + expect((result["safety"] as { debuggerCalls: number; focusChanges: number }).debuggerCalls).toBe(0); + }); + + it("rejects a ref that did not come from the page map before touching the page", async () => { + const { api, scripts } = apiFor({ + 7: { id: 7, url: "http://127.0.0.1:3000/map", discarded: false, frozen: false }, + }); + const loaded = loadExtension(api); + let result: Record = {}; + await loaded.BTMCP_TABS.act(api, { requestId: "a", chromeTabId: 7, ref: "#go", op: "click", approvedUrl: "http://127.0.0.1:3000/map" }, (_id, payload) => { + result = payload; + }); + expect(result["ok"]).toBe(false); + expect(String(result["error"])).toMatch(/snapshotTab/); + expect(scripts).toEqual([]); + }); + + it("acts in the main world and returns the what-changed delta", async () => { + const { api, calls, scripts } = apiFor({ + 7: { id: 7, url: "http://127.0.0.1:3000/map", discarded: false, frozen: false }, + }); + const loaded = loadExtension(api); + loaded.BTMCP_SAFETY.install(api); + let result: Record = {}; + await loaded.BTMCP_TABS.act( + api, + { requestId: "a", chromeTabId: 7, ref: "e1", op: "click", approvedUrl: "http://127.0.0.1:3000/map" }, + (_id, payload) => { + result = payload; + } + ); + expect(result["ok"]).toBe(true); + expect(result["trusted"]).toBe(false); + expect(result["outcome"]).toBe("worked"); + expect(result["delta"]).toMatchObject({ + urlChanged: true, + consoleErrors: ["MARKER-MAP-ERROR"], + failedRequests: [{ url: "http://127.0.0.1:3000/api/fail", status: 500 }], + }); + expect(scripts.some((script) => script.world === "MAIN")).toBe(true); + expect(calls.some((call) => call.method === "debugger.attach" || call.method === "windows.update" || call.method === "tabs.update")).toBe(false); + }); +}); diff --git a/browser-tools-mcp/test/unit/tab-listing.test.ts b/browser-tools-mcp/test/unit/tab-listing.test.ts index f284dcff..a5d35360 100644 --- a/browser-tools-mcp/test/unit/tab-listing.test.ts +++ b/browser-tools-mcp/test/unit/tab-listing.test.ts @@ -4,7 +4,9 @@ import { defaultGatePolicy } from "../../src/gates/capability"; import { TAB_COVERAGE } from "../../src/tabs/contract"; import { normalizeChromeTabListing, + parseActOnRef, parseChromeNavigate, + parseSemanticSnapshot, parseSoftInput, } from "../../src/tabs/listing"; @@ -113,3 +115,102 @@ describe("tab action results", () => { ).toThrow(/without a reason/); }); }); + +describe("semantic page map results", () => { + it("keeps role, name, text, state, visibility, and a stable ref, and strips secrets", () => { + const snapshot = parseSemanticSnapshot(4, { + url: "http://127.0.0.1:3000/map?token=ghp_abcdefghijklmnopqrstuvwxyz0123456789", + title: "Map", + asleep: false, + frozen: false, + skipped: null, + hostAccess: true, + truncated: false, + nodes: [ + { + ref: "e1", + role: "button", + name: "Run check", + text: "Run check", + state: { pressed: false }, + visible: true, + depth: 1, + }, + { + ref: "e2", + role: "checkbox", + name: "Agree", + text: "", + state: { checked: true, disabled: false }, + visible: false, + depth: 2, + }, + { ref: "not-a-ref", role: "button", name: "skip", text: "", state: {}, visible: true, depth: 1 }, + ], + }); + expect(snapshot.chromeTabId).toBe(4); + expect(snapshot.nodes).toHaveLength(2); + expect(snapshot.nodes[0]).toMatchObject({ + ref: "e1", + role: "button", + name: "Run check", + visible: true, + state: { pressed: false }, + }); + expect(snapshot.nodes[1]?.state).toEqual({ checked: true }); + expect(JSON.stringify(snapshot)).not.toContain("ghp_"); + }); + + it("returns the what-changed delta and refuses a focus change", () => { + expect( + parseActOnRef(4, "e1", { + outcome: "worked", + reason: null, + detail: "Untrusted click was delivered to the element.", + trusted: true, + delta: { + urlChanged: false, + previousUrl: "http://127.0.0.1:3000/map", + url: "http://127.0.0.1:3000/map?changed=1", + consoleErrors: ["MARKER-MAP-ERROR", ""], + failedRequests: [{ url: "http://127.0.0.1:3000/api/fail", status: 500 }, { url: "/api/down", status: 0 }], + }, + safety: { debuggerCalls: 0, focusChanges: 0 }, + }) + ).toEqual({ + chromeTabId: 4, + ref: "e1", + outcome: "worked", + reason: null, + detail: "Untrusted click was delivered to the element.", + trusted: false, + delta: { + urlChanged: true, + previousUrl: "http://127.0.0.1:3000/map", + url: "http://127.0.0.1:3000/map?changed=1", + consoleErrors: ["MARKER-MAP-ERROR"], + failedRequests: [ + { url: "http://127.0.0.1:3000/api/fail", status: 500 }, + { url: "/api/down", status: null }, + ], + }, + }); + + expect(() => + parseActOnRef(4, "e1", { + outcome: "worked", + detail: "", + delta: { previousUrl: "http://127.0.0.1/", url: "http://127.0.0.1/" }, + safety: { debuggerCalls: 0, focusChanges: 1 }, + }) + ).toThrow(/change focus/); + + expect(() => + parseActOnRef(4, "e1", { + outcome: "worked", + detail: "", + safety: { debuggerCalls: 0, focusChanges: 0 }, + }) + ).toThrow(/what changed/); + }); +}); diff --git a/chrome-extension/tab-actions.js b/chrome-extension/tab-actions.js index 13d9ee4c..7ca433d9 100644 --- a/chrome-extension/tab-actions.js +++ b/chrome-extension/tab-actions.js @@ -1,9 +1,10 @@ /** * chrome.tabs actions for the service worker. * - * List, read, background navigate, and soft click/type. No debugger API. - * Navigation always passes active: false and never focuses a window. - * Background navigate stays in the visible window. + * List, read, semantic snapshot, background navigate, and soft click/type. + * No debugger API. Navigation always passes active: false and never focuses + * a window. Background navigate stays in the visible window. Acting on a + * snapshot ref uses the same soft click and type gates. */ var BTMCP_TABS = (function () { function storageGet(area, keys) { @@ -168,6 +169,14 @@ var BTMCP_TABS = (function () { return btmcpSoftAct(command); } + function semanticEntry() { + return btmcpSemanticSnapshot(); + } + + function actOnRefEntry(command) { + return btmcpActOnRef(command); + } + function watchOpened(chromeApi) { var opened = []; function onCreated(tab) { @@ -550,10 +559,192 @@ var BTMCP_TABS = (function () { } } + function emptySnapshot(tab, extra) { + return Object.assign( + { + url: typeof tab.url === "string" ? tab.url : "", + title: typeof tab.title === "string" ? tab.title : "", + asleep: tab.discarded === true, + frozen: tab.frozen === true, + skipped: null, + hostAccess: false, + truncated: false, + nodes: [], + }, + extra || {} + ); + } + + async function snapshot(chromeApi, message, respond) { + var chromeTabId = message.chromeTabId; + if (typeof chromeTabId !== "number") { + fail(respond, message.requestId, "chrome-tab-snapshot-result", "chromeTabId must be a chrome.tabs id from listTabs."); + return; + } + var state = await readHumanState(chromeApi); + var beforeSafety = BTMCP_SAFETY.snapshot(); + try { + var tab = await chromeApi.tabs.get(chromeTabId); + if (!tab) throw new Error("Tab not found."); + if (tab.incognito === true) throw new Error("Incognito is skipped."); + if (excluded(tab.url || "", state.excludeOrigins)) throw new Error("origin is excluded"); + var gate = BTMCP_GATES.confirmBeforeAct({ + action: "readTab", + popupEnabled: state.popupEnabled, + panelEnabled: false, + approvedUrl: message.approvedUrl, + liveUrl: tab.url || "", + hideBanner: message.hideBanner === true, + }); + if (!gate.allowed) throw new Error(gate.reason); + if (tab.discarded === true) { + respond(message.requestId, { + type: "chrome-tab-snapshot-result", + ok: true, + safety: delta(beforeSafety), + snapshot: emptySnapshot(tab, { asleep: true, skipped: "asleep", hostAccess: true }), + }); + return; + } + if (tab.frozen === true) { + respond(message.requestId, { + type: "chrome-tab-snapshot-result", + ok: true, + safety: delta(beforeSafety), + snapshot: emptySnapshot(tab, { frozen: true, skipped: "frozen", hostAccess: true }), + }); + return; + } + var blocked = unsupportedPage(tab.url || ""); + if (blocked) { + respond(message.requestId, { + type: "chrome-tab-snapshot-result", + ok: true, + safety: delta(beforeSafety), + snapshot: emptySnapshot(tab, { skipped: blocked, hostAccess: false }), + }); + return; + } + var granted = await hostGranted(chromeApi, tab.url); + if (!granted) { + respond(message.requestId, { + type: "chrome-tab-snapshot-result", + ok: true, + safety: delta(beforeSafety), + snapshot: emptySnapshot(tab, { skipped: "host access is not granted", hostAccess: false }), + }); + return; + } + var injected = await runInTab(chromeApi, chromeTabId, "ISOLATED", semanticEntry); + var page = injected && injected[0] && injected[0].result ? injected[0].result : null; + if (!page || !Array.isArray(page.nodes)) throw new Error("The page map did not return nodes."); + respond(message.requestId, { + type: "chrome-tab-snapshot-result", + ok: true, + safety: delta(beforeSafety), + snapshot: emptySnapshot(tab, { + asleep: false, + frozen: false, + skipped: null, + hostAccess: true, + title: page.title || tab.title || "", + truncated: page.truncated === true, + nodes: page.nodes, + }), + }); + } catch (error) { + fail(respond, message.requestId, "chrome-tab-snapshot-result", error); + } + } + + function pageDelta(beforeUrl, afterUrl, pageDeltaValue) { + var reported = pageDeltaValue && typeof pageDeltaValue === "object" ? pageDeltaValue : {}; + var previousUrl = typeof reported.previousUrl === "string" && reported.previousUrl ? reported.previousUrl : beforeUrl; + var nextUrl = typeof reported.url === "string" && reported.url ? reported.url : afterUrl || previousUrl; + if (afterUrl && afterUrl !== beforeUrl) nextUrl = afterUrl; + return { + urlChanged: nextUrl !== previousUrl, + previousUrl: previousUrl, + url: nextUrl, + consoleErrors: Array.isArray(reported.consoleErrors) ? reported.consoleErrors : [], + failedRequests: Array.isArray(reported.failedRequests) ? reported.failedRequests : [], + }; + } + + async function act(chromeApi, message, respond) { + var chromeTabId = message.chromeTabId; + var ref = message.ref; + var op = message.op === "type" ? "type" : "click"; + if (typeof chromeTabId !== "number") { + fail(respond, message.requestId, "act-ref-result", "chromeTabId must be a chrome.tabs id from listTabs."); + return; + } + if (!btmcpValidRef(ref)) { + fail(respond, message.requestId, "act-ref-result", "ref must be a ref from snapshotTab"); + return; + } + if (op === "type" && (typeof message.text !== "string" || message.text.length > 20000)) { + fail(respond, message.requestId, "act-ref-result", "text is required"); + return; + } + var state = await readHumanState(chromeApi); + var beforeSafety = BTMCP_SAFETY.snapshot(); + var beforeUrl = ""; + try { + var tab = await chromeApi.tabs.get(chromeTabId); + if (!tab) throw new Error("Tab not found."); + beforeUrl = typeof tab.url === "string" ? tab.url : ""; + if (tab.incognito === true) throw new Error("Incognito is skipped."); + if (tab.discarded === true) throw new Error("Tab is asleep. It was not reloaded."); + if (tab.frozen === true) throw new Error("Tab is frozen and was skipped."); + var blocked = unsupportedPage(beforeUrl); + if (blocked) throw new Error(blocked); + var gate = BTMCP_GATES.confirmBeforeAct({ + action: op === "type" ? "softType" : "softClick", + popupEnabled: state.popupEnabled, + panelEnabled: false, + approvedUrl: message.approvedUrl, + liveUrl: beforeUrl, + hideBanner: message.hideBanner === true, + }); + if (!gate.allowed) throw new Error(gate.reason); + var granted = await hostGranted(chromeApi, beforeUrl); + if (!granted) throw new Error("host access is not granted"); + var injected = await runInTab(chromeApi, chromeTabId, "MAIN", actOnRefEntry, [ + { op: op, ref: ref, text: message.text || "", submit: message.submit === true }, + ]); + var result = injected && injected[0] ? injected[0].result : null; + if (!result || (result.outcome !== "worked" && result.outcome !== "needs a real gesture")) { + throw new Error("The action did not report an outcome."); + } + if (result.outcome === "needs a real gesture" && typeof result.reason !== "string") { + throw new Error("The action refused without a reason."); + } + var after = await chromeApi.tabs.get(chromeTabId); + var afterUrl = after && typeof after.url === "string" ? after.url : beforeUrl; + respond(message.requestId, { + type: "act-ref-result", + ok: true, + chromeTabId: chromeTabId, + ref: ref, + outcome: result.outcome, + reason: result.outcome === "worked" ? null : result.reason, + detail: typeof result.detail === "string" ? result.detail : "", + trusted: false, + delta: pageDelta(beforeUrl, afterUrl, result.delta), + safety: delta(beforeSafety), + }); + } catch (error) { + fail(respond, message.requestId, "act-ref-result", error); + } + } + return { list: list, read: read, + snapshot: snapshot, soft: soft, + act: act, navigateChrome: navigateChrome, navigateConnected: navigateConnected, }; diff --git a/chrome-extension/tab-page.js b/chrome-extension/tab-page.js index c81e5cb6..0e44df6b 100644 --- a/chrome-extension/tab-page.js +++ b/chrome-extension/tab-page.js @@ -1,9 +1,15 @@ /** * Runs inside a tab via chrome.scripting.executeScript. * - * btmcpReadPage stays in the isolated world. btmcpSoftAct runs in the page's - * main world so the native value setter and execCommand reach React, Vue, and - * rich editors. Neither function calls the debugger API or changes window focus. + * btmcpReadPage and btmcpSemanticSnapshot stay in the isolated world. + * btmcpSoftAct and btmcpActOnRef run in the page's main world so the native + * value setter and execCommand reach React, Vue, and rich editors. + * None of them calls the debugger API or focuses the browser window. + * + * A semantic snapshot stamps data-btmcp-ref on each mapped element. The same + * element keeps that ref until the document is replaced. Acting on a ref uses + * the soft click and type gates, then reports URL, console, and failed-request + * changes that happened after the action. */ function btmcpScrubText(value) { @@ -332,3 +338,453 @@ function btmcpTypeInto(element, text, submit) { if (editorSubmit) return editorSubmit; return btmcpWorked(submit ? "Text inserted and the form submitted." : "Inserted text with execCommand."); } + +var BTMCP_DELTA_WAIT_MS = 300; +var BTMCP_MAP_LIMIT = 80; +var BTMCP_REF_ATTR = "data-btmcp-ref"; +var BTMCP_REF_SEQ = "data-btmcp-ref-seq"; +var BTMCP_WATCH_KEY = "__btmcpChangeWatch"; + +function btmcpValidRef(ref) { + return typeof ref === "string" && /^e[1-9][0-9]{0,4}$/.test(ref); +} + +function btmcpPageUrl() { + try { + if (typeof location !== "undefined" && location && typeof location.href === "string") return location.href; + } catch (error) { + /* a detached frame can throw on location */ + } + return ""; +} + +function btmcpDeltaWaitMs() { + var value = typeof BTMCP_DELTA_WAIT_MS === "number" ? BTMCP_DELTA_WAIT_MS : 300; + if (!isFinite(value) || value < 0) return 300; + if (value > 1000) return 1000; + return value; +} + +function btmcpTag(element) { + return String(element && element.tagName ? element.tagName : "").toUpperCase(); +} + +function btmcpFlatText(node) { + if (!node) return ""; + var raw = ""; + if (typeof node.innerText === "string" && node.innerText) raw = node.innerText; + else if (typeof node.textContent === "string") raw = node.textContent; + return btmcpScrubText(raw.replace(/\s+/g, " ").trim()); +} + +function btmcpInputType(element) { + return ((element.getAttribute && element.getAttribute("type")) || element.type || "").toLowerCase(); +} + +function btmcpAccessibleName(element) { + var tag = btmcpTag(element); + var type = btmcpInputType(element); + var labelledby = element.getAttribute && element.getAttribute("aria-labelledby"); + if (labelledby && document.getElementById) { + var ids = String(labelledby).split(/\s+/); + var parts = []; + for (var i = 0; i < ids.length; i++) { + if (!ids[i]) continue; + var target = document.getElementById(ids[i]); + var piece = btmcpFlatText(target); + if (piece) parts.push(piece); + } + if (parts.length) return parts.join(" ").slice(0, 120); + } + var aria = element.getAttribute && element.getAttribute("aria-label"); + if (aria && String(aria).trim()) return btmcpScrubText(String(aria).replace(/\s+/g, " ").trim()).slice(0, 120); + if (element.labels && element.labels.length) { + var fromLabels = btmcpFlatText(element.labels[0]); + if (fromLabels) return fromLabels.slice(0, 120); + } + if (element.id && document.getElementsByTagName) { + var labels = document.getElementsByTagName("label"); + for (var l = 0; l < labels.length; l++) { + var label = labels[l]; + var htmlFor = label.htmlFor || (label.getAttribute && label.getAttribute("for")); + if (htmlFor && htmlFor === element.id) { + var named = btmcpFlatText(label); + if (named) return named.slice(0, 120); + } + } + } + if (element.closest) { + var wrap = element.closest("label"); + if (wrap && wrap !== element) { + var wrapped = btmcpFlatText(wrap); + if (wrapped) return wrapped.slice(0, 120); + } + } + if (tag === "IMG") { + var alt = element.getAttribute && element.getAttribute("alt"); + if (alt && String(alt).trim()) return btmcpScrubText(String(alt).trim()).slice(0, 120); + } + if (tag !== "INPUT" && tag !== "TEXTAREA" && tag !== "SELECT") { + var text = btmcpFlatText(element); + if (text) return text.slice(0, 120); + } + var title = element.getAttribute && element.getAttribute("title"); + if (title && String(title).trim()) return btmcpScrubText(String(title).trim()).slice(0, 120); + if (type === "password") return "password"; + if (tag === "INPUT" || tag === "TEXTAREA") { + var placeholder = element.getAttribute && element.getAttribute("placeholder"); + if (placeholder && String(placeholder).trim()) return btmcpScrubText(String(placeholder).trim()).slice(0, 120); + } + return ""; +} + +function btmcpElementText(element) { + var tag = btmcpTag(element); + var type = btmcpInputType(element); + if (type === "password" || type === "file" || type === "hidden") return ""; + if (tag === "INPUT" || tag === "TEXTAREA") { + var value = element.value == null ? "" : String(element.value); + return btmcpScrubText(value.replace(/\s+/g, " ").trim()).slice(0, 160); + } + if (tag === "SELECT" && element.options && element.selectedIndex >= 0) { + return btmcpFlatText(element.options[element.selectedIndex]).slice(0, 160); + } + return btmcpFlatText(element).slice(0, 160); +} + +function btmcpAttrFlag(element, name) { + var value = element.getAttribute && element.getAttribute(name); + if (value === "true") return true; + if (value === "false") return false; + if (value === "mixed") return "mixed"; + return null; +} + +function btmcpElementState(element, role) { + var state = {}; + if (element.disabled === true || btmcpAttrFlag(element, "aria-disabled") === true) state.disabled = true; + var checkable = + role === "checkbox" || + role === "radio" || + role === "switch" || + role === "menuitemcheckbox" || + role === "menuitemradio" || + (element.getAttribute && element.getAttribute("aria-checked") != null); + if (checkable) { + var checked = btmcpAttrFlag(element, "aria-checked"); + if (checked === null && typeof element.checked === "boolean") checked = element.checked; + if (checked !== null) state.checked = checked; + } + var expanded = btmcpAttrFlag(element, "aria-expanded"); + if (expanded === true || expanded === false) state.expanded = expanded; + var pressed = btmcpAttrFlag(element, "aria-pressed"); + if (pressed !== null) state.pressed = pressed; + var selected = btmcpAttrFlag(element, "aria-selected"); + if (selected === null && role === "option" && element.selected === true) selected = true; + if (selected === true || selected === false) state.selected = selected; + if (element.required === true || btmcpAttrFlag(element, "aria-required") === true) state.required = true; + if (btmcpAttrFlag(element, "aria-invalid") === true) state.invalid = true; + if (element.readOnly === true || btmcpAttrFlag(element, "aria-readonly") === true) state.readonly = true; + return state; +} + +function btmcpImplicitRole(element) { + var explicit = element.getAttribute && element.getAttribute("role"); + if (explicit && String(explicit).trim()) return String(explicit).trim().toLowerCase().split(/\s+/)[0]; + var tag = btmcpTag(element); + if (tag === "A") return element.hasAttribute && element.hasAttribute("href") ? "link" : ""; + if (tag === "BUTTON") return "button"; + if (tag === "SUMMARY") return "button"; + if (tag === "SELECT") return element.multiple ? "listbox" : "combobox"; + if (tag === "TEXTAREA") return "textbox"; + if (tag === "OPTION") return "option"; + if (tag === "INPUT") { + var type = btmcpInputType(element) || "text"; + if (type === "hidden") return ""; + if (type === "button" || type === "submit" || type === "reset" || type === "image" || type === "file") return "button"; + if (type === "checkbox") return "checkbox"; + if (type === "radio") return "radio"; + if (type === "range") return "slider"; + if (type === "number") return "spinbutton"; + if (type === "search") return "searchbox"; + return "textbox"; + } + if (tag === "IMG") return "img"; + if (tag === "NAV") return "navigation"; + if (tag === "MAIN") return "main"; + if (tag === "FORM") return "form"; + if (tag === "HEADER") return "banner"; + if (tag === "FOOTER") return "contentinfo"; + if (tag === "ASIDE") return "complementary"; + if (tag === "DIALOG") return "dialog"; + if (tag === "UL" || tag === "OL") return "list"; + if (tag === "LI") return "listitem"; + if (tag === "TABLE") return "table"; + if (/^H[1-6]$/.test(tag)) return "heading"; + if (element.isContentEditable) return "textbox"; + return ""; +} + +function btmcpSkipSubtree(element) { + var tag = btmcpTag(element); + if (tag === "SCRIPT" || tag === "STYLE" || tag === "NOSCRIPT" || tag === "TEMPLATE" || tag === "HEAD") return true; + if (element.hidden === true) return true; + if (element.hasAttribute && element.hasAttribute("hidden")) return true; + if (element.getAttribute && element.getAttribute("aria-hidden") === "true") return true; + var style = null; + try { + style = window.getComputedStyle ? window.getComputedStyle(element) : null; + } catch (error) { + style = null; + } + if (style && style.display === "none") return true; + return false; +} + +function btmcpElementVisible(element) { + var style = null; + try { + style = window.getComputedStyle ? window.getComputedStyle(element) : null; + } catch (error) { + style = null; + } + if (style && (style.visibility === "hidden" || style.visibility === "collapse" || String(style.opacity) === "0")) { + return false; + } + var rect = element.getBoundingClientRect ? element.getBoundingClientRect() : null; + if (!rect || rect.width <= 0 || rect.height <= 0) return false; + return true; +} + +function btmcpRefOwner(ref) { + if (!document.querySelector || !btmcpValidRef(ref)) return null; + try { + return document.querySelector("[" + BTMCP_REF_ATTR + '="' + ref + '"]'); + } catch (error) { + return null; + } +} + +function btmcpAssignRef(element) { + var existing = element.getAttribute && element.getAttribute(BTMCP_REF_ATTR); + if (btmcpValidRef(existing) && btmcpRefOwner(existing) === element) return existing; + var root = document.documentElement || document.body; + var seq = 0; + if (root && root.getAttribute) { + seq = Number(root.getAttribute(BTMCP_REF_SEQ) || "0"); + if (!isFinite(seq) || seq < 0) seq = 0; + } + var ref = ""; + var guard = 0; + do { + seq += 1; + if (seq > 99999) seq = 1; + ref = "e" + seq; + guard += 1; + } while (btmcpRefOwner(ref) && guard < 5); + if (root && root.setAttribute) root.setAttribute(BTMCP_REF_SEQ, String(seq)); + if (element.setAttribute) element.setAttribute(BTMCP_REF_ATTR, ref); + return ref; +} + +function btmcpSemanticSnapshot() { + var nodes = []; + var truncated = false; + function consider(element, depth) { + if (!element || !element.tagName || truncated) return; + if (btmcpSkipSubtree(element)) return; + var role = btmcpImplicitRole(element); + if (role && role !== "presentation" && role !== "none") { + if (nodes.length >= BTMCP_MAP_LIMIT) { + truncated = true; + return; + } + nodes.push({ + ref: btmcpAssignRef(element), + role: role, + name: btmcpAccessibleName(element), + text: btmcpElementText(element), + state: btmcpElementState(element, role), + visible: btmcpElementVisible(element), + depth: depth, + }); + } + var children = element.children || []; + for (var i = 0; i < children.length; i++) consider(children[i], depth + 1); + } + var root = document.body || document.documentElement; + if (root) consider(root, 0); + return { + title: btmcpScrubText(document.title || "").slice(0, 200), + nodes: nodes, + truncated: truncated, + }; +} + +function btmcpPushCapped(list, item, cap) { + if (list.length >= cap) return; + list.push(item); +} + +function btmcpNoteUrl(url) { + var raw = String(url || ""); + try { + if (raw && typeof location !== "undefined" && location && location.href) raw = new URL(raw, location.href).href; + } catch (error) { + /* keep the raw string */ + } + return btmcpScrubText(raw).slice(0, 300); +} + +function btmcpEnsureWatch() { + var existing = window[BTMCP_WATCH_KEY]; + if (existing && existing.installed) return existing; + var state = { installed: true, consoleErrors: [], failedRequests: [] }; + + function noteError(text) { + var scrubbed = btmcpScrubText(String(text == null ? "" : text)).replace(/\s+/g, " ").trim().slice(0, 240); + if (!scrubbed) return; + btmcpPushCapped(state.consoleErrors, scrubbed, 40); + } + + function noteFailure(url, status) { + var code = typeof status === "number" && isFinite(status) ? status : null; + btmcpPushCapped(state.failedRequests, { url: btmcpNoteUrl(url), status: code }, 40); + } + + if (window.console && typeof window.console.error === "function") { + var originalError = window.console.error; + window.console.error = function () { + try { + var parts = []; + for (var i = 0; i < arguments.length; i++) { + var value = arguments[i]; + if (value instanceof Error) parts.push(value.message); + else parts.push(String(value)); + } + noteError(parts.join(" ")); + } catch (error) { + /* keep the page console working */ + } + return originalError.apply(window.console, arguments); + }; + } + + if (window.addEventListener) { + window.addEventListener("error", function (event) { + var message = event && (event.message || (event.error && event.error.message)); + noteError(message || "error"); + }); + window.addEventListener("unhandledrejection", function (event) { + var reason = event && event.reason; + noteError(reason instanceof Error ? reason.message : reason == null ? "unhandled rejection" : String(reason)); + }); + } + + if (typeof window.fetch === "function") { + var originalFetch = window.fetch; + window.fetch = function (input) { + var requestUrl = ""; + try { + if (typeof input === "string") requestUrl = input; + else if (input && typeof input.url === "string") requestUrl = input.url; + } catch (error) { + requestUrl = ""; + } + return originalFetch.apply(this, arguments).then( + function (response) { + try { + if (response && typeof response.status === "number" && response.status >= 400) { + noteFailure((response && response.url) || requestUrl, response.status); + } + } catch (error) { + /* ignore observer failures */ + } + return response; + }, + function (error) { + noteFailure(requestUrl, null); + throw error; + } + ); + }; + } + + if (window.XMLHttpRequest && window.XMLHttpRequest.prototype) { + var proto = window.XMLHttpRequest.prototype; + var originalOpen = proto.open; + var originalSend = proto.send; + if (typeof originalOpen === "function" && typeof originalSend === "function") { + proto.open = function (method, url) { + try { + this.__btmcpUrl = url; + } catch (error) { + /* the request still proceeds */ + } + return originalOpen.apply(this, arguments); + }; + proto.send = function () { + var xhr = this; + if (xhr.addEventListener) { + xhr.addEventListener("loadend", function () { + var status = xhr.status; + if (status === 0 || status >= 400) noteFailure(xhr.__btmcpUrl || xhr.responseURL || "", status === 0 ? null : status); + }); + } + return originalSend.apply(this, arguments); + }; + } + } + + try { + Object.defineProperty(window, BTMCP_WATCH_KEY, { value: state, configurable: true }); + } catch (error) { + window[BTMCP_WATCH_KEY] = state; + } + return state; +} + +function btmcpWatchCursor() { + var watch = btmcpEnsureWatch(); + return { + url: btmcpPageUrl(), + consoleAt: watch.consoleErrors.length, + networkAt: watch.failedRequests.length, + }; +} + +function btmcpWatchDelta(cursor) { + var watch = btmcpEnsureWatch(); + var url = btmcpPageUrl(); + var previous = cursor && typeof cursor.url === "string" ? cursor.url : ""; + return { + urlChanged: url !== previous, + previousUrl: previous, + url: url, + consoleErrors: watch.consoleErrors.slice(cursor && cursor.consoleAt ? cursor.consoleAt : 0).slice(0, 8), + failedRequests: watch.failedRequests.slice(cursor && cursor.networkAt ? cursor.networkAt : 0).slice(0, 8), + }; +} + +function btmcpActOnRef(command) { + var ref = command && command.ref; + var cursor = btmcpWatchCursor(); + function finish(result) { + return new Promise(function (resolve) { + setTimeout(function () { + result.delta = btmcpWatchDelta(cursor); + resolve(result); + }, btmcpDeltaWaitMs()); + }); + } + if (!btmcpValidRef(ref)) { + return finish(btmcpGesture("UI that needs the page drawn", "The ref is not from snapshotTab.")); + } + var result = btmcpSoftAct({ + op: command && command.op, + selector: "[" + BTMCP_REF_ATTR + '="' + ref + '"]', + text: command && command.text, + submit: command && command.submit === true, + }); + return finish(result); +} diff --git a/chrome-extension/worker.js b/chrome-extension/worker.js index 854c1aa1..304a95b0 100644 --- a/chrome-extension/worker.js +++ b/chrome-extension/worker.js @@ -4,8 +4,9 @@ * Owns the toolbar-popup switch (session storage, so it resets when the * browser quits) and answers capability probes with a tab URL read at that * moment via chrome.tabs.get. Tab list, read, background navigate, and soft - * input go through BTMCP_TABS. Those paths do not call the debugger API and do - * not activate a tab or focus a window. DevTools capture stays in devtools.js. + * input, the semantic page map, and acting on a ref go through BTMCP_TABS. + * Those paths do not call the debugger API and do not activate a tab or focus + * a window. DevTools capture stays in devtools.js. */ importScripts("shared.js", "gates.js", "tab-safety.js", "tab-page.js", "tab-actions.js"); @@ -150,8 +151,12 @@ async function ensureConnected() { void BTMCP_TABS.navigateChrome(api, message, reply); } else if (message && message.type === "read-chrome-tab") { void BTMCP_TABS.read(api, message, reply); + } else if (message && message.type === "snapshot-chrome-tab") { + void BTMCP_TABS.snapshot(api, message, reply); } else if (message && message.type === "soft-input") { void BTMCP_TABS.soft(api, message, reply); + } else if (message && message.type === "act-chrome-tab") { + void BTMCP_TABS.act(api, message, reply); } else if (message && message.type === "ping") { sendOn(ws, { type: "pong", id: message.id }); }