From 5aab1982472014c5a7213cb67193aa6e30e78a29 Mon Sep 17 00:00:00 2001 From: drakezhang Date: Wed, 28 Oct 2026 22:44:39 +0800 Subject: [PATCH 1/5] feat: add native drag gestures with VOM targets --- .github/workflows/ci.yml | 24 + apps/extension/package.json | 4 + .../extension/src/cooperation/action-guard.ts | 8 +- .../src/cooperation/controller.test.ts | 39 +- apps/extension/src/cooperation/controller.ts | 30 +- apps/extension/src/debug/manager.ts | 1 + apps/extension/src/entrypoints/audit/App.tsx | 1 + .../src/session-manager/ref-store.ts | 12 +- .../__tests__/download-lifecycle.test.ts | 6 +- .../src/tools/__tests__/drag.browser.test.ts | 550 ++++++++++++++++++ .../src/tools/__tests__/drag.test.ts | 183 ++++++ .../__tests__/file-transfer-trigger.test.ts | 8 +- .../__tests__/transfer-transaction.test.ts | 21 + .../src/tools/background-execution.ts | 1 + apps/extension/src/tools/click-overlay.ts | 5 +- apps/extension/src/tools/dispatcher.ts | 17 +- apps/extension/src/tools/download.ts | 6 +- apps/extension/src/tools/drag.ts | 367 ++++++++++++ apps/extension/src/tools/interaction.ts | 30 +- apps/extension/src/tools/observation.ts | 1 + ...sfer-trigger.ts => pointer-transaction.ts} | 60 +- .../src/tools/transfer-transaction.ts | 16 +- apps/extension/src/tools/upload.ts | 4 +- .../src/tools/vom/semantic-graph/structure.ts | 1 + .../src/tools/vom/visual-observation.ts | 1 + apps/extension/src/transport/types.ts | 30 + crates/bsk-cli/skill/SKILL.md | 3 +- .../skill/references/interaction-details.md | 30 + crates/bsk-cli/src/cli/drag.rs | 205 +++++++ crates/bsk-cli/src/cli/mod.rs | 4 + crates/bsk-cli/src/daemon/ipc.rs | 1 + crates/bsk-cli/src/daemon/queue.rs | 8 +- crates/bsk-cli/src/main.rs | 1 + crates/bsk-cli/tests/tools_ipc.rs | 2 + .../bsk-protocol/schema/tool_drag_params.json | 164 ++++++ .../bsk-protocol/schema/tool_drag_result.json | 116 ++++ .../schema/tool_observe_result.json | 7 + .../schema/tool_snapshot_result.json | 7 + crates/bsk-protocol/src/bin/dump-schema.rs | 2 + crates/bsk-protocol/src/method.rs | 4 + crates/bsk-protocol/src/tools/drag.rs | 61 ++ crates/bsk-protocol/src/tools/mod.rs | 2 + crates/bsk-protocol/src/tools/observation.rs | 9 + .../regression/drag-interactions/README.md | 48 ++ .../drag-interactions.case.json | 70 +++ .../drag-interactions.fixture.mjs | 56 ++ .../drag-interactions/libraries.tsx | 61 ++ .../regression/drag-interactions/run.mjs | 205 +++++++ evals/browser/lib/bsk-runner.mjs | 12 + evals/browser/lib/case-loader.mjs | 18 +- evals/browser/lib/operations.mjs | 2 + evals/browser/tests/drag-workflow.test.mjs | 88 +++ evals/browser/tests/eval.test.mjs | 17 +- .../i18n/src/locales/de-DE/extension.json | 1 + .../i18n/src/locales/en-US/extension.json | 1 + .../i18n/src/locales/es-ES/extension.json | 1 + .../i18n/src/locales/fr-FR/extension.json | 1 + .../i18n/src/locales/it-IT/extension.json | 1 + .../i18n/src/locales/ja-JP/extension.json | 1 + .../i18n/src/locales/ko-KR/extension.json | 1 + .../i18n/src/locales/pt-BR/extension.json | 1 + .../i18n/src/locales/zh-CN/extension.json | 1 + .../i18n/src/locales/zh-TW/extension.json | 1 + packages/vom/src/__tests__/render.test.ts | 79 +++ packages/vom/src/render.ts | 40 +- packages/vom/src/types.ts | 2 + pnpm-lock.yaml | 64 ++ 67 files changed, 2759 insertions(+), 65 deletions(-) create mode 100644 apps/extension/src/tools/__tests__/drag.browser.test.ts create mode 100644 apps/extension/src/tools/__tests__/drag.test.ts create mode 100644 apps/extension/src/tools/__tests__/transfer-transaction.test.ts create mode 100644 apps/extension/src/tools/drag.ts rename apps/extension/src/tools/{file-transfer-trigger.ts => pointer-transaction.ts} (68%) create mode 100644 crates/bsk-cli/src/cli/drag.rs create mode 100644 crates/bsk-protocol/schema/tool_drag_params.json create mode 100644 crates/bsk-protocol/schema/tool_drag_result.json create mode 100644 crates/bsk-protocol/src/tools/drag.rs create mode 100644 evals/browser/cases/regression/drag-interactions/README.md create mode 100644 evals/browser/cases/regression/drag-interactions/drag-interactions.case.json create mode 100644 evals/browser/cases/regression/drag-interactions/drag-interactions.fixture.mjs create mode 100644 evals/browser/cases/regression/drag-interactions/libraries.tsx create mode 100644 evals/browser/cases/regression/drag-interactions/run.mjs create mode 100644 evals/browser/tests/drag-workflow.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ecaffcbb..34e46a7de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -284,6 +284,12 @@ jobs: BSK_CLICK_CHROME: google-chrome run: pnpm exec vitest run src/tools/extract/extract.browser.test.ts + - name: Run drag browser regression + working-directory: apps/extension + env: + BSK_DRAG_CHROME: google-chrome + run: pnpm exec vitest run src/tools/__tests__/drag.browser.test.ts + - name: Run fill browser regression working-directory: apps/extension env: @@ -363,6 +369,24 @@ jobs: path: evals/browser/results/js-dialog-control/ if-no-files-found: ignore + - name: Build extension for the isolated drag daemon + env: + BSK_DAEMON_WS_URL: ws://127.0.0.1:52839 + run: pnpm --filter @browser-skill/extension build + + - name: Validate drag CLI, native drops, recovery and cross-origin frame refs + env: + BSK_DRAG_CHROME: ${{ env.BSK_VIDEO_CHROME }} + run: node evals/browser/cases/regression/drag-interactions/run.mjs + + - name: Upload drag regression evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: drag-regression + path: evals/browser/results/drag-interactions/ + if-no-files-found: ignore + node-scripts: name: Node script tests runs-on: ubuntu-latest diff --git a/apps/extension/package.json b/apps/extension/package.json index 0bedec366..b45ba32fe 100644 --- a/apps/extension/package.json +++ b/apps/extension/package.json @@ -23,6 +23,8 @@ "react-dom": "^19.0.0" }, "devDependencies": { + "@dnd-kit/core": "6.3.1", + "@dnd-kit/sortable": "10.0.0", "@tailwindcss/vite": "^4.3.0", "@testing-library/dom": "^10.4.1", "@testing-library/react": "^16.3.2", @@ -32,7 +34,9 @@ "@types/react-dom": "^19.0.0", "@vitejs/plugin-react": "^6.0.0", "@wxt-dev/module-react": "^1.1.3", + "esbuild": "0.27.7", "happy-dom": "^15.11.7", + "sortablejs": "1.15.6", "tailwindcss": "^4.3.0", "typescript": "^5.8.0", "vitest": "^4.1.6", diff --git a/apps/extension/src/cooperation/action-guard.ts b/apps/extension/src/cooperation/action-guard.ts index 530476a53..1424d3c8d 100644 --- a/apps/extension/src/cooperation/action-guard.ts +++ b/apps/extension/src/cooperation/action-guard.ts @@ -43,10 +43,12 @@ export function guardedRunner(cdp: T, options: ActionGuard const key = method === "Input.dispatchKeyEvent"; const focus = method === "Emulation.setFocusEmulationEnabled"; const drag = method === "Input.dispatchDragEvent"; + const cancelDrag = method === "Input.cancelDragging"; const chooser = method === "Page.setInterceptFileChooserDialog"; - const identity = `${target.tabId}:${target.sessionId ?? ""}:${chooser ? "chooser" : drag ? "drag" : focus ? "focus" : mouse ? "mouse:" + String(p?.button) : "key:" + String(p?.key)}`; + const identity = `${target.tabId}:${target.sessionId ?? ""}:${cancelDrag ? "mouse:left" : chooser ? "chooser" : drag ? "drag" : focus ? "focus" : mouse ? "mouse:" + String(p?.button) : "key:" + String(p?.key)}`; const release = - ((mouse && p?.type === "mouseReleased") || + (cancelDrag || + (mouse && p?.type === "mouseReleased") || (key && p?.type === "keyUp") || (focus && p?.enabled === false) || (chooser && p?.enabled === false) || @@ -96,7 +98,7 @@ export function guardedRunner(cdp: T, options: ActionGuard (drag && p?.type === "dragEnter") ) leases.set(identity, cdp.getAttachmentId?.(target.tabId)); - if (release) leases.delete(identity); + if (release && !cancelDrag) leases.delete(identity); }; if (cdp.sendGuarded) return cdp.sendGuarded(target, method, params, { diff --git a/apps/extension/src/cooperation/controller.test.ts b/apps/extension/src/cooperation/controller.test.ts index 2461893f2..f3d880711 100644 --- a/apps/extension/src/cooperation/controller.test.ts +++ b/apps/extension/src/cooperation/controller.test.ts @@ -595,18 +595,22 @@ describe("shared-tab dispatch fences", () => { const run = ( f: Awaited>, context: string, - action: { ref?: string; dependsOn?: string[]; key?: string }, + action: { ref?: string; dependsOn?: string[]; key?: string; to?: string }, ) => { f.change({ structure: f.controller.view(7)!.revision + 100 }); return f.controller.run( { id: "live", - method: action.key ? "tool.press" : "tool.click", + method: action.to ? "tool.drag" : action.key ? "tool.press" : "tool.click", params: { session_id: "shared", tab_id: 7, context_id: context, - ...(action.ref ? { ref: action.ref } : {}), + ...(action.to + ? { source: { ref: action.ref }, target: { ref: action.to } } + : action.ref + ? { ref: action.ref } + : {}), ...(action.key ? { key: action.key } : {}), ...(action.dependsOn ? { depends_on: action.dependsOn } : {}), }, @@ -620,7 +624,7 @@ describe("shared-tab dispatch fences", () => { /** One learning refusal, then `attempts` fresh-sync actions all succeed. */ const learnsThenActs = async ( build: (tick: number) => VomScene, - action: { ref?: string; dependsOn?: string[] }, + action: { ref?: string; dependsOn?: string[]; to?: string }, attempts = 2, ) => { live(build); @@ -634,7 +638,7 @@ describe("shared-tab dispatch fences", () => { /** Every fresh-sync action keeps being refused without dispatching input. */ const keepsRefusing = async ( build: (tick: number) => VomScene, - action: { ref?: string; dependsOn?: string[]; key?: string }, + action: { ref?: string; dependsOn?: string[]; key?: string; to?: string }, ) => { live(build); refs(); @@ -649,6 +653,31 @@ describe("shared-tab dispatch fences", () => { }; const clock = (tick: number) => `10:00:0${tick}`; + it("learns unrelated live text for a drag while protecting both endpoints", () => + learnsThenActs( + page((tick) => [ + [1, null, "main"], + [2, 1, "region", { name: "Source" }], + [3, 2, "button", { name: "Card" }], + [5, 1, "region", { name: "Destination" }], + [6, 5, "button", { name: "Drop here" }], + [4, 1, "span", { text: clock(tick) }], + ]), + { ref: "e3", to: "e6" }, + )); + it("keeps a drag destination exact when dependencies mention only its source", () => + keepsRefusing( + page((tick) => [ + [1, null, "main"], + [2, 1, "region", { name: "Source" }], + [3, 2, "button", { name: "Card" }], + [5, 1, "region", { name: "Destination" }], + [6, 5, "button", { name: "Drop here" }], + [7, 5, "span", { text: clock(tick) }], + ]), + { ref: "e3", to: "e6", dependsOn: ["e3"] }, + )); + it("tolerates a clock beside a button directly inside
", () => learnsThenActs( page((tick) => [ diff --git a/apps/extension/src/cooperation/controller.ts b/apps/extension/src/cooperation/controller.ts index aed052490..ced2ea090 100644 --- a/apps/extension/src/cooperation/controller.ts +++ b/apps/extension/src/cooperation/controller.ts @@ -7,7 +7,7 @@ import { import { resolveActionTarget } from "@/tools/interaction"; import { captureVomObservation, handleSnapshot } from "@/tools/observation"; import { type CdpRunner, chromeTabsApi, isRpcError, resolveTargetTab } from "@/tools/shared"; -import type { RequestFrame, RpcError } from "@/transport/types"; +import type { DragTarget, RequestFrame, RpcError } from "@/transport/types"; import { guardedRunner } from "./action-guard"; import { type ControlAuthority, ControlAuthorityError } from "./control-authority"; import { @@ -888,6 +888,8 @@ export class CooperationController implements ControlAuthority { ref?: string; selector?: string; depends_on?: string[]; + source?: DragTarget; + target?: DragTarget; } | undefined; if (req.method === "tool.session_start") { @@ -1029,6 +1031,7 @@ export class CooperationController implements ControlAuthority { let verifiedFrames = context?.frames; let evidenceTargets: { backendNodeId: number; frameId?: string }[] | undefined; let actionTarget: { backendNodeId: number; frameId?: string } | undefined; + let destinationTarget: typeof actionTarget; state.working = true; const intervention = new AbortController(); state.interruption = intervention; @@ -1137,6 +1140,13 @@ export class CooperationController implements ControlAuthority { return sameObservationEvidence(context.evidence!, evidence!); }; const matches = (keys?: ReadonlySet) => { + // A drag decision also depends on its destination. Explicit dependencies + // or learned live text must never relax the destination's own region. + if ( + destinationTarget && + !sameObservationEvidence(context.evidence!, evidence!, destinationTarget, keys) + ) + return false; if (evidenceTargets) { return ( keyboardRegionMatches() && @@ -1201,17 +1211,31 @@ export class CooperationController implements ControlAuthority { context.refs !== session.refStore.revision ) return failure("This action requires the context returned by the latest cooperate sync"); - if (params.ref || params.selector) { + const primaryTarget = req.method === "tool.drag" ? params.source : params; + if (primaryTarget?.ref || primaryTarget?.selector) { const resolved = await resolveActionTarget( guardedRunner(cdp), session, target, - params, + primaryTarget, "shared action", + req.method === "tool.drag", ); if (isRpcError(resolved)) return resolved; actionTarget = resolved; } + if (req.method === "tool.drag" && (params.target?.ref || params.target?.selector)) { + const resolved = await resolveActionTarget( + guardedRunner(cdp), + session, + target, + params.target, + "drag destination", + true, + ); + if (isRpcError(resolved)) return resolved; + destinationTarget = resolved; + } if (params.depends_on !== undefined) { if ( !Array.isArray(params.depends_on) || diff --git a/apps/extension/src/debug/manager.ts b/apps/extension/src/debug/manager.ts index ecdb13a2d..a5917abef 100644 --- a/apps/extension/src/debug/manager.ts +++ b/apps/extension/src/debug/manager.ts @@ -51,6 +51,7 @@ const ACTIONS = new Set([ "tool.hover", "tool.evaluate", "tool.wheel", + "tool.drag", "tool.scroll_to", "tool.focus", "tool.blur", diff --git a/apps/extension/src/entrypoints/audit/App.tsx b/apps/extension/src/entrypoints/audit/App.tsx index 2b7bb5a6a..6d2259f2a 100644 --- a/apps/extension/src/entrypoints/audit/App.tsx +++ b/apps/extension/src/entrypoints/audit/App.tsx @@ -31,6 +31,7 @@ const methodKeys = { "tool.focus": "focus", "tool.blur": "blur", "tool.wheel": "scroll", + "tool.drag": "drag", "tool.scroll_to": "scroll", "tool.upload": "upload", "tool.download": "download", diff --git a/apps/extension/src/session-manager/ref-store.ts b/apps/extension/src/session-manager/ref-store.ts index 4d4dc9fe5..64f139aa6 100644 --- a/apps/extension/src/session-manager/ref-store.ts +++ b/apps/extension/src/session-manager/ref-store.ts @@ -35,6 +35,11 @@ export type RefInput = export class RefStore { private map = new Map(); private generation = 0; + private observationToken = crypto.randomUUID(); + + get observationId(): string { + return this.observationToken; + } private readonly documents = new Map(); get revision(): number { @@ -70,6 +75,7 @@ export class RefStore { for (const [ref, input] of entries) next.set(normaliseRef(ref), this.entry(input, generation)); this.map = next; this.generation = generation; + this.observationToken = crypto.randomUUID(); } set( @@ -106,12 +112,16 @@ export class RefStore { changed = true; } } - if (changed) this.generation++; + if (changed) { + this.generation++; + this.observationToken = crypto.randomUUID(); + } } clear(): void { this.generation++; this.map.clear(); + this.observationToken = crypto.randomUUID(); } entries(): IterableIterator<[string, RefEntry]> { diff --git a/apps/extension/src/tools/__tests__/download-lifecycle.test.ts b/apps/extension/src/tools/__tests__/download-lifecycle.test.ts index 41468fcb1..51c6946d9 100644 --- a/apps/extension/src/tools/__tests__/download-lifecycle.test.ts +++ b/apps/extension/src/tools/__tests__/download-lifecycle.test.ts @@ -3,7 +3,7 @@ import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "@/browser-dr import type { InputPassthroughMessage } from "@/lib/input-passthrough-bridge"; import { SessionManager } from "@/session-manager/manager"; import { handleDownload } from "../download"; -import { fileTransferTriggerDeps } from "../file-transfer-trigger"; +import { pointerTransactionDeps } from "../pointer-transaction"; function deferred() { let resolve!: (value: T) => void; @@ -107,7 +107,7 @@ it("does not detach the replacement attachment when an old cleanup release times vi.useFakeTimers(); const f = await fixture(); const abort = new AbortController(); - const scope = fileTransferTriggerDeps(f.deps, abort.signal); + const scope = pointerTransactionDeps(f.deps, abort.signal); const release = deferred(); const releasing = deferred(); const send = vi.mocked(f.api.sendCommand).getMockImplementation()!; @@ -210,7 +210,7 @@ it.each([ attaching.resolve(); return attach.promise; }); - const scope = fileTransferTriggerDeps(f.deps, abort.signal); + const scope = pointerTransactionDeps(f.deps, abort.signal); const pending = scope.deps.cdp.sendToTarget!( { tabId: 7, sessionId }, "Input.dispatchMouseEvent", diff --git a/apps/extension/src/tools/__tests__/drag.browser.test.ts b/apps/extension/src/tools/__tests__/drag.browser.test.ts new file mode 100644 index 000000000..1cc14bac2 --- /dev/null +++ b/apps/extension/src/tools/__tests__/drag.browser.test.ts @@ -0,0 +1,550 @@ +// @vitest-environment node + +import { readFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { build } from "esbuild"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph"; +import { SessionManager } from "@/session-manager/manager"; +import type { DragParams } from "@/transport/types"; +import { handleDrag } from "../drag"; +import { handleObserve } from "../observation"; +import type { CdpRunner } from "../shared"; + +type Send = >( + method: string, + params?: object, + sessionId?: string, +) => Promise; +const { dragMarkup, dragScript } = await import( + new URL( + "../../../../../evals/browser/cases/regression/drag-interactions/drag-interactions.fixture.mjs", + import.meta.url, + ).href +); +let send: Send; +let close: () => void; +let run: Promise; +if (process.env.BSK_DRAG_CHROME) { + beforeAll(async () => { + const { withChrome } = await import( + new URL( + "../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs", + import.meta.url, + ).href + ); + const ready = Promise.withResolvers(), + done = Promise.withResolvers(); + close = done.resolve; + run = withChrome( + { executable: process.env.BSK_DRAG_CHROME, deviceScale: 1, zoom: 1 }, + async (s: Send) => { + ready.resolve(s); + await done.promise; + }, + ); + run.catch(ready.reject); + send = await ready.promise; + }, 30000); + afterAll(async () => { + close?.(); + await run; + }, 30000); +} + +type Hook = (method: string, params?: Record) => void | Promise; +async function withPage( + test: (h: Awaited>) => Promise, + background = false, +) { + const { targetId } = await send<{ targetId: string }>("Target.createTarget", { + url: "about:blank", + background, + }); + try { + await test(await harness(targetId, background)); + } finally { + await send("Target.closeTarget", { targetId }); + } +} +async function harness(targetId: string, background: boolean) { + const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId, + flatten: true, + }); + if (!background) await send("Page.bringToFront", {}, sessionId); + const evaluate = async (expression: string): Promise => { + const reply = await send<{ result: { value: T }; exceptionDetails?: unknown }>( + "Runtime.evaluate", + { expression, returnByValue: true, awaitPromise: true }, + sessionId, + ); + expect(reply.exceptionDetails).toBeUndefined(); + return reply.result.value; + }; + await evaluate(`document.body.innerHTML=${JSON.stringify(dragMarkup)};${dragScript}`); + const manager = new SessionManager({ + agentWindow: { + create: async () => ({ windowId: 100, initialTabIds: [] }), + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("aa11"); + let hook: Hook = () => {}; + const calls: { method: string; params?: Record }[] = []; + const forward = async (target: CdpTarget, method: string, params?: object): Promise => { + calls.push({ method, params: params as Record }); + const result = await send(method, params, target.sessionId ?? sessionId); + await hook(method, params as Record); + return result; + }; + const cdp: CdpRunner = { + send: (_tab, method, params) => forward({ tabId: 4 }, method, params), + sendToTarget: forward, + }; + const frames = async () => { + type Tree = { frame: { id: string; name?: string }; childFrames?: Tree[] }; + const { frameTree } = await send<{ frameTree: Tree }>("Page.getFrameTree", {}, sessionId); + const result: CdpFrame[] = []; + const visit = async (tree: Tree, parentFrameId?: string) => { + const owner = parentFrameId + ? await send<{ backendNodeId: number }>( + "DOM.getFrameOwner", + { frameId: tree.frame.id }, + sessionId, + ) + : undefined; + result.push({ + frameId: tree.frame.id, + parentFrameId, + target: { tabId: 4 }, + ownerBackendNodeId: owner?.backendNodeId, + }); + for (const child of tree.childFrames ?? []) await visit(child, tree.frame.id); + }; + await visit(frameTree); + return { rootFrameId: frameTree.frame.id, frames: result }; + }; + cdp.getFrameGraph = frames; + const deps = { + cdp, + tabsApi: { + get: async (id: number) => ({ id, windowId: 100, active: !background }) as chrome.tabs.Tab, + query: async () => [{ id: 4, windowId: 100, active: !background } as chrome.tabs.Tab], + }, + }; + return { + ctx, + cdp, + calls, + evaluate, + frames, + setHook: (h: Hook) => { + hook = h; + }, + drag: (params: Partial, signal?: AbortSignal) => + handleDrag( + manager, + { + session_id: "aa11", + source: { selector: "#pointer" }, + delta: { x: 120, y: 0 }, + duration_ms: 100, + ...params, + }, + { ...deps, signal }, + ), + observe: () => + handleObserve(manager, { session_id: "aa11" }, { ...deps, conditionalSurfaceProbe: false }), + ref: async (expression: string, ref = "e1", frameId?: string) => { + const { result } = await send<{ result: { objectId: string } }>( + "Runtime.evaluate", + { expression }, + sessionId, + ); + const { node } = await send<{ node: { backendNodeId: number } }>( + "DOM.describeNode", + { objectId: result.objectId }, + sessionId, + ); + await send("Runtime.releaseObject", { objectId: result.objectId }, sessionId); + ctx.refStore.set(ref, node.backendNodeId, { tabId: 4, frameId }); + }, + }; +} + +let libraryBundle: string | undefined; +async function dndKitBundle() { + if (!libraryBundle) { + const result = await build({ + entryPoints: [ + new URL( + "../../../../../evals/browser/cases/regression/drag-interactions/libraries.tsx", + import.meta.url, + ).pathname, + ], + bundle: true, + write: false, + format: "iife", + platform: "browser", + nodePaths: [new URL("../../../node_modules", import.meta.url).pathname], + define: { "process.env.NODE_ENV": '"production"' }, + }); + libraryBundle = result.outputFiles[0].text; + } + return libraryBundle; +} +const ok = (result: unknown) => expect(result, JSON.stringify(result)).not.toHaveProperty("code"); +describe.skipIf(!process.env.BSK_DRAG_CHROME)("real browser drag", () => { + it.each(["mouse", "pointer", "hold"])("moves %s with real input and releases", async (id) => { + await withPage(async (h) => { + ok( + await h.drag({ + source: { selector: `#${id}` }, + delta: { x: 120, y: 35 }, + hold_ms: id === "hold" ? 260 : 0, + }), + ); + expect(await h.evaluate(`window.moved.${id}`)).toEqual({ dx: 120, dy: 35 }); + expect( + await h.evaluate( + `events.filter(e=>e.id===${JSON.stringify(id)}&&e.type.endsWith('down')).every(e=>e.trusted)`, + ), + ).toBe(true); + expect( + await h.evaluate( + `events.filter(e=>e.type===${JSON.stringify(id === "mouse" ? "mouseup" : "pointerup")}).at(-1).buttons`, + ), + ).toBe(0); + if (id === "hold") + expect(await h.evaluate("window.held")).toBeGreaterThanOrEqual(260); + }); + }); + it("preserves middle-button and modifiers", async () => + withPage(async (h) => { + ok(await h.drag({ source: { selector: "#mouse" }, button: "middle", modifiers: ["shift"] })); + expect(await h.evaluate("events.find(e=>e.type==='mousedown')")).toMatchObject({ + button: 1, + buttons: 4, + shift: true, + trusted: true, + }); + expect(await h.evaluate("window.moved.mouse.dx")).toBe(120); + })); + it("performs native HTML drag with DataTransfer and trusted drop", async () => + withPage(async (h) => { + ok( + await h.drag({ + source: { selector: "#native" }, + target: { selector: "#drop" }, + delta: undefined, + duration_ms: 250, + }), + ); + expect(await h.evaluate("window.drops")).toBe(1); + expect(await h.evaluate("native.parentElement.id")).toBe("drop"); + expect( + await h.evaluate( + "events.filter(e=>['dragstart','dragover','drop','dragend'].includes(e.type)).map(e=>[e.type,e.trusted])", + ), + ).toEqual( + expect.arrayContaining([ + ["dragstart", true], + ["dragover", true], + ["drop", true], + ["dragend", true], + ]), + ); + })); + it("changes a native range value", async () => + withPage(async (h) => { + ok(await h.drag({ source: { selector: "#range" }, delta: { x: 70, y: 0 } })); + expect(await h.evaluate("Number(range.value)")).toBeGreaterThan(70); + })); + it("resizes a panel through its handle", async () => + withPage(async (h) => { + ok(await h.drag({ source: { selector: "#handle" }, delta: { x: 50, y: 30 } })); + expect(await h.evaluate("[resize.style.width,resize.style.height]")).toEqual([ + "190px", + "100px", + ]); + })); + it("reorders pointer-based cards", async () => + withPage(async (h) => { + ok( + await h.drag({ + source: { selector: "#first" }, + target: { selector: "#third" }, + delta: undefined, + }), + ); + expect(await h.evaluate("Array.from(sort.children,e=>e.id)")).toEqual([ + "second", + "third", + "first", + ]); + })); + it("draws on canvas and moves SVG content", async () => + withPage(async (h) => { + ok( + await h.drag({ + source: { selector: "#canvas", offset: { x: -100, y: 0 } }, + delta: { x: 100, y: 0 }, + }), + ); + expect( + await h.evaluate( + "Array.from(canvas.getContext('2d').getImageData(0,0,360,120).data).some(v=>v!==0)", + ), + ).toBe(true); + ok(await h.drag({ source: { selector: "#svg" }, delta: { x: 80, y: 20 } })); + expect(await h.evaluate("[svg.getAttribute('x'),svg.getAttribute('y')]")).toEqual([ + "100", + "30", + ]); + })); + it("uses VOM refs for native source and an empty drop area", async () => + withPage(async (h) => { + const observation = await h.observe(); + ok(observation); + if ("code" in observation) throw new Error(observation.message); + const find = (name: string) => { + const line = observation.text.split("\n").find((l) => l.includes('"' + name + '"')); + const ref = line?.match(/@e\d+/)?.[0]; + expect(ref, observation.text).toBeTruthy(); + return ref!; + }; + ok( + await h.drag({ + source: { ref: find("Native card") }, + target: { ref: find("Done") }, + delta: undefined, + observation_id: observation.observation_id, + }), + ); + expect(await h.evaluate("window.drops")).toBe(1); + })); + it("distinguishes duplicate names using VOM refs", async () => + withPage(async (h) => { + await h.evaluate( + `mouse.setAttribute('aria-label','Item');pointer.setAttribute('aria-label','Item')`, + ); + const observation = await h.observe(); + if ("code" in observation) throw new Error(observation.message); + const refs = observation.text + .split("\n") + .filter((l) => l.includes('"Item"')) + .map((l) => l.match(/@e\d+/)?.[0]) + .filter(Boolean); + expect(refs).toHaveLength(2); + ok(await h.drag({ source: { ref: refs[1] }, delta: { x: 60, y: 0 } })); + expect(await h.evaluate("window.moved")).toEqual({ pointer: { dx: 60, dy: 0 } }); + })); + it("rejects an old observation token before input", async () => + withPage(async (h) => { + const old = await h.observe(); + if ("code" in old) throw new Error(old.message); + await h.observe(); + expect(await h.drag({ observation_id: old.observation_id })).toMatchObject({ + code: "not_found", + data: { effect_state: "none" }, + }); + expect( + h.calls.some( + (call) => + call.method === "Input.dispatchMouseEvent" && call.params?.type === "mousePressed", + ), + ).toBe(false); + })); + it.each([ + "ambiguous", + "covered", + "hidden", + "offscreen", + "zero", + "wrong-tab", + "detached", + "disabled", + "inert", + ])("rejects %s endpoints without pressing", async (kind) => + withPage(async (h) => { + let params: Partial = {}; + if (kind === "ambiguous") params = { source: { selector: ".box" } }; + if (kind === "covered") + await h.evaluate( + "document.body.insertAdjacentHTML('beforeend','
')", + ); + if (kind === "disabled") await h.evaluate("pointer.setAttribute('aria-disabled','true')"); + if (kind === "inert") await h.evaluate("pointer.inert=true"); + if (kind === "hidden") await h.evaluate("pointer.style.display='none'"); + if (kind === "offscreen") params = { delta: { x: 10000, y: 0 } }; + if (kind === "zero") params = { delta: { x: 0, y: 0 } }; + if (kind === "wrong-tab") { + h.ctx.refStore.set("e1", 1, { tabId: 99 }); + params = { source: { ref: "e1" } }; + } + if (kind === "detached") { + await h.ref("pointer"); + await h.evaluate("pointer.remove()"); + params = { source: { ref: "e1" } }; + } + expect(await h.drag(params)).toHaveProperty("code"); + expect( + h.calls.some( + (call) => + call.method === "Input.dispatchMouseEvent" && call.params?.type === "mousePressed", + ), + ).toBe(false); + })); + it.each([ + "hold", + "move", + "native", + ])("cancels during %s, releases input and allows the next drag", async (stage) => + withPage(async (h) => { + const controller = new AbortController(); + let moves = 0; + h.setHook((method, p) => { + if (method !== "Input.dispatchMouseEvent") return; + if (stage === "hold" && p?.type === "mousePressed") controller.abort(); + if (stage !== "hold" && p?.type === "mouseMoved" && p.buttons && ++moves === 2) + controller.abort(); + }); + const result = await h.drag( + stage === "native" + ? { source: { selector: "#native" }, target: { selector: "#drop" }, delta: undefined } + : { hold_ms: stage === "hold" ? 300 : 0 }, + controller.signal, + ); + expect(result).toMatchObject({ code: "cancelled", data: { effect_state: "unknown" } }); + expect(await h.evaluate("window.drops")).toBe(0); + h.setHook(() => {}); + ok(await h.drag({ source: { selector: "#mouse" }, delta: { x: 50, y: 0 } })); + expect(await h.evaluate("window.moved.mouse")).toEqual({ dx: 50, dy: 0 }); + })); + it("rejects a source replaced by hover before pressing", async () => + withPage(async (h) => { + h.setHook(async (method, p) => { + if (method === "Input.dispatchMouseEvent" && p?.type === "mouseMoved") { + h.setHook(() => {}); + await h.evaluate("pointer.replaceWith(pointer.cloneNode(true))"); + } + }); + expect(await h.drag({})).toHaveProperty("code"); + expect(h.calls.some((c) => c.params?.type === "mousePressed")).toBe(false); + })); + it("retires a gesture when its document changes and releases its button", async () => + withPage(async (h) => { + h.setHook((method, p) => { + if (method === "Input.dispatchMouseEvent" && p?.type === "mousePressed") + h.ctx.refStore.invalidateTab(4); + }); + expect(await h.drag({})).toMatchObject({ + code: "cdp_failed", + data: { effect_state: "unknown" }, + }); + expect(h.calls.filter((c) => c.params?.type === "mouseReleased")).toHaveLength(1); + expect(await h.evaluate("window.moved.pointer")).toBeUndefined(); + })); + it("times out while holding and does not resume input later", async () => + withPage(async (h) => { + expect(await h.drag({ hold_ms: 400, timeout_ms: 180 })).toMatchObject({ code: "timeout" }); + const count = h.calls.filter((c) => c.method === "Input.dispatchMouseEvent").length; + await new Promise((r) => setTimeout(r, 450)); + expect(h.calls.filter((c) => c.method === "Input.dispatchMouseEvent")).toHaveLength(count); + expect(await h.evaluate("events.filter(e=>e.type==='pointerup').at(-1).buttons")).toBe(0); + })); + it("retires a delayed press reply and never resumes the gesture", async () => + withPage(async (h) => { + const delayed = Promise.withResolvers(); + h.setHook((method, p) => + method === "Input.dispatchMouseEvent" && p?.type === "mousePressed" + ? delayed.promise + : undefined, + ); + expect(await h.drag({ timeout_ms: 180 })).toMatchObject({ + code: "timeout", + data: { effect_state: "unknown" }, + }); + const count = h.calls.filter((c) => c.method === "Input.dispatchMouseEvent").length; + delayed.resolve(); + await new Promise((r) => setTimeout(r, 80)); + expect(h.calls.filter((c) => c.method === "Input.dispatchMouseEvent")).toHaveLength(count); + expect(await h.evaluate("window.moved.pointer")).toBeUndefined(); + })); + it("supports a shadow-root handle via ref", async () => + withPage(async (h) => { + await h.evaluate( + `const host=document.createElement('div');document.body.prepend(host);const shadow=host.attachShadow({mode:'open'});shadow.innerHTML='
Handle
';window.shadowHandle=shadow.firstChild;installDrag(shadowHandle,'pointer',(x,y)=>window.shadowMove=[x,y]);`, + ); + await h.ref("window.shadowHandle"); + ok(await h.drag({ source: { ref: "e1" }, delta: { x: 40, y: 10 } })); + expect(await h.evaluate("window.shadowMove")).toEqual([40, 10]); + })); + it("supports an iframe ref and remeasures after scrolling", async () => + withPage(async (h) => { + await h.evaluate( + `document.body.style.paddingTop='1800px';const f=document.createElement('iframe');document.body.prepend(f);f.contentDocument.body.innerHTML='
Inside
';const el=f.contentDocument.querySelector('#inside');el.onpointerdown=e=>{el.setPointerCapture(e.pointerId);window.frameStart=[e.clientX,e.clientY]};el.onpointermove=e=>{if(e.buttons)window.frameMove=[e.clientX-frameStart[0],e.clientY-frameStart[1]]};window.frameElementForTest=el;`, + ); + const graph = await h.frames(); + const frame = graph.frames.find((f) => f.parentFrameId)!; + await h.ref("window.frameElementForTest", "e1", frame.frameId); + ok(await h.drag({ source: { ref: "e1" }, delta: { x: 60, y: 20 } })); + expect(await h.evaluate("window.frameMove")).toEqual([60, 20]); + })); + it("delivers input in a background tab and restores readiness", async () => + withPage(async (h) => { + ok(await h.drag({ source: { selector: "#mouse" } })); + expect(await h.evaluate("window.moved.mouse.dx")).toBe(120); + }, true)); + it.each([false, true])("reorders real SortableJS cards (fallback=%s)", async (forceFallback) => + withPage(async (h) => { + const require = createRequire(import.meta.url); + const library = await readFile(require.resolve("sortablejs/Sortable.min.js"), "utf8"); + await h.evaluate( + `document.body.innerHTML='
Alpha
Beta
Gamma
';${library};Sortable.create(document.querySelector('#items'),{forceFallback:${forceFallback},animation:0,onEnd:()=>window.sorted=true});void 0;`, + ); + ok( + await h.drag({ + source: { selector: "#alpha" }, + target: { selector: "#gamma", offset: { x: 0, y: 20 } }, + delta: undefined, + duration_ms: 700, + }), + ); + await expect + .poll(() => h.evaluate("Array.from(document.querySelector('#items').children,e=>e.id)")) + .toEqual(["beta", "gamma", "alpha"]); + expect(await h.evaluate("window.sorted")).toBe(true); + })); + it("reorders real dnd-kit cards with a distance activation threshold", async () => + withPage(async (h) => { + await h.evaluate(`document.body.innerHTML='
';${await dndKitBundle()}`); + await expect.poll(() => h.evaluate("!!document.querySelector('#gamma')")).toBe(true); + ok( + await h.drag({ + source: { selector: "#alpha" }, + target: { selector: "#gamma" }, + delta: undefined, + duration_ms: 700, + }), + ); + await expect + .poll(() => h.evaluate("Array.from(document.querySelector('#items').children,e=>e.id)")) + .toEqual(["beta", "gamma", "alpha"]); + })); + it("clears background focus emulation on cancellation", async () => + withPage(async (h) => { + const abort = new AbortController(); + h.setHook((method, p) => { + if (method === "Input.dispatchMouseEvent" && p?.type === "mousePressed") abort.abort(); + }); + expect(await h.drag({ hold_ms: 300 }, abort.signal)).toMatchObject({ code: "cancelled" }); + expect( + h.calls.some( + (c) => c.method === "Emulation.setFocusEmulationEnabled" && c.params?.enabled === false, + ), + ).toBe(true); + }, true)); +}); diff --git a/apps/extension/src/tools/__tests__/drag.test.ts b/apps/extension/src/tools/__tests__/drag.test.ts new file mode 100644 index 000000000..707f7d5a7 --- /dev/null +++ b/apps/extension/src/tools/__tests__/drag.test.ts @@ -0,0 +1,183 @@ +import { expect, it, vi } from "vitest"; +import { guardedRunner } from "@/cooperation/action-guard"; +import { INPUT_PASSTHROUGH } from "@/lib/input-passthrough-bridge"; +import type { DragParams } from "@/transport/types"; +import { withClickOverlay } from "../click-overlay"; +import { validateDrag } from "../drag"; +import { pointerTransactionDeps } from "../pointer-transaction"; +import type { CdpRunner } from "../shared"; + +it.each([ + { source: { selector: "" } }, + { source: { ref: "e1", selector: "#one" } }, + { delta: { x: NaN, y: 1 } }, + { target: { selector: "#two" } }, + { duration_ms: 0 }, + { duration_ms: Infinity }, + { hold_ms: -1 }, + { source: { selector: "#one", offset: { x: 1, y: Infinity } } }, + { modifiers: ["unknown"] }, +])("rejects malformed drag parameters: %j", (invalid) => { + expect( + validateDrag({ + session_id: "s", + source: { ref: "e1" }, + delta: { x: 2, y: 0 }, + ...invalid, + } as DragParams), + ).toMatchObject({ code: "invalid_params" }); +}); +it("cancels native drag and releases the last position after control is revoked", async () => { + let revoked = false; + const sent: { method: string; params?: object }[] = []; + const raw: CdpRunner = { + send: vi.fn(async (_tab, method, params) => { + sent.push({ method, params }); + return {}; + }) as CdpRunner["send"], + }; + const cdp = guardedRunner(raw, { + prepare: async () => { + if (revoked) throw new Error("revoked"); + }, + dispatch: () => { + if (revoked) throw new Error("revoked"); + }, + }); + const scope = pointerTransactionDeps( + { cdp, tabsApi: { get: vi.fn(), query: vi.fn() } }, + undefined, + true, + ); + await scope.deps.cdp.send(4, "Input.dispatchMouseEvent", { + type: "mousePressed", + button: "left", + buttons: 1, + x: 10, + y: 20, + }); + await scope.deps.cdp.send(4, "Input.dispatchMouseEvent", { + type: "mouseMoved", + button: "left", + buttons: 1, + x: 80, + y: 90, + }); + revoked = true; + await scope.cleanup(Date.now() + 1000); + expect(sent.map((s) => s.method)).toEqual([ + "Input.dispatchMouseEvent", + "Input.dispatchMouseEvent", + "Input.cancelDragging", + "Input.dispatchMouseEvent", + ]); + expect(sent.at(-1)?.params).toMatchObject({ type: "mouseReleased", x: 80, y: 90 }); + await expect( + scope.deps.cdp.send(4, "Input.dispatchMouseEvent", { type: "mouseMoved" }), + ).rejects.toThrow(); +}); +it("does not allow an unrelated request to cancel dragging after revocation", async () => { + const raw: CdpRunner = { send: vi.fn() as CdpRunner["send"] }; + const cdp = guardedRunner(raw, { + prepare: async () => { + throw new Error("revoked"); + }, + }); + await expect(cdp.send(4, "Input.cancelDragging", {})).rejects.toThrow("revoked"); + expect(raw.send).not.toHaveBeenCalled(); +}); + +it("does not release a replacement attachment after native cancellation settles", async () => { + let attachment = "original"; + const cdp: CdpRunner = { + getAttachmentId: () => attachment, + send: vi.fn(async (_tab, method) => { + if (method === "Input.cancelDragging") attachment = "replacement"; + return {}; + }) as CdpRunner["send"], + }; + const scope = pointerTransactionDeps( + { cdp, tabsApi: { get: vi.fn(), query: vi.fn() } }, + undefined, + true, + ); + await scope.deps.cdp.send(4, "Input.dispatchMouseEvent", { + type: "mousePressed", + button: "left", + x: 10, + y: 20, + }); + await scope.deps.cdp.send(4, "Input.dispatchMouseEvent", { + type: "mouseMoved", + button: "left", + x: 20, + y: 20, + }); + await scope.cleanup(Date.now() + 1000); + expect(vi.mocked(cdp.send).mock.calls.map(([, method]) => method)).toEqual([ + "Input.dispatchMouseEvent", + "Input.dispatchMouseEvent", + "Input.cancelDragging", + ]); +}); + +it("does not complete a drop when native cancellation cannot be confirmed", async () => { + const cdp: CdpRunner = { + send: vi.fn(async (_tab, method) => { + if (method === "Input.cancelDragging") throw new Error("disconnected"); + return {}; + }) as CdpRunner["send"], + detach: vi.fn(async () => {}), + }; + const scope = pointerTransactionDeps( + { cdp, tabsApi: { get: vi.fn(), query: vi.fn() } }, + undefined, + true, + ); + await scope.deps.cdp.send(4, "Input.dispatchMouseEvent", { + type: "mousePressed", + button: "left", + x: 10, + y: 20, + }); + await scope.deps.cdp.send(4, "Input.dispatchMouseEvent", { + type: "mouseMoved", + button: "left", + x: 20, + y: 20, + }); + await expect(scope.cleanup(Date.now() + 1000)).rejects.toThrow("disconnected"); + expect(cdp.detach).toHaveBeenCalledOnce(); + expect( + vi + .mocked(cdp.send) + .mock.calls.some(([, , p]) => (p as { type?: string })?.type === "mouseReleased"), + ).toBe(false); +}); + +it("refuses a long gesture before input when overlay passthrough cannot cover it", async () => { + let passthrough = false; + const cdp: CdpRunner = { + send: vi.fn(async () => ({ + result: { value: passthrough ? "clear" : "covered" }, + })) as CdpRunner["send"], + }; + const action = vi.fn(async () => ({ done: true })); + const result = await withClickOverlay( + 4, + { x: 10, y: 20 }, + { + cdp, + sendInputPassthrough: async (_tab, message) => { + passthrough = message.phase === "begin"; + return { type: INPUT_PASSTHROUGH, ok: true }; + }, + }, + action, + true, + 6000, + ); + expect(result).toMatchObject({ code: "cdp_failed", data: { effect_state: "none" } }); + expect(action).not.toHaveBeenCalled(); + expect(passthrough).toBe(false); +}); diff --git a/apps/extension/src/tools/__tests__/file-transfer-trigger.test.ts b/apps/extension/src/tools/__tests__/file-transfer-trigger.test.ts index 2aa4f1a64..78559349c 100644 --- a/apps/extension/src/tools/__tests__/file-transfer-trigger.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer-trigger.test.ts @@ -1,5 +1,5 @@ import { afterEach, expect, it, vi } from "vitest"; -import { fileTransferTriggerDeps } from "../file-transfer-trigger"; +import { pointerTransactionDeps } from "../pointer-transaction"; import type { CdpRunner } from "../shared"; afterEach(() => vi.useRealTimers()); @@ -11,7 +11,7 @@ it("does not send cleanup input to a replacement attachment", async () => { send: vi.fn(async () => ({})) as CdpRunner["send"], getAttachmentId: () => attachment, }; - const scope = fileTransferTriggerDeps( + const scope = pointerTransactionDeps( { cdp, tabsApi: { get: vi.fn(), query: vi.fn() } }, abort.signal, ); @@ -35,7 +35,7 @@ it("bounds a lost button-release reply and detaches without replaying input", as send: vi.fn(async () => ({})) as CdpRunner["send"], detach: vi.fn(async () => {}), }; - const scope = fileTransferTriggerDeps( + const scope = pointerTransactionDeps( { cdp, tabsApi: { get: vi.fn(), query: vi.fn() } }, abort.signal, ); @@ -56,7 +56,7 @@ it("fences an already pending release rather than sending another one", async () send: vi.fn(async () => ({})) as CdpRunner["send"], detach: vi.fn(async () => {}), }; - const scope = fileTransferTriggerDeps( + const scope = pointerTransactionDeps( { cdp, tabsApi: { get: vi.fn(), query: vi.fn() } }, abort.signal, ); diff --git a/apps/extension/src/tools/__tests__/transfer-transaction.test.ts b/apps/extension/src/tools/__tests__/transfer-transaction.test.ts new file mode 100644 index 000000000..7466b7404 --- /dev/null +++ b/apps/extension/src/tools/__tests__/transfer-transaction.test.ts @@ -0,0 +1,21 @@ +import { expect, it } from "vitest"; +import { waitBounded } from "../transfer-transaction"; + +it.each([ + "aborted", + "timeout", +])("consumes late failures after an already %s boundary", async (kind) => { + const controller = new AbortController(); + if (kind === "aborted") controller.abort(); + const pending = Promise.withResolvers(); + await expect( + waitBounded( + pending.promise, + kind === "timeout" ? Date.now() - 1 : Date.now() + 1000, + controller.signal, + "expired", + ), + ).rejects.toMatchObject({ kind }); + pending.reject(new Error("late CDP failure")); + await new Promise((resolve) => setTimeout(resolve, 0)); +}); diff --git a/apps/extension/src/tools/background-execution.ts b/apps/extension/src/tools/background-execution.ts index 3987ac40a..9b81e2d22 100644 --- a/apps/extension/src/tools/background-execution.ts +++ b/apps/extension/src/tools/background-execution.ts @@ -29,6 +29,7 @@ const pageTools = new Set([ "tool.click", "tool.hover", "tool.wheel", + "tool.drag", "tool.scroll_to", "tool.focus", "tool.blur", diff --git a/apps/extension/src/tools/click-overlay.ts b/apps/extension/src/tools/click-overlay.ts index 8edac05e7..0040a166e 100644 --- a/apps/extension/src/tools/click-overlay.ts +++ b/apps/extension/src/tools/click-overlay.ts @@ -28,6 +28,7 @@ export async function withClickOverlay( deps: ClickOverlayDeps, click: (beforePress: () => Promise) => Promise, alwaysBypass = false, + inputBudgetMs = 1_000, ): Promise { const send = deps.sendInputPassthrough ?? sendInputPassthrough; let bypass = false; @@ -56,8 +57,8 @@ export async function withClickOverlay( const unregisterCleanup = deps.registerClickCleanup?.(cleanup); let passthroughExpiresAt = Infinity; const expired = () => Date.now() >= passthroughExpiresAt; - // Reserve one second for input delivery; completed clicks still use the full lease. - const pressDeadlineReached = () => Date.now() >= passthroughExpiresAt - 1_000; + // Reserve enough lease for the gesture; slow delivery still reports an unknown outcome. + const pressDeadlineReached = () => Date.now() >= passthroughExpiresAt - inputBudgetMs; const notReady = () => rpcError( "cdp_failed", diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index de3bffdf2..4cfab0fa1 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -12,6 +12,7 @@ import type { ClickParams, ConsoleParams, DownloadParams, + DragParams, EmulateParams, EvaluateParams, ExtractParams, @@ -57,6 +58,7 @@ import { handleConsole } from "./console"; import { handleDebug } from "./debug"; import { handleDialog } from "./dialog-control"; import { handleDownload } from "./download"; +import { handleDrag } from "./drag"; import { type EmulateCdpRunner, handleEmulate } from "./emulate"; import { classifyCdpError } from "./errors"; import { handleEvaluate } from "./evaluate"; @@ -168,7 +170,7 @@ export interface DispatcherDeps { } /** Tools whose page input can make the page open another tab or window. */ -const OPENS_TABS = new Set(["tool.click", "tool.press"]); +const OPENS_TABS = new Set(["tool.click", "tool.press", "tool.drag"]); /** * Routes RPC requests pushed by the daemon over the Transport to the @@ -898,6 +900,18 @@ export class ToolDispatcher { ); return this.rememberHover((req.params as HoverParams).session_id, result); } + case "tool.drag": + return this.withHoverReassert( + req.params as DragParams, + () => + handleDrag( + this.sessions, + req.params as DragParams, + cdp ? { cdp, tabsApi: chromeTabsApi, signal, bypassOverlay, onInputSent } : undefined, + ), + { releaseAfter: true }, + signal, + ); case "tool.wheel": return this.withHoverReassert( req.params as WheelParams, @@ -1238,6 +1252,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.reload": case "tool.click": case "tool.hover": + case "tool.drag": case "tool.wheel": case "tool.scroll_to": case "tool.focus": diff --git a/apps/extension/src/tools/download.ts b/apps/extension/src/tools/download.ts index ec3311c4a..c7cbd9f98 100644 --- a/apps/extension/src/tools/download.ts +++ b/apps/extension/src/tools/download.ts @@ -10,9 +10,9 @@ import { type DownloadsApi, type NavigationTargetsApi, } from "./download-capture"; -import { fileTransferTriggerDeps } from "./file-transfer-trigger"; import { withInputReady } from "./input-readiness"; import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction"; +import { pointerTransactionDeps } from "./pointer-transaction"; import { enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab } from "./shared"; let downloadActive = false; @@ -48,7 +48,7 @@ export async function handleDownload( const timeoutMs = params.timeout_ms ?? 120_000; const deadline = Date.now() + timeoutMs; - let trigger: ReturnType | undefined; + let trigger: ReturnType | undefined; const capture = await captureBrowserDownload({ cdp: deps.cdp, target: address.cdpTarget, @@ -62,7 +62,7 @@ export async function handleDownload( expectedFrameId: address.frameId, trigger: (markDispatched, captureSignal) => { const signal = deps.signal ? AbortSignal.any([captureSignal, deps.signal]) : captureSignal; - trigger = fileTransferTriggerDeps(deps, signal); + trigger = pointerTransactionDeps(deps, signal); const clickDeps = trigger.deps; return withInputReady( ctx, diff --git a/apps/extension/src/tools/drag.ts b/apps/extension/src/tools/drag.ts new file mode 100644 index 000000000..82bd0df0e --- /dev/null +++ b/apps/extension/src/tools/drag.ts @@ -0,0 +1,367 @@ +import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import type { SessionManager } from "@/session-manager/manager"; +import type { DragParams, DragPoint, DragResult, DragTarget, RpcError } from "@/transport/types"; +import { withClickOverlay } from "./click-overlay"; +import { attachDialogs, markDialogCursor } from "./dialogs"; +import { + type ResolvedNodeGeometry, + resolveNodeGeometry, + scrollElementAndFramesIntoView, +} from "./frame-geometry"; +import { withInputReady } from "./input-readiness"; +import { type InteractionDeps, modifiersBitfield, resolveActionTarget } from "./interaction"; +import { pointerTransactionDeps } from "./pointer-transaction"; +import { + type CdpRunner, + chromeTabsApi, + enforceAgentWindow, + isRpcError, + lookupSession, + resolveTargetTab, + sendToCdpTarget, +} from "./shared"; +import { pointBelongsToTarget } from "./target-hit"; +import { BoundedWaitError, waitBounded } from "./transfer-transaction"; + +const validPoint = (p: DragPoint | undefined) => + !!p && Number.isFinite(p.x) && Number.isFinite(p.y); +function validTarget(t: DragTarget | undefined): boolean { + return ( + !!t && + [t.ref, t.selector].filter((v) => typeof v === "string" && v.trim()).length === 1 && + !(t.ref !== undefined && t.selector !== undefined) && + (t.offset === undefined || validPoint(t.offset)) + ); +} +export function validateDrag(p: DragParams): RpcError | null { + if ( + !p || + !validTarget(p.source) || + (p.target !== undefined && !validTarget(p.target)) || + (p.target !== undefined) === (p.delta !== undefined) || + (p.delta !== undefined && (!validPoint(p.delta) || (p.delta.x === 0 && p.delta.y === 0))) + ) + return { + code: "invalid_params", + message: "drag requires a source and exactly one target or finite nonzero delta", + }; + for (const [value, min, max] of [ + [p.hold_ms ?? 0, 0, 5000], + [p.duration_ms ?? 500, 1, 5000], + [p.timeout_ms ?? 30000, 1, 2147483647], + ]) + if (!Number.isSafeInteger(value) || value < min || value > max) + return { + code: "invalid_params", + message: "drag requires hold 0..5000ms, duration 1..5000ms and a positive timeout", + }; + if ( + (p.button !== undefined && !["left", "middle", "right"].includes(p.button)) || + (p.modifiers !== undefined && + (!Array.isArray(p.modifiers) || + p.modifiers.some((m) => !["alt", "ctrl", "shift", "meta"].includes(m)))) || + (p.observation_id !== undefined && (typeof p.observation_id !== "string" || !p.observation_id)) + ) + return { code: "invalid_params", message: "invalid drag button, modifiers or observation_id" }; + return null; +} + +/** One request owns all input, including compensation after timeout or human takeover. */ +export async function handleDrag( + manager: SessionManager, + params: DragParams, + deps: InteractionDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi }, +): Promise { + if (deps.signal?.aborted) + return { code: "cancelled", message: "drag aborted", data: { effect_state: "none" } }; + const invalid = validateDrag(params); + if (invalid) return invalid; + const ctx = lookupSession(manager, params, "drag"); + if (isRpcError(ctx)) return ctx; + const observationId = ctx.refStore.observationId; + if (params.observation_id !== undefined && params.observation_id !== observationId) + return { + code: "not_found", + message: "observation expired; observe again before dragging", + data: { effect_state: "none" }, + }; + const deadline = Date.now() + (params.timeout_ms ?? 30000); + const transaction = pointerTransactionDeps(deps, deps.signal, true); + const inputDeps = transaction.deps; + let attempted = false; + let document: { tabId: number; revision: number } | undefined; + const check = () => { + if (deps.signal?.aborted) throw new DOMException("drag aborted", "AbortError"); + if (Date.now() >= deadline) throw new DOMException("drag timed out", "TimeoutError"); + if (document && ctx.refStore.documentRevision(document.tabId) !== document.revision) + throw new Error("document changed during drag"); + if (ctx.refStore.observationId !== observationId) + throw new Error("observation or document changed during drag"); + }; + // Check identity even between frames of a gesture. The transaction also fences + // late CDP continuations after the outer time boundary has retired the operation. + const cdp = new Proxy(inputDeps.cdp, { + get(target, key) { + const value = Reflect.get(target, key); + return typeof value === "function" + ? (...args: unknown[]) => { + check(); + return value.apply(target, args); + } + : value; + }, + }) as CdpRunner; + let outcome: DragResult | RpcError; + try { + outcome = await waitBounded( + (async (): Promise => { + check(); + const tab = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(tab)) return tab; + const denied = enforceAgentWindow(ctx, tab, "drag"); + if (denied) return denied; + document = { tabId: tab.tabId, revision: ctx.refStore.documentRevision(tab.tabId) }; + cdp.trackSessionTab?.(ctx.sessionId, tab.tabId); + const resolve = (spec: DragTarget) => + resolveActionTarget(cdp, ctx, tab, spec, "drag", true); + const source = await resolve(params.source); + check(); + if (isRpcError(source)) return source; + const target = params.target ? await resolve(params.target) : undefined; + check(); + if (target && isRpcError(target)) return target; + return withInputReady( + ctx, + tab.tabId, + { ...deps, deadline, registerCleanup: inputDeps.registerClickCleanup }, + async (input): Promise => { + const address = (r: typeof source) => ({ + target: r.cdpTarget, + backendNodeId: r.backendNodeId, + frameId: r.frameId, + }); + // Scroll both ends first, then measure both again. Scrolling the second + // endpoint must never leave us using stale coordinates for the first. + for (const node of target ? [source, target] : [source]) { + const error = await scrollElementAndFramesIntoView( + cdp, + tab.tabId, + node.cdpTarget, + node.backendNodeId, + node.frameId, + ); + if (error) return error; + } + const startGeometry = await resolveNodeGeometry(cdp, tab.tabId, address(source)); + if (isRpcError(startGeometry)) return startGeometry; + const endGeometry = target + ? await resolveNodeGeometry(cdp, tab.tabId, address(target)) + : undefined; + if (endGeometry && isRpcError(endGeometry)) return endGeometry; + const offset = (p: DragPoint, o?: DragPoint): DragPoint => ({ + x: p.x + (o?.x ?? 0), + y: p.y + (o?.y ?? 0), + }); + if ( + (source.cdpTarget.sessionId && params.source.offset) || + (target?.cdpTarget.sessionId && params.target?.offset) + ) + return { + code: "unsupported", + message: + "offsets within out-of-process frames are not supported; use an element ref at the desired point", + }; + const start = offset(startGeometry.actionPoint, params.source.offset); + const end = endGeometry + ? offset(endGeometry.actionPoint, params.target?.offset) + : offset(start, params.delta); + const viewport = startGeometry.topViewport; + if ( + [start, end].some( + (p) => + !validPoint(p) || + p.x < 0 || + p.y < 0 || + p.x >= viewport.width || + p.y >= viewport.height, + ) || + (start.x === end.x && start.y === end.y) + ) + return { + code: "invalid_params", + message: "drag needs distinct endpoints inside the visible viewport", + }; + const button = params.button ?? "left"; + const buttons = { left: 1, right: 2, middle: 4 }[button]; + const modifiers = modifiersBitfield(params.modifiers); + const cursor = markDialogCursor(deps.cdp, tab.tabId); + return withClickOverlay( + tab.tabId, + start, + { ...inputDeps, cdp }, + async (verifyOverlay): Promise => { + await cdp.send(tab.tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + ...start, + modifiers, + }); + const blocked = await verifyOverlay(); + if (blocked) return blocked; + const group = `drag-hit-${crypto.randomUUID()}`; + try { + const { object } = await sendToCdpTarget<{ object?: { objectId?: string } }>( + cdp, + source.cdpTarget, + "DOM.resolveNode", + { backendNodeId: source.backendNodeId, objectGroup: group }, + ); + if (!object?.objectId) + return { code: "not_found", message: "drag source is detached" }; + const state = await sendToCdpTarget<{ result?: { value?: boolean } }>( + cdp, + source.cdpTarget, + "Runtime.callFunctionOn", + { + objectId: object.objectId, + returnByValue: true, + functionDeclaration: `function() { return this.isConnected && !this.matches(':disabled,[aria-disabled="true"]') && !this.closest('[inert]'); }`, + }, + ); + if (state.result?.value !== true) + return { + code: "permission_denied", + message: "drag source is detached, disabled or inert", + }; + const endpoints: [typeof source, ResolvedNodeGeometry, DragPoint][] = [ + [source, startGeometry, start], + ]; + if (target && endGeometry) endpoints.push([target, endGeometry, end]); + for (const [node, geometry, point] of endpoints) { + const local = { + x: geometry.targetActionPoint.x + point.x - geometry.actionPoint.x, + y: geometry.targetActionPoint.y + point.y - geometry.actionPoint.y, + }; + if (!(await pointBelongsToTarget(cdp, node, local.x, local.y, group, false))) + return { + code: "permission_denied", + message: + "drag endpoint is covered or no longer belongs to the requested element", + }; + } + } finally { + const cleanup = deps.cdp.resourceCleanup ?? deps.cdp; + for (const node of target ? [source, target] : [source]) + await sendToCdpTarget(cleanup, node.cdpTarget, "Runtime.releaseObjectGroup", { + objectGroup: group, + }).catch(() => {}); + } + await cdp.verifyPointerTarget?.({ ...source, tabId: tab.tabId }, start); + input.markSent(); + attempted = true; + deps.onInputSent?.(tab.tabId); + await cdp.send(tab.tabId, "Input.dispatchMouseEvent", { + type: "mousePressed", + ...start, + button, + buttons, + modifiers, + clickCount: 1, + }); + const pause = async (ms: number) => { + check(); + if (ms <= 0) return; + let timer: ReturnType | undefined; + try { + await waitBounded( + new Promise((resolve) => { + timer = setTimeout(resolve, ms); + }), + deadline, + deps.signal, + "drag timed out", + ); + } catch (error) { + if (error instanceof BoundedWaitError) + throw new DOMException( + "drag " + error.kind, + error.kind === "timeout" ? "TimeoutError" : "AbortError", + ); + throw error; + } finally { + clearTimeout(timer); + } + check(); + }; + await pause(params.hold_ms ?? 0); + const duration = params.duration_ms ?? 500; + const steps = Math.max(2, Math.ceil(duration / 16)); + const started = Date.now(); + for (let step = 1; step <= steps; step++) { + await pause(started + (duration * step) / steps - Date.now()); + const ratio = step / steps; + await cdp.send(tab.tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: start.x + (end.x - start.x) * ratio, + y: start.y + (end.y - start.y) * ratio, + button, + buttons, + modifiers, + }); + } + // A second move at the destination also serves HTML dragover consumers. + await cdp.send(tab.tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + ...end, + button, + buttons, + modifiers, + }); + await cdp.send(tab.tabId, "Input.dispatchMouseEvent", { + type: "mouseReleased", + ...end, + button, + buttons: 0, + modifiers, + clickCount: 1, + }); + return attachDialogs(deps.cdp, tab.tabId, cursor, { + tab_id: tab.tabId, + source: start, + target: end, + }); + }, + true, + (params.hold_ms ?? 0) + (params.duration_ms ?? 500) + 1_000, + ); + }, + ); + })(), + deadline, + deps.signal, + "drag timed out", + ); + } catch (error) { + const aborted = + deps.signal?.aborted || + (error instanceof BoundedWaitError && error.kind === "aborted") || + (error instanceof Error && error.name === "AbortError"); + const timeout = + (error instanceof BoundedWaitError && error.kind === "timeout") || + (error instanceof Error && error.name === "TimeoutError"); + outcome = { + code: aborted ? "cancelled" : timeout ? "timeout" : "cdp_failed", + message: aborted ? "drag aborted" : error instanceof Error ? error.message : String(error), + }; + } + try { + await transaction.cleanup(Date.now() + 2000); + } catch { + outcome = { + code: "cdp_failed", + message: "drag cleanup could not be confirmed; observe before further input", + }; + } + return isRpcError(outcome) + ? { ...outcome, data: { ...outcome.data, effect_state: attempted ? "unknown" : "none" } } + : outcome; +} diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 5d6a16ade..703eeed3c 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -149,6 +149,7 @@ export async function resolveBackendNode( target: { tabId: number }, params: { ref?: string; selector?: string }, toolName: string, + strictSelector = false, ): Promise< | { backendNodeId: number; @@ -200,11 +201,25 @@ export async function resolveBackendNode( message: "DOM.getDocument returned no root nodeId", }; } - const found = await cdp.send<{ nodeId?: number }>(target.tabId, "DOM.querySelector", { - nodeId: rootNodeId, - selector: params.selector, - }); - if (typeof found.nodeId !== "number" || found.nodeId === 0) { + const query = { nodeId: rootNodeId, selector: params.selector }; + let nodeId: number | undefined; + if (strictSelector) { + const found = await cdp.send<{ nodeIds: number[] }>( + target.tabId, + "DOM.querySelectorAll", + query, + ); + if (found.nodeIds.length > 1) + return { + code: "invalid_params", + message: `${toolName}: selector matched ${found.nodeIds.length} elements; use a VOM ref or a narrower selector`, + }; + nodeId = found.nodeIds[0]; + } else { + nodeId = (await cdp.send<{ nodeId?: number }>(target.tabId, "DOM.querySelector", query)) + .nodeId; + } + if (typeof nodeId !== "number" || nodeId === 0) { return rpcError( "not_found", "selector_not_found", @@ -214,7 +229,7 @@ export async function resolveBackendNode( const described = await cdp.send<{ node?: { backendNodeId?: number } }>( target.tabId, "DOM.describeNode", - { nodeId: found.nodeId }, + { nodeId }, ); const backendNodeId = described.node?.backendNodeId; if (typeof backendNodeId !== "number") { @@ -439,8 +454,9 @@ export async function resolveActionTarget( target: ResolvedTargetTab, params: { ref?: string; selector?: string }, toolName: string, + strictSelector = false, ): Promise { - const node = await resolveBackendNode(cdp, ctx, target, params, toolName); + const node = await resolveBackendNode(cdp, ctx, target, params, toolName, strictSelector); return isRpcError(node) ? node : { tab: target, ...node }; } diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 9bd445134..8f679c939 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -1302,6 +1302,7 @@ async function handleVomObservation( ); readSucceeded = true; return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + observation_id: ctx.refStore.observationId, text: observation.text, ref_count: observation.refs.length, tab_id: target.tabId, diff --git a/apps/extension/src/tools/file-transfer-trigger.ts b/apps/extension/src/tools/pointer-transaction.ts similarity index 68% rename from apps/extension/src/tools/file-transfer-trigger.ts rename to apps/extension/src/tools/pointer-transaction.ts index 4b71d0629..49a99e23d 100644 --- a/apps/extension/src/tools/file-transfer-trigger.ts +++ b/apps/extension/src/tools/pointer-transaction.ts @@ -3,28 +3,45 @@ import type { InteractionDeps } from "./interaction"; import { type CdpRunner, sendToCdpTarget } from "./shared"; import { waitBounded } from "./transfer-transaction"; -/** A retired file-transfer click must never resume input when a CDP reply arrives late. */ -export function fileTransferTriggerDeps(deps: InteractionDeps, signal?: AbortSignal) { +/** A retired pointer transaction must never resume input when a CDP reply arrives late. */ +export function pointerTransactionDeps( + deps: InteractionDeps, + signal?: AbortSignal, + cancelDragging = false, +) { let retired = false; let release: { tabId: number; params: object; attachmentId?: string } | undefined; let releasing = false; + let movedWhilePressed = false; const clickCleanups = new Set<() => Promise>(); const check = () => { if (retired || signal?.aborted) - throw new DOMException("file-transfer trigger aborted", "AbortError"); + throw new DOMException("pointer transaction aborted", "AbortError"); }; const send = async (target: CdpTarget, method: string, params?: object): Promise => { check(); const mouse = params as { type?: string } | undefined; const onDispatch = () => { check(); + if ( + release?.attachmentId !== undefined && + deps.cdp.getAttachmentId?.(target.tabId) !== release.attachmentId + ) + throw new Error("pointer attachment changed"); if (method === "Input.dispatchMouseEvent") { if (mouse?.type === "mousePressed") { release = { tabId: target.tabId, - params: { ...params, type: "mouseReleased" }, + params: { ...params, type: "mouseReleased", buttons: 0 }, attachmentId: deps.cdp.getAttachmentId?.(target.tabId), }; + } else if (mouse?.type === "mouseMoved" && release) { + release.params = { + ...release.params, + x: (params as { x: number }).x, + y: (params as { y: number }).y, + }; + movedWhilePressed = true; } else if (mouse?.type === "mouseReleased") { releasing = true; } @@ -33,10 +50,7 @@ export function fileTransferTriggerDeps(deps: InteractionDeps, signal?: AbortSig const result = deps.cdp.sendGuarded ? await deps.cdp.sendGuarded(target, method, params, { signal, - attachmentId: - method === "Input.dispatchMouseEvent" && mouse?.type === "mouseReleased" - ? release?.attachmentId - : undefined, + attachmentId: release?.attachmentId, onDispatch, }) : await (() => { @@ -79,6 +93,30 @@ export function fileTransferTriggerDeps(deps: InteractionDeps, signal?: AbortSig }; if (!ownsAttachment()) return; try { + // Native HTML drag state also belongs to this gesture. Cancel it before releasing; + // a plain release could otherwise commit the pending drop after cancellation. + if ( + cancelDragging && + movedWhilePressed && + (pending.params as { button?: string }).button === "left" && + !releasing + ) { + const cancel = deps.cdp.sendGuarded + ? deps.cdp.sendGuarded( + { tabId: pending.tabId }, + "Input.cancelDragging", + {}, + { attachmentId: pending.attachmentId }, + ) + : deps.cdp.send(pending.tabId, "Input.cancelDragging", {}); + await waitBounded( + cancel, + Math.min(deadline, Date.now() + 300), + undefined, + "drag cancellation timed out", + ); + } + if (!ownsAttachment()) return; // Never replay an in-flight release or send cleanup on a new attachment. const cleanup = releasing ? detach() @@ -92,8 +130,8 @@ export function fileTransferTriggerDeps(deps: InteractionDeps, signal?: AbortSig }, ) : deps.cdp.send(pending.tabId, "Input.dispatchMouseEvent", pending.params); - if (!cleanup) throw new Error("file-transfer input cleanup unavailable"); - await waitBounded(cleanup, deadline, undefined, "file-transfer input cleanup timed out"); + if (!cleanup) throw new Error("pointer input cleanup unavailable"); + await waitBounded(cleanup, deadline, undefined, "pointer input cleanup timed out"); } catch (error) { // Replacement may happen while release is pending. Recheck ownership, // and have the driver enforce it again when executing the fallback. @@ -127,7 +165,7 @@ export function fileTransferTriggerDeps(deps: InteractionDeps, signal?: AbortSig Promise.allSettled(cleanups), deadline, undefined, - "file-transfer trigger cleanup timed out", + "pointer transaction cleanup timed out", ); for (const result of results) if (result.status === "rejected") throw result.reason; }, diff --git a/apps/extension/src/tools/transfer-transaction.ts b/apps/extension/src/tools/transfer-transaction.ts index 6debaebd0..6874e8d55 100644 --- a/apps/extension/src/tools/transfer-transaction.ts +++ b/apps/extension/src/tools/transfer-transaction.ts @@ -1,5 +1,5 @@ -// Shared time boundary for browser-side file-transfer transactions. Concrete -// upload/download mechanisms own their state machines and effect semantics. +// Shared time boundary for browser-side transactions. Concrete operations +// own their state machines and effect semantics. export class BoundedWaitError extends Error { constructor( @@ -21,15 +21,21 @@ export async function waitBounded( timeoutMessage: string, ): Promise { const remaining = remainingMs(deadline); - if (signal?.aborted) throw new BoundedWaitError("aborted", "file transfer aborted"); - if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage); + if (signal?.aborted || remaining === 0) { + // Work may already be in flight when its caller retires it. Consume a late + // rejection even though this boundary no longer waits for the result. + void promise.catch(() => {}); + throw signal?.aborted + ? new BoundedWaitError("aborted", "operation aborted") + : new BoundedWaitError("timeout", timeoutMessage); + } let timer: ReturnType | undefined; let onAbort: (() => void) | undefined; const boundary = new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining); if (signal) { - onAbort = () => reject(new BoundedWaitError("aborted", "file transfer aborted")); + onAbort = () => reject(new BoundedWaitError("aborted", "operation aborted")); signal.addEventListener("abort", onAbort, { once: true }); } }); diff --git a/apps/extension/src/tools/upload.ts b/apps/extension/src/tools/upload.ts index 6bca00f22..dec191a81 100644 --- a/apps/extension/src/tools/upload.ts +++ b/apps/extension/src/tools/upload.ts @@ -10,9 +10,9 @@ import type { SessionManager } from "@/session-manager/manager"; import type { RpcError, UploadParams, UploadResult } from "@/transport/types"; import { uploadThroughFileDrop } from "./file-drop-transaction"; import { uploadThroughActivatedFileInput } from "./file-input-transaction"; -import { fileTransferTriggerDeps } from "./file-transfer-trigger"; import { withInputReady } from "./input-readiness"; import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction"; +import { pointerTransactionDeps } from "./pointer-transaction"; import { type CdpRunner, enforceAgentWindow, @@ -67,7 +67,7 @@ export async function handleUpload( }); if (isRpcError(transaction)) return transaction; } else { - const trigger = fileTransferTriggerDeps(deps, deps.signal); + const trigger = pointerTransactionDeps(deps, deps.signal); const transaction = await uploadThroughActivatedFileInput({ cdp: deps.cdp, actionTarget: address, diff --git a/apps/extension/src/tools/vom/semantic-graph/structure.ts b/apps/extension/src/tools/vom/semantic-graph/structure.ts index 8d6be24be..7d148033b 100644 --- a/apps/extension/src/tools/vom/semantic-graph/structure.ts +++ b/apps/extension/src/tools/vom/semantic-graph/structure.ts @@ -194,6 +194,7 @@ function baseDecision( const role = output.role?.toLowerCase() ?? ""; if (!TRANSPARENT_ROLES.has(role)) return { kind: "keep", reason: "semantic" }; if ( + isVomReferenceNode(output) || output.name || output.text || output.value || diff --git a/apps/extension/src/tools/vom/visual-observation.ts b/apps/extension/src/tools/vom/visual-observation.ts index 96fe461c1..ac4233d0d 100644 --- a/apps/extension/src/tools/vom/visual-observation.ts +++ b/apps/extension/src/tools/vom/visual-observation.ts @@ -332,6 +332,7 @@ export async function publishObservationPage( state.token = nextToken; state.revision = store.revision; const result: ObserveResult = { + observation_id: store.observationId, text: rendered!.text, ref_count: refs.length, tab_id: tabId, diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index ffb0ed81e..04cd85ce0 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -452,6 +452,7 @@ export interface SnapshotParams { } export interface SnapshotResult { + observation_id?: string; text: string; ref_count: number; tab_id: number; @@ -597,6 +598,35 @@ export interface HoverResult { dialogs?: JavaScriptDialogInfo[]; } +export interface DragPoint { + x: number; + y: number; +} +export interface DragTarget { + ref?: string; + selector?: string; + offset?: DragPoint; +} +export interface DragParams { + session_id: string; + tab_id?: number; + source: DragTarget; + target?: DragTarget; + delta?: DragPoint; + button?: MouseButton; + modifiers?: KeyModifier[]; + hold_ms?: number; + duration_ms?: number; + timeout_ms?: number; + observation_id?: string; +} +export interface DragResult { + tab_id: number; + source: DragPoint; + target: DragPoint; + dialogs?: JavaScriptDialogInfo[]; +} + export interface WheelParams { session_id: string; ref?: string; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index e5ceb22ba..e9f1f7ead 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -92,6 +92,7 @@ Use a ref and context ID from the latest sync (one action per context): | Fill a field | `bsk fill @e3 --value "text" --session --context ID` | | Select an option | `bsk select @e3 --value "option-value" --session --context ID` | | Press a key | `bsk press Enter --ref @e3 --session --context ID` | +| Drag or hold and drag | `bsk drag @e3 --to @e8 --hold 300ms --session --context ID` | | Reveal a hover menu | `bsk hover @e3 --session --context ID` | | Reveal an element | `bsk scroll-to @e3 --session --context ID` | | Scroll with wheel input | `bsk wheel --delta-y 600 --session --context ID` | @@ -119,7 +120,7 @@ Load further references only as needed. | Website debugging, reproduction evidence, or request rules/replay | [Debugging](references/debugging.md) | | Required profile, existing user tab, multiple/background tabs, or remote tab ownership | [Tabs and profiles](references/tabs-and-profiles.md) | | Missing CLI, daemon startup failure, sandboxed startup, connection failure, or remote pairing | [Environment](references/environment.md) | -| Waits, hover menus/probing, scrolling, `next_cursor`/`@more`, console/network, emulation, evaluation, or recording | [Interaction details](references/interaction-details.md) | +| Dragging, waits, hover menus/probing, scrolling, `next_cursor`/`@more`, console/network, emulation, evaluation, or recording | [Interaction details](references/interaction-details.md) | | Screenshot, full-page capture, or `[visual:screenshot]`/Canvas interaction | [Screenshots and Canvas](references/screenshots-and-canvas.md) | | Video recording or MP4 export | [Video](references/video.md) | | Upload or download | [Files](references/files.md) | diff --git a/crates/bsk-cli/skill/references/interaction-details.md b/crates/bsk-cli/skill/references/interaction-details.md index 9f9732153..42ef361a9 100644 --- a/crates/bsk-cli/skill/references/interaction-details.md +++ b/crates/bsk-cli/skill/references/interaction-details.md @@ -12,6 +12,36 @@ scroll distance. An optional target is scrolled into view first; without one, input lands at the viewport centre. Observe to check the page's response. +### Dragging + +Use `bsk drag --to --session `, +or replace `--to` with `--dx --dy ` for sliders, resize handles, +and panning. Prefer refs from the latest VOM observation. Selectors must match +exactly one element in the main document; frame/shadow targets require refs. + +`--hold 500ms` waits after pressing, before moving; `--duration 800ms` controls +travel pacing. Both are bounded at 5s each. `--button` and `--modifiers` work as +for click. `--from-offset-x/y` and `--to-offset-x/y` adjust the endpoints from +their visible action points in top-level viewport CSS pixels. Offsets inside +out-of-process frames are unsupported. Use the actual thumb/handle when present. + +Observation JSON includes `observation_id`; supplying `--observation-id ` +rejects a superseded ref map before input. New observe/snapshot/continuation +replaces refs. Keep the matching token and refs together; the token does not +replace `--context` when shared-control coordination requires it. + +Both endpoints must be visible after preparation. Drag does not auto-scroll +while held, cross tabs, or synthesize touch gestures. Canvas screenshot-bound +visual refs are not DOM targets; canvas selectors can receive mouse drags. +VOM `[draggable]`, `[drag-handle]` and `[drop-target]` are evidence-based hints, +not promises that an application will accept a gesture. Some custom targets +have no detectable hint and need a known selector. + +Success means the input sequence was delivered, not that a drop was accepted. +Observe afterwards and verify the value, order or destination. Cancellation +releases owned input at its last position; it cannot roll back page handlers. +After `effect_state=unknown`, observe before deciding whether to retry. + ### Filling text `fill` supports enabled, editable text inputs, textareas, and contenteditable diff --git a/crates/bsk-cli/src/cli/drag.rs b/crates/bsk-cli/src/cli/drag.rs new file mode 100644 index 000000000..3cf333147 --- /dev/null +++ b/crates/bsk-cli/src/cli/drag.rs @@ -0,0 +1,205 @@ +//! `bsk drag`: an entire press, optional hold, move and release in one request. +use super::{ + dialogs::print_dialog_summaries, + ensure_daemon::ensure_daemon, + error::{CliError, Format}, + interaction::{CliMouseButton, parse_modifiers, split_target}, + navigate::parse_timeout_ms, +}; +use anyhow::Context; +use bsk_protocol::{ + Method, + tools::{DragParams, DragPoint, DragResult, DragTarget}, +}; +use clap::Args; +use std::time::Duration; + +#[derive(Debug, Clone, Args)] +pub struct DragArgs { + /// Source ref or CSS selector (selectors must match exactly one element). + pub source: String, + /// Destination ref or CSS selector; mutually exclusive with --dx/--dy. + #[arg(long, conflicts_with_all = ["dx", "dy"], required_unless_present_any = ["dx", "dy"])] + pub to: Option, + #[arg(long, allow_negative_numbers = true)] + pub dx: Option, + #[arg(long, allow_negative_numbers = true)] + pub dy: Option, + /// Source offset from its visible action point, in viewport CSS pixels. + #[arg(long, default_value_t = 0.0, allow_negative_numbers = true)] + pub from_offset_x: f64, + #[arg(long, default_value_t = 0.0, allow_negative_numbers = true)] + pub from_offset_y: f64, + #[arg(long, requires = "to", allow_negative_numbers = true)] + pub to_offset_x: Option, + #[arg(long, requires = "to", allow_negative_numbers = true)] + pub to_offset_y: Option, + #[arg(long)] + pub session: String, + #[arg(long)] + pub tab_id: Option, + #[arg(long)] + pub observation_id: Option, + #[arg(long, value_enum, default_value_t = CliMouseButton::Left)] + pub button: CliMouseButton, + #[arg(long, default_value = "")] + pub modifiers: String, + #[arg(long, default_value = "0ms", value_parser = parse_hold_ms)] + pub hold: u32, + #[arg(long, default_value = "500ms", value_parser = parse_timeout_ms)] + pub duration: u32, + #[arg(long, default_value = "30s", value_parser = parse_timeout_ms)] + pub timeout: u32, +} + +fn target(value: String, x: f64, y: f64) -> Result { + if value.trim().is_empty() || !x.is_finite() || !y.is_finite() { + return Err(CliError::Local(anyhow::anyhow!( + "drag target must be nonempty and offsets finite" + ))); + } + let (ref_, selector) = split_target(Some(value), None, None)?; + Ok(DragTarget { + ref_, + selector, + offset: if x == 0.0 && y == 0.0 { + None + } else { + Some(DragPoint { x, y }) + }, + }) +} + +pub fn dispatch(args: DragArgs, format: Format) -> Result<(), CliError> { + let source = target(args.source, args.from_offset_x, args.from_offset_y)?; + let destination = args + .to + .map(|s| { + target( + s, + args.to_offset_x.unwrap_or(0.0), + args.to_offset_y.unwrap_or(0.0), + ) + }) + .transpose()?; + let delta = if destination.is_none() { + Some(DragPoint { + x: args.dx.unwrap_or(0.0), + y: args.dy.unwrap_or(0.0), + }) + } else { + None + }; + if delta + .as_ref() + .is_some_and(|d| !d.x.is_finite() || !d.y.is_finite() || (d.x == 0.0 && d.y == 0.0)) + || args.hold > 5000 + || !(1..=5000).contains(&args.duration) + || args.timeout == 0 + { + return Err(CliError::Local(anyhow::anyhow!( + "drag requires a finite nonzero delta, hold 0..5000ms, duration 1..5000ms and positive timeout" + ))); + } + let modifiers = + parse_modifiers(&args.modifiers).map_err(|e| CliError::Local(anyhow::anyhow!(e)))?; + let params = DragParams { + session_id: args.session, + tab_id: args.tab_id, + source, + target: destination, + delta, + button: Some(args.button.into()), + modifiers: Some(modifiers), + hold_ms: Some(args.hold), + duration_ms: Some(args.duration), + timeout_ms: Some(args.timeout), + observation_id: args.observation_id, + }; + let info = ensure_daemon().context("ensure daemon is running")?; + let reply: DragResult = super::business_rpc::call( + info.sock_path, + "drag-1", + Method::ToolDrag, + Some(params), + Duration::from_millis(u64::from(args.timeout) + 15000), + )?; + match format { + Format::Json => println!( + "{}", + serde_json::to_string_pretty(&reply).map_err(|e| CliError::Local(e.into()))? + ), + Format::Human => { + println!( + "drag input delivered tab={} from=({}, {}) to=({}, {}); observe to verify", + reply.tab_id, reply.source.x, reply.source.y, reply.target.x, reply.target.y + ); + print_dialog_summaries(&reply.dialogs); + } + } + Ok(()) +} + +fn parse_hold_ms(value: &str) -> Result { + if matches!(value, "0" | "0ms" | "0s") { + Ok(0) + } else { + parse_timeout_ms(value) + } +} +#[cfg(test)] +mod tests { + use super::*; + use crate::cli::{Cli, Command}; + use clap::Parser; + #[test] + fn parses_drag_destinations_hold_and_negative_offsets() { + for tail in [ + vec!["--to", "@e8"], + vec!["--dx", "-240", "--dy", "20", "--hold", "500ms"], + ] { + let mut argv = vec!["bsk", "drag", "@e3", "--session", "test"]; + argv.extend(tail); + let Command::Drag(args) = Cli::try_parse_from(argv).unwrap().command else { + panic!("drag") + }; + assert_eq!(args.source, "@e3"); + assert_eq!(args.duration, 500); + } + assert!( + Cli::try_parse_from([ + "bsk", + "drag", + "e3", + "--to", + "e8", + "--dx", + "1", + "--session", + "test" + ]) + .is_err() + ); + assert!(Cli::try_parse_from(["bsk", "drag", "e3", "--session", "test"]).is_err()); + } + #[test] + fn invalid_drag_is_rejected_before_starting_daemon() { + for tail in [ + vec!["--dx", "0"], + vec!["--dx", "NaN"], + vec!["--to", ""], + vec!["--to", "e8", "--hold", "6s"], + vec!["--to", "e8", "--modifiers", "unknown"], + ] { + let mut argv = vec!["bsk", "drag", "e3", "--session", "test"]; + argv.extend(tail); + let Command::Drag(args) = Cli::try_parse_from(argv).unwrap().command else { + panic!("drag") + }; + assert!(matches!( + dispatch(args, Format::Json), + Err(CliError::Local(_)) + )); + } + } +} diff --git a/crates/bsk-cli/src/cli/mod.rs b/crates/bsk-cli/src/cli/mod.rs index 35121d748..6a3fe6df8 100644 --- a/crates/bsk-cli/src/cli/mod.rs +++ b/crates/bsk-cli/src/cli/mod.rs @@ -14,6 +14,7 @@ pub mod dialog_control; pub mod dialogs; pub mod doctor; pub mod download; +pub mod drag; pub mod emulate; pub mod ensure_daemon; pub mod error; @@ -206,6 +207,9 @@ pub enum Command { /// Dispatch a native mouse-wheel event at the viewport centre or an element. Wheel(WheelArgs), + /// Drag an element to another element or by a relative offset. + Drag(drag::DragArgs), + /// Scroll a snapshot ref or CSS selector into the visible viewport. #[command(name = "scroll-to")] ScrollTo(ScrollToArgs), diff --git a/crates/bsk-cli/src/daemon/ipc.rs b/crates/bsk-cli/src/daemon/ipc.rs index f5c8b0512..66d6bf21f 100644 --- a/crates/bsk-cli/src/daemon/ipc.rs +++ b/crates/bsk-cli/src/daemon/ipc.rs @@ -295,6 +295,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc) -> RpcHandler | Method::ToolReload | Method::ToolClick | Method::ToolHover + | Method::ToolDrag | Method::ToolWheel | Method::ToolScrollTo | Method::ToolFocus diff --git a/crates/bsk-cli/src/daemon/queue.rs b/crates/bsk-cli/src/daemon/queue.rs index 99476e467..4640c5cfd 100644 --- a/crates/bsk-cli/src/daemon/queue.rs +++ b/crates/bsk-cli/src/daemon/queue.rs @@ -1098,7 +1098,7 @@ fn not_dispatched_error( fn is_native_input(method: &Method) -> bool { matches!( method, - Method::ToolClick | Method::ToolPress | Method::ToolWheel + Method::ToolClick | Method::ToolPress | Method::ToolWheel | Method::ToolDrag ) } @@ -1744,8 +1744,10 @@ mod link_error_tests { WaitOutcome::TimedOutAfterResponse, ]; for stage in stages { - let click = settled(Method::ToolClick, stage); - assert_eq!(click.data.unwrap()["reason"], "input_outcome_unknown"); + for method in [Method::ToolClick, Method::ToolDrag] { + let input = settled(method, stage); + assert_eq!(input.data.unwrap()["reason"], "input_outcome_unknown"); + } let upload = settled(Method::ToolUpload, stage); let upload = upload.data.unwrap(); diff --git a/crates/bsk-cli/src/main.rs b/crates/bsk-cli/src/main.rs index f017784e2..d929fc361 100644 --- a/crates/bsk-cli/src/main.rs +++ b/crates/bsk-cli/src/main.rs @@ -93,6 +93,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> { Command::Reload(args) => cli::navigate::dispatch_reload(args, format), Command::Click(args) => cli::interaction::dispatch_click(args, format), Command::Hover(args) => cli::interaction::dispatch_hover(args, format), + Command::Drag(args) => cli::drag::dispatch(args, format), Command::Wheel(args) => cli::wheel::dispatch(args, format), Command::ScrollTo(args) => cli::scroll::dispatch(args, format), Command::Focus(args) => cli::interaction::dispatch_focus(args, format), diff --git a/crates/bsk-cli/tests/tools_ipc.rs b/crates/bsk-cli/tests/tools_ipc.rs index 117308c19..787e2a09c 100644 --- a/crates/bsk-cli/tests/tools_ipc.rs +++ b/crates/bsk-cli/tests/tools_ipc.rs @@ -500,6 +500,7 @@ async fn snapshot_returns_text_and_ref_count() { let _: SnapshotParams = serde_json::from_value(req.params.clone().unwrap()).unwrap(); ResponseBody::Ok( serde_json::to_value(SnapshotResult { + observation_id: None, text: "@e1 link \"home\"\n@e2 button \"submit\"\n".into(), ref_count: 2, tab_id: 13, @@ -540,6 +541,7 @@ async fn observe_returns_semantic_text_and_ref_count() { let _: ObserveParams = serde_json::from_value(req.params.clone().unwrap()).unwrap(); ResponseBody::Ok( serde_json::to_value(ObserveResult { + observation_id: None, next_cursor: None, text: "@vom 1\n @e1 button \"Products\" [hover: Shoes]\n".into(), ref_count: 1, diff --git a/crates/bsk-protocol/schema/tool_drag_params.json b/crates/bsk-protocol/schema/tool_drag_params.json new file mode 100644 index 000000000..b889433bc --- /dev/null +++ b/crates/bsk-protocol/schema/tool_drag_params.json @@ -0,0 +1,164 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DragParams", + "type": "object", + "required": [ + "session_id", + "source" + ], + "properties": { + "button": { + "anyOf": [ + { + "$ref": "#/definitions/MouseButton" + }, + { + "type": "null" + } + ] + }, + "delta": { + "anyOf": [ + { + "$ref": "#/definitions/DragPoint" + }, + { + "type": "null" + } + ] + }, + "duration_ms": { + "description": "Travel duration, excluding hold. Defaults to 500 ms.", + "type": [ + "integer", + "null" + ], + "format": "uint32", + "maximum": 5000.0, + "minimum": 1.0 + }, + "hold_ms": { + "description": "Delay after pressing and before moving. Defaults to zero.", + "type": [ + "integer", + "null" + ], + "format": "uint32", + "maximum": 5000.0, + "minimum": 0.0 + }, + "modifiers": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/KeyModifier" + } + }, + "observation_id": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "source": { + "$ref": "#/definitions/DragTarget" + }, + "tab_id": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "target": { + "description": "Exactly one of target or delta is required.", + "anyOf": [ + { + "$ref": "#/definitions/DragTarget" + }, + { + "type": "null" + } + ] + }, + "timeout_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 1.0 + } + }, + "definitions": { + "DragPoint": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number", + "format": "double" + }, + "y": { + "type": "number", + "format": "double" + } + } + }, + "DragTarget": { + "type": "object", + "properties": { + "offset": { + "description": "Offset from the visible action point, in top-level viewport CSS pixels.", + "anyOf": [ + { + "$ref": "#/definitions/DragPoint" + }, + { + "type": "null" + } + ] + }, + "ref": { + "type": [ + "string", + "null" + ] + }, + "selector": { + "type": [ + "string", + "null" + ] + } + } + }, + "KeyModifier": { + "description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).", + "type": "string", + "enum": [ + "alt", + "ctrl", + "meta", + "shift" + ] + }, + "MouseButton": { + "description": "Mouse button selector for `tool.click`.", + "type": "string", + "enum": [ + "left", + "middle", + "right" + ] + } + } +} diff --git a/crates/bsk-protocol/schema/tool_drag_result.json b/crates/bsk-protocol/schema/tool_drag_result.json new file mode 100644 index 000000000..26a897f7c --- /dev/null +++ b/crates/bsk-protocol/schema/tool_drag_result.json @@ -0,0 +1,116 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DragResult", + "type": "object", + "required": [ + "source", + "tab_id", + "target" + ], + "properties": { + "dialogs": { + "description": "Input delivery completed; application acceptance must be observed separately.", + "type": "array", + "items": { + "$ref": "#/definitions/JavaScriptDialogInfo" + } + }, + "source": { + "$ref": "#/definitions/DragPoint" + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "target": { + "$ref": "#/definitions/DragPoint" + } + }, + "definitions": { + "DragPoint": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number", + "format": "double" + }, + "y": { + "type": "number", + "format": "double" + } + } + }, + "JavaScriptDialogHandledAction": { + "description": "How the extension resolved the dialog so CDP could continue.", + "type": "string", + "enum": [ + "accepted", + "dismissed" + ] + }, + "JavaScriptDialogInfo": { + "description": "One observed + handled JavaScript dialog during a tool call.", + "type": "object", + "required": [ + "handled", + "message", + "sequence", + "tab_id", + "type" + ], + "properties": { + "default_prompt": { + "type": [ + "string", + "null" + ] + }, + "handled": { + "$ref": "#/definitions/JavaScriptDialogHandledAction" + }, + "has_browser_handler": { + "type": [ + "boolean", + "null" + ] + }, + "message": { + "type": "string" + }, + "sequence": { + "description": "Monotonic per-tab sequence for ordering within a session.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "type": { + "$ref": "#/definitions/JavaScriptDialogType" + }, + "url": { + "type": [ + "string", + "null" + ] + } + } + }, + "JavaScriptDialogType": { + "description": "Native JS dialog kind reported by CDP.", + "type": "string", + "enum": [ + "alert", + "confirm", + "prompt", + "beforeunload" + ] + } + } +} diff --git a/crates/bsk-protocol/schema/tool_observe_result.json b/crates/bsk-protocol/schema/tool_observe_result.json index 890e5fab7..5f2acc17e 100644 --- a/crates/bsk-protocol/schema/tool_observe_result.json +++ b/crates/bsk-protocol/schema/tool_observe_result.json @@ -42,6 +42,13 @@ "null" ] }, + "observation_id": { + "description": "Token identifying the current ref map; pass to drag to reject stale refs.", + "type": [ + "string", + "null" + ] + }, "ref_count": { "description": "Number of `@e` refs registered for this session by this observation. Equivalent to the size of the new ref-store map.", "type": "integer", diff --git a/crates/bsk-protocol/schema/tool_snapshot_result.json b/crates/bsk-protocol/schema/tool_snapshot_result.json index da4ff111c..14132ac91 100644 --- a/crates/bsk-protocol/schema/tool_snapshot_result.json +++ b/crates/bsk-protocol/schema/tool_snapshot_result.json @@ -14,6 +14,13 @@ "$ref": "#/definitions/JavaScriptDialogInfo" } }, + "observation_id": { + "description": "Token identifying the current ref map; pass to drag to reject stale refs.", + "type": [ + "string", + "null" + ] + }, "ref_count": { "description": "Number of `@e` refs registered for this session by this snapshot. Equivalent to the size of the new ref-store map.", "type": "integer", diff --git a/crates/bsk-protocol/src/bin/dump-schema.rs b/crates/bsk-protocol/src/bin/dump-schema.rs index acedcf7d6..9afdc2b31 100644 --- a/crates/bsk-protocol/src/bin/dump-schema.rs +++ b/crates/bsk-protocol/src/bin/dump-schema.rs @@ -88,6 +88,8 @@ fn main() { dump!(ClickResult, "tool_click_result"); dump!(HoverParams, "tool_hover_params"); dump!(HoverResult, "tool_hover_result"); + dump!(DragParams, "tool_drag_params"); + dump!(DragResult, "tool_drag_result"); dump!(WheelParams, "tool_wheel_params"); dump!(WheelResult, "tool_wheel_result"); dump!(ScrollToParams, "tool_scroll_to_params"); diff --git a/crates/bsk-protocol/src/method.rs b/crates/bsk-protocol/src/method.rs index 2596c4f85..ef37a333d 100644 --- a/crates/bsk-protocol/src/method.rs +++ b/crates/bsk-protocol/src/method.rs @@ -84,6 +84,8 @@ pub enum Method { ToolClick, #[serde(rename = "tool.hover")] ToolHover, + #[serde(rename = "tool.drag")] + ToolDrag, #[serde(rename = "tool.wheel")] ToolWheel, #[serde(rename = "tool.scroll_to")] @@ -210,6 +212,7 @@ impl Method { | Method::ToolNavigateForward | Method::ToolReload | Method::ToolClick + | Method::ToolDrag | Method::ToolWheel | Method::ToolScrollTo | Method::ToolFocus @@ -487,6 +490,7 @@ mod tests { assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput); assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput); assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation); + assert_eq!(Method::ToolDrag.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::ToolWheel.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::ToolScrollTo.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::ToolFocus.effect(), MethodEffect::BrowserMutation); diff --git a/crates/bsk-protocol/src/tools/drag.rs b/crates/bsk-protocol/src/tools/drag.rs new file mode 100644 index 000000000..c6bc845cf --- /dev/null +++ b/crates/bsk-protocol/src/tools/drag.rs @@ -0,0 +1,61 @@ +//! One bounded mouse drag, with element identity at both ends. +use super::{JavaScriptDialogInfo, KeyModifier, MouseButton}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct DragPoint { + pub x: f64, + pub y: f64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct DragTarget { + #[serde(rename = "ref", default, skip_serializing_if = "Option::is_none")] + pub ref_: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + /// Offset from the visible action point, in top-level viewport CSS pixels. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub offset: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct DragParams { + pub session_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tab_id: Option, + pub source: DragTarget, + /// Exactly one of target or delta is required. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delta: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub button: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub modifiers: Option>, + /// Delay after pressing and before moving. Defaults to zero. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 0, max = 5000))] + pub hold_ms: Option, + /// Travel duration, excluding hold. Defaults to 500 ms. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1, max = 5000))] + pub duration_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observation_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1))] + pub timeout_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct DragResult { + pub tab_id: i64, + pub source: DragPoint, + pub target: DragPoint, + /// Input delivery completed; application acceptance must be observed separately. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dialogs: Vec, +} diff --git a/crates/bsk-protocol/src/tools/mod.rs b/crates/bsk-protocol/src/tools/mod.rs index 8c51128ea..4ef6d3fab 100644 --- a/crates/bsk-protocol/src/tools/mod.rs +++ b/crates/bsk-protocol/src/tools/mod.rs @@ -4,6 +4,7 @@ pub mod console; pub mod cooperate; pub mod debug; pub mod dialog; +pub mod drag; pub mod emulate; pub mod extract; pub mod file_transfer; @@ -29,6 +30,7 @@ pub use console::*; pub use cooperate::*; pub use debug::*; pub use dialog::*; +pub use drag::*; pub use emulate::*; pub use extract::*; pub use file_transfer::*; diff --git a/crates/bsk-protocol/src/tools/observation.rs b/crates/bsk-protocol/src/tools/observation.rs index f88f33b7e..13e6cf48c 100644 --- a/crates/bsk-protocol/src/tools/observation.rs +++ b/crates/bsk-protocol/src/tools/observation.rs @@ -32,6 +32,9 @@ pub struct SnapshotParams { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct SnapshotResult { + /// Token identifying the current ref map; pass to drag to reject stale refs. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observation_id: Option, /// Indented aria-snapshot text. Refs are rendered as `@e` so the /// agent can copy them into subsequent `tool.click` / `tool.fill` /// selectors. @@ -129,6 +132,9 @@ pub struct HoverProbeReport { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct ObserveResult { + /// Token identifying the current ref map; pass to drag to reject stale refs. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observation_id: Option, /// Continue omitted content from this same observation. #[serde(default, skip_serializing_if = "Option::is_none")] pub next_cursor: Option, @@ -348,6 +354,7 @@ mod tests { #[test] fn snapshot_result_round_trips_with_ref_count() { let r = SnapshotResult { + observation_id: None, text: "root\n @e1 button \"submit\"\n".into(), ref_count: 1, tab_id: 42, @@ -406,6 +413,7 @@ mod tests { #[test] fn observe_result_round_trips_with_ref_count() { let r = ObserveResult { + observation_id: None, next_cursor: None, text: "@vom 1\n @e1 button \"submit\"\n".into(), ref_count: 1, @@ -450,6 +458,7 @@ mod tests { #[test] fn observe_result_round_trips_with_hover_probe_report() { let r = ObserveResult { + observation_id: None, next_cursor: None, text: "@vom 1\n".into(), ref_count: 0, diff --git a/evals/browser/cases/regression/drag-interactions/README.md b/evals/browser/cases/regression/drag-interactions/README.md new file mode 100644 index 000000000..071f6b913 --- /dev/null +++ b/evals/browser/cases/regression/drag-interactions/README.md @@ -0,0 +1,48 @@ +# Drag interactions + +This synthetic corpus tests complete mouse gestures through `tool.drag` and the +`bsk drag` CLI. The shared page contains mouse listeners, pointer capture, a +long-press threshold, native HTML drag with DataTransfer, a range input, a resize +handle, sortable cards, Canvas drawing and SVG movement. Browser assertions read +actual DOM order/values, pixels and trusted events rather than command receipts. + +The browser suite also exercises real SortableJS in native and fallback modes, +real dnd-kit with a distance activation threshold, VOM duplicate-name refs, +observation expiry, shadow roots, iframe coordinates, background focus, +modifiers, invalid/covered/hidden/detached targets, cancellation during hold, +movement and native dragging, delayed replies, and recovery after interruption. +Dependencies for third-party fixtures are test-only and version-locked. + +Run with an installed Chrome executable (no browser download required): + +```sh +BSK_DRAG_CHROME=/path/to/chrome pnpm --filter @browser-skill/extension exec vitest run \ + src/tools/__tests__/drag.browser.test.ts src/tools/__tests__/drag.test.ts +pnpm eval:browser:check +``` + +The suite owns an isolated browser/profile and closes each page. CI enables the +browser tests explicitly; without the environment variable they are skipped. + +For the full CLI/IPC/WS/installed-extension path, use current stable Chrome or +Chrome for Testing with `Extensions.loadUnpacked` support: + +```sh +cargo build -p bsk --locked +BSK_DAEMON_WS_URL=ws://127.0.0.1:52839 pnpm --filter @browser-skill/extension build +BSK_DRAG_CHROME=/path/to/chrome node evals/browser/cases/regression/drag-interactions/run.mjs +``` + +This runner owns its daemon home, browser/profile and session; it stops the +session even on failure. Results go to ignored `evals/browser/results/drag-interactions`. +It validates shared-control checks, timeout recovery and cross-origin iframe refs +as well as page outcomes. To use another +port, match the extension build URL with `BSK_DRAG_PORT`. + +The corpus also supports the existing smoke runner against a connected test +extension: `pnpm eval:browser smoke --case drag-interactions --bsk ./target/debug/bsk`. + +Scope: desktop mouse input, same-tab visible endpoints, element references or +strict main-document selectors. No automatic scrolling during a held gesture, +touch gestures or screenshot-bound Canvas endpoints. A cancelled gesture may +already have changed application state; input cleanup is not rollback. diff --git a/evals/browser/cases/regression/drag-interactions/drag-interactions.case.json b/evals/browser/cases/regression/drag-interactions/drag-interactions.case.json new file mode 100644 index 000000000..cec3673fa --- /dev/null +++ b/evals/browser/cases/regression/drag-interactions/drag-interactions.case.json @@ -0,0 +1,70 @@ +{ + "$schema": "../../../schemas/case.schema.json", + "schemaVersion": 1, + "id": "drag-interactions", + "title": "Native drag and pointer movement", + "suite": "regression", + "tags": ["drag", "pointer", "regression"], + "fixture": { + "startPath": "/drag-interactions" + }, + "prompts": { + "en": "Open {url}. Drag Mouse box right by 120 pixels and drag Native card into Done. Click Check drag results and confirm DRAG-OK. Close the session.", + "zh-CN": "打开 {url}，把 Mouse box 向右拖动120像素，把 Native card 拖入 Done。点击 Check drag results 并确认 DRAG-OK，最后关闭会话。" + }, + "coverage": [ + "session.start", + "session.stop", + "page.navigate", + "inspect.observe", + "interact.drag", + "interact.click" + ], + "assertions": { + "site": [ + { + "label": "both drag mechanisms changed the page", + "type": "drag.checked", + "where": { + "data.drops": 1, + "data.mouse": 120 + }, + "minCount": 1 + } + ], + "response": [ + { + "label": "verified drag outcome", + "includes": "DRAG-OK" + } + ], + "adapter": [] + }, + "smoke": { + "steps": [ + { + "action": "navigate" + }, + { + "action": "observe" + }, + { + "action": "drag", + "selector": "#mouse", + "dx": 120 + }, + { + "action": "drag", + "selector": "#native", + "to": "#drop" + }, + { + "action": "click", + "selector": "#check" + }, + { + "action": "observe" + } + ] + } +} diff --git a/evals/browser/cases/regression/drag-interactions/drag-interactions.fixture.mjs b/evals/browser/cases/regression/drag-interactions/drag-interactions.fixture.mjs new file mode 100644 index 000000000..4062ee2dd --- /dev/null +++ b/evals/browser/cases/regression/drag-interactions/drag-interactions.fixture.mjs @@ -0,0 +1,56 @@ +import { page } from "../../../lib/fixtures.mjs"; + +export const dragMarkup = ` +
Mouse box
Pointer box
Hold box
+
Native card
Done
+ +
+
First
Second
Third
+ + +`; + +export const dragScript = ` +window.events=[]; window.moved={}; window.held=0; window.drops=0; +for(const type of ['mousedown','mousemove','mouseup','pointerdown','pointermove','pointerup','dragstart','dragover','drop','dragend']) + document.addEventListener(type,e=>events.push({type,id:e.target.id,trusted:e.isTrusted,buttons:e.buttons,button:e.button,shift:e.shiftKey,x:e.clientX,y:e.clientY}),true); +function installDrag(el,kind,apply,minHold=0){ + let start; + el.addEventListener(kind+'down',e=>{e.preventDefault();start={x:e.clientX,y:e.clientY,t:performance.now()};if(kind==='pointer')el.setPointerCapture(e.pointerId)}); + window.addEventListener(kind+'move',e=>{if(!start||!e.buttons||performance.now()-start.t{if(start)window.held=performance.now()-start.t;start=null}); +} +for(const [id,kind,min] of [['mouse','mouse',0],['pointer','pointer',0],['hold','pointer',220]]){ + const el=document.getElementById(id);installDrag(el,kind,(x,y)=>el.style.transform='translate('+x+'px,'+y+'px)',min); +} +installDrag(document.getElementById('handle'),'pointer',(x,y)=>{resize.style.width=(140+x)+'px';resize.style.height=(70+y)+'px'}); +let sorted; +for(const el of document.querySelectorAll('#sort>div')){ + el.addEventListener('pointerdown',e=>{e.preventDefault();sorted=el}); + el.addEventListener('pointerup',()=>{if(sorted&&sorted!==el)sort.insertBefore(sorted,el.nextSibling);sorted=null}); +} +native.addEventListener('dragstart',e=>e.dataTransfer.setData('text/plain','card-17')); +drop.addEventListener('drop',e=>{e.preventDefault();if(e.dataTransfer.getData('text/plain')==='card-17'){drop.append(native);window.drops++;}}); +const canvas=document.getElementById('canvas'),pen=canvas.getContext('2d'); +installDrag(canvas,'pointer',(x,y,e)=>{const r=canvas.getBoundingClientRect();pen.fillRect(e.clientX-r.left,e.clientY-r.top,4,4)}); +installDrag(document.getElementById('svg'),'pointer',(x,y)=>{svg.setAttribute('x',20+x);svg.setAttribute('y',10+y)}); +check.onclick=()=>{const data={drops:window.drops,mouse:window.moved.mouse?.dx??0}; + result.textContent=data.drops===1&&data.mouse>=100?'DRAG-OK':'DRAG-INCOMPLETE'; + window.browserEval?.send('drag.checked',data);}; +`; + +export default { + id: "drag-interactions", + routes: ["/drag-interactions"], + render() { + return page({ title: "Drag interactions", body: dragMarkup, script: dragScript }); + }, +}; diff --git a/evals/browser/cases/regression/drag-interactions/libraries.tsx b/evals/browser/cases/regression/drag-interactions/libraries.tsx new file mode 100644 index 000000000..1f4bbfb6c --- /dev/null +++ b/evals/browser/cases/regression/drag-interactions/libraries.tsx @@ -0,0 +1,61 @@ +import { + closestCenter, + DndContext, + type DragEndEvent, + PointerSensor, + useSensor, + useSensors, +} from "@dnd-kit/core"; +import { + arrayMove, + SortableContext, + useSortable, + verticalListSortingStrategy, +} from "@dnd-kit/sortable"; +import React, { useState } from "react"; +import { createRoot } from "react-dom/client"; + +function Item({ id }: { id: string }) { + const { attributes, listeners, setNodeRef, transform, transition } = useSortable({ id }); + return ( +
+ {id} +
+ ); +} +function App() { + const [items, setItems] = useState(["alpha", "beta", "gamma"]); + const sensors = useSensors(useSensor(PointerSensor, { activationConstraint: { distance: 8 } })); + function end({ active, over }: DragEndEvent) { + if (over && active.id !== over.id) + setItems((current) => + arrayMove(current, current.indexOf(String(active.id)), current.indexOf(String(over.id))), + ); + } + return ( + + +
+ {items.map((id) => ( + + ))} +
+ + + ); +} +createRoot(document.getElementById("app")!).render(); diff --git a/evals/browser/cases/regression/drag-interactions/run.mjs b/evals/browser/cases/regression/drag-interactions/run.mjs new file mode 100644 index 000000000..cb748c845 --- /dev/null +++ b/evals/browser/cases/regression/drag-interactions/run.mjs @@ -0,0 +1,205 @@ +// Full CLI -> daemon -> installed MV3 extension, in an isolated browser/profile. +import assert from "node:assert/strict"; +import { execFile, spawn } from "node:child_process"; +import { once } from "node:events"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { withChrome } from "../snapshot-coordinates/chrome.mjs"; +import { dragMarkup, dragScript } from "./drag-interactions.fixture.mjs"; + +const exec = promisify(execFile); +const chrome = process.env.BSK_DRAG_CHROME; +assert(chrome, "Set BSK_DRAG_CHROME to a Chrome executable with extension debugging support"); +const bsk = resolve(process.env.BSK_DRAG_BSK ?? "target/debug/bsk"); +const extension = resolve(process.env.BSK_DRAG_EXTENSION ?? "apps/extension/dist/chrome-mv3"); +const port = Number(process.env.BSK_DRAG_PORT ?? 52839); +const home = await mkdtemp(join(tmpdir(), "bsk-drag-home-")); +const env = { ...process.env, BSK_HOME: home, BSK_AUTO_UPDATE: "off", BSK_AUTO_START: "0" }; +const output = resolve(process.env.BSK_DRAG_OUTPUT ?? "evals/browser/results/drag-interactions"); +const transcript = []; +const browserLog = []; +async function cli(args) { + try { + const { stdout } = await exec(bsk, [...args, "--json"], { + env, + timeout: 45000, + maxBuffer: 8 * 1024 * 1024, + }); + const result = JSON.parse(stdout); + transcript.push({ args, result }); + return result; + } catch (error) { + transcript.push({ args, error: error.stdout || error.message }); + throw error; + } +} +async function eventually(fn) { + let error; + for (let attempt = 0; attempt < 100; attempt++) { + try { + return await fn(); + } catch (e) { + error = e; + } + await new Promise((r) => setTimeout(r, 100)); + } + throw error; +} +const server = createServer((req, res) => { + if (req.url === "/frames") { + res.setHeader("Content-Type", "text/html; charset=utf-8"); + res.end( + `Frame drag`, + ); + return; + } + if (req.url === "/child") { + res.setHeader("Content-Type", "text/html; charset=utf-8"); + res.end(`
Frame handle
`); + return; + } + res.setHeader("Content-Type", "text/html; charset=utf-8"); + res.end(`Drag regression${dragMarkup}`); +}); +server.listen(0, "127.0.0.1"); +await once(server, "listening"); +const daemon = spawn(bsk, ["daemon", "start", "--foreground", "--port", String(port)], { + env, + stdio: ["ignore", "ignore", "pipe"], +}); +let daemonLog = ""; +daemon.stderr.on("data", (data) => { + daemonLog += data; +}); +try { + await eventually(() => cli(["status"])); + await withChrome( + { + executable: chrome, + deviceScale: 1, + zoom: 1, + extensionPath: extension, + headless: true, + disableGpu: false, + extensionViaCdp: true, + onEvent: (event) => { + if ( + event.method === "Runtime.consoleAPICalled" || + event.method === "Runtime.exceptionThrown" + ) + browserLog.push(event); + }, + }, + async (send) => { + const worker = await eventually(async () => { + const targets = (await send("Target.getTargets")).targetInfos; + const t = targets.find( + (t) => + t.type === "service_worker" && + t.url.startsWith("chrome-extension://") && + t.url.endsWith("/background.js"), + ); + assert(t); + return t; + }); + const { sessionId: workerSession } = await send("Target.attachToTarget", { + targetId: worker.targetId, + flatten: true, + }); + await send("Runtime.enable", {}, workerSession); + transcript.push({ browser: await send("Browser.getVersion") }); + await eventually(async () => { + const r = await cli(["browsers"]); + assert.equal((Array.isArray(r) ? r : r.browsers).length, 1); + }); + const { session_id: session } = await cli(["session", "start"]); + const sessionArgs = ["--session", session]; + try { + await cli(["navigate", `http://127.0.0.1:${server.address().port}/`, ...sessionArgs]); + const action = async (args) => { + const c = await cli(["cooperate", "sync", ...sessionArgs]); + assert(c.context_id); + return cli([ + ...(typeof args === "function" ? args(c) : args), + ...sessionArgs, + "--context", + c.context_id, + ]); + }; + const verify = async (expression) => { + const r = await action(["evaluate", expression]); + assert.equal(r.ok, true); + return r.value; + }; + await action(["drag", "#mouse", "--dx", "120"]); + assert.equal(await verify("window.moved.mouse.dx"), 120); + await action(["drag", "#hold", "--dx", "100", "--hold", "300ms"]); + assert.equal(await verify("window.moved.hold.dx"), 100); + await action(["drag", "#native", "--to", "#drop"]); + assert.equal(await verify("window.drops"), 1); + await action(["drag", "#range", "--dx", "70"]); + assert((await verify("Number(range.value)")) > 70); + await action(["drag", "#handle", "--dx", "50", "--dy", "30"]); + assert.deepEqual(await verify("[resize.style.width,resize.style.height]"), [ + "190px", + "100px", + ]); + await action(["drag", "#first", "--to", "#third"]); + assert.deepEqual(await verify("Array.from(sort.children,e=>e.id)"), [ + "second", + "third", + "first", + ]); + await assert.rejects( + action(["drag", "#pointer", "--dx", "30", "--hold", "1s", "--timeout", "300ms"]), + (error) => JSON.parse(error.stdout).code === "timeout", + ); + await action(["drag", "#pointer", "--dx", "50"]); + assert.equal(await verify("window.moved.pointer.dx"), 50); + await action(["navigate", `http://127.0.0.1:${server.address().port}/frames`]); + await action((context) => { + const line = context.observation.text + .split("\n") + .find((line) => line.includes('[drag-handle] "Frame handle"')); + const ref = line?.match(/@e\d+/)?.[0]; + assert(ref, context.observation.text); + return [ + "drag", + ref, + "--dx", + "60", + "--dy", + "20", + "--observation-id", + context.observation.observation_id, + ]; + }); + assert.deepEqual(await verify("window.frameMove"), [60, 20]); + console.log( + "PASS installed extension: mouse, hold, native drop, range, resize, sort, timeout recovery, cross-origin frame ref", + ); + } finally { + await cli(["session", "stop", session]); + } + }, + ); +} finally { + if (daemon.exitCode === null && daemon.signalCode === null) { + const exited = once(daemon, "exit"); + daemon.kill(); + await exited; + } + server.closeAllConnections(); + await new Promise((r) => server.close(r)); + await mkdir(output, { recursive: true }); + await writeFile(join(output, "transcript.json"), JSON.stringify(transcript, null, 2)); + await writeFile(join(output, "daemon.log"), daemonLog); + await writeFile(join(output, "browser.json"), JSON.stringify(browserLog, null, 2)); + await rm(home, { recursive: true, force: true }); +} diff --git a/evals/browser/lib/bsk-runner.mjs b/evals/browser/lib/bsk-runner.mjs index 4a780f8b8..6c53d5317 100644 --- a/evals/browser/lib/bsk-runner.mjs +++ b/evals/browser/lib/bsk-runner.mjs @@ -187,6 +187,18 @@ export async function runBskSmokeTask({ ...tabArgs(step, variables), ]); break; + case "drag": + result = await bsk([ + "drag", + String(resolveValue(step.ref ?? step.selector, variables)), + ...(step.to !== undefined ? ["--to", String(resolveValue(step.to, variables))] : []), + ...["dx", "dy", "hold", "duration"].flatMap((key) => + step[key] === undefined ? [] : [`--${key}`, String(step[key])], + ), + ...session, + ...tabArgs(step, variables), + ]); + break; case "fill": result = await bsk( [ diff --git a/evals/browser/lib/case-loader.mjs b/evals/browser/lib/case-loader.mjs index 522741f09..151c28c3d 100644 --- a/evals/browser/lib/case-loader.mjs +++ b/evals/browser/lib/case-loader.mjs @@ -66,9 +66,25 @@ function validateWorkflowStep(errors, step, path) { if (step.evidence !== undefined && !EVIDENCE_PATTERN.test(step.evidence)) { errors.push(`${path}.evidence must be an identifier`); } - if (["click", "hover", "fill", "select"].includes(step.action) && !step.selector && !step.ref) { + if ( + ["click", "hover", "fill", "select", "drag"].includes(step.action) && + !step.selector && + !step.ref + ) { errors.push(`${path} requires selector or ref`); } + if (step.action === "drag") { + const hasDelta = step.dx !== undefined || step.dy !== undefined; + if ((step.to !== undefined) === hasDelta) errors.push(`${path} requires either to or dx/dy`); + if (step.to !== undefined) checkString(errors, step.to, `${path}.to`); + for (const key of ["dx", "dy"]) + if (step[key] !== undefined && !Number.isFinite(step[key])) + errors.push(`${path}.${key} must be finite`); + if (hasDelta && (step.dx ?? 0) === 0 && (step.dy ?? 0) === 0) + errors.push(`${path} requires a nonzero delta`); + for (const key of ["hold", "duration"]) + if (step[key] !== undefined) checkString(errors, step[key], `${path}.${key}`); + } if (step.action === "fill") { if (typeof step.value !== "string") errors.push(`${path}.value must be a string`); if (step.noClear !== undefined && typeof step.noClear !== "boolean") { diff --git a/evals/browser/lib/operations.mjs b/evals/browser/lib/operations.mjs index da698e12a..0ca64f689 100644 --- a/evals/browser/lib/operations.mjs +++ b/evals/browser/lib/operations.mjs @@ -15,6 +15,7 @@ export const OPERATION_CATALOG = [ { operation: "inspect.network" }, { operation: "interact.click" }, { operation: "interact.hover" }, + { operation: "interact.drag" }, { operation: "interact.fill" }, { operation: "interact.select" }, { operation: "interact.press" }, @@ -46,6 +47,7 @@ export const WORKFLOW_ACTION_OPERATIONS = { network: "inspect.network", click: "interact.click", hover: "interact.hover", + drag: "interact.drag", fill: "interact.fill", select: "interact.select", press: "interact.press", diff --git a/evals/browser/tests/drag-workflow.test.mjs b/evals/browser/tests/drag-workflow.test.mjs new file mode 100644 index 000000000..623d6adb6 --- /dev/null +++ b/evals/browser/tests/drag-workflow.test.mjs @@ -0,0 +1,88 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { runBskSmokeTask } from "../lib/bsk-runner.mjs"; +import { validateCaseManifest } from "../lib/case-loader.mjs"; + +const manifest = JSON.parse( + await readFile( + new URL("../cases/regression/drag-interactions/drag-interactions.case.json", import.meta.url), + "utf8", + ), +); +test("drag manifests validate a unique destination or finite nonzero displacement", () => { + for (const step of [ + { action: "drag", ref: "@e1", to: "@e2" }, + { action: "drag", selector: "#handle", dx: -100, hold: "300ms", duration: "1s" }, + ]) { + const copy = structuredClone(manifest); + copy.smoke.steps = [step]; + assert.deepEqual(validateCaseManifest(copy), []); + } + for (const step of [ + { action: "drag", to: "#end" }, + { action: "drag", selector: "#start" }, + { action: "drag", selector: "#start", to: "#end", dx: 10 }, + { action: "drag", selector: "#start", to: "" }, + { action: "drag", selector: "#start", dx: 0 }, + { action: "drag", selector: "#start", dx: Infinity }, + { action: "drag", selector: "#start", dy: "20" }, + ]) { + const copy = structuredClone(manifest); + copy.smoke.steps = [step]; + assert.notDeepEqual(validateCaseManifest(copy), []); + } +}); +test("drag smoke routes refs, destination, hold and signed displacement, then stops the session", async () => { + const outputDirectory = await mkdtemp(join(tmpdir(), "drag-workflow-")); + const calls = []; + try { + const report = await runBskSmokeTask({ + bskCommand: "bsk", + outputDirectory, + runId: "drag-test", + server: { reset() {}, snapshot: () => ({ events: [] }) }, + serverInfo: { baseUrl: "http://localhost:4173" }, + task: { + id: "drag-test", + suite: "regression", + tags: [], + startPath: "/drag-interactions", + siteAssertions: [], + responseAssertions: [], + adapterAssertions: [], + smokeSteps: [ + { action: "drag", ref: "@e1", to: "@e2" }, + { action: "drag", selector: "#handle", dx: -100, dy: 20, hold: "300ms", duration: "1s" }, + ], + }, + executeProcess: async (_command, args) => { + calls.push(args); + let response = {}; + if (args[0] === "session" && args[1] === "start") response = { session_id: "test" }; + if (args[0] === "session" && args[1] === "stop") response = { stopped: ["test"] }; + return { exitCode: 0, stdout: JSON.stringify(response), stderr: "" }; + }, + }); + assert.equal(report.executionError, undefined); + const drags = calls.filter((args) => args[0] === "drag"); + assert.deepEqual(drags[0].slice(0, 4), ["drag", "@e1", "--to", "@e2"]); + assert.deepEqual(drags[1].slice(0, 10), [ + "drag", + "#handle", + "--dx", + "-100", + "--dy", + "20", + "--hold", + "300ms", + "--duration", + "1s", + ]); + assert.equal(report.evidence.sessionStopped, true); + } finally { + await rm(outputDirectory, { recursive: true, force: true }); + } +}); diff --git a/evals/browser/tests/eval.test.mjs b/evals/browser/tests/eval.test.mjs index 03f647539..58b4fdbf5 100644 --- a/evals/browser/tests/eval.test.mjs +++ b/evals/browser/tests/eval.test.mjs @@ -42,6 +42,7 @@ test("case manifests are discovered, ordered, and grouped into suites", () => { "diagnostics", "mobile-emulation", "generated-form", + "drag-interactions", "fill-editor-roots", "oopif-scrollbars", "snapshot-coordinates", @@ -50,16 +51,16 @@ test("case manifests are discovered, ordered, and grouped into suites", () => { assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, { core: 6, matrix: 1, - regression: 3, + regression: 4, }); }); test("repository validation links every case to a fixture and valid workflow evidence", () => { assert.deepEqual(validateRepositoryCases(cases, fixtureRegistry), []); const summary = repositorySummary(cases, fixtureRegistry); - assert.equal(summary.cases, 10); - assert.equal(summary.fixtureModules, 11); - assert.equal(summary.fixtureRoutes, 19); + assert.equal(summary.cases, 11); + assert.equal(summary.fixtureModules, 12); + assert.equal(summary.fixtureRoutes, 20); }); test("manifest validation rejects unknown operations and incomplete workflow steps", () => { @@ -276,11 +277,11 @@ test("report summary keeps partial verification distinct from full verification" assert.equal(rows[0].errors, 1); }); -test("coverage inventory contains all 28 operations and three manual lanes", () => { +test("coverage inventory contains all 29 operations and three manual lanes", () => { const coverage = buildOperationCoverage(cases); const operations = coverage.map(({ operation }) => operation); - assert.equal(operations.length, 28); - assert.equal(new Set(operations).size, 28); + assert.equal(operations.length, 29); + assert.equal(new Set(operations).size, 29); assert.deepEqual( coverage.filter(({ directSmoke }) => !directSmoke).map(({ operation }) => operation), ["tabs.borrow", "tabs.return", "assist.request-help"], @@ -360,7 +361,7 @@ test("CLI accepts pnpm separator and supports suite and tag filters", async () = timeoutMs: 5_000, }); assert.equal(coverageResult.exitCode, 0, coverageResult.stderr); - assert.equal(JSON.parse(coverageResult.stdout).length, 28); + assert.equal(JSON.parse(coverageResult.stdout).length, 29); const coreResult = await runProcess( process.execPath, diff --git a/packages/i18n/src/locales/de-DE/extension.json b/packages/i18n/src/locales/de-DE/extension.json index 78b56dd9e..cedb201c3 100644 --- a/packages/i18n/src/locales/de-DE/extension.json +++ b/packages/i18n/src/locales/de-DE/extension.json @@ -329,6 +329,7 @@ "press": "Taste drücken", "select": "Option auswählen", "hover": "Hover", + "drag": "Ziehen", "focus": "Fokussieren", "blur": "Fokus entfernen", "scroll": "Scrollen", diff --git a/packages/i18n/src/locales/en-US/extension.json b/packages/i18n/src/locales/en-US/extension.json index 4e7c2b0d4..6cf643486 100644 --- a/packages/i18n/src/locales/en-US/extension.json +++ b/packages/i18n/src/locales/en-US/extension.json @@ -329,6 +329,7 @@ "press": "Press key", "select": "Select option", "hover": "Hover", + "drag": "Drag", "focus": "Focus", "blur": "Blur", "scroll": "Scroll", diff --git a/packages/i18n/src/locales/es-ES/extension.json b/packages/i18n/src/locales/es-ES/extension.json index 978ce0a7e..58f10c55f 100644 --- a/packages/i18n/src/locales/es-ES/extension.json +++ b/packages/i18n/src/locales/es-ES/extension.json @@ -329,6 +329,7 @@ "press": "Pulsar tecla", "select": "Seleccionar opción", "hover": "Pasar el ratón por encima", + "drag": "Arrastrar", "focus": "Enfocar", "blur": "Quitar el foco", "scroll": "Desplazar", diff --git a/packages/i18n/src/locales/fr-FR/extension.json b/packages/i18n/src/locales/fr-FR/extension.json index a4c19439f..238070e69 100644 --- a/packages/i18n/src/locales/fr-FR/extension.json +++ b/packages/i18n/src/locales/fr-FR/extension.json @@ -329,6 +329,7 @@ "press": "Appuyer sur une touche", "select": "Sélectionner une option", "hover": "Survoler", + "drag": "Glisser", "focus": "Mettre le focus", "blur": "Retirer le focus", "scroll": "Faire défiler", diff --git a/packages/i18n/src/locales/it-IT/extension.json b/packages/i18n/src/locales/it-IT/extension.json index b70b592dd..2a618f969 100644 --- a/packages/i18n/src/locales/it-IT/extension.json +++ b/packages/i18n/src/locales/it-IT/extension.json @@ -329,6 +329,7 @@ "press": "Premi tasto", "select": "Seleziona opzione", "hover": "Passa sopra", + "drag": "Trascinare", "focus": "Attiva focus", "blur": "Rimuovi focus", "scroll": "Scorri", diff --git a/packages/i18n/src/locales/ja-JP/extension.json b/packages/i18n/src/locales/ja-JP/extension.json index e69643802..55d028d03 100644 --- a/packages/i18n/src/locales/ja-JP/extension.json +++ b/packages/i18n/src/locales/ja-JP/extension.json @@ -329,6 +329,7 @@ "press": "キーを押す", "select": "オプションを選択", "hover": "ホバー", + "drag": "ドラッグ", "focus": "フォーカス", "blur": "フォーカス解除", "scroll": "スクロール", diff --git a/packages/i18n/src/locales/ko-KR/extension.json b/packages/i18n/src/locales/ko-KR/extension.json index 621fbb2b8..6454a002d 100644 --- a/packages/i18n/src/locales/ko-KR/extension.json +++ b/packages/i18n/src/locales/ko-KR/extension.json @@ -329,6 +329,7 @@ "press": "키 누르기", "select": "옵션 선택", "hover": "마우스 올리기", + "drag": "드래그", "focus": "초점 맞추기", "blur": "초점 해제", "scroll": "스크롤", diff --git a/packages/i18n/src/locales/pt-BR/extension.json b/packages/i18n/src/locales/pt-BR/extension.json index e15ee0d4d..52491ab67 100644 --- a/packages/i18n/src/locales/pt-BR/extension.json +++ b/packages/i18n/src/locales/pt-BR/extension.json @@ -329,6 +329,7 @@ "press": "Pressionar tecla", "select": "Selecionar opção", "hover": "Passar o mouse", + "drag": "Arrastar", "focus": "Focar", "blur": "Remover foco", "scroll": "Rolar", diff --git a/packages/i18n/src/locales/zh-CN/extension.json b/packages/i18n/src/locales/zh-CN/extension.json index e9c9e55d6..694ce197f 100644 --- a/packages/i18n/src/locales/zh-CN/extension.json +++ b/packages/i18n/src/locales/zh-CN/extension.json @@ -329,6 +329,7 @@ "press": "按键", "select": "选择选项", "hover": "悬停", + "drag": "拖动", "focus": "聚焦", "blur": "移开焦点", "scroll": "滚动", diff --git a/packages/i18n/src/locales/zh-TW/extension.json b/packages/i18n/src/locales/zh-TW/extension.json index 39533c48f..89712a0d7 100644 --- a/packages/i18n/src/locales/zh-TW/extension.json +++ b/packages/i18n/src/locales/zh-TW/extension.json @@ -329,6 +329,7 @@ "press": "按鍵", "select": "選擇選項", "hover": "懸停", + "drag": "拖曳", "focus": "聚焦", "blur": "移開焦點", "scroll": "捲動", diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index 431c4f03a..98c5ff46f 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -1209,3 +1209,82 @@ describe("disabled form state", () => { expect(out.text).not.toContain('button "Cancel" [disabled]'); }); }); + +describe("drag affordances", () => { + it.each([ + { attrs: { draggable: "true" }, hint: "draggable" }, + { cursor: "grab", hint: "drag-handle" }, + { cursor: "col-resize", hint: "drag-handle" }, + { attrs: { ondrop: "accept(event)" }, hint: "drop-target" }, + ])("keeps an addressable drag candidate without inventing a button: %j", ({ hint, ...props }) => { + const out = renderVom( + scene([ + node({ + id: 1, + backendNodeId: 42, + role: "generic", + name: "Item", + rect: { x: 0, y: 0, w: 100, h: 50 }, + ...props, + }), + ]), + ); + expect(out.refs).toHaveLength(1); + expect(out.text).toContain(`[${hint}]`); + expect(out.text).not.toContain("button"); + }); + it.each([ + { attrs: { draggable: "false" } }, + { cursor: "default" }, + { cursor: "grab", inert: true }, + { cursor: "grab", pointerEvents: "none" }, + ])("does not manufacture drag refs from %j", (props) => { + const out = renderVom( + scene([ + node({ + id: 1, + backendNodeId: 42, + role: "generic", + rect: { x: 0, y: 0, w: 100, h: 50 }, + ...props, + }), + ]), + ); + expect(out.refs).toHaveLength(0); + }); +}); + +it("does not turn inherited drag cursors into duplicate handles", () => { + const out = renderVom( + scene([ + node({ + id: 1, + backendNodeId: 10, + role: "generic", + name: "Card", + cursor: "grab", + rect: { x: 0, y: 0, w: 100, h: 80 }, + }), + node({ + id: 2, + parentId: 1, + backendNodeId: 20, + role: "generic", + name: "Title", + cursor: "grab", + rect: { x: 0, y: 0, w: 100, h: 20 }, + }), + node({ + id: 3, + parentId: 1, + backendNodeId: 30, + role: "generic", + name: "Resize", + cursor: "ew-resize", + rect: { x: 90, y: 20, w: 10, h: 60 }, + }), + ]), + ); + expect(out.refs.map((ref) => ref.backendNodeId)).toEqual([10, 30]); + expect(out.text).not.toContain('[drag-handle] "Title"'); +}); diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 71302796d..49b177398 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -188,7 +188,26 @@ function isUrlBearingStructuralAction(node: VomNode): boolean { return name !== undefined && isProbableNavigationTarget(name); } +/** Hints describe observed evidence, not a promise that a drop will be accepted. */ +function dragHint(node: VomNode): string | undefined { + if ( + node.disabled || + node.inert || + node.pointerEvents === "none" || + ["statictext", "inlinetextbox"].includes(normalizedRole(node)) + ) + return undefined; + if (node.dragHint !== undefined) return node.dragHint || undefined; + if (node.attrs?.draggable?.toLowerCase() === "true") return "draggable"; + if (node.attrs?.ondrop !== undefined || node.attrs?.ondragover !== undefined) + return "drop-target"; + if (["grab", "grabbing", "move"].includes(node.cursor ?? "") || node.cursor?.endsWith("-resize")) + return "drag-handle"; + return undefined; +} + function shouldRender(node: VomNode): boolean { + if (dragHint(node)) return true; if (!node.role) return false; const role = normalizedRole(node); @@ -202,7 +221,9 @@ function shouldRender(node: VomNode): boolean { export function isVomReferenceNode(node: VomNode): boolean { return ( node.referenceable !== false && - (INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node)) + (INTERACTIVE_ROLES.has(normalizedRole(node)) || + isUrlBearingStructuralAction(node) || + dragHint(node) !== undefined) ); } @@ -370,6 +391,19 @@ function isAncestorOf( } export function applyVomInteractionRecovery(nodes: VomNode[]): VomNode[] { + // CSS cursors inherit. Expose the handle once instead of assigning refs to + // every decorative span inside it; retain explicit nested draggable/drop cues. + const byId = new Map(nodes.map((node) => [node.id, node])); + nodes = nodes.map((node) => { + const parent = node.parentId === null ? undefined : byId.get(node.parentId); + return dragHint(node) === "drag-handle" && + parent && + dragHint(parent) && + parent.cursor === node.cursor && + parent.frameId === node.frameId + ? { ...node, dragHint: false } + : node; + }); const parentMap = buildParentMap(nodes); const children = buildChildren(nodes); const recoveredIds = new Set(); @@ -719,8 +753,10 @@ function renderNodeLine( surface: CondSurface | undefined = undefined, redactValues = false, ): string { - let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`; + let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role ?? "generic"}`; + const hint = dragHint(node); + if (hint) line += ` [${hint}]`; const name = cleaned(node.name); if (name) line += ` ${JSON.stringify(name)}`; if (context.length > 0) line += ` [ctx: ${context.join(" > ")}]`; diff --git a/packages/vom/src/types.ts b/packages/vom/src/types.ts index 50d7a58c9..b7b0f4b98 100644 --- a/packages/vom/src/types.ts +++ b/packages/vom/src/types.ts @@ -51,6 +51,8 @@ export interface VomNode { position: string; pointerEvents: string; cursor?: string; + /** Derived interaction hint; false suppresses an inherited cursor hint. */ + dragHint?: "draggable" | "drag-handle" | "drop-target" | false; /** Browser-reported clickability, independent of role and inline attributes. */ clickable?: boolean; attrs?: Record; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cf11d4367..85f0786d7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -47,6 +47,12 @@ importers: specifier: ^19.0.0 version: 19.2.6(react@19.2.6) devDependencies: + '@dnd-kit/core': + specifier: 6.3.1 + version: 6.3.1(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@dnd-kit/sortable': + specifier: 10.0.0 + version: 10.0.0(@dnd-kit/core@6.3.1(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react@19.2.6) '@tailwindcss/vite': specifier: ^4.3.0 version: 4.3.0(vite@8.0.13(@types/node@25.8.0)(esbuild@0.27.7)(jiti@2.7.0)) @@ -74,9 +80,15 @@ importers: '@wxt-dev/module-react': specifier: ^1.1.3 version: 1.2.2(vite@8.0.13(@types/node@25.8.0)(esbuild@0.27.7)(jiti@2.7.0))(wxt@0.20.26(@types/node@25.8.0)(jiti@2.7.0)(rolldown@1.2.4)(rollup@4.60.4)) + esbuild: + specifier: 0.27.7 + version: 0.27.7 happy-dom: specifier: ^15.11.7 version: 15.11.7 + sortablejs: + specifier: 1.15.6 + version: 1.15.6 tailwindcss: specifier: ^4.3.0 version: 4.3.0 @@ -926,6 +938,28 @@ packages: engines: {node: '>= 0.10.4'} hasBin: true + '@dnd-kit/accessibility@3.1.1': + resolution: {integrity: sha512-2P+YgaXF+gRsIihwwY1gCsQSYnu9Zyj2py8kY5fFvUM1qm2WA2u639R6YNVfU4GWr+ZM5mqEsfHZZLoRONbemw==} + peerDependencies: + react: '>=16.8.0' + + '@dnd-kit/core@6.3.1': + resolution: {integrity: sha512-xkGBRQQab4RLwgXxoqETICr6S5JlogafbhNsidmrkVv2YRs5MLwpjoF2qpiGjQt8S9AoxtIV603s0GIUpY5eYQ==} + peerDependencies: + react: '>=16.8.0' + react-dom: '>=16.8.0' + + '@dnd-kit/sortable@10.0.0': + resolution: {integrity: sha512-+xqhmIIzvAYMGfBYYnbKuNicfSsk4RksY2XdmJhT+HAC01nix6fHCztU68jooFiMUB01Ky3F0FyOvhG/BZrWkg==} + peerDependencies: + '@dnd-kit/core': ^6.3.0 + react: '>=16.8.0' + + '@dnd-kit/utilities@3.2.2': + resolution: {integrity: sha512-+MKAJEOfaBe5SmV6t34p80MMKhjvUz0vRrvVJbPT0WElzaOJ/1xs+D+KDv+tD/NE5ujfrChEcshd4fLn0wpiqg==} + peerDependencies: + react: '>=16.8.0' + '@dual-bundle/import-meta-resolve@4.2.1': resolution: {integrity: sha512-id+7YRUgoUX6CgV0DtuhirQWodeeA7Lf4i2x71JS/vtA5pRb/hIGWlw+G6MeXvsM+MXrz0VAydTGElX1rAfgPg==} @@ -3725,6 +3759,9 @@ packages: sonic-boom@4.2.1: resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} + sortablejs@1.15.6: + resolution: {integrity: sha512-aNfiuwMEpfBM/CN6LY0ibyhxPfPbyFeBTYJKCvzkJ2GkUpazIt3H+QIPAMHwqQ7tMKaHz1Qj+rJJCqljnf4p3A==} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -4879,6 +4916,31 @@ snapshots: transitivePeerDependencies: - supports-color + '@dnd-kit/accessibility@3.1.1(react@19.2.6)': + dependencies: + react: 19.2.6 + tslib: 2.8.1 + + '@dnd-kit/core@6.3.1(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + dependencies: + '@dnd-kit/accessibility': 3.1.1(react@19.2.6) + '@dnd-kit/utilities': 3.2.2(react@19.2.6) + react: 19.2.6 + react-dom: 19.2.6(react@19.2.6) + tslib: 2.8.1 + + '@dnd-kit/sortable@10.0.0(@dnd-kit/core@6.3.1(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react@19.2.6)': + dependencies: + '@dnd-kit/core': 6.3.1(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@dnd-kit/utilities': 3.2.2(react@19.2.6) + react: 19.2.6 + tslib: 2.8.1 + + '@dnd-kit/utilities@3.2.2(react@19.2.6)': + dependencies: + react: 19.2.6 + tslib: 2.8.1 + '@dual-bundle/import-meta-resolve@4.2.1': {} '@emnapi/core@1.10.0': @@ -7531,6 +7593,8 @@ snapshots: dependencies: atomic-sleep: 1.0.0 + sortablejs@1.15.6: {} + source-map-js@1.2.1: {} source-map-support@0.5.21: From 301d02e5d415cf180a4539bbdcd3ecacedaa2e71 Mon Sep 17 00:00:00 2001 From: drakezhang Date: Wed, 28 Oct 2026 23:02:35 +0800 Subject: [PATCH 2/5] docs: keep browser skill within CRLF size budget --- crates/bsk-cli/skill/SKILL.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index e9f1f7ead..0cdbc73df 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -92,7 +92,7 @@ Use a ref and context ID from the latest sync (one action per context): | Fill a field | `bsk fill @e3 --value "text" --session --context ID` | | Select an option | `bsk select @e3 --value "option-value" --session --context ID` | | Press a key | `bsk press Enter --ref @e3 --session --context ID` | -| Drag or hold and drag | `bsk drag @e3 --to @e8 --hold 300ms --session --context ID` | +| Drag | `bsk drag @e3 --to @e8 --session --context ID` | | Reveal a hover menu | `bsk hover @e3 --session --context ID` | | Reveal an element | `bsk scroll-to @e3 --session --context ID` | | Scroll with wheel input | `bsk wheel --delta-y 600 --session --context ID` | @@ -110,9 +110,8 @@ The agent decides. On `dialog_pending`, ## Read details only when needed -Resolve references from this skill's directory, not the working directory. -Read the matching reference before acting; do not preload all references. -Load further references only as needed. +Resolve references relative to this skill. Read the matching reference before +acting; load others only as needed. | When | Read | | --- | --- | From 26182e7ce07aaa1e409b9966fa2ada30ce37daac Mon Sep 17 00:00:00 2001 From: drakezhang Date: Wed, 28 Oct 2026 23:18:41 +0800 Subject: [PATCH 3/5] fix: wait for remote capacity cleanup in integration test --- .../src/daemon/remote/authorization.rs | 22 ++++++++++----- crates/bsk-cli/tests/remote_server.rs | 28 +++++++++++++++++-- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/crates/bsk-cli/src/daemon/remote/authorization.rs b/crates/bsk-cli/src/daemon/remote/authorization.rs index 83951a68c..68b4a6341 100644 --- a/crates/bsk-cli/src/daemon/remote/authorization.rs +++ b/crates/bsk-cli/src/daemon/remote/authorization.rs @@ -147,7 +147,8 @@ impl AuthorizationStore { .truncate(false) .read(true) .write(true) - .open(self.path.with_extension("lock"))?; + .open(self.path.with_extension("lock")) + .context("open authorization write lock")?; #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; @@ -165,17 +166,24 @@ impl AuthorizationStore { } std::thread::sleep(Duration::from_millis(10)); } - Err(error) => return Err(error.into()), + Err(error) => return Err(error).context("acquire authorization write lock"), } } let mut state = self.read_state()?; let result = work(&mut state)?; state.version = 1; - let mut file = tempfile::NamedTempFile::new_in(parent)?; + let mut file = tempfile::NamedTempFile::new_in(parent) + .context("create temporary authorization snapshot")?; serde_json::to_writer(file.as_file_mut(), &state)?; - file.as_file_mut().flush()?; - file.as_file().sync_all()?; - file.persist(&self.path).map_err(|err| err.error)?; + file.as_file_mut() + .flush() + .context("flush authorization snapshot")?; + file.as_file() + .sync_all() + .context("sync authorization snapshot")?; + file.persist(&self.path) + .map_err(|err| err.error) + .context("replace authorization snapshot")?; if let Ok(directory) = fs::File::open(parent) { let _ = directory.sync_all(); } @@ -186,7 +194,7 @@ impl AuthorizationStore { let state: State = match fs::read(&self.path) { Ok(bytes) => serde_json::from_slice(&bytes).context("invalid authorization store")?, Err(err) if err.kind() == std::io::ErrorKind::NotFound => State::default(), - Err(err) => return Err(err.into()), + Err(err) => return Err(err).context("read authorization snapshot"), }; ensure!( state.version <= 1, diff --git a/crates/bsk-cli/tests/remote_server.rs b/crates/bsk-cli/tests/remote_server.rs index d0eaeb9ec..da2f81106 100644 --- a/crates/bsk-cli/tests/remote_server.rs +++ b/crates/bsk-cli/tests/remote_server.rs @@ -217,9 +217,31 @@ async fn browser_capacity_does_not_block_renewal_or_replacement() { &store.authenticate(&renewed).unwrap().device_id, ]); closed(&mut replacement).await; - let (mut ws, _) = tokio_tungstenite::connect_async(server.request(&second, ORIGIN)) - .await - .unwrap(); + // The close frame precedes server cleanup and release of its capacity slot. + // Wait for that release, but do not hide authorization or transport errors. + let (mut ws, _) = tokio::time::timeout(Duration::from_secs(3), async { + loop { + match tokio_tungstenite::connect_async(server.request(&second, ORIGIN)).await { + Ok(connection) => break connection, + Err(tokio_tungstenite::tungstenite::Error::Http(response)) + if response.status() == 503 => + { + assert_eq!(response.headers()["retry-after"], "5"); + assert_eq!( + serde_json::from_slice::( + response.body().as_deref().unwrap_or_default() + ) + .unwrap(), + json!({"error": "connection_capacity_reached"}) + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + Err(error) => panic!("second browser connection failed: {error}"), + } + } + }) + .await + .expect("revoked connection must release browser capacity"); handshake(&mut ws, "second").await; } From 18e0287dcfcb30cf5aaf1c10d833f421c2ac76f0 Mon Sep 17 00:00:00 2001 From: drakezhang Date: Wed, 28 Oct 2026 23:57:22 +0800 Subject: [PATCH 4/5] fix: validate drag endpoints and preserve interaction semantics --- apps/extension/src/cooperation/controller.ts | 4 +- .../src/cooperation/pointer-target.ts | 77 +------- .../src/tools/__tests__/drag.browser.test.ts | 46 +++++ .../src/tools/__tests__/target-hit.test.ts | 167 ++++++++++++++++++ apps/extension/src/tools/drag.ts | 29 ++- apps/extension/src/tools/geometry.ts | 36 +++- apps/extension/src/tools/shared.ts | 2 + apps/extension/src/tools/target-hit.ts | 64 ++++++- .../regression/drag-interactions/README.md | 5 +- .../regression/drag-interactions/run.mjs | 164 ++++++++++++++++- packages/vom/src/__tests__/render.test.ts | 96 ++++++++++ packages/vom/src/render.ts | 15 +- 12 files changed, 600 insertions(+), 105 deletions(-) create mode 100644 apps/extension/src/tools/__tests__/target-hit.test.ts diff --git a/apps/extension/src/cooperation/controller.ts b/apps/extension/src/cooperation/controller.ts index ced2ea090..0323c4af7 100644 --- a/apps/extension/src/cooperation/controller.ts +++ b/apps/extension/src/cooperation/controller.ts @@ -1289,9 +1289,9 @@ export class CooperationController implements ControlAuthority { sent = true; nativeInputStarted ||= startsInput; }, - verifyPointerTarget: async (node, point) => { + verifyPointerTarget: async (node, point, offset) => { try { - await pointerTargetValidator(cdp)(node, point); + await pointerTargetValidator(cdp)(node, point, offset); } catch (error) { revoked = error instanceof Error ? error.message : String(error); throw error; diff --git a/apps/extension/src/cooperation/pointer-target.ts b/apps/extension/src/cooperation/pointer-target.ts index 1486b391c..76b6f2411 100644 --- a/apps/extension/src/cooperation/pointer-target.ts +++ b/apps/extension/src/cooperation/pointer-target.ts @@ -1,14 +1,11 @@ -import { cdpTargetKey } from "@/browser-driver/frame-graph"; import { resolveNodeGeometry } from "@/tools/frame-geometry"; -import { projectPolygon } from "@/tools/geometry"; -import { GeometryContext } from "@/tools/geometry/frame-context"; -import { type CdpRunner, isRpcError, sendToCdpTarget } from "@/tools/shared"; -import { pointBelongsToTarget } from "@/tools/target-hit"; +import { type CdpRunner, isRpcError } from "@/tools/shared"; +import { pointReachesTarget } from "@/tools/target-hit"; export function pointerTargetValidator( cdp: CdpRunner, ): NonNullable { - return async (target, point) => { + return async (target, point, offset = { x: 0, y: 0 }) => { const geometry = await resolveNodeGeometry(cdp, target.tabId, { target: target.cdpTarget, backendNodeId: target.backendNodeId, @@ -16,71 +13,11 @@ export function pointerTargetValidator( }); if ( isRpcError(geometry) || - Math.abs(geometry.actionPoint.x - point.x) > 2 || - Math.abs(geometry.actionPoint.y - point.y) > 2 + Math.abs(geometry.actionPoint.x + offset.x - point.x) > 2 || + Math.abs(geometry.actionPoint.y + offset.y - point.y) > 2 ) throw new Error("The pointer target moved or disappeared; synchronize before clicking"); - const objectGroup = `shared-hit-${crypto.randomUUID()}`; - try { - if ( - !(await pointBelongsToTarget( - cdp, - target, - geometry.targetActionPoint.x, - geometry.targetActionPoint.y, - objectGroup, - false, - )) - ) - throw new Error( - "The pointer no longer hits the observed target; synchronize before clicking", - ); - if (target.frameId && target.cdpTarget.sessionId) { - const context = new GeometryContext(cdp, target.tabId); - const ancestry = await context.ancestry(target.frameId); - const projection = await context.targetProjection(target.frameId); - if (!ancestry || !projection) throw new Error("The pointer frame path is unavailable"); - let local = geometry.targetActionPoint; - let edge = 0; - for (const child of ancestry) { - const parent = await context.frame(child.parentFrameId!); - if (!parent) throw new Error("The pointer frame path changed"); - if (cdpTargetKey(child.target) === cdpTargetKey(parent.target)) continue; - const mapping = projection.edges[edge++]; - if (!mapping || child.ownerBackendNodeId === undefined) - throw new Error("The pointer frame mapping is unavailable"); - local = projectPolygon([local], mapping)[0]; - // OOPIF hit testing stops at its owning iframe. Verify that owner in - // every parent target, rather than comparing unrelated frame ids. - try { - if ( - !(await pointBelongsToTarget( - cdp, - { - cdpTarget: parent.target, - backendNodeId: child.ownerBackendNodeId, - frameId: parent.frameId, - }, - local.x, - local.y, - objectGroup, - false, - )) - ) - throw new Error("The pointer target is covered; synchronize before clicking"); - } finally { - await sendToCdpTarget(cdp, parent.target, "Runtime.releaseObjectGroup", { - objectGroup, - }).catch(() => {}); - } - } - if (Math.abs(local.x - point.x) > 2 || Math.abs(local.y - point.y) > 2) - throw new Error("The pointer frame projection changed; synchronize before clicking"); - } - } finally { - await sendToCdpTarget(cdp, target.cdpTarget, "Runtime.releaseObjectGroup", { - objectGroup, - }).catch(() => {}); - } + if (!(await pointReachesTarget(cdp, target, point))) + throw new Error("The pointer target is covered or unavailable; synchronize before clicking"); }; } diff --git a/apps/extension/src/tools/__tests__/drag.browser.test.ts b/apps/extension/src/tools/__tests__/drag.browser.test.ts index 1cc14bac2..27424ae81 100644 --- a/apps/extension/src/tools/__tests__/drag.browser.test.ts +++ b/apps/extension/src/tools/__tests__/drag.browser.test.ts @@ -5,6 +5,7 @@ import { createRequire } from "node:module"; import { build } from "esbuild"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph"; +import { pointerTargetValidator } from "@/cooperation/pointer-target"; import { SessionManager } from "@/session-manager/manager"; import type { DragParams } from "@/transport/types"; import { handleDrag } from "../drag"; @@ -127,6 +128,7 @@ async function harness(targetId: string, background: boolean) { return { rootFrameId: frameTree.frame.id, frames: result }; }; cdp.getFrameGraph = frames; + cdp.verifyPointerTarget = pointerTargetValidator(cdp); const deps = { cdp, tabsApi: { @@ -304,6 +306,50 @@ describe.skipIf(!process.env.BSK_DRAG_CHROME)("real browser drag", () => { "30", ]); })); + it.each([-100, 100])("preserves a source offset of %s with the production validator", async (x) => + withPage(async (h) => { + ok( + await h.drag({ + source: { selector: "#canvas", offset: { x, y: 20 } }, + delta: { x: 20, y: 10 }, + }), + ); + const expected = await h.evaluate<{ x: number; y: number }>( + `(()=>{const r=canvas.getBoundingClientRect();return {x:r.x+r.width/2+${x},y:r.y+r.height/2+20}})()`, + ); + const down = await h.evaluate<{ x: number; y: number }>( + "events.find(e=>e.id==='canvas'&&e.type==='pointerdown')", + ); + expect(down.x).toBeCloseTo(expected.x, 0); + expect(down.y).toBeCloseTo(expected.y, 0); + expect(await h.evaluate("window.moved.canvas")).toEqual({ dx: 20, dy: 10 }); + })); + it.each([ + "covered", + "moved", + "outside", + ])("rejects an offset point that is %s before pressing", async (kind) => + withPage(async (h) => { + if (kind === "covered") + await h.evaluate( + `canvas.scrollIntoView({block:'center'});const r=canvas.getBoundingClientRect();const cover=document.createElement('div');cover.style='position:fixed;z-index:1000;width:30px;height:30px;background:red;left:'+(r.x+r.width/2-115)+'px;top:'+(r.y+r.height/2-15)+'px';document.body.append(cover)`, + ); + if (kind === "moved") + h.setHook(async (method, p) => { + if (method === "Input.dispatchMouseEvent" && p?.type === "mouseMoved") { + h.setHook(() => {}); + await h.evaluate("canvas.style.transform='translateX(20px)'"); + } + }); + const result = await h.drag({ + source: { selector: "#canvas", offset: { x: kind === "outside" ? 300 : -100, y: 0 } }, + delta: { x: 20, y: 0 }, + }); + expect(result).toMatchObject({ code: expect.any(String), data: { effect_state: "none" } }); + if (kind === "moved") + expect(result).toHaveProperty("message", expect.stringMatching(/moved/)); + expect(h.calls.some((call) => call.params?.type === "mousePressed")).toBe(false); + })); it("uses VOM refs for native source and an empty drop area", async () => withPage(async (h) => { const observation = await h.observe(); diff --git a/apps/extension/src/tools/__tests__/target-hit.test.ts b/apps/extension/src/tools/__tests__/target-hit.test.ts new file mode 100644 index 000000000..ba773bc73 --- /dev/null +++ b/apps/extension/src/tools/__tests__/target-hit.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, it } from "vitest"; +import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph"; +import { projectPolygon, type Quad, unprojectPoint } from "../geometry"; +import type { CdpRunner } from "../shared"; +import { pointReachesTarget } from "../target-hit"; + +describe("inverse input projection", () => { + it.each([ + [ + { x: 100, y: 50 }, + { x: 500, y: 50 }, + { x: 500, y: 350 }, + { x: 100, y: 350 }, + ], + [ + { x: 120, y: 20 }, + { x: 500, y: 70 }, + { x: 420, y: 400 }, + { x: 50, y: 300 }, + ], + [ + { x: 400, y: 20 }, + { x: 350, y: 300 }, + { x: 50, y: 250 }, + { x: 100, y: 0 }, + ], + ])("round-trips an offset point through translated, rotated and perspective frames", (...quad) => { + const edge = { sourceViewport: { width: 600, height: 400 }, destinationQuad: quad as Quad }; + const source = { x: 110, y: 280 }; + const actual = unprojectPoint(projectPolygon([source], edge)[0], edge); + expect(actual?.x).toBeCloseTo(source.x, 8); + expect(actual?.y).toBeCloseTo(source.y, 8); + }); + + it("rejects a singular frame projection", () => { + expect( + unprojectPoint( + { x: 1, y: 1 }, + { + sourceViewport: { width: 100, height: 100 }, + destinationQuad: Array.from({ length: 4 }, () => ({ x: 0, y: 0 })) as Quad, + }, + ), + ).toBeNull(); + }); +}); + +function nestedFrames(blocked?: string) { + const root = { tabId: 4 }; + const parent = { tabId: 4, sessionId: "parent" }; + const child = { tabId: 4, sessionId: "child" }; + const graph: CdpFrameGraph = { + rootFrameId: "root-frame", + frames: [ + { frameId: "root-frame", target: root }, + { + frameId: "parent-frame", + parentFrameId: "root-frame", + target: parent, + ownerBackendNodeId: 100, + }, + { + frameId: "child-frame", + parentFrameId: "parent-frame", + target: child, + ownerBackendNodeId: 200, + }, + ], + }; + const hits: { target: string; x: number; y: number }[] = []; + const released: string[] = []; + const request = async (target: CdpTarget, method: string, params?: object): Promise => { + const key = target.sessionId ?? "root"; + const p = params as Record; + const sizes = { root: [1000, 800], parent: [400, 300], child: [200, 100] }; + const [width, height] = sizes[key as keyof typeof sizes]; + switch (method) { + case "Page.getLayoutMetrics": + return { + cssLayoutViewport: { clientWidth: width, clientHeight: height, pageX: 0, pageY: 0 }, + }; + case "Runtime.evaluate": + return { result: { value: { width, height } } }; + case "DOM.getBoxModel": + return { + model: { + content: + p.backendNodeId === 100 + ? [100, 50, 500, 50, 500, 350, 100, 350] + : [20, 30, 220, 30, 220, 130, 20, 130], + }, + }; + case "DOM.getNodeForLocation": + hits.push({ target: key, x: p.x as number, y: p.y as number }); + return { + backendNodeId: blocked === key ? 999 : { root: 100, parent: 200, child: 9 }[key], + frameId: `${key}-frame`, + }; + case "DOM.resolveNode": + return { object: { objectId: `${key}:${p.backendNodeId}` } }; + case "Runtime.callFunctionOn": + return { + result: { value: p.objectId === (p.arguments as { objectId: string }[])[0].objectId }, + }; + case "Runtime.releaseObjectGroup": + released.push(key); + return {}; + default: + throw new Error(`Unexpected method ${method}`); + } + }; + const cdp: CdpRunner = { + send: ((_tab, method, params) => request(root, method, params)) as CdpRunner["send"], + sendToTarget: request as CdpRunner["sendToTarget"], + getFrameGraph: async () => graph, + }; + return { + cdp, + hits, + released, + target: { tabId: 4, cdpTarget: child, backendNodeId: 9, frameId: "child-frame" }, + }; +} + +describe("pointer hit through frame ancestors", () => { + it("checks every owner and the leaf at the inverse-projected root input point", async () => { + const h = nestedFrames(); + expect(await pointReachesTarget(h.cdp, h.target, { x: 220, y: 130 })).toBe(true); + expect(h.hits).toEqual([ + { target: "root", x: 220, y: 130 }, + { target: "parent", x: 120, y: 80 }, + { target: "child", x: 100, y: 50 }, + ]); + expect(h.released).toEqual(["root", "parent", "child"]); + }); + + it.each([ + "root", + "parent", + "child", + ])("rejects an overlay in %s and releases handles", async (blocked) => { + const h = nestedFrames(blocked); + expect(await pointReachesTarget(h.cdp, h.target, { x: 220, y: 130 })).toBe(false); + expect(h.hits.at(-1)?.target).toBe(blocked); + expect(h.released).toEqual(h.hits.map((hit) => hit.target)); + }); + + it("fails closed when an out-of-process target has no frame identity", async () => { + const h = nestedFrames(); + expect( + await pointReachesTarget(h.cdp, { ...h.target, frameId: undefined }, { x: 220, y: 130 }), + ).toBe(false); + expect(h.hits).toEqual([]); + }); + + it("fails closed when the frame belongs to a replacement target", async () => { + const h = nestedFrames(); + expect( + await pointReachesTarget( + h.cdp, + { ...h.target, cdpTarget: { tabId: 4, sessionId: "retired-child" } }, + { x: 220, y: 130 }, + ), + ).toBe(false); + expect(h.hits).toEqual([]); + }); +}); diff --git a/apps/extension/src/tools/drag.ts b/apps/extension/src/tools/drag.ts index 82bd0df0e..b95e43a74 100644 --- a/apps/extension/src/tools/drag.ts +++ b/apps/extension/src/tools/drag.ts @@ -3,11 +3,7 @@ import type { SessionManager } from "@/session-manager/manager"; import type { DragParams, DragPoint, DragResult, DragTarget, RpcError } from "@/transport/types"; import { withClickOverlay } from "./click-overlay"; import { attachDialogs, markDialogCursor } from "./dialogs"; -import { - type ResolvedNodeGeometry, - resolveNodeGeometry, - scrollElementAndFramesIntoView, -} from "./frame-geometry"; +import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry"; import { withInputReady } from "./input-readiness"; import { type InteractionDeps, modifiersBitfield, resolveActionTarget } from "./interaction"; import { pointerTransactionDeps } from "./pointer-transaction"; @@ -20,7 +16,7 @@ import { resolveTargetTab, sendToCdpTarget, } from "./shared"; -import { pointBelongsToTarget } from "./target-hit"; +import { pointReachesTarget } from "./target-hit"; import { BoundedWaitError, waitBounded } from "./transfer-transaction"; const validPoint = (p: DragPoint | undefined) => @@ -102,6 +98,7 @@ export async function handleDrag( // late CDP continuations after the outer time boundary has retired the operation. const cdp = new Proxy(inputDeps.cdp, { get(target, key) { + if (key === "resourceCleanup") return deps.cdp.resourceCleanup ?? deps.cdp; const value = Reflect.get(target, key); return typeof value === "function" ? (...args: unknown[]) => { @@ -232,16 +229,10 @@ export async function handleDrag( code: "permission_denied", message: "drag source is detached, disabled or inert", }; - const endpoints: [typeof source, ResolvedNodeGeometry, DragPoint][] = [ - [source, startGeometry, start], - ]; - if (target && endGeometry) endpoints.push([target, endGeometry, end]); - for (const [node, geometry, point] of endpoints) { - const local = { - x: geometry.targetActionPoint.x + point.x - geometry.actionPoint.x, - y: geometry.targetActionPoint.y + point.y - geometry.actionPoint.y, - }; - if (!(await pointBelongsToTarget(cdp, node, local.x, local.y, group, false))) + const endpoints: [typeof source, DragPoint][] = [[source, start]]; + if (target) endpoints.push([target, end]); + for (const [node, point] of endpoints) { + if (!(await pointReachesTarget(cdp, { ...node, tabId: tab.tabId }, point))) return { code: "permission_denied", message: @@ -255,7 +246,11 @@ export async function handleDrag( objectGroup: group, }).catch(() => {}); } - await cdp.verifyPointerTarget?.({ ...source, tabId: tab.tabId }, start); + await cdp.verifyPointerTarget?.( + { ...source, tabId: tab.tabId }, + start, + params.source.offset, + ); input.markSent(); attempted = true; deps.onInputSent?.(tab.tabId); diff --git a/apps/extension/src/tools/geometry.ts b/apps/extension/src/tools/geometry.ts index 247ca02cd..e1dc2ffc7 100644 --- a/apps/extension/src/tools/geometry.ts +++ b/apps/extension/src/tools/geometry.ts @@ -54,8 +54,9 @@ export function viewportPolygon(viewport: Size): Quad { return rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height }); } -/** Project a normalized unit-square point into an arbitrary content quad. */ -export function projectUnitPoint(point: Point, quad: Quad): Point { +function quadProjection( + quad: Quad, +): [number, number, number, number, number, number, number, number] { const [p0, p1, p2, p3] = quad; const dx1 = p1.x - p2.x; const dx2 = p3.x - p2.x; @@ -83,10 +84,7 @@ export function projectUnitPoint(point: Point, quad: Quad): Point { } else { const denominator = dx1 * dy2 - dx2 * dy1; if (Math.abs(denominator) < 1e-9) { - return { - x: p0.x + (p1.x - p0.x) * point.x + (p3.x - p0.x) * point.y, - y: p0.y + (p1.y - p0.y) * point.x + (p3.y - p0.y) * point.y, - }; + return [p1.x - p0.x, p3.x - p0.x, p0.x, p1.y - p0.y, p3.y - p0.y, p0.y, 0, 0]; } g = (dx3 * dy2 - dx2 * dy3) / denominator; h = (dx1 * dy3 - dx3 * dy1) / denominator; @@ -98,6 +96,12 @@ export function projectUnitPoint(point: Point, quad: Quad): Point { f = p0.y; } + return [a, b, c, d, e, f, g, h]; +} + +/** Project a normalized unit-square point into an arbitrary content quad. */ +export function projectUnitPoint(point: Point, quad: Quad): Point { + const [a, b, c, d, e, f, g, h] = quadProjection(quad); const scale = g * point.x + h * point.y + 1; return { x: (a * point.x + b * point.y + c) / scale, @@ -105,6 +109,26 @@ export function projectUnitPoint(point: Point, quad: Quad): Point { }; } +/** Map the actual parent input point back into a child target's viewport. */ +export function unprojectPoint(point: Point, edge: ProjectiveEdge): Point | null { + const { width, height } = edge.sourceViewport; + if (width <= 0 || height <= 0) return null; + const [a, b, c, d, e, f, g, h] = quadProjection(edge.destinationQuad); + const ax = a - point.x * g; + const bx = b - point.x * h; + const ay = d - point.y * g; + const by = e - point.y * h; + const determinant = ax * by - bx * ay; + if (Math.abs(determinant) < 1e-9) return null; + const x = point.x - c; + const y = point.y - f; + const local = { + x: ((x * by - bx * y) / determinant) * width, + y: ((ax * y - x * ay) / determinant) * height, + }; + return Number.isFinite(local.x) && Number.isFinite(local.y) ? local : null; +} + export function projectPolygon(polygon: Polygon, edge: ProjectiveEdge): Polygon { const { width, height } = edge.sourceViewport; if (width <= 0 || height <= 0) return []; diff --git a/apps/extension/src/tools/shared.ts b/apps/extension/src/tools/shared.ts index 91f9c5095..c2e68fd23 100644 --- a/apps/extension/src/tools/shared.ts +++ b/apps/extension/src/tools/shared.ts @@ -71,6 +71,8 @@ export interface CdpRunner { verifyPointerTarget?( target: { tabId: number; cdpTarget: CdpTarget; backendNodeId: number; frameId?: string }, point: { x: number; y: number }, + /** Intentional displacement from the observed action point, in viewport CSS pixels. */ + offset?: { x: number; y: number }, ): Promise; send(tabId: number, method: string, params?: object): Promise; sendToTarget?(target: CdpTarget, method: string, params?: object): Promise; diff --git a/apps/extension/src/tools/target-hit.ts b/apps/extension/src/tools/target-hit.ts index 4159d8b70..bc6052729 100644 --- a/apps/extension/src/tools/target-hit.ts +++ b/apps/extension/src/tools/target-hit.ts @@ -1,5 +1,6 @@ -import type { CdpTarget } from "@/browser-driver/frame-graph"; -import { cssViewport, type LayoutMetrics } from "./geometry/frame-context"; +import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph"; +import { type Point, unprojectPoint } from "./geometry"; +import { cssViewport, GeometryContext, type LayoutMetrics } from "./geometry/frame-context"; import { type CdpRunner, sendToCdpTarget } from "./shared"; interface RuntimeReply { @@ -77,3 +78,62 @@ export async function pointBelongsToTarget( if (reply.exceptionDetails) return false; return reply.result?.value === true; } + +/** Check the actual root input point through every target boundary, including + * parent-page overlays that an OOPIF-local hit test cannot see. */ +export async function pointReachesTarget( + cdp: CdpRunner, + target: { tabId: number; cdpTarget: CdpTarget; backendNodeId: number; frameId?: string }, + point: Point, +): Promise { + const objectGroup = `pointer-hit-${crypto.randomUUID()}`; + const touched = new Map(); + const hit = async (node: typeof target, local: Point) => { + touched.set(cdpTargetKey(node.cdpTarget), node.cdpTarget); + return pointBelongsToTarget(cdp, node, local.x, local.y, objectGroup, false); + }; + try { + let local = point; + if (target.cdpTarget.sessionId) { + if (!target.frameId) return false; + const context = new GeometryContext(cdp, target.tabId); + const frame = await context.frame(target.frameId); + if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(target.cdpTarget)) return false; + const ancestry = await context.ancestry(target.frameId); + const projection = await context.targetProjection(target.frameId); + if (!ancestry || !projection) return false; + const owners: (typeof target)[] = []; + for (const child of ancestry) { + const parent = child.parentFrameId && (await context.frame(child.parentFrameId)); + if (!parent) return false; + if (cdpTargetKey(child.target) === cdpTargetKey(parent.target)) continue; + if (child.ownerBackendNodeId === undefined) return false; + owners.push({ + tabId: target.tabId, + cdpTarget: parent.target, + backendNodeId: child.ownerBackendNodeId, + frameId: parent.frameId, + }); + } + if (owners.length !== projection.edges.length) return false; + // Descend from the dispatched point, not from an independently clipped + // element centroid: clipping and CSS transforms can make those differ. + for (let index = owners.length - 1; index >= 0; index--) { + if (!(await hit(owners[index], local))) return false; + const projected = unprojectPoint(local, projection.edges[index]); + if (!projected) return false; + local = projected; + } + } + return await hit(target, local); + } finally { + const cleanup = cdp.resourceCleanup ?? cdp; + await Promise.all( + [...touched.values()].map((address) => + sendToCdpTarget(cleanup, address, "Runtime.releaseObjectGroup", { objectGroup }).catch( + () => {}, + ), + ), + ); + } +} diff --git a/evals/browser/cases/regression/drag-interactions/README.md b/evals/browser/cases/regression/drag-interactions/README.md index 071f6b913..fe11b5123 100644 --- a/evals/browser/cases/regression/drag-interactions/README.md +++ b/evals/browser/cases/regression/drag-interactions/README.md @@ -36,7 +36,10 @@ BSK_DRAG_CHROME=/path/to/chrome node evals/browser/cases/regression/drag-interac This runner owns its daemon home, browser/profile and session; it stops the session even on failure. Results go to ignored `evals/browser/results/drag-interactions`. It validates shared-control checks, timeout recovery and cross-origin iframe refs -as well as page outcomes. To use another +as well as page outcomes. It also checks precise source/destination offsets through +the production pointer validator, preserves fill/click behavior on draggable cards, +and rejects drops covered by the top page or an intermediate iframe before pressing. +To use another port, match the extension build URL with `BSK_DRAG_PORT`. The corpus also supports the existing smoke runner against a connected test diff --git a/evals/browser/cases/regression/drag-interactions/run.mjs b/evals/browser/cases/regression/drag-interactions/run.mjs index cb748c845..49552e992 100644 --- a/evals/browser/cases/regression/drag-interactions/run.mjs +++ b/evals/browser/cases/regression/drag-interactions/run.mjs @@ -49,6 +49,56 @@ async function eventually(fn) { throw error; } const server = createServer((req, res) => { + const url = new URL(req.url, "http://127.0.0.1"); + if (url.pathname === "/precision") { + res.setHeader("Content-Type", "text/html; charset=utf-8"); + res.end(`Precision drag and card semantics +
Draft
+
Open card
+ `); + return; + } + if (url.pathname === "/frame-drops") { + res.setHeader("Content-Type", "text/html; charset=utf-8"); + res.end(`Nested frame drop
Frame source
+ `); + return; + } + if (url.pathname === "/frame-middle") { + res.setHeader("Content-Type", "text/html; charset=utf-8"); + res.end(` + `); + return; + } + if (url.pathname === "/drop-child") { + res.setHeader("Content-Type", "text/html; charset=utf-8"); + res.end(`
Nested destination
`); + return; + } if (req.url === "/frames") { res.setHeader("Content-Type", "text/html; charset=utf-8"); res.end( @@ -122,11 +172,14 @@ try { const sessionArgs = ["--session", session]; try { await cli(["navigate", `http://127.0.0.1:${server.address().port}/`, ...sessionArgs]); + let observation; const action = async (args) => { const c = await cli(["cooperate", "sync", ...sessionArgs]); assert(c.context_id); + // Unchanged/delta replies retain the preceding observation's refs. + observation = c.observation ?? observation; return cli([ - ...(typeof args === "function" ? args(c) : args), + ...(typeof args === "function" ? args({ ...c, observation }) : args), ...sessionArgs, "--context", c.context_id, @@ -137,6 +190,16 @@ try { assert.equal(r.ok, true); return r.value; }; + const observedRef = (context, label, role) => { + const line = context.observation.text + .split("\n") + .find((line) => line.includes(`"${label}"`)); + assert(line, context.observation.text); + if (role) assert.match(line, new RegExp(`\\b${role}\\b`)); + const ref = line.match(/@e\d+/)?.[0]; + assert(ref, line); + return ref; + }; await action(["drag", "#mouse", "--dx", "120"]); assert.equal(await verify("window.moved.mouse.dx"), 120); await action(["drag", "#hold", "--dx", "100", "--hold", "300ms"]); @@ -181,8 +244,105 @@ try { ]; }); assert.deepEqual(await verify("window.frameMove"), [60, 20]); + await action(["navigate", `http://127.0.0.1:${server.address().port}/precision`]); + await action([ + "drag", + "#precision", + "--dx", + "60", + "--dy", + "10", + "--from-offset-x", + "-100", + "--from-offset-y", + "-20", + ]); + assert.deepEqual(await verify("[points[0],points.at(-1)]"), [ + [80, 40], + [140, 50], + ]); + assert.equal( + await verify("precision.getContext('2d').getImageData(140,50,1,1).data[3]"), + 255, + ); + await action(["evaluate", "points=[]"]); + await action([ + "drag", + "#precision", + "--to", + "#precision", + "--from-offset-x", + "-100", + "--to-offset-x", + "100", + ]); + assert.deepEqual(await verify("[points[0],points.at(-1)]"), [ + [80, 60], + [280, 60], + ]); + await action(["evaluate", "points=[];cover.hidden=false"]); + await assert.rejects( + action([ + "drag", + "#precision", + "--dx", + "60", + "--from-offset-x", + "-100", + "--from-offset-y", + "-20", + ]), + (error) => JSON.parse(error.stdout).data?.effect_state === "none", + ); + assert.equal(await verify("points.length"), 0); + await action(["evaluate", "cover.hidden=true"]); + await action((c) => [ + "fill", + observedRef(c, "Editable card", "textbox"), + "--value", + "Updated draft", + ]); + assert.equal( + await verify("document.querySelector('[contenteditable]').textContent"), + "Updated draft", + ); + await action((c) => ["click", observedRef(c, "Clickable card", "button")]); + assert.equal(await verify("cardClicks"), 1); + for (const cover of ["top", "middle", "none"]) { + await action([ + "navigate", + `http://127.0.0.1:${server.address().port}/frame-drops?cover=${cover}`, + ]); + const gesture = () => + action((c) => [ + "drag", + observedRef(c, "Frame source"), + "--to", + observedRef(c, "Nested destination"), + ]); + if (cover === "none") { + await gesture(); + assert.deepEqual(await verify("counts"), { + presses: 1, + starts: 1, + intended: 1, + wrong: 0, + }); + } else { + await assert.rejects( + gesture(), + (error) => JSON.parse(error.stdout).data?.effect_state === "none", + ); + assert.deepEqual(await verify("counts"), { + presses: 0, + starts: 0, + intended: 0, + wrong: 0, + }); + } + } console.log( - "PASS installed extension: mouse, hold, native drop, range, resize, sort, timeout recovery, cross-origin frame ref", + "PASS installed extension: mouse, hold, native drop, range, resize, sort, timeout recovery, cross-origin frame ref, precise offsets, card semantics, nested frame occlusion", ); } finally { await cli(["session", "stop", session]); diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index 98c5ff46f..b66029ad4 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -1212,6 +1212,102 @@ describe("disabled form state", () => { describe("drag affordances", () => { it.each([ + "", + "true", + "plaintext-only", + ])("preserves contenteditable=%s semantics on draggable cards", (contenteditable) => { + const out = renderVom( + scene([ + node({ + id: 1, + backendNodeId: 42, + role: "group", + attrs: { contenteditable, draggable: "true", "aria-label": "Message editor" }, + value: "Draft message", + rect: { x: 0, y: 0, w: 200, h: 80 }, + }), + ]), + ); + expect(out.text).toContain( + '@e1 textbox [draggable] "Message editor" [filled] ="Draft message"', + ); + expect(out.refs).toEqual([ + expect.objectContaining({ backendNodeId: 42, role: "textbox", name: "Message editor" }), + ]); + }); + it.each & { hint: string }>([ + { attrs: { draggable: "true" }, hint: "draggable" }, + { attrs: { ondrop: "accept(event)" }, hint: "drop-target" }, + { cursor: "grab", hint: "drag-handle" }, + ])("recovers clickable semantics alongside $hint", ({ hint, attrs, ...props }) => { + const out = renderVom( + scene([ + node({ + id: 1, + role: "group", + text: "Open settings", + attrs: { onclick: "handleOpen()", ...attrs }, + ...props, + }), + ]), + ); + expect(out.text).toContain(`@e1 button [${hint}] "Open settings"`); + expect(out.refs).toEqual([ + expect.objectContaining({ backendNodeId: 1, role: "button", name: "Open settings" }), + ]); + }); + it("preserves native roles and state when adding a drag hint", () => { + const out = renderVom( + scene([ + node({ + id: 1, + tag: "input", + role: "checkbox", + name: "Select card", + checked: true, + value: "card-1", + attrs: { draggable: "true", onclick: "selectCard()" }, + }), + ]), + ); + expect(out.text).toContain('@e1 checkbox [draggable] "Select card" [checked] ="card-1"'); + expect(out.refs[0].role).toBe("checkbox"); + }); + it.each(["combobox", "switch"])("recovers an explicit %s role on a draggable card", (role) => { + const out = renderVom( + scene([ + node({ + id: 1, + role: "group", + attrs: { draggable: "true", role, tabindex: "0", "aria-label": "Card setting" }, + }), + ]), + ); + expect(out.text).toContain(`@e1 ${role} [draggable] "Card setting"`); + expect(out.refs[0].role).toBe(role); + }); + it("keeps clickable children when their inherited drag hint is suppressed", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "generic", name: "Card", cursor: "grab" }), + node({ + id: 2, + parentId: 1, + role: "generic", + text: "Edit card", + clickable: true, + cursor: "grab", + }), + ]), + ); + expect(out.text).toContain('@e1 generic [drag-handle] "Card"'); + expect(out.text).toContain('@e2 button "Edit card"'); + expect(coreRefs(out.refs)).toEqual([ + { ref: "e1", backendNodeId: 1 }, + { ref: "e2", backendNodeId: 2 }, + ]); + }); + it.each & { hint: string }>([ { attrs: { draggable: "true" }, hint: "draggable" }, { cursor: "grab", hint: "drag-handle" }, { cursor: "col-resize", hint: "drag-handle" }, diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 49b177398..f0477bf0c 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -218,12 +218,16 @@ function shouldRender(node: VomNode): boolean { return cleaned(node.name) !== undefined; } -export function isVomReferenceNode(node: VomNode): boolean { +function isSemanticReferenceNode(node: VomNode): boolean { return ( node.referenceable !== false && - (INTERACTIVE_ROLES.has(normalizedRole(node)) || - isUrlBearingStructuralAction(node) || - dragHint(node) !== undefined) + (INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node)) + ); +} + +export function isVomReferenceNode(node: VomNode): boolean { + return ( + isSemanticReferenceNode(node) || (node.referenceable !== false && dragHint(node) !== undefined) ); } @@ -411,7 +415,8 @@ export function applyVomInteractionRecovery(nodes: VomNode[]): VomNode[] { const recoveredNameById = new Map(); for (const node of nodes) { - if (isVomReferenceNode(node)) continue; + // Drag evidence adds a capability; it must not suppress existing role/name recovery. + if (isSemanticReferenceNode(node)) continue; const role = recoveredRole(node, children); if (!role || !passesRecoveryGuards(node, children)) continue; recoveredIds.add(node.id); From 8465b04b880e428e1445a25fb9ae48115b8dcd0e Mon Sep 17 00:00:00 2001 From: drakezhang Date: Thu, 29 Oct 2026 00:19:42 +0800 Subject: [PATCH 5/5] test: update scroll occlusion error assertions --- apps/extension/src/tools/__tests__/scroll.browser.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/extension/src/tools/__tests__/scroll.browser.test.ts b/apps/extension/src/tools/__tests__/scroll.browser.test.ts index fa3ebb5d2..db2dac7bd 100644 --- a/apps/extension/src/tools/__tests__/scroll.browser.test.ts +++ b/apps/extension/src/tools/__tests__/scroll.browser.test.ts @@ -472,7 +472,7 @@ describe.skipIf(!process.env.BSK_SCROLL_CHROME)("native target hit testing", () expect(await pointBelongsToTarget(h.cdp, target, point.x, point.y, "hit-oracle")).toBe( false, ); - await expect(verify(target, point)).rejects.toThrow(/no longer hits/); + await expect(verify(target, point)).rejects.toThrow(/covered/); expect(await h.evaluate("window.hits")).toBe(1); } }, @@ -530,7 +530,7 @@ describe.skipIf(!process.env.BSK_SCROLL_CHROME)("native target hit testing", () expect(await pointBelongsToTarget(h.cdp, target, point.x, point.y, "hit-oracle")).toBe( false, ); - await expect(verify(target, point)).rejects.toThrow(/covered|no longer hits/); + await expect(verify(target, point)).rejects.toThrow(/covered/); expect(await h.evaluate("window.hits")).toBe(background ? 2 : 1); await h.evaluate("cover.remove()"); } finally {