feat: add native drag gestures with VOM targets

.github/workflows/ci.yml

@@ -284,6 +284,12 @@ jobs:
           BSK_CLICK_CHROME: google-chrome
         run: pnpm exec vitest run src/tools/extract/extract.browser.test.ts
 
+      - name: Run drag browser regression
+        working-directory: apps/extension
+        env:
+          BSK_DRAG_CHROME: google-chrome
+        run: pnpm exec vitest run src/tools/__tests__/drag.browser.test.ts
+
       - name: Run fill browser regression
         working-directory: apps/extension
         env:
@@ -363,6 +369,24 @@ jobs:
           path: evals/browser/results/js-dialog-control/
           if-no-files-found: ignore
 
+      - name: Build extension for the isolated drag daemon
+        env:
+          BSK_DAEMON_WS_URL: ws://127.0.0.1:52839
+        run: pnpm --filter @browser-skill/extension build
+
+      - name: Validate drag CLI, native drops, recovery and cross-origin frame refs
+        env:
+          BSK_DRAG_CHROME: ${{ env.BSK_VIDEO_CHROME }}
+        run: node evals/browser/cases/regression/drag-interactions/run.mjs
+
+      - name: Upload drag regression evidence
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: drag-regression
+          path: evals/browser/results/drag-interactions/
+          if-no-files-found: ignore
+
   node-scripts:
     name: Node script tests
     runs-on: ubuntu-latest

apps/extension/package.json

@@ -23,6 +23,8 @@
     "react-dom": "^19.0.0"
   },
   "devDependencies": {
+    "@dnd-kit/core": "6.3.1",
+    "@dnd-kit/sortable": "10.0.0",
     "@tailwindcss/vite": "^4.3.0",
     "@testing-library/dom": "^10.4.1",
     "@testing-library/react": "^16.3.2",
@@ -32,7 +34,9 @@
     "@types/react-dom": "^19.0.0",
     "@vitejs/plugin-react": "^6.0.0",
     "@wxt-dev/module-react": "^1.1.3",
+    "esbuild": "0.27.7",
     "happy-dom": "^15.11.7",
+    "sortablejs": "1.15.6",
     "tailwindcss": "^4.3.0",
     "typescript": "^5.8.0",
     "vitest": "^4.1.6",

apps/extension/src/cooperation/action-guard.ts

@@ -43,10 +43,12 @@ export function guardedRunner<T extends CdpRunner>(cdp: T, options: ActionGuard
     const key = method === "Input.dispatchKeyEvent";
     const focus = method === "Emulation.setFocusEmulationEnabled";
     const drag = method === "Input.dispatchDragEvent";
+    const cancelDrag = method === "Input.cancelDragging";
     const chooser = method === "Page.setInterceptFileChooserDialog";
-    const identity = `${target.tabId}:${target.sessionId ?? ""}:${chooser ? "chooser" : drag ? "drag" : focus ? "focus" : mouse ? "mouse:" + String(p?.button) : "key:" + String(p?.key)}`;
+    const identity = `${target.tabId}:${target.sessionId ?? ""}:${cancelDrag ? "mouse:left" : chooser ? "chooser" : drag ? "drag" : focus ? "focus" : mouse ? "mouse:" + String(p?.button) : "key:" + String(p?.key)}`;
     const release =
-      ((mouse && p?.type === "mouseReleased") ||
+      (cancelDrag ||
+        (mouse && p?.type === "mouseReleased") ||
         (key && p?.type === "keyUp") ||
         (focus && p?.enabled === false) ||
         (chooser && p?.enabled === false) ||
@@ -96,7 +98,7 @@ export function guardedRunner<T extends CdpRunner>(cdp: T, options: ActionGuard
         (drag && p?.type === "dragEnter")
       )
         leases.set(identity, cdp.getAttachmentId?.(target.tabId));
-      if (release) leases.delete(identity);
+      if (release && !cancelDrag) leases.delete(identity);
     };
     if (cdp.sendGuarded)
       return cdp.sendGuarded(target, method, params, {

apps/extension/src/cooperation/controller.test.ts

@@ -595,18 +595,22 @@ describe("shared-tab dispatch fences", () => {
     const run = (
       f: Awaited<ReturnType<typeof fixture>>,
       context: string,
-      action: { ref?: string; dependsOn?: string[]; key?: string },
+      action: { ref?: string; dependsOn?: string[]; key?: string; to?: string },
     ) => {
       f.change({ structure: f.controller.view(7)!.revision + 100 });
       return f.controller.run(
         {
           id: "live",
-          method: action.key ? "tool.press" : "tool.click",
+          method: action.to ? "tool.drag" : action.key ? "tool.press" : "tool.click",
           params: {
             session_id: "shared",
             tab_id: 7,
             context_id: context,
-            ...(action.ref ? { ref: action.ref } : {}),
+            ...(action.to
+              ? { source: { ref: action.ref }, target: { ref: action.to } }
+              : action.ref
+                ? { ref: action.ref }
+                : {}),
             ...(action.key ? { key: action.key } : {}),
             ...(action.dependsOn ? { depends_on: action.dependsOn } : {}),
           },
@@ -620,7 +624,7 @@ describe("shared-tab dispatch fences", () => {
     /** One learning refusal, then `attempts` fresh-sync actions all succeed. */
     const learnsThenActs = async (
       build: (tick: number) => VomScene,
-      action: { ref?: string; dependsOn?: string[] },
+      action: { ref?: string; dependsOn?: string[]; to?: string },
       attempts = 2,
     ) => {
       live(build);
@@ -634,7 +638,7 @@ describe("shared-tab dispatch fences", () => {
     /** Every fresh-sync action keeps being refused without dispatching input. */
     const keepsRefusing = async (
       build: (tick: number) => VomScene,
-      action: { ref?: string; dependsOn?: string[]; key?: string },
+      action: { ref?: string; dependsOn?: string[]; key?: string; to?: string },
     ) => {
       live(build);
       refs();
@@ -649,6 +653,31 @@ describe("shared-tab dispatch fences", () => {
     };
     const clock = (tick: number) => `10:00:0${tick}`;
 
+    it("learns unrelated live text for a drag while protecting both endpoints", () =>
+      learnsThenActs(
+        page((tick) => [
+          [1, null, "main"],
+          [2, 1, "region", { name: "Source" }],
+          [3, 2, "button", { name: "Card" }],
+          [5, 1, "region", { name: "Destination" }],
+          [6, 5, "button", { name: "Drop here" }],
+          [4, 1, "span", { text: clock(tick) }],
+        ]),
+        { ref: "e3", to: "e6" },
+      ));
+    it("keeps a drag destination exact when dependencies mention only its source", () =>
+      keepsRefusing(
+        page((tick) => [
+          [1, null, "main"],
+          [2, 1, "region", { name: "Source" }],
+          [3, 2, "button", { name: "Card" }],
+          [5, 1, "region", { name: "Destination" }],
+          [6, 5, "button", { name: "Drop here" }],
+          [7, 5, "span", { text: clock(tick) }],
+        ]),
+        { ref: "e3", to: "e6", dependsOn: ["e3"] },
+      ));
+
     it("tolerates a clock beside a button directly inside <main>", () =>
       learnsThenActs(
         page((tick) => [

apps/extension/src/cooperation/controller.ts

@@ -7,7 +7,7 @@ import {
 import { resolveActionTarget } from "@/tools/interaction";
 import { captureVomObservation, handleSnapshot } from "@/tools/observation";
 import { type CdpRunner, chromeTabsApi, isRpcError, resolveTargetTab } from "@/tools/shared";
-import type { RequestFrame, RpcError } from "@/transport/types";
+import type { DragTarget, RequestFrame, RpcError } from "@/transport/types";
 import { guardedRunner } from "./action-guard";
 import { type ControlAuthority, ControlAuthorityError } from "./control-authority";
 import {
@@ -888,6 +888,8 @@ export class CooperationController implements ControlAuthority {
           ref?: string;
           selector?: string;
           depends_on?: string[];
+          source?: DragTarget;
+          target?: DragTarget;
         }
       | undefined;
     if (req.method === "tool.session_start") {
@@ -1029,6 +1031,7 @@ export class CooperationController implements ControlAuthority {
     let verifiedFrames = context?.frames;
     let evidenceTargets: { backendNodeId: number; frameId?: string }[] | undefined;
     let actionTarget: { backendNodeId: number; frameId?: string } | undefined;
+    let destinationTarget: typeof actionTarget;
     state.working = true;
     const intervention = new AbortController();
     state.interruption = intervention;
@@ -1137,6 +1140,13 @@ export class CooperationController implements ControlAuthority {
         return sameObservationEvidence(context.evidence!, evidence!);
       };
       const matches = (keys?: ReadonlySet<string>) => {
+        // A drag decision also depends on its destination. Explicit dependencies
+        // or learned live text must never relax the destination's own region.
+        if (
+          destinationTarget &&
+          !sameObservationEvidence(context.evidence!, evidence!, destinationTarget, keys)
+        )
+          return false;
         if (evidenceTargets) {
           return (
             keyboardRegionMatches() &&
@@ -1201,17 +1211,31 @@ export class CooperationController implements ControlAuthority {
         context.refs !== session.refStore.revision
       )
         return failure("This action requires the context returned by the latest cooperate sync");
-      if (params.ref || params.selector) {
+      const primaryTarget = req.method === "tool.drag" ? params.source : params;
+      if (primaryTarget?.ref || primaryTarget?.selector) {
         const resolved = await resolveActionTarget(
           guardedRunner(cdp),
           session,
           target,
-          params,
+          primaryTarget,
           "shared action",
+          req.method === "tool.drag",
         );
         if (isRpcError(resolved)) return resolved;
         actionTarget = resolved;
       }
+      if (req.method === "tool.drag" && (params.target?.ref || params.target?.selector)) {
+        const resolved = await resolveActionTarget(
+          guardedRunner(cdp),
+          session,
+          target,
+          params.target,
+          "drag destination",
+          true,
+        );
+        if (isRpcError(resolved)) return resolved;
+        destinationTarget = resolved;
+      }
       if (params.depends_on !== undefined) {
         if (
           !Array.isArray(params.depends_on) ||
@@ -1265,9 +1289,9 @@ export class CooperationController implements ControlAuthority {
             sent = true;
             nativeInputStarted ||= startsInput;
           },
-          verifyPointerTarget: async (node, point) => {
+          verifyPointerTarget: async (node, point, offset) => {
             try {
-              await pointerTargetValidator(cdp)(node, point);
+              await pointerTargetValidator(cdp)(node, point, offset);
             } catch (error) {
               revoked = error instanceof Error ? error.message : String(error);
               throw error;

apps/extension/src/cooperation/pointer-target.ts

@@ -1,86 +1,23 @@
-import { cdpTargetKey } from "@/browser-driver/frame-graph";
 import { resolveNodeGeometry } from "@/tools/frame-geometry";
-import { projectPolygon } from "@/tools/geometry";
-import { GeometryContext } from "@/tools/geometry/frame-context";
-import { type CdpRunner, isRpcError, sendToCdpTarget } from "@/tools/shared";
-import { pointBelongsToTarget } from "@/tools/target-hit";
+import { type CdpRunner, isRpcError } from "@/tools/shared";
+import { pointReachesTarget } from "@/tools/target-hit";
 
 export function pointerTargetValidator(
   cdp: CdpRunner,
 ): NonNullable<CdpRunner["verifyPointerTarget"]> {
-  return async (target, point) => {
+  return async (target, point, offset = { x: 0, y: 0 }) => {
     const geometry = await resolveNodeGeometry(cdp, target.tabId, {
       target: target.cdpTarget,
       backendNodeId: target.backendNodeId,
       frameId: target.frameId,
     });
     if (
       isRpcError(geometry) ||
-      Math.abs(geometry.actionPoint.x - point.x) > 2 ||
-      Math.abs(geometry.actionPoint.y - point.y) > 2
+      Math.abs(geometry.actionPoint.x + offset.x - point.x) > 2 ||
+      Math.abs(geometry.actionPoint.y + offset.y - point.y) > 2
     )
       throw new Error("The pointer target moved or disappeared; synchronize before clicking");
-    const objectGroup = `shared-hit-${crypto.randomUUID()}`;
-    try {
-      if (
-        !(await pointBelongsToTarget(
-          cdp,
-          target,
-          geometry.targetActionPoint.x,
-          geometry.targetActionPoint.y,
-          objectGroup,
-          false,
-        ))
-      )
-        throw new Error(
-          "The pointer no longer hits the observed target; synchronize before clicking",
-        );
-      if (target.frameId && target.cdpTarget.sessionId) {
-        const context = new GeometryContext(cdp, target.tabId);
-        const ancestry = await context.ancestry(target.frameId);
-        const projection = await context.targetProjection(target.frameId);
-        if (!ancestry || !projection) throw new Error("The pointer frame path is unavailable");
-        let local = geometry.targetActionPoint;
-        let edge = 0;
-        for (const child of ancestry) {
-          const parent = await context.frame(child.parentFrameId!);
-          if (!parent) throw new Error("The pointer frame path changed");
-          if (cdpTargetKey(child.target) === cdpTargetKey(parent.target)) continue;
-          const mapping = projection.edges[edge++];
-          if (!mapping || child.ownerBackendNodeId === undefined)
-            throw new Error("The pointer frame mapping is unavailable");
-          local = projectPolygon([local], mapping)[0];
-          // OOPIF hit testing stops at its owning iframe. Verify that owner in
-          // every parent target, rather than comparing unrelated frame ids.
-          try {
-            if (
-              !(await pointBelongsToTarget(
-                cdp,
-                {
-                  cdpTarget: parent.target,
-                  backendNodeId: child.ownerBackendNodeId,
-                  frameId: parent.frameId,
-                },
-                local.x,
-                local.y,
-                objectGroup,
-                false,
-              ))
-            )
-              throw new Error("The pointer target is covered; synchronize before clicking");
-          } finally {
-            await sendToCdpTarget(cdp, parent.target, "Runtime.releaseObjectGroup", {
-              objectGroup,
-            }).catch(() => {});
-          }
-        }
-        if (Math.abs(local.x - point.x) > 2 || Math.abs(local.y - point.y) > 2)
-          throw new Error("The pointer frame projection changed; synchronize before clicking");
-      }
-    } finally {
-      await sendToCdpTarget(cdp, target.cdpTarget, "Runtime.releaseObjectGroup", {
-        objectGroup,
-      }).catch(() => {});
-    }
+    if (!(await pointReachesTarget(cdp, target, point)))
+      throw new Error("The pointer target is covered or unavailable; synchronize before clicking");
   };
 }

apps/extension/src/debug/manager.ts

@@ -51,6 +51,7 @@ const ACTIONS = new Set([
   "tool.hover",
   "tool.evaluate",
   "tool.wheel",
+  "tool.drag",
   "tool.scroll_to",
   "tool.focus",
   "tool.blur",

apps/extension/src/entrypoints/audit/App.tsx

@@ -31,6 +31,7 @@ const methodKeys = {
   "tool.focus": "focus",
   "tool.blur": "blur",
   "tool.wheel": "scroll",
+  "tool.drag": "drag",
   "tool.scroll_to": "scroll",
   "tool.upload": "upload",
   "tool.download": "download",

apps/extension/src/session-manager/ref-store.ts

@@ -35,6 +35,11 @@ export type RefInput =
 export class RefStore {
   private map = new Map<string, RefEntry>();
   private generation = 0;
+  private observationToken = crypto.randomUUID();
+
+  get observationId(): string {
+    return this.observationToken;
+  }
   private readonly documents = new Map<number, number>();
 
   get revision(): number {
@@ -70,6 +75,7 @@ export class RefStore {
     for (const [ref, input] of entries) next.set(normaliseRef(ref), this.entry(input, generation));
     this.map = next;
     this.generation = generation;
+    this.observationToken = crypto.randomUUID();
   }
 
   set(
@@ -106,12 +112,16 @@ export class RefStore {
         changed = true;
       }
     }
-    if (changed) this.generation++;
+    if (changed) {
+      this.generation++;
+      this.observationToken = crypto.randomUUID();
+    }
   }
 
   clear(): void {
     this.generation++;
     this.map.clear();
+    this.observationToken = crypto.randomUUID();
   }
 
   entries(): IterableIterator<[string, RefEntry]> {