Late navigations, streamed planner answers, and declines that skip instead of stall

extension/README.md

@@ -41,8 +41,11 @@ Fast mode (Jev) is on by default.
   - Requests that find or read information use fast mode: Jev picks each click and scroll; the LLM writes typed text and the answer. When Jev is blocked, stuck or guessing, it asks the LLM for that one step and keeps driving, instead of handing the whole task over.
   - Requests with several separate parts ("when is X, and also when is Y") and requests that change something (buy, book, send, delete, submit) go to the LLM planner, which thinks and decides but hands legwork (opening results, emails, items; moving through lists) back to Jev as sub-goals with `delegate`.
   - Jev requests stay within its 32k-token context: crowded pages keep the elements most relevant to the task. If a Jev call fails, the LLM picks that step. The planner's older page views are trimmed to their text, so later steps stay fast.
-- Answers stream into the panel as they are written; Jev then checks them against every page it saw (are the facts on those pages, and is every part of the request answered?), and an answer that fails is redone (first by Jev looking closer, then by the model).
-- Before an irreversible click (buy, place order, checkout, send, delete, publish, unsubscribe...) the panel shows **"Hold on — this can't be undone"** with Allow and Cancel. Cancel ends the task there; say "go ahead" to continue.
+- Pages often navigate a moment after an action (a search that submits once its suggestions load). WebPilot notices when the page starts changing after it last looked, and looks again instead of clicking on the page being left.
+- Answers stream into the panel as they are written, including the planner's (it replies in plain text rather than through a tool call, whose arguments only arrive at the end); Jev then checks them against every page it saw (are the facts on those pages, and is every part of the request answered?), and an answer that fails is redone (first by Jev looking closer, then by the model).
+- Requests that only find or read information never send, buy or delete anything: those controls aren't even considered.
+- When a request does ask for something irreversible (buy, place order, checkout, send, delete, publish, unsubscribe...), the panel shows **"Hold on — this can't be undone"** before each such click, with Allow and Skip. Skip leaves that action undone (it isn't asked about again during the task) and WebPilot carries on with the rest, saying in the answer what it skipped. The stop button ends the whole task.
+- Sites are picked for your country, from your browser's language and time zone (walmart.ca rather than walmart.com in Canada).
 - WebPilot never types into password, one-time-code or card fields and never makes up usernames or other personal details. On a sign-in page your password manager has already filled in, clicking "Log in" asks first (**"Hold on — sign in?"**); otherwise it asks you to sign in yourself and then continue.
 - A follow-up like "no, it's on Gmail" or "try again" continues a request that didn't finish.
 

extension/sidepanel.css

@@ -265,6 +265,7 @@ button { font: inherit; color: inherit; }
 .confirm h3 { margin: 4px 0 4px; font-size: 13px; }
 .confirm p { margin: 0 0 10px; color: var(--wp-text-secondary); overflow-wrap: anywhere; }
 .confirm .action { color: var(--wp-text); font-family: var(--wp-font-mono); font-size: 12px; }
+.confirm .hint { margin: 8px 0 0; font-size: 11.5px; color: var(--wp-text-faint); }
 .row { display: flex; gap: 8px; }
 
 .btn {

extension/src/dev-bridge.ts

@@ -81,12 +81,20 @@ async function runCommand(engine: Engine, command: RunCommand) {
         return;
     }
     if (command.url) await waitForTabComplete(tabId);
+    let answerStarted = false;
     const hooks = {
         overrides: typeof command.fastMode === "boolean" ? { fastMode: command.fastMode } : undefined,
         // The task reports the confirmation itself (a "confirm" event); just answer it.
         confirm: async () => command.confirm === "allow",
         onEvent: (event: TaskEvent) => {
-            if (event.type === "answer-delta") return;
+            if (event.type === "answer-delta") {
+                // Only when the answer starts showing, for timing.
+                if (answerStarted) return;
+                answerStarted = true;
+                post("/bridge/event", { id: command.id, event: { type: "answer-start" } }).catch(() => {});
+                return;
+            }
+            if (event.type === "answer-reset") answerStarted = false;
             post("/bridge/event", { id: command.id, event }).catch(() => {});
         },
     };
@@ -96,6 +104,7 @@ async function runCommand(engine: Engine, command: RunCommand) {
         for (const message of [command.goal, ...(command.followUps || [])]) {
             if (turns.length) post("/bridge/event", { id: command.id, event: { type: "status", message: `Follow-up: ${message}` } }).catch(() => {});
             const turnStarted = Date.now();
+            answerStarted = false;
             const turn = await engine.run(tabId, message, hooks);
             turns.push({ user: message, answer: turn.answer || "", error: turn.error, mode: turn.mode, stats: turn.stats, durationMs: Date.now() - turnStarted });
         }

extension/src/dom-browser.ts

@@ -1,5 +1,6 @@
 /// <reference types="chrome" />
 
+import { STALE_PAGE_MESSAGE } from "../../lib/cdp/driver";
 import { axTreeToSnapshot } from "../../lib/cdp/snapshot";
 import type { TaskBrowser } from "../../lib/core/run-task";
 import { clickElement, pressKeyInPage, readPageText, snapshotPage, typeIntoElement, waitForQuiet } from "./page-scripts";
@@ -37,8 +38,21 @@ function refId(ref: string): number {
  * Cross-origin frames are not read.
  */
 export class DomBrowser implements TaskBrowser {
+    /** The URL of the last snapshot, to catch navigations that happen after it. */
+    private snapshotUrl = "";
+
     constructor(public readonly tabId: number) {}
 
+    /** Refuses refs from a snapshot of a page that is being left (see CdpBrowser). */
+    private async ensureSnapshotCurrent() {
+        const tab = await chrome.tabs.get(this.tabId);
+        if (tab.status !== "loading" && (!this.snapshotUrl || tab.url === this.snapshotUrl)) return;
+        const deadline = Date.now() + LOAD_TIMEOUT_MS;
+        while (Date.now() < deadline && (await chrome.tabs.get(this.tabId)).status === "loading") await sleep(50);
+        await sleep(SETTLE_MS);
+        throw new Error(STALE_PAGE_MESSAGE);
+    }
+
     private async exec<Args extends unknown[], Result>(func: (...args: Args) => Result, args: Args, world: "ISOLATED" | "MAIN" = "ISOLATED"): Promise<Awaited<Result>> {
         let results: Array<{ result?: unknown }>;
         try {
@@ -88,11 +102,13 @@ export class DomBrowser implements TaskBrowser {
 
     async snapshot(): Promise<string> {
         const page = await this.exec(snapshotPage, [MAX_NODES]);
+        this.snapshotUrl = page.url;
         return axTreeToSnapshot(page.nodes, page, { refPrefix: DOM_REF_PREFIX });
     }
 
     async click(ref: string): Promise<string> {
         const id = refId(ref);
+        await this.ensureSnapshotCurrent();
         await this.withSettle(async () => {
             const result = await this.exec(clickElement, [id]);
             if (!result.ok) throw new Error(result.error);
@@ -102,6 +118,7 @@ export class DomBrowser implements TaskBrowser {
 
     async type(ref: string, _element: string, text: string, submit: boolean): Promise<string> {
         const id = refId(ref);
+        await this.ensureSnapshotCurrent();
         await this.withSettle(async () => {
             const result = await this.exec(typeIntoElement, [id, text]);
             if (!result.ok) throw new Error(result.error);

extension/src/engine.ts

@@ -245,14 +245,16 @@ export class Engine {
                     hooks.confirm?.(action, url).then(settle, () => settle(false));
                 }),
                 onTrace: addTrace,
+                locale: { language: navigator.language, timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone },
                 onEvent: (event) => {
                     hooks.onEvent?.(event);
                     if (event.type !== "answer-delta" && event.type !== "answer-reset") {
                         const { type, ...data } = event;
                         addTrace(type, type === "done" ? { mode: event.mode } : data);
                     }
                     if (event.type === "step") {
-                        addStep({ lane: event.source, label: ACTION_LABELS[event.action] || event.action, detail: event.detail ? event.detail.slice(0, 140) : undefined });
+                        const label = ACTION_LABELS[event.action] || event.action;
+                        addStep({ lane: event.source, label: event.status ? `${event.status}: ${label}` : label, detail: event.detail ? event.detail.slice(0, 140) : undefined });
                     } else if (event.type === "handoff") {
                         addStep({ lane: "note", label: "handing to the model", detail: event.reason });
                     } else if (event.type === "status") {

extension/src/sidepanel.tsx

@@ -237,8 +237,9 @@ function ConfirmCard({ pending, onAnswer }: { pending: PendingConfirm; onAnswer:
             </p>
             <div className="row">
                 <button className="btn primary" onClick={() => onAnswer(true)}>Allow</button>
-                <button className="btn" onClick={() => onAnswer(false)}>Cancel</button>
+                <button className="btn" onClick={() => onAnswer(false)}>Skip</button>
             </div>
+            <p className="hint">Skip leaves this undone and carries on with the rest. The stop button ends the task.</p>
         </div>
     );
 }

lib/cdp/driver.ts

@@ -22,6 +22,8 @@ export interface CdpTransport {
 const NAVIGATION_START_WINDOW_MS = 400;
 const LOAD_TIMEOUT_MS = 12_000;
 const SETTLE_MS = 250;
+/** How long a navigation request may take to start loading before it's treated as same-document. */
+const NAVIGATION_REQUEST_GRACE_MS = 1500;
 const NAVIGATE_RESPONSE_TIMEOUT_MS = 10_000;
 
 const KEY_CODES: Record<string, { code: string; keyCode: number; text?: string }> = {
@@ -38,8 +40,18 @@ function sleep(ms: number) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+/** Thrown when refs from the last snapshot would act on a page that has since changed. */
+export const STALE_PAGE_MESSAGE = "The page changed since the last snapshot. Take a new snapshot.";
+
 export class CdpBrowser implements FastModeBrowser {
     private enabled = false;
+    private mainFrameId: string | null = null;
+    /** The main frame is loading a new document. */
+    private loading = false;
+    /** When the page asked to navigate; a request that never starts loading (same-document) expires. */
+    private navigationRequestedAt = 0;
+    /** The main frame started navigating after the last snapshot, so its refs may point at the old page. */
+    private navigatedSinceSnapshot = false;
 
     constructor(private transport: CdpTransport) {}
 
@@ -48,9 +60,48 @@ export class CdpBrowser implements FastModeBrowser {
         await this.transport.send("Page.enable");
         await this.transport.send("DOM.enable");
         await this.transport.send("Accessibility.enable");
+        const { frameTree } = await this.transport.send<{ frameTree: { frame: { id: string } } }>("Page.getFrameTree");
+        this.mainFrameId = frameTree.frame.id;
+        // Sites often navigate a moment after an action (a search that submits
+        // after its suggestions load). Tracking it lets clicks and typing refuse
+        // refs from a snapshot of the page being left.
+        this.transport.onEvent((method, params) => {
+            const frameId = typeof params.frameId === "string" ? params.frameId : undefined;
+            const frame = (params.frame || {}) as { id?: string; parentId?: string };
+            const main = frameId === undefined || frameId === this.mainFrameId;
+            if (method === "Page.frameRequestedNavigation" && main && params.disposition === "currentTab") {
+                this.navigationRequestedAt = Date.now();
+                this.navigatedSinceSnapshot = true;
+            } else if (method === "Page.frameStartedLoading" && main) {
+                this.loading = true;
+                this.navigatedSinceSnapshot = true;
+            } else if (method === "Page.frameNavigated" && !frame.parentId) {
+                if (frame.id) this.mainFrameId = frame.id;
+                this.navigatedSinceSnapshot = true;
+            } else if (method === "Page.navigatedWithinDocument" && main) {
+                this.navigationRequestedAt = 0;
+            } else if ((method === "Page.frameStoppedLoading" && main) || method === "Page.loadEventFired" || method === "Inspector.detached") {
+                this.loading = false;
+                this.navigationRequestedAt = 0;
+            }
+        });
         this.enabled = true;
     }
 
+    /**
+     * Before acting on a ref: if the page navigated since the snapshot the ref
+     * came from, wait for the new page and ask for a fresh snapshot instead of
+     * clicking on (or typing into) the page being left.
+     */
+    private async ensureSnapshotCurrent() {
+        if (!this.navigatedSinceSnapshot) return;
+        const deadline = Date.now() + LOAD_TIMEOUT_MS;
+        const pending = () => this.loading || (this.navigationRequestedAt > 0 && Date.now() - this.navigationRequestedAt < NAVIGATION_REQUEST_GRACE_MS);
+        while (pending() && Date.now() < deadline) await sleep(50);
+        await sleep(SETTLE_MS);
+        throw new Error(STALE_PAGE_MESSAGE);
+    }
+
     async pageInfo(): Promise<{ url: string; title: string }> {
         const { result } = await this.transport.send<{ result: { value?: { url: string; title: string } } }>("Runtime.evaluate", {
             expression: "({ url: location.href, title: document.title })",
@@ -61,6 +112,7 @@ export class CdpBrowser implements FastModeBrowser {
 
     async snapshot(): Promise<string> {
         await this.enable();
+        this.navigatedSinceSnapshot = false;
         const [{ nodes }, page] = await Promise.all([
             this.transport.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree"),
             this.pageInfo(),
@@ -122,6 +174,8 @@ export class CdpBrowser implements FastModeBrowser {
 
     async click(ref: string): Promise<string> {
         const backendNodeId = await this.resolve(ref);
+        await this.enable();
+        await this.ensureSnapshotCurrent();
         await this.withSettle(async () => {
             const point = await this.center(backendNodeId);
             if (!point) {
@@ -139,6 +193,8 @@ export class CdpBrowser implements FastModeBrowser {
 
     async type(ref: string, _element: string, text: string, submit: boolean): Promise<string> {
         const backendNodeId = await this.resolve(ref);
+        await this.enable();
+        await this.ensureSnapshotCurrent();
         await this.withSettle(async () => {
             await this.transport.send("DOM.scrollIntoViewIfNeeded", { backendNodeId }).catch(() => {});
             await this.transport.send("DOM.focus", { backendNodeId });