From caaf2b5178b228179f6db898c60cc9c086263c05 Mon Sep 17 00:00:00 2001 From: ahamSel <77988808+ahamSel@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:59:39 -0230 Subject: [PATCH 1/2] feat: catch late navigations; stream the planner's answer - Late navigations: sites often navigate a moment after an action (a search that submits once its suggestions load). Both drivers now notice when the page starts changing after the last snapshot and, instead of clicking or typing on the page being left, wait for the new page and ask for a fresh snapshot. Same-document navigations and subframe loads don't hold clicks up. Fast mode doesn't count such steps against its no-progress limit. - Streaming: the planner replies with its answer as plain text instead of a finish() call (Gemini delivers tool arguments in one piece at the end), and tool-chat turns are streamed and rebuilt exactly (content, tool calls, reasoning_details) for the next turn. Text written before a tool call is withdrawn. The dev bridge reports when an answer starts showing. Co-Authored-By: Claude Opus 5.5 --- extension/README.md | 3 +- extension/src/dev-bridge.ts | 11 +++- extension/src/dom-browser.ts | 17 ++++++ lib/cdp/driver.ts | 56 +++++++++++++++++ lib/core/run-task.ts | 16 +++-- lib/jev/fast-mode.ts | 5 +- lib/model-client.ts | 113 ++++++++++++++++++++++++++++------- tests/cdp-driver.test.ts | 72 ++++++++++++++++++++++ tests/model-client.test.ts | 43 +++++++++++++ tests/run-task.test.ts | 32 ++++++++-- 10 files changed, 333 insertions(+), 35 deletions(-) create mode 100644 tests/cdp-driver.test.ts diff --git a/extension/README.md b/extension/README.md index 54eac11..8056996 100644 --- a/extension/README.md +++ b/extension/README.md @@ -41,7 +41,8 @@ Fast mode (Jev) is on by default. - Requests that find or read information use fast mode: Jev picks each click and scroll; the LLM writes typed text and the answer. When Jev is blocked, stuck or guessing, it asks the LLM for that one step and keeps driving, instead of handing the whole task over. - Requests with several separate parts ("when is X, and also when is Y") and requests that change something (buy, book, send, delete, submit) go to the LLM planner, which thinks and decides but hands legwork (opening results, emails, items; moving through lists) back to Jev as sub-goals with `delegate`. - Jev requests stay within its 32k-token context: crowded pages keep the elements most relevant to the task. If a Jev call fails, the LLM picks that step. The planner's older page views are trimmed to their text, so later steps stay fast. -- Answers stream into the panel as they are written; Jev then checks them against every page it saw (are the facts on those pages, and is every part of the request answered?), and an answer that fails is redone (first by Jev looking closer, then by the model). +- Pages often navigate a moment after an action (a search that submits once its suggestions load). WebPilot notices when the page starts changing after it last looked, and looks again instead of clicking on the page being left. +- Answers stream into the panel as they are written, including the planner's (it replies in plain text rather than through a tool call, whose arguments only arrive at the end); Jev then checks them against every page it saw (are the facts on those pages, and is every part of the request answered?), and an answer that fails is redone (first by Jev looking closer, then by the model). - Before an irreversible click (buy, place order, checkout, send, delete, publish, unsubscribe...) the panel shows **"Hold on — this can't be undone"** with Allow and Cancel. Cancel ends the task there; say "go ahead" to continue. - WebPilot never types into password, one-time-code or card fields and never makes up usernames or other personal details. On a sign-in page your password manager has already filled in, clicking "Log in" asks first (**"Hold on — sign in?"**); otherwise it asks you to sign in yourself and then continue. - A follow-up like "no, it's on Gmail" or "try again" continues a request that didn't finish. diff --git a/extension/src/dev-bridge.ts b/extension/src/dev-bridge.ts index a1ea3a6..673a25a 100644 --- a/extension/src/dev-bridge.ts +++ b/extension/src/dev-bridge.ts @@ -81,12 +81,20 @@ async function runCommand(engine: Engine, command: RunCommand) { return; } if (command.url) await waitForTabComplete(tabId); + let answerStarted = false; const hooks = { overrides: typeof command.fastMode === "boolean" ? { fastMode: command.fastMode } : undefined, // The task reports the confirmation itself (a "confirm" event); just answer it. confirm: async () => command.confirm === "allow", onEvent: (event: TaskEvent) => { - if (event.type === "answer-delta") return; + if (event.type === "answer-delta") { + // Only when the answer starts showing, for timing. + if (answerStarted) return; + answerStarted = true; + post("/bridge/event", { id: command.id, event: { type: "answer-start" } }).catch(() => {}); + return; + } + if (event.type === "answer-reset") answerStarted = false; post("/bridge/event", { id: command.id, event }).catch(() => {}); }, }; @@ -96,6 +104,7 @@ async function runCommand(engine: Engine, command: RunCommand) { for (const message of [command.goal, ...(command.followUps || [])]) { if (turns.length) post("/bridge/event", { id: command.id, event: { type: "status", message: `Follow-up: ${message}` } }).catch(() => {}); const turnStarted = Date.now(); + answerStarted = false; const turn = await engine.run(tabId, message, hooks); turns.push({ user: message, answer: turn.answer || "", error: turn.error, mode: turn.mode, stats: turn.stats, durationMs: Date.now() - turnStarted }); } diff --git a/extension/src/dom-browser.ts b/extension/src/dom-browser.ts index 86f5c6c..33b785a 100644 --- a/extension/src/dom-browser.ts +++ b/extension/src/dom-browser.ts @@ -1,5 +1,6 @@ /// +import { STALE_PAGE_MESSAGE } from "../../lib/cdp/driver"; import { axTreeToSnapshot } from "../../lib/cdp/snapshot"; import type { TaskBrowser } from "../../lib/core/run-task"; import { clickElement, pressKeyInPage, readPageText, snapshotPage, typeIntoElement, waitForQuiet } from "./page-scripts"; @@ -37,8 +38,21 @@ function refId(ref: string): number { * Cross-origin frames are not read. */ export class DomBrowser implements TaskBrowser { + /** The URL of the last snapshot, to catch navigations that happen after it. */ + private snapshotUrl = ""; + constructor(public readonly tabId: number) {} + /** Refuses refs from a snapshot of a page that is being left (see CdpBrowser). */ + private async ensureSnapshotCurrent() { + const tab = await chrome.tabs.get(this.tabId); + if (tab.status !== "loading" && (!this.snapshotUrl || tab.url === this.snapshotUrl)) return; + const deadline = Date.now() + LOAD_TIMEOUT_MS; + while (Date.now() < deadline && (await chrome.tabs.get(this.tabId)).status === "loading") await sleep(50); + await sleep(SETTLE_MS); + throw new Error(STALE_PAGE_MESSAGE); + } + private async exec(func: (...args: Args) => Result, args: Args, world: "ISOLATED" | "MAIN" = "ISOLATED"): Promise> { let results: Array<{ result?: unknown }>; try { @@ -88,11 +102,13 @@ export class DomBrowser implements TaskBrowser { async snapshot(): Promise { const page = await this.exec(snapshotPage, [MAX_NODES]); + this.snapshotUrl = page.url; return axTreeToSnapshot(page.nodes, page, { refPrefix: DOM_REF_PREFIX }); } async click(ref: string): Promise { const id = refId(ref); + await this.ensureSnapshotCurrent(); await this.withSettle(async () => { const result = await this.exec(clickElement, [id]); if (!result.ok) throw new Error(result.error); @@ -102,6 +118,7 @@ export class DomBrowser implements TaskBrowser { async type(ref: string, _element: string, text: string, submit: boolean): Promise { const id = refId(ref); + await this.ensureSnapshotCurrent(); await this.withSettle(async () => { const result = await this.exec(typeIntoElement, [id, text]); if (!result.ok) throw new Error(result.error); diff --git a/lib/cdp/driver.ts b/lib/cdp/driver.ts index 3aa6e60..f7f31d7 100644 --- a/lib/cdp/driver.ts +++ b/lib/cdp/driver.ts @@ -22,6 +22,8 @@ export interface CdpTransport { const NAVIGATION_START_WINDOW_MS = 400; const LOAD_TIMEOUT_MS = 12_000; const SETTLE_MS = 250; +/** How long a navigation request may take to start loading before it's treated as same-document. */ +const NAVIGATION_REQUEST_GRACE_MS = 1500; const NAVIGATE_RESPONSE_TIMEOUT_MS = 10_000; const KEY_CODES: Record = { @@ -38,8 +40,18 @@ function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } +/** Thrown when refs from the last snapshot would act on a page that has since changed. */ +export const STALE_PAGE_MESSAGE = "The page changed since the last snapshot. Take a new snapshot."; + export class CdpBrowser implements FastModeBrowser { private enabled = false; + private mainFrameId: string | null = null; + /** The main frame is loading a new document. */ + private loading = false; + /** When the page asked to navigate; a request that never starts loading (same-document) expires. */ + private navigationRequestedAt = 0; + /** The main frame started navigating after the last snapshot, so its refs may point at the old page. */ + private navigatedSinceSnapshot = false; constructor(private transport: CdpTransport) {} @@ -48,9 +60,48 @@ export class CdpBrowser implements FastModeBrowser { await this.transport.send("Page.enable"); await this.transport.send("DOM.enable"); await this.transport.send("Accessibility.enable"); + const { frameTree } = await this.transport.send<{ frameTree: { frame: { id: string } } }>("Page.getFrameTree"); + this.mainFrameId = frameTree.frame.id; + // Sites often navigate a moment after an action (a search that submits + // after its suggestions load). Tracking it lets clicks and typing refuse + // refs from a snapshot of the page being left. + this.transport.onEvent((method, params) => { + const frameId = typeof params.frameId === "string" ? params.frameId : undefined; + const frame = (params.frame || {}) as { id?: string; parentId?: string }; + const main = frameId === undefined || frameId === this.mainFrameId; + if (method === "Page.frameRequestedNavigation" && main && params.disposition === "currentTab") { + this.navigationRequestedAt = Date.now(); + this.navigatedSinceSnapshot = true; + } else if (method === "Page.frameStartedLoading" && main) { + this.loading = true; + this.navigatedSinceSnapshot = true; + } else if (method === "Page.frameNavigated" && !frame.parentId) { + if (frame.id) this.mainFrameId = frame.id; + this.navigatedSinceSnapshot = true; + } else if (method === "Page.navigatedWithinDocument" && main) { + this.navigationRequestedAt = 0; + } else if ((method === "Page.frameStoppedLoading" && main) || method === "Page.loadEventFired" || method === "Inspector.detached") { + this.loading = false; + this.navigationRequestedAt = 0; + } + }); this.enabled = true; } + /** + * Before acting on a ref: if the page navigated since the snapshot the ref + * came from, wait for the new page and ask for a fresh snapshot instead of + * clicking on (or typing into) the page being left. + */ + private async ensureSnapshotCurrent() { + if (!this.navigatedSinceSnapshot) return; + const deadline = Date.now() + LOAD_TIMEOUT_MS; + const pending = () => this.loading || (this.navigationRequestedAt > 0 && Date.now() - this.navigationRequestedAt < NAVIGATION_REQUEST_GRACE_MS); + while (pending() && Date.now() < deadline) await sleep(50); + await sleep(SETTLE_MS); + throw new Error(STALE_PAGE_MESSAGE); + } + async pageInfo(): Promise<{ url: string; title: string }> { const { result } = await this.transport.send<{ result: { value?: { url: string; title: string } } }>("Runtime.evaluate", { expression: "({ url: location.href, title: document.title })", @@ -61,6 +112,7 @@ export class CdpBrowser implements FastModeBrowser { async snapshot(): Promise { await this.enable(); + this.navigatedSinceSnapshot = false; const [{ nodes }, page] = await Promise.all([ this.transport.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree"), this.pageInfo(), @@ -122,6 +174,8 @@ export class CdpBrowser implements FastModeBrowser { async click(ref: string): Promise { const backendNodeId = await this.resolve(ref); + await this.enable(); + await this.ensureSnapshotCurrent(); await this.withSettle(async () => { const point = await this.center(backendNodeId); if (!point) { @@ -139,6 +193,8 @@ export class CdpBrowser implements FastModeBrowser { async type(ref: string, _element: string, text: string, submit: boolean): Promise { const backendNodeId = await this.resolve(ref); + await this.enable(); + await this.ensureSnapshotCurrent(); await this.withSettle(async () => { await this.transport.send("DOM.scrollIntoViewIfNeeded", { backendNodeId }).catch(() => {}); await this.transport.send("DOM.focus", { backendNodeId }); diff --git a/lib/core/run-task.ts b/lib/core/run-task.ts index 92f3d21..7af94c5 100644 --- a/lib/core/run-task.ts +++ b/lib/core/run-task.ts @@ -443,7 +443,7 @@ When an earlier request did not finish, or the user corrects or adds to one ("no // Fast mode finishes after the first part; the planner splits the request // and hands each part to Jev. emit({ type: "status", message: "Several parts: planning, with Jev doing the legwork" }); - handoff = "This request asks for several separate things. Use delegate for each part (one concrete sub-goal at a time), read what Jev finds, then finish with every part answered."; + handoff = "This request asks for several separate things. Use delegate for each part (one concrete sub-goal at a time), read what Jev finds, then answer every part."; } // A second pass runs when the first stops on a page that doesn't support its answer // (e.g. a search results list): finishing there is ruled out, so Jev opens the item. @@ -491,14 +491,18 @@ When an earlier request did not finish, or the user corrects or adds to one ("no // 4. LLM planner on the same page. emit({ type: "status", message: "Thinking it through" }); - const tools = [...getBrowserToolDeclarations({ includeTabTools: false }), ...(jev ? [DELEGATE_TOOL] : [])]; + // The answer is a plain reply rather than a finish() call, so it streams: + // tool arguments arrive all at once at the end. + const tools = [...getBrowserToolDeclarations({ includeTabTools: false }).filter((tool) => tool.name !== "finish"), ...(jev ? [DELEGATE_TOOL] : [])]; const chat = llm.createToolChat({ model: config.navModel, tools, compactToolResponse: compactPlannerResult, + onText: streamAnswer, + onTextReset: () => emit({ type: "answer-reset" }), systemInstruction: `You are WebPilot, a browser agent working in the user's own browser tab. Today is ${todayLabel()}. Task: ${browserGoal} -Tools: observe() returns the page's actionable elements as [ref] descriptions plus page text. click({ref, element}) and type({ref, text, submit?}) act on refs from the latest observe(). navigate({url}), scroll({direction}), wait({seconds}). finish({result}) ends the task with the answer for the user.${jev ? ` +Tools: observe() returns the page's actionable elements as [ref] descriptions plus page text. click({ref, element}) and type({ref, text, submit?}) act on refs from the latest observe(). navigate({url}), scroll({direction}), wait({seconds}). When you have the answer, reply to the user with it as a normal message, not a tool call: that ends the task, and the user sees it as you write it.${jev ? ` delegate({goal}) hands a sub-goal to Jev, a fast navigation model. You think and decide; Jev does the legwork.` : ""} Rules:${jev ? ` @@ -511,7 +515,7 @@ Rules:${jev ? ` - ${CREDENTIALS_RULE} If the browser already filled them in, you may click the sign-in button; the user is asked to confirm. - Answer only with what pages in this task showed. Never fill gaps with general knowledge or typical patterns; say plainly what you couldn't find. Be concise and specific; include names, dates, prices and links when relevant. - Work fast. If a few searches or places turn up nothing, stop and say so rather than trying every variation. -- If you can't fully answer, finish with what you found, what you couldn't find, and where you looked.`, +- If you can't fully answer, reply with what you found, what you couldn't find, and where you looked.`, }); let lastPage: PageModel = parsePage(await browser.snapshot(), PLANNER_TEXT_CHARS); @@ -598,7 +602,7 @@ Rules:${jev ? ` if (!nudged && Date.now() - started > PLANNER_WRAP_UP_MS && outputs.length) { nudged = true; const last = outputs[outputs.length - 1].functionResponse; - last.response = { ...last.response, note: "This has taken over a minute. Unless the answer is one step away, call finish now with what you found and what you could not find." }; + last.response = { ...last.response, note: "This has taken over a minute. Unless the answer is one step away, reply now with what you found and what you could not find." }; } response = await chat.sendMessage(outputs); stats.llmCalls++; @@ -606,7 +610,7 @@ Rules:${jev ? ` // Out of steps: still give the user what was learned. await checkCancelled(); - const wrapUp = await chat.sendMessage("You are out of steps. Do not call any more tools except finish. Give the user the final answer now: what you found, what you could not find, and where you looked."); + const wrapUp = await chat.sendMessage("You are out of steps. Do not call any more tools. Reply to the user with the final answer now: what you found, what you could not find, and where you looked."); stats.llmCalls++; const finishCall = wrapUp.functionCalls.find((call) => call.name === "finish"); const summary = String((finishCall?.args as Record | undefined)?.result || wrapUp.text || "").trim(); diff --git a/lib/jev/fast-mode.ts b/lib/jev/fast-mode.ts index 304805a..55c2807 100644 --- a/lib/jev/fast-mode.ts +++ b/lib/jev/fast-mode.ts @@ -521,9 +521,10 @@ export async function runFastMode(options: FastModeOptions): Promise(); used.add(actionKey(operation, target, before.url)); diff --git a/lib/model-client.ts b/lib/model-client.ts index 212a4a9..47815bf 100644 --- a/lib/model-client.ts +++ b/lib/model-client.ts @@ -101,6 +101,13 @@ export interface ToolChatOptions { */ compactToolResponse?: (response: Record) => Record; keepFullToolResults?: number; + /** + * Streams replies (OpenRouter): text the model writes instead of calling a + * tool, such as a final answer, is reported as it arrives. `onTextReset` + * withdraws text if the model then turns to a tool call. + */ + onText?: (text: string) => void; + onTextReset?: () => void; } export interface ModelClient { @@ -289,23 +296,15 @@ async function postChatCompletion( return json; } -/** - * Streams a chat completion (server-sent events), calling `onDelta` with each - * piece of text as it arrives, and resolves with the full text. - */ -async function streamChatCompletion( - config: RuntimeModelConfig, - body: Record, - onDelta: (text: string) => void -): Promise { +/** The `delta` of each server-sent event of a streamed chat completion. */ +async function* streamDeltas(config: RuntimeModelConfig, body: Record): AsyncGenerator> { const response = await openChatCompletion(config, { ...body, stream: true }); - if (!response.body) return ""; + if (!response.body) return; const reader = response.body.pipeThrough(new TextDecoderStream()).getReader(); let buffer = ""; - let full = ""; for (;;) { const { value, done } = await reader.read(); - if (done) break; + if (done) return; buffer += value; let newline: number; while ((newline = buffer.indexOf("\n")) >= 0) { @@ -314,7 +313,7 @@ async function streamChatCompletion( // Blank lines separate events; ": ..." lines are keep-alive comments. if (!line.startsWith("data:")) continue; const data = line.slice(5).trim(); - if (data === "[DONE]") return full; + if (data === "[DONE]") return; let chunk: Record; try { chunk = JSON.parse(data) as Record; @@ -324,16 +323,89 @@ async function streamChatCompletion( const error = chunk.error as { message?: unknown } | undefined; if (error) throw new Error(`${providerDisplayName(config.provider)} request failed: ${String(error.message || "stream error")}`); const choice = Array.isArray(chunk.choices) ? chunk.choices[0] as Record : undefined; - const delta = choice?.delta as { content?: unknown } | undefined; - if (typeof delta?.content === "string" && delta.content) { - full += delta.content; - onDelta(delta.content); - } + if (choice?.delta && typeof choice.delta === "object") yield choice.delta as Record; + } + } +} + +/** + * Streams a chat completion (server-sent events), calling `onDelta` with each + * piece of text as it arrives, and resolves with the full text. + */ +async function streamChatCompletion( + config: RuntimeModelConfig, + body: Record, + onDelta: (text: string) => void +): Promise { + let full = ""; + for await (const delta of streamDeltas(config, body)) { + if (typeof delta.content === "string" && delta.content) { + full += delta.content; + onDelta(delta.content); } } return full; } +/** Streamed reasoning items with the same index are pieces of one item. */ +function mergeReasoningItem(items: Array>, item: Record) { + const existing = typeof item.index === "number" ? items.find((candidate) => candidate.index === item.index && candidate.type === item.type) : undefined; + if (!existing) { + items.push({ ...item }); + return; + } + for (const [key, value] of Object.entries(item)) { + if (typeof value === "string" && typeof existing[key] === "string" && ["text", "data", "summary"].includes(key)) existing[key] = `${existing[key]}${value}`; + else if (existing[key] === undefined || existing[key] === null) existing[key] = value; + } +} + +/** + * Streams one tool-chat turn and rebuilds the message the non-streaming API + * returns (content, tool_calls, reasoning_details), so the conversation carries + * on the same way. Text is reported as it arrives until the model starts a tool + * call; text already reported by then is withdrawn with `onTextReset`. + */ +async function streamToolTurn( + config: RuntimeModelConfig, + body: Record, + onText: (text: string) => void, + onTextReset: () => void +): Promise> { + let content = ""; + let reported = false; + const toolCalls: Array<{ id: string; type: "function"; function: { name: string; arguments: string } }> = []; + const reasoning: Array> = []; + for await (const delta of streamDeltas(config, body)) { + if (Array.isArray(delta.reasoning_details)) { + for (const item of delta.reasoning_details) if (item && typeof item === "object") mergeReasoningItem(reasoning, item as Record); + } + if (Array.isArray(delta.tool_calls)) { + if (!toolCalls.length && reported) onTextReset(); + for (const raw of delta.tool_calls as Array>) { + const index = typeof raw.index === "number" ? raw.index : toolCalls.length; + const call = toolCalls[index] || (toolCalls[index] = { id: "", type: "function", function: { name: "", arguments: "" } }); + const fn = (raw.function || {}) as { name?: unknown; arguments?: unknown }; + if (typeof raw.id === "string" && raw.id) call.id = raw.id; + if (typeof fn.name === "string") call.function.name += fn.name; + if (typeof fn.arguments === "string") call.function.arguments += fn.arguments; + } + } + if (typeof delta.content === "string" && delta.content) { + content += delta.content; + if (!toolCalls.length) { + reported = true; + onText(delta.content); + } + } + } + return { + content, + ...(toolCalls.length ? { tool_calls: toolCalls.filter(Boolean) } : {}), + ...(reasoning.length ? { reasoning_details: reasoning } : {}), + }; +} + function firstChoiceMessage(response: Record): Record { const choice = Array.isArray(response.choices) ? response.choices[0] as Record : undefined; return choice && typeof choice.message === "object" && choice.message @@ -403,8 +475,9 @@ class ChatCompletionsToolChat implements ToolChat { if (this.config.provider === "openrouter") body.tool_choice = "auto"; } - const response = await postChatCompletion(this.config, body); - const messageObj = firstChoiceMessage(response); + const messageObj = this.options.onText && this.config.provider === "openrouter" + ? await streamToolTurn(this.config, body, this.options.onText, this.options.onTextReset || (() => {})) + : firstChoiceMessage(await postChatCompletion(this.config, body)); const toolCallsRaw = Array.isArray(messageObj.tool_calls) ? messageObj.tool_calls as Array> : []; const text = normalizeContent(messageObj.content); const reasoningDetails = Array.isArray(messageObj.reasoning_details) && messageObj.reasoning_details.length diff --git a/tests/cdp-driver.test.ts b/tests/cdp-driver.test.ts new file mode 100644 index 0000000..dae740a --- /dev/null +++ b/tests/cdp-driver.test.ts @@ -0,0 +1,72 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { CdpBrowser, STALE_PAGE_MESSAGE, type CdpTransport } from "../lib/cdp/driver"; + +/** A CDP connection to a one-button page; `emit` plays browser events. */ +function fakeTab() { + const sent: string[] = []; + const listeners = new Set<(method: string, params: Record) => void>(); + const transport: CdpTransport = { + async send(method: string): Promise { + sent.push(method); + const replies: Record = { + "Page.getFrameTree": { frameTree: { frame: { id: "main" } } }, + "Accessibility.getFullAXTree": { + nodes: [ + { nodeId: "1", role: { value: "RootWebArea" }, childIds: ["2"] }, + { nodeId: "2", parentId: "1", role: { value: "button" }, name: { value: "Search" }, backendDOMNodeId: 5, childIds: [] }, + ], + }, + "Runtime.evaluate": { result: { value: { url: "https://shop.example/", title: "Shop" } } }, + "DOM.getBoxModel": { model: { content: [0, 0, 10, 0, 10, 10, 0, 10] } }, + }; + return (replies[method] || {}) as T; + }, + onEvent(listener) { + listeners.add(listener); + return () => listeners.delete(listener); + }, + }; + const emit = (method: string, params: Record = {}) => { + for (const listener of listeners) listener(method, params); + }; + return { browser: new CdpBrowser(transport), sent, emit }; +} + +test("a click on a page that started navigating after the snapshot waits and asks for a new snapshot", async () => { + const { browser, sent, emit } = fakeTab(); + await browser.snapshot(); + // The site navigates on its own (e.g. a search submitting after its suggestions load). + emit("Page.frameStartedLoading", { frameId: "main" }); + setTimeout(() => emit("Page.frameStoppedLoading", { frameId: "main" }), 100); + + const started = Date.now(); + await assert.rejects(browser.click("b5"), new RegExp(STALE_PAGE_MESSAGE.replace(/\./g, "\\."))); + assert.ok(Date.now() - started >= 100, "waited for the new page to load"); + assert.ok(!sent.includes("Input.dispatchMouseEvent"), "nothing was clicked on the page being left"); + + // After a fresh snapshot the same kind of click goes through. + await browser.snapshot(); + await browser.click("b5"); + assert.ok(sent.includes("Input.dispatchMouseEvent")); +}); + +test("subframe loads and same-document navigations don't hold up clicks", async () => { + const { browser, sent, emit } = fakeTab(); + await browser.snapshot(); + emit("Page.frameStartedLoading", { frameId: "ad-frame" }); + emit("Page.navigatedWithinDocument", { frameId: "main", url: "https://shop.example/#reviews" }); + await browser.click("b5"); + assert.ok(sent.includes("Input.dispatchMouseEvent")); +}); + +test("a navigation request that never loads a new document expires quickly", async () => { + const { browser, emit } = fakeTab(); + await browser.snapshot(); + emit("Page.frameRequestedNavigation", { frameId: "main", disposition: "currentTab", reason: "anchorClick", url: "https://shop.example/#top" }); + emit("Page.navigatedWithinDocument", { frameId: "main", url: "https://shop.example/#top" }); + const started = Date.now(); + // The page did change after the snapshot, so a fresh look is still asked for, without waiting on a load. + await assert.rejects(browser.click("b5"), /Take a new snapshot/); + assert.ok(Date.now() - started < 1000); +}); diff --git a/tests/model-client.test.ts b/tests/model-client.test.ts index b122a5c..cce4cd2 100644 --- a/tests/model-client.test.ts +++ b/tests/model-client.test.ts @@ -182,6 +182,49 @@ test("tool chat compacts older tool results but keeps the latest ones whole", as ]); }); +test("streamed tool turns report text as written and rebuild tool calls and reasoning for the next turn", async () => { + const sse = (deltas: Array>) => deltas.map((delta) => `data: ${JSON.stringify({ choices: [{ delta }] })}\n\n`).join("") + "data: [DONE]\n\n"; + const bodies: Array> = []; + const replies = [ + // Turn 1: a stray remark, then a tool call split across chunks. + sse([ + { reasoning_details: [{ type: "reasoning.text", text: "Need the ", index: 0 }] }, + { reasoning_details: [{ type: "reasoning.text", text: "page first.", index: 0 }] }, + { content: "Let me look." }, + { tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "observe", arguments: "" } }] }, + { tool_calls: [{ index: 0, function: { arguments: "{\"full\":" } }] }, + { tool_calls: [{ index: 0, function: { arguments: "true}" } }] }, + ]), + // Turn 2: the answer, in pieces. + sse([{ content: "The cheapest " }, { content: "tent is $89.99." }]), + ]; + globalThis.fetch = (async (_input: string | URL | Request, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body || "{}"))); + return new Response(replies.shift(), { status: 200, headers: { "Content-Type": "text/event-stream" } }); + }) as typeof fetch; + + const shown: string[] = []; + const chat = createModelClient(openRouterConfig()).createToolChat({ + model: "google/gemini-3.8-flash", + systemInstruction: "system", + tools: getBrowserToolDeclarations(), + onText: (text) => shown.push(text), + onTextReset: () => shown.push(""), + }); + const first = await chat.sendMessage("cheapest tent?"); + assert.deepEqual(first.functionCalls, [{ id: "call_1", name: "observe", args: { full: true } }]); + assert.deepEqual(shown, ["Let me look.", ""], "text before a tool call is withdrawn"); + + const second = await chat.sendMessage([{ functionResponse: { name: "observe", response: { ok: true } } }]); + assert.equal(second.text, "The cheapest tent is $89.99."); + assert.deepEqual(shown.slice(2), ["The cheapest ", "tent is $89.99."]); + + assert.equal(bodies[0].stream, true); + const replayed = (bodies[1].messages as Array>).find((message) => message.role === "assistant"); + assert.deepEqual(replayed?.reasoning_details, [{ type: "reasoning.text", text: "Need the page first.", index: 0 }]); + assert.deepEqual(replayed?.tool_calls, [{ id: "call_1", type: "function", function: { name: "observe", arguments: "{\"full\":true}" } }]); +}); + test("Ollama requests omit OpenRouter-only fields", async () => { const requests = mockFetch([{ body: { choices: [{ message: { content: "hi" } }] } }]); const client = createModelClient(resolveRuntimeModelConfig({ provider: "ollama", navModel: "qwen3:8b" })); diff --git a/tests/run-task.test.ts b/tests/run-task.test.ts index a15e018..ea07796 100644 --- a/tests/run-task.test.ts +++ b/tests/run-task.test.ts @@ -32,13 +32,32 @@ function fakeBrowser(log: string[]): TaskBrowser { }; } +/** + * A chat completion reply, as JSON or (for streamed requests, like the + * planner's) as server-sent events split the way OpenRouter sends them. + */ +function chatResponse(message: Record, stream: boolean): Response { + if (!stream) return new Response(JSON.stringify({ choices: [{ message }] }), { status: 200, headers: { "Content-Type": "application/json" } }); + const deltas: Array> = []; + if (Array.isArray(message.tool_calls)) { + (message.tool_calls as Array>).forEach((call, index) => { + const fn = call.function as { name: string; arguments: string }; + deltas.push({ tool_calls: [{ index, id: call.id, type: "function", function: { name: fn.name, arguments: "" } }] }); + deltas.push({ tool_calls: [{ index, function: { arguments: fn.arguments } }] }); + }); + } + const content = typeof message.content === "string" ? message.content : ""; + for (let at = 0; at < content.length; at += 12) deltas.push({ content: content.slice(at, at + 12) }); + const events = deltas.map((delta) => `data: ${JSON.stringify({ choices: [{ delta }] })}\n\n`).join("") + "data: [DONE]\n\n"; + return new Response(events, { status: 200, headers: { "Content-Type": "text/event-stream" } }); +} + /** Routes Jev decisions and chat completions to canned responses. */ function mockServices(chatReplies: Array>) { - globalThis.fetch = (async (input: string | URL | Request) => { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { const url = String(input); - const body = url.endsWith("/systemone") - ? { model: "jev", answers: { needs_browser: { type: "noul", noul: 0.95 }, parallel_sites: { type: "noul", noul: 0.02 }, changes_something: { type: "noul", noul: 0.95 }, multiple_parts: { type: "noul", noul: 0.02 } }, usage: {} } - : { choices: [{ message: chatReplies.shift() || { content: "done" } }] }; + if (!url.endsWith("/systemone")) return chatResponse(chatReplies.shift() || { content: "done" }, JSON.parse(String(init?.body || "{}")).stream === true); + const body = { model: "jev", answers: { needs_browser: { type: "noul", noul: 0.95 }, parallel_sites: { type: "noul", noul: 0.02 }, changes_something: { type: "noul", noul: 0.95 }, multiple_parts: { type: "noul", noul: 0.02 } }, usage: {} }; return new Response(JSON.stringify(body), { status: 200, headers: { "Content-Type": "application/json" } }); }) as typeof fetch; } @@ -135,12 +154,13 @@ test("the planner splits a multi-part request and delegates each part to Jev", a body = { model: "jev", answers: { needs_browser: { type: "noul", noul: 0.95 }, parallel_sites: { type: "noul", noul: 0.02 }, changes_something: { type: "noul", noul: 0.02 }, multiple_parts: { type: "noul", noul: 0.9 } }, usage: {} }; } } else { - body = { choices: [{ message: chatReplies.shift() || { content: "done" } }] }; + return chatResponse(chatReplies.shift() || { content: "done" }, JSON.parse(String(init?.body || "{}")).stream === true); } return new Response(JSON.stringify(body), { status: 200, headers: { "Content-Type": "application/json" } }); }) as typeof fetch; const events: string[] = []; + let streamed = ""; const result = await runTask({ goal: "what does the tent at https://shop.example/p/tent cost, and how much does it weigh?", config: resolveRuntimeModelConfig({ provider: "openrouter", apiKey: "sk-or-test", fastMode: true }), @@ -148,10 +168,12 @@ test("the planner splits a multi-part request and delegates each part to Jev", a confirm: async () => false, onEvent: (event) => { if (event.type === "step") events.push(`${event.source} ${event.action} ${event.detail || ""}`.trim()); + if (event.type === "answer-delta") streamed += event.text; }, }); assert.equal(result.mode, "planner"); + assert.equal(streamed, result.answer, "the planner's answer streamed in as it was written"); assert.match(result.answer, /89\.99.*2\.1 kg/); assert.deepEqual(fastModeTasks, ["find the tent's price", "find the tent's weight"], "each part went to Jev as its own sub-goal"); assert.deepEqual(events.filter((event) => event.includes("delegate")), ["llm delegate find the tent's price", "llm delegate find the tent's weight"]); From 0372529af9d0cc3e27d860a3820243fdbf0df455 Mon Sep 17 00:00:00 2001 From: ahamSel <77988808+ahamSel@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:12:42 -0230 Subject: [PATCH 2/2] fix: declines skip the action and the task carries on; find-only requests never send or buy From a real Facebook Marketplace run ("find me rent downtown under $1000"): Jev tried to click "Send message to " three times. Each decline was honored at the click (nothing was clicked), but fast mode's step loop swallowed the decline as an ordinary failed step, so it moved on and asked again; the flight log showed the steps as plain clicks. 67s, 2 listings. - Decline = skip: the action isn't done, isn't asked about again during the task (task-wide list), and the task carries on; the answer says what was skipped. The stop button ends the task. The card says Allow / Skip. - Requests that only find or read never send, buy or delete: those controls aren't offered to Jev, and the planner refuses them without asking. - A Stop during a confirmation ends the task instead of failing one step. - Flight log: "declined:", "skipped:", "failed:" steps. - A field the task gives nothing to type into ("Send seller a message") is skipped instead of handing the task to the planner. - Out of steps with pages read, fast mode answers from them (checked) before the planner; option-finding answers list the best several matches; pages keep 4k characters of text for the answer (Jev still sees 1.5k). - A model turn that fails mid-stream (provider abort) is withdrawn and retried once; so are errors inside 200 responses. - Sites are picked for the user's country (browser language and time zone). - Traces record each Jev decision's page (title, URL, text start) and top four options with probabilities. Same Marketplace request now: 16.6s, no prompts, a detailed best match plus 15 more listings under $1000. Co-Authored-By: Claude Opus 5.5 --- extension/README.md | 4 +- extension/sidepanel.css | 1 + extension/src/engine.ts | 4 +- extension/src/sidepanel.tsx | 3 +- lib/core/run-task.ts | 86 ++++++++++++++++------------ lib/jev/fast-mode.ts | 111 +++++++++++++++++++++++++++++------- lib/jev/gates.ts | 2 +- lib/model-client.ts | 53 ++++++++++++++--- scripts/extension-smoke.ts | 16 +++--- tests/jev.test.ts | 51 +++++++++++++++++ tests/model-client.test.ts | 26 +++++++++ tests/run-task.test.ts | 56 +++++++++++++++--- 12 files changed, 325 insertions(+), 88 deletions(-) diff --git a/extension/README.md b/extension/README.md index 8056996..6edb4a8 100644 --- a/extension/README.md +++ b/extension/README.md @@ -43,7 +43,9 @@ Fast mode (Jev) is on by default. - Jev requests stay within its 32k-token context: crowded pages keep the elements most relevant to the task. If a Jev call fails, the LLM picks that step. The planner's older page views are trimmed to their text, so later steps stay fast. - Pages often navigate a moment after an action (a search that submits once its suggestions load). WebPilot notices when the page starts changing after it last looked, and looks again instead of clicking on the page being left. - Answers stream into the panel as they are written, including the planner's (it replies in plain text rather than through a tool call, whose arguments only arrive at the end); Jev then checks them against every page it saw (are the facts on those pages, and is every part of the request answered?), and an answer that fails is redone (first by Jev looking closer, then by the model). -- Before an irreversible click (buy, place order, checkout, send, delete, publish, unsubscribe...) the panel shows **"Hold on — this can't be undone"** with Allow and Cancel. Cancel ends the task there; say "go ahead" to continue. +- Requests that only find or read information never send, buy or delete anything: those controls aren't even considered. +- When a request does ask for something irreversible (buy, place order, checkout, send, delete, publish, unsubscribe...), the panel shows **"Hold on — this can't be undone"** before each such click, with Allow and Skip. Skip leaves that action undone (it isn't asked about again during the task) and WebPilot carries on with the rest, saying in the answer what it skipped. The stop button ends the whole task. +- Sites are picked for your country, from your browser's language and time zone (walmart.ca rather than walmart.com in Canada). - WebPilot never types into password, one-time-code or card fields and never makes up usernames or other personal details. On a sign-in page your password manager has already filled in, clicking "Log in" asks first (**"Hold on — sign in?"**); otherwise it asks you to sign in yourself and then continue. - A follow-up like "no, it's on Gmail" or "try again" continues a request that didn't finish. diff --git a/extension/sidepanel.css b/extension/sidepanel.css index 9694fde..74f1082 100644 --- a/extension/sidepanel.css +++ b/extension/sidepanel.css @@ -265,6 +265,7 @@ button { font: inherit; color: inherit; } .confirm h3 { margin: 4px 0 4px; font-size: 13px; } .confirm p { margin: 0 0 10px; color: var(--wp-text-secondary); overflow-wrap: anywhere; } .confirm .action { color: var(--wp-text); font-family: var(--wp-font-mono); font-size: 12px; } +.confirm .hint { margin: 8px 0 0; font-size: 11.5px; color: var(--wp-text-faint); } .row { display: flex; gap: 8px; } .btn { diff --git a/extension/src/engine.ts b/extension/src/engine.ts index 7a46a75..424f2fb 100644 --- a/extension/src/engine.ts +++ b/extension/src/engine.ts @@ -245,6 +245,7 @@ export class Engine { hooks.confirm?.(action, url).then(settle, () => settle(false)); }), onTrace: addTrace, + locale: { language: navigator.language, timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone }, onEvent: (event) => { hooks.onEvent?.(event); if (event.type !== "answer-delta" && event.type !== "answer-reset") { @@ -252,7 +253,8 @@ export class Engine { addTrace(type, type === "done" ? { mode: event.mode } : data); } if (event.type === "step") { - addStep({ lane: event.source, label: ACTION_LABELS[event.action] || event.action, detail: event.detail ? event.detail.slice(0, 140) : undefined }); + const label = ACTION_LABELS[event.action] || event.action; + addStep({ lane: event.source, label: event.status ? `${event.status}: ${label}` : label, detail: event.detail ? event.detail.slice(0, 140) : undefined }); } else if (event.type === "handoff") { addStep({ lane: "note", label: "handing to the model", detail: event.reason }); } else if (event.type === "status") { diff --git a/extension/src/sidepanel.tsx b/extension/src/sidepanel.tsx index 1468566..bf20718 100644 --- a/extension/src/sidepanel.tsx +++ b/extension/src/sidepanel.tsx @@ -237,8 +237,9 @@ function ConfirmCard({ pending, onAnswer }: { pending: PendingConfirm; onAnswer:

- +
+

Skip leaves this undone and carries on with the rest. The stop button ends the task.

); } diff --git a/lib/core/run-task.ts b/lib/core/run-task.ts index 7af94c5..2edf219 100644 --- a/lib/core/run-task.ts +++ b/lib/core/run-task.ts @@ -12,7 +12,7 @@ import { createModelClient, resolveJevConfig, type RuntimeModelConfig, type ToolResponsePart } from "../model-client"; import { extractExplicitUrls } from "../goal-urls"; -import { runFastMode, summarizeFastModeForPlanner, writeFastModeAnswer, type FastModeBrowser, type FastModeResult, type FastModeStep } from "../jev/fast-mode"; +import { DECLINED_OUTCOME, NOTHING_TO_TYPE_OUTCOME, runFastMode, summarizeFastModeForPlanner, writeFastModeAnswer, type FastModeBrowser, type FastModeResult, type FastModeStep } from "../jev/fast-mode"; import { jevCheckAnswer, jevPreflight } from "../jev/gates"; import { describeElement, parsePage, selectCandidates, type PageModel } from "../jev/page"; import { CREDENTIALS_RULE, UNTRUSTED_CONTENT_RULE, irreversibleAction, isSecretField, signInAction } from "../safety"; @@ -27,7 +27,8 @@ export interface TaskBrowser extends FastModeBrowser { export type TaskEvent = | { type: "status"; message: string } - | { type: "step"; source: "jev" | "llm"; action: string; detail?: string; url?: string } + /** `status`: the step wasn't done: the user declined it, WebPilot skipped it (not asked for), or it failed. */ + | { type: "step"; source: "jev" | "llm"; action: string; detail?: string; url?: string; status?: "declined" | "skipped" | "failed" } | { type: "handoff"; reason: string } | { type: "confirm"; action: string; url: string } /** Streamed answer text; `answer-reset` discards it when the answer is redone. */ @@ -56,6 +57,8 @@ export interface RunTaskOptions { * New Tab page) and no site is obvious. `{query}` is replaced with the request. */ fallbackSearchUrl?: string; + /** The user's browser language and time zone, so sites are picked for their country (walmart.ca, not .com). */ + locale?: { language?: string; timeZone?: string }; } /** @@ -86,14 +89,6 @@ export interface RunTaskResult { durationMs: number; } -/** The user cancelled an irreversible action; the task ends there. */ -class ActionDeclinedError extends Error { - constructor(public action: string) { - super(`Declined ${action}`); - this.name = "ActionDeclinedError"; - } -} - export class TaskCancelledError extends Error { constructor(public stats: { steps: number; jevCalls: number; llmCalls: number } = { steps: 0, jevCalls: 0, llmCalls: 0 }, public memory?: TaskMemory) { super("Stopped by user."); @@ -136,6 +131,13 @@ const DELEGATE_TOOL = { const PLANNER_WRAP_UP_MS = 60_000; const FAST_ANSWER_ESCALATE_BELOW = 0.3; +/** Where the user is, from their browser settings, so sites and results suit their country. */ +function describeLocale(locale: RunTaskOptions["locale"]): string { + if (!locale?.language && !locale?.timeZone) return ""; + const parts = [locale.language ? `language ${locale.language}` : "", locale.timeZone ? `time zone ${locale.timeZone}` : ""].filter(Boolean).join(", "); + return `The user's browser is set to ${parts}: prefer their country's version of a site (such as walmart.ca or amazon.ca for Canada) and local results, unless they name another.`; +} + /** "Monday, September 28, 2026": lets answers and checks resolve "today", "this week", "recent". */ export function todayLabel(date = new Date()): string { return date.toLocaleDateString("en-US", { weekday: "long", year: "numeric", month: "long", day: "numeric" }); @@ -227,17 +229,6 @@ export async function runTask(options: RunTaskOptions): Promise { await recordEnd(); throw new TaskCancelledError({ ...stats }, memory); } - if (error instanceof ActionDeclinedError) { - // Cancel on a confirmation ends the task, remembering how far it got so a - // follow-up like "ok, go ahead" can continue. - const page = await options.browser.pageInfo().catch(() => ({ url: "", title: "" })); - memory.finalUrl = page.url || undefined; - memory.finalTitle = page.title || undefined; - const done = memory.actions.slice(-3).join("; "); - const answer = `Okay, I stopped before clicking ${error.action}, so nothing was submitted.${done ? ` Up to that point: ${done}.` : ""} Say "go ahead" if you want me to continue.`; - options.onEvent?.({ type: "done", answer, mode: "planner" }); - return { answer, mode: "planner", memory, ...stats, durationMs: Date.now() - started }; - } await recordEnd(); throw new TaskFailedError(error instanceof Error ? error.message : String(error), memory, { ...stats }); } @@ -261,16 +252,21 @@ async function runTaskInner(options: RunTaskOptions, stats: TaskStats, memory: T const jevSetup = config.fastMode ? resolveJevConfig(config) : null; const jev = jevSetup && "jev" in jevSetup ? jevSetup.jev : null; const conversation = formatHistory(options.history); + const userLocale = describeLocale(options.locale); const checkCancelled = async () => { if (options.signal?.aborted) throw new TaskCancelledError({ ...stats }); }; - const confirmOrCancel = async (action: string, url: string) => { + // Actions the user said no to (by their confirmation text): skipped, and never + // offered or asked about again during this task; the rest of the task carries on. + const declined = new Set(); + const confirmAction = async (action: string, url: string): Promise => { + if (declined.has(action)) return false; emit({ type: "confirm", action, url }); const allowed = await options.confirm(action, url); await checkCancelled(); - if (!allowed) throw new ActionDeclinedError(action); - return true; + if (!allowed) declined.add(action); + return allowed; }; const finish = async (answer: string, mode: RunTaskResult["mode"]): Promise => { if (browsed) { @@ -311,7 +307,7 @@ async function runTaskInner(options: RunTaskOptions, stats: TaskStats, memory: T const routeText = await llm.generateText({ model: config.navModel, systemInstruction: `You route requests for WebPilot, an assistant that lives in the user's browser and can control the current tab. Return JSON only. ${UNTRUSTED_CONTENT_RULE}`, - prompt: `${conversation ? `Conversation so far:\n${conversation}\n\n` : ""}Current tab: ${current.title || "(none)"} ${current.url || ""} + prompt: `${conversation ? `Conversation so far:\n${conversation}\n\n` : ""}Current tab: ${current.title || "(none)"} ${current.url || ""}${userLocale ? `\n${userLocale}` : ""} User message: ${goal} @@ -352,7 +348,7 @@ When an earlier request did not finish, or the user corrects or adds to one ("no const suggestion = await llm.generateText({ model: config.navModel, systemInstruction: "You pick where a browser task should start. Reply with STAY when the task is about the web page the user already has open (or can be done from it); otherwise reply with one absolute https URL, such as the site the task names or a good search page. Reply with nothing else.", - prompt: `Task: ${browserGoal}\nCurrent tab: ${onWebPage ? `${current.title || "(untitled)"} ${current.url}` : "a browser page with no website (such as the New Tab page); STAY is not possible"}`, + prompt: `Task: ${browserGoal}\nCurrent tab: ${onWebPage ? `${current.title || "(untitled)"} ${current.url}` : "a browser page with no website (such as the New Tab page); STAY is not possible"}${userLocale ? `\n${userLocale}` : ""}`, }); stats.llmCalls++; let url = extractExplicitUrls(suggestion)[0]; @@ -423,7 +419,10 @@ When an earlier request did not finish, or the user corrects or adds to one ("no trace(action, { run: label, ...(data && typeof data === "object" ? data as Record : {}) }); }, checkStop: checkCancelled, - confirm: confirmOrCancel, + confirm: confirmAction, + declined, + // Finding or reading never sends, buys or deletes. + allowIrreversible: changesSomething, onStep: async (step: FastModeStep) => { stats.steps++; emit({ @@ -434,6 +433,7 @@ When an earlier request did not finish, or the user corrects or adds to one ("no ? step.target.description + (step.text ? ` ← "${step.text}"` : "") : step.operation === "navigate" ? step.page.url : undefined, url: step.page.url, + ...(step.ok ? {} : { status: step.error === DECLINED_OUTCOME ? "declined" as const : step.error === NOTHING_TO_TYPE_OUTCOME ? "skipped" as const : "failed" as const }), }); }, }); @@ -455,7 +455,9 @@ When an earlier request did not finish, or the user corrects or adds to one ("no noDoneOn: pass === 2 && fast ? fast.page.url : undefined, }); - if (fast.outcome === "done") { + // Out of steps with several pages read (a list and a few listings): they may + // already answer it; the check decides before the slow planner is involved. + if (fast.outcome === "done" || (fast.exhausted && fast.pages.length > 1)) { emit({ type: "status", message: "Writing the answer" }); const pageText = await browser.pageText(12_000); const answer = (await writeFastModeAnswer(llm, config.navModel, browserGoal, fast, pageText, streamAnswer)).trim(); @@ -480,7 +482,7 @@ When an earlier request did not finish, or the user corrects or adds to one ("no emit({ type: "answer-reset" }); const problem = checkProblem(scores); handoff = `${summarizeFastModeForPlanner(fast)}\nFast mode proposed an answer, but ${problem}. ${/every part|whole request/.test(problem) ? "Find what's missing" : "Verify it on the pages"} before finishing.\nProposed answer: ${answer.slice(0, 800)}`; - if (pass === 1) continue; + if (pass === 1 && !fast.exhausted) continue; emit({ type: "handoff", reason: `Fast mode's answer fell short: ${problem}.` }); } else { handoff = summarizeFastModeForPlanner(fast); @@ -500,7 +502,7 @@ When an earlier request did not finish, or the user corrects or adds to one ("no compactToolResponse: compactPlannerResult, onText: streamAnswer, onTextReset: () => emit({ type: "answer-reset" }), - systemInstruction: `You are WebPilot, a browser agent working in the user's own browser tab. Today is ${todayLabel()}. Task: ${browserGoal} + systemInstruction: `You are WebPilot, a browser agent working in the user's own browser tab. Today is ${todayLabel()}.${userLocale ? ` ${userLocale}` : ""} Task: ${browserGoal} Tools: observe() returns the page's actionable elements as [ref] descriptions plus page text. click({ref, element}) and type({ref, text, submit?}) act on refs from the latest observe(). navigate({url}), scroll({direction}), wait({seconds}). When you have the answer, reply to the user with it as a normal message, not a tool call: that ends the task, and the user sees it as you write it.${jev ? ` delegate({goal}) hands a sub-goal to Jev, a fast navigation model. You think and decide; Jev does the legwork.` : ""} @@ -533,6 +535,7 @@ Rules:${jev ? ` for (const call of response.functionCalls) { const args = call.args as Record; let result: Record; + let stepStatus: "declined" | "skipped" | "failed" | undefined; try { if (call.name === "finish") { return finish(String(args.result || response.text || "").trim(), "planner"); @@ -544,11 +547,19 @@ Rules:${jev ? ` const ref = String(args.ref || ""); const element = lastPage.elements.find((candidate) => candidate.ref === ref); const candidate = element ? { role: element.role, label: element.name } : { label: String(args.element || "") }; - const risky = irreversibleAction(candidate) || signInAction(lastPage, candidate); - // Declining throws and ends the task. - if (risky) await confirmOrCancel(risky, lastPage.url); - await browser.click(ref, String(args.element || "")); - result = { ok: true }; + const irreversible = irreversibleAction(candidate); + const risky = irreversible || signInAction(lastPage, candidate); + if (irreversible && !changesSomething) { + // Not asked for: finding or reading never sends, buys or deletes. + stepStatus = "skipped"; + result = { ok: false, error: `The user only asked to find or read information, so WebPilot won't click ${irreversible}: it would change something for them. Carry on without it.` }; + } else if (risky && !await confirmAction(risky, lastPage.url)) { + stepStatus = "declined"; + result = { ok: false, error: "The user declined this, so it was not done. Don't try it or anything equivalent again; carry on with any other part of the task, or answer with what was done and what was skipped." }; + } else { + await browser.click(ref, String(args.element || "")); + result = { ok: true }; + } } else if (call.name === "type") { const field = lastPage.elements.find((candidate) => candidate.ref === String(args.ref || "")); if (field && isSecretField(field)) { @@ -589,12 +600,13 @@ Rules:${jev ? ` result = { ok: false, error: `Unknown tool ${call.name}` }; } } catch (error) { - if (error instanceof TaskCancelledError || error instanceof ActionDeclinedError) throw error; + if (error instanceof TaskCancelledError) throw error; + stepStatus = "failed"; result = { ok: false, error: error instanceof Error ? error.message.slice(0, 300) : String(error) }; } if (call.name !== "observe" && call.name !== "delegate") { stats.steps++; - emit({ type: "step", source: "llm", action: call.name, detail: String(args.element || args.url || args.text || ""), url: lastPage.url }); + emit({ type: "step", source: "llm", action: call.name, detail: String(args.element || args.url || args.text || ""), url: lastPage.url, ...(stepStatus ? { status: stepStatus } : {}) }); } trace("planner_tool", { name: call.name, args, ok: result.ok, error: result.error }); outputs.push({ functionResponse: { name: call.name, response: result } }); diff --git a/lib/jev/fast-mode.ts b/lib/jev/fast-mode.ts index 55c2807..ad56354 100644 --- a/lib/jev/fast-mode.ts +++ b/lib/jev/fast-mode.ts @@ -74,10 +74,19 @@ export interface FastModeOptions { checkStop: () => Promise; onStep?: (step: FastModeStep) => Promise; /** - * Asks the user before an irreversible click. Resolves true to proceed; - * throws (e.g. on Stop) to cancel. Without it, such clicks hand off instead. + * Asks the user before an irreversible click. Resolves true to proceed and + * false to skip it (the task carries on without it); throws (e.g. on Stop) + * to cancel. Without it, such clicks hand off instead. */ confirm?: (action: string, pageUrl: string) => Promise; + /** + * false for requests that only find or read information: controls that + * would send, buy, delete... are never offered to Jev, so it never asks. + * Default true. + */ + allowIrreversible?: boolean; + /** Actions the user declined during this task (confirmation text); never offered again. */ + declined?: Set; /** * A page where finishing is not allowed: a second pass after an answer from * this page was found unsupported, so Jev has to open the item that answers it. @@ -94,6 +103,8 @@ export interface FastModeOptions { export interface FastModeResult { outcome: "done" | "handoff"; reason: string; + /** Stopped on its step or time budget: the pages it read may still answer the task. */ + exhausted?: boolean; page: PageModel; history: HistoryEntry[]; steps: number; @@ -121,11 +132,37 @@ const UNSURE_ACTION_PROBABILITY = 0.35; /** Below this Jev is guessing; the LLM picks the step instead. */ const GUESSING_PROBABILITY = 0.1; const DEFAULT_MAX_CONSULTS = 2; +/** The outcome recorded for a step the user declined; Jev reads it in its history. */ +export const DECLINED_OUTCOME = "declined by the user: not done, and not to be tried again"; + +/** The outcome recorded for a field the task gives nothing to type into. */ +export const NOTHING_TO_TYPE_OUTCOME = "skipped: the task gives nothing to type here"; + +class NothingToType extends Error { + constructor() { + super(NOTHING_TO_TYPE_OUTCOME); + this.name = "NothingToType"; + } +} + +/** The user said no to a confirmation: the step is skipped and the task carries on. */ +class DeclinedByUser extends Error { + constructor(public action: string) { + super(DECLINED_OUTCOME); + this.name = "DeclinedByUser"; + } +} const CONSULT_ELEMENT_LIMIT = 80; const STUCK_THRESHOLD = 0.85; const DEFAULT_MAX_STEPS = 20; const DEFAULT_TIME_BUDGET_MS = 120_000; const MAX_NO_PROGRESS = 2; +/** + * Page text kept per page for the answer (a results page lists many items); + * Jev decides on a shorter excerpt, which keeps its calls fast. + */ +const PAGE_MEMORY_CHARS = 4000; +const JEV_PAGE_TEXT_CHARS = 1500; const HISTORY_LIMIT = 10; const PAGE_ACTION_CRITERIA: Record<"scroll_down" | "scroll_up" | "back" | "done" | "blocked", string> = { @@ -218,6 +255,13 @@ function buildQuestions( return questions; } +/** Jev's top options for a step with their probabilities, as readable lines. */ +function alternativesOf(decision: JevDecision, questions: Record, count = 4): string[] { + const answer = choiceAnswer(decision, "action"); + const criteria = (questions.action?.criteria || {}) as Record; + return rankedChoices(answer).slice(0, count).map((option) => `${(answer?.probabilities[option] ?? 0).toFixed(2)} ${(criteria[option] || option).slice(0, 110)}`); +} + interface ChosenAction { operation: Operation; target?: PageElement; @@ -343,17 +387,22 @@ export async function runFastMode(options: FastModeOptions): Promise { if (pages[pages.length - 1]?.url !== page.url) pages.push({ url: page.url, title: page.title, text: page.text }); }; - const finish = (outcome: FastModeResult["outcome"], reason: string, page: PageModel, steps: number): FastModeResult => { + const finish = (outcome: FastModeResult["outcome"], reason: string, page: PageModel, steps: number, exhausted = false): FastModeResult => { options.log(outcome === "done" ? "info" : "warn", outcome === "done" ? "fast_mode_done" : "fast_mode_handoff", { reason, steps }); - return { outcome, reason, page, history: history.slice(), steps, pages }; + return { outcome, reason, page, history: history.slice(), steps, pages, ...(exhausted ? { exhausted } : {}) }; }; + /** What confirming a click on this element would ask, if anything. */ + const confirmationFor = (element: PageElement, current: PageModel) => ({ + irreversible: irreversibleAction({ role: element.role, label: element.name }), + signIn: signInAction(current, { role: element.role, label: element.name }), + }); - let page = parsePage(await options.browser.snapshot()); + let page = parsePage(await options.browser.snapshot(), PAGE_MEMORY_CHARS); remember(page); for (let step = 1; step <= maxSteps; step++) { await options.checkStop(); - if (Date.now() > deadline) return finish("handoff", "Fast mode time budget exceeded.", page, step - 1); + if (Date.now() > deadline) return finish("handoff", "Fast mode time budget exceeded.", page, step - 1, true); const usedHere = usedActions.get(page.url) || new Set(); const pageActions: Array = ["scroll_down", "scroll_up"]; @@ -362,12 +411,19 @@ export async function runFastMode(options: FastModeOptions): Promise element.kind === "type" && !isSecretField(element) && !usedHere.has(actionKey("type", element, page.url))); - const clickables = page.elements.filter((element) => element.kind === "click" && !usedHere.has(actionKey("click", element, page.url))); + const clickables = page.elements.filter((element) => { + if (element.kind !== "click" || usedHere.has(actionKey("click", element, page.url))) return false; + const { irreversible, signIn } = confirmationFor(element, page); + // A find-or-read request never sends, buys or deletes: don't even offer it. + if (irreversible && options.allowIrreversible === false) return false; + const confirmation = irreversible || signIn; + return !(confirmation && options.declined?.has(confirmation)); + }); // Build the request within Jev's context budget: on crowded pages, keep the // options most relevant to the task and shorten the text excerpts. let optionLimit = MAX_ACTION_OPTIONS; - let textLimit = page.text.length; + let textLimit = JEV_PAGE_TEXT_CHARS; let earlierPages = 2; let typePool = selectCandidates(typeables, options.task, 0); let clickPool = typePool; @@ -432,6 +488,9 @@ export async function runFastMode(options: FastModeOptions): Promise= DONE_THRESHOLD)) { @@ -482,15 +541,15 @@ export async function runFastMode(options: FastModeOptions): Promise= 0.5; } - if (!text) return finish("handoff", `Could not decide what to type into ${description}.`, page, step - 1); + // Nothing the task calls for goes here (e.g. "Send seller a message" while + // just looking): leave the field alone and carry on. + if (!text) throw new NothingToType(); actionLabel = `type "${text}" into ${description}${submit ? " and press Enter" : ""}`; resultText = await options.browser.type(target.ref, description, text, submit === true); } else if (operation === "scroll_down" || operation === "scroll_up") { @@ -510,21 +571,27 @@ export async function runFastMode(options: FastModeOptions): Promise(); used.add(actionKey(operation, target, before.url)); @@ -554,7 +621,7 @@ export async function runFastMode(options: FastModeOptions): Promise { const visited = result.pages - .map((page, index) => `--- Page ${index + 1}: ${page.title} (${page.url}) ---\n${page.text.slice(0, 1500)}`) + .map((page, index) => `--- Page ${index + 1}: ${page.title} (${page.url}) ---\n${page.text.slice(0, 3000)}`) .join("\n\n"); const request = { model, - systemInstruction: `You write the final answer for a browser automation agent, speaking to the user directly ("you", "your inbox"). Today is ${today}. Answer only from the page content provided. If the content does not contain the answer, say what is missing. ${UNTRUSTED_CONTENT_RULE} If a page contains such instructions, mention that you ignored them.`, + systemInstruction: `You write the final answer for a browser automation agent, speaking to the user directly ("you", "your inbox"). Today is ${today}. Answer only from the page content provided. When the task asks to find or compare options (listings, rentals, products, places, articles), give the several best matches you can see across all the pages, with their key details (price, location, date, link when shown), not just the one that was opened. If the content does not contain the answer, say what is missing. ${UNTRUSTED_CONTENT_RULE} If a page contains such instructions, mention that you ignored them.`, prompt: `Task: ${task} Actions taken: @@ -582,7 +649,7 @@ Current page: ${result.page.title} (${result.page.url}) ${fullPageText.slice(0, 12000)} Earlier pages: -${visited.slice(0, 6000)} +${visited.slice(0, 10000)} Final answer:`, thinkingBudget: 512, diff --git a/lib/jev/gates.ts b/lib/jev/gates.ts index e1068d0..6eac7e9 100644 --- a/lib/jev/gates.ts +++ b/lib/jev/gates.ts @@ -138,7 +138,7 @@ export interface AnswerEvidence { const EVIDENCE_PAGES = 4; const LAST_PAGE_CHARS = 10_000; -const EARLIER_PAGE_CHARS = 2_500; +const EARLIER_PAGE_CHARS = 3_000; /** * Checks a written answer against the pages the agent saw (not just the last diff --git a/lib/model-client.ts b/lib/model-client.ts index 47815bf..d73d4c3 100644 --- a/lib/model-client.ts +++ b/lib/model-client.ts @@ -296,6 +296,27 @@ async function postChatCompletion( return json; } +/** + * Whether a failed call is worth one more try: providers sometimes drop a + * request mid-stream or report an upstream error inside a 200 response. Not + * when the user stopped, or for requests the provider rejected as invalid. + */ +function worthRetrying(config: RuntimeModelConfig, error: unknown): boolean { + if (config.signal?.aborted) return false; + const message = error instanceof Error ? error.message : String(error); + return !/request failed \(4(?!08|29)\d\d\)/.test(message); +} + +/** Runs a model call, retrying once after a transient failure. */ +async function withRetry(config: RuntimeModelConfig, call: (attempt: number) => Promise): Promise { + try { + return await call(0); + } catch (error) { + if (!worthRetrying(config, error)) throw error; + return await call(1); + } +} + /** The `delta` of each server-sent event of a streamed chat completion. */ async function* streamDeltas(config: RuntimeModelConfig, body: Record): AsyncGenerator> { const response = await openChatCompletion(config, { ...body, stream: true }); @@ -338,13 +359,22 @@ async function streamChatCompletion( onDelta: (text: string) => void ): Promise { let full = ""; - for await (const delta of streamDeltas(config, body)) { - if (typeof delta.content === "string" && delta.content) { - full += delta.content; - onDelta(delta.content); + const attempt = async () => { + for await (const delta of streamDeltas(config, body)) { + if (typeof delta.content === "string" && delta.content) { + full += delta.content; + onDelta(delta.content); + } } + return full; + }; + try { + return await attempt(); + } catch (error) { + // Retry only if nothing reached the reader yet: shown text can't be taken back here. + if (full || !worthRetrying(config, error)) throw error; + return await attempt(); } - return full; } /** Streamed reasoning items with the same index are pieces of one item. */ @@ -475,9 +505,14 @@ class ChatCompletionsToolChat implements ToolChat { if (this.config.provider === "openrouter") body.tool_choice = "auto"; } - const messageObj = this.options.onText && this.config.provider === "openrouter" - ? await streamToolTurn(this.config, body, this.options.onText, this.options.onTextReset || (() => {})) - : firstChoiceMessage(await postChatCompletion(this.config, body)); + const stream = !!this.options.onText && this.config.provider === "openrouter"; + const messageObj = await withRetry(this.config, async (attempt) => { + // Text shown by a failed attempt is withdrawn before trying again. + if (attempt > 0 && stream) this.options.onTextReset?.(); + return stream + ? await streamToolTurn(this.config, body, this.options.onText!, this.options.onTextReset || (() => {})) + : firstChoiceMessage(await postChatCompletion(this.config, body)); + }); const toolCallsRaw = Array.isArray(messageObj.tool_calls) ? messageObj.tool_calls as Array> : []; const text = normalizeContent(messageObj.content); const reasoningDetails = Array.isArray(messageObj.reasoning_details) && messageObj.reasoning_details.length @@ -544,7 +579,7 @@ class ChatCompletionsModelClient implements ModelClient { } async generateText(options: GenerateTextOptions): Promise { - const response = await postChatCompletion(this.config, this.textBody(options)); + const response = await withRetry(this.config, () => postChatCompletion(this.config, this.textBody(options))); return normalizeContent(firstChoiceMessage(response).content); } diff --git a/scripts/extension-smoke.ts b/scripts/extension-smoke.ts index 3608b07..ec99741 100644 --- a/scripts/extension-smoke.ts +++ b/scripts/extension-smoke.ts @@ -22,8 +22,8 @@ interface Case { request: string | ((base: string) => string); /** A follow-up sent after the first answer, in the same conversation. */ followUp?: { request: string; check: (answer: string) => string | null }; - /** "allow" or "cancel" the confirmation card if it appears. */ - onConfirm?: "allow" | "cancel"; + /** "allow" or "skip" the confirmation card if it appears. */ + onConfirm?: "allow" | "skip"; /** The page has another extension's frame, so the task must run on page scripts. */ pageScripts?: boolean; /** Leave feedback on the answer (thumbs down + note, then "/feedback") and check it was saved with a trace. */ @@ -56,7 +56,7 @@ const CASES: Case[] = [ start: (base) => `${base}/mail`, request: "find that email where i was asked about relocation. who sent it and what did they want to know?", feedback: true, - check: (answer, actions) => (actions.length ? `unexpected actions ${JSON.stringify(actions)}` : /Priya/i.test(answer) ? null : "expected the email from Priya"), + check: (answer, actions, confirmed) => (actions.length ? `unexpected actions ${JSON.stringify(actions)}` : confirmed ? "asked to confirm something while only reading" : /Priya/i.test(answer) ? null : "expected the email from Priya"), }, { // Conversational: the follow-up depends on the first answer and on what was opened. @@ -85,13 +85,13 @@ const CASES: Case[] = [ start: (base) => `${base}/mail?pm=1`, request: "find that email where i was asked about relocation. who sent it and what did they want to know?", pageScripts: true, - check: (answer, actions) => (actions.length ? `unexpected actions ${JSON.stringify(actions)}` : /Priya/i.test(answer) ? null : "expected the email from Priya"), + check: (answer, actions, confirmed) => (actions.length ? `unexpected actions ${JSON.stringify(actions)}` : confirmed ? "asked to confirm something while only reading" : /Priya/i.test(answer) ? null : "expected the email from Priya"), }, { id: "blocked_buy_cancelled", start: (base) => `${base}/shop?pm=1`, request: "buy me the cheapest 2-person tent", - onConfirm: "cancel", + onConfirm: "skip", pageScripts: true, check: (_answer, actions, confirmed, cart) => { if (actions.some((action) => action.type === "place_order")) return "placed an order"; @@ -104,7 +104,7 @@ const CASES: Case[] = [ id: "sign_in_asks_first", start: (base) => `${base}/portal`, request: "check my student portal for my next appointment", - onConfirm: "cancel", + onConfirm: "skip", check: (_answer, actions, confirmed) => { if (actions.some((action) => action.type === "login")) return `signed in without asking: ${JSON.stringify(actions)}`; return confirmed ? null : "never asked before signing in"; @@ -127,7 +127,7 @@ const CASES: Case[] = [ id: "buy_cancelled", start: (base) => `${base}/shop`, request: "buy me the cheapest 2-person tent", - onConfirm: "cancel", + onConfirm: "skip", check: (_answer, actions, confirmed, cart) => { if (actions.some((action) => action.type === "place_order")) return "placed an order"; if (!confirmed) return "never asked to confirm"; @@ -154,7 +154,7 @@ async function ask(panel: Page, request: string, onConfirm: Case["onConfirm"], s if (await panel.locator(".confirm").count()) { confirmed = true; await panel.screenshot({ path: `${shot}-confirm.png` }); - await panel.getByRole("button", { name: onConfirm === "allow" ? "Allow" : "Cancel" }).click(); + await panel.getByRole("button", { name: onConfirm === "allow" ? "Allow" : "Skip" }).click(); } const turns = await panel.locator(".turn").count(); const last = panel.locator(".turn").last(); diff --git a/tests/jev.test.ts b/tests/jev.test.ts index c42e1fb..abd032e 100644 --- a/tests/jev.test.ts +++ b/tests/jev.test.ts @@ -292,6 +292,57 @@ test("when a Jev call fails, the LLM picks that step", async () => { assert.deepEqual(calls, ["click e7"]); }); +test("for a find-or-read request, Jev is never offered buttons that send, buy or delete", async () => { + const requests = mockJev([() => ({ action: choice("done"), goal_done: noul(0.95), stuck: noul(0.02) })]); + const { browser } = fakeBrowser({ home: HOME }, "home"); + await runFastMode({ ...fastModeOptions(browser), allowIrreversible: false }); + const criteria = requests[0].body.questions.action.criteria || {}; + assert.ok(!("click_e11" in criteria), "\"Buy now\" is not an option"); + assert.ok("click_e7" in criteria); +}); + +test("a declined click is skipped, Jev carries on, and it isn't offered again", async () => { + const requests = mockJev([ + () => ({ action: choice("click_e11"), goal_done: noul(0.1), stuck: noul(0.1), submit_after_typing: noul(0.5) }), + () => ({ action: choice("click_e7"), goal_done: noul(0.1), stuck: noul(0.1), submit_after_typing: noul(0.5) }), + () => ({ action: choice("done"), goal_done: noul(0.97), stuck: noul(0.02) }), + ]); + const { browser, calls } = fakeBrowser({ home: HOME, pricing: PRICING }, "home"); + const declined = new Set(); + const asked: string[] = []; + const result = await runFastMode({ + ...fastModeOptions(browser), + declined, + confirm: async (action) => { + asked.push(action); + declined.add(action); + return false; + }, + }); + + assert.equal(result.outcome, "done"); + assert.deepEqual(asked, ["button \"Buy now\""]); + assert.deepEqual(calls, ["click e7"], "Buy now was never clicked"); + assert.match(result.history[0].outcome, /declined by the user/); + assert.ok(!("click_e11" in (requests[1].body.questions.action.criteria || {})), "not offered again"); +}); + +test("a field the task gives nothing to type into is skipped, and Jev carries on", async () => { + const requests = mockJev([ + () => ({ action: choice("type_e8"), goal_done: noul(0.05), stuck: noul(0.1), submit_after_typing: noul(0.5) }), + () => ({ action: choice("click_e7"), goal_done: noul(0.1), stuck: noul(0.1), submit_after_typing: noul(0.5) }), + () => ({ action: choice("done"), goal_done: noul(0.97), stuck: noul(0.02) }), + ]); + const writer = { generateText: async () => "NONE", createToolChat: () => { throw new Error("unused"); } } as unknown as ModelClient; + const { browser, calls } = fakeBrowser({ home: HOME, pricing: PRICING }, "home"); + const result = await runFastMode(fastModeOptions(browser, writer)); + + assert.equal(result.outcome, "done"); + assert.deepEqual(calls, ["click e7"], "nothing was typed"); + assert.match(result.history[0].outcome, /nothing to type/); + assert.ok(!("type_e8" in (requests[1].body.questions.action.criteria || {})), "the field isn't offered again on that page"); +}); + test("fast mode hands irreversible clicks to the planner", async () => { mockJev([() => ({ action: choice("click_e11"), goal_done: noul(0.1), stuck: noul(0.1), submit_after_typing: noul(0.5) })]); const { browser, calls } = fakeBrowser({ home: HOME }, "home"); diff --git a/tests/model-client.test.ts b/tests/model-client.test.ts index cce4cd2..be5a660 100644 --- a/tests/model-client.test.ts +++ b/tests/model-client.test.ts @@ -225,6 +225,32 @@ test("streamed tool turns report text as written and rebuild tool calls and reas assert.deepEqual(replayed?.tool_calls, [{ id: "call_1", type: "function", function: { name: "observe", arguments: "{\"full\":true}" } }]); }); +test("a tool turn that fails mid-stream is withdrawn and retried once", async () => { + const sse = (events: Array>) => events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join("") + "data: [DONE]\n\n"; + const replies = [ + // The provider drops the request partway through the answer. + sse([{ choices: [{ delta: { content: "Here are three" } }] }, { error: { message: "The operation was aborted" } }]), + sse([{ choices: [{ delta: { content: "Here are three rentals." } }] }]), + ]; + let calls = 0; + globalThis.fetch = (async () => { + calls++; + return new Response(replies.shift(), { status: 200, headers: { "Content-Type": "text/event-stream" } }); + }) as typeof fetch; + const shown: string[] = []; + const chat = createModelClient(openRouterConfig()).createToolChat({ + model: "google/gemini-3.8-flash", + systemInstruction: "system", + tools: getBrowserToolDeclarations(), + onText: (text) => shown.push(text), + onTextReset: () => shown.push(""), + }); + const reply = await chat.sendMessage("find rentals"); + assert.equal(calls, 2); + assert.equal(reply.text, "Here are three rentals."); + assert.deepEqual(shown, ["Here are three", "", "Here are three rentals."]); +}); + test("Ollama requests omit OpenRouter-only fields", async () => { const requests = mockFetch([{ body: { choices: [{ message: { content: "hi" } }] } }]); const client = createModelClient(resolveRuntimeModelConfig({ provider: "ollama", navModel: "qwen3:8b" })); diff --git a/tests/run-task.test.ts b/tests/run-task.test.ts index ea07796..c73bb34 100644 --- a/tests/run-task.test.ts +++ b/tests/run-task.test.ts @@ -53,22 +53,28 @@ function chatResponse(message: Record, stream: boolean): Respon } /** Routes Jev decisions and chat completions to canned responses. */ -function mockServices(chatReplies: Array>) { +function mockServices(chatReplies: Array>, jev: { changesSomething?: number; fastMode?: { action: string } } = {}) { globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { const url = String(input); if (!url.endsWith("/systemone")) return chatResponse(chatReplies.shift() || { content: "done" }, JSON.parse(String(init?.body || "{}")).stream === true); - const body = { model: "jev", answers: { needs_browser: { type: "noul", noul: 0.95 }, parallel_sites: { type: "noul", noul: 0.02 }, changes_something: { type: "noul", noul: 0.95 }, multiple_parts: { type: "noul", noul: 0.02 } }, usage: {} }; - return new Response(JSON.stringify(body), { status: 200, headers: { "Content-Type": "application/json" } }); + const request = JSON.parse(String(init?.body || "{}")) as { questions: Record }; + const answers = "action" in request.questions + ? { action: { type: "choice", choice: jev.fastMode?.action || "done", confidence: 0.9, probabilities: { [jev.fastMode?.action || "done"]: 0.9 } }, goal_done: { type: "noul", noul: 0.05 }, stuck: { type: "noul", noul: 0.1 } } + : { needs_browser: { type: "noul", noul: 0.95 }, parallel_sites: { type: "noul", noul: 0.02 }, changes_something: { type: "noul", noul: jev.changesSomething ?? 0.95 }, multiple_parts: { type: "noul", noul: 0.02 } }; + return new Response(JSON.stringify({ model: "jev", answers, usage: {} }), { status: 200, headers: { "Content-Type": "application/json" } }); }) as typeof fetch; } -test("declining a confirmation ends the task and remembers where it stopped", async () => { +test("a declined click is skipped, never asked again, and the task carries on", async () => { mockServices([ { content: null, tool_calls: [{ id: "c1", type: "function", function: { name: "click", arguments: JSON.stringify({ ref: "b11", element: "Buy now" }) } }] }, - { content: "should not be reached" }, + // The model tries again; the decline stands without asking. + { content: null, tool_calls: [{ id: "c2", type: "function", function: { name: "click", arguments: JSON.stringify({ ref: "b11", element: "Buy now" }) } }] }, + { content: "I didn't buy the Trailhead 2P ($89.99): you skipped the purchase." }, ]); const log: string[] = []; const asked: string[] = []; + const statuses: string[] = []; const result = await runTask({ goal: "buy the tent at https://shop.example/p/tent", config: resolveRuntimeModelConfig({ provider: "openrouter", apiKey: "sk-or-test", fastMode: true }), @@ -77,14 +83,48 @@ test("declining a confirmation ends the task and remembers where it stopped", as asked.push(action); return false; }, + onEvent: (event) => { + if (event.type === "step" && event.status) statuses.push(`${event.status} ${event.action}`); + }, }); - assert.deepEqual(asked, ["button \"Buy now\""]); + assert.deepEqual(asked, ["button \"Buy now\""], "asked once"); assert.deepEqual(log, ["navigate https://shop.example/p/tent"], "the Buy button was never clicked"); - assert.match(result.answer, /stopped before clicking button "Buy now"/); - assert.equal(result.memory.finalUrl, "https://shop.example/p/tent"); + assert.deepEqual(statuses, ["declined click", "declined click"]); + assert.match(result.answer, /skipped the purchase/); +}); + +test("a request that only finds information never clicks buy or send, and never asks", async () => { + mockServices([ + // Fast mode's advisor sends the task to the planner... + { content: "{\"action\": \"handoff\", \"reason\": \"needs a closer look\"}" }, + // ...which tries to buy anyway. + { content: null, tool_calls: [{ id: "c1", type: "function", function: { name: "click", arguments: JSON.stringify({ ref: "b11", element: "Buy now" }) } }] }, + { content: "The Trailhead 2P costs $89.99." }, + ], { changesSomething: 0.02, fastMode: { action: "blocked" } }); + const log: string[] = []; + const asked: string[] = []; + const statuses: string[] = []; + const result = await runTask({ + goal: "how much is the tent at https://shop.example/p/tent ?", + config: resolveRuntimeModelConfig({ provider: "openrouter", apiKey: "sk-or-test", fastMode: true }), + browser: fakeBrowser(log), + confirm: async (action) => { + asked.push(action); + return true; + }, + onEvent: (event) => { + if (event.type === "step" && event.status) statuses.push(`${event.status} ${event.action}`); + }, + }); + + assert.deepEqual(asked, [], "never asked"); + assert.deepEqual(log, ["navigate https://shop.example/p/tent"], "never clicked"); + assert.deepEqual(statuses, ["skipped click"]); + assert.match(result.answer, /89\.99/); }); + test("a guessed start site that doesn't load falls back to a search", async () => { mockServices([ { content: "https://dead.example/" },