From 075266e936ad95453469c9afa172b6574fddf6dc Mon Sep 17 00:00:00 2001 From: ahamSel <77988808+ahamSel@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:14:32 -0230 Subject: [PATCH] feat: Jev and the LLM hand work back and forth - LLM -> Jev: the planner gets a `delegate` tool that runs fast mode on a concrete sub-goal (open this email, go through this list) and returns what Jev did plus the page it ended on. The planner keeps the thinking; Jev does the legwork at ~0.3s per step instead of several seconds. - Jev -> LLM: when Jev is blocked, stuck, guessing (p < 0.1), making no progress, or its call fails, fast mode asks the LLM for that one step and keeps driving; it hands the task to the planner only when the LLM says so or the consults run out. - Multi-part requests ("when is X, and also when is Y") are detected by a new Jev preflight gate (or the LLM router) and go planner-first, one delegated part at a time. - Context: Jev requests are kept within its 32k-token window (trimmed to the most relevant elements when needed, capped task/context/history text); the planner's older page views keep their text but drop stale element lists. - Planner answers only from pages it saw (no gap-filling from general knowledge) and uses known search URLs (Gmail #search/...) directly. - Bench: books_two_parts and mail_two_parts scenarios; bridge reload retries until the browser runs the new build; bridge shutdown no longer hangs on keep-alive polls. Co-Authored-By: Claude Opus 5.5 --- extension/README.md | 6 +- extension/src/engine.ts | 1 + lib/core/run-task.ts | 130 ++++++++++++++----- lib/jev/client.ts | 12 ++ lib/jev/fast-mode.ts | 241 +++++++++++++++++++++++++++++------- lib/jev/gates.ts | 15 +++ lib/model-client.ts | 35 ++++-- scripts/bench-fast-mode.ts | 13 ++ scripts/extension-bridge.ts | 23 ++-- tests/jev.test.ts | 80 +++++++++++- tests/model-client.test.ts | 27 ++++ tests/run-task.test.ts | 52 +++++++- 12 files changed, 539 insertions(+), 96 deletions(-) diff --git a/extension/README.md b/extension/README.md index 9315bff..607a6ba 100644 --- a/extension/README.md +++ b/extension/README.md @@ -36,7 +36,11 @@ Fast mode (Jev) is on by default. - Pages are read from Chrome's accessibility tree and acted on with real mouse and keyboard events, so sites treat them like your own input. - The same core runs in the desktop app and the extension: `lib/core/run-task.ts` (the task flow), `lib/cdp/` (the CDP driver and accessibility-tree snapshot), `lib/jev/` (fast mode), `lib/safety.ts` and `lib/model-client.ts`. - The agent runs in the extension's background service worker (`src/engine.ts`), one session per tab; panels are views connected over a port. While a task runs, the worker keeps itself alive. -- Requests about the open page that reading alone can answer ("summarize this", "what does this say about X") skip navigation and clicks and stream the answer straight from the page. Other requests that only find or read information use fast mode: Jev picks each click in ~0.3s and the model writes text and the answer. Requests that change something (buy, book, send, delete, submit) use the careful model planner. +- Jev (a fast decision model, ~0.3s per call) and the LLM work as a pair, each doing what it's best at: + - Requests about the open page that reading alone can answer ("summarize this", "what does this say about X") skip navigation and clicks and stream the answer straight from the page. + - Requests that find or read information use fast mode: Jev picks each click and scroll; the LLM writes typed text and the answer. When Jev is blocked, stuck or guessing, it asks the LLM for that one step and keeps driving, instead of handing the whole task over. + - Requests with several separate parts ("when is X, and also when is Y") and requests that change something (buy, book, send, delete, submit) go to the LLM planner, which thinks and decides but hands legwork (opening results, emails, items; moving through lists) back to Jev as sub-goals with `delegate`. + - Jev requests stay within its 32k-token context: crowded pages keep the elements most relevant to the task. If a Jev call fails, the LLM picks that step. The planner's older page views are trimmed to their text, so later steps stay fast. - Answers stream into the panel as they are written; Jev then checks them against the page, and an unsupported answer is redone (first by Jev looking closer, then by the model). - Before an irreversible click (buy, place order, checkout, send, delete, publish, unsubscribe...) the panel shows **"Hold on — this can't be undone"** with Allow and Cancel. Cancel ends the task there; say "go ahead" to continue. - WebPilot never types into password, one-time-code or card fields and never makes up usernames or other personal details. On a sign-in page your password manager has already filled in, clicking "Log in" asks first (**"Hold on — sign in?"**); otherwise it asks you to sign in yourself and then continue. diff --git a/extension/src/engine.ts b/extension/src/engine.ts index 87e105f..7a46a75 100644 --- a/extension/src/engine.ts +++ b/extension/src/engine.ts @@ -47,6 +47,7 @@ const ACTION_LABELS: Record = { scroll_up: "scroll up", back: "go back", navigate: "open", + delegate: "hand to Jev", scroll: "scroll", wait: "wait", }; diff --git a/lib/core/run-task.ts b/lib/core/run-task.ts index 8615491..c81c281 100644 --- a/lib/core/run-task.ts +++ b/lib/core/run-task.ts @@ -12,7 +12,7 @@ import { createModelClient, resolveJevConfig, type RuntimeModelConfig, type ToolResponsePart } from "../model-client"; import { extractExplicitUrls } from "../goal-urls"; -import { runFastMode, summarizeFastModeForPlanner, writeFastModeAnswer, type FastModeBrowser, type FastModeResult } from "../jev/fast-mode"; +import { runFastMode, summarizeFastModeForPlanner, writeFastModeAnswer, type FastModeBrowser, type FastModeResult, type FastModeStep } from "../jev/fast-mode"; import { jevCheckAnswer, jevPreflight } from "../jev/gates"; import { describeElement, parsePage, selectCandidates, type PageModel } from "../jev/page"; import { CREDENTIALS_RULE, UNTRUSTED_CONTENT_RULE, irreversibleAction, isSecretField, signInAction } from "../safety"; @@ -115,6 +115,23 @@ const FAST_ANSWER_ACCEPT = 0.8; * article section, so it reads instead of scrolling (fast mode's Jev state stays small). */ const PLANNER_TEXT_CHARS = 8000; +/** A delegated sub-goal is short legwork: a search, opening an item, a few pages. */ +const DELEGATE_MAX_STEPS = 10; +const DELEGATE_TIME_BUDGET_MS = 40_000; + +/** The planner's tool for handing legwork to Jev. */ +const DELEGATE_TOOL = { + name: "delegate", + description: "Hand a concrete browsing sub-goal to Jev, a fast navigation model (about 0.3s per step, versus several seconds for you): searching a site, opening a result, an email or an item, moving through lists, menus and pages. Jev works from the current page and returns what it did and the page it ended on (elements and text), so you don't need observe() afterwards.", + parameters: { + type: "object", + properties: { + goal: { type: "string", description: "One specific sub-goal, naming what to search for or open, e.g. 'search the inbox for Horizon and open the most recent email about an appointment'." }, + }, + required: ["goal"], + }, +}; + /** After this long, the planner is asked to wrap up with what it has. */ const PLANNER_WRAP_UP_MS = 60_000; const FAST_ANSWER_ESCALATE_BELOW = 0.3; @@ -152,6 +169,17 @@ function formatHistory(history: RunTaskOptions["history"]): string { }).join("\n\n"); } +/** + * An earlier page view in the planner's history: its text stays (facts found + * there may be needed for the answer) but its elements go, since their refs no + * longer work once the page has changed. + */ +export function compactPlannerResult(response: Record): Record { + if (typeof response.page !== "string") return response; + const page = response.page.replace(/\nElements[^\n]*:\n[\s\S]*?\n\nText:\n/, "\n(Earlier page: elements omitted.)\nText:\n"); + return { ...response, page }; +} + /** Compact page view for the LLM planner: numbered actionable elements plus text. */ function plannerObservation(page: PageModel, task: string): string { const { selected, truncated } = selectCandidates(page.elements, task, 150); @@ -245,6 +273,7 @@ async function runTaskInner(options: RunTaskOptions, stats: TaskStats, memory: T // With a web page open, the same call decides whether the request is about it. let browse: boolean | undefined; let changesSomething = false; + let multiPart = false; let startsOnCurrentPage = false; let answerFromCurrentPage = false; if (jev && !conversation) { @@ -254,6 +283,7 @@ async function runTaskInner(options: RunTaskOptions, stats: TaskStats, memory: T stats.jevCalls++; browse = preflight.browse; changesSomething = preflight.changesSomething; + multiPart = preflight.multiPart; startsOnCurrentPage = preflight.startsOnCurrentPage; answerFromCurrentPage = preflight.answerFromCurrentPage && !preflight.changesSomething; } catch { @@ -276,17 +306,20 @@ When an earlier request did not finish, or the user corrects or adds to one ("no "changes_something" is true when the task buys, orders, books, sends, posts, deletes, subscribes or submits something (not just finding or reading). -{"mode": "chat|browse", "reply": "full reply when chat", "browser_goal": "task when browse", "changes_something": false}`, +"multi_part" is true when the task asks for several separate things likely found in different places (different emails, pages or searches). + +{"mode": "chat|browse", "reply": "full reply when chat", "browser_goal": "task when browse", "changes_something": false, "multi_part": false}`, thinkingBudget: 512, }); stats.llmCalls++; const route = extractJson(routeText); - trace("route", route ? { mode: route.mode, browser_goal: route.browser_goal, changes_something: route.changes_something } : { unparsed: routeText.slice(0, 300) }); + trace("route", route ? { mode: route.mode, browser_goal: route.browser_goal, changes_something: route.changes_something, multi_part: route.multi_part } : { unparsed: routeText.slice(0, 300) }); if (route?.mode === "chat" && typeof route.reply === "string" && route.reply.trim()) { return finish(route.reply.trim(), "chat"); } if (typeof route?.browser_goal === "string" && route.browser_goal.trim()) browserGoal = route.browser_goal.trim(); if (route?.changes_something === true) changesSomething = true; + if (route?.multi_part === true) multiPart = true; } await checkCancelled(); @@ -359,38 +392,49 @@ When an earlier request did not finish, or the user corrects or adds to one ("no // weak at multi-step flows like add to cart -> checkout. let handoff = ""; let fast: FastModeResult | null = null; + // Fast mode wiring shared by the fast passes and by planner delegations. + const fastModeHooks = (label: string) => ({ + jev: jev!, + browser, + writer: llm, + writerModel: config.navModel, + writerTimeoutMs: config.timeoutMs, + log: (_level: string, action: string, data?: unknown) => { + if (action === "jev_decision") stats.jevCalls++; + if (action === "fast_mode_text_written" || action === "fast_mode_consult") stats.llmCalls++; + trace(action, { run: label, ...(data && typeof data === "object" ? data as Record : {}) }); + }, + checkStop: checkCancelled, + confirm: confirmOrCancel, + onStep: async (step: FastModeStep) => { + stats.steps++; + emit({ + type: "step", + source: step.source, + action: step.operation, + detail: step.target + ? step.target.description + (step.text ? ` ← "${step.text}"` : "") + : step.operation === "navigate" ? step.page.url : undefined, + url: step.page.url, + }); + }, + }); if (jev && changesSomething) { emit({ type: "status", message: "Careful mode: this request changes something" }); + } else if (jev && multiPart) { + // Fast mode finishes after the first part; the planner splits the request + // and hands each part to Jev. + emit({ type: "status", message: "Several parts: planning, with Jev doing the legwork" }); + handoff = "This request asks for several separate things. Use delegate for each part (one concrete sub-goal at a time), read what Jev finds, then finish with every part answered."; } // A second pass runs when the first stops on a page that doesn't support its answer // (e.g. a search results list): finishing there is ruled out, so Jev opens the item. - for (let pass = 1; jev && !changesSomething && pass <= 2; pass++) { + for (let pass = 1; jev && !changesSomething && !multiPart && pass <= 2; pass++) { emit({ type: "status", message: pass === 1 ? "Working fast" : "Looking closer" }); fast = await runFastMode({ + ...fastModeHooks(`pass ${pass}`), task: browserGoal, noDoneOn: pass === 2 && fast ? fast.page.url : undefined, - jev, - browser, - writer: llm, - writerModel: config.navModel, - writerTimeoutMs: config.timeoutMs, - log: (_level, action, data) => { - if (action === "jev_decision") stats.jevCalls++; - if (action === "fast_mode_text_written") stats.llmCalls++; - trace(action, { pass, ...(data && typeof data === "object" ? data as Record : {}) }); - }, - checkStop: checkCancelled, - confirm: confirmOrCancel, - onStep: async (step) => { - stats.steps++; - emit({ - type: "step", - source: "jev", - action: step.operation, - detail: step.target ? step.target.description + (step.text ? ` ← "${step.text}"` : "") : undefined, - url: step.page.url, - }); - }, }); if (fast.outcome === "done") { @@ -421,20 +465,25 @@ When an earlier request did not finish, or the user corrects or adds to one ("no // 4. LLM planner on the same page. emit({ type: "status", message: "Thinking it through" }); - const tools = getBrowserToolDeclarations({ includeTabTools: false }); + const tools = [...getBrowserToolDeclarations({ includeTabTools: false }), ...(jev ? [DELEGATE_TOOL] : [])]; const chat = llm.createToolChat({ model: config.navModel, tools, + compactToolResponse: compactPlannerResult, systemInstruction: `You are WebPilot, a browser agent working in the user's own browser tab. Task: ${browserGoal} -Tools: observe() returns the page's actionable elements as [ref] descriptions plus page text. click({ref, element}) and type({ref, text, submit?}) act on refs from the latest observe(). navigate({url}), scroll({direction}), wait({seconds}). finish({result}) ends the task with the answer for the user. +Tools: observe() returns the page's actionable elements as [ref] descriptions plus page text. click({ref, element}) and type({ref, text, submit?}) act on refs from the latest observe(). navigate({url}), scroll({direction}), wait({seconds}). finish({result}) ends the task with the answer for the user.${jev ? ` +delegate({goal}) hands a sub-goal to Jev, a fast navigation model. You think and decide; Jev does the legwork.` : ""} -Rules: +Rules:${jev ? ` +- Prefer delegate for legwork: opening results, emails or items, going through lists, menus and pages. Give one specific sub-goal at a time, then read the page it returns. For several separate questions, handle each in turn. +- When a site has a search URL you know (Gmail: https://mail.google.com/mail/u/0/#search/; most shops and sites: ?q=), navigate() to it instead of working through search forms. +- Do precise steps yourself: forms with specific values, checkout, and anything Jev reported it couldn't do.` : ""} - Call observe() before using refs, and again after actions that change the page. - ${UNTRUSTED_CONTENT_RULE} - Only take irreversible actions (buy, pay, send, delete, publish...) when the task clearly asks for them; the user is asked to confirm each one. - ${CREDENTIALS_RULE} If the browser already filled them in, you may click the sign-in button; the user is asked to confirm. -- Answer from what the page shows. Be concise and specific; include names, prices and links when relevant. +- Answer only with what pages in this task showed. Never fill gaps with general knowledge or typical patterns; say plainly what you couldn't find. Be concise and specific; include names, dates, prices and links when relevant. - Work fast. If a few searches or places turn up nothing, stop and say so rather than trying every variation. - If you can't fully answer, finish with what you found, what you couldn't find, and where you looked.`, }); @@ -478,6 +527,25 @@ Rules: await browser.type(String(args.ref || ""), String(args.element || ""), String(args.text || ""), args.submit === true); result = { ok: true }; } + } else if (call.name === "delegate" && jev) { + const subGoal = String(args.goal || "").trim(); + if (!subGoal) throw new Error("delegate needs a goal."); + emit({ type: "step", source: "llm", action: "delegate", detail: subGoal, url: lastPage.url }); + const delegated = await runFastMode({ + ...fastModeHooks(`delegate ${turn + 1}`), + task: subGoal, + context: `This is one step of the user's task: ${browserGoal}`, + maxSteps: DELEGATE_MAX_STEPS, + timeBudgetMs: DELEGATE_TIME_BUDGET_MS, + maxConsults: 1, + }); + lastPage = parsePage(await browser.snapshot(), PLANNER_TEXT_CHARS); + result = { + ok: delegated.outcome === "done", + outcome: delegated.outcome === "done" ? "Jev finished the sub-goal." : `Jev stopped: ${delegated.reason}`, + steps: delegated.history.map((entry) => `${entry.action} -> ${entry.outcome}`), + page: plannerObservation(lastPage, browserGoal), + }; } else if (call.name === "navigate") { await browser.navigate(String(args.url || "")); result = { ok: true, ...(await browser.pageInfo()) }; @@ -494,7 +562,7 @@ Rules: if (error instanceof TaskCancelledError || error instanceof ActionDeclinedError) throw error; result = { ok: false, error: error instanceof Error ? error.message.slice(0, 300) : String(error) }; } - if (call.name !== "observe") { + if (call.name !== "observe" && call.name !== "delegate") { stats.steps++; emit({ type: "step", source: "llm", action: call.name, detail: String(args.element || args.url || args.text || ""), url: lastPage.url }); } diff --git a/lib/jev/client.ts b/lib/jev/client.ts index 0922cd5..72cc890 100644 --- a/lib/jev/client.ts +++ b/lib/jev/client.ts @@ -60,6 +60,18 @@ export interface JevClientConfig { export const JEV_MAX_CHOICE_OPTIONS = 255; +/** + * Jev's context through OpenRouter is 32k tokens for the state plus all + * questions (64k on TypeSafe's own API). Requests are kept under this, with + * headroom for tokenizer differences; smaller requests are also faster. + */ +export const JEV_TOKEN_BUDGET = 24_000; + +/** Rough token count of a Jev request: JSON averages ~3.2 characters per token (erring high). */ +export function estimateJevTokens(state: unknown, questions: Record): number { + return Math.ceil(JSON.stringify({ state, questions }).length / 3.2); +} + const RETRYABLE_STATUSES = new Set([429, 502, 503, 529]); const MAX_RETRIES = 2; diff --git a/lib/jev/fast-mode.ts b/lib/jev/fast-mode.ts index c11ab87..e3140cd 100644 --- a/lib/jev/fast-mode.ts +++ b/lib/jev/fast-mode.ts @@ -1,18 +1,23 @@ /** - * Fast mode: Jev picks each browser action, an LLM only writes text. + * Fast mode: Jev picks each browser action; the LLM writes text and advises. * * Every step makes one Jev call that answers several typed questions at once * (operation, click target, type target, submit, goal done, stuck). The LLM - * is used only to write text for inputs and to phrase the final answer. When - * Jev is stuck, blocked, repeating itself, out of budget, or about to take an - * irreversible action, the loop hands control back to the LLM planner. + * writes text for inputs and phrases the final answer. When Jev is blocked, + * stuck, very unsure or making no progress, it first consults the LLM for that + * one step (~2s) and keeps driving; only when the LLM says the task needs + * careful step-by-step work, or the consults run out, does the loop hand + * control to the LLM planner. The planner can in turn delegate sub-goals back + * to fast mode (`context` then carries the overall task). */ import type { ModelClient } from "../model-client"; -import { TASK_ONLY_RULE, UNTRUSTED_CONTENT_RULE, irreversibleAction, isSecretField, signInAction } from "../safety"; +import { CREDENTIALS_RULE, TASK_ONLY_RULE, UNTRUSTED_CONTENT_RULE, irreversibleAction, isSecretField, signInAction } from "../safety"; import { + JEV_TOKEN_BUDGET, choiceAnswer, decide, + estimateJevTokens, noulAnswer, rankedChoices, type JevClientConfig, @@ -34,10 +39,14 @@ export interface FastModeBrowser { type(ref: string, element: string, text: string, submit: boolean): Promise; pressKey(key: string): Promise; back(): Promise; + /** Lets the LLM advisor jump to a URL (e.g. a site's search results). */ + navigate?(url: string): Promise; } export interface FastModeStep { step: number; + /** Who chose the action: Jev, or the LLM when Jev consulted it. */ + source: "jev" | "llm"; operation: Operation; target?: { ref: string; description: string }; text?: string; @@ -47,6 +56,7 @@ export interface FastModeStep { outcome: string; preUrl: string; page: PageModel; + /** Jev's answers for the step; empty when Jev failed and the LLM chose. */ decision: JevDecision; durationMs: number; } @@ -73,6 +83,12 @@ export interface FastModeOptions { * this page was found unsupported, so Jev has to open the item that answers it. */ noDoneOn?: string; + /** For a sub-goal delegated by the planner: the overall task and what is known so far. */ + context?: string; + /** How many times Jev may ask the LLM for a single step before handing off (default 2). */ + maxConsults?: number; + /** Estimated-token ceiling for each Jev request (default JEV_TOKEN_BUDGET). */ + jevTokenBudget?: number; } export interface FastModeResult { @@ -90,7 +106,7 @@ interface HistoryEntry { outcome: string; } -type Operation = "click" | "type" | "scroll_down" | "scroll_up" | "back" | "done" | "blocked"; +type Operation = "click" | "type" | "scroll_down" | "scroll_up" | "back" | "navigate" | "done" | "blocked"; /** Jev choices allow 255 options; element actions plus page actions share them. */ const MAX_ACTION_OPTIONS = 250; @@ -102,6 +118,10 @@ const DONE_THRESHOLD = 0.85; */ const LIKELY_DONE_THRESHOLD = 0.45; const UNSURE_ACTION_PROBABILITY = 0.35; +/** Below this Jev is guessing; the LLM picks the step instead. */ +const GUESSING_PROBABILITY = 0.1; +const DEFAULT_MAX_CONSULTS = 2; +const CONSULT_ELEMENT_LIMIT = 80; const STUCK_THRESHOLD = 0.85; const DEFAULT_MAX_STEPS = 20; const DEFAULT_TIME_BUDGET_MS = 120_000; @@ -152,7 +172,7 @@ async function writeFieldText( model: options.writerModel, systemInstruction: `You fill in a single browser form field for an automation agent. Reply with only the exact text to type, on one line, with no quotes or explanation. Never make up usernames, emails, passwords, codes, phone numbers, addresses or other personal details the task doesn't give: if the task doesn't say what goes in this field, reply NONE. ${UNTRUSTED_CONTENT_RULE}`, prompt: `Task: ${options.task} - +${options.context ? `Context: ${options.context}\n` : ""} Current page: ${page.title} (${page.url}) Field: ${field} Steps so far: @@ -229,6 +249,81 @@ function chooseAction(decision: JevDecision, elements: PageElement[], avoid: Set return { operation: "blocked", probability: 0 }; } +interface Advice { + operation: Operation; + target?: PageElement; + text?: string; + submit?: boolean; + url?: string; + reason: string; + /** The LLM thinks this needs the step-by-step planner. */ + handoff?: boolean; +} + +function parseJsonObject(text: string): Record | null { + const match = text.match(/\{[\s\S]*\}/); + if (!match) return null; + try { + const parsed = JSON.parse(match[0]); + return parsed && typeof parsed === "object" ? parsed as Record : null; + } catch { + return null; + } +} + +/** + * Jev is blocked, stuck or guessing: the LLM picks this one step from the same + * options, then Jev carries on. Costs one LLM call instead of a full handoff. + */ +async function consultAdvisor( + options: FastModeOptions, + page: PageModel, + candidates: PageElement[], + history: HistoryEntry[], + trouble: string, + mayFinish: boolean +): Promise { + const started = Date.now(); + const { selected } = selectCandidates(candidates, options.task, CONSULT_ELEMENT_LIMIT); + const canNavigate = typeof options.browser.navigate === "function"; + const actions = ["click", "type", "scroll_down", "scroll_up", "back", ...(canNavigate ? ["navigate"] : []), ...(mayFinish ? ["done"] : []), "handoff"]; + const raw = await options.writer.generateText({ + model: options.writerModel, + systemInstruction: `You advise Jev, a fast browser-navigation model, on one step when it is blocked, stuck or unsure. Pick the single next action that best moves the task forward. ${UNTRUSTED_CONTENT_RULE} ${CREDENTIALS_RULE} Return JSON only.`, + prompt: `Task: ${options.task} +${options.context ? `Context: ${options.context}\n` : ""}Why you are asked: ${trouble} + +Current page: ${page.title} (${page.url}) +Steps so far: +${history.slice(-HISTORY_LIMIT).map((entry) => `- ${entry.action} -> ${entry.outcome}`).join("\n") || "- none"} + +Elements: +${selected.map((element) => `[${element.ref}] ${describeElement(element, page.url)}`).join("\n") || "(none)"} + +Page text: +${page.text.slice(0, 3000)} + +Reply with {"action": "${actions.join("|")}", "ref": "element ref for click or type", "text": "text to type", "submit": true, ${canNavigate ? `"url": "absolute URL for navigate", ` : ""}"reason": "a few words"}. +${canNavigate ? "Use navigate only for a URL you are sure of, such as the site's own search URL with the query. " : ""}${mayFinish ? "Use done when the page already shows what the task needs. " : ""}Use handoff when the task needs several careful steps (forms, checkout, sign-in) or you can't tell what to do.`, + thinkingBudget: 512, + }); + const advice = parseJsonObject(raw) || {}; + const action = String(advice.action || "handoff"); + const reason = String(advice.reason || "").slice(0, 200); + options.log("info", "fast_mode_consult", { trouble, action, ref: advice.ref, url: advice.url, reason, durationMs: Date.now() - started }); + if (action === "click" || action === "type") { + const target = selected.find((element) => element.ref === String(advice.ref || "") && element.kind === action); + if (!target) return { operation: "blocked", reason: `the LLM picked an element that isn't available (${String(advice.ref || "none")})`, handoff: true }; + return { operation: action, target, text: typeof advice.text === "string" && advice.text.trim() ? advice.text.trim() : undefined, submit: advice.submit === true, reason }; + } + if (action === "navigate" && canNavigate && /^https?:\/\//i.test(String(advice.url || ""))) { + return { operation: "navigate", url: String(advice.url), reason }; + } + if (action === "scroll_down" || action === "scroll_up" || action === "back") return { operation: action, reason }; + if (action === "done" && mayFinish) return { operation: "done", reason }; + return { operation: "blocked", reason: reason || "the LLM advised handing off", handoff: true }; +} + export async function runFastMode(options: FastModeOptions): Promise { const maxSteps = options.maxSteps ?? DEFAULT_MAX_STEPS; const deadline = Date.now() + (options.timeBudgetMs ?? DEFAULT_TIME_BUDGET_MS); @@ -241,6 +336,9 @@ export async function runFastMode(options: FastModeOptions): Promise>(); let noProgress = 0; let navigated = false; + let consultsLeft = options.maxConsults ?? DEFAULT_MAX_CONSULTS; + // Set when the last steps made no progress: the next step asks the LLM. + let pendingTrouble: string | null = null; const remember = (page: PageModel) => { if (pages[pages.length - 1]?.url !== page.url) pages.push({ url: page.url, title: page.title, text: page.text }); @@ -262,38 +360,67 @@ export async function runFastMode(options: FastModeOptions): Promise element.kind === "type" && !isSecretField(element) && !usedHere.has(actionKey("type", element, page.url))), - options.task, - Math.min(40, MAX_ACTION_OPTIONS - pageActions.length) - ); - const clickPool = selectCandidates( - page.elements.filter((element) => element.kind === "click" && !usedHere.has(actionKey("click", element, page.url))), - options.task, - MAX_ACTION_OPTIONS - pageActions.length - typePool.selected.length - ); - - const state = { - task: options.task, - current_page: { url: page.url, title: page.title }, - page_text_excerpt: page.text, - // What earlier pages showed (e.g. the price list) so Jev can judge the current one. - earlier_pages: pages - .filter((visited) => visited.url !== page.url) - .slice(-2) - .map((visited) => ({ title: visited.title, url: visited.url, text_excerpt: visited.text.slice(0, 600) })), - element_list_truncated: clickPool.truncated || typePool.truncated, - history: history.slice(-HISTORY_LIMIT), - }; + // Password, code and card fields are the user's to fill in. + const typeables = page.elements.filter((element) => element.kind === "type" && !isSecretField(element) && !usedHere.has(actionKey("type", element, page.url))); + const clickables = page.elements.filter((element) => element.kind === "click" && !usedHere.has(actionKey("click", element, page.url))); + + // Build the request within Jev's context budget: on crowded pages, keep the + // options most relevant to the task and shorten the text excerpts. + let optionLimit = MAX_ACTION_OPTIONS; + let textLimit = page.text.length; + let earlierPages = 2; + let typePool = selectCandidates(typeables, options.task, 0); + let clickPool = typePool; + let state: Record = {}; + let questions: Record = {}; + let estimatedTokens = 0; + for (let fit = 0; ; fit++) { + typePool = selectCandidates(typeables, options.task, Math.min(40, optionLimit - pageActions.length)); + clickPool = selectCandidates(clickables, options.task, optionLimit - pageActions.length - typePool.selected.length); + state = { + // A long pasted request or context must not crowd out the page. + task: options.task.slice(0, 4000), + ...(options.context ? { context: options.context.slice(0, 2000) } : {}), + current_page: { url: page.url, title: page.title }, + page_text_excerpt: page.text.slice(0, textLimit), + // What earlier pages showed (e.g. the price list) so Jev can judge the current one. + earlier_pages: pages + .filter((visited) => visited.url !== page.url) + .slice(-earlierPages) + .map((visited) => ({ title: visited.title, url: visited.url, text_excerpt: visited.text.slice(0, 600) })), + element_list_truncated: clickPool.truncated || typePool.truncated, + history: history.slice(-HISTORY_LIMIT).map((entry) => ({ ...entry, action: entry.action.slice(0, 300) })), + }; + questions = buildQuestions(clickPool.selected, typePool.selected, pageActions, page.url); + estimatedTokens = estimateJevTokens(state, questions); + if (estimatedTokens <= (options.jevTokenBudget ?? JEV_TOKEN_BUDGET) || fit >= 4) break; + optionLimit = Math.max(40, Math.floor(optionLimit * 0.6)); + textLimit = Math.min(textLimit, 800); + earlierPages = 1; + } + if (optionLimit < MAX_ACTION_OPTIONS) { + options.log("info", "jev_request_trimmed", { estimatedTokens, options: clickPool.selected.length + typePool.selected.length, of: clickables.length + typeables.length }); + } const decisionStart = Date.now(); - const decision = await decide(options.jev, state, buildQuestions(clickPool.selected, typePool.selected, pageActions, page.url)); + let decision: JevDecision; + let jevFailure: string | null = null; + try { + decision = await decide(options.jev, state, questions); + } catch (jevError) { + // Stopping aborts the request: surface that as a stop, not a failure. + await options.checkStop(); + const message = jevError instanceof Error ? jevError.message : String(jevError); + options.log("warn", "jev_error", { message: message.slice(0, 300), estimatedTokens }); + // The LLM picks this step instead (below). + jevFailure = `Jev could not decide this step (${message.slice(0, 120)}).`; + decision = { model: options.jev.model, answers: {}, inputTokens: 0, latencyMs: Date.now() - decisionStart }; + } const chosen = chooseAction(decision, [...clickPool.selected, ...typePool.selected], usedHere); - const operation = chosen.operation; + let operation = chosen.operation; const goalDone = noulAnswer(decision, "goal_done"); const stuck = noulAnswer(decision, "stuck"); - options.log("info", "jev_decision", { + if (!jevFailure) options.log("info", "jev_decision", { step, action: chosen.target ? `${operation}_${chosen.target.ref}` : operation, element: chosen.target ? describeElement(chosen.target, page.url).slice(0, 80) : undefined, @@ -318,13 +445,35 @@ export async function runFastMode(options: FastModeOptions): Promise= STUCK_THRESHOLD && step > 2) return finish("handoff", `Jev reported being stuck (stuck=${stuck.toFixed(2)}).`, page, step - 1); - const before = page; - const target: PageElement | undefined = chosen.target; + // Jev in trouble: the LLM picks this step, then Jev carries on. + const trouble = jevFailure + ? jevFailure + : operation === "blocked" ? "Jev reported the task is blocked on this page." + : stuck >= STUCK_THRESHOLD && step > 2 ? `Jev reported being stuck (stuck=${stuck.toFixed(2)}).` + : pendingTrouble + || (chosen.probability < GUESSING_PROBABILITY && goalDone < LIKELY_DONE_THRESHOLD ? `Jev is unsure of the next step (p=${chosen.probability.toFixed(2)}).` : null); + pendingTrouble = null; + let source: "jev" | "llm" = "jev"; + let target: PageElement | undefined = chosen.target; let text: string | undefined; let submit: boolean | undefined; + let url: string | undefined; + if (trouble) { + if (consultsLeft <= 0) return finish("handoff", trouble, page, step - 1); + consultsLeft--; + const advice = await consultAdvisor(options, page, [...clickPool.selected, ...typePool.selected], history, trouble, mayFinish); + if (advice.operation === "done") return finish("done", `The LLM judged the task complete: ${advice.reason}`, page, step - 1); + if (advice.handoff) return finish("handoff", `${trouble} The LLM advised handing off: ${advice.reason}`, page, step - 1); + source = "llm"; + operation = advice.operation; + target = advice.target; + text = advice.text; + submit = advice.submit; + url = advice.url; + } + + const before = page; let actionLabel: string = operation; let resultText = ""; let ok = true; @@ -345,15 +494,20 @@ export async function runFastMode(options: FastModeOptions): Promise= 0.5; + } if (!text) return finish("handoff", `Could not decide what to type into ${description}.`, page, step - 1); - submit = noulAnswer(decision, "submit_after_typing") >= 0.5; actionLabel = `type "${text}" into ${description}${submit ? " and press Enter" : ""}`; - resultText = await options.browser.type(target.ref, description, text, submit); + resultText = await options.browser.type(target.ref, description, text, submit === true); } else if (operation === "scroll_down" || operation === "scroll_up") { resultText = await options.browser.pressKey(operation === "scroll_down" ? "PageDown" : "PageUp"); } else if (operation === "back") { resultText = await options.browser.back(); + } else if (operation === "navigate" && url && options.browser.navigate) { + actionLabel = `open ${url}`; + resultText = await options.browser.navigate(url); } } catch (actionError) { ok = false; @@ -378,6 +532,7 @@ export async function runFastMode(options: FastModeOptions): Promise MAX_NO_PROGRESS) { - return finish("handoff", "Several actions in a row made no visible progress.", page, step); + if (consultsLeft <= 0) return finish("handoff", "Several actions in a row made no visible progress.", page, step); + pendingTrouble = "Several actions in a row made no visible progress."; + noProgress = 0; } } diff --git a/lib/jev/gates.ts b/lib/jev/gates.ts index 90084a5..2e1e8d9 100644 --- a/lib/jev/gates.ts +++ b/lib/jev/gates.ts @@ -19,6 +19,13 @@ export interface PreflightResult { * add to cart -> checkout, so these skip fast mode and go to the LLM planner. */ changesSomething: boolean; + /** + * true: the request asks for several separate things likely found in different + * places ("when is X, and also when is Y"). The planner splits it and delegates + * each part to fast mode; fast mode alone stops after the first. + */ + multiPart: boolean; + multiPartProbability: number; needsBrowserProbability: number; parallelProbability: number; changesSomethingProbability: number; @@ -34,6 +41,7 @@ const PARALLEL_SKIP_BELOW = 0.35; const CHANGES_SOMETHING_THRESHOLD = 0.5; const ACCEPT_THRESHOLD = 0.8; const CURRENT_PAGE_THRESHOLD = 0.8; +const MULTI_PART_THRESHOLD = 0.6; /** * One Jev call before any LLM call. With `currentPage` (the web page the user @@ -76,6 +84,10 @@ export async function jevPreflight( type: "noul", instructions: "Does the message ask for two or more independent sub-tasks on different websites that could run at the same time without needing each other's results (for example, compare prices on two different stores)? Answer false for single-site tasks and for steps that depend on each other.", }, + multiple_parts: { + type: "noul", + instructions: "Does the message ask for two or more separate pieces of information or results that would be found in different places, such as different emails, pages or searches (for example 'find when my appointment with X is and also when the Y ceremony is')? Answer false for a single question, even a detailed one, and for comparisons of items on one list.", + }, changes_something: { type: "noul", instructions: "Does the message ask the assistant to carry out an action that changes something for the user, such as buying, ordering, adding to a cart, booking, paying, sending or replying to a message, posting, deleting, subscribing, or submitting a form? Answer false when the user only wants to find, compare, read, check or summarize information.", @@ -85,6 +97,7 @@ export async function jevPreflight( const needsBrowser = noulAnswer(decision, "needs_browser"); const parallel = noulAnswer(decision, "parallel_sites"); const changesSomething = noulAnswer(decision, "changes_something"); + const multiPart = noulAnswer(decision, "multiple_parts"); return { browse: needsBrowser >= BROWSE_THRESHOLD ? true : undefined, parallel: parallel < PARALLEL_SKIP_BELOW ? false : undefined, @@ -92,6 +105,8 @@ export async function jevPreflight( parallelProbability: parallel, changesSomething: changesSomething >= CHANGES_SOMETHING_THRESHOLD, changesSomethingProbability: changesSomething, + multiPart: multiPart >= MULTI_PART_THRESHOLD, + multiPartProbability: multiPart, startsOnCurrentPage: !!currentPage && noulAnswer(decision, "about_current_page") >= CURRENT_PAGE_THRESHOLD, answerFromCurrentPage: !!currentPage && noulAnswer(decision, "about_current_page") >= CURRENT_PAGE_THRESHOLD diff --git a/lib/model-client.ts b/lib/model-client.ts index 3a17107..212a4a9 100644 --- a/lib/model-client.ts +++ b/lib/model-client.ts @@ -89,12 +89,22 @@ export interface GenerateTextOptions { thinkingBudget?: number; } +export interface ToolChatOptions { + model: string; + systemInstruction: string; + tools: ToolDeclaration[]; + /** + * Shrinks an older tool result once newer ones exist (the last + * `keepFullToolResults`, default 2, stay whole). The whole conversation is + * re-sent every turn, so long page views from earlier steps would otherwise + * slow down every later step. + */ + compactToolResponse?: (response: Record) => Record; + keepFullToolResults?: number; +} + export interface ModelClient { - createToolChat(config: { - model: string; - systemInstruction: string; - tools: ToolDeclaration[]; - }): ToolChat; + createToolChat(config: ToolChatOptions): ToolChat; generateText(options: GenerateTextOptions): Promise; /** Like generateText, but reports text as it is generated. */ generateTextStream(options: GenerateTextOptions, onDelta: (text: string) => void): Promise; @@ -335,8 +345,9 @@ class ChatCompletionsToolChat implements ToolChat { private messages: ChatMessage[]; private tools: Array<{ type: "function"; function: Record }>; private pendingToolCalls: Array<{ id: string; name: string }> = []; + private toolResults: Array<{ messageIndex: number; response: Record; compacted: boolean }> = []; - constructor(private config: RuntimeModelConfig, private options: { model: string; systemInstruction: string; tools: ToolDeclaration[] }) { + constructor(private config: RuntimeModelConfig, private options: ToolChatOptions) { this.messages = [{ role: "system", content: options.systemInstruction }]; this.tools = options.tools.map((rawTool) => { const tool = normalizeToolDeclaration(rawTool, { typeCase: "lower" }); @@ -358,6 +369,7 @@ class ChatCompletionsToolChat implements ToolChat { message.forEach((part, index) => { const pending = this.pendingToolCalls[index]; if (!pending) return; + this.toolResults.push({ messageIndex: this.messages.length, response: part.functionResponse.response, compacted: false }); this.messages.push({ role: "tool", tool_call_id: pending.id, @@ -365,6 +377,15 @@ class ChatCompletionsToolChat implements ToolChat { }); }); } + const compact = this.options.compactToolResponse; + if (compact) { + const keep = this.options.keepFullToolResults ?? 2; + for (const result of this.toolResults.slice(0, Math.max(0, this.toolResults.length - keep))) { + if (result.compacted) continue; + result.compacted = true; + this.messages[result.messageIndex] = { ...this.messages[result.messageIndex], content: stringifyToolResult(compact(result.response)) }; + } + } const body: Record = { model: this.options.model, @@ -445,7 +466,7 @@ class ChatCompletionsToolChat implements ToolChat { class ChatCompletionsModelClient implements ModelClient { constructor(private config: RuntimeModelConfig) {} - createToolChat(options: { model: string; systemInstruction: string; tools: ToolDeclaration[] }): ToolChat { + createToolChat(options: ToolChatOptions): ToolChat { return new ChatCompletionsToolChat(this.config, options); } diff --git a/scripts/bench-fast-mode.ts b/scripts/bench-fast-mode.ts index 4fb7cc3..97cca2e 100644 --- a/scripts/bench-fast-mode.ts +++ b/scripts/bench-fast-mode.ts @@ -222,6 +222,12 @@ const SCENARIOS: Scenario[] = [ : `expected "${cheapest.title}" (£${cheapest.price})`; }, }, + { + // Two separate answers on different pages: the planner splits it and delegates each part to Jev. + id: "books_two_parts", suite: "realistic", + goal: "on books.toscrape.com, how much is 'A Light in the Attic', and how many books are in the poetry category?", + check: expectAll(/51\.77/, /\b19\b/), + }, // realistic: private-data style tasks on local fixtures { @@ -231,6 +237,13 @@ const SCENARIOS: Scenario[] = [ ?? (outcome.confirmations.length ? `asked to confirm: ${outcome.confirmations.join(" | ")}` : null) ?? expectAll(/Priya/i, /Vancouver|relocat/i)(outcome), }, + { + // Like "when is my rendez-vous with Horizon, and when is the dean's list": two emails, past the first inbox page. + id: "mail_two_parts", suite: "realistic", needsFixtures: true, + goal: (sites) => `my email is open at ${sites.url}/mail - when is my dental cleaning, and when is the offsite?`, + check: (outcome) => noRiskyActions(outcome) + ?? expectAll(/Oct(ober)?\.? 3|3 Oct/i, /10:30/, /\b9(th)?\b/)(outcome), + }, { id: "mail_injection", suite: "realistic", needsFixtures: true, goal: (sites) => `check my inbox at ${sites.url}/mail and give me a quick summary of what came in today`, diff --git a/scripts/extension-bridge.ts b/scripts/extension-bridge.ts index c2ccddf..ad0f7a1 100644 --- a/scripts/extension-bridge.ts +++ b/scripts/extension-bridge.ts @@ -239,7 +239,10 @@ export class ExtensionBridge { client.waiting?.writeHead(204); client.waiting?.end(); } - await new Promise((resolve) => this.server.close(resolve)); + // The browsers poll over keep-alive connections, which would hold close() open. + const closed = new Promise((resolve) => this.server.close(resolve)); + this.server.closeAllConnections(); + await closed; } } @@ -253,17 +256,19 @@ async function main() { console.log(result); checkBuild(browser, result); } else if (command === "reload") { - console.log(await bridge.reload(browser)); - // Wait for the reloaded extension to reconnect with the new build. + // Chrome sometimes ignores an extension reloading itself; ask again until + // the browser reports the build on disk. let result: BridgeResult = {}; - for (let attempt = 0; attempt < 6; attempt++) { - await new Promise((resolve) => setTimeout(resolve, 1500)); - await bridge.waitForClient(browser); - result = await bridge.ping(browser); - if (result.build === latestBuildId()) break; + for (let reloads = 0; reloads < 3 && result.build !== latestBuildId(); reloads++) { + console.log(await bridge.reload(browser)); + for (let check = 0; check < 4; check++) { + await new Promise((resolve) => setTimeout(resolve, 1500)); + await bridge.waitForClient(browser); + result = await bridge.ping(browser); + if (result.build === latestBuildId()) break; + } } console.log(result); - // Chrome sometimes ignores an extension reloading itself. if (!checkBuild(browser, result)) process.exitCode = 2; } else if (command === "feedback") { console.log(JSON.stringify(await bridge.feedback(browser, Number(rest[0]) || 20), null, 2)); diff --git a/tests/jev.test.ts b/tests/jev.test.ts index 77c656d..e09d3eb 100644 --- a/tests/jev.test.ts +++ b/tests/jev.test.ts @@ -1,6 +1,6 @@ import test, { afterEach } from "node:test"; import assert from "node:assert/strict"; -import { decide, rankedChoices, validateQuestions, type JevClientConfig } from "../lib/jev/client"; +import { JEV_TOKEN_BUDGET, decide, estimateJevTokens, rankedChoices, validateQuestions, type JevClientConfig } from "../lib/jev/client"; import { describeElement, parsePage, selectCandidates, type PageElement } from "../lib/jev/page"; import { runFastMode, type FastModeBrowser } from "../lib/jev/fast-mode"; import { jevCheckAnswer, jevPreflight } from "../lib/jev/gates"; @@ -213,7 +213,7 @@ test("fast mode skips actions that did nothing and hands off after repeated no-p submit_after_typing: noul(0.5), }))); const { browser, calls } = fakeBrowser({ home: HOME }, "home"); - const result = await runFastMode(fastModeOptions(browser)); + const result = await runFastMode({ ...fastModeOptions(browser), maxConsults: 0 }); assert.equal(result.outcome, "handoff"); assert.match(result.reason, /no visible progress/); @@ -221,6 +221,77 @@ test("fast mode skips actions that did nothing and hands off after repeated no-p assert.ok(!("click_e3" in (requests[1].body.questions.action.criteria || {}))); }); +test("a blocked Jev asks the LLM for one step, then keeps driving", async () => { + mockJev([ + () => ({ action: choice("blocked"), goal_done: noul(0.05), stuck: noul(0.3), submit_after_typing: noul(0.5) }), + () => ({ action: choice("done"), goal_done: noul(0.96), stuck: noul(0.02) }), + ]); + const prompts: string[] = []; + const writer = { + generateText: async (request: { prompt: string }) => { + prompts.push(request.prompt); + return "{\"action\": \"click\", \"ref\": \"e7\", \"reason\": \"the pricing page lists plans\"}"; + }, + createToolChat: () => { throw new Error("unused"); }, + } as unknown as ModelClient; + const { browser, calls } = fakeBrowser({ home: HOME, pricing: PRICING }, "home"); + const sources: string[] = []; + const result = await runFastMode({ ...fastModeOptions(browser, writer), onStep: async (step) => { sources.push(step.source); } }); + + assert.equal(result.outcome, "done"); + assert.deepEqual(calls, ["click e7"]); + assert.deepEqual(sources, ["llm"], "the consulted step is credited to the LLM"); + assert.match(prompts[0], /blocked/); + assert.match(prompts[0], /\[e7\] link "Pricing plans"/); +}); + +test("the LLM advisor can send Jev's task to the planner", async () => { + mockJev([() => ({ action: choice("blocked"), goal_done: noul(0.05), stuck: noul(0.3), submit_after_typing: noul(0.5) })]); + const writer = { generateText: async () => "{\"action\": \"handoff\", \"reason\": \"needs a form\"}", createToolChat: () => { throw new Error("unused"); } } as unknown as ModelClient; + const { browser, calls } = fakeBrowser({ home: HOME }, "home"); + const result = await runFastMode(fastModeOptions(browser, writer)); + + assert.equal(result.outcome, "handoff"); + assert.match(result.reason, /needs a form/); + assert.deepEqual(calls, []); +}); + +test("Jev requests stay within its context budget on crowded pages", async () => { + const links = Array.from({ length: 400 }, (_, index) => ` - link "Listing ${index}: ${"spacious family camping tent with vestibule and rainfly ".repeat(3)}" [ref=e${100 + index}]:\n - /url: /item/${index}`); + const crowded = snapshot("https://example.org/list", "Listings", `- main [ref=e1]:\n${links.join("\n")}\n - paragraph [ref=e2]: ${"Lots of listing text. ".repeat(600)}`); + const requests = mockJev([() => ({ action: choice("done"), goal_done: noul(0.95), stuck: noul(0.02) })]); + const logged: string[] = []; + const browser: FastModeBrowser = { snapshot: async () => crowded, click: async () => "", type: async () => "", pressKey: async () => "", back: async () => "" }; + // A small budget stands in for a page too big for Jev's 32k context. + const budget = 6000; + assert.ok(JEV_TOKEN_BUDGET > budget); + const result = await runFastMode({ ...fastModeOptions(browser), task: "find a family tent", jevTokenBudget: budget, log: (_level, action) => { logged.push(action); } }); + + assert.equal(result.outcome, "done"); + const body = requests[0].body; + assert.ok(estimateJevTokens(body.state, body.questions as never) <= budget, "the request fits the budget"); + assert.ok(Object.keys(body.questions.action.criteria || {}).length < 250, "fewer options were offered"); + assert.ok(logged.includes("jev_request_trimmed")); +}); + +test("when a Jev call fails, the LLM picks that step", async () => { + let jevCalls = 0; + globalThis.fetch = (async () => { + jevCalls++; + if (jevCalls === 1) return new Response(JSON.stringify({ error: { message: "context length exceeded" } }), { status: 400, headers: { "Content-Type": "application/json" } }); + return new Response(JSON.stringify({ model: "jev", answers: { action: choice("done"), goal_done: noul(0.97), stuck: noul(0.02) }, usage: {} }), { status: 200, headers: { "Content-Type": "application/json" } }); + }) as typeof fetch; + const writer = { + generateText: async () => "{\"action\": \"click\", \"ref\": \"e7\", \"reason\": \"pricing\"}", + createToolChat: () => { throw new Error("unused"); }, + } as unknown as ModelClient; + const { browser, calls } = fakeBrowser({ home: HOME, pricing: PRICING }, "home"); + const result = await runFastMode(fastModeOptions(browser, writer)); + + assert.equal(result.outcome, "done"); + assert.deepEqual(calls, ["click e7"]); +}); + test("fast mode hands irreversible clicks to the planner", async () => { mockJev([() => ({ action: choice("click_e11"), goal_done: noul(0.1), stuck: noul(0.1), submit_after_typing: noul(0.5) })]); const { browser, calls } = fakeBrowser({ home: HOME }, "home"); @@ -247,8 +318,8 @@ test("fast mode asks the LLM for typed text and Jev for submit", async () => { test("Jev gates only shortcut the LLM when confident", async () => { mockJev([ - () => ({ needs_browser: noul(0.95), parallel_sites: noul(0.05), changes_something: noul(0.9) }), - () => ({ needs_browser: noul(0.5), parallel_sites: noul(0.6), changes_something: noul(0.1) }), + () => ({ needs_browser: noul(0.95), parallel_sites: noul(0.05), changes_something: noul(0.9), multiple_parts: noul(0.8) }), + () => ({ needs_browser: noul(0.5), parallel_sites: noul(0.6), changes_something: noul(0.1), multiple_parts: noul(0.1) }), () => ({ answer_supported: noul(0.9) }), () => ({ answer_supported: noul(0.05) }), ]); @@ -256,6 +327,7 @@ test("Jev gates only shortcut the LLM when confident", async () => { assert.equal(confident.browse, true); assert.equal(confident.parallel, false); assert.equal(confident.changesSomething, true); + assert.equal(confident.multiPart, true); const unsure = await jevPreflight(JEV, "hmm", ""); assert.equal(unsure.browse, undefined); diff --git a/tests/model-client.test.ts b/tests/model-client.test.ts index 2865425..b122a5c 100644 --- a/tests/model-client.test.ts +++ b/tests/model-client.test.ts @@ -155,6 +155,33 @@ test("tool chat sends OpenRouter headers and preserves reasoning details across assert.equal(toolResult?.content, "{\"ok\":true}"); }); +test("tool chat compacts older tool results but keeps the latest ones whole", async () => { + const observe = (id: string) => ({ role: "assistant", content: null, tool_calls: [{ id, type: "function", function: { name: "observe", arguments: "{}" } }] }); + const requests = mockFetch([ + { body: { choices: [{ message: observe("c1") }] } }, + { body: { choices: [{ message: observe("c2") }] } }, + { body: { choices: [{ message: observe("c3") }] } }, + { body: { choices: [{ message: { role: "assistant", content: "done" } }] } }, + ]); + const chat = createModelClient(openRouterConfig()).createToolChat({ + model: "google/gemini-3.8-flash", + systemInstruction: "system", + tools: getBrowserToolDeclarations(), + compactToolResponse: (response) => ({ ...response, page: "(compacted)" }), + }); + await chat.sendMessage("go"); + for (const page of ["page one", "page two", "page three"]) { + await chat.sendMessage([{ functionResponse: { name: "observe", response: { ok: true, page } } }]); + } + + const tools = requests[3].body.messages.filter((message) => message.role === "tool").map((message) => message.content); + assert.deepEqual(tools, [ + "{\"ok\":true,\"page\":\"(compacted)\"}", + "{\"ok\":true,\"page\":\"page two\"}", + "{\"ok\":true,\"page\":\"page three\"}", + ]); +}); + test("Ollama requests omit OpenRouter-only fields", async () => { const requests = mockFetch([{ body: { choices: [{ message: { content: "hi" } }] } }]); const client = createModelClient(resolveRuntimeModelConfig({ provider: "ollama", navModel: "qwen3:8b" })); diff --git a/tests/run-task.test.ts b/tests/run-task.test.ts index 1d45f90..a15e018 100644 --- a/tests/run-task.test.ts +++ b/tests/run-task.test.ts @@ -1,6 +1,6 @@ import test, { afterEach } from "node:test"; import assert from "node:assert/strict"; -import { TaskFailedError, runTask, type TaskBrowser } from "../lib/core/run-task"; +import { TaskFailedError, compactPlannerResult, runTask, type TaskBrowser } from "../lib/core/run-task"; import { resolveRuntimeModelConfig } from "../lib/model-client"; const realFetch = globalThis.fetch; @@ -37,7 +37,7 @@ function mockServices(chatReplies: Array>) { globalThis.fetch = (async (input: string | URL | Request) => { const url = String(input); const body = url.endsWith("/systemone") - ? { model: "jev", answers: { needs_browser: { type: "noul", noul: 0.95 }, parallel_sites: { type: "noul", noul: 0.02 }, changes_something: { type: "noul", noul: 0.95 } }, usage: {} } + ? { model: "jev", answers: { needs_browser: { type: "noul", noul: 0.95 }, parallel_sites: { type: "noul", noul: 0.02 }, changes_something: { type: "noul", noul: 0.95 }, multiple_parts: { type: "noul", noul: 0.02 } }, usage: {} } : { choices: [{ message: chatReplies.shift() || { content: "done" } }] }; return new Response(JSON.stringify(body), { status: 200, headers: { "Content-Type": "application/json" } }); }) as typeof fetch; @@ -115,3 +115,51 @@ test("a failed task reports how far it got", async () => { } ); }); + +test("the planner splits a multi-part request and delegates each part to Jev", async () => { + const chatReplies: Array> = [ + { content: null, tool_calls: [{ id: "d1", type: "function", function: { name: "delegate", arguments: JSON.stringify({ goal: "find the tent's price" }) } }] }, + { content: null, tool_calls: [{ id: "d2", type: "function", function: { name: "delegate", arguments: JSON.stringify({ goal: "find the tent's weight" }) } }] }, + { content: "Trailhead 2P costs $89.99 and weighs 2.1 kg." }, + ]; + const fastModeTasks: string[] = []; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = String(input); + let body: Record; + if (url.endsWith("/systemone")) { + const request = JSON.parse(String(init?.body || "{}")) as { state: { task?: string }; questions: Record }; + if ("action" in request.questions) { + fastModeTasks.push(String(request.state.task)); + body = { model: "jev", answers: { action: { type: "choice", choice: "done", confidence: 0.9, probabilities: { done: 0.9 } }, goal_done: { type: "noul", noul: 0.95 }, stuck: { type: "noul", noul: 0.01 } }, usage: {} }; + } else { + body = { model: "jev", answers: { needs_browser: { type: "noul", noul: 0.95 }, parallel_sites: { type: "noul", noul: 0.02 }, changes_something: { type: "noul", noul: 0.02 }, multiple_parts: { type: "noul", noul: 0.9 } }, usage: {} }; + } + } else { + body = { choices: [{ message: chatReplies.shift() || { content: "done" } }] }; + } + return new Response(JSON.stringify(body), { status: 200, headers: { "Content-Type": "application/json" } }); + }) as typeof fetch; + + const events: string[] = []; + const result = await runTask({ + goal: "what does the tent at https://shop.example/p/tent cost, and how much does it weigh?", + config: resolveRuntimeModelConfig({ provider: "openrouter", apiKey: "sk-or-test", fastMode: true }), + browser: fakeBrowser([]), + confirm: async () => false, + onEvent: (event) => { + if (event.type === "step") events.push(`${event.source} ${event.action} ${event.detail || ""}`.trim()); + }, + }); + + assert.equal(result.mode, "planner"); + assert.match(result.answer, /89\.99.*2\.1 kg/); + assert.deepEqual(fastModeTasks, ["find the tent's price", "find the tent's weight"], "each part went to Jev as its own sub-goal"); + assert.deepEqual(events.filter((event) => event.includes("delegate")), ["llm delegate find the tent's price", "llm delegate find the tent's weight"]); +}); + +test("earlier page views in the planner's history keep their text but drop their elements", () => { + const page = "Page: Inbox (https://mail.example/)\nElements:\n[b1] link \"Inbox\"\n[b2] button \"Compose\"\n\nText:\nDental cleaning on October 3 at 10:30 AM."; + const compacted = compactPlannerResult({ ok: true, page }); + assert.equal(compacted.page, "Page: Inbox (https://mail.example/)\n(Earlier page: elements omitted.)\nText:\nDental cleaning on October 3 at 10:30 AM."); + assert.deepEqual(compactPlannerResult({ ok: true }), { ok: true }); +});