From 0e21fb20441b3ccb6cff1a9416fb09cdd3870ef0 Mon Sep 17 00:00:00 2001 From: ahamSel <77988808+ahamSel@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:16:02 -0230 Subject: [PATCH] feat(extension): per-tab panels, conversations with memory, faster answers Extension: - The task engine runs in the background service worker with one session per tab (conversation, running task, pending confirmation); tasks keep running when the panel closes or the user switches tabs. Panels are tab-specific (sidePanel.setOptions per tab) and connect over a port. - Conversations are saved locally (chrome.storage.local, last 100) with a History view to reopen and continue them; + starts a new one. - Each turn keeps hidden memory (pages visited, where it ended, actions) that follow-ups use, navigating back when the tab has moved on. - Ctrl+Shift+Space opens WebPilot for the current tab; the input is focused on open; Esc stops a task; answers stream in. - Dev builds include a test bridge (dev-bridge.ts + scripts/ extension-bridge.ts) so the scenario harness can run in the user's real browsers and reload the extension after a rebuild; it only trusts a server holding the build's random secret. Release builds compile it out and drop its permissions. Core (lib/core/run-task.ts, lib/jev, lib/model-client.ts): - Jev's preflight also decides whether the request is about the open page and whether reading it is enough; "summarize this page" skips navigation and clicks and streams the answer. - Streaming text generation (generateTextStream, SSE). - When a fast-mode answer is unsupported, Jev gets a second pass that may not finish on that page (e.g. opens the email instead of answering from the search list) before the LLM planner takes over. - Cancel on a confirmation ends the task with a summary and memory, so "go ahead" can continue. Results in the user's real browsers through the dev bridge (realistic suite, fast mode): Chrome 9/9 median 7.6s, Edge 9/9 median 7.0s. Extension smoke 7/7 including a follow-up conversation. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 1 + extension/README.md | 35 +++- extension/manifest.json | 26 ++- extension/sidepanel.css | 38 ++++ extension/src/background.ts | 41 ++++- extension/src/conversations.ts | 51 ++++++ extension/src/dev-bridge.ts | 145 ++++++++++++++++ extension/src/engine.ts | 275 +++++++++++++++++++++++++++++ extension/src/protocol.ts | 76 ++++++++ extension/src/sidepanel.tsx | 307 ++++++++++++++++++--------------- lib/core/run-task.ts | 153 +++++++++++++--- lib/jev/fast-mode.ts | 20 ++- lib/jev/gates.ts | 37 +++- lib/model-client.ts | 89 ++++++++-- package.json | 1 + scripts/bench-fast-mode.ts | 70 +++++++- scripts/build-extension.mjs | 69 ++++++-- scripts/extension-bridge.ts | 252 +++++++++++++++++++++++++++ scripts/extension-smoke.ts | 105 ++++++++--- tests/jev.test.ts | 14 ++ tests/run-task.test.ts | 67 +++++++ 21 files changed, 1625 insertions(+), 247 deletions(-) create mode 100644 extension/src/conversations.ts create mode 100644 extension/src/dev-bridge.ts create mode 100644 extension/src/engine.ts create mode 100644 extension/src/protocol.ts create mode 100644 scripts/extension-bridge.ts create mode 100644 tests/run-task.test.ts diff --git a/.gitignore b/.gitignore index 86a6ad7..149f8ce 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,4 @@ next-env.d.ts # Screenshots (testing) /screenshots /extension/dist +/extension/.dev-bridge-token diff --git a/extension/README.md b/extension/README.md index 1c40bd8..33c6cbd 100644 --- a/extension/README.md +++ b/extension/README.md @@ -13,7 +13,14 @@ npm run extension:build Then open `chrome://extensions`, turn on **Developer mode**, click **Load unpacked**, and pick `extension/dist`. Click the WebPilot toolbar icon to open the side panel. -For development, `npm run extension:dev` rebuilds on every change; click the reload icon on the extension card to pick it up. +`extension:build` is the release build. For development use the dev build, which adds a test bridge (see Testing): `npm run extension:build:dev`, or `npm run extension:dev` to rebuild on every change. + +## Using it + +- Click the WebPilot icon, or press **Ctrl+Shift+Space** (change it at `chrome://extensions/shortcuts`), to open WebPilot **for the current tab**. Every tab has its own panel, conversation and running task; switching tabs switches panels, and a task keeps running if you close its panel or switch away. +- It's a conversation: follow-ups like "what about the second one?" or "open it" continue from the previous answers. Each turn also keeps hidden notes (pages visited, where it ended, what it clicked) so later messages can pick up from there, even after the tab has moved on. +- **History** (clock icon) lists past conversations. Opening one continues it in the current tab; WebPilot doesn't restore the old pages but knows what it did and navigates back if needed. **+** starts a new conversation. +- Enter sends, Shift+Enter adds a line, Esc stops a running task. ## First run @@ -23,11 +30,13 @@ Fast mode (Jev) is on by default. ## How it works -- The agent runs in the side panel page and controls the active tab through `chrome.debugger` (the Chrome DevTools Protocol). Chrome shows a "WebPilot started debugging this browser" bar while a task runs; it goes away when the task ends, and clicking **Cancel** on it stops the task. +- WebPilot controls the tab you ask it to through `chrome.debugger` (the Chrome DevTools Protocol), attaching only when a task needs the page. Chrome shows a "WebPilot started debugging this browser" bar while a task runs; it goes away when the task ends, and clicking **Cancel** on it stops the task. - Pages are read from Chrome's accessibility tree and acted on with real mouse and keyboard events, so sites treat them like your own input. - The same core runs in the desktop app and the extension: `lib/core/run-task.ts` (the task flow), `lib/cdp/` (the CDP driver and accessibility-tree snapshot), `lib/jev/` (fast mode), `lib/safety.ts` and `lib/model-client.ts`. -- Requests that only find or read information use fast mode: Jev picks each click in ~0.3s and the model writes text and the answer. Requests that change something (buy, book, send, delete, submit) use the careful model planner. -- Before an irreversible click (buy, place order, checkout, send, delete, publish, unsubscribe...) the panel shows **"Hold on — this can't be undone"** with Allow and Cancel. +- The agent runs in the extension's background service worker (`src/engine.ts`), one session per tab; panels are views connected over a port. While a task runs, the worker keeps itself alive. +- Requests about the open page that reading alone can answer ("summarize this", "what does this say about X") skip navigation and clicks and stream the answer straight from the page. Other requests that only find or read information use fast mode: Jev picks each click in ~0.3s and the model writes text and the answer. Requests that change something (buy, book, send, delete, submit) use the careful model planner. +- Answers stream into the panel as they are written; Jev then checks them against the page, and an unsupported answer is redone (first by Jev looking closer, then by the model). +- Before an irreversible click (buy, place order, checkout, send, delete, publish, unsubscribe...) the panel shows **"Hold on — this can't be undone"** with Allow and Cancel. Cancel ends the task there; say "go ahead" to continue. ## Permissions @@ -36,7 +45,7 @@ Fast mode (Jev) is on by default. | `sidePanel` | The WebPilot panel | | `debugger` | Reading and controlling the tab you ask it to work in | | `tabs` | Knowing which tab is active and its title | -| `storage` | Your settings and OpenRouter key (stored locally) | +| `storage` | Your settings, OpenRouter key and conversation history (stored locally) | | `identity` | "Connect OpenRouter" sign-in | | `https://openrouter.ai/*` | Model and Jev requests | @@ -44,7 +53,7 @@ Chrome does not allow extensions to control `chrome://` pages or the Chrome Web ## Privacy -Page text, element labels and URLs of the tab WebPilot works in are sent to OpenRouter to run the model, and in fast mode to TypeSafe (through OpenRouter) for Jev. Nothing is sent while no task is running. Settings stay in the browser's local extension storage. +Page text, element labels and URLs of the tab WebPilot works in are sent to OpenRouter to run the model, and in fast mode to TypeSafe (through OpenRouter) for Jev. Nothing is sent while no task is running. Settings and conversation history (at most the 100 most recent conversations) stay in the browser's local extension storage on this device; they are never synced or uploaded. History has Clear all. ## Testing @@ -61,3 +70,17 @@ The full scenario suite runs against the extension's engine too: ```bash npm run bench:fast-mode -- --engine cdp --suite all ``` + +### Testing in your own browsers (dev bridge) + +Dev builds include a bridge that lets the scenario harness run tasks inside the real browsers where the dev extension is installed, with their logins. Each task opens its own window and closes it afterwards, so your tabs are left alone. + +```bash +node scripts/build-extension.mjs --dev --out ~/WebPilot-extension # load this folder once per browser +npx tsx scripts/extension-bridge.ts ping chrome # or edge, brave... +npx tsx scripts/extension-bridge.ts reload edge # pick up a rebuild without clicking +npx tsx scripts/extension-bridge.ts run chrome "summarize this page" --url https://example.com +npm run bench:fast-mode -- --engine extension --browser chrome --suite realistic --modes fast +``` + +The extension polls `http://127.0.0.1:4466` and only accepts commands from a server that knows the random secret generated for that build (`extension/.dev-bridge-token`, gitignored). Release builds (`npm run extension:build`) compile the bridge out and drop its permissions (`alarms`, localhost). diff --git a/extension/manifest.json b/extension/manifest.json index 880026b..85a22af 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -11,19 +11,33 @@ "128": "icons/128x128.png" }, "action": { - "default_title": "Open WebPilot", + "default_title": "Open WebPilot (Ctrl+Shift+Space)", "default_icon": { "16": "icons/16x16.png", "32": "icons/32x32.png" } }, - "side_panel": { - "default_path": "sidepanel.html" - }, "background": { "service_worker": "background.js", "type": "module" }, - "permissions": ["sidePanel", "debugger", "tabs", "storage", "identity"], - "host_permissions": ["https://openrouter.ai/*", "http://127.0.0.1/*", "http://localhost/*"] + "permissions": [ + "sidePanel", + "debugger", + "tabs", + "storage", + "identity" + ], + "host_permissions": [ + "https://openrouter.ai/*" + ], + "commands": { + "_execute_action": { + "suggested_key": { + "default": "Ctrl+Shift+Space", + "mac": "MacCtrl+Shift+Space" + }, + "description": "Open WebPilot for this tab" + } + } } diff --git a/extension/sidepanel.css b/extension/sidepanel.css index ca93ec4..67a5069 100644 --- a/extension/sidepanel.css +++ b/extension/sidepanel.css @@ -366,3 +366,41 @@ button { font: inherit; color: inherit; } @media (prefers-reduced-motion: reduce) { *, *::before, *::after { animation: none !important; transition: none !important; } } + +/* ---------- history ---------- */ +.history-item { + display: flex; + align-items: center; + gap: 4px; + margin-bottom: 6px; + border: 1px solid var(--wp-border-soft); + border-radius: var(--wp-radius-md); + background: var(--wp-surface); + animation: rise 0.2s ease-out both; +} +.history-item.current { border-color: var(--wp-accent-glow); } +.history-open { + flex: 1; + min-width: 0; + text-align: left; + padding: 8px 10px; + border: 0; + background: transparent; + cursor: pointer; +} +.history-title { display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.history-open small { display: block; color: var(--wp-text-faint); font-size: 11px; } +.history-item:hover { border-color: var(--wp-border); } + +/* Answer text still streaming in. */ +.answer.streaming > :last-child::after { + content: ""; + display: inline-block; + width: 7px; + height: 13px; + margin-left: 3px; + vertical-align: -2px; + background: var(--wp-accent); + animation: blink 0.9s steps(2) infinite; +} +@keyframes blink { 50% { opacity: 0; } } diff --git a/extension/src/background.ts b/extension/src/background.ts index 9572daa..bf61e0c 100644 --- a/extension/src/background.ts +++ b/extension/src/background.ts @@ -1,11 +1,40 @@ /// -// The toolbar button opens WebPilot in the side panel. The agent itself runs in -// the side panel page, which stays alive while it is open (MV3 service workers -// are suspended when idle, so they cannot host a long-running task). +import { startDevBridge } from "./dev-bridge"; +import { Engine } from "./engine"; +import { PANEL_PORT_PREFIX } from "./protocol"; + +declare const __DEV_BRIDGE__: boolean; + +const engine = new Engine(); + +/** + * Every tab gets its own side panel (and its own conversation): clicking the + * toolbar button or pressing the shortcut opens the panel for the current tab + * only. Both calls stay synchronous so they run inside the user gesture. + */ +function openPanel(tabId: number) { + chrome.sidePanel.setOptions({ tabId, path: `sidepanel.html?tabId=${tabId}`, enabled: true }).catch(() => {}); + chrome.sidePanel.open({ tabId }).catch(() => {}); +} + chrome.runtime.onInstalled.addListener(() => { - chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true }).catch(() => {}); + // Earlier versions opened one shared panel on click; per-tab panels are opened here instead. + chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: false }).catch(() => {}); +}); + +chrome.action.onClicked.addListener((tab) => { + if (tab.id !== undefined) openPanel(tab.id); }); -chrome.runtime.onStartup.addListener(() => { - chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true }).catch(() => {}); + +chrome.runtime.onConnect.addListener((port) => { + if (!port.name.startsWith(PANEL_PORT_PREFIX)) return; + const tabId = Number(port.name.slice(PANEL_PORT_PREFIX.length)); + if (Number.isInteger(tabId)) engine.connect(port, tabId); }); + +chrome.tabs.onRemoved.addListener((tabId) => { + engine.dispose(tabId); +}); + +if (__DEV_BRIDGE__) startDevBridge(engine); diff --git a/extension/src/conversations.ts b/extension/src/conversations.ts new file mode 100644 index 0000000..8845291 --- /dev/null +++ b/extension/src/conversations.ts @@ -0,0 +1,51 @@ +/// + +/** + * Local conversation history: each conversation (including the hidden browser + * memory of its turns) is stored in chrome.storage.local, on this device only, + * never synced. An index lists them for the History view; only the most recent + * MAX_CONVERSATIONS are kept. + */ + +import type { Conversation, ConversationSummary } from "./protocol"; + +const INDEX_KEY = "webpilot.conversations"; +const CONVERSATION_PREFIX = "webpilot.conversation."; +const MAX_CONVERSATIONS = 100; +const TITLE_LENGTH = 70; + +export function newConversationId(): string { + return `c${Date.now().toString(36)}${Math.random().toString(36).slice(2, 7)}`; +} + +export async function listConversations(): Promise { + const index = (await chrome.storage.local.get(INDEX_KEY))[INDEX_KEY] as ConversationSummary[] | undefined; + return (index || []).slice().sort((left, right) => right.updatedAt - left.updatedAt); +} + +export async function loadConversation(id: string): Promise { + const key = `${CONVERSATION_PREFIX}${id}`; + return ((await chrome.storage.local.get(key))[key] as Conversation | undefined) || null; +} + +export async function saveConversation(conversation: Conversation): Promise { + if (!conversation.turns.length) return; + const title = conversation.turns[0].user.replace(/\s+/g, " ").slice(0, TITLE_LENGTH); + const saved: Conversation = { ...conversation, title, turns: conversation.turns.map((turn) => ({ ...turn, streaming: undefined })) }; + const summary: ConversationSummary = { id: saved.id, title, updatedAt: saved.updatedAt, turnCount: saved.turns.length }; + const index = [summary, ...(await listConversations()).filter((item) => item.id !== saved.id)]; + const dropped = index.slice(MAX_CONVERSATIONS); + await chrome.storage.local.set({ [`${CONVERSATION_PREFIX}${saved.id}`]: saved, [INDEX_KEY]: index.slice(0, MAX_CONVERSATIONS) }); + if (dropped.length) await chrome.storage.local.remove(dropped.map((item) => `${CONVERSATION_PREFIX}${item.id}`)); +} + +export async function deleteConversation(id: string): Promise { + const index = (await listConversations()).filter((item) => item.id !== id); + await chrome.storage.local.set({ [INDEX_KEY]: index }); + await chrome.storage.local.remove(`${CONVERSATION_PREFIX}${id}`); +} + +export async function clearConversations(): Promise { + const index = await listConversations(); + await chrome.storage.local.remove([INDEX_KEY, ...index.map((item) => `${CONVERSATION_PREFIX}${item.id}`)]); +} diff --git a/extension/src/dev-bridge.ts b/extension/src/dev-bridge.ts new file mode 100644 index 0000000..5373a9d --- /dev/null +++ b/extension/src/dev-bridge.ts @@ -0,0 +1,145 @@ +/// + +/** + * Dev-only bridge: lets the scenario harness (scripts/extension-bridge.ts) run + * tasks in the real browser the extension is installed in. Only dev builds + * (`npm run extension:build:dev`) include it; release builds compile it out. + * + * The extension long-polls http://127.0.0.1:4466 and accepts a command only if + * the response carries the random secret generated for this build, so another + * local process cannot drive the user's browser. Each task runs in a new window + * that is closed afterwards, leaving the user's own tabs alone. + */ + +import type { TaskEvent } from "../../lib/core/run-task"; +import type { Engine } from "./engine"; + +declare const __DEV_BRIDGE_TOKEN__: string; + +const BRIDGE_URL = "http://127.0.0.1:4466"; +const RETRY_MS = 3000; + +interface RunCommand { + id: string; + type: "run"; + goal: string; + url?: string; + confirm?: "allow" | "deny"; + fastMode?: boolean; + keepOpen?: boolean; +} + +type Command = RunCommand | { id: string; type: "reload" } | { id: string; type: "ping" }; + +function browserName(): string { + const brands = (navigator as Navigator & { userAgentData?: { brands?: Array<{ brand: string }> } }).userAgentData?.brands || []; + const names = brands.map((brand) => brand.brand.toLowerCase()); + if (names.some((name) => name.includes("edge"))) return "edge"; + if (names.some((name) => name.includes("brave"))) return "brave"; + if (names.some((name) => name.includes("opera"))) return "opera"; + if (names.some((name) => name.includes("google chrome"))) return "chrome"; + return "chromium"; +} + +async function post(path: string, body: unknown): Promise { + return fetch(`${BRIDGE_URL}${path}`, { + method: "POST", + headers: { "Content-Type": "application/json", "x-webpilot-bridge": __DEV_BRIDGE_TOKEN__ }, + body: JSON.stringify(body), + }); +} + +function waitForTabComplete(tabId: number): Promise { + return new Promise((resolve) => { + const timer = setTimeout(done, 15_000); + function done() { + clearTimeout(timer); + chrome.tabs.onUpdated.removeListener(listener); + resolve(); + } + function listener(updatedId: number, change: { status?: string }) { + if (updatedId === tabId && change.status === "complete") done(); + } + chrome.tabs.onUpdated.addListener(listener); + }); +} + +async function runCommand(engine: Engine, command: RunCommand) { + const window = await chrome.windows.create({ url: command.url || "about:blank", focused: false, width: 1280, height: 860 }); + const tabId = window?.tabs?.[0]?.id; + if (tabId === undefined) { + await post("/bridge/result", { id: command.id, error: "Could not open a test window." }); + return; + } + if (command.url) await waitForTabComplete(tabId); + const started = Date.now(); + try { + const turn = await engine.run(tabId, command.goal, { + overrides: typeof command.fastMode === "boolean" ? { fastMode: command.fastMode } : undefined, + // The task reports the confirmation itself (a "confirm" event); just answer it. + confirm: async () => command.confirm === "allow", + onEvent: (event: TaskEvent) => { + if (event.type === "answer-delta") return; + post("/bridge/event", { id: command.id, event }).catch(() => {}); + }, + }); + await post("/bridge/result", { + id: command.id, + answer: turn.answer || "", + error: turn.error, + mode: turn.mode, + stats: turn.stats, + steps: turn.steps.length, + durationMs: Date.now() - started, + }); + } catch (error) { + await post("/bridge/result", { id: command.id, error: error instanceof Error ? error.message : String(error) }); + } finally { + if (!command.keepOpen && window?.id !== undefined) await chrome.windows.remove(window.id).catch(() => {}); + } +} + +let polling = false; + +async function pollLoop(engine: Engine) { + if (polling) return; + polling = true; + const client = browserName(); + try { + for (;;) { + let response: Response; + try { + response = await fetch(`${BRIDGE_URL}/bridge/next?client=${client}`, { headers: { "x-webpilot-bridge": __DEV_BRIDGE_TOKEN__ } }); + } catch { + await new Promise((resolve) => setTimeout(resolve, RETRY_MS)); + continue; + } + // Only trust a server that knows this build's secret. + if (response.status !== 200 || response.headers.get("x-webpilot-bridge") !== __DEV_BRIDGE_TOKEN__) { + await response.body?.cancel().catch(() => {}); + if (response.status !== 204) await new Promise((resolve) => setTimeout(resolve, RETRY_MS)); + continue; + } + const command = await response.json() as Command; + if (command.type === "ping") { + await post("/bridge/result", { id: command.id, client, version: chrome.runtime.getManifest().version }).catch(() => {}); + } else if (command.type === "reload") { + await post("/bridge/result", { id: command.id, reloading: true }).catch(() => {}); + chrome.runtime.reload(); + } else if (command.type === "run") { + runCommand(engine, command).catch(() => {}); + } + } + } finally { + polling = false; + } +} + +export function startDevBridge(engine: Engine) { + pollLoop(engine); + // The service worker is suspended when idle; the alarm wakes it to resume polling. + chrome.alarms.create("webpilot-dev-bridge", { periodInMinutes: 0.5 }); + chrome.alarms.onAlarm.addListener((alarm) => { + if (alarm.name === "webpilot-dev-bridge") pollLoop(engine); + }); +} diff --git a/extension/src/engine.ts b/extension/src/engine.ts new file mode 100644 index 0000000..8332f6f --- /dev/null +++ b/extension/src/engine.ts @@ -0,0 +1,275 @@ +/// + +/** + * The task engine, running in the extension's background service worker. + * + * Each tab has its own session: the conversation it shows, at most one running + * task and any pending confirmation. Tasks keep running when the panel is closed + * or the user switches tabs; panels are views that connect over a port. + * Conversations (with each turn's hidden browser memory) are saved locally + * (conversations.ts) so they can be reopened from History and continued; which + * conversation a tab shows is kept in chrome.storage.session. + */ + +import { TaskCancelledError, runTask, type TaskEvent } from "../../lib/core/run-task"; +import { loadSettings, modelConfigFor, type ExtensionSettings } from "./settings"; +import { clearConversations, deleteConversation, listConversations, loadConversation, newConversationId, saveConversation } from "./conversations"; +import type { EngineMessage, LogStep, PanelMessage, SessionState, Turn } from "./protocol"; +import { TabBrowser } from "./tab-browser"; + +const TAB_KEY_PREFIX = "webpilot.tab."; +const KEEPALIVE_MS = 20_000; +const BROADCAST_THROTTLE_MS = 60; + +const ACTION_LABELS: Record = { + click: "click", + type: "type", + scroll_down: "scroll down", + scroll_up: "scroll up", + back: "go back", + navigate: "open", + scroll: "scroll", + wait: "wait", +}; + +export interface RunHooks { + /** Overrides asking the user (the dev bridge answers confirmations itself). */ + confirm?: (action: string, url: string) => Promise; + onEvent?: (event: TaskEvent) => void; + /** Per-run settings overrides (the dev bridge compares fast mode on and off). */ + overrides?: Partial; +} + +interface Session { + state: SessionState; + createdAt: number; + controller?: AbortController; + confirmResolve?: (allowed: boolean) => void; + broadcastTimer?: ReturnType; +} + +function newId(): string { + return Math.random().toString(36).slice(2, 10); +} + +export class Engine { + private sessions = new Map(); + private ports = new Map>(); + private keepalive: ReturnType | null = null; + + private async session(tabId: number): Promise { + const existing = this.sessions.get(tabId); + if (existing) return existing; + const key = `${TAB_KEY_PREFIX}${tabId}`; + const conversationId = (await chrome.storage.session.get(key))[key] as string | undefined; + const conversation = conversationId ? await loadConversation(conversationId) : null; + const created = this.sessionFor(tabId, conversation?.id || newConversationId(), conversation?.turns || [], conversation?.createdAt); + this.sessions.set(tabId, created); + return created; + } + + private sessionFor(tabId: number, conversationId: string, turns: Turn[], createdAt = Date.now()): Session { + // A turn without an end was cut off by a service-worker restart. + const settled = turns.map((turn) => (turn.finishedAt ? turn : { ...turn, streaming: undefined, error: turn.error || "Interrupted.", finishedAt: Date.now() })); + chrome.storage.session.set({ [`${TAB_KEY_PREFIX}${tabId}`]: conversationId }).catch(() => {}); + return { state: { tabId, conversationId, turns: settled, running: false, pending: null }, createdAt }; + } + + private persist(session: Session) { + saveConversation({ + id: session.state.conversationId, + title: "", + createdAt: session.createdAt, + updatedAt: Date.now(), + turns: session.state.turns, + }).catch(() => {}); + } + + private async sendHistory(tabId: number) { + const items = await listConversations(); + const message: EngineMessage = { type: "history", items }; + for (const port of this.ports.get(tabId) || []) { + try { + port.postMessage(message); + } catch { + // The panel went away. + } + } + } + + private broadcast(session: Session, immediate = false) { + const send = () => { + session.broadcastTimer = undefined; + const message: EngineMessage = { type: "state", state: session.state }; + for (const port of this.ports.get(session.state.tabId) || []) { + try { + port.postMessage(message); + } catch { + // The panel went away; onDisconnect cleans up. + } + } + }; + if (immediate) { + if (session.broadcastTimer) clearTimeout(session.broadcastTimer); + send(); + } else if (!session.broadcastTimer) { + session.broadcastTimer = setTimeout(send, BROADCAST_THROTTLE_MS); + } + } + + private updateKeepalive() { + const running = [...this.sessions.values()].some((session) => session.state.running); + if (running && !this.keepalive) { + // Extension API calls reset the service worker's idle timer while a task runs. + this.keepalive = setInterval(() => chrome.runtime.getPlatformInfo().catch(() => {}), KEEPALIVE_MS); + } else if (!running && this.keepalive) { + clearInterval(this.keepalive); + this.keepalive = null; + } + } + + async connect(port: chrome.runtime.Port, tabId: number) { + const ports = this.ports.get(tabId) || new Set(); + ports.add(port); + this.ports.set(tabId, ports); + port.onDisconnect.addListener(() => ports.delete(port)); + port.onMessage.addListener((message: PanelMessage) => this.handle(tabId, message)); + this.broadcast(await this.session(tabId), true); + } + + private async handle(tabId: number, message: PanelMessage) { + if (message.type === "run") this.run(tabId, message.goal).catch(() => {}); + if (message.type === "stop") await this.stop(tabId); + if (message.type === "confirm") await this.resolveConfirm(tabId, message.id, message.allowed); + if (message.type === "clear") await this.clear(tabId); + if (message.type === "history") await this.sendHistory(tabId); + if (message.type === "open") await this.open(tabId, message.conversationId); + if (message.type === "delete") { + await deleteConversation(message.conversationId); + await this.sendHistory(tabId); + } + if (message.type === "clear-history") { + await clearConversations(); + await this.sendHistory(tabId); + } + } + + async run(tabId: number, goal: string, hooks: RunHooks = {}): Promise { + const session = await this.session(tabId); + if (session.state.running) throw new Error("A task is already running in this tab."); + const settings = { ...(await loadSettings()), ...hooks.overrides }; + const turn: Turn = { id: newId(), user: goal.trim(), steps: [], startedAt: Date.now() }; + const history = session.state.turns.filter((item) => item.answer).map((item) => ({ user: item.user, assistant: item.answer!, memory: item.memory })); + session.state = { ...session.state, turns: [...session.state.turns, turn], running: true }; + session.controller = new AbortController(); + this.updateKeepalive(); + this.broadcast(session, true); + this.persist(session); + + const update = (patch: (current: Turn) => Turn, immediate = false) => { + session.state = { + ...session.state, + turns: session.state.turns.map((item) => (item.id === turn.id ? patch(item) : item)), + }; + this.broadcast(session, immediate); + }; + const addStep = (step: Omit) => update((current) => ({ ...current, steps: [...current.steps, { ...step, atMs: Date.now() - turn.startedAt }] })); + + const browser = new TabBrowser(tabId); + try { + if (!settings.apiKey) throw new Error("Connect OpenRouter in Settings first."); + const result = await runTask({ + goal: turn.user, + config: modelConfigFor(settings), + browser, + history, + signal: session.controller.signal, + confirm: (action, url) => new Promise((resolve) => { + const id = newId(); + session.state = { ...session.state, pending: { id, action, url } }; + this.broadcast(session, true); + let settled = false; + const settle = (allowed: boolean) => { + if (settled) return; + settled = true; + session.confirmResolve = undefined; + session.state = { ...session.state, pending: null }; + addStep({ lane: "note", label: allowed ? "allowed" : "cancelled", detail: action }); + resolve(allowed); + }; + session.confirmResolve = settle; + hooks.confirm?.(action, url).then(settle, () => settle(false)); + }), + onEvent: (event) => { + hooks.onEvent?.(event); + if (event.type === "step") { + addStep({ lane: event.source, label: ACTION_LABELS[event.action] || event.action, detail: event.detail ? event.detail.slice(0, 140) : undefined }); + } else if (event.type === "handoff") { + addStep({ lane: "note", label: "handing to the model", detail: event.reason }); + } else if (event.type === "status") { + addStep({ lane: "note", label: event.message.toLowerCase() }); + } else if (event.type === "answer-delta") { + update((current) => ({ ...current, streaming: (current.streaming || "") + event.text })); + } else if (event.type === "answer-reset") { + update((current) => ({ ...current, streaming: undefined })); + } + }, + }); + update((current) => ({ ...current, streaming: undefined, answer: result.answer, mode: result.mode, memory: result.memory, finishedAt: Date.now(), stats: { jevCalls: result.jevCalls, llmCalls: result.llmCalls } }), true); + } catch (error) { + const message = error instanceof TaskCancelledError ? "Stopped." : error instanceof Error ? error.message : String(error); + update((current) => ({ ...current, streaming: undefined, error: message, finishedAt: Date.now() }), true); + } finally { + await browser.detach(); + session.controller = undefined; + session.confirmResolve = undefined; + session.state = { ...session.state, running: false, pending: null }; + this.broadcast(session, true); + this.persist(session); + this.updateKeepalive(); + } + return session.state.turns.find((item) => item.id === turn.id)!; + } + + async stop(tabId: number) { + const session = await this.session(tabId); + session.controller?.abort(); + session.confirmResolve?.(false); + } + + async resolveConfirm(tabId: number, id: string, allowed: boolean) { + const session = await this.session(tabId); + if (session.state.pending?.id === id) session.confirmResolve?.(allowed); + } + + /** Starts a new conversation in the tab; the old one stays in History. */ + async clear(tabId: number) { + const session = await this.session(tabId); + if (session.state.running) return; + const fresh = this.sessionFor(tabId, newConversationId(), []); + this.sessions.set(tabId, fresh); + this.broadcast(fresh, true); + } + + /** Shows a saved conversation in the tab so it can be continued. */ + async open(tabId: number, conversationId: string) { + const session = await this.session(tabId); + if (session.state.running) return; + const conversation = await loadConversation(conversationId); + if (!conversation) return; + const opened = this.sessionFor(tabId, conversation.id, conversation.turns, conversation.createdAt); + this.sessions.set(tabId, opened); + this.broadcast(opened, true); + } + + /** The tab closed: stop its task. */ + async dispose(tabId: number) { + const session = this.sessions.get(tabId); + session?.controller?.abort(); + session?.confirmResolve?.(false); + this.sessions.delete(tabId); + this.ports.delete(tabId); + // The conversation itself stays in History. + await chrome.storage.session.remove(`${TAB_KEY_PREFIX}${tabId}`).catch(() => {}); + } +} diff --git a/extension/src/protocol.ts b/extension/src/protocol.ts new file mode 100644 index 0000000..ae497f1 --- /dev/null +++ b/extension/src/protocol.ts @@ -0,0 +1,76 @@ +/** Messages and state shared by the background engine, side panels and the dev bridge. */ + +import type { TaskMemory } from "../../lib/core/run-task"; + +export interface LogStep { + lane: "jev" | "llm" | "note"; + label: string; + detail?: string; + atMs: number; +} + +export interface Turn { + id: string; + user: string; + /** Final answer. */ + answer?: string; + /** Answer text streaming in while the task runs. */ + streaming?: string; + error?: string; + steps: LogStep[]; + startedAt: number; + finishedAt?: number; + mode?: "chat" | "fast" | "planner"; + stats?: { jevCalls: number; llmCalls: number }; + /** Hidden browser state for follow-ups; never shown in the chat. */ + memory?: TaskMemory; +} + +/** A saved conversation, stored locally in chrome.storage.local. */ +export interface Conversation { + id: string; + title: string; + createdAt: number; + updatedAt: number; + turns: Turn[]; +} + +export interface ConversationSummary { + id: string; + title: string; + updatedAt: number; + turnCount: number; +} + +export interface PendingConfirm { + id: string; + action: string; + url: string; +} + +/** Everything one tab's panel shows. */ +export interface SessionState { + tabId: number; + conversationId: string; + turns: Turn[]; + running: boolean; + pending: PendingConfirm | null; +} + +export type PanelMessage = + | { type: "run"; goal: string } + | { type: "stop" } + | { type: "confirm"; id: string; allowed: boolean } + /** Start a new conversation in this tab. */ + | { type: "clear" } + | { type: "history" } + | { type: "open"; conversationId: string } + | { type: "delete"; conversationId: string } + | { type: "clear-history" }; + +export type EngineMessage = + | { type: "state"; state: SessionState } + | { type: "history"; items: ConversationSummary[] }; + +/** Panels connect with chrome.runtime.connect({ name: PANEL_PORT_PREFIX + tabId }). */ +export const PANEL_PORT_PREFIX = "webpilot-panel:"; diff --git a/extension/src/sidepanel.tsx b/extension/src/sidepanel.tsx index 7a8b408..bffb8ae 100644 --- a/extension/src/sidepanel.tsx +++ b/extension/src/sidepanel.tsx @@ -1,37 +1,11 @@ /// -import { useCallback, useEffect, useRef, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { createRoot } from "react-dom/client"; import ReactMarkdown from "react-markdown"; -import { TaskCancelledError, runTask, type TaskEvent } from "../../lib/core/run-task"; -import { TabBrowser } from "./tab-browser"; import { connectOpenRouter } from "./openrouter-auth"; -import { loadSettings, modelConfigFor, saveSettings, type ExtensionSettings } from "./settings"; - -interface LogStep { - lane: "jev" | "llm" | "note"; - label: string; - detail?: string; - atMs: number; -} - -interface Turn { - id: number; - user: string; - answer?: string; - error?: string; - steps: LogStep[]; - startedAt: number; - finishedAt?: number; - mode?: "chat" | "fast" | "planner"; - stats?: { jevCalls: number; llmCalls: number }; -} - -interface PendingConfirm { - action: string; - url: string; - resolve: (allowed: boolean) => void; -} +import { PANEL_PORT_PREFIX, type ConversationSummary, type EngineMessage, type PanelMessage, type PendingConfirm, type SessionState, type Turn } from "./protocol"; +import { loadSettings, saveSettings, type ExtensionSettings } from "./settings"; const EXAMPLES = [ { text: "Summarize this page", note: "Reads the tab you have open" }, @@ -39,17 +13,6 @@ const EXAMPLES = [ { text: "What's trending on Hacker News right now?", note: "Opens the site and reads the front page" }, ]; -const ACTION_LABELS: Record = { - click: "click", - type: "type", - scroll_down: "scroll down", - scroll_up: "scroll up", - back: "go back", - navigate: "open", - scroll: "scroll", - wait: "wait", -}; - function seconds(ms: number): string { return `${(ms / 1000).toFixed(1)}s`; } @@ -62,18 +25,78 @@ function hostOf(url: string): string { } } -/** - * The tab WebPilot works in: the active tab of this window. When the panel page - * itself is open as a tab (e.g. in tests), the most recently used other tab. - */ -async function activeTab(): Promise { - const panelUrl = chrome.runtime.getURL(""); +/** The tab this panel belongs to: from the panel URL, or the active tab as a fallback. */ +async function panelTabId(): Promise { + const fromUrl = Number(new URLSearchParams(location.search).get("tabId")); + if (Number.isInteger(fromUrl) && fromUrl > 0) return fromUrl; const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }); - if (tab && !tab.url?.startsWith(panelUrl)) return tab; - const tabs = await chrome.tabs.query({ currentWindow: true }); - return tabs - .filter((candidate) => !(candidate.url || "").startsWith(panelUrl)) - .sort((left, right) => (right.lastAccessed || 0) - (left.lastAccessed || 0))[0]; + return tab?.id; +} + +function HistoryIcon() { + return ( + + ); +} + +function TrashIcon() { + return ( + + ); +} + +function relativeTime(timestamp: number): string { + const minutes = Math.round((Date.now() - timestamp) / 60_000); + if (minutes < 1) return "just now"; + if (minutes < 60) return `${minutes}m ago`; + const hours = Math.round(minutes / 60); + if (hours < 24) return `${hours}h ago`; + const days = Math.round(hours / 24); + return days < 7 ? `${days}d ago` : new Date(timestamp).toLocaleDateString(); +} + +function History({ items, currentId, onOpen, onDelete, onClearAll, onClose }: { + items: ConversationSummary[] | null; + currentId?: string; + onOpen: (id: string) => void; + onDelete: (id: string) => void; + onClearAll: () => void; + onClose: () => void; +}) { + return ( +
+
+ +

History

+ {!!items?.length && } +
+ {items === null &&

Loading…

} + {items?.length === 0 &&

No conversations yet. They are saved on this device only.

} + {items?.map((item) => ( +
+ + +
+ ))} + {!!items?.length &&

Saved on this device only. Opening one continues it in this tab; WebPilot remembers what it did.

} +
+ ); +} + +function ClearIcon() { + return ( + + ); } function Logo() { @@ -147,7 +170,7 @@ function FlightLog({ turn, live }: { turn: Turn; live: boolean }) { ); } -function ConfirmCard({ pending }: { pending: PendingConfirm }) { +function ConfirmCard({ pending, onAnswer }: { pending: PendingConfirm; onAnswer: (allowed: boolean) => void }) { return (

Hold on — this can't be undone

@@ -155,8 +178,8 @@ function ConfirmCard({ pending }: { pending: PendingConfirm }) { WebPilot wants to click {pending.action} on {hostOf(pending.url)}.

- - + +
); @@ -241,109 +264,100 @@ function Settings({ settings, onChange, onClose }: { settings: ExtensionSettings function App() { const [settings, setSettings] = useState(null); - const [view, setView] = useState<"chat" | "settings">("chat"); - const [turns, setTurns] = useState([]); - const [input, setInput] = useState(""); - const [running, setRunning] = useState(false); - const [pending, setPending] = useState(null); + const [view, setView] = useState<"chat" | "settings" | "history">("chat"); + const [history, setHistory] = useState(null); + const [tabId, setTabId] = useState(null); const [tabTitle, setTabTitle] = useState(""); - const controllerRef = useRef(null); + const [session, setSession] = useState(null); + const [input, setInput] = useState(""); + const portRef = useRef(null); const scrollRef = useRef(null); - const nextId = useRef(1); + const inputRef = useRef(null); useEffect(() => { loadSettings().then(setSettings); - const refresh = () => activeTab().then((tab) => setTabTitle(tab?.title || "")); - refresh(); - chrome.tabs.onActivated.addListener(refresh); - chrome.tabs.onUpdated.addListener(refresh); + panelTabId().then((id) => setTabId(id ?? null)); + }, []); + + // Connect to this tab's session in the background engine; reconnect if the + // service worker restarts. + useEffect(() => { + if (tabId === null) return; + let disposed = false; + const connect = () => { + if (disposed) return; + const port = chrome.runtime.connect({ name: `${PANEL_PORT_PREFIX}${tabId}` }); + port.onMessage.addListener((message: EngineMessage) => { + if (message.type === "state") setSession(message.state); + if (message.type === "history") setHistory(message.items); + }); + port.onDisconnect.addListener(() => { + portRef.current = null; + setTimeout(connect, 300); + }); + portRef.current = port; + }; + connect(); + const refreshTitle = () => chrome.tabs.get(tabId).then((tab) => setTabTitle(tab.title || "")).catch(() => {}); + refreshTitle(); + const onUpdated = (updatedId: number) => { + if (updatedId === tabId) refreshTitle(); + }; + chrome.tabs.onUpdated.addListener(onUpdated); return () => { - chrome.tabs.onActivated.removeListener(refresh); - chrome.tabs.onUpdated.removeListener(refresh); + disposed = true; + chrome.tabs.onUpdated.removeListener(onUpdated); + portRef.current?.disconnect(); }; - }, []); + }, [tabId]); + + const running = !!session?.running; + const turns: Turn[] = session?.turns || []; useEffect(() => { scrollRef.current?.scrollTo({ top: scrollRef.current.scrollHeight }); - }, [turns, pending]); + }, [session]); - const updateSettings = (next: ExtensionSettings) => { - setSettings(next); - saveSettings(next); - }; + // Ready to type as soon as the panel opens and after each answer. + useEffect(() => { + if (!running && view === "chat") inputRef.current?.focus(); + }, [running, view, settings]); - const updateTurn = (id: number, patch: (turn: Turn) => Turn) => { - setTurns((current) => current.map((turn) => (turn.id === id ? patch(turn) : turn))); - }; + const send = (message: PanelMessage) => portRef.current?.postMessage(message); - const run = useCallback(async (goal: string) => { + const run = (goal: string) => { if (!settings || !goal.trim() || running) return; if (!settings.apiKey) { setView("settings"); return; } - const id = nextId.current++; - const startedAt = Date.now(); - const history = turns.filter((turn) => turn.answer).map((turn) => ({ user: turn.user, assistant: turn.answer! })); - setTurns((current) => [...current, { id, user: goal.trim(), steps: [], startedAt }]); + send({ type: "run", goal: goal.trim() }); setInput(""); - setRunning(true); - const controller = new AbortController(); - controllerRef.current = controller; - const addStep = (step: Omit) => updateTurn(id, (turn) => ({ ...turn, steps: [...turn.steps, { ...step, atMs: Date.now() - startedAt }] })); - - let browser: TabBrowser | null = null; - try { - const tab = await activeTab(); - if (!tab?.id) throw new Error("No tab to work in. Open a tab and try again."); - // Attaches to the tab only when the task needs the page; Chrome pages - // (New Tab, chrome://) are first navigated to the task's website. - browser = new TabBrowser(tab.id); - const result = await runTask({ - goal: goal.trim(), - config: modelConfigFor(settings), - browser, - history, - signal: controller.signal, - confirm: (action, url) => new Promise((resolve) => { - setPending({ - action, - url, - resolve: (allowed) => { - setPending(null); - addStep({ lane: "note", label: allowed ? "allowed" : "cancelled", detail: action }); - resolve(allowed); - }, - }); - }), - onEvent: (event: TaskEvent) => { - if (event.type === "step") { - addStep({ lane: event.source, label: ACTION_LABELS[event.action] || event.action, detail: event.detail ? event.detail.slice(0, 120) : undefined }); - } else if (event.type === "handoff") { - addStep({ lane: "note", label: "handing to the model", detail: event.reason }); - } else if (event.type === "status") { - addStep({ lane: "note", label: event.message.toLowerCase() }); - } - }, - }); - updateTurn(id, (turn) => ({ ...turn, answer: result.answer, mode: result.mode, finishedAt: Date.now(), stats: { jevCalls: result.jevCalls, llmCalls: result.llmCalls } })); - } catch (error) { - const message = error instanceof TaskCancelledError ? "Stopped." : error instanceof Error ? error.message : String(error); - updateTurn(id, (turn) => ({ ...turn, error: message, finishedAt: Date.now() })); - } finally { - await browser?.detach(); - controllerRef.current = null; - setPending(null); - setRunning(false); - } - }, [settings, running, turns]); + }; - const stop = () => { - controllerRef.current?.abort(); - pending?.resolve(false); + const updateSettings = (next: ExtensionSettings) => { + setSettings(next); + saveSettings(next); }; if (!settings) return null; + if (view === "history") { + return ( +
+ { + send({ type: "open", conversationId }); + setView("chat"); + }} + onDelete={(conversationId) => send({ type: "delete", conversationId })} + onClearAll={() => send({ type: "clear-history" })} + onClose={() => setView("chat")} + /> +
+ ); + } if (view === "settings") { return (
@@ -358,8 +372,14 @@ function App() {
WebPilot
- {tabTitle || "No tab"} + {tabTitle || "This tab"}
+ {turns.length > 0 && !running && ( + + )} + {!running && ( + + )} @@ -367,7 +387,7 @@ function App() { {!turns.length && (

Where to?

-

Ask about this page or anything on the web. WebPilot works in your current tab and asks before anything irreversible.

+

Ask about this page or anything on the web. WebPilot works in this tab and asks before anything irreversible.

{EXAMPLES.map((example, index) => (