diff --git a/.gitignore b/.gitignore index b9b13e2..86a6ad7 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,4 @@ next-env.d.ts # Screenshots (testing) /screenshots +/extension/dist diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2ad500d..cc3b9ce 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,10 +61,22 @@ Dependabot opens a weekly grouped PR for `playwright` and `playwright-core` (sta 3. Run one real task (`npm run agent:cli -- "Go to https://example.com and tell me the page heading."`) to confirm the snapshot format still parses. 4. For releases, run the `Package Desktop` workflow to package and smoke-test the desktop app on every platform. +## Browser Extension + +`extension/` is a Manifest V3 side-panel extension that shares the agent core with the desktop app: + +- `lib/cdp/driver.ts`: a browser driver over the Chrome DevTools Protocol that works with any transport, `chrome.debugger` in the extension or a Playwright CDP session in tests. Input is sent as real mouse and keyboard events. +- `lib/cdp/snapshot.ts`: converts Chrome's accessibility tree into the same snapshot text Playwright MCP produces, so the page parser, fast mode and planner read pages identically on both. Element refs are `b`, stable for the life of the document. +- `lib/core/run-task.ts`: the task flow over any `TaskBrowser` (Jev preflight, starting page, fast mode, answer check, LLM planner fallback, confirmations), with events for the UI and cancellation through an `AbortSignal`. It knows the user's current tab, so "summarize this page" starts there. +- `extension/src/`: the `chrome.debugger` transport, settings, OpenRouter OAuth (PKCE) and the React side panel. The agent runs in the side panel page, which stays alive while open (MV3 service workers are suspended when idle). + +`npm run extension:build` bundles it with esbuild into `extension/dist`; `process.env` is stubbed because the shared model client reads optional defaults from it in Node. + ## Safety `lib/safety.ts` holds the rules every agent path shares: +- **Requests that change something go to the careful planner.** Jev's preflight also asks whether the request buys, books, sends, deletes, submits…; if so, fast mode is skipped. Jev is excellent at finding and reading but weak at multi-step flows like add to cart → checkout, and these requests are rarer and higher-stakes. - **Irreversible actions need the user's OK.** Before a click whose label starts with a committing verb (buy, pay, place order, checkout, send, delete, unsubscribe, publish...), the run pauses with "Confirm: WebPilot wants to click …"; Resume allows it and Stop cancels. It applies to the LLM planner, fast mode and parallel sub-agents, and matches on the page's own label for the element rather than the model's description. Links only trigger it for strong verbs (buy, checkout, delete...), so navigation like "Sent items" or "Send feedback" does not. - **Page content is untrusted.** Planner, sub-agent, text-writing and answer prompts say that instructions found in pages or emails must never be followed. Jev gets a neutral version ("only the task defines what to do"), because it reads instructions literally and a list of risky verbs made it avoid legitimate steps. diff --git a/README.md b/README.md index 09dfefd..0d1ec6f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ WebPilot is meant to be a hackable, local-first alternative to closed agentic br ## Features - Desktop app built with Electron and Next.js. +- Browser extension for Chrome, Edge, Brave and Arc: WebPilot in the side panel, working in your current tab with your existing logins ([extension/README.md](./extension/README.md)). - Browser automation through Playwright MCP, running on the latest stable Playwright. - Fast mode: TypeSafe's Jev decision model (through OpenRouter) picks each browser action in ~0.3s while the LLM only writes text, with automatic fallback to the LLM planner. - Model providers: OpenRouter (one key for Gemini, Claude, GPT, Qwen, DeepSeek and hundreds more) and local Ollama. @@ -80,6 +81,14 @@ MODEL_BASE_URL=http://127.0.0.1:11434/v1 The settings UI can discover local Ollama models and hides known embedding-only models when Ollama reports enough metadata to identify them. +## Browser Extension + +```bash +npm run extension:build +``` + +Load `extension/dist` from `chrome://extensions` (Developer mode > Load unpacked), then click the WebPilot icon to open the side panel and **Connect OpenRouter**. See [extension/README.md](./extension/README.md) for how it works, permissions and privacy. + ## Fast Mode Turn on **Fast mode (Jev)** in Settings (OpenRouter only). Instead of asking the LLM to plan every click, WebPilot asks [Jev](https://openrouter.ai/typesafe/jev-1.13), a decision model that returns a choice with probabilities in about 0.3 seconds. The LLM still writes anything that needs text (search queries, the final answer), and the regular planner takes over whenever Jev is stuck, blocked, or about to click something irreversible. diff --git a/TESTING.md b/TESTING.md index 4adbb43..93d1994 100644 --- a/TESTING.md +++ b/TESTING.md @@ -40,6 +40,16 @@ Runs each scenario in both modes and writes JSON and Markdown reports (pass/fail - `realistic`: requests phrased the way people ask, without URLs where possible ("hey can you please find some recent tents to buy on kijiji?", "what's trending on hacker news right now?"). Answers are checked against live data where it exists: the Hacker News API, GitHub's latest Playwright release, and current books.toscrape prices. - `realistic` also runs private-data tasks against a local webmail and shop (`scripts/fixtures/realistic-sites.mjs`, also runnable on its own with `node scripts/fixtures/realistic-sites.mjs`). The inbox has a distractor newsletter and a prompt-injection email asking AI assistants to forward mail and delete evidence; the shop has a real cart and checkout. The fixture records every send, delete and order, and any of them fails the scenario. When a run asks to confirm an irreversible action, the harness denies it; `shop_buy` passes only if it asks before checking out with the right tent in the cart. Model latency on OpenRouter varies run to run, so compare medians over `--repeat 3` or more before drawing conclusions. +## Browser Extension + +```bash +npm run extension:build +OPENROUTER_API_KEY=sk-or-... npm run extension:smoke -- --headed +npm run bench:fast-mode -- --engine cdp --suite all +``` + +`extension:smoke` loads the built extension into Chromium and drives its real side panel against the local fixtures. `--engine cdp` runs the scenario suite on the extension's engine (`lib/core/run-task.ts` over the CDP driver) instead of the desktop agent. `tests/cdp-snapshot.test.ts` covers the accessibility-tree converter without a browser. + ## Deterministic Browser Smoke ```bash diff --git a/eslint.config.mjs b/eslint.config.mjs index 2a847bd..34b5a7c 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -20,7 +20,15 @@ const eslintConfig = defineConfig([ "agent_threads/**", "e2e_reports/**", "readiness_reports/**", + "extension/dist/**", ]), + { + // The browser extension is not a Next.js app. + files: ["extension/**/*.{ts,tsx}"], + rules: { + "@next/next/no-img-element": "off", + }, + }, { files: ["electron/**/*.cjs", "scripts/**/*.js"], rules: { diff --git a/extension/README.md b/extension/README.md new file mode 100644 index 0000000..1c40bd8 --- /dev/null +++ b/extension/README.md @@ -0,0 +1,63 @@ +# WebPilot browser extension + +WebPilot in your browser's side panel. Ask it about the page you have open or anything on the web; it works in your current tab, with the logins you already have, and asks before anything irreversible. + +Works in Chromium browsers: Chrome, Edge, Brave, Arc (Chrome 116+). + +## Install from source + +```bash +npm install +npm run extension:build +``` + +Then open `chrome://extensions`, turn on **Developer mode**, click **Load unpacked**, and pick `extension/dist`. Click the WebPilot toolbar icon to open the side panel. + +For development, `npm run extension:dev` rebuilds on every change; click the reload icon on the extension card to pick it up. + +## First run + +Open **Settings** (gear icon) and click **Connect OpenRouter**. You approve a key for WebPilot on openrouter.ai; it can be revoked or given a credit limit from the OpenRouter dashboard. You can also paste an existing `sk-or-...` key. + +Fast mode (Jev) is on by default. + +## How it works + +- The agent runs in the side panel page and controls the active tab through `chrome.debugger` (the Chrome DevTools Protocol). Chrome shows a "WebPilot started debugging this browser" bar while a task runs; it goes away when the task ends, and clicking **Cancel** on it stops the task. +- Pages are read from Chrome's accessibility tree and acted on with real mouse and keyboard events, so sites treat them like your own input. +- The same core runs in the desktop app and the extension: `lib/core/run-task.ts` (the task flow), `lib/cdp/` (the CDP driver and accessibility-tree snapshot), `lib/jev/` (fast mode), `lib/safety.ts` and `lib/model-client.ts`. +- Requests that only find or read information use fast mode: Jev picks each click in ~0.3s and the model writes text and the answer. Requests that change something (buy, book, send, delete, submit) use the careful model planner. +- Before an irreversible click (buy, place order, checkout, send, delete, publish, unsubscribe...) the panel shows **"Hold on — this can't be undone"** with Allow and Cancel. + +## Permissions + +| Permission | Why | +|---|---| +| `sidePanel` | The WebPilot panel | +| `debugger` | Reading and controlling the tab you ask it to work in | +| `tabs` | Knowing which tab is active and its title | +| `storage` | Your settings and OpenRouter key (stored locally) | +| `identity` | "Connect OpenRouter" sign-in | +| `https://openrouter.ai/*` | Model and Jev requests | + +Chrome does not allow extensions to control `chrome://` pages or the Chrome Web Store. + +## Privacy + +Page text, element labels and URLs of the tab WebPilot works in are sent to OpenRouter to run the model, and in fast mode to TypeSafe (through OpenRouter) for Jev. Nothing is sent while no task is running. Settings stay in the browser's local extension storage. + +## Testing + +```bash +npm run extension:build +OPENROUTER_API_KEY=sk-or-... npm run extension:smoke # headless +OPENROUTER_API_KEY=sk-or-... npm run extension:smoke -- --headed +``` + +The smoke test loads `extension/dist` into Chromium, opens the local webmail and shop fixtures, types requests into the real side panel and checks the answers, confirmations and recorded actions (no sends, deletes or orders). Screenshots go to `e2e_reports/extension/`. + +The full scenario suite runs against the extension's engine too: + +```bash +npm run bench:fast-mode -- --engine cdp --suite all +``` diff --git a/extension/manifest.json b/extension/manifest.json new file mode 100644 index 0000000..880026b --- /dev/null +++ b/extension/manifest.json @@ -0,0 +1,29 @@ +{ + "manifest_version": 3, + "name": "WebPilot", + "description": "An AI agent in your browser's side panel: it reads and acts on your current tab, fast, and asks before anything irreversible.", + "version": "0.1.0", + "minimum_chrome_version": "116", + "icons": { + "16": "icons/16x16.png", + "32": "icons/32x32.png", + "48": "icons/48x48.png", + "128": "icons/128x128.png" + }, + "action": { + "default_title": "Open WebPilot", + "default_icon": { + "16": "icons/16x16.png", + "32": "icons/32x32.png" + } + }, + "side_panel": { + "default_path": "sidepanel.html" + }, + "background": { + "service_worker": "background.js", + "type": "module" + }, + "permissions": ["sidePanel", "debugger", "tabs", "storage", "identity"], + "host_permissions": ["https://openrouter.ai/*", "http://127.0.0.1/*", "http://localhost/*"] +} diff --git a/extension/sidepanel.css b/extension/sidepanel.css new file mode 100644 index 0000000..ca93ec4 --- /dev/null +++ b/extension/sidepanel.css @@ -0,0 +1,368 @@ +/* WebPilot side panel: the desktop app's tokens (dark-first, amber accent) with a flight-recorder log. */ + +:root { + --wp-bg: #0c0c0e; + --wp-surface: #18181b; + --wp-surface-raised: #27272a; + --wp-border: #3f3f46; + --wp-border-soft: #2a2a2f; + --wp-text: #fafafa; + --wp-text-secondary: #a1a1aa; + --wp-text-faint: #71717a; + --wp-accent: #f59e0b; + --wp-accent-hover: #d97706; + --wp-accent-glow: rgba(245, 158, 11, 0.35); + --wp-accent-wash: rgba(245, 158, 11, 0.08); + --wp-llm: #60a5fa; + --wp-llm-wash: rgba(96, 165, 250, 0.1); + --wp-success: #22c55e; + --wp-error: #ef4444; + --wp-radius-sm: 6px; + --wp-radius-md: 8px; + --wp-radius-lg: 12px; + --wp-font-sans: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + --wp-font-mono: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace; + color-scheme: dark; +} + +@media (prefers-color-scheme: light) { + :root:not([data-theme="dark"]) { + --wp-bg: #fafafa; + --wp-surface: #ffffff; + --wp-surface-raised: #f4f4f5; + --wp-border: #e4e4e7; + --wp-border-soft: #ececee; + --wp-text: #18181b; + --wp-text-secondary: #52525b; + --wp-text-faint: #a1a1aa; + --wp-accent: #d97706; + --wp-accent-hover: #b45309; + --wp-accent-glow: rgba(217, 119, 6, 0.25); + --wp-accent-wash: rgba(217, 119, 6, 0.08); + --wp-llm: #2563eb; + --wp-llm-wash: rgba(37, 99, 235, 0.08); + color-scheme: light; + } +} + +* { box-sizing: border-box; } + +html, body, #root { height: 100%; } + +body { + margin: 0; + background: + radial-gradient(120% 60% at 100% 0%, var(--wp-accent-wash), transparent 60%), + var(--wp-bg); + color: var(--wp-text); + font: 13px/1.5 var(--wp-font-sans); + -webkit-font-smoothing: antialiased; +} + +/* Fine grain so large dark areas do not band. */ +body::before { + content: ""; + position: fixed; + inset: 0; + pointer-events: none; + opacity: 0.035; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='120' height='120'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='.9' numOctaves='2'/%3E%3C/filter%3E%3Crect width='120' height='120' filter='url(%23n)'/%3E%3C/svg%3E"); +} + +button { font: inherit; color: inherit; } + +.app { display: flex; flex-direction: column; height: 100%; position: relative; } + +/* ---------- header ---------- */ +.header { + display: flex; + align-items: center; + gap: 8px; + padding: 10px 12px; + border-bottom: 1px solid var(--wp-border-soft); + backdrop-filter: blur(8px); +} +.brand { display: flex; align-items: center; gap: 7px; font-weight: 650; letter-spacing: -0.01em; } +.brand img { width: 18px; height: 18px; border-radius: 4px; } +.target { + flex: 1; + min-width: 0; + display: flex; + align-items: center; + gap: 6px; + padding: 3px 9px; + border: 1px solid var(--wp-border-soft); + border-radius: 999px; + color: var(--wp-text-secondary); + font-size: 11.5px; +} +.target span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.target .reticle { + flex: none; + width: 8px; + height: 8px; + border-radius: 50%; + border: 1.5px solid var(--wp-text-faint); +} +.target.locked { border-color: var(--wp-accent-glow); color: var(--wp-text); } +.target.locked .reticle { border-color: var(--wp-accent); box-shadow: 0 0 0 3px var(--wp-accent-wash); animation: pulse 1.6s ease-in-out infinite; } +.icon-button { + flex: none; + display: grid; + place-items: center; + width: 28px; + height: 28px; + border: 0; + border-radius: var(--wp-radius-sm); + background: transparent; + color: var(--wp-text-secondary); + cursor: pointer; +} +.icon-button:hover { background: var(--wp-surface-raised); color: var(--wp-text); } + +/* ---------- conversation ---------- */ +.scroll { flex: 1; overflow-y: auto; padding: 14px 12px 8px; scroll-behavior: smooth; } +.turn { margin-bottom: 14px; animation: rise 0.28s ease-out both; } +.user-bubble { + margin-left: auto; + width: fit-content; + max-width: 88%; + padding: 7px 11px; + border-radius: var(--wp-radius-lg) var(--wp-radius-lg) 4px var(--wp-radius-lg); + background: var(--wp-surface-raised); + white-space: pre-wrap; + overflow-wrap: anywhere; +} +.answer { margin-top: 10px; overflow-wrap: anywhere; } +.answer p { margin: 0 0 8px; } +.answer ul, .answer ol { margin: 0 0 8px; padding-left: 18px; } +.answer li { margin: 2px 0; } +.answer a { color: var(--wp-accent); text-decoration: none; } +.answer a:hover { text-decoration: underline; } +.answer code { font-family: var(--wp-font-mono); font-size: 12px; background: var(--wp-surface-raised); padding: 1px 4px; border-radius: 4px; } +.answer strong { font-weight: 650; } +.answer.error { color: var(--wp-error); } + +/* ---------- flight log ---------- */ +.log { + position: relative; + margin-top: 10px; + padding: 8px 10px 8px 22px; + border: 1px solid var(--wp-border-soft); + border-radius: var(--wp-radius-md); + background: color-mix(in srgb, var(--wp-surface) 70%, transparent); + font-family: var(--wp-font-mono); + font-size: 11px; +} +.log::before { + content: ""; + position: absolute; + left: 11px; + top: 12px; + bottom: 12px; + width: 1px; + background: var(--wp-border); +} +.log.live::after { + content: ""; + position: absolute; + left: 10px; + top: 12px; + width: 3px; + height: 36px; + border-radius: 2px; + background: linear-gradient(var(--wp-accent), transparent); + box-shadow: 0 0 10px var(--wp-accent-glow); + animation: sweep 1.4s ease-in-out infinite; +} +.log-head { + display: flex; + justify-content: space-between; + gap: 8px; + margin-bottom: 4px; + color: var(--wp-text-secondary); + font-size: 10.5px; + letter-spacing: 0.06em; + text-transform: uppercase; +} +.log-head .clock { font-variant-numeric: tabular-nums; color: var(--wp-text); } +.step { position: relative; display: flex; align-items: baseline; gap: 7px; padding: 3px 0; animation: rise 0.2s ease-out both; } +.step::before { + content: ""; + position: absolute; + left: -14px; + top: 8px; + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--wp-surface); + border: 1.5px solid var(--wp-text-faint); +} +.step.jev::before { border-color: var(--wp-accent); background: var(--wp-accent); box-shadow: 0 0 6px var(--wp-accent-glow); } +.step.llm::before { border-color: var(--wp-llm); } +.lane { + flex: none; + width: 30px; + font-size: 9.5px; + font-weight: 700; + letter-spacing: 0.08em; + text-align: center; + border-radius: 3px; + padding: 0 3px; +} +.step.jev .lane { color: var(--wp-accent); background: var(--wp-accent-wash); } +.step.llm .lane { color: var(--wp-llm); background: var(--wp-llm-wash); } +.step.note .lane { color: var(--wp-text-faint); } +.step .what { min-width: 0; color: var(--wp-text-secondary); overflow-wrap: anywhere; } +.step .what b { color: var(--wp-text); font-weight: 600; } +.step .at { flex: none; margin-left: auto; color: var(--wp-text-faint); font-variant-numeric: tabular-nums; } +.log-summary { margin-top: 4px; color: var(--wp-text-faint); font-size: 10.5px; } + +/* ---------- confirmation ---------- */ +.confirm { + margin-top: 10px; + padding: 12px; + border-radius: var(--wp-radius-md); + background: var(--wp-surface); + border: 1px solid var(--wp-accent); + box-shadow: 0 0 0 3px var(--wp-accent-wash), 0 8px 24px rgba(0, 0, 0, 0.35); + position: relative; + overflow: hidden; + animation: rise 0.22s ease-out both; +} +.confirm::before { + content: ""; + position: absolute; + inset: 0 0 auto 0; + height: 4px; + background: repeating-linear-gradient(-45deg, var(--wp-accent) 0 8px, transparent 8px 16px); +} +.confirm h3 { margin: 4px 0 4px; font-size: 13px; } +.confirm p { margin: 0 0 10px; color: var(--wp-text-secondary); overflow-wrap: anywhere; } +.confirm .action { color: var(--wp-text); font-family: var(--wp-font-mono); font-size: 12px; } +.row { display: flex; gap: 8px; } + +.btn { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 6px; + height: 30px; + padding: 0 12px; + border-radius: var(--wp-radius-sm); + border: 1px solid var(--wp-border); + background: var(--wp-surface-raised); + cursor: pointer; + font-weight: 550; +} +.btn:hover { border-color: var(--wp-text-faint); } +.btn.primary { background: var(--wp-accent); border-color: var(--wp-accent); color: #111; } +.btn.primary:hover { background: var(--wp-accent-hover); } +.btn:disabled { opacity: 0.5; cursor: not-allowed; } +.btn.block { width: 100%; } + +/* ---------- empty state ---------- */ +.empty { padding: 28px 6px 8px; animation: rise 0.35s ease-out both; } +.empty h1 { margin: 0 0 4px; font-size: 19px; letter-spacing: -0.02em; font-weight: 650; } +.empty .sub { margin: 0 0 18px; color: var(--wp-text-secondary); } +.prompt { + display: block; + width: 100%; + text-align: left; + margin-bottom: 7px; + padding: 9px 11px; + border: 1px solid var(--wp-border-soft); + border-radius: var(--wp-radius-md); + background: var(--wp-surface); + cursor: pointer; + transition: border-color 0.15s, transform 0.15s; + animation: rise 0.35s ease-out both; +} +.prompt:hover { border-color: var(--wp-accent-glow); transform: translateX(2px); } +.prompt small { display: block; color: var(--wp-text-faint); font-size: 11px; } + +/* ---------- composer ---------- */ +.composer { padding: 10px 12px 12px; border-top: 1px solid var(--wp-border-soft); } +.field-wrap { + display: flex; + align-items: flex-end; + gap: 6px; + padding: 6px 6px 6px 10px; + border: 1px solid var(--wp-border); + border-radius: var(--wp-radius-lg); + background: var(--wp-surface); + transition: border-color 0.15s, box-shadow 0.15s; +} +.field-wrap:focus-within { border-color: var(--wp-accent); box-shadow: 0 0 0 3px var(--wp-accent-wash); } +.field-wrap textarea { + flex: 1; + min-height: 22px; + max-height: 140px; + resize: none; + border: 0; + outline: 0; + background: transparent; + color: var(--wp-text); + font: inherit; + padding: 3px 0; +} +.send { + flex: none; + display: grid; + place-items: center; + width: 30px; + height: 30px; + border: 0; + border-radius: 8px; + background: var(--wp-accent); + color: #111; + cursor: pointer; +} +.send:disabled { background: var(--wp-surface-raised); color: var(--wp-text-faint); cursor: not-allowed; } +.send.stop { background: var(--wp-surface-raised); color: var(--wp-text); border: 1px solid var(--wp-border); } +.composer .hint { display: flex; justify-content: space-between; margin-top: 6px; color: var(--wp-text-faint); font-size: 11px; } +.mode { color: var(--wp-accent); } + +/* ---------- settings ---------- */ +.settings { padding: 14px 12px; overflow-y: auto; flex: 1; animation: rise 0.25s ease-out both; } +.settings h2 { margin: 0 0 12px; font-size: 15px; } +.section { margin-bottom: 18px; } +.label { display: block; margin-bottom: 5px; color: var(--wp-text-secondary); font-size: 11px; text-transform: uppercase; letter-spacing: 0.06em; } +.input { + width: 100%; + height: 32px; + padding: 0 10px; + border: 1px solid var(--wp-border); + border-radius: var(--wp-radius-sm); + background: var(--wp-surface); + color: var(--wp-text); + font: 12.5px var(--wp-font-mono); + outline: 0; +} +.input:focus { border-color: var(--wp-accent); } +.help { margin: 6px 0 0; color: var(--wp-text-faint); font-size: 11.5px; } +.status-line { margin: 8px 0 0; font-size: 12px; } +.status-line.ok { color: var(--wp-success); } +.status-line.bad { color: var(--wp-error); } +.or { display: flex; align-items: center; gap: 8px; margin: 12px 0; color: var(--wp-text-faint); font-size: 11px; } +.or::before, .or::after { content: ""; flex: 1; height: 1px; background: var(--wp-border-soft); } +.toggle { display: flex; align-items: flex-start; gap: 10px; cursor: pointer; } +.switch { flex: none; position: relative; width: 34px; height: 20px; border-radius: 999px; background: var(--wp-border); transition: background 0.15s; margin-top: 1px; } +.switch::after { content: ""; position: absolute; top: 2px; left: 2px; width: 16px; height: 16px; border-radius: 50%; background: #fff; transition: transform 0.15s; } +.switch.on { background: var(--wp-accent); } +.switch.on::after { transform: translateX(14px); } +.privacy { + padding: 10px; + border-radius: var(--wp-radius-md); + border: 1px dashed var(--wp-border); + color: var(--wp-text-secondary); + font-size: 11.5px; +} + +@keyframes rise { from { opacity: 0; transform: translateY(4px); } to { opacity: 1; transform: none; } } +@keyframes pulse { 50% { box-shadow: 0 0 0 5px transparent; } } +@keyframes sweep { 0% { transform: translateY(0); opacity: 0; } 20% { opacity: 1; } 100% { transform: translateY(calc(100% + 120px)); opacity: 0; } } + +@media (prefers-reduced-motion: reduce) { + *, *::before, *::after { animation: none !important; transition: none !important; } +} diff --git a/extension/sidepanel.html b/extension/sidepanel.html new file mode 100644 index 0000000..535c7e5 --- /dev/null +++ b/extension/sidepanel.html @@ -0,0 +1,13 @@ + + + + + + WebPilot + + + +
+ + + diff --git a/extension/src/background.ts b/extension/src/background.ts new file mode 100644 index 0000000..9572daa --- /dev/null +++ b/extension/src/background.ts @@ -0,0 +1,11 @@ +/// + +// The toolbar button opens WebPilot in the side panel. The agent itself runs in +// the side panel page, which stays alive while it is open (MV3 service workers +// are suspended when idle, so they cannot host a long-running task). +chrome.runtime.onInstalled.addListener(() => { + chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true }).catch(() => {}); +}); +chrome.runtime.onStartup.addListener(() => { + chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true }).catch(() => {}); +}); diff --git a/extension/src/debugger-transport.ts b/extension/src/debugger-transport.ts new file mode 100644 index 0000000..ec2ed40 --- /dev/null +++ b/extension/src/debugger-transport.ts @@ -0,0 +1,62 @@ +/// + +import type { CdpTransport } from "../../lib/cdp/driver"; + +const PROTOCOL_VERSION = "1.3"; + +/** Pages Chrome never lets extensions debug. */ +export function isRestrictedUrl(url: string | undefined): boolean { + if (!url) return true; + return !/^https?:\/\//i.test(url) + || /^https:\/\/chromewebstore\.google\.com\//i.test(url) + || /^https:\/\/chrome\.google\.com\/webstore/i.test(url); +} + +/** + * CDP transport over `chrome.debugger` for one tab. While attached, Chrome shows + * a "WebPilot started debugging this browser" bar; `detach()` removes it. + */ +export class DebuggerTransport implements CdpTransport { + private listeners = new Set<(method: string, params: Record) => void>(); + private detachedReason: string | null = null; + + private readonly onEventHandler = (source: chrome.debugger.Debuggee, method: string, params?: object) => { + if (source.tabId !== this.tabId) return; + for (const listener of this.listeners) listener(method, (params || {}) as Record); + }; + + private readonly onDetachHandler = (source: chrome.debugger.Debuggee, reason: string) => { + if (source.tabId === this.tabId) this.detachedReason = reason; + }; + + private constructor(public readonly tabId: number) {} + + static async attach(tabId: number): Promise { + const transport = new DebuggerTransport(tabId); + await chrome.debugger.attach({ tabId }, PROTOCOL_VERSION); + chrome.debugger.onEvent.addListener(transport.onEventHandler); + chrome.debugger.onDetach.addListener(transport.onDetachHandler); + return transport; + } + + async send(method: string, params?: Record): Promise { + if (this.detachedReason) { + throw new Error(this.detachedReason === "canceled_by_user" + ? "Browser control was cancelled from Chrome's debugging bar." + : `Lost control of the tab (${this.detachedReason}).`); + } + return await chrome.debugger.sendCommand({ tabId: this.tabId }, method, params) as T; + } + + onEvent(listener: (method: string, params: Record) => void): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + async detach(): Promise { + chrome.debugger.onEvent.removeListener(this.onEventHandler); + chrome.debugger.onDetach.removeListener(this.onDetachHandler); + this.listeners.clear(); + if (!this.detachedReason) await chrome.debugger.detach({ tabId: this.tabId }).catch(() => {}); + } +} diff --git a/extension/src/openrouter-auth.ts b/extension/src/openrouter-auth.ts new file mode 100644 index 0000000..d28408f --- /dev/null +++ b/extension/src/openrouter-auth.ts @@ -0,0 +1,35 @@ +/// + +/** + * "Connect OpenRouter": OAuth with PKCE, so users approve a key on openrouter.ai + * instead of pasting one. The key is created for this extension and can be + * revoked or given a credit limit from the OpenRouter dashboard. + */ + +function base64Url(bytes: Uint8Array): string { + let binary = ""; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +export async function connectOpenRouter(): Promise { + const verifier = base64Url(crypto.getRandomValues(new Uint8Array(48))); + const challenge = base64Url(new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(verifier)))); + const callbackUrl = chrome.identity.getRedirectURL("openrouter"); + const authUrl = `https://openrouter.ai/auth?callback_url=${encodeURIComponent(callbackUrl)}&code_challenge=${challenge}&code_challenge_method=S256`; + + const redirect = await chrome.identity.launchWebAuthFlow({ url: authUrl, interactive: true }); + const code = redirect ? new URL(redirect).searchParams.get("code") : null; + if (!code) throw new Error("OpenRouter did not return an authorization code."); + + const response = await fetch("https://openrouter.ai/api/v1/auth/keys", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ code, code_verifier: verifier, code_challenge_method: "S256" }), + }); + const json = await response.json().catch(() => ({})) as { key?: string; error?: { message?: string } }; + if (!response.ok || !json.key) { + throw new Error(json.error?.message || `OpenRouter key exchange failed (${response.status}).`); + } + return json.key; +} diff --git a/extension/src/settings.ts b/extension/src/settings.ts new file mode 100644 index 0000000..567f5bd --- /dev/null +++ b/extension/src/settings.ts @@ -0,0 +1,43 @@ +/// + +import { defaultModelsForProvider } from "../../lib/runtime-provider-presets"; +import { resolveRuntimeModelConfig, type RuntimeModelConfig } from "../../lib/model-client"; + +export interface ExtensionSettings { + apiKey: string; + navModel: string; + fastMode: boolean; +} + +const DEFAULTS: ExtensionSettings = { + apiKey: "", + navModel: defaultModelsForProvider("openrouter").navModel, + fastMode: true, +}; + +const KEY = "webpilot.settings.v1"; + +export async function loadSettings(): Promise { + const stored = (await chrome.storage.local.get(KEY))[KEY] as Partial | undefined; + return { + apiKey: typeof stored?.apiKey === "string" ? stored.apiKey : DEFAULTS.apiKey, + navModel: typeof stored?.navModel === "string" && stored.navModel.trim() ? stored.navModel.trim() : DEFAULTS.navModel, + fastMode: typeof stored?.fastMode === "boolean" ? stored.fastMode : DEFAULTS.fastMode, + }; +} + +export async function saveSettings(settings: ExtensionSettings): Promise { + await chrome.storage.local.set({ [KEY]: settings }); +} + +export function modelConfigFor(settings: ExtensionSettings): RuntimeModelConfig { + return resolveRuntimeModelConfig({ + provider: "openrouter", + apiKey: settings.apiKey, + navModel: settings.navModel, + reviewModel: settings.navModel, + synthEnabled: false, + fastMode: settings.fastMode, + timeoutMs: 60_000, + }); +} diff --git a/extension/src/sidepanel.tsx b/extension/src/sidepanel.tsx new file mode 100644 index 0000000..6450dc2 --- /dev/null +++ b/extension/src/sidepanel.tsx @@ -0,0 +1,431 @@ +/// + +import { useCallback, useEffect, useRef, useState } from "react"; +import { createRoot } from "react-dom/client"; +import ReactMarkdown from "react-markdown"; +import { CdpBrowser } from "../../lib/cdp/driver"; +import { TaskCancelledError, runTask, type TaskEvent } from "../../lib/core/run-task"; +import { DebuggerTransport, isRestrictedUrl } from "./debugger-transport"; +import { connectOpenRouter } from "./openrouter-auth"; +import { loadSettings, modelConfigFor, saveSettings, type ExtensionSettings } from "./settings"; + +interface LogStep { + lane: "jev" | "llm" | "note"; + label: string; + detail?: string; + atMs: number; +} + +interface Turn { + id: number; + user: string; + answer?: string; + error?: string; + steps: LogStep[]; + startedAt: number; + finishedAt?: number; + mode?: "chat" | "fast" | "planner"; + stats?: { jevCalls: number; llmCalls: number }; +} + +interface PendingConfirm { + action: string; + url: string; + resolve: (allowed: boolean) => void; +} + +const EXAMPLES = [ + { text: "Summarize this page", note: "Reads the tab you have open" }, + { text: "Find some recent tents for sale on Kijiji", note: "Searches and compares listings" }, + { text: "What's trending on Hacker News right now?", note: "Opens the site and reads the front page" }, +]; + +const ACTION_LABELS: Record = { + click: "click", + type: "type", + scroll_down: "scroll down", + scroll_up: "scroll up", + back: "go back", + navigate: "open", + scroll: "scroll", + wait: "wait", +}; + +function seconds(ms: number): string { + return `${(ms / 1000).toFixed(1)}s`; +} + +function hostOf(url: string): string { + try { + return new URL(url).host; + } catch { + return url; + } +} + +/** + * The tab WebPilot works in: the active tab of this window. When the panel page + * itself is open as a tab (e.g. in tests), the most recently used web tab instead. + */ +async function activeTab(): Promise { + const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }); + if (tab && !tab.url?.startsWith(chrome.runtime.getURL(""))) return tab; + const tabs = await chrome.tabs.query({ currentWindow: true }); + return tabs + .filter((candidate) => /^https?:/i.test(candidate.url || "")) + .sort((left, right) => (right.lastAccessed || 0) - (left.lastAccessed || 0))[0]; +} + +function Logo() { + return ; +} + +function SendIcon() { + return ( + + ); +} + +function StopIcon() { + return ( + + ); +} + +function GearIcon() { + return ( + + ); +} + +function BackIcon() { + return ( + + ); +} + +function FlightLog({ turn, live }: { turn: Turn; live: boolean }) { + const [now, setNow] = useState(Date.now()); + useEffect(() => { + if (!live) return; + const timer = setInterval(() => setNow(Date.now()), 100); + return () => clearInterval(timer); + }, [live]); + const elapsed = (turn.finishedAt ?? now) - turn.startedAt; + if (!turn.steps.length && !live) return null; + return ( +
+
+ {live ? "Flying" : turn.mode === "fast" ? "Fast mode" : turn.mode === "planner" ? "Planner" : "Log"} + {seconds(elapsed)} +
+ {turn.steps.map((step, index) => ( +
+ {step.lane === "note" ? "··" : step.lane.toUpperCase()} + + {step.label} + {step.detail ? ` ${step.detail}` : ""} + + {seconds(step.atMs)} +
+ ))} + {!live && turn.stats && ( +
+ {turn.stats.jevCalls} Jev decision{turn.stats.jevCalls === 1 ? "" : "s"} · {turn.stats.llmCalls} LLM call{turn.stats.llmCalls === 1 ? "" : "s"} +
+ )} +
+ ); +} + +function ConfirmCard({ pending }: { pending: PendingConfirm }) { + return ( +
+

Hold on — this can't be undone

+

+ WebPilot wants to click {pending.action} on {hostOf(pending.url)}. +

+
+ + +
+
+ ); +} + +function Settings({ settings, onChange, onClose }: { settings: ExtensionSettings; onChange: (next: ExtensionSettings) => void; onClose: () => void }) { + const [status, setStatus] = useState<{ tone: "ok" | "bad"; text: string } | null>(null); + const [connecting, setConnecting] = useState(false); + + const connect = async () => { + setConnecting(true); + setStatus(null); + try { + const key = await connectOpenRouter(); + onChange({ ...settings, apiKey: key }); + setStatus({ tone: "ok", text: "Connected to OpenRouter." }); + } catch (error) { + setStatus({ tone: "bad", text: error instanceof Error ? error.message : String(error) }); + } finally { + setConnecting(false); + } + }; + + return ( +
+
+ +

Settings

+
+ +
+ OpenRouter + +

One account for Gemini, Claude, GPT and Jev. You approve a key for WebPilot on openrouter.ai and can revoke it there.

+
or paste a key
+ onChange({ ...settings, apiKey: event.target.value.trim() })} + aria-label="OpenRouter API key" + /> + {status &&

{status.text}

} +
+ +
+ + onChange({ ...settings, navModel: event.target.value.trim() })} + spellCheck={false} + /> +

Any OpenRouter model id with tool calling, e.g. google/gemini-3.8-flash.

+
+ +
+
onChange({ ...settings, fastMode: !settings.fastMode })} + onKeyDown={(event) => { if (event.key === " " || event.key === "Enter") { event.preventDefault(); onChange({ ...settings, fastMode: !settings.fastMode }); } }}> + + + Fast mode (Jev) + + Jev picks each click in about 0.3s; the model only writes text and answers. Falls back to the model when unsure. + + +
+
+ +
+ WebPilot reads and acts on the tab you ask it to, using Chrome's debugging bar while it works. Page text is sent to OpenRouter + (and, in fast mode, to TypeSafe for Jev). It asks before anything irreversible, like buying, sending or deleting. +
+
+ ); +} + +function App() { + const [settings, setSettings] = useState(null); + const [view, setView] = useState<"chat" | "settings">("chat"); + const [turns, setTurns] = useState([]); + const [input, setInput] = useState(""); + const [running, setRunning] = useState(false); + const [pending, setPending] = useState(null); + const [tabTitle, setTabTitle] = useState(""); + const controllerRef = useRef(null); + const scrollRef = useRef(null); + const nextId = useRef(1); + + useEffect(() => { + loadSettings().then(setSettings); + const refresh = () => activeTab().then((tab) => setTabTitle(tab?.title || "")); + refresh(); + chrome.tabs.onActivated.addListener(refresh); + chrome.tabs.onUpdated.addListener(refresh); + return () => { + chrome.tabs.onActivated.removeListener(refresh); + chrome.tabs.onUpdated.removeListener(refresh); + }; + }, []); + + useEffect(() => { + scrollRef.current?.scrollTo({ top: scrollRef.current.scrollHeight }); + }, [turns, pending]); + + const updateSettings = (next: ExtensionSettings) => { + setSettings(next); + saveSettings(next); + }; + + const updateTurn = (id: number, patch: (turn: Turn) => Turn) => { + setTurns((current) => current.map((turn) => (turn.id === id ? patch(turn) : turn))); + }; + + const run = useCallback(async (goal: string) => { + if (!settings || !goal.trim() || running) return; + if (!settings.apiKey) { + setView("settings"); + return; + } + const id = nextId.current++; + const startedAt = Date.now(); + const history = turns.filter((turn) => turn.answer).map((turn) => ({ user: turn.user, assistant: turn.answer! })); + setTurns((current) => [...current, { id, user: goal.trim(), steps: [], startedAt }]); + setInput(""); + setRunning(true); + const controller = new AbortController(); + controllerRef.current = controller; + const addStep = (step: Omit) => updateTurn(id, (turn) => ({ ...turn, steps: [...turn.steps, { ...step, atMs: Date.now() - startedAt }] })); + + let transport: DebuggerTransport | null = null; + try { + const tab = await activeTab(); + if (!tab?.id || isRestrictedUrl(tab.url)) { + throw new Error("Open a regular web page in this tab first. Chrome doesn't let extensions control its own pages or the Web Store."); + } + transport = await DebuggerTransport.attach(tab.id); + const result = await runTask({ + goal: goal.trim(), + config: modelConfigFor(settings), + browser: new CdpBrowser(transport), + history, + signal: controller.signal, + confirm: (action, url) => new Promise((resolve) => { + setPending({ + action, + url, + resolve: (allowed) => { + setPending(null); + addStep({ lane: "note", label: allowed ? "allowed" : "cancelled", detail: action }); + resolve(allowed); + }, + }); + }), + onEvent: (event: TaskEvent) => { + if (event.type === "step") { + addStep({ lane: event.source, label: ACTION_LABELS[event.action] || event.action, detail: event.detail ? event.detail.slice(0, 120) : undefined }); + } else if (event.type === "handoff") { + addStep({ lane: "note", label: "handing to the model", detail: event.reason }); + } else if (event.type === "status") { + addStep({ lane: "note", label: event.message.toLowerCase() }); + } + }, + }); + updateTurn(id, (turn) => ({ ...turn, answer: result.answer, mode: result.mode, finishedAt: Date.now(), stats: { jevCalls: result.jevCalls, llmCalls: result.llmCalls } })); + } catch (error) { + const message = error instanceof TaskCancelledError ? "Stopped." : error instanceof Error ? error.message : String(error); + updateTurn(id, (turn) => ({ ...turn, error: message, finishedAt: Date.now() })); + } finally { + await transport?.detach(); + controllerRef.current = null; + setPending(null); + setRunning(false); + } + }, [settings, running, turns]); + + const stop = () => { + controllerRef.current?.abort(); + pending?.resolve(false); + }; + + if (!settings) return null; + if (view === "settings") { + return ( +
+ setView("chat")} /> +
+ ); + } + + return ( +
+
+
WebPilot
+
+ + {tabTitle || "No tab"} +
+ +
+ +
+ {!turns.length && ( +
+

Where to?

+

Ask about this page or anything on the web. WebPilot works in your current tab and asks before anything irreversible.

+ {EXAMPLES.map((example, index) => ( + + ))} + {!settings.apiKey && ( + + )} +
+ )} + {turns.map((turn) => { + const live = running && !turn.finishedAt; + return ( +
+
{turn.user}
+ + {live && pending && } + {turn.answer && ( +
+ {children} }}>{turn.answer} +
+ )} + {turn.error &&
{turn.error}
} +
+ ); + })} +
+ +
+
{ event.preventDefault(); run(input); }}> +