From 58c2164f2587e8b2abfdbbd8e2db93a58ddf2f13 Mon Sep 17 00:00:00 2001 From: ahamSel <77988808+ahamSel@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:34:38 -0230 Subject: [PATCH] fix(extension): start tasks from the New Tab page and other Chrome pages Chrome forbids debugging its own pages (New Tab, chrome://, the Web Store), and the side panel refused to run any task from them. Now: - The tab is attached to chrome.debugger only when a task first needs the page (TabBrowser), so chat-only replies never touch the tab or show the debugging bar. - A task starting on a Chrome page navigates the tab to its website with the tabs API first, then takes control. - If no site is obvious from a Chrome page, the task starts from a Google search instead of failing. Extension smoke adds two cases starting from a blank tab; 6/6 pass. Co-Authored-By: Claude Opus 5.5 --- extension/src/sidepanel.tsx | 24 ++++----- extension/src/tab-browser.ts | 100 +++++++++++++++++++++++++++++++++++ lib/core/run-task.ts | 20 ++++--- scripts/extension-smoke.ts | 17 +++++- 4 files changed, 141 insertions(+), 20 deletions(-) create mode 100644 extension/src/tab-browser.ts diff --git a/extension/src/sidepanel.tsx b/extension/src/sidepanel.tsx index 6450dc2..7a8b408 100644 --- a/extension/src/sidepanel.tsx +++ b/extension/src/sidepanel.tsx @@ -3,9 +3,8 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { createRoot } from "react-dom/client"; import ReactMarkdown from "react-markdown"; -import { CdpBrowser } from "../../lib/cdp/driver"; import { TaskCancelledError, runTask, type TaskEvent } from "../../lib/core/run-task"; -import { DebuggerTransport, isRestrictedUrl } from "./debugger-transport"; +import { TabBrowser } from "./tab-browser"; import { connectOpenRouter } from "./openrouter-auth"; import { loadSettings, modelConfigFor, saveSettings, type ExtensionSettings } from "./settings"; @@ -65,14 +64,15 @@ function hostOf(url: string): string { /** * The tab WebPilot works in: the active tab of this window. When the panel page - * itself is open as a tab (e.g. in tests), the most recently used web tab instead. + * itself is open as a tab (e.g. in tests), the most recently used other tab. */ async function activeTab(): Promise { + const panelUrl = chrome.runtime.getURL(""); const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }); - if (tab && !tab.url?.startsWith(chrome.runtime.getURL(""))) return tab; + if (tab && !tab.url?.startsWith(panelUrl)) return tab; const tabs = await chrome.tabs.query({ currentWindow: true }); return tabs - .filter((candidate) => /^https?:/i.test(candidate.url || "")) + .filter((candidate) => !(candidate.url || "").startsWith(panelUrl)) .sort((left, right) => (right.lastAccessed || 0) - (left.lastAccessed || 0))[0]; } @@ -292,17 +292,17 @@ function App() { controllerRef.current = controller; const addStep = (step: Omit) => updateTurn(id, (turn) => ({ ...turn, steps: [...turn.steps, { ...step, atMs: Date.now() - startedAt }] })); - let transport: DebuggerTransport | null = null; + let browser: TabBrowser | null = null; try { const tab = await activeTab(); - if (!tab?.id || isRestrictedUrl(tab.url)) { - throw new Error("Open a regular web page in this tab first. Chrome doesn't let extensions control its own pages or the Web Store."); - } - transport = await DebuggerTransport.attach(tab.id); + if (!tab?.id) throw new Error("No tab to work in. Open a tab and try again."); + // Attaches to the tab only when the task needs the page; Chrome pages + // (New Tab, chrome://) are first navigated to the task's website. + browser = new TabBrowser(tab.id); const result = await runTask({ goal: goal.trim(), config: modelConfigFor(settings), - browser: new CdpBrowser(transport), + browser, history, signal: controller.signal, confirm: (action, url) => new Promise((resolve) => { @@ -331,7 +331,7 @@ function App() { const message = error instanceof TaskCancelledError ? "Stopped." : error instanceof Error ? error.message : String(error); updateTurn(id, (turn) => ({ ...turn, error: message, finishedAt: Date.now() })); } finally { - await transport?.detach(); + await browser?.detach(); controllerRef.current = null; setPending(null); setRunning(false); diff --git a/extension/src/tab-browser.ts b/extension/src/tab-browser.ts new file mode 100644 index 0000000..0dbe381 --- /dev/null +++ b/extension/src/tab-browser.ts @@ -0,0 +1,100 @@ +/// + +import { CdpBrowser } from "../../lib/cdp/driver"; +import type { TaskBrowser } from "../../lib/core/run-task"; +import { DebuggerTransport, isRestrictedUrl } from "./debugger-transport"; + +const TAB_LOAD_TIMEOUT_MS = 15_000; + +/** Resolves once the tab finishes loading (or after a timeout). */ +function waitForTabLoad(tabId: number): Promise { + return new Promise((resolve) => { + const done = () => { + clearTimeout(timer); + chrome.tabs.onUpdated.removeListener(listener); + resolve(); + }; + const listener = (updatedId: number, change: { status?: string }) => { + if (updatedId === tabId && change.status === "complete") done(); + }; + const timer = setTimeout(done, TAB_LOAD_TIMEOUT_MS); + chrome.tabs.onUpdated.addListener(listener); + }); +} + +/** + * The browser for one tab, attached to `chrome.debugger` only when a task first + * needs to read or act on the page. Until then (and for chat-only replies) the + * tab is untouched and Chrome shows no debugging bar. + * + * Chrome forbids debugging its own pages (New Tab, chrome://, the Web Store), but + * the tabs API may still navigate them, so a task that starts on one is moved to + * its website first and controlled from there. + */ +export class TabBrowser implements TaskBrowser { + private transport: DebuggerTransport | null = null; + private cdp: CdpBrowser | null = null; + + constructor(public readonly tabId: number) {} + + private async tab(): Promise { + return await chrome.tabs.get(this.tabId); + } + + private async attached(): Promise { + if (this.cdp) return this.cdp; + const tab = await this.tab(); + if (isRestrictedUrl(tab.url)) { + throw new Error("This tab shows a Chrome page that extensions can't control. Mention a website to open, or switch to a regular web page."); + } + this.transport = await DebuggerTransport.attach(this.tabId); + this.cdp = new CdpBrowser(this.transport); + return this.cdp; + } + + async pageInfo(): Promise<{ url: string; title: string }> { + if (this.cdp) return this.cdp.pageInfo(); + const tab = await this.tab(); + return { url: tab.url || "", title: tab.title || "" }; + } + + async navigate(url: string): Promise { + if (!this.cdp && isRestrictedUrl((await this.tab()).url)) { + await chrome.tabs.update(this.tabId, { url }); + await waitForTabLoad(this.tabId); + await this.attached(); + return ""; + } + return (await this.attached()).navigate(url); + } + + async snapshot(): Promise { + return (await this.attached()).snapshot(); + } + + async click(ref: string): Promise { + return (await this.attached()).click(ref); + } + + async type(ref: string, element: string, text: string, submit: boolean): Promise { + return (await this.attached()).type(ref, element, text, submit); + } + + async pressKey(key: string): Promise { + return (await this.attached()).pressKey(key); + } + + async back(): Promise { + return (await this.attached()).back(); + } + + async pageText(maxChars?: number): Promise { + return (await this.attached()).pageText(maxChars); + } + + async detach(): Promise { + await this.transport?.detach(); + this.transport = null; + this.cdp = null; + } +} diff --git a/lib/core/run-task.ts b/lib/core/run-task.ts index 614e4f8..95347cb 100644 --- a/lib/core/run-task.ts +++ b/lib/core/run-task.ts @@ -43,6 +43,11 @@ export interface RunTaskOptions { onEvent?: (event: TaskEvent) => void; signal?: AbortSignal; maxPlannerSteps?: number; + /** + * Where to start when the current tab is not a web page (e.g. the browser's + * New Tab page) and no site is obvious. `{query}` is replaced with the request. + */ + fallbackSearchUrl?: string; } export interface RunTaskResult { @@ -180,19 +185,22 @@ Use "chat" when no browser action or live page is needed (greetings, questions a await browser.navigate(explicitUrl); stats.steps++; } else { + const onWebPage = isWebPage(current.url); const suggestion = await llm.generateText({ model: config.navModel, - systemInstruction: "You pick where a browser task should start. Reply with STAY when the task is about the page the user already has open (or can be done from it); otherwise reply with one absolute https URL. Reply with nothing else.", - prompt: `Task: ${browserGoal}\nCurrent tab: ${current.title || "(blank)"} ${current.url || ""}`, + systemInstruction: "You pick where a browser task should start. Reply with STAY when the task is about the web page the user already has open (or can be done from it); otherwise reply with one absolute https URL, such as the site the task names or a good search page. Reply with nothing else.", + prompt: `Task: ${browserGoal}\nCurrent tab: ${onWebPage ? `${current.title || "(untitled)"} ${current.url}` : "a browser page with no website (such as the New Tab page); STAY is not possible"}`, }); stats.llmCalls++; - const url = extractExplicitUrls(suggestion)[0]; - if (url && !/\bSTAY\b/.test(suggestion)) { + let url = extractExplicitUrls(suggestion)[0]; + if ((!url || /\bSTAY\b/.test(suggestion)) && onWebPage) url = ""; + if (!url && !onWebPage) { + url = (options.fallbackSearchUrl || "https://www.google.com/search?q={query}").replace("{query}", encodeURIComponent(browserGoal)); + } + if (url) { emit({ type: "step", source: "llm", action: "navigate", detail: url, url }); await browser.navigate(url); stats.steps++; - } else if (!isWebPage(current.url)) { - throw new Error("Open a web page in this tab, or mention the site to use."); } } await checkCancelled(); diff --git a/scripts/extension-smoke.ts b/scripts/extension-smoke.ts index 07b7bb9..b5fc71b 100644 --- a/scripts/extension-smoke.ts +++ b/scripts/extension-smoke.ts @@ -19,13 +19,26 @@ import { startRealisticSites } from "./fixtures/realistic-sites.mjs"; interface Case { id: string; start: (base: string) => string; - request: string; + request: string | ((base: string) => string); /** "allow" or "cancel" the confirmation card if it appears. */ onConfirm?: "allow" | "cancel"; check: (answer: string, actions: Array<{ type: string }>, confirmed: boolean, cart: string[]) => string | null; } const CASES: Case[] = [ + { + // Chrome's own pages can't be debugged; the tab is navigated to the site first. + id: "from_blank_tab_with_site", + start: () => "about:blank", + request: (base) => `what's the cheapest 2-person tent at ${base}/shop ?`, + check: (answer) => (/Trailhead/i.test(answer) && /89\.99/.test(answer) ? null : "expected Trailhead 2P at $89.99"), + }, + { + id: "from_blank_tab_no_site", + start: () => "about:blank", + request: "look up the Voyager 1 article on wikipedia and tell me its launch date", + check: (answer) => (/September 5,? 1977|5 September 1977/i.test(answer) ? null : "expected September 5, 1977"), + }, { id: "current_page_question", start: (base) => `${base}/shop`, @@ -75,7 +88,7 @@ async function runCase(context: BrowserContext, panel: Page, base: string, testC await panel.waitForSelector("textarea"); const started = Date.now(); - await panel.fill("textarea", testCase.request); + await panel.fill("textarea", typeof testCase.request === "function" ? testCase.request(base) : testCase.request); await panel.keyboard.press("Enter"); let confirmed = false;