Extension: start tasks from the New Tab page and other Chrome pages

extension/src/sidepanel.tsx

@@ -3,9 +3,8 @@
 import { useCallback, useEffect, useRef, useState } from "react";
 import { createRoot } from "react-dom/client";
 import ReactMarkdown from "react-markdown";
-import { CdpBrowser } from "../../lib/cdp/driver";
 import { TaskCancelledError, runTask, type TaskEvent } from "../../lib/core/run-task";
-import { DebuggerTransport, isRestrictedUrl } from "./debugger-transport";
+import { TabBrowser } from "./tab-browser";
 import { connectOpenRouter } from "./openrouter-auth";
 import { loadSettings, modelConfigFor, saveSettings, type ExtensionSettings } from "./settings";
 
@@ -65,14 +64,15 @@ function hostOf(url: string): string {
 
 /**
  * The tab WebPilot works in: the active tab of this window. When the panel page
- * itself is open as a tab (e.g. in tests), the most recently used web tab instead.
+ * itself is open as a tab (e.g. in tests), the most recently used other tab.
  */
 async function activeTab(): Promise<chrome.tabs.Tab | undefined> {
+    const panelUrl = chrome.runtime.getURL("");
     const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });
-    if (tab && !tab.url?.startsWith(chrome.runtime.getURL(""))) return tab;
+    if (tab && !tab.url?.startsWith(panelUrl)) return tab;
     const tabs = await chrome.tabs.query({ currentWindow: true });
     return tabs
-        .filter((candidate) => /^https?:/i.test(candidate.url || ""))
+        .filter((candidate) => !(candidate.url || "").startsWith(panelUrl))
         .sort((left, right) => (right.lastAccessed || 0) - (left.lastAccessed || 0))[0];
 }
 
@@ -292,17 +292,17 @@ function App() {
         controllerRef.current = controller;
         const addStep = (step: Omit<LogStep, "atMs">) => updateTurn(id, (turn) => ({ ...turn, steps: [...turn.steps, { ...step, atMs: Date.now() - startedAt }] }));
 
-        let transport: DebuggerTransport | null = null;
+        let browser: TabBrowser | null = null;
         try {
             const tab = await activeTab();
-            if (!tab?.id || isRestrictedUrl(tab.url)) {
-                throw new Error("Open a regular web page in this tab first. Chrome doesn't let extensions control its own pages or the Web Store.");
-            }
-            transport = await DebuggerTransport.attach(tab.id);
+            if (!tab?.id) throw new Error("No tab to work in. Open a tab and try again.");
+            // Attaches to the tab only when the task needs the page; Chrome pages
+            // (New Tab, chrome://) are first navigated to the task's website.
+            browser = new TabBrowser(tab.id);
             const result = await runTask({
                 goal: goal.trim(),
                 config: modelConfigFor(settings),
-                browser: new CdpBrowser(transport),
+                browser,
                 history,
                 signal: controller.signal,
                 confirm: (action, url) => new Promise<boolean>((resolve) => {
@@ -331,7 +331,7 @@ function App() {
             const message = error instanceof TaskCancelledError ? "Stopped." : error instanceof Error ? error.message : String(error);
             updateTurn(id, (turn) => ({ ...turn, error: message, finishedAt: Date.now() }));
         } finally {
-            await transport?.detach();
+            await browser?.detach();
             controllerRef.current = null;
             setPending(null);
             setRunning(false);

extension/src/tab-browser.ts

@@ -0,0 +1,100 @@
+/// <reference types="chrome" />
+
+import { CdpBrowser } from "../../lib/cdp/driver";
+import type { TaskBrowser } from "../../lib/core/run-task";
+import { DebuggerTransport, isRestrictedUrl } from "./debugger-transport";
+
+const TAB_LOAD_TIMEOUT_MS = 15_000;
+
+/** Resolves once the tab finishes loading (or after a timeout). */
+function waitForTabLoad(tabId: number): Promise<void> {
+    return new Promise((resolve) => {
+        const done = () => {
+            clearTimeout(timer);
+            chrome.tabs.onUpdated.removeListener(listener);
+            resolve();
+        };
+        const listener = (updatedId: number, change: { status?: string }) => {
+            if (updatedId === tabId && change.status === "complete") done();
+        };
+        const timer = setTimeout(done, TAB_LOAD_TIMEOUT_MS);
+        chrome.tabs.onUpdated.addListener(listener);
+    });
+}
+
+/**
+ * The browser for one tab, attached to `chrome.debugger` only when a task first
+ * needs to read or act on the page. Until then (and for chat-only replies) the
+ * tab is untouched and Chrome shows no debugging bar.
+ *
+ * Chrome forbids debugging its own pages (New Tab, chrome://, the Web Store), but
+ * the tabs API may still navigate them, so a task that starts on one is moved to
+ * its website first and controlled from there.
+ */
+export class TabBrowser implements TaskBrowser {
+    private transport: DebuggerTransport | null = null;
+    private cdp: CdpBrowser | null = null;
+
+    constructor(public readonly tabId: number) {}
+
+    private async tab(): Promise<chrome.tabs.Tab> {
+        return await chrome.tabs.get(this.tabId);
+    }
+
+    private async attached(): Promise<CdpBrowser> {
+        if (this.cdp) return this.cdp;
+        const tab = await this.tab();
+        if (isRestrictedUrl(tab.url)) {
+            throw new Error("This tab shows a Chrome page that extensions can't control. Mention a website to open, or switch to a regular web page.");
+        }
+        this.transport = await DebuggerTransport.attach(this.tabId);
+        this.cdp = new CdpBrowser(this.transport);
+        return this.cdp;
+    }
+
+    async pageInfo(): Promise<{ url: string; title: string }> {
+        if (this.cdp) return this.cdp.pageInfo();
+        const tab = await this.tab();
+        return { url: tab.url || "", title: tab.title || "" };
+    }
+
+    async navigate(url: string): Promise<string> {
+        if (!this.cdp && isRestrictedUrl((await this.tab()).url)) {
+            await chrome.tabs.update(this.tabId, { url });
+            await waitForTabLoad(this.tabId);
+            await this.attached();
+            return "";
+        }
+        return (await this.attached()).navigate(url);
+    }
+
+    async snapshot(): Promise<string> {
+        return (await this.attached()).snapshot();
+    }
+
+    async click(ref: string): Promise<string> {
+        return (await this.attached()).click(ref);
+    }
+
+    async type(ref: string, element: string, text: string, submit: boolean): Promise<string> {
+        return (await this.attached()).type(ref, element, text, submit);
+    }
+
+    async pressKey(key: string): Promise<string> {
+        return (await this.attached()).pressKey(key);
+    }
+
+    async back(): Promise<string> {
+        return (await this.attached()).back();
+    }
+
+    async pageText(maxChars?: number): Promise<string> {
+        return (await this.attached()).pageText(maxChars);
+    }
+
+    async detach(): Promise<void> {
+        await this.transport?.detach();
+        this.transport = null;
+        this.cdp = null;
+    }
+}

lib/core/run-task.ts

@@ -43,6 +43,11 @@ export interface RunTaskOptions {
     onEvent?: (event: TaskEvent) => void;
     signal?: AbortSignal;
     maxPlannerSteps?: number;
+    /**
+     * Where to start when the current tab is not a web page (e.g. the browser's
+     * New Tab page) and no site is obvious. `{query}` is replaced with the request.
+     */
+    fallbackSearchUrl?: string;
 }
 
 export interface RunTaskResult {
@@ -180,19 +185,22 @@ Use "chat" when no browser action or live page is needed (greetings, questions a
         await browser.navigate(explicitUrl);
         stats.steps++;
     } else {
+        const onWebPage = isWebPage(current.url);
         const suggestion = await llm.generateText({
             model: config.navModel,
-            systemInstruction: "You pick where a browser task should start. Reply with STAY when the task is about the page the user already has open (or can be done from it); otherwise reply with one absolute https URL. Reply with nothing else.",
-            prompt: `Task: ${browserGoal}\nCurrent tab: ${current.title || "(blank)"} ${current.url || ""}`,
+            systemInstruction: "You pick where a browser task should start. Reply with STAY when the task is about the web page the user already has open (or can be done from it); otherwise reply with one absolute https URL, such as the site the task names or a good search page. Reply with nothing else.",
+            prompt: `Task: ${browserGoal}\nCurrent tab: ${onWebPage ? `${current.title || "(untitled)"} ${current.url}` : "a browser page with no website (such as the New Tab page); STAY is not possible"}`,
         });
         stats.llmCalls++;
-        const url = extractExplicitUrls(suggestion)[0];
-        if (url && !/\bSTAY\b/.test(suggestion)) {
+        let url = extractExplicitUrls(suggestion)[0];
+        if ((!url || /\bSTAY\b/.test(suggestion)) && onWebPage) url = "";
+        if (!url && !onWebPage) {
+            url = (options.fallbackSearchUrl || "https://www.google.com/search?q={query}").replace("{query}", encodeURIComponent(browserGoal));
+        }
+        if (url) {
             emit({ type: "step", source: "llm", action: "navigate", detail: url, url });
             await browser.navigate(url);
             stats.steps++;
-        } else if (!isWebPage(current.url)) {
-            throw new Error("Open a web page in this tab, or mention the site to use.");
         }
     }
     await checkCancelled();

scripts/extension-smoke.ts

@@ -19,13 +19,26 @@ import { startRealisticSites } from "./fixtures/realistic-sites.mjs";
 interface Case {
     id: string;
     start: (base: string) => string;
-    request: string;
+    request: string | ((base: string) => string);
     /** "allow" or "cancel" the confirmation card if it appears. */
     onConfirm?: "allow" | "cancel";
     check: (answer: string, actions: Array<{ type: string }>, confirmed: boolean, cart: string[]) => string | null;
 }
 
 const CASES: Case[] = [
+    {
+        // Chrome's own pages can't be debugged; the tab is navigated to the site first.
+        id: "from_blank_tab_with_site",
+        start: () => "about:blank",
+        request: (base) => `what's the cheapest 2-person tent at ${base}/shop ?`,
+        check: (answer) => (/Trailhead/i.test(answer) && /89\.99/.test(answer) ? null : "expected Trailhead 2P at $89.99"),
+    },
+    {
+        id: "from_blank_tab_no_site",
+        start: () => "about:blank",
+        request: "look up the Voyager 1 article on wikipedia and tell me its launch date",
+        check: (answer) => (/September 5,? 1977|5 September 1977/i.test(answer) ? null : "expected September 5, 1977"),
+    },
     {
         id: "current_page_question",
         start: (base) => `${base}/shop`,
@@ -75,7 +88,7 @@ async function runCase(context: BrowserContext, panel: Page, base: string, testC
     await panel.waitForSelector("textarea");
 
     const started = Date.now();
-    await panel.fill("textarea", testCase.request);
+    await panel.fill("textarea", typeof testCase.request === "function" ? testCase.request(base) : testCase.request);
     await panel.keyboard.press("Enter");
 
     let confirmed = false;