webfetch: cap the page and let the model say what it is looking for

packages/redrob/src/tool/webfetch-focus.ts

@@ -0,0 +1,60 @@
+/**
+ * Keep a fetched page small enough that reading it does not cost more than the answer is worth.
+ *
+ * A page converted to markdown is routinely 25-35k characters (~8k tokens), and every one of those
+ * tokens is re-sent on every later step of the turn. Two cuts: when the caller says what it is
+ * looking for, only the lines around the matches are kept; in all cases the result is capped, with a
+ * note saying how much was dropped and how to see more.
+ */
+
+export const DEFAULT_MAX_CHARS = 12_000
+const CONTEXT_LINES = 8
+
+export function focus(text: string, query: string | undefined, maxChars = DEFAULT_MAX_CHARS): string {
+  const limit = Math.max(500, maxChars)
+  const terms = (query ?? "")
+    .split(/[\s,]+/)
+    .map((t) => t.trim().toLowerCase())
+    .filter((t) => t.length >= 2)
+
+  let body = text
+  let focused = 0
+  if (terms.length > 0) {
+    const lines = text.split("\n")
+    const keep = new Array<boolean>(lines.length).fill(false)
+    for (let i = 0; i < lines.length; i++) {
+      const lower = lines[i].toLowerCase()
+      if (!terms.some((t) => lower.includes(t))) continue
+      focused++
+      for (let j = Math.max(0, i - CONTEXT_LINES); j <= Math.min(lines.length - 1, i + CONTEXT_LINES); j++) keep[j] = true
+    }
+    if (focused > 0) {
+      const out: string[] = []
+      let gap = false
+      for (let i = 0; i < lines.length; i++) {
+        if (keep[i]) {
+          if (gap) out.push("[...]")
+          out.push(lines[i])
+          gap = false
+        } else gap = true
+      }
+      body = out.join("\n")
+    }
+  }
+
+  const note: string[] = []
+  if (terms.length > 0) {
+    note.push(
+      focused > 0
+        ? `[focused on "${query}": ${focused} matching lines with ${CONTEXT_LINES} lines of context, out of ${text.split("\n").length} lines]`
+        : `[no line matched "${query}"; showing the start of the page]`,
+    )
+  }
+  if (body.length > limit) {
+    body = body.slice(0, limit)
+    note.push(
+      `[truncated: ${text.length.toLocaleString()} characters total, showing ${limit.toLocaleString()}. Pass query="..." to keep only the relevant parts, or raise max_chars.]`,
+    )
+  }
+  return note.length ? `${body}\n\n${note.join("\n")}` : body
+}

packages/redrob/src/tool/webfetch.ts

@@ -4,6 +4,7 @@ import { Parser } from "htmlparser2"
 import * as Tool from "./tool"
 import TurndownService from "turndown"
 import DESCRIPTION from "./webfetch.txt"
+import { DEFAULT_MAX_CHARS, focus } from "./webfetch-focus"
 import { isImageAttachment } from "@/util/media"
 
 const MAX_RESPONSE_SIZE = 5 * 1024 * 1024 // 5MB
@@ -19,6 +20,13 @@ export const Parameters = Schema.Struct({
     })
     .pipe(Schema.withDecodingDefault(Effect.succeed("markdown" as const))),
   timeout: Schema.optional(Schema.Number).annotate({ description: "Optional timeout in seconds (max 120)" }),
+  query: Schema.optional(Schema.String).annotate({
+    description:
+      "What you are looking for on the page (keywords). Only the lines around matches are returned, which keeps a long page cheap to read.",
+  }),
+  max_chars: Schema.optional(Schema.Number).annotate({
+    description: `Maximum characters to return (default ${DEFAULT_MAX_CHARS}). Longer content is cut with a note.`,
+  }),
 })
 
 export const WebFetchTool = Tool.define(
@@ -126,29 +134,25 @@ export const WebFetchTool = Tool.define(
           const content = new TextDecoder().decode(arrayBuffer)
 
           // Handle content based on requested format and actual content type
+          const trim = (text: string) => focus(text, params.query, params.max_chars)
           switch (params.format) {
             case "markdown":
               if (contentType.includes("text/html")) {
-                const markdown = convertHTMLToMarkdown(content)
-                return {
-                  output: markdown,
-                  title,
-                  metadata: {},
-                }
+                return { output: trim(convertHTMLToMarkdown(content)), title, metadata: {} }
               }
-              return { output: content, title, metadata: {} }
+              return { output: trim(content), title, metadata: {} }
 
             case "text":
               if (contentType.includes("text/html")) {
-                return { output: extractTextFromHTML(content), title, metadata: {} }
+                return { output: trim(extractTextFromHTML(content)), title, metadata: {} }
               }
-              return { output: content, title, metadata: {} }
+              return { output: trim(content), title, metadata: {} }
 
             case "html":
-              return { output: content, title, metadata: {} }
+              return { output: trim(content), title, metadata: {} }
 
             default:
-              return { output: content, title, metadata: {} }
+              return { output: trim(content), title, metadata: {} }
           }
         }).pipe(Effect.orDie),
     }

packages/redrob/src/tool/webfetch.txt

@@ -10,4 +10,4 @@ Usage notes:
   - HTTP URLs will be automatically upgraded to HTTPS
   - Format options: "markdown" (default), "text", or "html"
   - This tool is read-only and does not modify any files
-  - Results may be summarized if the content is very large
+  - Output is capped (default 12,000 characters). Pass `query` with the keywords you are after so only the matching parts of a long page come back; pass `max_chars` to raise the cap when you really need the whole page

packages/redrob/test/tool/__snapshots__/parameters.test.ts.snap

@@ -387,6 +387,14 @@ exports[`tool parameters JSON Schema (wire shape) webfetch 1`] = `
       ],
       "type": "string",
     },
+    "max_chars": {
+      "description": "Maximum characters to return (default 12000). Longer content is cut with a note.",
+      "type": "number",
+    },
+    "query": {
+      "description": "What you are looking for on the page (keywords). Only the lines around matches are returned, which keeps a long page cheap to read.",
+      "type": "string",
+    },
     "timeout": {
       "description": "Optional timeout in seconds (max 120)",
       "type": "number",