From 2ec8ed0878e01b7a91c1f09fdb4da067aaef1bdc Mon Sep 17 00:00:00 2001 From: Janghoon Lee <44862514+savagemanage@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:35:27 +0000 Subject: [PATCH 1/2] webfetch: cap the page and let the model say what it is looking for A fetched page converted to markdown is routinely 25-35k characters, and in the benchmark runs (10 tasks, claude-opus-5) the two research tasks pulled 25k and 35k characters into the context in single webfetch calls, which every later step of the turn then re-sent. The tool now takes `query` (keywords; only the lines around matches come back, with eight lines of context and the gaps marked) and `max_chars` (default 12,000; longer content is cut with a note that says how much there was and how to see more). Images and the fetch itself are unchanged; the generic 50 KB tool-output cap still applies after this. --- packages/redrob/src/tool/webfetch-focus.ts | 60 ++++++++++++++++++++++ packages/redrob/src/tool/webfetch.ts | 26 ++++++---- packages/redrob/src/tool/webfetch.txt | 2 +- 3 files changed, 76 insertions(+), 12 deletions(-) create mode 100644 packages/redrob/src/tool/webfetch-focus.ts diff --git a/packages/redrob/src/tool/webfetch-focus.ts b/packages/redrob/src/tool/webfetch-focus.ts new file mode 100644 index 0000000000..138ab3d80e --- /dev/null +++ b/packages/redrob/src/tool/webfetch-focus.ts @@ -0,0 +1,60 @@ +/** + * Keep a fetched page small enough that reading it does not cost more than the answer is worth. + * + * A page converted to markdown is routinely 25-35k characters (~8k tokens), and every one of those + * tokens is re-sent on every later step of the turn. Two cuts: when the caller says what it is + * looking for, only the lines around the matches are kept; in all cases the result is capped, with a + * note saying how much was dropped and how to see more. + */ + +export const DEFAULT_MAX_CHARS = 12_000 +const CONTEXT_LINES = 8 + +export function focus(text: string, query: string | undefined, maxChars = DEFAULT_MAX_CHARS): string { + const limit = Math.max(500, maxChars) + const terms = (query ?? "") + .split(/[\s,]+/) + .map((t) => t.trim().toLowerCase()) + .filter((t) => t.length >= 2) + + let body = text + let focused = 0 + if (terms.length > 0) { + const lines = text.split("\n") + const keep = new Array(lines.length).fill(false) + for (let i = 0; i < lines.length; i++) { + const lower = lines[i].toLowerCase() + if (!terms.some((t) => lower.includes(t))) continue + focused++ + for (let j = Math.max(0, i - CONTEXT_LINES); j <= Math.min(lines.length - 1, i + CONTEXT_LINES); j++) keep[j] = true + } + if (focused > 0) { + const out: string[] = [] + let gap = false + for (let i = 0; i < lines.length; i++) { + if (keep[i]) { + if (gap) out.push("[...]") + out.push(lines[i]) + gap = false + } else gap = true + } + body = out.join("\n") + } + } + + const note: string[] = [] + if (terms.length > 0) { + note.push( + focused > 0 + ? `[focused on "${query}": ${focused} matching lines with ${CONTEXT_LINES} lines of context, out of ${text.split("\n").length} lines]` + : `[no line matched "${query}"; showing the start of the page]`, + ) + } + if (body.length > limit) { + body = body.slice(0, limit) + note.push( + `[truncated: ${text.length.toLocaleString()} characters total, showing ${limit.toLocaleString()}. Pass query="..." to keep only the relevant parts, or raise max_chars.]`, + ) + } + return note.length ? `${body}\n\n${note.join("\n")}` : body +} diff --git a/packages/redrob/src/tool/webfetch.ts b/packages/redrob/src/tool/webfetch.ts index 4277dc7e97..b0f92e9109 100644 --- a/packages/redrob/src/tool/webfetch.ts +++ b/packages/redrob/src/tool/webfetch.ts @@ -4,6 +4,7 @@ import { Parser } from "htmlparser2" import * as Tool from "./tool" import TurndownService from "turndown" import DESCRIPTION from "./webfetch.txt" +import { DEFAULT_MAX_CHARS, focus } from "./webfetch-focus" import { isImageAttachment } from "@/util/media" const MAX_RESPONSE_SIZE = 5 * 1024 * 1024 // 5MB @@ -19,6 +20,13 @@ export const Parameters = Schema.Struct({ }) .pipe(Schema.withDecodingDefault(Effect.succeed("markdown" as const))), timeout: Schema.optional(Schema.Number).annotate({ description: "Optional timeout in seconds (max 120)" }), + query: Schema.optional(Schema.String).annotate({ + description: + "What you are looking for on the page (keywords). Only the lines around matches are returned, which keeps a long page cheap to read.", + }), + max_chars: Schema.optional(Schema.Number).annotate({ + description: `Maximum characters to return (default ${DEFAULT_MAX_CHARS}). Longer content is cut with a note.`, + }), }) export const WebFetchTool = Tool.define( @@ -126,29 +134,25 @@ export const WebFetchTool = Tool.define( const content = new TextDecoder().decode(arrayBuffer) // Handle content based on requested format and actual content type + const trim = (text: string) => focus(text, params.query, params.max_chars) switch (params.format) { case "markdown": if (contentType.includes("text/html")) { - const markdown = convertHTMLToMarkdown(content) - return { - output: markdown, - title, - metadata: {}, - } + return { output: trim(convertHTMLToMarkdown(content)), title, metadata: {} } } - return { output: content, title, metadata: {} } + return { output: trim(content), title, metadata: {} } case "text": if (contentType.includes("text/html")) { - return { output: extractTextFromHTML(content), title, metadata: {} } + return { output: trim(extractTextFromHTML(content)), title, metadata: {} } } - return { output: content, title, metadata: {} } + return { output: trim(content), title, metadata: {} } case "html": - return { output: content, title, metadata: {} } + return { output: trim(content), title, metadata: {} } default: - return { output: content, title, metadata: {} } + return { output: trim(content), title, metadata: {} } } }).pipe(Effect.orDie), } diff --git a/packages/redrob/src/tool/webfetch.txt b/packages/redrob/src/tool/webfetch.txt index 169aadefa3..e10bcb2856 100644 --- a/packages/redrob/src/tool/webfetch.txt +++ b/packages/redrob/src/tool/webfetch.txt @@ -10,4 +10,4 @@ Usage notes: - HTTP URLs will be automatically upgraded to HTTPS - Format options: "markdown" (default), "text", or "html" - This tool is read-only and does not modify any files - - Results may be summarized if the content is very large + - Output is capped (default 12,000 characters). Pass `query` with the keywords you are after so only the matching parts of a long page come back; pass `max_chars` to raise the cap when you really need the whole page From 204127865b3fa56382d036863adb469c21cf00db Mon Sep 17 00:00:00 2001 From: Janghoon Lee <44862514+savagemanage@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:55:18 +0000 Subject: [PATCH 2/2] test: webfetch wire-schema snapshot gains query and max_chars --- .../test/tool/__snapshots__/parameters.test.ts.snap | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/redrob/test/tool/__snapshots__/parameters.test.ts.snap b/packages/redrob/test/tool/__snapshots__/parameters.test.ts.snap index 51ff867ea4..a8d85a4882 100644 --- a/packages/redrob/test/tool/__snapshots__/parameters.test.ts.snap +++ b/packages/redrob/test/tool/__snapshots__/parameters.test.ts.snap @@ -387,6 +387,14 @@ exports[`tool parameters JSON Schema (wire shape) webfetch 1`] = ` ], "type": "string", }, + "max_chars": { + "description": "Maximum characters to return (default 12000). Longer content is cut with a note.", + "type": "number", + }, + "query": { + "description": "What you are looking for on the page (keywords). Only the lines around matches are returned, which keeps a long page cheap to read.", + "type": "string", + }, "timeout": { "description": "Optional timeout in seconds (max 120)", "type": "number",