diff --git a/packages/redrob/src/tool/webfetch-focus.ts b/packages/redrob/src/tool/webfetch-focus.ts new file mode 100644 index 0000000000..138ab3d80e --- /dev/null +++ b/packages/redrob/src/tool/webfetch-focus.ts @@ -0,0 +1,60 @@ +/** + * Keep a fetched page small enough that reading it does not cost more than the answer is worth. + * + * A page converted to markdown is routinely 25-35k characters (~8k tokens), and every one of those + * tokens is re-sent on every later step of the turn. Two cuts: when the caller says what it is + * looking for, only the lines around the matches are kept; in all cases the result is capped, with a + * note saying how much was dropped and how to see more. + */ + +export const DEFAULT_MAX_CHARS = 12_000 +const CONTEXT_LINES = 8 + +export function focus(text: string, query: string | undefined, maxChars = DEFAULT_MAX_CHARS): string { + const limit = Math.max(500, maxChars) + const terms = (query ?? "") + .split(/[\s,]+/) + .map((t) => t.trim().toLowerCase()) + .filter((t) => t.length >= 2) + + let body = text + let focused = 0 + if (terms.length > 0) { + const lines = text.split("\n") + const keep = new Array(lines.length).fill(false) + for (let i = 0; i < lines.length; i++) { + const lower = lines[i].toLowerCase() + if (!terms.some((t) => lower.includes(t))) continue + focused++ + for (let j = Math.max(0, i - CONTEXT_LINES); j <= Math.min(lines.length - 1, i + CONTEXT_LINES); j++) keep[j] = true + } + if (focused > 0) { + const out: string[] = [] + let gap = false + for (let i = 0; i < lines.length; i++) { + if (keep[i]) { + if (gap) out.push("[...]") + out.push(lines[i]) + gap = false + } else gap = true + } + body = out.join("\n") + } + } + + const note: string[] = [] + if (terms.length > 0) { + note.push( + focused > 0 + ? `[focused on "${query}": ${focused} matching lines with ${CONTEXT_LINES} lines of context, out of ${text.split("\n").length} lines]` + : `[no line matched "${query}"; showing the start of the page]`, + ) + } + if (body.length > limit) { + body = body.slice(0, limit) + note.push( + `[truncated: ${text.length.toLocaleString()} characters total, showing ${limit.toLocaleString()}. Pass query="..." to keep only the relevant parts, or raise max_chars.]`, + ) + } + return note.length ? `${body}\n\n${note.join("\n")}` : body +} diff --git a/packages/redrob/src/tool/webfetch.ts b/packages/redrob/src/tool/webfetch.ts index 4277dc7e97..b0f92e9109 100644 --- a/packages/redrob/src/tool/webfetch.ts +++ b/packages/redrob/src/tool/webfetch.ts @@ -4,6 +4,7 @@ import { Parser } from "htmlparser2" import * as Tool from "./tool" import TurndownService from "turndown" import DESCRIPTION from "./webfetch.txt" +import { DEFAULT_MAX_CHARS, focus } from "./webfetch-focus" import { isImageAttachment } from "@/util/media" const MAX_RESPONSE_SIZE = 5 * 1024 * 1024 // 5MB @@ -19,6 +20,13 @@ export const Parameters = Schema.Struct({ }) .pipe(Schema.withDecodingDefault(Effect.succeed("markdown" as const))), timeout: Schema.optional(Schema.Number).annotate({ description: "Optional timeout in seconds (max 120)" }), + query: Schema.optional(Schema.String).annotate({ + description: + "What you are looking for on the page (keywords). Only the lines around matches are returned, which keeps a long page cheap to read.", + }), + max_chars: Schema.optional(Schema.Number).annotate({ + description: `Maximum characters to return (default ${DEFAULT_MAX_CHARS}). Longer content is cut with a note.`, + }), }) export const WebFetchTool = Tool.define( @@ -126,29 +134,25 @@ export const WebFetchTool = Tool.define( const content = new TextDecoder().decode(arrayBuffer) // Handle content based on requested format and actual content type + const trim = (text: string) => focus(text, params.query, params.max_chars) switch (params.format) { case "markdown": if (contentType.includes("text/html")) { - const markdown = convertHTMLToMarkdown(content) - return { - output: markdown, - title, - metadata: {}, - } + return { output: trim(convertHTMLToMarkdown(content)), title, metadata: {} } } - return { output: content, title, metadata: {} } + return { output: trim(content), title, metadata: {} } case "text": if (contentType.includes("text/html")) { - return { output: extractTextFromHTML(content), title, metadata: {} } + return { output: trim(extractTextFromHTML(content)), title, metadata: {} } } - return { output: content, title, metadata: {} } + return { output: trim(content), title, metadata: {} } case "html": - return { output: content, title, metadata: {} } + return { output: trim(content), title, metadata: {} } default: - return { output: content, title, metadata: {} } + return { output: trim(content), title, metadata: {} } } }).pipe(Effect.orDie), } diff --git a/packages/redrob/src/tool/webfetch.txt b/packages/redrob/src/tool/webfetch.txt index 169aadefa3..e10bcb2856 100644 --- a/packages/redrob/src/tool/webfetch.txt +++ b/packages/redrob/src/tool/webfetch.txt @@ -10,4 +10,4 @@ Usage notes: - HTTP URLs will be automatically upgraded to HTTPS - Format options: "markdown" (default), "text", or "html" - This tool is read-only and does not modify any files - - Results may be summarized if the content is very large + - Output is capped (default 12,000 characters). Pass `query` with the keywords you are after so only the matching parts of a long page come back; pass `max_chars` to raise the cap when you really need the whole page diff --git a/packages/redrob/test/tool/__snapshots__/parameters.test.ts.snap b/packages/redrob/test/tool/__snapshots__/parameters.test.ts.snap index 51ff867ea4..a8d85a4882 100644 --- a/packages/redrob/test/tool/__snapshots__/parameters.test.ts.snap +++ b/packages/redrob/test/tool/__snapshots__/parameters.test.ts.snap @@ -387,6 +387,14 @@ exports[`tool parameters JSON Schema (wire shape) webfetch 1`] = ` ], "type": "string", }, + "max_chars": { + "description": "Maximum characters to return (default 12000). Longer content is cut with a note.", + "type": "number", + }, + "query": { + "description": "What you are looking for on the page (keywords). Only the lines around matches are returned, which keeps a long page cheap to read.", + "type": "string", + }, "timeout": { "description": "Optional timeout in seconds (max 120)", "type": "number",