From 239619c84713aaf309eb35917fe86dfe1087ef63 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 21 Aug 2026 03:25:12 +0300 Subject: [PATCH] Give clickRef to prima only, the one agent whose context carries refs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #110 said what should happen and then did the opposite: A tool's schema is shared with every caller, and Tester never receives ref-bearing snapshots, so it must not be shown a ref parameter it could only fill by inventing one. The ref tools live in the boat that uses them. The squashed commit put clickRef in createCodeceptJSTools, handing it to Tester, Captain, Rerunner and Driller, and added interactiveAriaWithRefs so the Tester started receiving ref-bearing snapshots after all. In the Tester the tool was worse than dead weight. ACTION_TOOLS never listed clickRef, so a successful ref-click did not count as browser progress in shouldStopForStalledExecution and prepareInstructionsForNextStep never named it. The locator rule told the model to prefer clickRef while the loop treated using it as standing still — a real session spent its last iterations that way and was stopped by the stall detector after a ref-click that had worked. clickRef moves into createRefTools, alongside the createIframeTools precedent in the same file, and prima spreads it in. The paragraphs steering the model to it come out of the shared locator rule and out of click() and form()'s descriptions, since a shared description must not name a tool the caller does not have; prima states it in its own and . Tester returns to getInteractiveARIA(), an exact revert of the #110 hunk. The invariant is refs in context if and only if a ref tool is present. Prima has both, and tests/integration/prima-do.test.ts already guards that its tool list keeps clickRef. Tester, Captain, Rerunner and Navigator have neither. Driller calls annotatePageElements but discards the annotated result, so no refs reach its prompts. A unit test holds the split in place. Co-Authored-By: Claude Opus 5 (1M context) --- boat/prima/src/prima.ts | 4 +- src/ai/rules.ts | 4 -- src/ai/tester.ts | 12 ++---- src/ai/tools.ts | 80 ++++++++++++++++++++-------------------- tests/unit/tools.test.ts | 12 +++++- 5 files changed, 55 insertions(+), 57 deletions(-) diff --git a/boat/prima/src/prima.ts b/boat/prima/src/prima.ts index 600e9d9d..990ea397 100644 --- a/boat/prima/src/prima.ts +++ b/boat/prima/src/prima.ts @@ -10,7 +10,7 @@ import { z } from 'zod'; import { ActionResult } from '../../../src/action-result.ts'; import { getPreviousResearch } from '../../../src/ai/researcher/cache.ts'; import { actionRule, locatorRule } from '../../../src/ai/rules.ts'; -import { createAgentTools, createCodeceptJSTools } from '../../../src/ai/tools.ts'; +import { createAgentTools, createCodeceptJSTools, createRefTools } from '../../../src/ai/tools.ts'; import { getAliveEndpoint, launchServer, listInstances, stopServer } from '../../../src/browser-server.ts'; import { ConfigCommand } from '../../../src/commands/config-command.ts'; import { ConfigMissingError, ConfigParser, type ExplorbotConfig, outputPath } from '../../../src/config.ts'; @@ -148,7 +148,7 @@ export class Prima { const deps = { explorer: this.bot.getExplorer(), stateManager: this.bot.stateManager(), ai: provider }; const ledger: LedgerEntry[] = instructions.map((text) => ({ text, status: 'open', proof: '' })); const descent = { markup: false }; - const tools = { ...createCodeceptJSTools(deps, task), ...this.testerTools(deps), context: this.contextTool(descent), completed: this.completedTool(), blocked: this.blockedTool() }; + const tools = { ...createCodeceptJSTools(deps, task), ...createRefTools(deps, task), ...this.testerTools(deps), context: this.contextTool(descent), completed: this.completedTool(), blocked: this.blockedTool() }; conversation.addUserText(await this.instructionPrompt(instructions, await this.capturedResult(previousState))); const used: string[] = []; diff --git a/src/ai/rules.ts b/src/ai/rules.ts index d1ddaaef..2b887d75 100644 --- a/src/ai/rules.ts +++ b/src/ai/rules.ts @@ -4,10 +4,6 @@ export const recommendedCodeceptCommands = ['I.click', 'I.type', 'I.fillField', const locatorPriorityRule = dedent` - When the page context shows the element a ref, such as [ref=e14], there is no locator to select: click it with clickRef - and that ref. A ref names one exact element, so it never matches the wrong one and never has to be narrowed. Everything - below is for elements the context gives no ref for. - Use the following priority when selecting locators: 1. ARIA locators (first choice) - target browser's accessibility tree, most reliable diff --git a/src/ai/tester.ts b/src/ai/tester.ts index 76802e18..2b81ea60 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -11,7 +11,7 @@ import { Observability } from '../observability.ts'; import type { StateTransition } from '../state-manager.ts'; import { Stats } from '../stats.ts'; import { type Test, TestResult, type TestResultType } from '../test-plan.ts'; -import { compactAriaSnapshot, detectFocusArea } from '../utils/aria.ts'; +import { detectFocusArea } from '../utils/aria.ts'; import { ErrorPageError, isErrorPage } from '../utils/error-page.ts'; import { createDebug, tag } from '../utils/logger.ts'; import { loop } from '../utils/loop.ts'; @@ -602,7 +602,7 @@ export class Tester extends TaskAgent implements Agent { - ${await this.interactiveAriaWithRefs(currentState)} + ${currentState.getInteractiveARIA()} ${uiMapSection} @@ -639,17 +639,11 @@ export class Tester extends TaskAgent implements Agent { - ${await this.interactiveAriaWithRefs(currentState)} + ${currentState.getInteractiveARIA()} `; } - private async interactiveAriaWithRefs(state: ActionResult): Promise { - const withRefs = await Promise.resolve(this.explorer?.withPage?.((page: any) => page.locator('body').ariaSnapshot({ mode: 'ai' }))).catch(() => null); - if (!withRefs) return state.getInteractiveARIA(); - return compactAriaSnapshot(withRefs, false); - } - private finishTest(task: Test): void { if (!task.result) { if (task.hasAchievedAll()) task.finish(TestResult.PASSED); diff --git a/src/ai/tools.ts b/src/ai/tools.ts index 9911c1eb..d0811d7c 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -34,10 +34,6 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, description: dedent` Click an element by trying multiple CodeceptJS commands in order until one succeeds. - Use this only for elements the page context gives you no ref for. When the element shows a ref such as [ref=e14], - call clickRef with that ref instead — composing a locator for an element that already has a ref is wasted work, - and a locator can match several elements where a ref cannot. - Follow from system prompt for locator selection. I.click(locator) - click element matching locator @@ -151,41 +147,6 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, }, }), - clickRef: tool({ - description: dedent` - Click an element by the ref the page context gave it, e.g. [ref=e14]. - - Prefer this over click() whenever the element you want carries a ref. A ref names one exact element, so it - cannot match several by mistake and never needs disambiguating — it is the fastest way to click. - Only pass a ref that appears in the page context you were given. Never invent or guess one. - If it reports the ref is gone, the page has been rebuilt: get fresh context and use the new ref. - `, - inputSchema: z.object({ - ref: z.string().describe('The ref exactly as it appears in the page context, e.g. "e14"'), - element: z.string().describe('Role and name of the element you are clicking, for the record'), - }), - execute: async ({ ref, element }) => { - const activeNote = task.startNote(`Click ${element}`); - const previousState = ActionResult.fromState(stateManager.getCurrentState()!); - const action = explorer.action(); - const named = await describeRef(explorer, ref); - const run = `I.usePlaywrightTo(${JSON.stringify(`click ${element}`)}, async ({ page }) => page.locator(${JSON.stringify(`aria-ref=${ref}`)}).click())`; - - if (!(await action.attempt(run, `Click ${element}`))) { - activeNote.commit(TestResult.FAILED); - return failedToolResult('clickRef', `Ref ${ref} could not be clicked: ${errorText(action.lastError)}`, { - suggestion: 'The ref may belong to an older version of the page. Get fresh context and use the ref it gives, or fall back to click() with a locator.', - }); - } - - // a ref belongs to this session only, so the run is reported as the locator a later test can replay - const code = named ? `I.click(${JSON.stringify(named)})` : run; - const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, code); - await commitNote(activeNote, TestResult.PASSED, toolResult, action); - return successToolResult('clickRef', { ...toolResult, code }, action); - }, - }), - hover: tool({ description: dedent` Move the mouse cursor to an element to reveal hover-only controls. @@ -387,8 +348,6 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, Execute raw CodeceptJS code block with multiple commands. USE THIS TOOL for typing text into fields: I.fillField, I.type - Do not put a click on a ref-bearing element in here — clickRef with its ref is cheaper and cannot mis-target. - Follow from system prompt for available commands. Follow from system prompt for locator selection. @@ -502,6 +461,45 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, }; } +export function createRefTools({ explorer, stateManager }: ToolDeps, task: Task) { + return { + clickRef: tool({ + description: dedent` + Click an element by the ref the page context gave it, e.g. [ref=e14]. + + Prefer this over click() whenever the element you want carries a ref. A ref names one exact element, so it + cannot match several by mistake and never needs disambiguating — it is the fastest way to click. + Only pass a ref that appears in the page context you were given. Never invent or guess one. + If it reports the ref is gone, the page has been rebuilt: get fresh context and use the new ref. + `, + inputSchema: z.object({ + ref: z.string().describe('The ref exactly as it appears in the page context, e.g. "e14"'), + element: z.string().describe('Role and name of the element you are clicking, for the record'), + }), + execute: async ({ ref, element }) => { + const activeNote = task.startNote(`Click ${element}`); + const previousState = ActionResult.fromState(stateManager.getCurrentState()!); + const action = explorer.action(); + const named = await describeRef(explorer, ref); + const run = `I.usePlaywrightTo(${JSON.stringify(`click ${element}`)}, async ({ page }) => page.locator(${JSON.stringify(`aria-ref=${ref}`)}).click())`; + + if (!(await action.attempt(run, `Click ${element}`))) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('clickRef', `Ref ${ref} could not be clicked: ${errorText(action.lastError)}`, { + suggestion: 'The ref may belong to an older version of the page. Get fresh context and use the ref it gives, or fall back to click() with a locator.', + }); + } + + // a ref belongs to this session only, so the run is reported as the locator a later test can replay + const code = named ? `I.click(${JSON.stringify(named)})` : run; + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, code); + await commitNote(activeNote, TestResult.PASSED, toolResult, action); + return successToolResult('clickRef', { ...toolResult, code }, action); + }, + }), + }; +} + export function createIframeTools({ explorer, stateManager }: ToolDeps) { return { exitIframe: tool({ diff --git a/tests/unit/tools.test.ts b/tests/unit/tools.test.ts index 5277d147..2a1ccc87 100644 --- a/tests/unit/tools.test.ts +++ b/tests/unit/tools.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { createCodeceptJSTools, createIframeTools, createLearnExperienceTool } from '../../src/ai/tools.ts'; +import { createCodeceptJSTools, createIframeTools, createLearnExperienceTool, createRefTools } from '../../src/ai/tools.ts'; function fakeDeps(): any { return { @@ -51,6 +51,16 @@ describe('createCodeceptJSTools click validation', () => { }); }); +describe('createRefTools', () => { + it('keeps ref tools out of the shared CodeceptJS tools', () => { + expect(Object.keys(createCodeceptJSTools(fakeDeps(), fakeTask()))).not.toContain('clickRef'); + }); + + it('returns the clickRef tool for callers that supply refs', () => { + expect(Object.keys(createRefTools(fakeDeps(), fakeTask()))).toEqual(['clickRef']); + }); +}); + describe('createIframeTools', () => { it('always returns the exitIframe tool', () => { const tools = createIframeTools(fakeDeps());