diff --git a/boat/prima/src/prima.ts b/boat/prima/src/prima.ts index 600e9d9d..990ea397 100644 --- a/boat/prima/src/prima.ts +++ b/boat/prima/src/prima.ts @@ -10,7 +10,7 @@ import { z } from 'zod'; import { ActionResult } from '../../../src/action-result.ts'; import { getPreviousResearch } from '../../../src/ai/researcher/cache.ts'; import { actionRule, locatorRule } from '../../../src/ai/rules.ts'; -import { createAgentTools, createCodeceptJSTools } from '../../../src/ai/tools.ts'; +import { createAgentTools, createCodeceptJSTools, createRefTools } from '../../../src/ai/tools.ts'; import { getAliveEndpoint, launchServer, listInstances, stopServer } from '../../../src/browser-server.ts'; import { ConfigCommand } from '../../../src/commands/config-command.ts'; import { ConfigMissingError, ConfigParser, type ExplorbotConfig, outputPath } from '../../../src/config.ts'; @@ -148,7 +148,7 @@ export class Prima { const deps = { explorer: this.bot.getExplorer(), stateManager: this.bot.stateManager(), ai: provider }; const ledger: LedgerEntry[] = instructions.map((text) => ({ text, status: 'open', proof: '' })); const descent = { markup: false }; - const tools = { ...createCodeceptJSTools(deps, task), ...this.testerTools(deps), context: this.contextTool(descent), completed: this.completedTool(), blocked: this.blockedTool() }; + const tools = { ...createCodeceptJSTools(deps, task), ...createRefTools(deps, task), ...this.testerTools(deps), context: this.contextTool(descent), completed: this.completedTool(), blocked: this.blockedTool() }; conversation.addUserText(await this.instructionPrompt(instructions, await this.capturedResult(previousState))); const used: string[] = []; diff --git a/src/ai/rules.ts b/src/ai/rules.ts index d1ddaaef..2b887d75 100644 --- a/src/ai/rules.ts +++ b/src/ai/rules.ts @@ -4,10 +4,6 @@ export const recommendedCodeceptCommands = ['I.click', 'I.type', 'I.fillField', const locatorPriorityRule = dedent` - When the page context shows the element a ref, such as [ref=e14], there is no locator to select: click it with clickRef - and that ref. A ref names one exact element, so it never matches the wrong one and never has to be narrowed. Everything - below is for elements the context gives no ref for. - Use the following priority when selecting locators: 1. ARIA locators (first choice) - target browser's accessibility tree, most reliable diff --git a/src/ai/tester.ts b/src/ai/tester.ts index 76802e18..2b81ea60 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -11,7 +11,7 @@ import { Observability } from '../observability.ts'; import type { StateTransition } from '../state-manager.ts'; import { Stats } from '../stats.ts'; import { type Test, TestResult, type TestResultType } from '../test-plan.ts'; -import { compactAriaSnapshot, detectFocusArea } from '../utils/aria.ts'; +import { detectFocusArea } from '../utils/aria.ts'; import { ErrorPageError, isErrorPage } from '../utils/error-page.ts'; import { createDebug, tag } from '../utils/logger.ts'; import { loop } from '../utils/loop.ts'; @@ -602,7 +602,7 @@ export class Tester extends TaskAgent implements Agent { - ${await this.interactiveAriaWithRefs(currentState)} + ${currentState.getInteractiveARIA()} ${uiMapSection} @@ -639,17 +639,11 @@ export class Tester extends TaskAgent implements Agent { - ${await this.interactiveAriaWithRefs(currentState)} + ${currentState.getInteractiveARIA()} `; } - private async interactiveAriaWithRefs(state: ActionResult): Promise { - const withRefs = await Promise.resolve(this.explorer?.withPage?.((page: any) => page.locator('body').ariaSnapshot({ mode: 'ai' }))).catch(() => null); - if (!withRefs) return state.getInteractiveARIA(); - return compactAriaSnapshot(withRefs, false); - } - private finishTest(task: Test): void { if (!task.result) { if (task.hasAchievedAll()) task.finish(TestResult.PASSED); diff --git a/src/ai/tools.ts b/src/ai/tools.ts index 9911c1eb..d0811d7c 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -34,10 +34,6 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, description: dedent` Click an element by trying multiple CodeceptJS commands in order until one succeeds. - Use this only for elements the page context gives you no ref for. When the element shows a ref such as [ref=e14], - call clickRef with that ref instead — composing a locator for an element that already has a ref is wasted work, - and a locator can match several elements where a ref cannot. - Follow from system prompt for locator selection. I.click(locator) - click element matching locator @@ -151,41 +147,6 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, }, }), - clickRef: tool({ - description: dedent` - Click an element by the ref the page context gave it, e.g. [ref=e14]. - - Prefer this over click() whenever the element you want carries a ref. A ref names one exact element, so it - cannot match several by mistake and never needs disambiguating — it is the fastest way to click. - Only pass a ref that appears in the page context you were given. Never invent or guess one. - If it reports the ref is gone, the page has been rebuilt: get fresh context and use the new ref. - `, - inputSchema: z.object({ - ref: z.string().describe('The ref exactly as it appears in the page context, e.g. "e14"'), - element: z.string().describe('Role and name of the element you are clicking, for the record'), - }), - execute: async ({ ref, element }) => { - const activeNote = task.startNote(`Click ${element}`); - const previousState = ActionResult.fromState(stateManager.getCurrentState()!); - const action = explorer.action(); - const named = await describeRef(explorer, ref); - const run = `I.usePlaywrightTo(${JSON.stringify(`click ${element}`)}, async ({ page }) => page.locator(${JSON.stringify(`aria-ref=${ref}`)}).click())`; - - if (!(await action.attempt(run, `Click ${element}`))) { - activeNote.commit(TestResult.FAILED); - return failedToolResult('clickRef', `Ref ${ref} could not be clicked: ${errorText(action.lastError)}`, { - suggestion: 'The ref may belong to an older version of the page. Get fresh context and use the ref it gives, or fall back to click() with a locator.', - }); - } - - // a ref belongs to this session only, so the run is reported as the locator a later test can replay - const code = named ? `I.click(${JSON.stringify(named)})` : run; - const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, code); - await commitNote(activeNote, TestResult.PASSED, toolResult, action); - return successToolResult('clickRef', { ...toolResult, code }, action); - }, - }), - hover: tool({ description: dedent` Move the mouse cursor to an element to reveal hover-only controls. @@ -387,8 +348,6 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, Execute raw CodeceptJS code block with multiple commands. USE THIS TOOL for typing text into fields: I.fillField, I.type - Do not put a click on a ref-bearing element in here — clickRef with its ref is cheaper and cannot mis-target. - Follow from system prompt for available commands. Follow from system prompt for locator selection. @@ -502,6 +461,45 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, }; } +export function createRefTools({ explorer, stateManager }: ToolDeps, task: Task) { + return { + clickRef: tool({ + description: dedent` + Click an element by the ref the page context gave it, e.g. [ref=e14]. + + Prefer this over click() whenever the element you want carries a ref. A ref names one exact element, so it + cannot match several by mistake and never needs disambiguating — it is the fastest way to click. + Only pass a ref that appears in the page context you were given. Never invent or guess one. + If it reports the ref is gone, the page has been rebuilt: get fresh context and use the new ref. + `, + inputSchema: z.object({ + ref: z.string().describe('The ref exactly as it appears in the page context, e.g. "e14"'), + element: z.string().describe('Role and name of the element you are clicking, for the record'), + }), + execute: async ({ ref, element }) => { + const activeNote = task.startNote(`Click ${element}`); + const previousState = ActionResult.fromState(stateManager.getCurrentState()!); + const action = explorer.action(); + const named = await describeRef(explorer, ref); + const run = `I.usePlaywrightTo(${JSON.stringify(`click ${element}`)}, async ({ page }) => page.locator(${JSON.stringify(`aria-ref=${ref}`)}).click())`; + + if (!(await action.attempt(run, `Click ${element}`))) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('clickRef', `Ref ${ref} could not be clicked: ${errorText(action.lastError)}`, { + suggestion: 'The ref may belong to an older version of the page. Get fresh context and use the ref it gives, or fall back to click() with a locator.', + }); + } + + // a ref belongs to this session only, so the run is reported as the locator a later test can replay + const code = named ? `I.click(${JSON.stringify(named)})` : run; + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, code); + await commitNote(activeNote, TestResult.PASSED, toolResult, action); + return successToolResult('clickRef', { ...toolResult, code }, action); + }, + }), + }; +} + export function createIframeTools({ explorer, stateManager }: ToolDeps) { return { exitIframe: tool({ diff --git a/tests/unit/tools.test.ts b/tests/unit/tools.test.ts index 5277d147..2a1ccc87 100644 --- a/tests/unit/tools.test.ts +++ b/tests/unit/tools.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { createCodeceptJSTools, createIframeTools, createLearnExperienceTool } from '../../src/ai/tools.ts'; +import { createCodeceptJSTools, createIframeTools, createLearnExperienceTool, createRefTools } from '../../src/ai/tools.ts'; function fakeDeps(): any { return { @@ -51,6 +51,16 @@ describe('createCodeceptJSTools click validation', () => { }); }); +describe('createRefTools', () => { + it('keeps ref tools out of the shared CodeceptJS tools', () => { + expect(Object.keys(createCodeceptJSTools(fakeDeps(), fakeTask()))).not.toContain('clickRef'); + }); + + it('returns the clickRef tool for callers that supply refs', () => { + expect(Object.keys(createRefTools(fakeDeps(), fakeTask()))).toEqual(['clickRef']); + }); +}); + describe('createIframeTools', () => { it('always returns the exitIframe tool', () => { const tools = createIframeTools(fakeDeps());