From dc0307eedfe9d1b6cb55661ad54e642eda5383be Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sun, 30 Aug 2026 13:00:24 +0300 Subject: [PATCH] Pick the right element by its label when a click matches several A click that matched more than one element asked the model to choose between them, but described each candidate with markup that had been stripped of everything nested inside it. A button whose label lives in child spans arrived as an empty shell, so two menu items reading "New test" and "New tests from requirement" were indistinguishable and the pick was a guess that opened the wrong screen and reported success. Each candidate now carries its own visible text, whitespace collapsed and capped, and its markup comes from the element's own outer HTML cleaned through the project's class filter rather than from an upstream simplifier that removes non-interactive descendants. Labels and meaningful class names survive; layout and generated styling classes do not, so the 300-character budget is spent on signal. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JbhzmY1M51pyRysUx32MdP --- CHANGELOG.md | 13 ++++++ src/ai/tools.ts | 26 ++++++++---- src/utils/html.ts | 14 ++++++ tests/unit/matched-elements.test.ts | 66 +++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 9 deletions(-) create mode 100644 tests/unit/matched-elements.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a82436..d6faf29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 2026-08-30 + +### Changes + +- [Tester] When a click matches more than one element, the pick between them is now made from what + those elements actually say. Candidates were described by their markup alone, and the step that + trimmed that markup threw away everything nested inside — so a menu holding "New test" and "New + tests from requirement" offered two identical empty buttons to choose from, and the choice was a + coin flip that could open the wrong screen and report success. Each candidate now carries its own + visible text, and its markup keeps the label and meaningful class names while layout and generated + styling classes are dropped, so the description says what the element is instead of how it is + styled. + ## 2026-08-29 ### Changes diff --git a/src/ai/tools.ts b/src/ai/tools.ts index d85477f..835ad0a 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -8,8 +8,9 @@ import { Stats } from '../stats.ts'; import { type Task, TestResult } from '../test-plan.js'; import { LARGE_ARIA_CHANGE_THRESHOLD } from '../utils/aria.ts'; import { isFatalBrowserError } from '../utils/browser-errors.ts'; +import { cleanHtmlSnippet } from '../utils/html.ts'; import { createDebug, tag } from '../utils/logger.js'; -import { compactErrorMessage } from '../utils/strings.ts'; +import { compactErrorMessage, normalizeInlineText, truncate } from '../utils/strings.ts'; import { pause } from '../utils/loop.js'; import { WebElement } from '../utils/web-element.ts'; import type { ToolDeps } from './agent.ts'; @@ -1332,20 +1333,23 @@ export function clickFailureSuggestion(attempts: Array<{ error?: string }>): str } const MAX_DISAMBIGUATE_ELEMENTS = 10; +const MAX_DISAMBIGUATE_TEXT = 80; +const MAX_DISAMBIGUATE_HTML = 300; const MULTIPLE_ELEMENTS_PATTERN = 'multiple elements'; -async function extractWebElements(error: Error | null | undefined): Promise | null> { +async function extractWebElements(error: Error | null | undefined): Promise | null> { if (!error || error.name !== 'MultipleElementsFound') return null; - const elements = (error as any).webElements as Array<{ toAbsoluteXPath: () => Promise; toSimplifiedHTML: () => Promise }> | undefined; + const elements = (error as any).webElements as Array<{ toAbsoluteXPath: () => Promise; toOuterHTML: () => Promise; getText: () => Promise }> | undefined; if (!elements?.length) return null; - const result: Array<{ xpath: string; html: string }> = []; + const result: Array<{ xpath: string; html: string; text: string }> = []; for (let i = 0; i < Math.min(elements.length, MAX_DISAMBIGUATE_ELEMENTS); i++) { try { const xpath = await elements[i].toAbsoluteXPath(); - const html = await elements[i].toSimplifiedHTML(); - result.push({ xpath, html }); + const html = truncate(cleanHtmlSnippet(await elements[i].toOuterHTML()), MAX_DISAMBIGUATE_HTML); + const text = truncate(normalizeInlineText((await elements[i].getText()) || ''), MAX_DISAMBIGUATE_TEXT); + result.push({ xpath, html, text }); } catch (e) { debugLog('Failed to get details for element %d: %s', i, e); } @@ -1353,17 +1357,21 @@ async function extractWebElements(error: Error | null | undefined): Promise 0 ? result : null; } -async function formatMatchedElements(error: Error | null | undefined): Promise { +function formatElementList(details: Array<{ xpath: string; html: string; text: string }>): string { + return details.map((el, i) => `Element ${i + 1}:\nText: "${el.text}"\nXPath: ${el.xpath}\nHTML: ${el.html}`).join('\n\n'); +} + +export async function formatMatchedElements(error: Error | null | undefined): Promise { const details = await extractWebElements(error); if (!details) return 'Could not fetch element details. Repeat the action to get better info.'; - return details.map((el, i) => `Element ${i + 1}\nXPath: ${el.xpath}\nHTML: ${el.html}`).join('\n\n'); + return formatElementList(details); } async function disambiguateElements(error: Error | null | undefined, explanation: string, provider: AIProvider): Promise<{ position: number; xpath: string } | null> { const elementDetails = await extractWebElements(error); if (!elementDetails) return null; - const elementList = elementDetails.map((el, i) => `Element ${i + 1}:\nXPath: ${el.xpath}\nHTML: ${el.html}`).join('\n\n'); + const elementList = formatElementList(elementDetails); const schema = z.object({ position: z.number().nullable().describe('1-based position of the correct element, or null if none match'), diff --git a/src/utils/html.ts b/src/utils/html.ts index 266e207..e781034 100644 --- a/src/utils/html.ts +++ b/src/utils/html.ts @@ -1020,6 +1020,19 @@ export function htmlCombinedSnapshot(html: string, htmlConfig?: HtmlConfig['comb return serialize(document); } +/** + * Cleans a small HTML snippet for AI consumption: keeps structure and text, + * drops noisy attributes and generated class names + */ +export function cleanHtmlSnippet(html: string): string { + const fragment = parseFragment(html); + for (const node of fragment.childNodes) { + if (!('tagName' in node)) continue; + cleanAllElements(node as parse5TreeAdapter.Element); + } + return serialize(fragment); +} + /** * Creates text-only snapshot with markdown formatting */ @@ -1526,6 +1539,7 @@ function cleanElement(element: parse5TreeAdapter.Element): void { if (attr.name === 'class') { attr.value = attr.value .split(/\s+/) + .filter(Boolean) .filter((className) => !/\d/.test(className)) .filter((className) => !className.includes(':')) .filter((className) => !TAILWIND_CLASS_PATTERNS.some((pattern) => pattern.test(className))) diff --git a/tests/unit/matched-elements.test.ts b/tests/unit/matched-elements.test.ts new file mode 100644 index 0000000..51df638 --- /dev/null +++ b/tests/unit/matched-elements.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from 'bun:test'; +import { formatMatchedElements } from '../../src/ai/tools.ts'; + +function multipleElementsError(elements: Array<{ xpath: string; html: string; text: string }>): Error { + const error = new Error('Multiple elements found'); + error.name = 'MultipleElementsFound'; + (error as any).webElements = elements.map((el) => ({ + toAbsoluteXPath: async () => el.xpath, + toOuterHTML: async () => el.html, + getText: async () => el.text, + })); + return error; +} + +const TAILWIND_BUTTON = ''; + +describe('formatMatchedElements', () => { + it('reports visible text when simplified html has none', async () => { + const error = multipleElementsError([ + { xpath: '//html/body/ul/li[3]/button', html: TAILWIND_BUTTON, text: 'New test' }, + { xpath: '//html/body/ul/li[4]/div/button', html: TAILWIND_BUTTON, text: 'New tests from requirement' }, + ]); + + const formatted = await formatMatchedElements(error); + + expect(formatted).toContain('Element 1:\nText: "New test"'); + expect(formatted).toContain('Element 2:\nText: "New tests from requirement"'); + }); + + it('collapses whitespace of text spread across nested nodes', async () => { + const error = multipleElementsError([{ xpath: '//html/body/button', html: TAILWIND_BUTTON, text: '\n New\n test \n' }]); + + const formatted = await formatMatchedElements(error); + + expect(formatted).toContain('Text: "New test"'); + }); + + it('caps long text', async () => { + const error = multipleElementsError([{ xpath: '//html/body/button', html: TAILWIND_BUTTON, text: 'a'.repeat(500) }]); + + const formatted = await formatMatchedElements(error); + const text = formatted!.split('\n')[1]; + + expect(text.length).toBeLessThan(100); + expect(text).toEndWith('..."'); + }); + + it('drops utility classes and generated attributes from the html', async () => { + const error = multipleElementsError([{ xpath: '//html/body/button', html: TAILWIND_BUTTON, text: 'New test' }]); + + const formatted = await formatMatchedElements(error); + + expect(formatted).toContain('New test'); + expect(formatted).toContain('class="badge badge-type manual"'); + expect(formatted).not.toContain('inline-flex'); + expect(formatted).not.toContain('items-center'); + expect(formatted).not.toContain('w-full'); + expect(formatted).not.toContain('bg-amber-100'); + }); + + it('falls back when the error carries no elements', async () => { + const formatted = await formatMatchedElements(new Error('boom')); + + expect(formatted).toContain('Could not fetch element details'); + }); +});