diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a82436..d6faf29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 2026-08-30 + +### Changes + +- [Tester] When a click matches more than one element, the pick between them is now made from what + those elements actually say. Candidates were described by their markup alone, and the step that + trimmed that markup threw away everything nested inside — so a menu holding "New test" and "New + tests from requirement" offered two identical empty buttons to choose from, and the choice was a + coin flip that could open the wrong screen and report success. Each candidate now carries its own + visible text, and its markup keeps the label and meaningful class names while layout and generated + styling classes are dropped, so the description says what the element is instead of how it is + styled. + ## 2026-08-29 ### Changes diff --git a/src/ai/tools.ts b/src/ai/tools.ts index d85477f..835ad0a 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -8,8 +8,9 @@ import { Stats } from '../stats.ts'; import { type Task, TestResult } from '../test-plan.js'; import { LARGE_ARIA_CHANGE_THRESHOLD } from '../utils/aria.ts'; import { isFatalBrowserError } from '../utils/browser-errors.ts'; +import { cleanHtmlSnippet } from '../utils/html.ts'; import { createDebug, tag } from '../utils/logger.js'; -import { compactErrorMessage } from '../utils/strings.ts'; +import { compactErrorMessage, normalizeInlineText, truncate } from '../utils/strings.ts'; import { pause } from '../utils/loop.js'; import { WebElement } from '../utils/web-element.ts'; import type { ToolDeps } from './agent.ts'; @@ -1332,20 +1333,23 @@ export function clickFailureSuggestion(attempts: Array<{ error?: string }>): str } const MAX_DISAMBIGUATE_ELEMENTS = 10; +const MAX_DISAMBIGUATE_TEXT = 80; +const MAX_DISAMBIGUATE_HTML = 300; const MULTIPLE_ELEMENTS_PATTERN = 'multiple elements'; -async function extractWebElements(error: Error | null | undefined): Promise | null> { +async function extractWebElements(error: Error | null | undefined): Promise | null> { if (!error || error.name !== 'MultipleElementsFound') return null; - const elements = (error as any).webElements as Array<{ toAbsoluteXPath: () => Promise; toSimplifiedHTML: () => Promise }> | undefined; + const elements = (error as any).webElements as Array<{ toAbsoluteXPath: () => Promise; toOuterHTML: () => Promise; getText: () => Promise }> | undefined; if (!elements?.length) return null; - const result: Array<{ xpath: string; html: string }> = []; + const result: Array<{ xpath: string; html: string; text: string }> = []; for (let i = 0; i < Math.min(elements.length, MAX_DISAMBIGUATE_ELEMENTS); i++) { try { const xpath = await elements[i].toAbsoluteXPath(); - const html = await elements[i].toSimplifiedHTML(); - result.push({ xpath, html }); + const html = truncate(cleanHtmlSnippet(await elements[i].toOuterHTML()), MAX_DISAMBIGUATE_HTML); + const text = truncate(normalizeInlineText((await elements[i].getText()) || ''), MAX_DISAMBIGUATE_TEXT); + result.push({ xpath, html, text }); } catch (e) { debugLog('Failed to get details for element %d: %s', i, e); } @@ -1353,17 +1357,21 @@ async function extractWebElements(error: Error | null | undefined): Promise 0 ? result : null; } -async function formatMatchedElements(error: Error | null | undefined): Promise { +function formatElementList(details: Array<{ xpath: string; html: string; text: string }>): string { + return details.map((el, i) => `Element ${i + 1}:\nText: "${el.text}"\nXPath: ${el.xpath}\nHTML: ${el.html}`).join('\n\n'); +} + +export async function formatMatchedElements(error: Error | null | undefined): Promise { const details = await extractWebElements(error); if (!details) return 'Could not fetch element details. Repeat the action to get better info.'; - return details.map((el, i) => `Element ${i + 1}\nXPath: ${el.xpath}\nHTML: ${el.html}`).join('\n\n'); + return formatElementList(details); } async function disambiguateElements(error: Error | null | undefined, explanation: string, provider: AIProvider): Promise<{ position: number; xpath: string } | null> { const elementDetails = await extractWebElements(error); if (!elementDetails) return null; - const elementList = elementDetails.map((el, i) => `Element ${i + 1}:\nXPath: ${el.xpath}\nHTML: ${el.html}`).join('\n\n'); + const elementList = formatElementList(elementDetails); const schema = z.object({ position: z.number().nullable().describe('1-based position of the correct element, or null if none match'), diff --git a/src/utils/html.ts b/src/utils/html.ts index 266e207..e781034 100644 --- a/src/utils/html.ts +++ b/src/utils/html.ts @@ -1020,6 +1020,19 @@ export function htmlCombinedSnapshot(html: string, htmlConfig?: HtmlConfig['comb return serialize(document); } +/** + * Cleans a small HTML snippet for AI consumption: keeps structure and text, + * drops noisy attributes and generated class names + */ +export function cleanHtmlSnippet(html: string): string { + const fragment = parseFragment(html); + for (const node of fragment.childNodes) { + if (!('tagName' in node)) continue; + cleanAllElements(node as parse5TreeAdapter.Element); + } + return serialize(fragment); +} + /** * Creates text-only snapshot with markdown formatting */ @@ -1526,6 +1539,7 @@ function cleanElement(element: parse5TreeAdapter.Element): void { if (attr.name === 'class') { attr.value = attr.value .split(/\s+/) + .filter(Boolean) .filter((className) => !/\d/.test(className)) .filter((className) => !className.includes(':')) .filter((className) => !TAILWIND_CLASS_PATTERNS.some((pattern) => pattern.test(className))) diff --git a/tests/unit/matched-elements.test.ts b/tests/unit/matched-elements.test.ts new file mode 100644 index 0000000..51df638 --- /dev/null +++ b/tests/unit/matched-elements.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from 'bun:test'; +import { formatMatchedElements } from '../../src/ai/tools.ts'; + +function multipleElementsError(elements: Array<{ xpath: string; html: string; text: string }>): Error { + const error = new Error('Multiple elements found'); + error.name = 'MultipleElementsFound'; + (error as any).webElements = elements.map((el) => ({ + toAbsoluteXPath: async () => el.xpath, + toOuterHTML: async () => el.html, + getText: async () => el.text, + })); + return error; +} + +const TAILWIND_BUTTON = ''; + +describe('formatMatchedElements', () => { + it('reports visible text when simplified html has none', async () => { + const error = multipleElementsError([ + { xpath: '//html/body/ul/li[3]/button', html: TAILWIND_BUTTON, text: 'New test' }, + { xpath: '//html/body/ul/li[4]/div/button', html: TAILWIND_BUTTON, text: 'New tests from requirement' }, + ]); + + const formatted = await formatMatchedElements(error); + + expect(formatted).toContain('Element 1:\nText: "New test"'); + expect(formatted).toContain('Element 2:\nText: "New tests from requirement"'); + }); + + it('collapses whitespace of text spread across nested nodes', async () => { + const error = multipleElementsError([{ xpath: '//html/body/button', html: TAILWIND_BUTTON, text: '\n New\n test \n' }]); + + const formatted = await formatMatchedElements(error); + + expect(formatted).toContain('Text: "New test"'); + }); + + it('caps long text', async () => { + const error = multipleElementsError([{ xpath: '//html/body/button', html: TAILWIND_BUTTON, text: 'a'.repeat(500) }]); + + const formatted = await formatMatchedElements(error); + const text = formatted!.split('\n')[1]; + + expect(text.length).toBeLessThan(100); + expect(text).toEndWith('..."'); + }); + + it('drops utility classes and generated attributes from the html', async () => { + const error = multipleElementsError([{ xpath: '//html/body/button', html: TAILWIND_BUTTON, text: 'New test' }]); + + const formatted = await formatMatchedElements(error); + + expect(formatted).toContain('New test'); + expect(formatted).toContain('class="badge badge-type manual"'); + expect(formatted).not.toContain('inline-flex'); + expect(formatted).not.toContain('items-center'); + expect(formatted).not.toContain('w-full'); + expect(formatted).not.toContain('bg-amber-100'); + }); + + it('falls back when the error carries no elements', async () => { + const formatted = await formatMatchedElements(new Error('boom')); + + expect(formatted).toContain('Could not fetch element details'); + }); +});