From ef8976c56a4630431196928823cdb2450cb74e54 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 18 Aug 2026 00:51:34 +0300 Subject: [PATCH] Explain why navigation failed, and stop when the browser is gone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Navigation failures reported only where the browser ended up — "redirected to /users/sign_in and could not resolve" — which named the symptom and never the cause. The common case is a login page whose credentials were never provided as knowledge, but that is one cause among several, so the reason is now composed the same way for all of them: the blocker the AI reported through the stop tool, otherwise the step that kept failing, plus the `learn` command to run when nothing is known about the page. The reason reaches both callers. `visit()` puts it in the thrown error, and the `interact` tool appends it to its failure instead of discarding it. resolveState carried that logic inside a ~300 line method. It is now an orchestrator over value-returning helpers, so the loop body reads as its three real phases: ask the AI for a batch, feed failures back, run one block. No behavior moved in that step — the tests added here were written against the unmodified method first, and stayed green through it. Those tests pin the parts most likely to drift: an AI call and the first attempt share one loop iteration, code blocks are alternatives without an expected URL but a sequence with one, and progress is measured against two different baselines — the state resolveState started from for resolution, the state before the attempt for the ARIA diff. One behavior did change deliberately. The loop swallowed every error, including the BrowserRecoveryError raised when the browser cannot be restored, so a dead browser burned every remaining attempt and a model call with each. Fatal browser errors now propagate, which is what Action.attempt rethrows them for and what the interact tool's throwIfFatalBrowserError already expects. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 13 + src/ai/navigator.ts | 419 +++++++++++--------- src/ai/tools.ts | 5 +- tests/unit/navigator-failure-reason.test.ts | 46 +++ tests/unit/navigator-resolve-state.test.ts | 310 +++++++++++++++ 5 files changed, 614 insertions(+), 179 deletions(-) create mode 100644 tests/unit/navigator-failure-reason.test.ts create mode 100644 tests/unit/navigator-resolve-state.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index eaafe561..bd2d641f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 2026-08-18 + +### Changes + +- Navigation that fails now says why. It used to report only where the browser ended up + ("redirected to /users/sign_in and could not resolve"); it now names the blocker the AI reported, + or the step that kept failing, and points at `explorbot learn ""` when nothing is known + about the page — the usual case being a login page whose credentials were never provided. +- A failed `interact` step carries that same reason instead of a bare "Failed to execute", so it + reaches the test log instead of stopping at the navigator. +- A browser that crashes and cannot be restored now stops navigation at once. It used to spend every + remaining attempt, and a model call with each one, against a page that was already gone. + ## 2026-08-16 ### Changes diff --git a/src/ai/navigator.ts b/src/ai/navigator.ts index 94bd1979..a8d50163 100644 --- a/src/ai/navigator.ts +++ b/src/ai/navigator.ts @@ -9,11 +9,14 @@ import Explorer from '../explorer.ts'; import type { KnowledgeTracker } from '../knowledge-tracker.js'; import { type StateManager, normalizeUrl } from '../state-manager.js'; import { renderAssertion } from '../playwright-recorder.ts'; +import { isFatalBrowserError } from '../utils/browser-errors.ts'; +import { getCliName } from '../utils/cli-name.ts'; import { extractCodeBlocks } from '../utils/code-extractor.js'; import { HooksRunner } from '../utils/hooks-runner.ts'; import { createDebug, pluralize, tag } from '../utils/logger.js'; import { loop, pause } from '../utils/loop.js'; import { RulesLoader } from '../utils/rules-loader.ts'; +import { normalizeInlineText } from '../utils/strings.ts'; import { extractStatePath, matchesNavigationUrl } from '../utils/url-matcher.js'; import type { Agent, AgentDeps } from './agent.js'; import type { Conversation } from './conversation.js'; @@ -33,6 +36,7 @@ class Navigator implements Agent { private hooksRunner: HooksRunner; private MAX_ATTEMPTS = Number.parseInt(process.env.MAX_ATTEMPTS || '5'); + lastFailureReason: string | null = null; private systemPrompt = dedent` @@ -105,8 +109,12 @@ class Navigator implements Agent { private getComparableCurrentUrl(stateManager: any, expectedUrl: string): string { const currentState = stateManager.getCurrentState(); if (!currentState) return ''; - const current = /^https?:\/\//i.test(expectedUrl) ? currentState.fullUrl || currentState.url || '' : currentState.url || ''; - return current; + return this.comparableUrl(currentState, expectedUrl); + } + + private comparableUrl(state: { url?: string; fullUrl?: string }, expectedUrl: string): string { + if (/^https?:\/\//i.test(expectedUrl)) return state.fullUrl || state.url || ''; + return state.url || ''; } private isSameExpectedOrigin(expectedUrl: string, stateManager: any): boolean { @@ -162,7 +170,7 @@ class Navigator implements Agent { const resolved = await this.resolveState(originalMessage, actionResult, { action, expectedUrl: url }); if (!resolved) { - throw new Error(`Navigation to ${url} failed: redirected to ${actualPath} and could not resolve`); + throw this.navigationError(url, `redirected to ${actualPath} and could not resolve`); } } else if (action.lastError) { const actionResult = action.actionResult || ActionResult.fromState(action.stateManager.getCurrentState()!); @@ -174,7 +182,7 @@ class Navigator implements Agent { const resolved = await this.resolveState(originalMessage, actionResult, { action, expectedUrl: url }); if (!resolved) { - throw new Error(`Navigation to ${url} failed: ${action.lastError?.message}`); + throw this.navigationError(url, action.lastError?.message || 'Navigation failed'); } } await this.explorer.capture({ screenshot: true }); @@ -190,9 +198,25 @@ class Navigator implements Agent { } } + private navigationError(url: string, fallback: string): Error { + if (this.lastFailureReason) return new Error(`Navigation to ${url} failed: ${this.lastFailureReason}`); + return new Error(`Navigation to ${url} failed: ${fallback}`); + } + + private failureReason(stopReason: string | null, knowledge: string, url: string): string | null { + const reasons: string[] = []; + if (stopReason) reasons.push(stopReason); + if (!knowledge) { + const path = extractStatePath(url).split('?')[0].split('#')[0]; + reasons.push(`no knowledge is set for ${path} — teach it what this page needs (credentials, hints) with: ${getCliName()} learn "${path}" ""`); + } + return reasons.join('; ') || null; + } + async resolveState(message: string, actionResult: ActionResult, opts?: { action?: Action; expectedUrl?: string; onAttempt?: (attempt: { code: string; error?: string }) => void }): Promise { if (!this.provider) throw new Error('AI-assisted recovery is unavailable: no AI model is configured.'); + this.lastFailureReason = null; tag('info').log('AI Navigator resolving state at', actionResult.url); debugLog('Resolution message:', message); @@ -200,52 +224,9 @@ class Navigator implements Agent { const expectedUrl = opts?.expectedUrl; const knowledge = this.knowledgeTracker.renderRelevantContext(actionResult); - let experience = ''; - - if (!actionResult.isInsideIframe) { - const successful = this.experienceTracker.getSuccessfulExperience(actionResult); - if (successful.length > 0) { - tag('operation').log(`Found ${successful.length} experience ${pluralize(successful.length, 'file')} for: ${actionResult.url}`); - experience = `\nPast successful recipes recorded from prior runs for this page. Prefer these solutions first if they match the goal.\n\n${successful.join('\n\n')}\n`; - } - } - - const prompt = dedent` - - ${message} - - - - ${actionResult.toAiContext()} - - - ${await actionResult.combinedHtml()} - - - - - Identify the actual request of the user. - Identify what is expected by user. - Identify what might have caused the error. - Propose different solutions to achieve the result. - Solution should be valid CodeceptJS code. - Use only data from the context to plan the solution. - Try various ways to achieve the result - - - ${actionRule} - - ${unexpectedPopupRule} - - ${RulesLoader.loadRules('navigator', ['multiple-locator', 'output'], actionResult.url || '').replace('{{maxAttempts}}', String(this.MAX_ATTEMPTS))} - - ${experience} - - ${knowledge} - `; const conversation = this.provider.startConversation(this.systemPrompt, 'navigator'); - conversation.addUserText(prompt); + conversation.addUserText(await this.buildResolutionPrompt(message, actionResult)); let stopReason: string | null = null; const tools = { @@ -273,8 +254,9 @@ class Navigator implements Agent { let htmlContextAdded = false; let codeBlockIndex = 0; let totalAttempts = 0; + let lastFailure: string | null = null; const progressBlocks: string[] = []; - const batchFailures: Array<{ code: string; error: string; ariaChanges?: string | null; urlAfter?: string }> = []; + const batchFailures: BatchFailure[] = []; let resolved = false; await loop( @@ -283,7 +265,6 @@ class Navigator implements Agent { const result = await this.provider.invokeConversation(conversation, tools); if (!result) return; if (stopReason) { - tag('error').log(`Navigator stopped: ${stopReason}`); resolved = false; stop(); return; @@ -307,41 +288,8 @@ class Navigator implements Agent { return; } tag('operation').log('Feeding failures back to AI for a new batch...'); - let contextMsg = 'Previous solutions did not work. Analyze the failures and try DIFFERENT strategies (not syntactic variants of the same locator).\n\n'; - if (batchFailures.length > 0) { - const lines = batchFailures - .map((f) => { - const head = `- \`${f.code.split('\n')[0]}\` → ${f.error}`; - if (!f.ariaChanges) return head; - const trimmed = f.ariaChanges.split('\n').slice(0, 12).join('\n '); - return `${head}\n • ARIA changes after the action:\n ${trimmed}`; - }) - .join('\n'); - contextMsg += `\n${lines}\n\n\n`; - } - if (!htmlContextAdded) { - htmlContextAdded = true; - contextMsg += `Full HTML context:\n\n\n${await actionResult.combinedHtml()}\n\n\n`; - } - const pageReacted = batchFailures.some((f) => f.ariaChanges); - if (pageReacted) { - contextMsg += dedent` - Some steps in the previous batch did not throw, but the URL did not change to the expected target and the page changed in other ways — the ARIA diff for each such step is listed in above. - - Read those diffs and judge what each step actually triggered. Different action types produce different reactions; the diff is your only evidence of what happened. A diff might show, for example: a new alert / alertdialog / status / validation message appearing near a field or at page level; a modal, dialog, or wizard step opening; a banner, toast, or notification region appearing; a section expanding or collapsing; a tab or accordion switching content. A diff might also be empty or unrelated to the step — that is also a signal. - - Choose exactly ONE path based on what the diffs actually show — do not assume the previous step submitted any particular kind of data: - - A. The diff indicates the application requires something only the user can supply — for example: an authentication failure you cannot guess past, a captcha, a permission the test cannot satisfy, or knowledge that is not present in the provided context. Call the stop() tool and quote what you saw in the diff and what is needed. - - B. The diff indicates the next step is something you can perform from the existing knowledge / hint context — for example: re-emit a step with a value that exists in the knowledge but was used incorrectly; dismiss an unexpected modal; accept a confirmation; take a follow-up step the page now requires. Emit code blocks for that next step. Do NOT change the locator of a step that already produced a reaction. - - C. The diff is empty or unrelated to your step — the action likely missed its target. Propose a different locator strategy. - `; - } else { - contextMsg += 'Propose new solutions. If errors mention "intercepts pointer events" or timeouts on visible elements, an overlay is blocking — dismiss it first (Escape, click outside, Close button) before retrying the original action.'; - } - conversation.addUserText(contextMsg); + conversation.addUserText(await this.buildRetryFeedback(batchFailures, !htmlContextAdded, actionResult)); + htmlContextAdded = true; codeBlocks = []; batchFailures.length = 0; return; @@ -349,114 +297,59 @@ class Navigator implements Agent { codeBlockIndex++; totalAttempts++; - await action.exitIframe(); - const prevActionResult = action.actionResult ?? actionResult; const prevHash = prevActionResult.getStateHash(); - debugLog(`Attempting resolution: ${codeBlock}`); - const attemptOk = await action.attempt(codeBlock, message); - - const page = action.playwrightHelper?.page; - if (page) { - try { - await page.waitForLoadState('load', { timeout: 5000 }); - } catch { - // Navigation did not reach 'load' state within timeout; continue and verify URL - } - } - - if (attemptOk) opts?.onAttempt?.({ code: codeBlock }); - - if (!attemptOk) { - const raw = action.lastError?.message || 'attempt failed'; - const firstMeaningful = raw.split('\n').find((l) => l.trim() && !l.trim().startsWith('at ')) || raw; - const shortErr = firstMeaningful.replace(/\s+/g, ' ').trim().slice(0, 220); - batchFailures.push({ code: codeBlock, error: shortErr }); - opts?.onAttempt?.({ code: codeBlock, error: shortErr }); + const attempt = await this.executeAttempt(action, codeBlock, message); + opts?.onAttempt?.({ code: codeBlock, error: attempt.error }); + if (attempt.error) { + batchFailures.push({ code: codeBlock, error: attempt.error }); + lastFailure = attempt.error; } if (expectedUrl) { - if (page) { - try { - await page.waitForURL((url: URL) => matchesNavigationUrl(expectedUrl, `${url.pathname}${url.search}${url.hash}`), { timeout: 5000 }); - } catch { - // URL did not transition to expectedUrl within timeout - } - } - const freshState = await this.explorer.capture(); - const currentUrl = /^https?:\/\//i.test(expectedUrl) ? freshState.fullUrl || freshState.url || '' : freshState.url || ''; - const urlMatches = this.isSameExpectedOrigin(expectedUrl, action.stateManager) && matchesNavigationUrl(expectedUrl, currentUrl); - const stateChanged = freshState.getStateHash() !== actionResult.getStateHash(); - resolved = urlMatches && stateChanged; - - if (!resolved && attemptOk) { - let ariaChanges: string | null = null; - if (freshState.getStateHash() !== prevHash) { - try { - const diff = await freshState.diff(prevActionResult); - ariaChanges = diff.ariaChanged; - } catch (err) { - debugLog('Failed to compute pageDiff for failed URL verification:', err); - } - } + const check = await this.verifyNavigation(action, expectedUrl); + const freshHash = check.freshState.getStateHash(); + resolved = check.urlMatches && freshHash !== actionResult.getStateHash(); + + if (!resolved && attempt.ok) { + lastFailure = `URL did not change (still ${check.freshState.url})`; batchFailures.push({ code: codeBlock, - error: `URL did not change (still ${freshState.url})`, - ariaChanges, - urlAfter: freshState.url, + error: lastFailure, + ariaChanges: await this.ariaDiff(check.freshState, prevActionResult), + urlAfter: check.freshState.url, }); - tag('warning').log(`URL verification failed: expected ${expectedUrl}, got ${freshState.url}`); + tag('warning').log(`URL verification failed: expected ${expectedUrl}, got ${check.freshState.url}`); } - if (freshState.getStateHash() !== prevHash && (attemptOk || urlMatches)) { + if (freshHash !== prevHash && (attempt.ok || check.urlMatches)) { progressBlocks.push(codeBlock); } } else { - resolved = attemptOk; - if (attemptOk) progressBlocks.push(codeBlock); + resolved = attempt.ok; + if (attempt.ok) progressBlocks.push(codeBlock); } - if (resolved) { - tag('success').log('Navigation resolved successfully'); - let scenario = message.split('\n')[0]; - if (expectedUrl) { - const fromPath = extractStatePath(actionResult.url || ''); - const toPath = extractStatePath(expectedUrl); - scenario = `reach ${toPath} from ${fromPath}`; - } - const recipe = progressBlocks - .join('\n') - .split('\n') - .filter((line) => !/^\s*I\.amOnPage\s*\(/.test(line)) - .join('\n') - .trim(); - if (recipe) { - const body = `## FLOW: ${scenario}\n\n* ${scenario}\n\n\`\`\`js\n${recipe}\n\`\`\`\n\n---\n`; - this.experienceTracker.writeFlow(actionResult, body); - } - stop(); - return; - } + if (!resolved) return; + + tag('success').log('Navigation resolved successfully'); + this.saveFlow(message, expectedUrl, actionResult, progressBlocks); + stop(); }, { maxAttempts: this.MAX_ATTEMPTS * 2, observability: { agent: 'navigator', }, - catch: async (error) => { + catch: async ({ error }) => { + if (isFatalBrowserError(error)) throw error; debugLog(error); resolved = false; }, } ); - if (!resolved && expectedUrl) { - await (action.getActor() as any).wait(1); - if (this.isOnExpectedPage(expectedUrl, action.stateManager)) { - resolved = true; - tag('success').log('Navigation resolved after delayed redirect'); - } - } + if (!resolved && expectedUrl) resolved = await this.rescueDelayedRedirect(action, expectedUrl); if (!resolved && stopReason) { tag('error').log(`Navigator stopped: ${stopReason}`); @@ -464,24 +357,192 @@ class Navigator implements Agent { tag('error').log(`Navigation failed after ${totalAttempts} attempts`); } - if (!resolved && isInteractive()) { - const stopLine = stopReason ? `Navigator stopped: ${stopReason}\n` : ''; - const userInput = await pause(`${stopLine}Navigator failed to resolve. Current: ${action.stateManager.getCurrentState()?.url}\n` + `Target: ${expectedUrl ?? '(none)'}\nEnter CodeceptJS commands (or press Enter to skip):`); + if (!resolved && isInteractive()) resolved = await this.askUserToResolve(action, message, expectedUrl, stopReason); - if (userInput?.trim()) { - resolved = await action.attempt(userInput, message); - if (resolved && expectedUrl) { - await (action.getActor() as any).wait(1); - if (!this.isOnExpectedPage(expectedUrl, action.stateManager)) { - resolved = false; - } - } - } + if (!resolved) { + let cause = 'the AI proposed no working solution for this page'; + if (lastFailure) cause = `${totalAttempts} ${pluralize(totalAttempts, 'attempt')} failed, last: ${lastFailure}`; + if (stopReason) cause = stopReason; + this.lastFailureReason = this.failureReason(cause, knowledge, actionResult.url || ''); } return resolved; } + private async buildResolutionPrompt(message: string, actionResult: ActionResult): Promise { + let experience = ''; + if (!actionResult.isInsideIframe) { + const successful = this.experienceTracker.getSuccessfulExperience(actionResult); + if (successful.length > 0) { + tag('operation').log(`Found ${successful.length} experience ${pluralize(successful.length, 'file')} for: ${actionResult.url}`); + experience = `\nPast successful recipes recorded from prior runs for this page. Prefer these solutions first if they match the goal.\n\n${successful.join('\n\n')}\n`; + } + } + + return dedent` + + ${message} + + + + ${actionResult.toAiContext()} + + + ${await actionResult.combinedHtml()} + + + + + Identify the actual request of the user. + Identify what is expected by user. + Identify what might have caused the error. + Propose different solutions to achieve the result. + Solution should be valid CodeceptJS code. + Use only data from the context to plan the solution. + Try various ways to achieve the result + + + ${actionRule} + + ${unexpectedPopupRule} + + ${RulesLoader.loadRules('navigator', ['multiple-locator', 'output'], actionResult.url || '').replace('{{maxAttempts}}', String(this.MAX_ATTEMPTS))} + + ${experience} + + ${this.knowledgeTracker.renderRelevantContext(actionResult)} + `; + } + + private async buildRetryFeedback(failures: BatchFailure[], includeHtml: boolean, actionResult: ActionResult): Promise { + let contextMsg = 'Previous solutions did not work. Analyze the failures and try DIFFERENT strategies (not syntactic variants of the same locator).\n\n'; + + if (failures.length > 0) { + const lines = failures + .map((f) => { + const head = `- \`${f.code.split('\n')[0]}\` → ${f.error}`; + if (!f.ariaChanges) return head; + const trimmed = f.ariaChanges.split('\n').slice(0, 12).join('\n '); + return `${head}\n • ARIA changes after the action:\n ${trimmed}`; + }) + .join('\n'); + contextMsg += `\n${lines}\n\n\n`; + } + + if (includeHtml) { + contextMsg += `Full HTML context:\n\n\n${await actionResult.combinedHtml()}\n\n\n`; + } + + if (!failures.some((f) => f.ariaChanges)) { + return `${contextMsg}Propose new solutions. If errors mention "intercepts pointer events" or timeouts on visible elements, an overlay is blocking — dismiss it first (Escape, click outside, Close button) before retrying the original action.`; + } + + return ( + contextMsg + + dedent` + Some steps in the previous batch did not throw, but the URL did not change to the expected target and the page changed in other ways — the ARIA diff for each such step is listed in above. + + Read those diffs and judge what each step actually triggered. Different action types produce different reactions; the diff is your only evidence of what happened. A diff might show, for example: a new alert / alertdialog / status / validation message appearing near a field or at page level; a modal, dialog, or wizard step opening; a banner, toast, or notification region appearing; a section expanding or collapsing; a tab or accordion switching content. A diff might also be empty or unrelated to the step — that is also a signal. + + Choose exactly ONE path based on what the diffs actually show — do not assume the previous step submitted any particular kind of data: + + A. The diff indicates the application requires something only the user can supply — for example: an authentication failure you cannot guess past, a captcha, a permission the test cannot satisfy, or knowledge that is not present in the provided context. Call the stop() tool and quote what you saw in the diff and what is needed. + + B. The diff indicates the next step is something you can perform from the existing knowledge / hint context — for example: re-emit a step with a value that exists in the knowledge but was used incorrectly; dismiss an unexpected modal; accept a confirmation; take a follow-up step the page now requires. Emit code blocks for that next step. Do NOT change the locator of a step that already produced a reaction. + + C. The diff is empty or unrelated to your step — the action likely missed its target. Propose a different locator strategy. + ` + ); + } + + private async executeAttempt(action: Action, codeBlock: string, message: string): Promise<{ ok: boolean; error?: string }> { + await action.exitIframe(); + + debugLog(`Attempting resolution: ${codeBlock}`); + const ok = await action.attempt(codeBlock, message); + + const page = action.playwrightHelper?.page; + if (page) { + try { + await page.waitForLoadState('load', { timeout: 5000 }); + } catch { + // Navigation did not reach 'load' state within timeout; continue and verify URL + } + } + + if (ok) return { ok }; + + const raw = action.lastError?.message || 'attempt failed'; + const firstMeaningful = raw.split('\n').find((l) => l.trim() && !l.trim().startsWith('at ')) || raw; + return { ok, error: normalizeInlineText(firstMeaningful).slice(0, 220) || 'attempt failed' }; + } + + private async verifyNavigation(action: Action, expectedUrl: string): Promise<{ freshState: ActionResult; urlMatches: boolean }> { + const page = action.playwrightHelper?.page; + if (page) { + try { + await page.waitForURL((url: URL) => matchesNavigationUrl(expectedUrl, `${url.pathname}${url.search}${url.hash}`), { timeout: 5000 }); + } catch { + // URL did not transition to expectedUrl within timeout + } + } + + const freshState = await this.explorer.capture(); + const urlMatches = this.isSameExpectedOrigin(expectedUrl, action.stateManager) && matchesNavigationUrl(expectedUrl, this.comparableUrl(freshState, expectedUrl)); + + return { freshState, urlMatches }; + } + + private async ariaDiff(freshState: ActionResult, previous: ActionResult): Promise { + if (freshState.getStateHash() === previous.getStateHash()) return null; + try { + const diff = await freshState.diff(previous); + return diff.ariaChanged; + } catch (err) { + debugLog('Failed to compute pageDiff for failed URL verification:', err); + return null; + } + } + + private saveFlow(message: string, expectedUrl: string | undefined, actionResult: ActionResult, progressBlocks: string[]): void { + let scenario = message.split('\n')[0]; + if (expectedUrl) { + scenario = `reach ${extractStatePath(expectedUrl)} from ${extractStatePath(actionResult.url || '')}`; + } + + const recipe = progressBlocks + .join('\n') + .split('\n') + .filter((line) => !/^\s*I\.amOnPage\s*\(/.test(line)) + .join('\n') + .trim(); + if (!recipe) return; + + this.experienceTracker.writeFlow(actionResult, `## FLOW: ${scenario}\n\n* ${scenario}\n\n\`\`\`js\n${recipe}\n\`\`\`\n\n---\n`); + } + + private async rescueDelayedRedirect(action: Action, expectedUrl: string): Promise { + await (action.getActor() as any).wait(1); + if (!this.isOnExpectedPage(expectedUrl, action.stateManager)) return false; + tag('success').log('Navigation resolved after delayed redirect'); + return true; + } + + private async askUserToResolve(action: Action, message: string, expectedUrl: string | undefined, stopReason: string | null): Promise { + let stopLine = ''; + if (stopReason) stopLine = `Navigator stopped: ${stopReason}\n`; + + const userInput = await pause(`${stopLine}Navigator failed to resolve. Current: ${action.stateManager.getCurrentState()?.url}\nTarget: ${expectedUrl ?? '(none)'}\nEnter CodeceptJS commands (or press Enter to skip):`); + if (!userInput?.trim()) return false; + + const resolved = await action.attempt(userInput, message); + if (!resolved) return false; + if (!expectedUrl) return true; + + await (action.getActor() as any).wait(1); + return this.isOnExpectedPage(expectedUrl, action.stateManager); + } + private buildExperienceTools(): { learnExperience: unknown } | undefined { if (!this.experienceTracker) return undefined; const stateManager = this.stateManager; @@ -791,6 +852,8 @@ class Navigator implements Agent { } } +type BatchFailure = { code: string; error: string; ariaChanges?: string | null; urlAfter?: string }; + export type AssertionResult = { code: string; passed: boolean; proof: string[] }; export { Navigator }; diff --git a/src/ai/tools.ts b/src/ai/tools.ts index cb6894bd..9911c1eb 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -810,7 +810,10 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig }); } - return failedToolResult('interact', `Failed to execute: ${instruction}`, { + let reason = ''; + if (navigator.lastFailureReason) reason = `: ${navigator.lastFailureReason}`; + + return failedToolResult('interact', `Failed to execute: ${instruction}${reason}`, { ...toolResult, suggestion: 'The action could not be completed. Try a different instruction or use more specific element descriptions.', }); diff --git a/tests/unit/navigator-failure-reason.test.ts b/tests/unit/navigator-failure-reason.test.ts new file mode 100644 index 00000000..b0fff69e --- /dev/null +++ b/tests/unit/navigator-failure-reason.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'bun:test'; +import { Navigator } from '../../src/ai/navigator.ts'; + +describe('Navigator failure reason', () => { + function createNavigator(reason: string | null) { + const navigator = Object.create(Navigator.prototype) as Navigator; + (navigator as any).lastFailureReason = reason; + return navigator; + } + + it('reports what the navigator stopped on', () => { + const navigator = createNavigator('login form requires credentials, none were provided'); + const error = (navigator as any).navigationError('/defects', 'redirected to /users/sign_in and could not resolve'); + + expect(error.message).toBe('Navigation to /defects failed: login form requires credentials, none were provided'); + }); + + it('falls back when nothing reported a reason', () => { + const navigator = createNavigator(null); + const error = (navigator as any).navigationError('/defects', 'redirected to /users/sign_in and could not resolve'); + + expect(error.message).toBe('Navigation to /defects failed: redirected to /users/sign_in and could not resolve'); + }); + + it('names the page a run has nothing to go on, without its query', () => { + const navigator = createNavigator(null); + const reason = (navigator as any).failureReason(null, '', '/users/sign_in?info=You+must+be+logged+in'); + + expect(reason).toContain('no knowledge is set for /users/sign_in '); + expect(reason).toContain('learn "/users/sign_in"'); + expect(reason).not.toContain('info='); + }); + + it('keeps the stop reason, and stays quiet about knowledge it already had', () => { + const navigator = createNavigator(null); + const reason = (navigator as any).failureReason('the login form rejected the credentials in knowledge', '…', '/users/sign_in'); + + expect(reason).toBe('the login form rejected the credentials in knowledge'); + }); + + it('has nothing to add when the run knew the page and stopped for no stated reason', () => { + const navigator = createNavigator(null); + + expect((navigator as any).failureReason(null, '…', '/users/sign_in')).toBeNull(); + }); +}); diff --git a/tests/unit/navigator-resolve-state.test.ts b/tests/unit/navigator-resolve-state.test.ts new file mode 100644 index 00000000..95075d46 --- /dev/null +++ b/tests/unit/navigator-resolve-state.test.ts @@ -0,0 +1,310 @@ +import { describe, expect, it } from 'bun:test'; +import { Navigator } from '../../src/ai/navigator.ts'; + +const BASE_URL = 'http://localhost:3000'; + +function fakeActionResult(url = '/login', hash = 'start') { + return { + url, + fullUrl: `${BASE_URL}${url}`, + isInsideIframe: false, + toAiContext: () => 'page context', + combinedHtml: async () => '', + getStateHash: () => hash, + } as any; +} + +function createHarness( + options: { + responses?: string[]; + knowledge?: string; + ariaChanged?: string; + stopAt?: number; + stopReason?: string; + attempt?: (code: string, page: { url: string; hash: string }) => boolean; + onWait?: (page: { url: string; hash: string }) => void; + } = {} +) { + const page = { url: '/login', hash: 'start' }; + const sent: string[] = []; + const attempts: string[] = []; + const flows: string[] = []; + let responseIndex = 0; + + const state = () => ({ + url: page.url, + fullUrl: `${BASE_URL}${page.url}`, + getStateHash: () => page.hash, + diff: async () => ({ ariaChanged: options.ariaChanged ?? null }), + }); + + const action: any = { + lastError: null, + actionResult: null, + exitIframe: async () => {}, + attempt: async (code: string) => { + attempts.push(code); + const ok = options.attempt ? options.attempt(code, page) : true; + action.lastError = ok ? null : new Error('element not visible\n at Object.'); + return ok; + }, + stateManager: { getCurrentState: state }, + getActor: () => ({ wait: async () => options.onWait?.(page) }), + }; + + const navigator = Object.create(Navigator.prototype) as any; + navigator.MAX_ATTEMPTS = 5; + navigator.config = { playwright: { url: BASE_URL } }; + navigator.knowledgeTracker = { renderRelevantContext: () => options.knowledge ?? '' }; + navigator.experienceTracker = { + getSuccessfulExperience: () => [], + writeFlow: (_actionResult: any, body: string) => flows.push(body), + }; + navigator.provider = { + startConversation: () => ({ addUserText: (text: string) => sent.push(text) }), + invokeConversation: async (_conversation: any, tools: any) => { + const index = responseIndex++; + if (options.stopAt === index) await tools.stop.execute({ reason: options.stopReason ?? 'blocked' }); + return { response: { text: options.responses?.[index] ?? '' } }; + }, + }; + navigator.explorer = { action: () => action, capture: async () => state() }; + navigator.stateManager = action.stateManager; + + return { navigator, page, sent, attempts, flows }; +} + +describe('Navigator resolveState', () => { + it('resolves when a proposed step reaches the expected URL', async () => { + const harness = createHarness({ + responses: ["```js\nI.amOnPage('/login')\nI.click('#login-btn')\n```"], + attempt: (_code, page) => { + page.url = '/defects'; + page.hash = 'defects'; + return true; + }, + }); + + const resolved = await harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' }); + + expect(resolved).toBe(true); + expect(harness.flows[0]).toContain('## FLOW: reach /defects from /login'); + expect(harness.flows[0]).toContain("I.click('#login-btn')"); + expect(harness.flows[0]).not.toContain('amOnPage'); + }); + + it('resolves on a successful step when no URL is expected', async () => { + const harness = createHarness({ responses: ["```js\nI.click('Save')\n```"] }); + + const resolved = await harness.navigator.resolveState('click Save', fakeActionResult()); + + expect(resolved).toBe(true); + }); + + it('treats code blocks as alternatives when no URL is expected', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('a')\n```\n\n```js\nI.click('b')\n```"], + attempt: (code) => code.includes('a'), + }); + + const resolved = await harness.navigator.resolveState('click something', fakeActionResult()); + + expect(resolved).toBe(true); + expect(harness.attempts).toEqual(["I.click('a')"]); + }); + + it('spends the loop budget on the AI call and every executed step', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('a')\n```\n\n```js\nI.click('b')\n```\n\n```js\nI.click('c')\n```"], + attempt: () => false, + }); + harness.navigator.MAX_ATTEMPTS = 1; + + await harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' }); + + expect(harness.attempts).toEqual(["I.click('a')", "I.click('b')"]); + }); + + it('measures the state change against the state resolveState started from', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('Retry')\n```"], + attempt: (_code, page) => { + page.url = '/defects'; + return true; + }, + }); + + const resolved = await harness.navigator.resolveState('reach /defects', fakeActionResult('/login', 'start'), { expectedUrl: '/defects' }); + + expect(harness.sent[1]).toContain('URL did not change (still /defects)'); + expect(resolved).toBe(true); + }); + + it('reports a successful step to the attempt hook even when the URL check fails', async () => { + const reported: Array<{ code: string; error?: string }> = []; + const harness = createHarness({ + responses: ["```js\nI.click('Sign in')\n```"], + attempt: (_code, page) => { + page.hash = 'after-submit'; + return true; + }, + }); + + await harness.navigator.resolveState('reach /defects', fakeActionResult(), { + expectedUrl: '/defects', + onAttempt: (attempt: { code: string; error?: string }) => reported.push(attempt), + }); + + expect(reported).toHaveLength(1); + expect(reported[0].error).toBeUndefined(); + }); + + it('feeds a reacting page back to the AI together with its ARIA diff', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('Sign in')\n```"], + ariaChanged: '+ alert "Invalid email or password"', + attempt: (_code, page) => { + page.hash = 'after-submit'; + return true; + }, + }); + + const resolved = await harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' }); + + expect(resolved).toBe(false); + const retry = harness.sent[1]; + expect(retry).toContain(''); + expect(retry).toContain('URL did not change (still /login)'); + expect(retry).toContain('Invalid email or password'); + expect(retry).toContain('Full HTML context'); + expect(retry).toContain('Choose exactly ONE path'); + }); + + it('asks for different strategies when nothing on the page reacted', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('Sign in')\n```"], + attempt: () => false, + }); + + await harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' }); + + const retry = harness.sent[1]; + expect(retry).toContain('element not visible'); + expect(retry).toContain('intercepts pointer events'); + expect(retry).not.toContain('Choose exactly ONE path'); + }); + + it('adds the full HTML context to the retry prompt only once', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('a')\n```", "```js\nI.click('b')\n```", "```js\nI.click('c')\n```"], + attempt: () => false, + }); + + await harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' }); + + const retries = harness.sent.slice(1); + expect(retries.length).toBeGreaterThan(1); + expect(retries.filter((text) => text.includes('Full HTML context'))).toHaveLength(1); + }); + + it('stops without executing anything when the AI reports a blocker', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('Sign in')\n```"], + stopAt: 0, + stopReason: 'the sign-in form rejected every attempt and no credentials were provided', + knowledge: 'base facts', + }); + + const resolved = await harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' }); + + expect(resolved).toBe(false); + expect(harness.attempts).toEqual([]); + expect(harness.navigator.lastFailureReason).toContain('no credentials were provided'); + }); + + it('names the missing knowledge for the page when none is configured', async () => { + const harness = createHarness({ responses: [''], knowledge: '' }); + + await harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' }); + + expect(harness.navigator.lastFailureReason).toContain('no knowledge is set for /login'); + expect(harness.navigator.lastFailureReason).toContain('learn "/login"'); + }); + + it('joins every known reason', async () => { + const harness = createHarness({ + stopAt: 0, + stopReason: 'a human verification step blocks the flow', + knowledge: '', + }); + + await harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' }); + + expect(harness.navigator.lastFailureReason.startsWith('a human verification step blocks the flow; ')).toBe(true); + expect(harness.navigator.lastFailureReason).toContain('no knowledge is set for /login'); + }); + + it('explains the failure when the AI never reported a blocker', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('Sign in')\n```"], + knowledge: 'base facts', + attempt: () => false, + }); + + await harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' }); + + expect(harness.navigator.lastFailureReason).toContain('element not visible'); + }); + + it('explains the failure when the AI proposed no steps at all', async () => { + const harness = createHarness({ responses: [''], knowledge: 'base facts' }); + + await harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' }); + + expect(harness.navigator.lastFailureReason).toBeTruthy(); + }); + + it('aborts instead of burning attempts when the browser cannot be recovered', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('a')\n```", "```js\nI.click('b')\n```"], + attempt: () => { + throw new Error('Target page, context or browser has been closed'); + }, + }); + + await expect(harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' })).rejects.toThrow(/browser has been closed/); + expect(harness.attempts).toHaveLength(1); + }); + + it('keeps retrying when a step fails for an ordinary reason', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('a')\n```", "```js\nI.click('b')\n```"], + attempt: () => { + throw new Error('locator resolved to 3 elements'); + }, + }); + + const resolved = await harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' }); + + expect(resolved).toBe(false); + expect(harness.attempts.length).toBeGreaterThan(1); + }); + + it('accepts a redirect that lands after the loop gave up', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('Sign in')\n```"], + attempt: (_code, page) => { + page.hash = 'submitting'; + return true; + }, + onWait: (page) => { + page.url = '/defects'; + }, + }); + + const resolved = await harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' }); + + expect(resolved).toBe(true); + expect(harness.navigator.lastFailureReason).toBeNull(); + }); +});