Fix what the tester is told: dead vision tools, doomed clicks, and misrouted click failures

CHANGELOG.md

@@ -232,12 +232,12 @@ The per-host directory config-free runs introduced in the previous release moved
 - The API key typed into the global setup wizard is masked while typing.
 ### Configuration
 
-- **`action.timeout`** — Longest a single click, fill, or other page interaction may block before it is reported as failed. Default: `3000` (ms). Previously an interaction inherited `playwright.timeout`, so a click on a disabled or unreachable control could hold the test for the full 30 seconds Playwright allows by default.
+- **`action.timeout`** — Longest a single click, fill, or other page interaction may block before it is reported as failed. Default: `3000` (ms). Previously an interaction inherited `playwright.timeout`, so a click on a disabled or unreachable control could hold the test for the full 30 seconds Playwright allows by default. The limit covers the interaction only — `playwright.timeout` is restored for navigation, waits, and page capture as soon as the commands finish.
 
 ### Changes
 
 - Added Poolside as a supported AI provider — `poolside/laguna-xs-2.1` for the token-heavy `model` role, reached through the OpenAI-compatible endpoint at `https://inference.poolside.ai/v1` with `POOLSIDE_API_KEY`. It is also selectable config-free as `EXPLORBOT_AI_PROVIDER=poolside`. Poolside serves no `visionModel` or `agenticModel`: its models take text only, and its endpoint accepts a JSON schema without enforcing it, so pair it with another provider for those two roles.
-- [Tester] When the vision model is unavailable, `see` and `visualClick` are now withdrawn from the tools offered to the AI instead of staying available and answering every call with an error. The AI moves to ARIA snapshots and `xpathCheck` immediately rather than spending test steps on visual tools that cannot work.
+- When the vision model is unavailable, `see` and `visualClick` are now withdrawn from the tools offered to the AI instead of staying available and answering every call with an error. The AI moves to ARIA snapshots and `xpathCheck` immediately rather than spending test steps on visual tools that cannot work. The withdrawal is session-wide — Tester, Pilot, Captain, and Rerunner all stop offering them once the model has failed once.
 - [Tester] A failed click now says why it failed. Disabled controls, elements hidden behind an overlay, invisible elements, a wrong container, and a genuinely absent element are reported separately, each with the next step that fits. Previously a disabled button was described as covered by an overlay, and an element missing from a wrong container was described as missing from the page — so the AI kept re-clicking instead of fixing the real cause.
 - [Tester] ARIA locators must now be copied from the ARIA snapshot or UI map rather than guessed. When an element is absent from the snapshot, text or CSS is used instead of an invented role and name.
 - [Tester] Container locators are now used when a target may match several elements, rather than on every interaction. Every click must still offer one fallback without a container, since a stale container fails on its own.

docs/reference/configuration.md

@@ -533,6 +533,7 @@ export default {
   action: {
     delay: 1000,                   // Delay between actions (ms)
     retries: 3,                    // Retry failed actions
+    timeout: 3000,                 // Max time a single click/fill may block (ms)
   },
 
   // Regex to detect dynamic URL segments (IDs, slugs) for plan deduplication

src/action.ts

@@ -19,6 +19,8 @@ import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCo
 
 const debugLog = createDebug('explorbot:action');
 const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3;
+const DEFAULT_ACTION_TIMEOUT = 3000;
+const DEFAULT_PAGE_TIMEOUT = 3000;
 
 class Action {
   private actor: CodeceptJS.I;
@@ -301,6 +303,8 @@ class Action {
         throw new Error('No valid I.* or page.* commands found in code block');
       }
 
+      this.playwrightHelper?.page?.setDefaultTimeout(this.config.action?.timeout ?? DEFAULT_ACTION_TIMEOUT);
+
       if (isPlaywright) {
         const page = this.playwrightHelper.page;
         await playwrightSandbox(page, sanitizedCode);
@@ -312,6 +316,8 @@ class Action {
         this.lastValue = await returned;
       }
 
+      this.restorePageTimeout();
+
       if (executedSteps.length > 0) {
         codeString = executedSteps.join('\n');
       }
@@ -330,6 +336,7 @@ class Action {
       this.assertionSteps = [];
       throw err;
     } finally {
+      this.restorePageTimeout();
       detachMainDocumentResponse();
       if (groupId) await this.recorder!.endAction();
       detachStepLogger(stepListener);
@@ -379,6 +386,10 @@ class Action {
     return this.actionResult;
   }
 
+  private restorePageTimeout(): void {
+    this.playwrightHelper?.page?.setDefaultTimeout(this.config.playwright.timeout ?? DEFAULT_PAGE_TIMEOUT);
+  }
+
   private async waitForPageReadiness(page: any): Promise<void> {
     await waitForPageReadiness(page, {
       timeout: this.config.playwright.waitForTimeout,

src/ai/captain.ts

@@ -21,6 +21,7 @@ import type { Navigator } from './navigator.ts';
 import type { Provider } from './provider.ts';
 import { Researcher } from './researcher.ts';
 import { TaskAgent } from './task-agent.ts';
+import { withdrawVisionTools } from './tools.ts';
 
 const MAX_STEPS = 15;
 
@@ -437,6 +438,8 @@ export class Captain extends CaptainBase implements Agent {
           return;
         }
 
+        withdrawVisionTools(tools);
+
         const currentState = stateManager.getCurrentState();
         if (!currentState && this.getMode() !== 'idle') {
           stop();

src/ai/captain/web-mode.ts

@@ -18,7 +18,7 @@ export function WithWebMode<T extends Constructor>(Base: T) {
       });
       const { see, context, visualClick, learnExperience } = agentTools;
 
-      return {
+      const tools: Record<string, any> = {
         navigate: tool({
           description: 'Navigate to a URL or page description using AI-powered navigation.',
           inputSchema: z.object({
@@ -123,11 +123,14 @@ export function WithWebMode<T extends Constructor>(Base: T) {
         }),
 
         ...codeceptTools,
-        see,
         context,
-        visualClick,
         learnExperience,
       };
+
+      if (see) tools.see = see;
+      if (visualClick) tools.visualClick = visualClick;
+
+      return tools;
     }
 
     webModePrompt(): string {

src/ai/pilot.ts

@@ -22,6 +22,7 @@ import type { Provider } from './provider.ts';
 import type { Researcher } from './researcher.ts';
 import { capabilityGroundingRule, dataProtectionRules } from './rules.ts';
 import { isInteractive } from './task-agent.ts';
+import { withdrawVisionTools } from './tools.ts';
 
 const CHECK_TOOLS = ['verify', 'see', 'research', 'context'];
 const META_TOOLS = ['record', 'reset', 'stop', 'finish'];
@@ -669,6 +670,7 @@ export class Pilot implements Agent {
     if (xpathCheck) planning.xpathCheck = xpathCheck;
     if (learnExperience) planning.learnExperience = learnExperience;
     if (askUser) planning.askUser = askUser;
+    withdrawVisionTools(planning);
     return planning;
   }
 

src/ai/rerunner.ts

@@ -24,7 +24,7 @@ import type { Navigator } from './navigator.ts';
 import { Provider } from './provider.ts';
 import { actionRule, locatorRule, sectionContextRule } from './rules.ts';
 import { TaskAgent } from './task-agent.ts';
-import { createCodeceptJSTools } from './tools.ts';
+import { createCodeceptJSTools, withdrawVisionTools } from './tools.ts';
 
 const debugLog = createDebug('explorbot:rerunner');
 
@@ -361,6 +361,7 @@ export class Rerunner extends TaskAgent implements Agent {
           },
         }),
       };
+      withdrawVisionTools(tools);
 
       const conversation = this.provider.startConversation(this.getHealSystemPrompt(), 'rerunner');
       conversation.addUserText(this.getHealUserPrompt(failedCode, error));

src/ai/rules.ts

@@ -12,9 +12,10 @@ const locatorPriorityRule = dedent`
 
   1. ARIA locators (first choice) - target browser's accessibility tree, most reliable
      Use JSON format: { "role": "button", "text": "Login" }
-     Best for: buttons, links, inputs, form controls, dropdowns, checkboxes, radio buttons
+     Copy role and text VERBATIM from the ARIA snapshot or UI map — never guess the pair.
+     If the element is absent from the snapshot, do not invent one; use text or CSS instead.
 
-  2. Text locators (second choice) - use only when text is unique on the page
+  2. Text locators (second choice) - exact visible text, use only when unique on the page
      Example: 'Login', 'Submit', 'Username'
      Skip if the same text appears multiple times on the page
 
@@ -30,10 +31,10 @@ const locatorPriorityRule = dedent`
 
 const contextSimplificationRule = dedent`
   <context_simplification>
-  When container is available from UI map sections:
-  - Text + container is simplest and PREFERRED: I.click('Save', '.modal')
-  - ARIA + container for disambiguation: I.click({"role":"button","text":"Save"}, '.modal')
-  - ALWAYS use context parameter unless locator is XPath or unique ID
+  - Add a container when the target may match several elements, or the UI map gives a verified
+    section container: I.click('Save', '.modal')
+  - Skip the container when the locator is already unique (XPath, unique ID, unique text)
+  - A wrong or stale container guarantees failure: always add one fallback command WITHOUT a container
   - No need for complex ARIA when container narrows scope sufficiently
   </context_simplification>
 `;
@@ -63,7 +64,7 @@ const locatorStrategyRule = dedent`
 
   <good_aria_locator_example>
   { "role": "button", "text": "Login" },
-  { "role": "input", "text": "Name" },
+  { "role": "textbox", "text": "Name" },
   { "role": "link", "text": "Forgot your password?" },
   { "role": "link", "text": "Sign Up" },
   { "role": "button", "text": "Sign In" },
@@ -109,7 +110,7 @@ const locatorStrategyRule = dedent`
   </xpath_rules>
 
   <good locator example>
-    'div[role=input][placeholder="Name"]'
+    'input[placeholder="Name"]'
     '[aria-label="Name"]'
     'form#user_form input[name="name"]'
     '#content-top #user_name'
@@ -249,7 +250,8 @@ export const unexpectedPopupRule = dedent`
 
 export const sectionContextRule = dedent`
   <section_context_rule>
-  Context parameter is DEFAULT for all interactions. ALWAYS use container from UI map sections unless locator is XPath or unique ID.
+  Use a container from UI map sections when the target may match several elements. A container that is
+  wrong or stale guarantees failure, so prefer a verified one and keep a containerless fallback.
 
   1. Identify which section contains the target element
   2. Get the Context Locator from that section in the UI map
@@ -267,9 +269,10 @@ export const sectionContextRule = dedent`
   For CSS locators - prepend section context:
   - I.click('.main button.submit')  // instead of I.click('button.submit')
 
-  Only omit context when:
+  Omit context when the locator already resolves to one element:
   - Locator is XPath (already includes path context)
   - Locator is a unique ID (#specific-element)
+  - Text or ARIA that appears only once on the page
   </section_context_rule>
 
   ${unexpectedPopupRule}

src/ai/tester.ts

@@ -24,7 +24,7 @@ import { Provider } from './provider.ts';
 import { Researcher } from './researcher.ts';
 import { actionRule, capabilityGroundingRule, dataProtectionRules, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, sectionContextRule } from './rules.ts';
 import { TaskAgent } from './task-agent.ts';
-import { createCodeceptJSTools, createIframeTools } from './tools.ts';
+import { createCodeceptJSTools, createIframeTools, withdrawVisionTools } from './tools.ts';
 
 const debugLog = createDebug('explorbot:tester');
 
@@ -245,6 +245,7 @@ export class Tester extends TaskAgent implements Agent {
           if (currentState.isInsideIframe) {
             Object.assign(tools, createIframeTools(this.toolDeps));
           }
+          withdrawVisionTools(tools);
 
           debugLog(`Test ${task.scenario} iteration ${iteration}`);
 

src/ai/tools.ts

@@ -3,6 +3,7 @@ import dedent from 'dedent';
 import { z } from 'zod';
 import { ActionResult, type PageDiff, type ToolResultMetadata } from '../action-result.ts';
 import type { ExperienceTracker } from '../experience-tracker.ts';
+import { Stats } from '../stats.ts';
 import { type Task, TestResult } from '../test-plan.js';
 import { LARGE_ARIA_CHANGE_THRESHOLD } from '../utils/aria.ts';
 import { isFatalBrowserError } from '../utils/browser-errors.ts';
@@ -55,14 +56,13 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps,
         commands: z.array(z.string()).describe(dedent`
           FALLBACK LOCATORS for ONE element. All commands must click the SAME element.
           Never mix different elements — use separate click() calls instead.
+          REQUIRED: include at least one command WITHOUT a container — a wrong or stale container always fails.
           Order by reliability:
-          1. I.click(text, container) - PREFERRED when container is known - e.g. I.click("Save", ".modal")
+          1. I.click(text, container) - when the container is verified - e.g. I.click("Save", ".modal")
           2. I.click(ARIA, container) - e.g. I.click({"role":"button","text":"Save"}, ".modal")
           3. I.click(CSS, container) - e.g. I.click("#btn", ".modal")
           4. I.click(CSS) or I.click(XPath) - when locator already includes context (ID, XPath)
           5. I.clickXY(x, y) - coordinates fallback
-          IMPORTANT: Always include at least one command WITHOUT a container as fallback,
-          in case the element moved to a different section (e.g. I.click("Save") without container).
         `),
         explanation: z.string().describe('Why you are clicking this element'),
       }),
@@ -136,13 +136,7 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps,
         const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, commands[0]);
         await commitNote(activeNote, TestResult.FAILED, toolResult, action);
 
-        let suggestion = "Try xpathCheck() to find the element's actual position, see() for visual analysis, or visualClick() to click by visual appearance.";
-        const lastError = attempts[attempts.length - 1]?.error || '';
-        if (lastError.includes('was not found') || lastError.includes('not found by text')) {
-          suggestion = 'Element was not found in the DOM. Use xpathCheck() to locate it, context() to refresh snapshot, or visualClick() to click by visual appearance.';
-        } else if (lastError.includes('Timeout') || lastError.includes('intercept')) {
-          suggestion = 'Element exists but could not be clicked (possibly covered by overlay or not interactable). Try closing overlapping panels first, or use visualClick().';
-        }
+        const suggestion = clickFailureSuggestion(attempts);
 
         return failedToolResult(
           'click',
@@ -576,8 +570,6 @@ export function createLearnExperienceTool({ getExperienceTracker, getState }: {
 }
 
 export function createAgentTools({ explorer, stateManager, ai, researcher, navigator, supervisor, withExperience }: AgentToolDeps): any {
-  let visionDisabled = false;
-
   const tools: Record<string, any> = {
     see: tool({
       description: dedent`
@@ -596,7 +588,7 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
         request: z.string().describe('LLM-friendly description of the page contents to look for. 1-3 sentences. No more than 100 words.'),
       }),
       execute: async ({ request }) => {
-        if (visionDisabled) {
+        if (Stats.visionDisabled) {
           return failedToolResult('see', 'Vision tools are disabled for this session. Use context() to get fresh ARIA snapshot and analyze page state from ARIA data.');
         }
 
@@ -621,8 +613,7 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
         } catch (error) {
           throwIfFatalBrowserError(error);
           const errorMessage = errorText(error);
-          visionDisabled = true;
-          tag('warning').log('⚠️ Vision model is not available. Visual checks are disabled for this session.');
+          disableVision();
           return failedToolResult('see', `See tool failed: ${errorMessage}`, {
             suggestion: 'Vision is now disabled. Use context() to get fresh ARIA snapshot and analyze page state from ARIA data.',
           });
@@ -849,7 +840,7 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
         context: z.string().describe('What you already tried and why it failed - helps with accurate identification'),
       }),
       execute: async ({ element, context }) => {
-        if (visionDisabled) {
+        if (Stats.visionDisabled) {
           return failedToolResult('visualClick', 'Vision tools are disabled for this session. Use xpathCheck() to find the element, then click() with the discovered locator.');
         }
 
@@ -905,8 +896,7 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
         } catch (error) {
           throwIfFatalBrowserError(error);
           const errorMessage = errorText(error);
-          visionDisabled = true;
-          tag('warning').log('⚠️ Vision model is not available. Visual clicks are disabled for this session.');
+          disableVision();
           return failedToolResult('visualClick', `visualClick tool failed: ${errorMessage}`, {
             suggestion: 'Vision is now disabled. Use xpathCheck() to find the element, then click() with the discovered locator.',
           });
@@ -1056,6 +1046,12 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
     }),
   };
 
+  const disableVision = (): void => {
+    Stats.visionDisabled = true;
+    withdrawVisionTools(tools);
+    tag('warning').log('⚠️ Vision model is not available. Visual tools are disabled for this session.');
+  };
+
   if (withExperience !== false) {
     tools.learnExperience = createLearnExperienceTool({
       getExperienceTracker: () => stateManager.getExperienceTracker(),
@@ -1108,6 +1104,8 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
     });
   }
 
+  withdrawVisionTools(tools);
+
   return tools;
 }
 
@@ -1256,6 +1254,40 @@ function getMultipleElementsSuggestion(): string {
   `;
 }
 
+export function withdrawVisionTools(tools: Record<string, any>): void {
+  if (!Stats.visionDisabled) return;
+  Reflect.deleteProperty(tools, 'see');
+  Reflect.deleteProperty(tools, 'visualClick');
+}
+
+export function clickFailureSuggestion(attempts: Array<{ error?: string }>): string {
+  const errors = attempts.map((a) => a.error || '');
+
+  if (errors.some((e) => e.includes('not enabled'))) {
+    return 'Element exists but is DISABLED — clicking it again cannot work. A precondition is unmet: a required field is empty, nothing is selected, or a dialog is blocking. Satisfy it, then retry.';
+  }
+
+  if (errors.some((e) => e.includes('intercepts pointer events'))) {
+    return 'Element exists but another element covers it. Close the overlapping panel or dialog, then retry.';
+  }
+
+  if (errors.some((e) => e.includes('is not visible'))) {
+    return 'Element is in the DOM but not visible. Reveal it first — scroll to it, expand its section, or open the panel holding it.';
+  }
+
+  const notFound = errors.filter((e) => e.includes('was not found'));
+
+  if (notFound.length && notFound.every((e) => e.includes('was not found inside element'))) {
+    return 'Element was not found inside that container — the container is wrong or stale, and the element may exist elsewhere on the page. Retry the same locator WITHOUT a container, or verify the container with xpathCheck().';
+  }
+
+  if (notFound.length) {
+    return 'Element was not found in the DOM. Use xpathCheck() to locate it, context() to refresh snapshot, or visualClick() to click by visual appearance.';
+  }
+
+  return "Try xpathCheck() to find the element's actual position, see() for visual analysis, or visualClick() to click by visual appearance.";
+}
+
 const MAX_DISAMBIGUATE_ELEMENTS = 10;
 const MULTIPLE_ELEMENTS_PATTERN = 'multiple elements';