From 7c0f217515fb3ca5e6ba46217c72f807d10a70e6 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 13:33:41 +0300 Subject: [PATCH 01/27] docs: spec and plan for region-of-interest states --- .../plans/2026-08-29-region-states.md | 1292 +++++++++++++++++ .../specs/2026-08-29-region-states-design.md | 248 ++++ 2 files changed, 1540 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-29-region-states.md create mode 100644 docs/superpowers/specs/2026-08-29-region-states-design.md diff --git a/docs/superpowers/plans/2026-08-29-region-states.md b/docs/superpowers/plans/2026-08-29-region-states.md new file mode 100644 index 00000000..700e820f --- /dev/null +++ b/docs/superpowers/plans/2026-08-29-region-states.md @@ -0,0 +1,1292 @@ +# Region-of-Interest States Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Detect modals, drawers and soft-navigated subviews as first-class states by diffing HTML after each action, verify with a Playwright geometry probe whether the appeared region overlays the page, surface the region to Tester, Pilot, StateManager and experience files (`root:` frontmatter) — and **unify all overlay detection into `src/utils/overlay.ts`, deleting the old selector-heuristic path entirely**. + +**Architecture:** A structural pipeline orchestrated by `Action.capturePageState` with every decision function in `overlay.ts`: memoized parse5 diff vs previous state → `findAppearedSubRoot` (≥ 10K chars) → browser coverage probe → `classifyRegionCoverage` → `Overlay.fromSubRoot`. `Overlay` gains `type: 'drawer' | 'region'`, `root` and `present` (`detected` keeps meaning "verified overlaying"). Named regions enter the state hash (`baseHash` escape hatch for research keys); experience files carry `root:` and load only while a matching region is open. After the new path lands, the selector-based path (`extractVisibleOverlayHtml`, `OVERLAY_SELECTORS`, `captureOverlayHtml`, `overlayHtml`, Driller's private extractor) is removed — detection is ARIA + diff/geometry, nothing else. + +**Tech Stack:** Bun, TypeScript, parse5 (html-diff), Playwright `page.evaluate` (probe), gray-matter (experience frontmatter), bun:test. + +**Spec:** `docs/superpowers/specs/2026-08-29-region-states-design.md` — read it first; the plan argues from it, including the "Removed code" table Task 11 executes. + +## Global Constraints + +- Bun only — never Node.js; run tests with `bun test `. +- **Execute in the dedicated worktree branched off `main`** (created via `bunosh worktree:create`, which symlinks the main checkout's `node_modules`). Never touch the main checkout at `~/projects/explorbot` — it carries unrelated in-flight work. This plan's code quotes were taken from a tree that had small uncommitted changes to `src/ai/pilot.ts` and `src/ai/researcher/deep-analysis.ts`; the regions this plan edits exist identically on `main`, but re-read every file immediately before editing — line numbers are approximate anchors, the quoted code is the authoritative anchor, and where a quote differs slightly from what's on disk, the on-disk code wins as the base for the edit. +- Per-task commits stage **only the files named in the task** (`git add `), never `git add -A` — the dirty tree holds unrelated work. +- Code style (from CLAUDE.md): no comments unless stated; no ternary operators; no `...(cond ? {k:v} : {})` spread — plain `if`; premature exit over if/else; `?.` over `&&` chains; private methods after public; new types at end of file; `dedent` for prompts; `mdq()` for markdown (never regex/includes on markdown). +- Prompts and rules must be GENERAL — no examples from debug sessions, no site-specific selectors or class names. +- No AI calls anywhere in the detection path — detection is structural (data tier). +- Run `bun run format` after each code change, before each commit. +- Never trigger the regression CI workflow (`regression` label / `gh workflow run`) — local unit + integration tests are the feedback loop. + +--- + +### Task 1: Overlay core — types, root, `present`, `findAppearedSubRoot` + +**Files:** +- Modify: `src/utils/overlay.ts`, `src/utils/html-diff.ts` (one-line export) +- Test: `tests/unit/overlay-detection.test.ts` + +**Interfaces:** +- Consumes: `HtmlDiffPart` and `pathToXPath` from `html-diff.ts` (`pathToXPath` becomes exported); `extractHeadings` from `./html.js` (already imported in overlay.ts). +- Produces (later tasks rely on these exact names): + - `OverlayType = 'dialog' | 'modal' | 'drawer' | 'region'`; `OverlayData` gains `root?: string | null`. + - `Overlay` gains `readonly root: string | null`, `get present(): boolean`, `static fromSubRoot(subRoot: AppearedSubRoot, verdict: RegionVerdict): Overlay`, private `static nameFromHtml(html: string): string | null`. + - `findAppearedSubRoot(parts: HtmlDiffPart[]): AppearedSubRoot | null` in `overlay.ts`; `export interface AppearedSubRoot { container: string; elementXPath: string; subtree: string; size: number }` at end of `overlay.ts`. + - `RegionVerdict` is implemented in Task 2; for this task declare it in `overlay.ts`'s end-of-file types block: `export interface RegionVerdict { overlays: boolean; coverage: number }`. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/unit/overlay-detection.test.ts` (extend its imports with `findAppearedSubRoot` from `../../src/utils/overlay.ts` and `htmlDiff` from `../../src/utils/html-diff.ts`): + +```ts +describe('findAppearedSubRoot', () => { + const bigForm = Array.from({ length: 200 }, (_, i) => `
`).join(''); + const basePage = ''; + const pageWithDrawer = `

Edit User

${bigForm}
`; + + it('finds a large appeared element with container and element xpath', async () => { + const diff = await htmlDiff(basePage, pageWithDrawer); + const subRoot = findAppearedSubRoot(diff.parts); + expect(subRoot).not.toBeNull(); + expect(subRoot!.size).toBeGreaterThanOrEqual(10_000); + expect(subRoot!.container).toBe('body'); + expect(subRoot!.elementXPath).toBe('//body/div[2]'); + expect(subRoot!.subtree).toContain('Edit User'); + }); + + it('returns null when the appeared content is below the threshold', async () => { + const before = '

Users

'; + const after = '

Users

Saved successfully
'; + const diff = await htmlDiff(before, after); + expect(findAppearedSubRoot(diff.parts)).toBeNull(); + }); + + it('returns null when nothing appeared', async () => { + const diff = await htmlDiff(basePage, basePage); + expect(findAppearedSubRoot(diff.parts)).toBeNull(); + }); +}); + +describe('Overlay.fromSubRoot', () => { + const subRoot = { + container: 'aside.detail-panel', + elementXPath: '//body/div[2]', + subtree: '', + size: 12000, + }; + + it('overlaying with full coverage becomes a modal named by headings', () => { + const overlay = Overlay.fromSubRoot(subRoot, { overlays: true, coverage: 0.95 }); + expect(overlay.type).toBe('modal'); + expect(overlay.name).toBe('Edit User'); + expect(overlay.root).toBe('aside.detail-panel'); + expect(overlay.detected).toBe(true); + expect(overlay.present).toBe(true); + }); + + it('overlaying with partial coverage becomes a drawer', () => { + expect(Overlay.fromSubRoot(subRoot, { overlays: true, coverage: 0.3 }).type).toBe('drawer'); + }); + + it('inline verdict becomes a region: present but not detected', () => { + const overlay = Overlay.fromSubRoot(subRoot, { overlays: false, coverage: 0.3 }); + expect(overlay.type).toBe('region'); + expect(overlay.detected).toBe(false); + expect(overlay.present).toBe(true); + }); + + it('body container falls back to the element xpath as root', () => { + const overlay = Overlay.fromSubRoot({ ...subRoot, container: 'body' }, { overlays: true, coverage: 1 }); + expect(overlay.root).toBe('//body/div[2]'); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test tests/unit/overlay-detection.test.ts` +Expected: FAIL — `findAppearedSubRoot` / `fromSubRoot` do not exist. + +- [ ] **Step 3: Export `pathToXPath` from html-diff** + +In `src/utils/html-diff.ts` change `function pathToXPath(treePath: string): string {` to `export function pathToXPath(treePath: string): string {`. Nothing else in that file changes. + +- [ ] **Step 4: Extend Overlay and add `findAppearedSubRoot`** + +Rewrite `src/utils/overlay.ts` (keep `OVERLAY_SELECTORS`, `fromAria`, `resolve`, `fromHtml`, `captureConfig` bodies verbatim for now — they are deleted in Task 11, not here; `fromHtml` delegates to the new `nameFromHtml`): + +```ts +import { detectFocusArea } from './aria.js'; +import { type HtmlDiffPart, pathToXPath } from './html-diff.js'; +import { HTML_EXTRACTION_LIMITS, HTML_SELECTORS, HTML_VISIBILITY_LIMITS, type VisibleOverlayExtractionConfig, extractHeadings } from './html.js'; + +export const OVERLAY_SELECTORS = { /* unchanged */ } as const; + +export type OverlayType = 'dialog' | 'modal' | 'drawer' | 'region'; +export type OverlayData = { type?: OverlayType | null; name?: string | null; root?: string | null }; + +export class Overlay { + readonly type: OverlayType | null; + readonly name: string | null; + readonly root: string | null; + + constructor(data: OverlayData = {}) { + this.type = data.type ?? null; + this.name = data.name ?? null; + this.root = data.root ?? null; + } + + get detected(): boolean { + return this.type !== null && this.type !== 'region'; + } + + get present(): boolean { + return this.type !== null; + } + + static fromHtml(html: string): Overlay { + return new Overlay({ type: 'modal', name: Overlay.nameFromHtml(html) }); + } + + static fromSubRoot(subRoot: AppearedSubRoot, verdict: RegionVerdict): Overlay { + let type: OverlayType = 'region'; + if (verdict.overlays) { + type = 'drawer'; + if (verdict.coverage >= FULL_COVERAGE_RATIO) type = 'modal'; + } + let root = subRoot.container; + if (root === 'body') root = subRoot.elementXPath; + return new Overlay({ type, name: Overlay.nameFromHtml(subRoot.subtree), root }); + } + + static fromAria(snapshot: string | null): Overlay { /* unchanged */ } + static resolve(data: { overlayHtml?: string; overlay?: OverlayData | null; ariaSnapshot?: string | null }): Overlay { /* unchanged */ } + static captureConfig(): VisibleOverlayExtractionConfig { /* unchanged */ } + + private static nameFromHtml(html: string): string | null { + const headings = extractHeadings(html); + return [headings.h1, headings.h2, headings.h3, headings.h4].filter(Boolean).join(' ') || null; + } +} + +const SUBROOT_MIN_HTML = 10_000; +const FULL_COVERAGE_RATIO = 0.8; + +export function findAppearedSubRoot(parts: HtmlDiffPart[]): AppearedSubRoot | null { + let best: AppearedSubRoot | null = null; + for (const part of parts) { + const appeared = part.added.find((line) => line.startsWith('ELEMENT:')); + if (!appeared) continue; + if (part.subtree.length < SUBROOT_MIN_HTML) continue; + if (best && part.subtree.length <= best.size) continue; + best = { + container: part.container, + elementXPath: pathToXPath(appeared.slice('ELEMENT:'.length)), + subtree: part.subtree, + size: part.subtree.length, + }; + } + return best; +} + +export interface AppearedSubRoot { + container: string; + elementXPath: string; + subtree: string; + size: number; +} + +export interface RegionVerdict { + overlays: boolean; + coverage: number; +} +``` + +`/* unchanged */` markers mean: keep the existing bodies verbatim — do not retype them. Cycle check holds: overlay → html-diff → html, overlay → html, overlay → aria; nothing imports overlay from those three. + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `bun test tests/unit/overlay-detection.test.ts && bun test tests/unit/html-diff.test.ts && bun test tests/unit/aria.test.ts && bun test tests/unit/state-manager.test.ts` +Expected: PASS — `detected` semantics for `dialog`/`modal` are unchanged, and the pre-existing `extractVisibleOverlayHtml`/resolve tests still pass because that path is untouched until Task 11. If the first `findAppearedSubRoot` test's `container` assertion fails, inspect the actual value — `findStableContainer` returns `body` for top-level appended nodes because `html[1]/body[1]` is in `IGNORED_PATHS`. + +- [ ] **Step 6: Format and commit** + +```bash +bun run format +git add src/utils/overlay.ts src/utils/html-diff.ts tests/unit/overlay-detection.test.ts +git commit -m "feat: Overlay carries region types and root; detect appeared subroots from diff" +``` + +--- + +### Task 2: Coverage probe and classifier in overlay.ts + +**Files:** +- Modify: `src/utils/overlay.ts` +- Test: `tests/unit/overlay-detection.test.ts` + +**Interfaces:** +- Consumes: `RegionVerdict` (Task 1). +- Produces (Task 4 relies on): `classifyRegionCoverage(samples: RegionCoverageSamples | null): RegionVerdict`; `probeRegionCoverage(config: { xpath: string }): RegionCoverageSamples` (runs inside the browser); `getRegionCoverageProbeSource(): string`; `export interface RegionCoverageSamples { found: boolean; rect: { x: number; y: number; width: number; height: number }; viewport: { width: number; height: number }; position: string; zIndex: number; outsideHits: Array<'inside' | 'blocked' | 'page'>; siblingsInert: boolean; bodyScrollLocked: boolean }` at end of `overlay.ts`. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/unit/overlay-detection.test.ts` (import `classifyRegionCoverage`, `getRegionCoverageProbeSource` and type `RegionCoverageSamples` from `../../src/utils/overlay.ts`): + +```ts +const samplesBase = (): RegionCoverageSamples => ({ + found: true, + rect: { x: 0, y: 0, width: 1280, height: 720 }, + viewport: { width: 1280, height: 720 }, + position: 'fixed', + zIndex: 100, + outsideHits: [], + siblingsInert: false, + bodyScrollLocked: false, +}); + +describe('classifyRegionCoverage', () => { + it('full viewport coverage is overlaying', () => { + const verdict = classifyRegionCoverage(samplesBase()); + expect(verdict.overlays).toBe(true); + expect(verdict.coverage).toBeCloseTo(1); + }); + + it('partial floating region with all outside points blocked is overlaying', () => { + const samples = samplesBase(); + samples.rect = { x: 880, y: 0, width: 400, height: 720 }; + samples.outsideHits = ['blocked', 'blocked', 'blocked', 'blocked']; + const verdict = classifyRegionCoverage(samples); + expect(verdict.overlays).toBe(true); + expect(verdict.coverage).toBeLessThan(0.8); + }); + + it('inert siblings mean overlaying regardless of geometry', () => { + const samples = samplesBase(); + samples.rect = { x: 0, y: 0, width: 400, height: 400 }; + samples.siblingsInert = true; + expect(classifyRegionCoverage(samples).overlays).toBe(true); + }); + + it('static in-flow region with page hits outside is inline', () => { + const samples = samplesBase(); + samples.rect = { x: 200, y: 100, width: 800, height: 500 }; + samples.position = 'static'; + samples.zIndex = 0; + samples.outsideHits = ['page', 'page', 'page']; + expect(classifyRegionCoverage(samples).overlays).toBe(false); + }); + + it('missing element or null samples is inline with zero coverage', () => { + expect(classifyRegionCoverage(null)).toEqual({ overlays: false, coverage: 0 }); + const samples = samplesBase(); + samples.found = false; + expect(classifyRegionCoverage(samples)).toEqual({ overlays: false, coverage: 0 }); + }); +}); + +describe('getRegionCoverageProbeSource', () => { + it('serializes to a reconstructible function', () => { + const source = getRegionCoverageProbeSource(); + const fn = new Function(`return ${source}`)(); + expect(typeof fn).toBe('function'); + }); +}); +``` + +Run: `bun test tests/unit/overlay-detection.test.ts` — expected FAIL. + +- [ ] **Step 2: Implement classifier and probe** + +In `src/utils/overlay.ts`, below `findAppearedSubRoot`: + +```ts +export function classifyRegionCoverage(samples: RegionCoverageSamples | null): RegionVerdict { + if (!samples?.found) return { overlays: false, coverage: 0 }; + const viewportArea = samples.viewport.width * samples.viewport.height; + if (!viewportArea) return { overlays: false, coverage: 0 }; + + const rect = samples.rect; + const visibleWidth = Math.min(rect.x + rect.width, samples.viewport.width) - Math.max(rect.x, 0); + const visibleHeight = Math.min(rect.y + rect.height, samples.viewport.height) - Math.max(rect.y, 0); + const coverage = (Math.max(0, visibleWidth) * Math.max(0, visibleHeight)) / viewportArea; + + if (coverage >= FULL_COVERAGE_RATIO) return { overlays: true, coverage }; + if (samples.siblingsInert) return { overlays: true, coverage }; + + const floating = samples.position === 'fixed' || samples.position === 'absolute' || samples.zIndex > 0; + if (!floating) return { overlays: false, coverage }; + + const outside = samples.outsideHits; + if (outside.length > 0 && outside.every((hit) => hit !== 'page')) return { overlays: true, coverage }; + if (samples.bodyScrollLocked && outside.length > 0 && outside.filter((hit) => hit !== 'page').length * 2 >= outside.length) return { overlays: true, coverage }; + + return { overlays: false, coverage }; +} + +export function probeRegionCoverage(config: { xpath: string }): RegionCoverageSamples { + const samples: RegionCoverageSamples = { + found: false, + rect: { x: 0, y: 0, width: 0, height: 0 }, + viewport: { width: window.innerWidth, height: window.innerHeight }, + position: 'static', + zIndex: 0, + outsideHits: [], + siblingsInert: false, + bodyScrollLocked: false, + }; + + const result = document.evaluate(config.xpath, document, null, 9, null); + const node = result.singleNodeValue; + if (!node || node.nodeType !== 1) return samples; + const element = node as HTMLElement; + const rect = element.getBoundingClientRect(); + if (rect.width === 0 && rect.height === 0) return samples; + + const style = window.getComputedStyle(element); + samples.found = true; + samples.rect = { x: rect.x, y: rect.y, width: rect.width, height: rect.height }; + samples.position = style.position; + samples.zIndex = Number.parseInt(style.zIndex || '0', 10) || 0; + + const bodyStyle = window.getComputedStyle(document.body); + samples.bodyScrollLocked = bodyStyle.overflow === 'hidden' || bodyStyle.overflowY === 'hidden'; + + for (const sibling of Array.from(element.parentElement?.children || [])) { + if (sibling === element) continue; + if (!sibling.hasAttribute('inert') && sibling.getAttribute('aria-hidden') !== 'true') continue; + samples.siblingsInert = true; + break; + } + + function classifyHit(hit: Element | null): 'inside' | 'blocked' | 'page' { + if (!hit) return 'page'; + if (element.contains(hit)) return 'inside'; + let current: Element | null = hit; + for (let depth = 0; current && depth < 4; depth++) { + const hitStyle = window.getComputedStyle(current as HTMLElement); + const hitZ = Number.parseInt(hitStyle.zIndex || '0', 10) || 0; + if ((hitStyle.position === 'fixed' || hitStyle.position === 'absolute') && hitZ > 0) return 'blocked'; + current = current.parentElement; + } + return 'page'; + } + + const inset = 10; + const width = window.innerWidth; + const height = window.innerHeight; + const points: Array<[number, number]> = [ + [inset, inset], + [width - inset, inset], + [inset, height - inset], + [width - inset, height - inset], + [width / 2, inset], + [width / 2, height - inset], + [inset, height / 2], + [width - inset, height / 2], + ]; + for (const [x, y] of points) { + if (x >= rect.left && x <= rect.right && y >= rect.top && y <= rect.bottom) continue; + samples.outsideHits.push(classifyHit(document.elementFromPoint(x, y))); + } + + return samples; +} + +export function getRegionCoverageProbeSource(): string { + return probeRegionCoverage.toString(); +} +``` + +Add `RegionCoverageSamples` to the end-of-file types block. The probe runs in the browser via `new Function`, so it must stay self-contained — no imports, no outer-scope references; type annotations erase at runtime so `toString()` stays valid. `9` is `XPathResult.FIRST_ORDERED_NODE_TYPE` as a literal. + +- [ ] **Step 3: Run tests to verify they pass** + +Run: `bun test tests/unit/overlay-detection.test.ts` +Expected: PASS. + +- [ ] **Step 4: Format and commit** + +```bash +bun run format +git add src/utils/overlay.ts tests/unit/overlay-detection.test.ts +git commit -m "feat: region coverage probe and classifier in overlay module" +``` + +--- + +### Task 3: ActionResult — baseHash, region hash, diff memoization, tool-result payoff + +**Files:** +- Modify: `src/action-result.ts` +- Test: `tests/unit/action-result.test.ts`, `tests/unit/action-result-diff.test.ts` + +**Interfaces:** +- Consumes: `Overlay.present`, `Overlay.root` (Task 1). +- Produces (Tasks 4–10 rely on): `get baseHash(): string`; `getStateHash()` including `region_` for named present regions; memoized `diff(previous)` (same `previous.id` → same `Diff` instance); `public regionSubtree: string | undefined`; `PageDiff.areaOfInterest?: string`. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/unit/action-result.test.ts`: + +```ts +describe('region state hash', () => { + const html = '

Users

'; + + it('named region forks the hash; baseHash stays the page hash', () => { + const plain = new ActionResult({ url: 'https://app.example.com/users', html }); + const withRegion = new ActionResult({ + url: 'https://app.example.com/users', + html, + overlay: { type: 'drawer', name: 'Edit User', root: 'aside.panel' }, + }); + expect(withRegion.hash).not.toBe(plain.hash); + expect(withRegion.hash).toContain('region_edit_user'); + expect(withRegion.baseHash).toBe(plain.hash); + }); + + it('unnamed region does not fork the hash', () => { + const plain = new ActionResult({ url: 'https://app.example.com/users', html }); + const unnamed = new ActionResult({ url: 'https://app.example.com/users', html, overlay: { type: 'modal' } }); + expect(unnamed.hash).toBe(plain.hash); + }); +}); +``` + +Append to `tests/unit/action-result-diff.test.ts` (reuse that file's existing helpers for building states): + +```ts +describe('diff memoization and areaOfInterest', () => { + it('returns the same Diff instance for the same previous state', async () => { + const previous = new ActionResult({ id: 1, url: 'https://app.example.com/users', html: '

Users

' }); + const current = new ActionResult({ id: 2, url: 'https://app.example.com/users', html: '

Users

changed

' }); + const first = await current.diff(previous); + const second = await current.diff(previous); + expect(second).toBe(first); + }); + + it('reports the appeared region instead of a collapsed dump', async () => { + const previous = new ActionResult({ id: 1, url: 'https://app.example.com/users', html: '

Users

' }); + const current = new ActionResult({ + id: 2, + url: 'https://app.example.com/users', + html: '

Users

', + overlay: { type: 'drawer', name: 'Edit User', root: 'aside.panel' }, + }); + current.regionSubtree = ''; + const result = await current.toToolResult(previous, 'aside.panel'); + expect(result.pageDiff?.areaOfInterest).toBe('drawer "Edit User" opened, scope: aside.panel'); + expect(result.pageDiff?.htmlParts).toHaveLength(1); + expect(result.pageDiff?.htmlParts?.[0].container).toBe('aside.panel'); + expect(result.pageDiff?.htmlParts?.[0].subtree).toContain('Edit User'); + }); +}); +``` + +Run: `bun test tests/unit/action-result.test.ts tests/unit/action-result-diff.test.ts` — expected FAIL. + +- [ ] **Step 2: Implement hash changes** + +In `src/action-result.ts` replace `getStateHash()` (currently at :478) with: + +```ts + getStateHash(): string { + return this.computeStateHash(true); + } + + get baseHash(): string { + return this.computeStateHash(false); + } +``` + +and add the private method (after the public methods, near `consoleErrors`): + +```ts + private computeStateHash(includeRegion: boolean): string { + const parts: string[] = []; + + parts.push(this.relativeUrl || this.url || '/'); + + this.extractHeadings(this.html); + + if (this.h1) parts.push(`h1_${this.h1}`); + if (this.h2) parts.push(`h2_${this.h2}`); + if (includeRegion && this.overlay.present && this.overlay.name) parts.push(`region_${this.overlay.name}`); + + let stateString = slugify(parts.map((part) => part.substring(0, 100)).join('_')); + + if (stateString.length > 200) { + stateString = stateString.substring(0, 200); + if (stateString.endsWith('_')) { + stateString = stateString.slice(0, -1); + } + } + + return stateString; + } +``` + +`get hash()` already delegates to `getStateHash()` — leave it. + +- [ ] **Step 3: Implement diff memoization and regionSubtree** + +Add fields next to `overlay`: + +```ts + public regionSubtree: string | undefined = undefined; + private _diffCache: { previousId: number | undefined; diff: Diff } | null = null; +``` + +Replace `diff()` (currently `return Diff.create(this, previousState)`): + +```ts + async diff(previousState: ActionResult | null): Promise { + if (this._diffCache && this._diffCache.previousId === previousState?.id) return this._diffCache.diff; + const diff = await Diff.create(this, previousState); + this._diffCache = { previousId: previousState?.id, diff }; + return diff; + } +``` + +- [ ] **Step 4: Implement the tool-result payoff** + +Add to `PageDiff` interface: `areaOfInterest?: string;` + +In `toToolResult`, replace the block + +```ts + if (diff.htmlParts.length > 0) { + const collapsed = collapseHtmlParts(await diff.cleanedHtmlParts()); + if (collapsed.length > 0) { + pageDiff.htmlParts = collapsed; + } + } +``` + +with: + +```ts + if (this.overlay.present && !previousState.overlay.present) { + let area = `${this.overlay.type} "${this.overlay.name || 'unnamed'}" opened`; + if (this.overlay.root) area += `, scope: ${this.overlay.root}`; + pageDiff.areaOfInterest = area; + } + + if (pageDiff.areaOfInterest && this.regionSubtree && this.overlay.root) { + const htmlConfig = ConfigParser.getInstance().getConfig().html; + let subtree = await minifyHtml(htmlCombinedSnapshot(this.regionSubtree, htmlConfig?.combined)); + if (subtree.length > HTML_PART_SUBTREE_BUDGET) { + subtree = `${subtree.slice(0, HTML_PART_SUBTREE_BUDGET)}...`; + } + pageDiff.htmlParts = [{ container: this.overlay.root, subtree, added: [], removed: [] }]; + } else if (diff.htmlParts.length > 0) { + const collapsed = collapseHtmlParts(await diff.cleanedHtmlParts()); + if (collapsed.length > 0) { + pageDiff.htmlParts = collapsed; + } + } +``` + +(`minifyHtml`, `htmlCombinedSnapshot`, `ConfigParser` are already imported in this file.) + +- [ ] **Step 5: Run tests** + +Run: `bun test tests/unit/action-result.test.ts tests/unit/action-result-diff.test.ts tests/unit/action-result-memo.test.ts tests/unit/state-manager.test.ts` +Expected: PASS. If the `region_edit_user` assertion fails on slug shape, print the hash and adjust the expectation to the actual `slugify` output of `region_Edit User` — the invariant under test is fork + containment, not the separator. + +- [ ] **Step 6: Format and commit** + +```bash +bun run format +git add src/action-result.ts tests/unit/action-result.test.ts tests/unit/action-result-diff.test.ts +git commit -m "feat: region-aware state hash, baseHash, memoized diff and areaOfInterest tool results" +``` + +--- + +### Task 4: Detection pipeline in Action + +**Files:** +- Modify: `src/action.ts` + +**Interfaces:** +- Consumes: `findAppearedSubRoot`, `classifyRegionCoverage`, `Overlay.fromSubRoot`, `getRegionCoverageProbeSource`, type `RegionCoverageSamples` — all from `./utils/overlay.ts` (Tasks 1–2); `result.diff` memoization + `regionSubtree` (Task 3). +- Produces: every captured `ActionResult` may now carry a diff-detected `overlay` (`modal`/`drawer`/`region`) and `regionSubtree` before `stateManager.updateState` runs. No new exports. + +- [ ] **Step 1: Wire imports** + +In `src/action.ts` extend the existing `./utils/overlay.ts` import (currently `import { Overlay } from './utils/overlay.js';` or similar — check) to also bring `classifyRegionCoverage`, `findAppearedSubRoot`, `getRegionCoverageProbeSource` and type `RegionCoverageSamples`. + +- [ ] **Step 2: Hook detection before updateState** + +In `capturePageState` (src/action.ts:170-188), between `const result = new ActionResult({...})` and `this.stateManager.updateState(result, codeBlock)`: + +```ts + if (!frame) await this.detectRegionOfInterest(result).catch((err: Error) => debugLog('Region detection failed:', err.message)); + this.stateManager.updateState(result, codeBlock); +``` + +- [ ] **Step 3: Implement the private methods** + +After the existing private `captureOverlayHtml` (private methods stay after public ones): + +```ts + private async detectRegionOfInterest(result: ActionResult): Promise { + if (result.overlay.detected) return; + const previousState = this.stateManager.getCurrentState(); + if (!previousState) return; + const previous = ActionResult.fromState(previousState); + if (!previous.html || previous.html === result.html) return; + if (!result.isSameUrl({ url: previous.url })) return; + + const diff = await result.diff(previous); + const subRoot = findAppearedSubRoot(diff.htmlParts); + if (!subRoot) return; + + const samples = await this.probeRegion(subRoot.elementXPath); + const verdict = classifyRegionCoverage(samples); + result.overlay = Overlay.fromSubRoot(subRoot, verdict); + result.regionSubtree = subRoot.subtree; + debugLog(`Region of interest: ${result.overlay.type} "${result.overlay.name}" root=${result.overlay.root} coverage=${verdict.coverage.toFixed(2)}`); + } + + private async probeRegion(xpath: string): Promise { + return this.playwrightHelper.page + .evaluate( + ({ probeSource, config }: { probeSource: string; config: any }) => { + const probe = new Function(`return ${probeSource}`)() as (config: any) => any; + return probe(config); + }, + { probeSource: getRegionCoverageProbeSource(), config: { xpath } } + ) + .catch((err: Error) => { + debugLog('Region coverage probe failed:', err.message); + return null; + }); + } +``` + +Two guards matter and must not be dropped: `result.overlay.detected` (an ARIA-detected overlay already owns the state) and `isSameUrl` (URL changes are already full state changes with research; the diff path is only for in-place swaps). + +- [ ] **Step 4: Verify nothing regressed** + +Run: `bun test tests/unit/` +Expected: PASS (the glue has no unit test — its pure parts are covered by Tasks 1–3; end-to-end behavior is exercised by the local regression harness, which only the user runs). + +- [ ] **Step 5: Format and commit** + +```bash +bun run format +git add src/action.ts +git commit -m "feat: detect region of interest from page diff during capture" +``` + +--- + +### Task 5: StateManager records region states + +**Files:** +- Modify: `src/state-manager.ts` +- Test: `tests/unit/state-manager.test.ts` + +**Interfaces:** +- Consumes: `Overlay.present` (Task 1); region-aware `hash` (Task 3). +- Produces: transitions recorded for region open/close; `tag('data').log('state', …)` payload gains `region` when a region is present. Rename `hasDialogAppeared` → `hasRegionAppeared` (private — no external consumers). + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/unit/state-manager.test.ts` (reuse that file's existing StateManager construction): + +```ts +describe('region state transitions', () => { + const html = '

Users

'; + + it('records a transition when a named region opens and when it closes', () => { + const base = new ActionResult({ url: '/users', html }); + stateManager.updateState(base); + const historyAfterBase = stateManager.getStateHistory().length; + + const withDrawer = new ActionResult({ url: '/users', html, overlay: { type: 'drawer', name: 'Edit User', root: 'aside.panel' } }); + stateManager.updateState(withDrawer); + expect(stateManager.getStateHistory().length).toBe(historyAfterBase + 1); + + const closed = new ActionResult({ url: '/users', html }); + stateManager.updateState(closed); + expect(stateManager.getStateHistory().length).toBe(historyAfterBase + 2); + }); + + it('records a transition for an unnamed region via hasRegionAppeared', () => { + const base = new ActionResult({ url: '/users', html }); + stateManager.updateState(base); + const historyAfterBase = stateManager.getStateHistory().length; + + const unnamed = new ActionResult({ url: '/users', html, overlay: { type: 'modal' } }); + stateManager.updateState(unnamed); + expect(stateManager.getStateHistory().length).toBe(historyAfterBase + 1); + }); +}); +``` + +Run: `bun test tests/unit/state-manager.test.ts` — observe which assertions already pass (named open/close comes from the Task 3 hash fork); the tests pin the behavior either way. + +- [ ] **Step 2: Generalize the check** + +In `src/state-manager.ts`: + +```ts + const hashChanged = actionResult.hash !== previousHash; + const regionAppeared = !hashChanged && this.hasRegionAppeared(previousState, newState); + + if (hashChanged || regionAppeared) { +``` + +and rename/adjust the private method: + +```ts + private hasRegionAppeared(previousState: WebPageState | null, newState: WebPageState): boolean { + const prevFocus = previousState?.overlay ?? Overlay.fromAria(previousState?.ariaSnapshot ?? null); + const newFocus = newState.overlay ?? Overlay.fromAria(newState.ariaSnapshot ?? null); + return !prevFocus.present && newFocus.present; + } +``` + +Update the debug line inside the branch to `debugLog('State change detected: region of interest appeared');`. + +- [ ] **Step 3: Extend the remote state frame** + +In `emitStateChange`: + +```ts + const payload: Record = { url: state.fullUrl || state.url, path: state.url, title: state.title, h1: state.h1 }; + if (state.overlay?.present) payload.region = state.overlay.name || state.overlay.type; + tag('data').log('state', payload); +``` + +- [ ] **Step 4: Run tests** + +Run: `bun test tests/unit/state-manager.test.ts tests/unit/state-manager-events.test.ts` +Expected: PASS. + +- [ ] **Step 5: Format and commit** + +```bash +bun run format +git add src/state-manager.ts tests/unit/state-manager.test.ts +git commit -m "feat: record region-of-interest transitions in state manager" +``` + +--- + +### Task 6: Experience `root:` envelope + +**Files:** +- Modify: `src/experience-tracker.ts`, `src/action-result.ts`, `CLAUDE.md` +- Test: `tests/unit/experience-tracker.test.ts` + +**Interfaces:** +- Consumes: `Overlay.present` / `Overlay.root` (Task 1), region-hashed states (Task 3). +- Produces: experience frontmatter key `root` (single writer: `ExperienceTracker.ensureExperienceFile`); retrieval gate in `ActionResult.isRelevantExperienceRecord(record: WebPageState & { root?: string }, …)`. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/unit/experience-tracker.test.ts`, reusing that file's existing `beforeEach` setup (temp experience dir, tracker construction). The tests need only the `tracker` it already builds: + +```ts +describe('region experience root', () => { + const html = '

Users

'; + const regionOverlay = { type: 'drawer' as const, name: 'Edit User', root: 'aside.panel' }; + + it('writes root frontmatter for a region state', () => { + const regionState = new ActionResult({ url: '/users', html, overlay: regionOverlay }); + tracker.writeAction(regionState, { title: 'Save the edit form', code: 'I.click("Save")', explanation: '' }); + const { data } = tracker.readExperienceFile(regionState.getStateHash()); + expect(data.root).toBe('aside.panel'); + }); + + it('skips root-scoped records when no region is open, loads them when it matches', () => { + const regionState = new ActionResult({ url: '/users', html, overlay: regionOverlay }); + tracker.writeAction(regionState, { title: 'Save the edit form', code: 'I.click("Save")', explanation: '' }); + + const baseState = new ActionResult({ url: '/users', html }); + const baseContents = tracker.getRelevantExperience(baseState).map((e) => e.content); + expect(baseContents.join('\n')).not.toContain('Save the edit form'); + + const openState = new ActionResult({ url: '/users', html, overlay: regionOverlay }); + const openContents = tracker.getRelevantExperience(openState).map((e) => e.content); + expect(openContents.join('\n')).toContain('Save the edit form'); + + const otherRegion = new ActionResult({ url: '/users', html, overlay: { type: 'drawer' as const, name: 'Filters', root: 'div.filters' } }); + const otherContents = tracker.getRelevantExperience(otherRegion).map((e) => e.content); + expect(otherContents.join('\n')).not.toContain('Save the edit form'); + }); +}); +``` + +Run: `bun test tests/unit/experience-tracker.test.ts` — expected FAIL. + +- [ ] **Step 2: Implement the writer** + +In `src/experience-tracker.ts` `ensureExperienceFile` (currently :118), replace the frontmatter literal: + +```ts + if (!existsSync(filePath)) { + const frontmatter: Record = { + url: state.url ? extractStatePath(state.url) : '', + title: state.title, + }; + if (state.overlay.present && state.overlay.root) { + frontmatter.root = state.overlay.root; + } + this.writeExperienceFile(stateHash, '', frontmatter); + } +``` + +Plain `if` — no conditional spread. + +- [ ] **Step 3: Implement the retrieval gate** + +In `src/action-result.ts` `isRelevantExperienceRecord` (currently :261), widen the signature and add the gate as the first check after the null guard: + +```ts + isRelevantExperienceRecord(record: WebPageState & { root?: string }, options?: { includeDescendantExperience?: boolean }): boolean { + if (!record.url || !this.url) return false; + if (record.root) { + if (!this.overlay.present) return false; + if (this.overlay.root && this.overlay.root !== record.root) return false; + } + if (this.isMatchedBy(record)) return true; +``` + +(rest of the method unchanged). A record without `root` behaves exactly as today — envelope rule 3. The `root` gate comes first so behavior does not depend on heading coincidences between region and page states. + +- [ ] **Step 4: Document the envelope key** + +In `CLAUDE.md`, "Data Envelope Formats" table, Experience row: change the envelope cell from `sparse frontmatter` to `sparse frontmatter: url, title, optional root (region scoping selector — record loads only while a matching region is open)`. + +- [ ] **Step 5: Run tests** + +Run: `bun test tests/unit/experience-tracker.test.ts tests/unit/experience-compactor.test.ts tests/unit/historian-experience.test.ts` +Expected: PASS. + +- [ ] **Step 6: Format and commit** + +```bash +bun run format +git add src/experience-tracker.ts src/action-result.ts CLAUDE.md tests/unit/experience-tracker.test.ts +git commit -m "feat: root selector envelope key scopes experience to open regions" +``` + +--- + +### Task 7: Researcher — baseHash keys, widened overlay research + +**Files:** +- Modify: `src/ai/researcher.ts`, `src/ai/researcher/deep-analysis.ts` + +**Interfaces:** +- Consumes: `baseHash` (Task 3), `Overlay.present` (Task 1). +- Produces: research cache keyed by `baseHash` (region states share the page's research); `researchOverlay` fires for any named present region, not only dialog/modal. + +- [ ] **Step 1: Key the cache by baseHash** + +In `src/ai/researcher.ts`: + +At :78-80 replace the static helper body: + +```ts + static getCachedResearch(state: WebPageState): string { + return getCachedResearch(ActionResult.fromState(state).baseHash); + } +``` + +At :99 replace `const stateHash = state.hash || this.actionResult.getStateHash();` with: + +```ts + const stateHash = this.actionResult.baseHash; +``` + +Then run `grep -n "\.hash" src/ai/researcher.ts src/ai/researcher/*.ts` and audit each hit: cache reads/writes (`getCachedResearch`, `saveResearch`, `getPreviousResearch`, `researchPath` keys) move to `baseHash`; state-equality comparisons (e.g. `getStateHash() === getCurrentState()?.hash` at :154 and :317) stay full-hash — both sides use the same computation, so they remain consistent. + +- [ ] **Step 2: Widen researchOverlay** + +In `src/ai/researcher/deep-analysis.ts` at :93-95 replace: + +```ts + const focusArea = current.overlay; + if (!focusArea.detected || !focusArea.name) return null; + if (focusArea.type !== 'dialog' && focusArea.type !== 'modal') return null; +``` + +with: + +```ts + const focusArea = current.overlay; + if (!focusArea.present || !focusArea.name) return null; +``` + +- [ ] **Step 3: Run tests** + +Run: `bun test tests/unit/ && bun test tests/integration/` +Expected: PASS. Failures here mean a cache-key call site was converted that should not have been (or vice versa) — re-audit the grep list before changing anything else. + +- [ ] **Step 4: Format and commit** + +```bash +bun run format +git add src/ai/researcher.ts src/ai/researcher/deep-analysis.ts +git commit -m "feat: key research by base page hash and research any named region" +``` + +--- + +### Task 8: Tester context — focus scope root, area of interest + +**Files:** +- Modify: `src/ai/tester.ts` + +**Interfaces:** +- Consumes: `Overlay.present`/`root` (Task 1), `baseHash` (Task 3), widened `researchOverlay` (Task 7). +- Produces: `` carries the root selector; new `` block for inline regions injected once per state change; `pageStateHash` holds `baseHash`. + +- [ ] **Step 1: Track state-change trigger** + +In `reinjectContextIfNeeded` (src/ai/tester.ts:528), replace the tracking prologue: + +```ts + const isNewUrl = this.previousUrl !== currentUrl; + + this.previousUrl = currentUrl; + this.previousStateHash = currentStateHash; +``` + +with: + +```ts + const isNewUrl = this.previousUrl !== currentUrl; + const isNewState = !isNewUrl && this.previousStateHash !== null && this.previousStateHash !== currentStateHash; + + this.previousUrl = currentUrl; + this.previousStateHash = currentStateHash; +``` + +- [ ] **Step 2: Root selector in focus_scope** + +In the `if (focusArea.detected)` block (currently :558), add before `context +=`: + +```ts + let rootHint = ''; + if (focusArea.root) rootHint = `\nIts content lives inside \`${focusArea.root}\` — scope locators to it.`; +``` + +and change the first line of the dedent block to: + +``` + A ${focusArea.type}${areaName} is currently open above the page.${rootHint} +``` + +(the rest of the block unchanged — the strict "not actionable outside" wording stays, and stays gated on `detected`, i.e. on a probe-verified or ARIA-verified overlay). + +- [ ] **Step 3: Inline area_of_interest block** + +Immediately after the `if (focusArea.detected) { ... }` block add: + +```ts + if (!focusArea.detected && focusArea.present && isNewState) { + let rootHint = ''; + if (focusArea.root) rootHint = `\nIt lives inside \`${focusArea.root}\`.`; + context += dedent` + + A large new area "${focusArea.name || 'unnamed area'}" appeared on this page without navigation.${rootHint} + The scenario most likely continues inside this area — prefer its elements for your next actions. + The rest of the page (navigation, menus, filters) is still interactive and remains available. + + `; + } +``` + +General wording only — no element names, no site specifics. + +- [ ] **Step 4: baseHash for research keys and widened overlay-research gate** + +At :592 replace `this.pageStateHash = currentStateHash;` with: + +```ts + this.pageStateHash = currentState.baseHash; +``` + +At :630 replace the condition `if (focusArea.detected && focusArea.name && this.pageStateHash && this.pageActionResult)` with: + +```ts + if (focusArea.present && focusArea.name && this.pageStateHash && this.pageActionResult) { +``` + +- [ ] **Step 5: Run tests** + +Run: `bun test tests/unit/ && bun test tests/integration/` +Expected: PASS (prompt changes must go through the integration suite before pushing — house rule). + +- [ ] **Step 6: Format and commit** + +```bash +bun run format +git add src/ai/tester.ts +git commit -m "feat: tester context carries region root and inline area of interest" +``` + +--- + +### Task 9: Pilot state context and prompt + +**Files:** +- Modify: `src/ai/pilot.ts` +- Test: `tests/unit/pilot-state-context.test.ts` + +**Interfaces:** +- Consumes: `Overlay.present`/`root` (Task 1). +- Produces: `` shows `modal: (root: )` for verified overlays and `region: (inline, root: )` for inline regions; one general diagnostic bullet in the Pilot system prompt. + +**Note:** `src/ai/pilot.ts` and this test file carry uncommitted in-flight changes — read both fully before editing and integrate, do not revert anything. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/unit/pilot-state-context.test.ts`, following that file's existing pattern for building an `ActionResult` and reading `buildStateContext` output: + +```ts +it('shows verified overlay with its root', () => { + const state = new ActionResult({ url: '/users', html: '

Users

', overlay: { type: 'drawer', name: 'Edit User', root: 'aside.panel' } }); + const context = buildContext(state); + expect(context).toContain('modal: Edit User (root: aside.panel)'); +}); + +it('shows inline region distinctly from a modal', () => { + const state = new ActionResult({ url: '/users', html: '

Users

', overlay: { type: 'region', name: 'User Details', root: 'section.details' } }); + const context = buildContext(state); + expect(context).toContain('region: User Details (inline, root: section.details)'); + expect(context).not.toContain('modal: User Details'); +}); +``` + +(`buildContext` here stands for however the existing tests invoke `buildStateContext` — reuse their helper verbatim.) + +Run: `bun test tests/unit/pilot-state-context.test.ts` — expected FAIL. + +- [ ] **Step 2: Implement the state lines** + +In `src/ai/pilot.ts` `buildStateContext` (currently :828-834) replace: + +```ts + const focusArea = state.overlay; + if (focusArea.detected) { + lines.push(`modal: ${focusArea.name || focusArea.type}`); + } else { + lines.push('modal: none'); + } +``` + +with: + +```ts + const focusArea = state.overlay; + if (focusArea.detected) { + let line = `modal: ${focusArea.name || focusArea.type}`; + if (focusArea.root) line += ` (root: ${focusArea.root})`; + lines.push(line); + } else if (focusArea.present) { + let line = `region: ${focusArea.name || 'unnamed'} (inline`; + if (focusArea.root) line += `, root: ${focusArea.root}`; + lines.push(`${line})`); + } else { + lines.push('modal: none'); + } +``` + +- [ ] **Step 3: One general prompt bullet** + +In `getSystemPrompt`, in the "Diagnostic patterns" list, add one line: + +``` + - "region:" in → a large area appeared in place without navigation (subview, wizard step, panel). Direct Tester to act inside it; the rest of the page is still usable. +``` + +Nothing else in the prompt changes. + +- [ ] **Step 4: Run tests** + +Run: `bun test tests/unit/pilot-state-context.test.ts && bun test tests/integration/` +Expected: PASS. + +- [ ] **Step 5: Format and commit** + +```bash +bun run format +git add src/ai/pilot.ts tests/unit/pilot-state-context.test.ts +git commit -m "feat: pilot state context distinguishes overlaying modals from inline regions" +``` + +--- + +### Task 10: Driller — nested overlay context from pageDiff + +**Files:** +- Modify: `src/ai/driller.ts` +- Test: `tests/unit/driller.test.ts` (run, extend only if it covers `detectNestedOverlayContext`) + +**Interfaces:** +- Consumes: `pageDiff.htmlParts` / `pageDiff.areaOfInterest` from tool results (Task 3). +- Produces: `detectNestedOverlayContext` no longer queries the live DOM; `Driller.getVisibleOverlayHtml` is deleted along with its imports (`getVisibleOverlayHtmlExtractorSource`, `OVERLAY_SELECTORS`, and any `HTML_*` config constants imported only for it). + +- [ ] **Step 1: Replace the DOM query with the diff the result already carries** + +In `src/ai/driller.ts` `detectNestedOverlayContext` (currently :648), replace the overlay-fetch prologue: + +```ts + if (!result?.pageDiff?.ariaChanges || result.pageDiff.urlChanged) return null; + + const overlayHtml = await this.getVisibleOverlayHtml(); + if (!overlayHtml) return null; +``` + +with: + +```ts + if (!result?.pageDiff?.ariaChanges || result.pageDiff.urlChanged) return null; + + const parts = result.pageDiff.htmlParts ?? []; + let appeared = parts.filter((part: any) => part.added?.length > 0); + if (result.pageDiff.areaOfInterest) appeared = parts; + const overlayHtml = appeared.map((part: any) => part.subtree).join('\n'); + if (!overlayHtml) return null; +``` + +The rest of the method (the `` dedent block) is unchanged — `overlayHtml` keeps its name and role in the prompt. + +- [ ] **Step 2: Delete the private extractor** + +Remove the whole `private async getVisibleOverlayHtml()` method (currently :674-692). Then remove from the imports at the top of `driller.ts`: `getVisibleOverlayHtmlExtractorSource`, `OVERLAY_SELECTORS`, and each of `HTML_SELECTORS` / `HTML_EXTRACTION_LIMITS` / `HTML_VISIBILITY_LIMITS` **only if** `grep -n "" src/ai/driller.ts` shows no remaining use in this file. + +- [ ] **Step 3: Run tests** + +Run: `bun test tests/unit/driller.test.ts && bun test tests/unit/` +Expected: PASS. + +- [ ] **Step 4: Format and commit** + +```bash +bun run format +git add src/ai/driller.ts +git commit -m "refactor: driller reads nested overlays from page diff instead of DOM queries" +``` + +--- + +### Task 11: Delete the selector-based overlay path + +**Files:** +- Modify: `src/action.ts`, `src/action-result.ts`, `src/utils/overlay.ts`, `src/utils/html.ts` +- Test: `tests/unit/overlay-detection.test.ts` + +**Interfaces:** +- Consumes: everything new from Tasks 1–10 (the replacements must be in place first). +- Produces: `Overlay.resolve(data: { overlay?: OverlayData | null; ariaSnapshot?: string | null })` — narrowed signature, no `overlayHtml`. Deleted symbols (per the spec's "Removed code" table): `Action.captureOverlayHtml`, `ActionResultData.overlayHtml`, `Overlay.fromHtml`, `Overlay.captureConfig`, `OVERLAY_SELECTORS`, `extractVisibleOverlayHtml`, `getVisibleOverlayHtmlExtractorSource`, `VisibleOverlayExtractionConfig`. + +- [ ] **Step 1: Update the tests first** + +In `tests/unit/overlay-detection.test.ts`: +- Delete the `describe('extractVisibleOverlayHtml', …)` block and the `overlayConfig` helper plus the now-unused imports (`extractVisibleOverlayHtml`, `VisibleOverlayExtractionConfig`, `OVERLAY_SELECTORS`, `HTML_*` constants — keep any that other tests in the file still use). +- Rewrite the `Overlay.resolve` tests that pass `overlayHtml` (currently around :129 and :153) to assert the narrowed behavior: + +```ts +it('resolve prefers stored overlay data over aria', () => { + const overlay = Overlay.resolve({ overlay: { type: 'modal', name: 'Stored' }, ariaSnapshot: aria }); + expect(overlay.name).toBe('Stored'); +}); + +it('resolve falls back to aria detection', () => { + expect(Overlay.resolve({ ariaSnapshot: aria }).detected).toBe(true); +}); +``` + +(adapt the `aria` fixture to whatever the file already defines). + +Run: `bun test tests/unit/overlay-detection.test.ts` — expected FAIL (resolve still accepts overlayHtml, extractor still exists — the failures confirm the tests now demand the deletion). + +- [ ] **Step 2: Delete in overlay.ts** + +Remove `OVERLAY_SELECTORS`, `Overlay.fromHtml`, `Overlay.captureConfig`, and the `overlayHtml` branch of `resolve`: + +```ts + static resolve(data: { overlay?: OverlayData | null; ariaSnapshot?: string | null }): Overlay { + if (data.overlay) return new Overlay(data.overlay); + return Overlay.fromAria(data.ariaSnapshot ?? null); + } +``` + +Prune imports that only served the deleted code (`HTML_EXTRACTION_LIMITS`, `HTML_SELECTORS`, `HTML_VISIBILITY_LIMITS`, `VisibleOverlayExtractionConfig`). `nameFromHtml` stays — `fromSubRoot` uses it. + +- [ ] **Step 3: Delete in action.ts and action-result.ts** + +- `src/action.ts`: remove the `captureOverlayHtml` method; remove `let overlayHtml = '';`, `if (!frame) overlayHtml = await this.captureOverlayHtml();` and the `overlayHtml: overlayHtml || undefined,` constructor line in `capturePageState`; drop `getVisibleOverlayHtmlExtractorSource` from imports. +- `src/action-result.ts`: remove `overlayHtml?: string;` from `ActionResultData`. + +- [ ] **Step 4: Delete in html.ts** + +Remove `extractVisibleOverlayHtml`, `getVisibleOverlayHtmlExtractorSource`, and the `VisibleOverlayExtractionConfig` interface. For each limit field used only by them (`overlayHtmlLength`, `maxOverlayCount`, `minOverlayWidth`, `minOverlayHeight`, `maxViewportOverlayRatio`, `minOpacity`): run `grep -rn "" src/` and delete the field only when the extractor was its sole consumer — shared visibility limits used by other extractors stay. + +- [ ] **Step 5: Verify the path is gone** + +```bash +grep -rn "extractVisibleOverlayHtml\|getVisibleOverlayHtmlExtractorSource\|OVERLAY_SELECTORS\|captureConfig\|overlayHtml\|Overlay.fromHtml" src/ tests/ +``` + +Expected: no hits in `src/` (test-fixture prose mentioning "overlay" is fine; symbol references are not). + +- [ ] **Step 6: Run tests** + +Run: `bun test tests/unit/ && bun test tests/integration/` +Expected: PASS. + +- [ ] **Step 7: Format and commit** + +```bash +bun run format +git add src/action.ts src/action-result.ts src/utils/overlay.ts src/utils/html.ts tests/unit/overlay-detection.test.ts +git commit -m "refactor: remove selector-based overlay detection; overlay.ts is the single detection module" +``` + +--- + +### Task 12: Finalization + +**Files:** +- Modify: `CHANGELOG.md` (via the `/changelog` skill) + +- [ ] **Step 1: Full verification** + +```bash +bun run format +bun run lint:fix +bun test tests/unit/ +bun test tests/integration/ +``` + +Expected: everything green. Fix regressions before proceeding; do not skip failing tests. + +- [ ] **Step 2: Dedup pass** + +Run the code-duplication-detector agent over the changed files (house rule after major changes). Apply only findings that touch code introduced by this plan. + +- [ ] **Step 3: Changelog** + +Invoke the `/changelog` skill to add the entry for this feature, then commit: + +```bash +git add CHANGELOG.md +git commit -m "docs: changelog for region-of-interest states" +``` + +- [ ] **Step 4: Report** + +Report to the user: what was built, what was deleted (the spec's "Removed code" table), test results, and that end-to-end validation against a real app is available via the local `regression:*` bunosh commands — which only the user decides to run. Never trigger the regression CI workflow. diff --git a/docs/superpowers/specs/2026-08-29-region-states-design.md b/docs/superpowers/specs/2026-08-29-region-states-design.md new file mode 100644 index 00000000..7295dadf --- /dev/null +++ b/docs/superpowers/specs/2026-08-29-region-states-design.md @@ -0,0 +1,248 @@ +# Region-of-Interest States — Diff-Detected Modals, Drawers and Soft Navigation + +**Date:** 2026-08-29 +**Status:** Planned +**Plan:** `docs/superpowers/plans/2026-08-29-region-states.md` + +## Problem + +A state is `url + h1 + h2` (`ActionResult.getStateHash`, `src/action-result.ts:478`). The only +other state signal is `StateManager.hasDialogAppeared` (`src/state-manager.ts:209`), which fires +when the ARIA snapshot suddenly contains a dialog/modal node (`Overlay.fromAria` → +`detectFocusArea`). Everything that opens without an ARIA dialog role and without a URL change is +invisible as a state: + +- drawers and side panels rendered as plain positioned `
`s; +- soft navigation — a SPA swapping a large content region in place (wizard step, inline editor, + detail subview) with no full re-render and no URL change. + +The diff pipeline actually *sees* these. `htmlDiff` (`src/utils/html-diff.ts`) returns +`HtmlDiffPart[]` — each with a stable `container` selector and the appeared `subtree` — but +`collapseHtmlParts` (`src/action-result.ts:591`) treats any diff over 8K chars as a full page +re-render and collapses it to `...collapsed (12000 chars)...`. The one signal that says "a modal +just opened" is thrown away as noise. + +On top of that, overlay detection today is **scattered across three approaches in four files**: + +1. ARIA role detection — `detectFocusArea` in `aria.ts`, via `Overlay.fromAria`; +2. a selector-heuristic browser extractor — `extractVisibleOverlayHtml` in `html.ts`, driven by + `OVERLAY_SELECTORS` class-name patterns (`[class*="modal"]`, `[class*="drawer"]`…) and a + z-index geometry fallback, invoked from `Action.captureOverlayHtml` and independently from + `Driller.getVisibleOverlayHtml`; +3. the `overlayHtml` → `Overlay.resolve` priority chain in `ActionResult`. + +The class-name selector heuristic is exactly the kind of memorized surface form the project's +Regex-vs-AI doctrine rejects: it works only on sites that happen to name their CSS that way. + +Consequences: + +- Tester gets either the strict `` block (ARIA dialogs only) or nothing. For a + drawer without a role it keeps targeting elements behind the drawer. +- Pilot's `` says `modal: none` while half the screen is a drawer. +- Modal open/close cycling is invisible to `isInDeadLoop` — every hash in the window is the base + page. +- Experience recorded while a modal is open lands in the base page's experience file, with no + record that it only applies inside that modal. + +## Design + +### Unification: `overlay.ts` is the single detection module + +All area-of-interest semantics live in `src/utils/overlay.ts`. After this change there are +exactly **two** detection signals, both general: + +1. **ARIA** — `Overlay.fromAria` (role-based dialogs, free at capture time, needs no previous + state). `aria.ts` keeps only the ARIA-tree *primitives* (`detectFocusArea`, + `focusAreaControls`); their sole overlay-semantics consumer is `overlay.ts`. +2. **Diff + geometry** — the new pipeline below, for everything the ARIA tree does not label. + +The selector-heuristic path is **deleted entirely** (see "Removed code"). No third approach, no +class-name patterns, no priority chain. + +The pipeline, run after every action inside `Action.capturePageState`, before the (sync) +`stateManager.updateState` call — `Action` orchestrates (it is the only browser mover), every +decision function lives in `overlay.ts`: + +``` +capture html/aria + └─ same URL, not iframe, html changed, no ARIA overlay already detected + └─ diff vs previous state (parse5, memoized — shared with toToolResult) + └─ findAppearedSubRoot: appeared subtree ≥ 10K chars (overlay.ts) + └─ probeRegionCoverage in page.evaluate (overlay.ts source) + └─ classifyRegionCoverage (overlay.ts) + ├─ overlays the page → Overlay 'modal' | 'drawer' + └─ inline → Overlay 'region' +``` + +Detection is 100% structural — size threshold, diff paths, geometry. No AI in the path. AI enters +only downstream: `researchOverlay` describes the region, Tester/Pilot decide what to do in it. + +### 1. Appeared-subroot detection (`overlay.ts`, over `html-diff.ts` parts) + +`findAppearedSubRoot(parts: HtmlDiffPart[])` returns the largest part that contains an appeared +element (`ELEMENT:` line in `part.added`) and whose minified `subtree` is ≥ `SUBROOT_MIN_HTML` +(10 000 chars, unexported const — no config knob). `html-diff.ts` stays a generic diff engine; it +newly exports `pathToXPath` so overlay.ts can convert appeared-element paths. + +The part's `container` is by design an ancestor that exists in **both** snapshots +(`findStableContainer`) — it is never the appeared element, and for portal roots +(`#modal-root`-style, zero-height with fixed children) its geometry lies. So the result carries +both: + +- `container` — the stable scoping selector, handed to Tester and stored as the experience `root`; +- `elementXPath` — the appeared element itself (from the `ELEMENT:html[1]/body[1]/div[3]` path + via `pathToXPath`) — this is what the coverage probe measures. + +When `container` degrades to `body` (top-level appended node — the common portal case), the +`elementXPath` doubles as the root selector. + +### 2. Coverage verification (`overlay.ts`) + +Split into dumb browser-side collection and a pure classifier, because jsdom has no layout and +only the pure half can be unit-tested. Both halves live in `overlay.ts`: + +- **Browser probe** (`probeRegionCoverage`, shipped as a source string like the existing + extractor pattern): resolves the element by XPath, collects raw samples — bounding rect, + viewport size, computed position and z-index, `elementFromPoint` hits at sample points + **outside** the region's rect (classified as `inside` / `blocked` / `page`), sibling + `inert`/`aria-hidden` flags, body scroll lock. +- **Pure classifier** (`classifyRegionCoverage(samples)`): returns `{ overlays, coverage }`. + Overlaying = coverage ≥ 0.8, or siblings inerted, or a floating element whose outside sample + points are all blocked by a scrim rather than landing on page content. + +Probe failure (page navigating away, evaluate throws) degrades to `overlays: false` with a debug +log — a false "inline" verdict costs a softer prompt; a false "overlaying" verdict would make +Tester refuse legitimate navigation. + +### 3. Overlay carries the region (`overlay.ts`) + +`Overlay` is extended rather than a parallel concept added: + +- `type`: `'dialog' | 'modal' | 'drawer' | 'region' | null` — `region` means inline subview; +- `name`: heading-derived (h1–h4 join over the region HTML, private `nameFromHtml`); +- `root`: the scoping selector (container CSS, or element XPath when the container degraded to + `body`); +- `get detected()` — **keeps meaning "verified overlaying"** (`type` is dialog/modal/drawer). + Every existing consumer of `detected` (Tester ``, Pilot `modal:` line, + `hasDialogAppeared`) keeps its semantics. +- `get present()` — any region, inline included. New consumers that want "an area of interest + exists" use this. +- `Overlay.fromSubRoot(subRoot, verdict)` — verdict `overlays: true` with coverage ≥ 0.8 → + `modal`; overlaying with partial coverage → `drawer`; otherwise `region`. +- `Overlay.resolve` simplifies to two sources: stored `overlay` data, else `fromAria`. + +### 4. State identity (`src/action-result.ts`) + +- `getStateHash()` gains a `region_` part when `overlay.present && overlay.name`. + **Named regions only**: names come from headings (stable across runs); selectors with dynamic + classes never enter a hash. An unnamed region does not fork the state — which is why + `hasDialogAppeared` survives (generalized to `hasRegionAppeared` over `present`) as the + transition trigger for unnamed overlays. +- `baseHash` getter — the hash without the region part. The research cache and Tester's + `pageStateHash` key off `baseHash`, otherwise a modal open at capture time forks + `getCachedResearch` and poisons `researchOverlay`'s append-to-page-research flow. +- `diff(previous)` is memoized on `previous.id` so capture-time detection and `toToolResult` + share one parse5 pass. +- Side effect, intended: with the region in the hash, modal **close** also changes the hash — a + test cycling open/close now produces alternating hashes that `isInDeadLoop` can see. + +### 5. StateManager records region states (`src/state-manager.ts`) + +Named regions change the hash, so `updateState` records the transition through the existing +hash-changed path — region states land in `stateHistory`, `getRecentTransitions`, visit counts, +and the `tag('data').log('state', …)` remote frame (which gains a `region` field). Unnamed +regions go through `hasRegionAppeared` (the renamed, `present`-based `hasDialogAppeared`). + +### 6. Experience envelope: `root:` (`src/experience-tracker.ts`) + +Experience files for region states get a new frontmatter key: + +```markdown +--- +url: /users +title: Users — Admin +root: 'aside.detail-panel' +--- +``` + +Envelope checklist (per CLAUDE.md "Data Envelope Formats"): + +1. **Read deterministically by code** — retrieval gating below; never interpreted by the model. +2. **Scoped to URL/state** — per `.md` file; region states have their own hash, so + their file is created while the region is open and `root` comes from `state.overlay.root`. +3. **Optional with a default** — absent means "whole page"; every existing file on disk behaves + exactly as today. +4. **Single writer** — `ExperienceTracker.ensureExperienceFile` only. + +**Retrieval rule** (in `ActionResult.isRelevantExperienceRecord`, where matching already lives): +a record carrying `root` is loaded only when the current state has a region open — +`overlay.present` — and, when the current region's own `root` is known, the selectors match +exactly. Found by this state + root selector exists → the experience file is loaded; no region +open → the file is skipped, so drawer recipes stop polluting base-page context. Matching stays +structural (string equality), never semantic. + +### 7. Surfacing to the agents + +- **Tool results** (`toToolResult`): when the region appeared in this transition, `pageDiff` + gains `areaOfInterest` — e.g. `drawer "Edit User" opened, scope: aside.detail-panel` — and + `htmlParts` is replaced by a single part containing the region's cleaned snapshot within the + existing per-part budget, instead of the `...collapsed (12000 chars)...` marker. This is the + payoff: the diff signal that was discarded becomes the headline of the acting tool's result. +- **Tester** (`reinjectContextIfNeeded`): verified overlays keep the strict `` + block, now with the concrete root selector. Inline regions get a new, softer + `` block — injected once per state change (via the previously write-only + `previousStateHash`) — that names the region and its root but leaves page navigation + actionable. The strict "elements outside are not actionable" wording stays gated on the probe + verdict. The `researchOverlay` trigger widens from `detected` to `present`. +- **Pilot** (`buildStateContext`): the `modal:` line stays for verified overlays (its diagnostic + prompt patterns keep working) and gains the root; inline regions get a new + `region: (inline, root: )` line plus one general system-prompt bullet. +- **Researcher** (`deep-analysis.ts` `researchOverlay`): the guard widens from + `type === 'dialog' | 'modal'` to any named present region, so drawers and subviews get the same + incremental Extended Research treatment, still appended under the base page's research (keyed + by `baseHash`). +- **Driller** (`detectNestedOverlayContext`): stops re-querying the live DOM through the selector + extractor. The nested-overlay context is built from what the tool result already carries — the + appeared `pageDiff.htmlParts` subtrees (plus the region part when `areaOfInterest` is set). + What changed after the click *is* the nested UI; no second detection approach needed. + +## Removed code + +Unification means the selector-heuristic path is deleted, not deprecated: + +| Removed | Was | +|---|---| +| `Action.captureOverlayHtml` + `overlayHtml` capture in `capturePageState` | Selector-extractor invocation per capture | +| `ActionResultData.overlayHtml` + `Overlay.resolve`'s overlayHtml branch + `Overlay.fromHtml` (public) | Priority chain feeding heading-named modals | +| `OVERLAY_SELECTORS`, `Overlay.captureConfig` (`overlay.ts`) | Class-name patterns (`[class*="modal"]`…) | +| `extractVisibleOverlayHtml`, `getVisibleOverlayHtmlExtractorSource`, `VisibleOverlayExtractionConfig` (`html.ts`), plus limit fields used only by them | The selector/z-index browser extractor | +| `Driller.getVisibleOverlayHtml` | Driller's private extractor invocation | +| `extractVisibleOverlayHtml` describe-block and `overlayHtml` resolve tests | Tests of the removed path | + +**Accepted trade-off:** an overlay that is *already open at the very first capture* and carries +no ARIA dialog role is no longer detected (there is no previous state to diff). The moment any +action happens, the diff path sees it. This trades a narrow first-paint case for removing a +site-shape heuristic that violates the core "no memorized surface forms" principle. + +## Decisions + +| Decision | Choice | Why | +|---|---|---| +| Single detection home | `overlay.ts` owns every decision function; `aria.ts` keeps ARIA parsing primitives; `Action` only orchestrates | One place to reason about overlays; browser access stays in the Action tier | +| Old selector path | Deleted, including Driller's use (rebuilt on `pageDiff`) | User decision: unify, old code gone; class-name selectors are memorized surface forms | +| What the probe measures | The appeared element (`elementXPath`), never the diff `container` | Container is a both-sides ancestor; portal roots have lying geometry | +| `detected` semantics | Unchanged: verified overlaying only; new `present` for any region | A false overlay claim makes Tester refuse legitimate navigation — worse than no detection | +| Hash contribution | Named regions only; `baseHash` escape hatch for research keys | Heading names are stable; selectors are not; research must stay keyed to the page | +| Threshold | `SUBROOT_MIN_HTML = 10_000` on the minified subtree, unexported const | Single named constant; no config knob until someone needs one | +| `root` retrieval gating | Sync string equality against `overlay.root`, require `overlay.present` | Deterministic, no DOM query in the sync retrieval path | + +## Non-goals / follow-ups + +- **DOM-presence gating for experience `root`** (querySelector against stored HTML when the + current region is detected by ARIA and has no `root`). Needs an async retrieval path; revisit + if the equality rule proves too strict. +- **Region-scoped ARIA slices** for the Tester context. v1 hands the root selector and the + region snapshot via the tool result; slicing the ARIA tree to the region is a later refinement. +- **First-paint overlay detection without ARIA roles.** If the accepted trade-off above bites in + practice, the general fix is a geometry-only probe at first capture (top-most covering element), + not the return of class-name selectors. From aa62629743cebe73fb6507248d3ef0cfcdac69d5 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 13:37:24 +0300 Subject: [PATCH 02/27] feat: Overlay carries region types and root; detect appeared subroots from diff --- .../plans/2026-08-29-region-states.md | 12 ++-- src/utils/html-diff.ts | 2 +- src/utils/overlay.ts | 64 ++++++++++++++++-- tests/unit/overlay-detection.test.ts | 65 ++++++++++++++++++- 4 files changed, 130 insertions(+), 13 deletions(-) diff --git a/docs/superpowers/plans/2026-08-29-region-states.md b/docs/superpowers/plans/2026-08-29-region-states.md index 700e820f..b29a50a8 100644 --- a/docs/superpowers/plans/2026-08-29-region-states.md +++ b/docs/superpowers/plans/2026-08-29-region-states.md @@ -37,7 +37,7 @@ - `findAppearedSubRoot(parts: HtmlDiffPart[]): AppearedSubRoot | null` in `overlay.ts`; `export interface AppearedSubRoot { container: string; elementXPath: string; subtree: string; size: number }` at end of `overlay.ts`. - `RegionVerdict` is implemented in Task 2; for this task declare it in `overlay.ts`'s end-of-file types block: `export interface RegionVerdict { overlays: boolean; coverage: number }`. -- [ ] **Step 1: Write the failing tests** +- [x] **Step 1: Write the failing tests** Append to `tests/unit/overlay-detection.test.ts` (extend its imports with `findAppearedSubRoot` from `../../src/utils/overlay.ts` and `htmlDiff` from `../../src/utils/html-diff.ts`): @@ -105,16 +105,16 @@ describe('Overlay.fromSubRoot', () => { }); ``` -- [ ] **Step 2: Run tests to verify they fail** +- [x] **Step 2: Run tests to verify they fail** Run: `bun test tests/unit/overlay-detection.test.ts` Expected: FAIL — `findAppearedSubRoot` / `fromSubRoot` do not exist. -- [ ] **Step 3: Export `pathToXPath` from html-diff** +- [x] **Step 3: Export `pathToXPath` from html-diff** In `src/utils/html-diff.ts` change `function pathToXPath(treePath: string): string {` to `export function pathToXPath(treePath: string): string {`. Nothing else in that file changes. -- [ ] **Step 4: Extend Overlay and add `findAppearedSubRoot`** +- [x] **Step 4: Extend Overlay and add `findAppearedSubRoot`** Rewrite `src/utils/overlay.ts` (keep `OVERLAY_SELECTORS`, `fromAria`, `resolve`, `fromHtml`, `captureConfig` bodies verbatim for now — they are deleted in Task 11, not here; `fromHtml` delegates to the new `nameFromHtml`): @@ -207,12 +207,12 @@ export interface RegionVerdict { `/* unchanged */` markers mean: keep the existing bodies verbatim — do not retype them. Cycle check holds: overlay → html-diff → html, overlay → html, overlay → aria; nothing imports overlay from those three. -- [ ] **Step 5: Run tests to verify they pass** +- [x] **Step 5: Run tests to verify they pass** Run: `bun test tests/unit/overlay-detection.test.ts && bun test tests/unit/html-diff.test.ts && bun test tests/unit/aria.test.ts && bun test tests/unit/state-manager.test.ts` Expected: PASS — `detected` semantics for `dialog`/`modal` are unchanged, and the pre-existing `extractVisibleOverlayHtml`/resolve tests still pass because that path is untouched until Task 11. If the first `findAppearedSubRoot` test's `container` assertion fails, inspect the actual value — `findStableContainer` returns `body` for top-level appended nodes because `html[1]/body[1]` is in `IGNORED_PATHS`. -- [ ] **Step 6: Format and commit** +- [x] **Step 6: Format and commit** ```bash bun run format diff --git a/src/utils/html-diff.ts b/src/utils/html-diff.ts index 1771dbe2..25a4f06c 100644 --- a/src/utils/html-diff.ts +++ b/src/utils/html-diff.ts @@ -492,7 +492,7 @@ function buildContainerSelector(element: ElementNode, allElements: NodeMap): str return matchCount === 1 ? selector : null; } -function pathToXPath(treePath: string): string { +export function pathToXPath(treePath: string): string { const parts = treePath.split('/'); const bodyIdx = parts.findIndex((p) => p.startsWith('body')); if (bodyIdx === -1) return `//${parts.join('/')}`; diff --git a/src/utils/overlay.ts b/src/utils/overlay.ts index 8983bef0..841153c8 100644 --- a/src/utils/overlay.ts +++ b/src/utils/overlay.ts @@ -1,4 +1,5 @@ import { detectFocusArea } from './aria.js'; +import { type HtmlDiffPart, pathToXPath } from './html-diff.js'; import { HTML_EXTRACTION_LIMITS, HTML_SELECTORS, HTML_VISIBILITY_LIMITS, type VisibleOverlayExtractionConfig, extractHeadings } from './html.js'; export const OVERLAY_SELECTORS = { @@ -7,25 +8,41 @@ export const OVERLAY_SELECTORS = { overlaySemanticSelector: '[role="dialog"], [role="alertdialog"], [aria-modal="true"], [role="listbox"], [role="menu"], [role="tooltip"]', } as const; -export type OverlayData = { type?: 'dialog' | 'modal' | null; name?: string | null }; +export type OverlayType = 'dialog' | 'modal' | 'drawer' | 'region'; +export type OverlayData = { type?: OverlayType | null; name?: string | null; root?: string | null }; export class Overlay { - readonly type: 'dialog' | 'modal' | null; + readonly type: OverlayType | null; readonly name: string | null; + readonly root: string | null; constructor(data: OverlayData = {}) { this.type = data.type ?? null; this.name = data.name ?? null; + this.root = data.root ?? null; } get detected(): boolean { + return this.type !== null && this.type !== 'region'; + } + + get present(): boolean { return this.type !== null; } static fromHtml(html: string): Overlay { - const headings = extractHeadings(html); - const name = [headings.h1, headings.h2, headings.h3, headings.h4].filter(Boolean).join(' '); - return new Overlay({ type: 'modal', name: name || null }); + return new Overlay({ type: 'modal', name: Overlay.nameFromHtml(html) }); + } + + static fromSubRoot(subRoot: AppearedSubRoot, verdict: RegionVerdict): Overlay { + let type: OverlayType = 'region'; + if (verdict.overlays) { + type = 'drawer'; + if (verdict.coverage >= FULL_COVERAGE_RATIO) type = 'modal'; + } + let root = subRoot.container; + if (root === 'body') root = subRoot.elementXPath; + return new Overlay({ type, name: Overlay.nameFromHtml(subRoot.subtree), root }); } static fromAria(snapshot: string | null): Overlay { @@ -48,4 +65,41 @@ export class Overlay { geometryFallback: false, }; } + + private static nameFromHtml(html: string): string | null { + const headings = extractHeadings(html); + return [headings.h1, headings.h2, headings.h3, headings.h4].filter(Boolean).join(' ') || null; + } +} + +const SUBROOT_MIN_HTML = 10_000; +const FULL_COVERAGE_RATIO = 0.8; + +export function findAppearedSubRoot(parts: HtmlDiffPart[]): AppearedSubRoot | null { + let best: AppearedSubRoot | null = null; + for (const part of parts) { + const appeared = part.added.find((line) => line.startsWith('ELEMENT:')); + if (!appeared) continue; + if (part.subtree.length < SUBROOT_MIN_HTML) continue; + if (best && part.subtree.length <= best.size) continue; + best = { + container: part.container, + elementXPath: pathToXPath(appeared.slice('ELEMENT:'.length)), + subtree: part.subtree, + size: part.subtree.length, + }; + } + return best; +} + +export interface AppearedSubRoot { + container: string; + elementXPath: string; + subtree: string; + size: number; +} + +export interface RegionVerdict { + overlays: boolean; + coverage: number; } diff --git a/tests/unit/overlay-detection.test.ts b/tests/unit/overlay-detection.test.ts index 700861c7..5bce8fcd 100644 --- a/tests/unit/overlay-detection.test.ts +++ b/tests/unit/overlay-detection.test.ts @@ -2,8 +2,9 @@ import 'parse5'; import { JSDOM } from 'jsdom'; import { describe, expect, it } from 'vitest'; import { ActionResult } from '../../src/action-result.ts'; +import { htmlDiff } from '../../src/utils/html-diff.ts'; import { HTML_EXTRACTION_LIMITS, HTML_SELECTORS, HTML_VISIBILITY_LIMITS, type VisibleOverlayExtractionConfig, extractVisibleOverlayHtml } from '../../src/utils/html.ts'; -import { OVERLAY_SELECTORS, Overlay } from '../../src/utils/overlay.ts'; +import { OVERLAY_SELECTORS, Overlay, findAppearedSubRoot } from '../../src/utils/overlay.ts'; function overlayConfig(overrides: Partial = {}): VisibleOverlayExtractionConfig { return { @@ -160,3 +161,65 @@ describe('Overlay', () => { expect(new Overlay().detected).toBe(false); }); }); + +describe('findAppearedSubRoot', () => { + const bigForm = Array.from({ length: 200 }, (_, i) => `
`).join(''); + const basePage = ''; + const pageWithDrawer = `

Edit User

${bigForm}
`; + + it('finds a large appeared element with container and element xpath', async () => { + const diff = await htmlDiff(basePage, pageWithDrawer); + const subRoot = findAppearedSubRoot(diff.parts); + expect(subRoot).not.toBeNull(); + expect(subRoot!.size).toBeGreaterThanOrEqual(10_000); + expect(subRoot!.container).toBe('body'); + expect(subRoot!.elementXPath).toBe('//body/div[2]'); + expect(subRoot!.subtree).toContain('Edit User'); + }); + + it('returns null when the appeared content is below the threshold', async () => { + const before = '

Users

'; + const after = '

Users

Saved successfully
'; + const diff = await htmlDiff(before, after); + expect(findAppearedSubRoot(diff.parts)).toBeNull(); + }); + + it('returns null when nothing appeared', async () => { + const diff = await htmlDiff(basePage, basePage); + expect(findAppearedSubRoot(diff.parts)).toBeNull(); + }); +}); + +describe('Overlay.fromSubRoot', () => { + const subRoot = { + container: 'aside.detail-panel', + elementXPath: '//body/div[2]', + subtree: '', + size: 12000, + }; + + it('overlaying with full coverage becomes a modal named by headings', () => { + const overlay = Overlay.fromSubRoot(subRoot, { overlays: true, coverage: 0.95 }); + expect(overlay.type).toBe('modal'); + expect(overlay.name).toBe('Edit User'); + expect(overlay.root).toBe('aside.detail-panel'); + expect(overlay.detected).toBe(true); + expect(overlay.present).toBe(true); + }); + + it('overlaying with partial coverage becomes a drawer', () => { + expect(Overlay.fromSubRoot(subRoot, { overlays: true, coverage: 0.3 }).type).toBe('drawer'); + }); + + it('inline verdict becomes a region: present but not detected', () => { + const overlay = Overlay.fromSubRoot(subRoot, { overlays: false, coverage: 0.3 }); + expect(overlay.type).toBe('region'); + expect(overlay.detected).toBe(false); + expect(overlay.present).toBe(true); + }); + + it('body container falls back to the element xpath as root', () => { + const overlay = Overlay.fromSubRoot({ ...subRoot, container: 'body' }, { overlays: true, coverage: 1 }); + expect(overlay.root).toBe('//body/div[2]'); + }); +}); From d5ad0821562a2ae01eaea5989c552913936120ae Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 13:38:19 +0300 Subject: [PATCH 03/27] feat: region coverage probe and classifier in overlay module --- .../plans/2026-08-29-region-states.md | 8 +- src/utils/overlay.ts | 107 ++++++++++++++++++ tests/unit/overlay-detection.test.ts | 61 +++++++++- 3 files changed, 171 insertions(+), 5 deletions(-) diff --git a/docs/superpowers/plans/2026-08-29-region-states.md b/docs/superpowers/plans/2026-08-29-region-states.md index b29a50a8..15714275 100644 --- a/docs/superpowers/plans/2026-08-29-region-states.md +++ b/docs/superpowers/plans/2026-08-29-region-states.md @@ -232,7 +232,7 @@ git commit -m "feat: Overlay carries region types and root; detect appeared subr - Consumes: `RegionVerdict` (Task 1). - Produces (Task 4 relies on): `classifyRegionCoverage(samples: RegionCoverageSamples | null): RegionVerdict`; `probeRegionCoverage(config: { xpath: string }): RegionCoverageSamples` (runs inside the browser); `getRegionCoverageProbeSource(): string`; `export interface RegionCoverageSamples { found: boolean; rect: { x: number; y: number; width: number; height: number }; viewport: { width: number; height: number }; position: string; zIndex: number; outsideHits: Array<'inside' | 'blocked' | 'page'>; siblingsInert: boolean; bodyScrollLocked: boolean }` at end of `overlay.ts`. -- [ ] **Step 1: Write the failing tests** +- [x] **Step 1: Write the failing tests** Append to `tests/unit/overlay-detection.test.ts` (import `classifyRegionCoverage`, `getRegionCoverageProbeSource` and type `RegionCoverageSamples` from `../../src/utils/overlay.ts`): @@ -299,7 +299,7 @@ describe('getRegionCoverageProbeSource', () => { Run: `bun test tests/unit/overlay-detection.test.ts` — expected FAIL. -- [ ] **Step 2: Implement classifier and probe** +- [x] **Step 2: Implement classifier and probe** In `src/utils/overlay.ts`, below `findAppearedSubRoot`: @@ -403,12 +403,12 @@ export function getRegionCoverageProbeSource(): string { Add `RegionCoverageSamples` to the end-of-file types block. The probe runs in the browser via `new Function`, so it must stay self-contained — no imports, no outer-scope references; type annotations erase at runtime so `toString()` stays valid. `9` is `XPathResult.FIRST_ORDERED_NODE_TYPE` as a literal. -- [ ] **Step 3: Run tests to verify they pass** +- [x] **Step 3: Run tests to verify they pass** Run: `bun test tests/unit/overlay-detection.test.ts` Expected: PASS. -- [ ] **Step 4: Format and commit** +- [x] **Step 4: Format and commit** ```bash bun run format diff --git a/src/utils/overlay.ts b/src/utils/overlay.ts index 841153c8..df95ec5e 100644 --- a/src/utils/overlay.ts +++ b/src/utils/overlay.ts @@ -92,6 +92,102 @@ export function findAppearedSubRoot(parts: HtmlDiffPart[]): AppearedSubRoot | nu return best; } +export function classifyRegionCoverage(samples: RegionCoverageSamples | null): RegionVerdict { + if (!samples?.found) return { overlays: false, coverage: 0 }; + const viewportArea = samples.viewport.width * samples.viewport.height; + if (!viewportArea) return { overlays: false, coverage: 0 }; + + const rect = samples.rect; + const visibleWidth = Math.min(rect.x + rect.width, samples.viewport.width) - Math.max(rect.x, 0); + const visibleHeight = Math.min(rect.y + rect.height, samples.viewport.height) - Math.max(rect.y, 0); + const coverage = (Math.max(0, visibleWidth) * Math.max(0, visibleHeight)) / viewportArea; + + if (coverage >= FULL_COVERAGE_RATIO) return { overlays: true, coverage }; + if (samples.siblingsInert) return { overlays: true, coverage }; + + const floating = samples.position === 'fixed' || samples.position === 'absolute' || samples.zIndex > 0; + if (!floating) return { overlays: false, coverage }; + + const outside = samples.outsideHits; + if (outside.length > 0 && outside.every((hit) => hit !== 'page')) return { overlays: true, coverage }; + if (samples.bodyScrollLocked && outside.length > 0 && outside.filter((hit) => hit !== 'page').length * 2 >= outside.length) return { overlays: true, coverage }; + + return { overlays: false, coverage }; +} + +export function probeRegionCoverage(config: { xpath: string }): RegionCoverageSamples { + const samples: RegionCoverageSamples = { + found: false, + rect: { x: 0, y: 0, width: 0, height: 0 }, + viewport: { width: window.innerWidth, height: window.innerHeight }, + position: 'static', + zIndex: 0, + outsideHits: [], + siblingsInert: false, + bodyScrollLocked: false, + }; + + const result = document.evaluate(config.xpath, document, null, 9, null); + const node = result.singleNodeValue; + if (!node || node.nodeType !== 1) return samples; + const element = node as HTMLElement; + const rect = element.getBoundingClientRect(); + if (rect.width === 0 && rect.height === 0) return samples; + + const style = window.getComputedStyle(element); + samples.found = true; + samples.rect = { x: rect.x, y: rect.y, width: rect.width, height: rect.height }; + samples.position = style.position; + samples.zIndex = Number.parseInt(style.zIndex || '0', 10) || 0; + + const bodyStyle = window.getComputedStyle(document.body); + samples.bodyScrollLocked = bodyStyle.overflow === 'hidden' || bodyStyle.overflowY === 'hidden'; + + for (const sibling of Array.from(element.parentElement?.children || [])) { + if (sibling === element) continue; + if (!sibling.hasAttribute('inert') && sibling.getAttribute('aria-hidden') !== 'true') continue; + samples.siblingsInert = true; + break; + } + + function classifyHit(hit: Element | null): 'inside' | 'blocked' | 'page' { + if (!hit) return 'page'; + if (element.contains(hit)) return 'inside'; + let current: Element | null = hit; + for (let depth = 0; current && depth < 4; depth++) { + const hitStyle = window.getComputedStyle(current as HTMLElement); + const hitZ = Number.parseInt(hitStyle.zIndex || '0', 10) || 0; + if ((hitStyle.position === 'fixed' || hitStyle.position === 'absolute') && hitZ > 0) return 'blocked'; + current = current.parentElement; + } + return 'page'; + } + + const inset = 10; + const width = window.innerWidth; + const height = window.innerHeight; + const points: Array<[number, number]> = [ + [inset, inset], + [width - inset, inset], + [inset, height - inset], + [width - inset, height - inset], + [width / 2, inset], + [width / 2, height - inset], + [inset, height / 2], + [width - inset, height / 2], + ]; + for (const [x, y] of points) { + if (x >= rect.left && x <= rect.right && y >= rect.top && y <= rect.bottom) continue; + samples.outsideHits.push(classifyHit(document.elementFromPoint(x, y))); + } + + return samples; +} + +export function getRegionCoverageProbeSource(): string { + return probeRegionCoverage.toString(); +} + export interface AppearedSubRoot { container: string; elementXPath: string; @@ -103,3 +199,14 @@ export interface RegionVerdict { overlays: boolean; coverage: number; } + +export interface RegionCoverageSamples { + found: boolean; + rect: { x: number; y: number; width: number; height: number }; + viewport: { width: number; height: number }; + position: string; + zIndex: number; + outsideHits: Array<'inside' | 'blocked' | 'page'>; + siblingsInert: boolean; + bodyScrollLocked: boolean; +} diff --git a/tests/unit/overlay-detection.test.ts b/tests/unit/overlay-detection.test.ts index 5bce8fcd..c26437de 100644 --- a/tests/unit/overlay-detection.test.ts +++ b/tests/unit/overlay-detection.test.ts @@ -4,7 +4,7 @@ import { describe, expect, it } from 'vitest'; import { ActionResult } from '../../src/action-result.ts'; import { htmlDiff } from '../../src/utils/html-diff.ts'; import { HTML_EXTRACTION_LIMITS, HTML_SELECTORS, HTML_VISIBILITY_LIMITS, type VisibleOverlayExtractionConfig, extractVisibleOverlayHtml } from '../../src/utils/html.ts'; -import { OVERLAY_SELECTORS, Overlay, findAppearedSubRoot } from '../../src/utils/overlay.ts'; +import { OVERLAY_SELECTORS, Overlay, type RegionCoverageSamples, classifyRegionCoverage, findAppearedSubRoot, getRegionCoverageProbeSource } from '../../src/utils/overlay.ts'; function overlayConfig(overrides: Partial = {}): VisibleOverlayExtractionConfig { return { @@ -223,3 +223,62 @@ describe('Overlay.fromSubRoot', () => { expect(overlay.root).toBe('//body/div[2]'); }); }); + +const samplesBase = (): RegionCoverageSamples => ({ + found: true, + rect: { x: 0, y: 0, width: 1280, height: 720 }, + viewport: { width: 1280, height: 720 }, + position: 'fixed', + zIndex: 100, + outsideHits: [], + siblingsInert: false, + bodyScrollLocked: false, +}); + +describe('classifyRegionCoverage', () => { + it('full viewport coverage is overlaying', () => { + const verdict = classifyRegionCoverage(samplesBase()); + expect(verdict.overlays).toBe(true); + expect(verdict.coverage).toBeCloseTo(1); + }); + + it('partial floating region with all outside points blocked is overlaying', () => { + const samples = samplesBase(); + samples.rect = { x: 880, y: 0, width: 400, height: 720 }; + samples.outsideHits = ['blocked', 'blocked', 'blocked', 'blocked']; + const verdict = classifyRegionCoverage(samples); + expect(verdict.overlays).toBe(true); + expect(verdict.coverage).toBeLessThan(0.8); + }); + + it('inert siblings mean overlaying regardless of geometry', () => { + const samples = samplesBase(); + samples.rect = { x: 0, y: 0, width: 400, height: 400 }; + samples.siblingsInert = true; + expect(classifyRegionCoverage(samples).overlays).toBe(true); + }); + + it('static in-flow region with page hits outside is inline', () => { + const samples = samplesBase(); + samples.rect = { x: 200, y: 100, width: 800, height: 500 }; + samples.position = 'static'; + samples.zIndex = 0; + samples.outsideHits = ['page', 'page', 'page']; + expect(classifyRegionCoverage(samples).overlays).toBe(false); + }); + + it('missing element or null samples is inline with zero coverage', () => { + expect(classifyRegionCoverage(null)).toEqual({ overlays: false, coverage: 0 }); + const samples = samplesBase(); + samples.found = false; + expect(classifyRegionCoverage(samples)).toEqual({ overlays: false, coverage: 0 }); + }); +}); + +describe('getRegionCoverageProbeSource', () => { + it('serializes to a reconstructible function', () => { + const source = getRegionCoverageProbeSource(); + const fn = new Function(`return ${source}`)(); + expect(typeof fn).toBe('function'); + }); +}); From 5bb36643ba8c272f636d917faaf967092650f51c Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 13:39:38 +0300 Subject: [PATCH 04/27] feat: region-aware state hash, baseHash, memoized diff and areaOfInterest tool results --- .../plans/2026-08-29-region-states.md | 12 ++-- src/action-result.ts | 68 +++++++++++++------ tests/unit/action-result-diff.test.ts | 31 +++++++++ tests/unit/action-result.test.ts | 22 ++++++ 4 files changed, 107 insertions(+), 26 deletions(-) diff --git a/docs/superpowers/plans/2026-08-29-region-states.md b/docs/superpowers/plans/2026-08-29-region-states.md index 15714275..1559c882 100644 --- a/docs/superpowers/plans/2026-08-29-region-states.md +++ b/docs/superpowers/plans/2026-08-29-region-states.md @@ -428,7 +428,7 @@ git commit -m "feat: region coverage probe and classifier in overlay module" - Consumes: `Overlay.present`, `Overlay.root` (Task 1). - Produces (Tasks 4–10 rely on): `get baseHash(): string`; `getStateHash()` including `region_` for named present regions; memoized `diff(previous)` (same `previous.id` → same `Diff` instance); `public regionSubtree: string | undefined`; `PageDiff.areaOfInterest?: string`. -- [ ] **Step 1: Write the failing tests** +- [x] **Step 1: Write the failing tests** Append to `tests/unit/action-result.test.ts`: @@ -488,7 +488,7 @@ describe('diff memoization and areaOfInterest', () => { Run: `bun test tests/unit/action-result.test.ts tests/unit/action-result-diff.test.ts` — expected FAIL. -- [ ] **Step 2: Implement hash changes** +- [x] **Step 2: Implement hash changes** In `src/action-result.ts` replace `getStateHash()` (currently at :478) with: @@ -531,7 +531,7 @@ and add the private method (after the public methods, near `consoleErrors`): `get hash()` already delegates to `getStateHash()` — leave it. -- [ ] **Step 3: Implement diff memoization and regionSubtree** +- [x] **Step 3: Implement diff memoization and regionSubtree** Add fields next to `overlay`: @@ -551,7 +551,7 @@ Replace `diff()` (currently `return Diff.create(this, previousState)`): } ``` -- [ ] **Step 4: Implement the tool-result payoff** +- [x] **Step 4: Implement the tool-result payoff** Add to `PageDiff` interface: `areaOfInterest?: string;` @@ -592,12 +592,12 @@ with: (`minifyHtml`, `htmlCombinedSnapshot`, `ConfigParser` are already imported in this file.) -- [ ] **Step 5: Run tests** +- [x] **Step 5: Run tests** Run: `bun test tests/unit/action-result.test.ts tests/unit/action-result-diff.test.ts tests/unit/action-result-memo.test.ts tests/unit/state-manager.test.ts` Expected: PASS. If the `region_edit_user` assertion fails on slug shape, print the hash and adjust the expectation to the actual `slugify` output of `region_Edit User` — the invariant under test is fork + containment, not the separator. -- [ ] **Step 6: Format and commit** +- [x] **Step 6: Format and commit** ```bash bun run format diff --git a/src/action-result.ts b/src/action-result.ts index e63acaf6..6db6e458 100644 --- a/src/action-result.ts +++ b/src/action-result.ts @@ -49,6 +49,7 @@ export interface PageDiff { consoleErrors?: string[]; htmlParts?: HtmlDiffPart[]; iframes?: string; + areaOfInterest?: string; } export interface ToolResultMetadata { @@ -89,6 +90,8 @@ export class ActionResult implements ActionResultData { public links: Link[] = []; public verifications?: Record; public overlay: Overlay = new Overlay(); + public regionSubtree: string | undefined = undefined; + private _diffCache: { previousId: number | undefined; diff: Diff } | null = null; constructor(data: ActionResultData) { this.id = data.id; @@ -476,29 +479,18 @@ export class ActionResult implements ActionResultData { } getStateHash(): string { - const parts: string[] = []; - - parts.push(this.relativeUrl || this.url || '/'); - - this.extractHeadings(this.html); - - if (this.h1) parts.push(`h1_${this.h1}`); - if (this.h2) parts.push(`h2_${this.h2}`); - - let stateString = slugify(parts.map((part) => part.substring(0, 100)).join('_')); - - if (stateString.length > 200) { - stateString = stateString.substring(0, 200); - if (stateString.endsWith('_')) { - stateString = stateString.slice(0, -1); - } - } + return this.computeStateHash(true); + } - return stateString; + get baseHash(): string { + return this.computeStateHash(false); } async diff(previousState: ActionResult | null): Promise { - return Diff.create(this, previousState); + if (this._diffCache && this._diffCache.previousId === previousState?.id) return this._diffCache.diff; + const diff = await Diff.create(this, previousState); + this._diffCache = { previousId: previousState?.id, diff }; + return diff; } async toToolResult(previousState: ActionResult | null, locator: string): Promise { @@ -549,7 +541,20 @@ export class ActionResult implements ActionResultData { pageDiff.ariaChangeCount = diff.ariaChangeCount; } - if (diff.htmlParts.length > 0) { + if (this.overlay.present && !previousState.overlay.present) { + let area = `${this.overlay.type} "${this.overlay.name || 'unnamed'}" opened`; + if (this.overlay.root) area += `, scope: ${this.overlay.root}`; + pageDiff.areaOfInterest = area; + } + + if (pageDiff.areaOfInterest && this.regionSubtree && this.overlay.root) { + const htmlConfig = ConfigParser.getInstance().getConfig().html; + let subtree = await minifyHtml(htmlCombinedSnapshot(this.regionSubtree, htmlConfig?.combined)); + if (subtree.length > HTML_PART_SUBTREE_BUDGET) { + subtree = `${subtree.slice(0, HTML_PART_SUBTREE_BUDGET)}...`; + } + pageDiff.htmlParts = [{ container: this.overlay.root, subtree, added: [], removed: [] }]; + } else if (diff.htmlParts.length > 0) { const collapsed = collapseHtmlParts(await diff.cleanedHtmlParts()); if (collapsed.length > 0) { pageDiff.htmlParts = collapsed; @@ -565,6 +570,29 @@ export class ActionResult implements ActionResultData { return result; } + private computeStateHash(includeRegion: boolean): string { + const parts: string[] = []; + + parts.push(this.relativeUrl || this.url || '/'); + + this.extractHeadings(this.html); + + if (this.h1) parts.push(`h1_${this.h1}`); + if (this.h2) parts.push(`h2_${this.h2}`); + if (includeRegion && this.overlay.present && this.overlay.name) parts.push(`region_${this.overlay.name}`); + + let stateString = slugify(parts.map((part) => part.substring(0, 100)).join('_')); + + if (stateString.length > 200) { + stateString = stateString.substring(0, 200); + if (stateString.endsWith('_')) { + stateString = stateString.slice(0, -1); + } + } + + return stateString; + } + private consoleErrors(): string[] { const errors: string[] = []; diff --git a/tests/unit/action-result-diff.test.ts b/tests/unit/action-result-diff.test.ts index e4bde0df..235340af 100644 --- a/tests/unit/action-result-diff.test.ts +++ b/tests/unit/action-result-diff.test.ts @@ -169,3 +169,34 @@ describe('ActionResult Diff', () => { expect(diff.ariaChanged).not.toBeNull(); }); }); + +describe('diff memoization and areaOfInterest', () => { + beforeEach(() => { + ConfigParser.resetForTesting(); + ConfigParser.setupTestConfig(); + }); + + test('returns the same Diff instance for the same previous state', async () => { + const previous = new ActionResult({ id: 1, url: 'https://app.example.com/users', html: '

Users

' }); + const current = new ActionResult({ id: 2, url: 'https://app.example.com/users', html: '

Users

changed

' }); + const first = await current.diff(previous); + const second = await current.diff(previous); + expect(second).toBe(first); + }); + + test('reports the appeared region instead of a collapsed dump', async () => { + const previous = new ActionResult({ id: 1, url: 'https://app.example.com/users', html: '

Users

' }); + const current = new ActionResult({ + id: 2, + url: 'https://app.example.com/users', + html: '

Users

', + overlay: { type: 'drawer', name: 'Edit User', root: 'aside.panel' }, + }); + current.regionSubtree = ''; + const result = await current.toToolResult(previous, 'aside.panel'); + expect(result.pageDiff?.areaOfInterest).toBe('drawer "Edit User" opened, scope: aside.panel'); + expect(result.pageDiff?.htmlParts).toHaveLength(1); + expect(result.pageDiff?.htmlParts?.[0].container).toBe('aside.panel'); + expect(result.pageDiff?.htmlParts?.[0].subtree).toContain('Edit User'); + }); +}); diff --git a/tests/unit/action-result.test.ts b/tests/unit/action-result.test.ts index e8a948ae..7f987d3a 100644 --- a/tests/unit/action-result.test.ts +++ b/tests/unit/action-result.test.ts @@ -140,3 +140,25 @@ describe('ActionResult', () => { }); }); }); + +describe('region state hash', () => { + const html = '

Users

'; + + it('named region forks the hash; baseHash stays the page hash', () => { + const plain = new ActionResult({ url: 'https://app.example.com/users', html }); + const withRegion = new ActionResult({ + url: 'https://app.example.com/users', + html, + overlay: { type: 'drawer', name: 'Edit User', root: 'aside.panel' }, + }); + expect(withRegion.hash).not.toBe(plain.hash); + expect(withRegion.hash).toContain('region_edit_user'); + expect(withRegion.baseHash).toBe(plain.hash); + }); + + it('unnamed region does not fork the hash', () => { + const plain = new ActionResult({ url: 'https://app.example.com/users', html }); + const unnamed = new ActionResult({ url: 'https://app.example.com/users', html, overlay: { type: 'modal' } }); + expect(unnamed.hash).toBe(plain.hash); + }); +}); From db0b5845f1ec7e89aa164e4b4996cfe192e3181d Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 13:40:24 +0300 Subject: [PATCH 05/27] feat: detect region of interest from page diff during capture --- .../plans/2026-08-29-region-states.md | 10 ++--- src/action.ts | 37 ++++++++++++++++++- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/docs/superpowers/plans/2026-08-29-region-states.md b/docs/superpowers/plans/2026-08-29-region-states.md index 1559c882..c8c1496c 100644 --- a/docs/superpowers/plans/2026-08-29-region-states.md +++ b/docs/superpowers/plans/2026-08-29-region-states.md @@ -616,11 +616,11 @@ git commit -m "feat: region-aware state hash, baseHash, memoized diff and areaOf - Consumes: `findAppearedSubRoot`, `classifyRegionCoverage`, `Overlay.fromSubRoot`, `getRegionCoverageProbeSource`, type `RegionCoverageSamples` — all from `./utils/overlay.ts` (Tasks 1–2); `result.diff` memoization + `regionSubtree` (Task 3). - Produces: every captured `ActionResult` may now carry a diff-detected `overlay` (`modal`/`drawer`/`region`) and `regionSubtree` before `stateManager.updateState` runs. No new exports. -- [ ] **Step 1: Wire imports** +- [x] **Step 1: Wire imports** In `src/action.ts` extend the existing `./utils/overlay.ts` import (currently `import { Overlay } from './utils/overlay.js';` or similar — check) to also bring `classifyRegionCoverage`, `findAppearedSubRoot`, `getRegionCoverageProbeSource` and type `RegionCoverageSamples`. -- [ ] **Step 2: Hook detection before updateState** +- [x] **Step 2: Hook detection before updateState** In `capturePageState` (src/action.ts:170-188), between `const result = new ActionResult({...})` and `this.stateManager.updateState(result, codeBlock)`: @@ -629,7 +629,7 @@ In `capturePageState` (src/action.ts:170-188), between `const result = new Actio this.stateManager.updateState(result, codeBlock); ``` -- [ ] **Step 3: Implement the private methods** +- [x] **Step 3: Implement the private methods** After the existing private `captureOverlayHtml` (private methods stay after public ones): @@ -671,12 +671,12 @@ After the existing private `captureOverlayHtml` (private methods stay after publ Two guards matter and must not be dropped: `result.overlay.detected` (an ARIA-detected overlay already owns the state) and `isSameUrl` (URL changes are already full state changes with research; the diff path is only for in-place swaps). -- [ ] **Step 4: Verify nothing regressed** +- [x] **Step 4: Verify nothing regressed** Run: `bun test tests/unit/` Expected: PASS (the glue has no unit test — its pure parts are covered by Tasks 1–3; end-to-end behavior is exercised by the local regression harness, which only the user runs). -- [ ] **Step 5: Format and commit** +- [x] **Step 5: Format and commit** ```bash bun run format diff --git a/src/action.ts b/src/action.ts index 5ec8a347..512bf8d5 100644 --- a/src/action.ts +++ b/src/action.ts @@ -13,7 +13,7 @@ import type { StateManager } from './state-manager.js'; import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts'; import { captureHtmlForSnapshot, getVisibleOverlayHtmlExtractorSource, htmlCombinedSnapshot, minifyHtml } from './utils/html.js'; import { createDebug, setStepSpanParent, tag } from './utils/logger.js'; -import { Overlay } from './utils/overlay.js'; +import { Overlay, type RegionCoverageSamples, classifyRegionCoverage, findAppearedSubRoot, getRegionCoverageProbeSource } from './utils/overlay.js'; import { sleep, waitForPageReadiness } from './utils/page-readiness.ts'; import { safeFilename } from './utils/strings.ts'; import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts'; @@ -184,6 +184,7 @@ class Action { overlayHtml: overlayHtml || undefined, iframeURL: frame ? frame.url?.() || 'iframe' : undefined, }); + if (!frame) await this.detectRegionOfInterest(result).catch((err: Error) => debugLog('Region detection failed:', err.message)); this.stateManager.updateState(result, codeBlock); return result; } catch (err) { @@ -205,6 +206,40 @@ class Action { ); } + private async detectRegionOfInterest(result: ActionResult): Promise { + if (result.overlay.detected) return; + const previousState = this.stateManager.getCurrentState(); + if (!previousState) return; + const previous = ActionResult.fromState(previousState); + if (!previous.html || previous.html === result.html) return; + if (!result.isSameUrl({ url: previous.url })) return; + + const diff = await result.diff(previous); + const subRoot = findAppearedSubRoot(diff.htmlParts); + if (!subRoot) return; + + const samples = await this.probeRegion(subRoot.elementXPath); + const verdict = classifyRegionCoverage(samples); + result.overlay = Overlay.fromSubRoot(subRoot, verdict); + result.regionSubtree = subRoot.subtree; + debugLog(`Region of interest: ${result.overlay.type} "${result.overlay.name}" root=${result.overlay.root} coverage=${verdict.coverage.toFixed(2)}`); + } + + private async probeRegion(xpath: string): Promise { + return this.playwrightHelper.page + .evaluate( + ({ probeSource, config }: { probeSource: string; config: any }) => { + const probe = new Function(`return ${probeSource}`)() as (config: any) => any; + return probe(config); + }, + { probeSource: getRegionCoverageProbeSource(), config: { xpath } } + ) + .catch((err: Error) => { + debugLog('Region coverage probe failed:', err.message); + return null; + }); + } + private async captureMainDocumentStatus(): Promise { if (this.mainDocumentStatus) return this.mainDocumentStatus; From d1c8b15261cd5025ab7c87e0ae8921855db85ae1 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 13:41:34 +0300 Subject: [PATCH 06/27] feat: record region-of-interest transitions in state manager --- .../plans/2026-08-29-region-states.md | 10 +++---- src/state-manager.ts | 16 ++++++----- tests/unit/state-manager.test.ts | 28 +++++++++++++++++++ 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/docs/superpowers/plans/2026-08-29-region-states.md b/docs/superpowers/plans/2026-08-29-region-states.md index c8c1496c..74b5a90b 100644 --- a/docs/superpowers/plans/2026-08-29-region-states.md +++ b/docs/superpowers/plans/2026-08-29-region-states.md @@ -696,7 +696,7 @@ git commit -m "feat: detect region of interest from page diff during capture" - Consumes: `Overlay.present` (Task 1); region-aware `hash` (Task 3). - Produces: transitions recorded for region open/close; `tag('data').log('state', …)` payload gains `region` when a region is present. Rename `hasDialogAppeared` → `hasRegionAppeared` (private — no external consumers). -- [ ] **Step 1: Write the failing tests** +- [x] **Step 1: Write the failing tests** Append to `tests/unit/state-manager.test.ts` (reuse that file's existing StateManager construction): @@ -732,7 +732,7 @@ describe('region state transitions', () => { Run: `bun test tests/unit/state-manager.test.ts` — observe which assertions already pass (named open/close comes from the Task 3 hash fork); the tests pin the behavior either way. -- [ ] **Step 2: Generalize the check** +- [x] **Step 2: Generalize the check** In `src/state-manager.ts`: @@ -755,7 +755,7 @@ and rename/adjust the private method: Update the debug line inside the branch to `debugLog('State change detected: region of interest appeared');`. -- [ ] **Step 3: Extend the remote state frame** +- [x] **Step 3: Extend the remote state frame** In `emitStateChange`: @@ -765,12 +765,12 @@ In `emitStateChange`: tag('data').log('state', payload); ``` -- [ ] **Step 4: Run tests** +- [x] **Step 4: Run tests** Run: `bun test tests/unit/state-manager.test.ts tests/unit/state-manager-events.test.ts` Expected: PASS. -- [ ] **Step 5: Format and commit** +- [x] **Step 5: Format and commit** ```bash bun run format diff --git a/src/state-manager.ts b/src/state-manager.ts index 03f7d2a0..5cc6151b 100644 --- a/src/state-manager.ts +++ b/src/state-manager.ts @@ -119,7 +119,9 @@ export class StateManager { */ private emitStateChange(event: StateTransition): void { const state = event.toState; - tag('data').log('state', { url: state.fullUrl || state.url, path: state.url, title: state.title, h1: state.h1 }); + const payload: Record = { url: state.fullUrl || state.url, path: state.url, title: state.title, h1: state.h1 }; + if (state.overlay?.present) payload.region = state.overlay.name || state.overlay.type; + tag('data').log('state', payload); this.stateChangeListeners.forEach((listener) => { try { @@ -143,9 +145,9 @@ export class StateManager { if (newState.url) this.allVisitedUrls.add(normalizeUrl(newState.url)); const hashChanged = actionResult.hash !== previousHash; - const dialogOpened = !hashChanged && this.hasDialogAppeared(previousState, newState); + const regionAppeared = !hashChanged && this.hasRegionAppeared(previousState, newState); - if (hashChanged || dialogOpened) { + if (hashChanged || regionAppeared) { const transition: StateTransition = { fromState: previousState, toState: newState, @@ -156,8 +158,8 @@ export class StateManager { this.stateHistory.push(transition); this.emitStateChange(transition); - if (dialogOpened) { - debugLog('State change detected: modal dialog appeared'); + if (regionAppeared) { + debugLog('State change detected: region of interest appeared'); } } @@ -206,10 +208,10 @@ export class StateManager { return newState; } - private hasDialogAppeared(previousState: WebPageState | null, newState: WebPageState): boolean { + private hasRegionAppeared(previousState: WebPageState | null, newState: WebPageState): boolean { const prevFocus = previousState?.overlay ?? Overlay.fromAria(previousState?.ariaSnapshot ?? null); const newFocus = newState.overlay ?? Overlay.fromAria(newState.ariaSnapshot ?? null); - return !prevFocus.detected && newFocus.detected; + return !prevFocus.present && newFocus.present; } /** diff --git a/tests/unit/state-manager.test.ts b/tests/unit/state-manager.test.ts index c956a2da..e6fbf523 100644 --- a/tests/unit/state-manager.test.ts +++ b/tests/unit/state-manager.test.ts @@ -385,4 +385,32 @@ describe('StateManager', () => { expect(stateManager.getListenerCount()).toBe(0); }); }); + + describe('region state transitions', () => { + const html = '

Users

'; + + it('records a transition when a named region opens and when it closes', () => { + const base = new ActionResult({ url: '/users', html }); + stateManager.updateState(base); + const historyAfterBase = stateManager.getStateHistory().length; + + const withDrawer = new ActionResult({ url: '/users', html, overlay: { type: 'drawer', name: 'Edit User', root: 'aside.panel' } }); + stateManager.updateState(withDrawer); + expect(stateManager.getStateHistory().length).toBe(historyAfterBase + 1); + + const closed = new ActionResult({ url: '/users', html }); + stateManager.updateState(closed); + expect(stateManager.getStateHistory().length).toBe(historyAfterBase + 2); + }); + + it('records a transition for an unnamed region via hasRegionAppeared', () => { + const base = new ActionResult({ url: '/users', html }); + stateManager.updateState(base); + const historyAfterBase = stateManager.getStateHistory().length; + + const unnamed = new ActionResult({ url: '/users', html, overlay: { type: 'modal' } }); + stateManager.updateState(unnamed); + expect(stateManager.getStateHistory().length).toBe(historyAfterBase + 1); + }); + }); }); From dab7b8f1a56f5f0b332e41f7538980726faabd6e Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 13:43:02 +0300 Subject: [PATCH 07/27] feat: root selector envelope key scopes experience to open regions --- CLAUDE.md | 2 +- .../plans/2026-08-29-region-states.md | 12 ++++---- src/action-result.ts | 6 +++- src/experience-tracker.ts | 5 +++- tests/unit/experience-tracker.test.ts | 29 +++++++++++++++++++ 5 files changed, 45 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 93517364..fd601e6b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -137,7 +137,7 @@ All persisted formats share one rule: **envelope keys (YAML frontmatter, HTML co | Format | Location & owner | Envelope | Body grammar | |---|---|---|---| | Knowledge | `knowledge/*.md`, KnowledgeTracker | `url`/`path`, `wait`, `waitForElement`, `noExperienceReading/Writing` | Free prose facts | -| Experience | `experience/.md`, ExperienceTracker | sparse frontmatter | `## FLOW:` / `## ACTION:` h2 blocks; bullets + ```js``` + `Solution:` line; h3 forbidden under blocks | +| Experience | `experience/.md`, ExperienceTracker | sparse frontmatter: `url`, `title`, optional `root` (region scoping selector — record loads only while a matching region is open) | `## FLOW:` / `## ACTION:` h2 blocks; bullets + ```js``` + `Solution:` line; h3 forbidden under blocks | | Test plan | `output/plans/*.md`, test-plan-markdown.ts | `` comment: `priority`, `style`; scenario heading, `url:` line, bullets as steps | Notes/results appended by runner | These are **data formats**: written and read back inside the runtime loop (knowledge/experience steer every run; plans are consumed by the runner and rerun). diff --git a/docs/superpowers/plans/2026-08-29-region-states.md b/docs/superpowers/plans/2026-08-29-region-states.md index 74b5a90b..bb159d72 100644 --- a/docs/superpowers/plans/2026-08-29-region-states.md +++ b/docs/superpowers/plans/2026-08-29-region-states.md @@ -790,7 +790,7 @@ git commit -m "feat: record region-of-interest transitions in state manager" - Consumes: `Overlay.present` / `Overlay.root` (Task 1), region-hashed states (Task 3). - Produces: experience frontmatter key `root` (single writer: `ExperienceTracker.ensureExperienceFile`); retrieval gate in `ActionResult.isRelevantExperienceRecord(record: WebPageState & { root?: string }, …)`. -- [ ] **Step 1: Write the failing tests** +- [x] **Step 1: Write the failing tests** Append to `tests/unit/experience-tracker.test.ts`, reusing that file's existing `beforeEach` setup (temp experience dir, tracker construction). The tests need only the `tracker` it already builds: @@ -827,7 +827,7 @@ describe('region experience root', () => { Run: `bun test tests/unit/experience-tracker.test.ts` — expected FAIL. -- [ ] **Step 2: Implement the writer** +- [x] **Step 2: Implement the writer** In `src/experience-tracker.ts` `ensureExperienceFile` (currently :118), replace the frontmatter literal: @@ -846,7 +846,7 @@ In `src/experience-tracker.ts` `ensureExperienceFile` (currently :118), replace Plain `if` — no conditional spread. -- [ ] **Step 3: Implement the retrieval gate** +- [x] **Step 3: Implement the retrieval gate** In `src/action-result.ts` `isRelevantExperienceRecord` (currently :261), widen the signature and add the gate as the first check after the null guard: @@ -862,16 +862,16 @@ In `src/action-result.ts` `isRelevantExperienceRecord` (currently :261), widen t (rest of the method unchanged). A record without `root` behaves exactly as today — envelope rule 3. The `root` gate comes first so behavior does not depend on heading coincidences between region and page states. -- [ ] **Step 4: Document the envelope key** +- [x] **Step 4: Document the envelope key** In `CLAUDE.md`, "Data Envelope Formats" table, Experience row: change the envelope cell from `sparse frontmatter` to `sparse frontmatter: url, title, optional root (region scoping selector — record loads only while a matching region is open)`. -- [ ] **Step 5: Run tests** +- [x] **Step 5: Run tests** Run: `bun test tests/unit/experience-tracker.test.ts tests/unit/experience-compactor.test.ts tests/unit/historian-experience.test.ts` Expected: PASS. -- [ ] **Step 6: Format and commit** +- [x] **Step 6: Format and commit** ```bash bun run format diff --git a/src/action-result.ts b/src/action-result.ts index 6db6e458..709b7590 100644 --- a/src/action-result.ts +++ b/src/action-result.ts @@ -261,8 +261,12 @@ export class ActionResult implements ActionResultData { return true; } - isRelevantExperienceRecord(record: WebPageState, options?: { includeDescendantExperience?: boolean }): boolean { + isRelevantExperienceRecord(record: WebPageState & { root?: string }, options?: { includeDescendantExperience?: boolean }): boolean { if (!record.url || !this.url) return false; + if (record.root) { + if (!this.overlay.present) return false; + if (this.overlay.root && this.overlay.root !== record.root) return false; + } if (this.isMatchedBy(record)) return true; if (!options?.includeDescendantExperience) return false; const cur = extractStatePath(this.url); diff --git a/src/experience-tracker.ts b/src/experience-tracker.ts index fd6b4c42..1fa74013 100644 --- a/src/experience-tracker.ts +++ b/src/experience-tracker.ts @@ -123,10 +123,13 @@ export class ExperienceTracker { const filePath = this.getExperienceFilePath(stateHash); if (!existsSync(filePath)) { - const frontmatter = { + const frontmatter: Record = { url: state.url ? extractStatePath(state.url) : '', title: state.title, }; + if (state.overlay.present && state.overlay.root) { + frontmatter.root = state.overlay.root; + } this.writeExperienceFile(stateHash, '', frontmatter); } diff --git a/tests/unit/experience-tracker.test.ts b/tests/unit/experience-tracker.test.ts index da94a7dd..3950f46c 100644 --- a/tests/unit/experience-tracker.test.ts +++ b/tests/unit/experience-tracker.test.ts @@ -421,4 +421,33 @@ describe('ExperienceTracker', () => { expect(toc[0].fileTag).toBe('A'); }); }); + + describe('region experience root', () => { + const html = '

Users

'; + const regionOverlay = { type: 'drawer' as const, name: 'Edit User', root: 'aside.panel' }; + + it('writes root frontmatter for a region state', () => { + const regionState = new ActionResult({ url: '/users', html, overlay: regionOverlay }); + experienceTracker.writeAction(regionState, { title: 'Save the edit form', code: 'I.click("Save")', explanation: '' }); + const { data } = experienceTracker.readExperienceFile(regionState.getStateHash()); + expect(data.root).toBe('aside.panel'); + }); + + it('skips root-scoped records when no region is open, loads them when it matches', () => { + const regionState = new ActionResult({ url: '/users', html, overlay: regionOverlay }); + experienceTracker.writeAction(regionState, { title: 'Save the edit form', code: 'I.click("Save")', explanation: '' }); + + const baseState = new ActionResult({ url: '/users', html }); + const baseContents = experienceTracker.getRelevantExperience(baseState).map((e) => e.content); + expect(baseContents.join('\n')).not.toContain('save the edit form'); + + const openState = new ActionResult({ url: '/users', html, overlay: regionOverlay }); + const openContents = experienceTracker.getRelevantExperience(openState).map((e) => e.content); + expect(openContents.join('\n')).toContain('save the edit form'); + + const otherRegion = new ActionResult({ url: '/users', html, overlay: { type: 'drawer' as const, name: 'Filters', root: 'div.filters' } }); + const otherContents = experienceTracker.getRelevantExperience(otherRegion).map((e) => e.content); + expect(otherContents.join('\n')).not.toContain('save the edit form'); + }); + }); }); From 2758f4c83ae4ec9b4e929a9c4126bb6587366dad Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 13:45:56 +0300 Subject: [PATCH 08/27] feat: key research by base page hash and research any named region --- docs/superpowers/plans/2026-08-29-region-states.md | 8 ++++---- src/ai/researcher.ts | 6 +++--- src/ai/researcher/deep-analysis.ts | 3 +-- tests/integration/researcher.test.ts | 10 +++++++--- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/superpowers/plans/2026-08-29-region-states.md b/docs/superpowers/plans/2026-08-29-region-states.md index bb159d72..8bcff2b2 100644 --- a/docs/superpowers/plans/2026-08-29-region-states.md +++ b/docs/superpowers/plans/2026-08-29-region-states.md @@ -890,7 +890,7 @@ git commit -m "feat: root selector envelope key scopes experience to open region - Consumes: `baseHash` (Task 3), `Overlay.present` (Task 1). - Produces: research cache keyed by `baseHash` (region states share the page's research); `researchOverlay` fires for any named present region, not only dialog/modal. -- [ ] **Step 1: Key the cache by baseHash** +- [x] **Step 1: Key the cache by baseHash** In `src/ai/researcher.ts`: @@ -910,7 +910,7 @@ At :99 replace `const stateHash = state.hash || this.actionResult.getStateHash() Then run `grep -n "\.hash" src/ai/researcher.ts src/ai/researcher/*.ts` and audit each hit: cache reads/writes (`getCachedResearch`, `saveResearch`, `getPreviousResearch`, `researchPath` keys) move to `baseHash`; state-equality comparisons (e.g. `getStateHash() === getCurrentState()?.hash` at :154 and :317) stay full-hash — both sides use the same computation, so they remain consistent. -- [ ] **Step 2: Widen researchOverlay** +- [x] **Step 2: Widen researchOverlay** In `src/ai/researcher/deep-analysis.ts` at :93-95 replace: @@ -927,12 +927,12 @@ with: if (!focusArea.present || !focusArea.name) return null; ``` -- [ ] **Step 3: Run tests** +- [x] **Step 3: Run tests** Run: `bun test tests/unit/ && bun test tests/integration/` Expected: PASS. Failures here mean a cache-key call site was converted that should not have been (or vice versa) — re-audit the grep list before changing anything else. -- [ ] **Step 4: Format and commit** +- [x] **Step 4: Format and commit** ```bash bun run format diff --git a/src/ai/researcher.ts b/src/ai/researcher.ts index 1beab792..c7c6fa7f 100644 --- a/src/ai/researcher.ts +++ b/src/ai/researcher.ts @@ -76,7 +76,7 @@ export class Researcher extends ResearcherBase implements Agent { } static getCachedResearch(state: WebPageState): string { - return getCachedResearch(state.hash || ''); + return getCachedResearch(ActionResult.fromState(state).baseHash); } getSystemMessage(): string { @@ -96,7 +96,7 @@ export class Researcher extends ResearcherBase implements Agent { const maxRetries = (this.config.ai?.agents?.researcher as any)?.retries ?? 2; let retriesLeft = opts._retriesLeft ?? maxRetries; this.actionResult = ActionResult.fromState(state); - const stateHash = state.hash || this.actionResult.getStateHash(); + const stateHash = this.actionResult.baseHash; const researchState = { ...state, hash: stateHash }; if (!force && stateHash) { @@ -268,7 +268,7 @@ export class Researcher extends ResearcherBase implements Agent { if (!interrupted() && deep) { try { - await this.performDeepAnalysis(state, result); + await this.performDeepAnalysis(researchState, result); } catch (err) { tag('warning').log(`Deep analysis failed, continuing with best-effort research: ${err instanceof Error ? err.message : err}`); } diff --git a/src/ai/researcher/deep-analysis.ts b/src/ai/researcher/deep-analysis.ts index c291c292..c1f8e2de 100644 --- a/src/ai/researcher/deep-analysis.ts +++ b/src/ai/researcher/deep-analysis.ts @@ -89,8 +89,7 @@ export function WithDeepAnalysis(Base: T) { async researchOverlay(current: ActionResult, previous: ActionResult, pageStateHash: string): Promise { const focusArea = current.overlay; - if (!focusArea.detected || !focusArea.name) return null; - if (focusArea.type !== 'dialog' && focusArea.type !== 'modal') return null; + if (!focusArea.present || !focusArea.name) return null; const cached = getCachedResearch(pageStateHash); if (!cached) return null; diff --git a/tests/integration/researcher.test.ts b/tests/integration/researcher.test.ts index 67054a1b..63f6a17a 100644 --- a/tests/integration/researcher.test.ts +++ b/tests/integration/researcher.test.ts @@ -90,6 +90,10 @@ function createMockDeps(state = fakeState) { }; } +function fakeStateBaseHash(): string { + return ActionResult.fromState(fakeState as any).baseHash; +} + function extractPromptText(entry: any): string { if (!entry?.body?.messages) return ''; return entry.body.messages @@ -194,7 +198,7 @@ describe('Researcher with aimock', () => { }); it('returns cached research verbatim and without an AI call', async () => { - saveResearch({ hash: fakeState.hash!, url: fakeState.url }, '## Cached Research\n\nPreviously analyzed page.'); + saveResearch({ hash: fakeStateBaseHash(), url: fakeState.url }, '## Cached Research\n\nPreviously analyzed page.'); const result = await researcher.research(fakeState, { fix: false }); @@ -203,7 +207,7 @@ describe('Researcher with aimock', () => { }); it('force flag bypasses cache', async () => { - saveResearch({ hash: fakeState.hash!, url: fakeState.url }, '## Cached Research\n\nOld cached content.'); + saveResearch({ hash: fakeStateBaseHash(), url: fakeState.url }, '## Cached Research\n\nOld cached content.'); const result = await researcher.research(fakeState, { fix: false, force: true }); @@ -215,7 +219,7 @@ describe('Researcher with aimock', () => { it('saves research result to cache after AI call', async () => { await researcher.research(fakeState, { fix: false }); - const cached = getCachedResearch(fakeState.hash!); + const cached = getCachedResearch(fakeStateBaseHash()); expect(cached).toContain('## Navigation'); expect(cached).toContain('Create Task'); }); From 1143c9b2e22888f4ce1b384369cb4b296ef3c2ad Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 13:47:06 +0300 Subject: [PATCH 09/27] feat: tester context carries region root and inline area of interest --- .../plans/2026-08-29-region-states.md | 12 +++++------ src/ai/tester.ts | 21 ++++++++++++++++--- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/docs/superpowers/plans/2026-08-29-region-states.md b/docs/superpowers/plans/2026-08-29-region-states.md index 8bcff2b2..8a03a260 100644 --- a/docs/superpowers/plans/2026-08-29-region-states.md +++ b/docs/superpowers/plans/2026-08-29-region-states.md @@ -951,7 +951,7 @@ git commit -m "feat: key research by base page hash and research any named regio - Consumes: `Overlay.present`/`root` (Task 1), `baseHash` (Task 3), widened `researchOverlay` (Task 7). - Produces: `` carries the root selector; new `` block for inline regions injected once per state change; `pageStateHash` holds `baseHash`. -- [ ] **Step 1: Track state-change trigger** +- [x] **Step 1: Track state-change trigger** In `reinjectContextIfNeeded` (src/ai/tester.ts:528), replace the tracking prologue: @@ -972,7 +972,7 @@ with: this.previousStateHash = currentStateHash; ``` -- [ ] **Step 2: Root selector in focus_scope** +- [x] **Step 2: Root selector in focus_scope** In the `if (focusArea.detected)` block (currently :558), add before `context +=`: @@ -989,7 +989,7 @@ and change the first line of the dedent block to: (the rest of the block unchanged — the strict "not actionable outside" wording stays, and stays gated on `detected`, i.e. on a probe-verified or ARIA-verified overlay). -- [ ] **Step 3: Inline area_of_interest block** +- [x] **Step 3: Inline area_of_interest block** Immediately after the `if (focusArea.detected) { ... }` block add: @@ -1009,7 +1009,7 @@ Immediately after the `if (focusArea.detected) { ... }` block add: General wording only — no element names, no site specifics. -- [ ] **Step 4: baseHash for research keys and widened overlay-research gate** +- [x] **Step 4: baseHash for research keys and widened overlay-research gate** At :592 replace `this.pageStateHash = currentStateHash;` with: @@ -1023,12 +1023,12 @@ At :630 replace the condition `if (focusArea.detected && focusArea.name && this. if (focusArea.present && focusArea.name && this.pageStateHash && this.pageActionResult) { ``` -- [ ] **Step 5: Run tests** +- [x] **Step 5: Run tests** Run: `bun test tests/unit/ && bun test tests/integration/` Expected: PASS (prompt changes must go through the integration suite before pushing — house rule). -- [ ] **Step 6: Format and commit** +- [x] **Step 6: Format and commit** ```bash bun run format diff --git a/src/ai/tester.ts b/src/ai/tester.ts index c180c5a5..5359489b 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -530,6 +530,7 @@ export class Tester extends TaskAgent implements Agent { const currentStateHash = currentState.hash; const isNewUrl = this.previousUrl !== currentUrl; + const isNewState = !isNewUrl && this.previousStateHash !== null && this.previousStateHash !== currentStateHash; this.previousUrl = currentUrl; this.previousStateHash = currentStateHash; @@ -557,9 +558,11 @@ export class Tester extends TaskAgent implements Agent { if (focusArea.detected) { const areaName = focusArea.name ? ` "${focusArea.name}"` : ''; + let rootHint = ''; + if (focusArea.root) rootHint = `\nIts content lives inside \`${focusArea.root}\` — scope locators to it.`; context += dedent` - A ${focusArea.type}${areaName} is currently open above the page. + A ${focusArea.type}${areaName} is currently open above the page.${rootHint} Scope all interactions to elements inside this ${focusArea.type}. Page navigation, filters, and tabs that exist outside it are not actionable while it is open and may share names or roles with elements inside it — prefer the locator inside the ${focusArea.type}. Use to confirm the element you target is actually inside the ${focusArea.type}. @@ -567,6 +570,18 @@ export class Tester extends TaskAgent implements Agent { `; } + if (!focusArea.detected && focusArea.present && isNewState) { + let rootHint = ''; + if (focusArea.root) rootHint = `\nIt lives inside \`${focusArea.root}\`.`; + context += dedent` + + A large new area "${focusArea.name || 'unnamed area'}" appeared on this page without navigation.${rootHint} + The scenario most likely continues inside this area — prefer its elements for your next actions. + The rest of the page (navigation, menus, filters) is still interactive and remains available. + + `; + } + if (currentState.isInsideIframe) { const iframeInfo = currentState.iframeURL || 'iframe context active'; context += dedent` @@ -589,7 +604,7 @@ export class Tester extends TaskAgent implements Agent { if (!alreadySeenUiMap) { research = await this.researcher.research(currentState).catch(this.skipResearch); } - this.pageStateHash = currentStateHash; + this.pageStateHash = currentState.baseHash; this.pageActionResult = currentState; let uiMapSection = ''; if (research) { @@ -627,7 +642,7 @@ export class Tester extends TaskAgent implements Agent { return context; } - if (focusArea.detected && focusArea.name && this.pageStateHash && this.pageActionResult) { + if (focusArea.present && focusArea.name && this.pageStateHash && this.pageActionResult) { const overlaySection = await this.researcher.researchOverlay(currentState, this.pageActionResult, this.pageStateHash).catch(this.skipResearch); if (overlaySection) { context += dedent` From 0eddbfc378ef9358fa10600c03a8017fa30699a7 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 13:48:13 +0300 Subject: [PATCH 10/27] feat: pilot state context distinguishes overlaying modals from inline regions --- .../superpowers/plans/2026-08-29-region-states.md | 10 +++++----- src/ai/pilot.ts | 9 ++++++++- tests/unit/pilot-state-context.test.ts | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/docs/superpowers/plans/2026-08-29-region-states.md b/docs/superpowers/plans/2026-08-29-region-states.md index 8a03a260..057cc1cc 100644 --- a/docs/superpowers/plans/2026-08-29-region-states.md +++ b/docs/superpowers/plans/2026-08-29-region-states.md @@ -1050,7 +1050,7 @@ git commit -m "feat: tester context carries region root and inline area of inter **Note:** `src/ai/pilot.ts` and this test file carry uncommitted in-flight changes — read both fully before editing and integrate, do not revert anything. -- [ ] **Step 1: Write the failing tests** +- [x] **Step 1: Write the failing tests** Append to `tests/unit/pilot-state-context.test.ts`, following that file's existing pattern for building an `ActionResult` and reading `buildStateContext` output: @@ -1073,7 +1073,7 @@ it('shows inline region distinctly from a modal', () => { Run: `bun test tests/unit/pilot-state-context.test.ts` — expected FAIL. -- [ ] **Step 2: Implement the state lines** +- [x] **Step 2: Implement the state lines** In `src/ai/pilot.ts` `buildStateContext` (currently :828-834) replace: @@ -1103,7 +1103,7 @@ with: } ``` -- [ ] **Step 3: One general prompt bullet** +- [x] **Step 3: One general prompt bullet** In `getSystemPrompt`, in the "Diagnostic patterns" list, add one line: @@ -1113,12 +1113,12 @@ In `getSystemPrompt`, in the "Diagnostic patterns" list, add one line: Nothing else in the prompt changes. -- [ ] **Step 4: Run tests** +- [x] **Step 4: Run tests** Run: `bun test tests/unit/pilot-state-context.test.ts && bun test tests/integration/` Expected: PASS. -- [ ] **Step 5: Format and commit** +- [x] **Step 5: Format and commit** ```bash bun run format diff --git a/src/ai/pilot.ts b/src/ai/pilot.ts index dc197301..139a60bd 100644 --- a/src/ai/pilot.ts +++ b/src/ai/pilot.ts @@ -827,7 +827,13 @@ export class Pilot implements Agent { const focusArea = state.overlay; if (focusArea.detected) { - lines.push(`modal: ${focusArea.name || focusArea.type}`); + let line = `modal: ${focusArea.name || focusArea.type}`; + if (focusArea.root) line += ` (root: ${focusArea.root})`; + lines.push(line); + } else if (focusArea.present) { + let line = `region: ${focusArea.name || 'unnamed'} (inline`; + if (focusArea.root) line += `, root: ${focusArea.root}`; + lines.push(`${line})`); } else { lines.push('modal: none'); } @@ -1131,6 +1137,7 @@ export class Pilot implements Agent { Diagnostic patterns (use , executed/element/skipped fields, ariaDiff): - Click failed + button in "disabled buttons" → required field missing. Instruct fill first. - "modal: none" but Tester targets a modal → modal closed; re-trigger. + - "region:" in → a large area appeared in place without navigation (subview, wizard step, panel). Direct Tester to act inside it; the rest of the page is still usable. - Action SUCCESS but ariaDiff empty → may have worked without visible DOM change; check result message. - MultipleElementsFound → xpathCheck() to identify the right one, then precise locator or visualClick(). - Wrong page (settings vs feature) → getVisitedStates() then back() or reset(). Don't try breadcrumbs (SPA back-nav is unreliable). diff --git a/tests/unit/pilot-state-context.test.ts b/tests/unit/pilot-state-context.test.ts index b5454676..2313d0ae 100644 --- a/tests/unit/pilot-state-context.test.ts +++ b/tests/unit/pilot-state-context.test.ts @@ -103,4 +103,19 @@ describe('Pilot buildStateContext — error signals', () => { const context = (pilot as any).buildStateContext(buildActionResult()); expect(context).toContain('network errors: none'); }); + + it('shows verified overlay with its root', () => { + const pilot = buildPilotWithStore(null); + const state = new ActionResult({ url: '/users', html: '

Users

', overlay: { type: 'drawer', name: 'Edit User', root: 'aside.panel' } }); + const context = (pilot as any).buildStateContext(state); + expect(context).toContain('modal: Edit User (root: aside.panel)'); + }); + + it('shows inline region distinctly from a modal', () => { + const pilot = buildPilotWithStore(null); + const state = new ActionResult({ url: '/users', html: '

Users

', overlay: { type: 'region', name: 'User Details', root: 'section.details' } }); + const context = (pilot as any).buildStateContext(state); + expect(context).toContain('region: User Details (inline, root: section.details)'); + expect(context).not.toContain('modal: User Details'); + }); }); From 114f1d9aa2dc10838710ad61b334c47b32a3499a Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 13:49:11 +0300 Subject: [PATCH 11/27] refactor: driller reads nested overlays from page diff instead of DOM queries --- .../plans/2026-08-29-region-states.md | 8 ++-- src/ai/driller.ts | 46 +++---------------- 2 files changed, 11 insertions(+), 43 deletions(-) diff --git a/docs/superpowers/plans/2026-08-29-region-states.md b/docs/superpowers/plans/2026-08-29-region-states.md index 057cc1cc..c9c39ff2 100644 --- a/docs/superpowers/plans/2026-08-29-region-states.md +++ b/docs/superpowers/plans/2026-08-29-region-states.md @@ -1138,7 +1138,7 @@ git commit -m "feat: pilot state context distinguishes overlaying modals from in - Consumes: `pageDiff.htmlParts` / `pageDiff.areaOfInterest` from tool results (Task 3). - Produces: `detectNestedOverlayContext` no longer queries the live DOM; `Driller.getVisibleOverlayHtml` is deleted along with its imports (`getVisibleOverlayHtmlExtractorSource`, `OVERLAY_SELECTORS`, and any `HTML_*` config constants imported only for it). -- [ ] **Step 1: Replace the DOM query with the diff the result already carries** +- [x] **Step 1: Replace the DOM query with the diff the result already carries** In `src/ai/driller.ts` `detectNestedOverlayContext` (currently :648), replace the overlay-fetch prologue: @@ -1163,16 +1163,16 @@ with: The rest of the method (the `` dedent block) is unchanged — `overlayHtml` keeps its name and role in the prompt. -- [ ] **Step 2: Delete the private extractor** +- [x] **Step 2: Delete the private extractor** Remove the whole `private async getVisibleOverlayHtml()` method (currently :674-692). Then remove from the imports at the top of `driller.ts`: `getVisibleOverlayHtmlExtractorSource`, `OVERLAY_SELECTORS`, and each of `HTML_SELECTORS` / `HTML_EXTRACTION_LIMITS` / `HTML_VISIBILITY_LIMITS` **only if** `grep -n "" src/ai/driller.ts` shows no remaining use in this file. -- [ ] **Step 3: Run tests** +- [x] **Step 3: Run tests** Run: `bun test tests/unit/driller.test.ts && bun test tests/unit/` Expected: PASS. -- [ ] **Step 4: Format and commit** +- [x] **Step 4: Format and commit** ```bash bun run format diff --git a/src/ai/driller.ts b/src/ai/driller.ts index 0543c661..257efb9b 100644 --- a/src/ai/driller.ts +++ b/src/ai/driller.ts @@ -6,23 +6,9 @@ import { setActivity } from '../activity.ts'; import { Observability } from '../observability.ts'; import { Plan, Test, TestResult } from '../test-plan.ts'; import { collectInteractiveNodes } from '../utils/aria.ts'; -import { - EXPLORBOT_ATTRS, - HTML_COMPOSITE_AREA_HINTS, - HTML_COMPOSITE_TARGET_ROLES, - HTML_EXTRACTION_LIMITS, - HTML_FORM_CONTROL_ROLES, - HTML_FORM_CONTROL_TAGS, - HTML_INTERACTIVE_ROLES, - HTML_SELECTORS, - HTML_VISIBILITY_LIMITS, - getComponentScopeHtmlExtractorSource, - getVisibleOverlayHtmlExtractorSource, - inferHtmlRole, -} from '../utils/html.ts'; +import { EXPLORBOT_ATTRS, HTML_COMPOSITE_AREA_HINTS, HTML_COMPOSITE_TARGET_ROLES, HTML_EXTRACTION_LIMITS, HTML_FORM_CONTROL_ROLES, HTML_FORM_CONTROL_TAGS, HTML_INTERACTIVE_ROLES, HTML_SELECTORS, getComponentScopeHtmlExtractorSource, inferHtmlRole } from '../utils/html.ts'; import { createDebug, tag } from '../utils/logger.ts'; import { loop, pause } from '../utils/loop.ts'; -import { OVERLAY_SELECTORS } from '../utils/overlay.ts'; import { annotatePageElements } from '../utils/web-annotate.ts'; import { eidxInContainer } from '../utils/web-eidx.ts'; import { WebElement } from '../utils/web-element.ts'; @@ -648,8 +634,11 @@ export class Driller extends TaskAgent implements Agent { private async detectNestedOverlayContext(component: ComponentInfo, result: any): Promise { if (!result?.pageDiff?.ariaChanges || result.pageDiff.urlChanged) return null; - const overlayHtml = await this.getVisibleOverlayHtml(); - if (!overlayHtml) return null; + const parts = result.pageDiff.htmlParts ?? []; + let appeared = parts.filter((part: any) => part.added?.length > 0); + if (result.pageDiff.areaOfInterest) appeared = parts; + const appearedHtml = appeared.map((part: any) => part.subtree).join('\n'); + if (!appearedHtml) return null; const state = this.stateManager.getCurrentState(); if (!state) return null; @@ -661,7 +650,7 @@ export class Driller extends TaskAgent implements Agent { Keep the recorded code reusable and include the parent-opening action when the nested element requires the overlay to be open. - ${overlayHtml} + ${appearedHtml} @@ -671,27 +660,6 @@ export class Driller extends TaskAgent implements Agent { `; } - private async getVisibleOverlayHtml(): Promise { - return this.explorer.withPage((page) => - page.evaluate( - ({ extractorSource, config }) => { - const extract = new Function(`return ${extractorSource}`)() as (config: any) => string; - return extract(config); - }, - { - extractorSource: getVisibleOverlayHtmlExtractorSource(), - config: { - interactiveContentSelector: HTML_SELECTORS.interactiveContent, - limits: HTML_EXTRACTION_LIMITS, - overlaySelectors: OVERLAY_SELECTORS.semanticOverlays, - overlaySemanticSelector: OVERLAY_SELECTORS.overlaySemanticSelector, - visibilityLimits: HTML_VISIBILITY_LIMITS, - }, - } - ) - ); - } - private async getComponentScopeHtml(component: ComponentInfo, originalState: ActionResult): Promise { const scopedHtml = await this.explorer.withPage((page) => page.evaluate( From 6370cfb1ec89579c3781813ff4835c8c3c816c60 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 13:52:53 +0300 Subject: [PATCH 12/27] refactor: remove selector-based overlay detection; overlay.ts is the single detection module --- .../plans/2026-08-29-region-states.md | 14 +- src/action-result.ts | 1 - src/action.ts | 15 +- src/utils/html.ts | 91 ----------- src/utils/overlay.ts | 26 +--- tests/unit/overlay-detection.test.ts | 144 ++---------------- 6 files changed, 19 insertions(+), 272 deletions(-) diff --git a/docs/superpowers/plans/2026-08-29-region-states.md b/docs/superpowers/plans/2026-08-29-region-states.md index c9c39ff2..1ecdad41 100644 --- a/docs/superpowers/plans/2026-08-29-region-states.md +++ b/docs/superpowers/plans/2026-08-29-region-states.md @@ -1192,7 +1192,7 @@ git commit -m "refactor: driller reads nested overlays from page diff instead of - Consumes: everything new from Tasks 1–10 (the replacements must be in place first). - Produces: `Overlay.resolve(data: { overlay?: OverlayData | null; ariaSnapshot?: string | null })` — narrowed signature, no `overlayHtml`. Deleted symbols (per the spec's "Removed code" table): `Action.captureOverlayHtml`, `ActionResultData.overlayHtml`, `Overlay.fromHtml`, `Overlay.captureConfig`, `OVERLAY_SELECTORS`, `extractVisibleOverlayHtml`, `getVisibleOverlayHtmlExtractorSource`, `VisibleOverlayExtractionConfig`. -- [ ] **Step 1: Update the tests first** +- [x] **Step 1: Update the tests first** In `tests/unit/overlay-detection.test.ts`: - Delete the `describe('extractVisibleOverlayHtml', …)` block and the `overlayConfig` helper plus the now-unused imports (`extractVisibleOverlayHtml`, `VisibleOverlayExtractionConfig`, `OVERLAY_SELECTORS`, `HTML_*` constants — keep any that other tests in the file still use). @@ -1213,7 +1213,7 @@ it('resolve falls back to aria detection', () => { Run: `bun test tests/unit/overlay-detection.test.ts` — expected FAIL (resolve still accepts overlayHtml, extractor still exists — the failures confirm the tests now demand the deletion). -- [ ] **Step 2: Delete in overlay.ts** +- [x] **Step 2: Delete in overlay.ts** Remove `OVERLAY_SELECTORS`, `Overlay.fromHtml`, `Overlay.captureConfig`, and the `overlayHtml` branch of `resolve`: @@ -1226,16 +1226,16 @@ Remove `OVERLAY_SELECTORS`, `Overlay.fromHtml`, `Overlay.captureConfig`, and the Prune imports that only served the deleted code (`HTML_EXTRACTION_LIMITS`, `HTML_SELECTORS`, `HTML_VISIBILITY_LIMITS`, `VisibleOverlayExtractionConfig`). `nameFromHtml` stays — `fromSubRoot` uses it. -- [ ] **Step 3: Delete in action.ts and action-result.ts** +- [x] **Step 3: Delete in action.ts and action-result.ts** - `src/action.ts`: remove the `captureOverlayHtml` method; remove `let overlayHtml = '';`, `if (!frame) overlayHtml = await this.captureOverlayHtml();` and the `overlayHtml: overlayHtml || undefined,` constructor line in `capturePageState`; drop `getVisibleOverlayHtmlExtractorSource` from imports. - `src/action-result.ts`: remove `overlayHtml?: string;` from `ActionResultData`. -- [ ] **Step 4: Delete in html.ts** +- [x] **Step 4: Delete in html.ts** Remove `extractVisibleOverlayHtml`, `getVisibleOverlayHtmlExtractorSource`, and the `VisibleOverlayExtractionConfig` interface. For each limit field used only by them (`overlayHtmlLength`, `maxOverlayCount`, `minOverlayWidth`, `minOverlayHeight`, `maxViewportOverlayRatio`, `minOpacity`): run `grep -rn "" src/` and delete the field only when the extractor was its sole consumer — shared visibility limits used by other extractors stay. -- [ ] **Step 5: Verify the path is gone** +- [x] **Step 5: Verify the path is gone** ```bash grep -rn "extractVisibleOverlayHtml\|getVisibleOverlayHtmlExtractorSource\|OVERLAY_SELECTORS\|captureConfig\|overlayHtml\|Overlay.fromHtml" src/ tests/ @@ -1243,12 +1243,12 @@ grep -rn "extractVisibleOverlayHtml\|getVisibleOverlayHtmlExtractorSource\|OVERL Expected: no hits in `src/` (test-fixture prose mentioning "overlay" is fine; symbol references are not). -- [ ] **Step 6: Run tests** +- [x] **Step 6: Run tests** Run: `bun test tests/unit/ && bun test tests/integration/` Expected: PASS. -- [ ] **Step 7: Format and commit** +- [x] **Step 7: Format and commit** ```bash bun run format diff --git a/src/action-result.ts b/src/action-result.ts index 709b7590..454eedaf 100644 --- a/src/action-result.ts +++ b/src/action-result.ts @@ -35,7 +35,6 @@ interface ActionResultData extends WebPageState { focusedElement?: FocusedElement | null; iframeURL?: string; links?: Link[]; - overlayHtml?: string; } export interface PageDiff { diff --git a/src/action.ts b/src/action.ts index 512bf8d5..58f9e5ed 100644 --- a/src/action.ts +++ b/src/action.ts @@ -11,7 +11,7 @@ import { Observability } from './observability.ts'; import type { PlaywrightRecorder } from './playwright-recorder.ts'; import type { StateManager } from './state-manager.js'; import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts'; -import { captureHtmlForSnapshot, getVisibleOverlayHtmlExtractorSource, htmlCombinedSnapshot, minifyHtml } from './utils/html.js'; +import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js'; import { createDebug, setStepSpanParent, tag } from './utils/logger.js'; import { Overlay, type RegionCoverageSamples, classifyRegionCoverage, findAppearedSubRoot, getRegionCoverageProbeSource } from './utils/overlay.js'; import { sleep, waitForPageReadiness } from './utils/page-readiness.ts'; @@ -146,13 +146,11 @@ class Action { let ariaSnapshot: string | null = null; let ariaSnapshotFile: string | undefined = undefined; let focusedElement: FocusedElement | null = null; - let overlayHtml = ''; try { const page = this.playwrightHelper.page; ariaSnapshot = await page.locator('body').ariaSnapshot(); focusedElement = await page.evaluate(readFocusedElement); - if (!frame) overlayHtml = await this.captureOverlayHtml(); } catch (err) { debugLog('ARIA snapshot failed:', err instanceof Error ? `${err.message}\n${err.stack}` : err); } @@ -181,7 +179,6 @@ class Action { ariaSnapshot, ariaSnapshotFile, focusedElement, - overlayHtml: overlayHtml || undefined, iframeURL: frame ? frame.url?.() || 'iframe' : undefined, }); if (!frame) await this.detectRegionOfInterest(result).catch((err: Error) => debugLog('Region detection failed:', err.message)); @@ -196,16 +193,6 @@ class Action { } } - private async captureOverlayHtml(): Promise { - return this.playwrightHelper.page.evaluate( - ({ extractorSource, config }: { extractorSource: string; config: any }) => { - const extract = new Function(`return ${extractorSource}`)() as (config: any) => string; - return extract(config); - }, - { extractorSource: getVisibleOverlayHtmlExtractorSource(), config: Overlay.captureConfig() } - ); - } - private async detectRegionOfInterest(result: ActionResult): Promise { if (result.overlay.detected) return; const previousState = this.stateManager.getCurrentState(); diff --git a/src/utils/html.ts b/src/utils/html.ts index 266e207b..132986f6 100644 --- a/src/utils/html.ts +++ b/src/utils/html.ts @@ -101,17 +101,12 @@ export const HTML_SELECTORS = { } as const; export const HTML_VISIBILITY_LIMITS = { - maxViewportOverlayRatio: 0.95, minOpacity: 0.1, - minOverlayHeight: 40, - minOverlayWidth: 80, } as const; export const HTML_EXTRACTION_LIMITS = { componentScopeHtmlLength: 8000, - maxOverlayCount: 3, maxScopeInteractiveCount: 16, - overlayHtmlLength: 6000, } as const; export const CODE_EDITOR_MARKERS = ['monaco', 'codemirror', 'ace', 'ace_editor', 'code'] as const; @@ -158,14 +153,6 @@ export const ELEMENT_EXTRACTION_CONFIG = { export type ElementExtractionConfig = typeof ELEMENT_EXTRACTION_CONFIG; export type RawElementData = NonNullable>; -export type VisibleOverlayExtractionConfig = { - interactiveContentSelector: string; - limits: typeof HTML_EXTRACTION_LIMITS; - overlaySelectors: readonly string[]; - overlaySemanticSelector: string; - visibilityLimits: typeof HTML_VISIBILITY_LIMITS; - geometryFallback?: boolean; -}; export type ComponentScopeExtractionConfig = { eidxAttr: string; interactiveControlSelector: string; @@ -449,80 +436,6 @@ export function getElementDataExtractorSource(): string { return extractElementData.toString(); } -export function extractVisibleOverlayHtml(config: VisibleOverlayExtractionConfig): string { - function isVisible(element: Element): boolean { - const html = element as HTMLElement; - const style = window.getComputedStyle(html); - const rect = html.getBoundingClientRect(); - if (rect.width === 0 && rect.height === 0) return false; - if (style.display === 'none' || style.visibility === 'hidden') return false; - if (Number.parseFloat(style.opacity || '1') < config.visibilityLimits.minOpacity) return false; - return true; - } - - function getUsefulContent(element: Element): { interactiveCount: number; text: string } { - const text = (element.textContent || '').replace(/\s+/g, ' ').trim(); - const interactiveCount = element.querySelectorAll(config.interactiveContentSelector).length; - return { interactiveCount, text }; - } - - function isLikelyFloatingOverlay(element: Element): boolean { - const html = element as HTMLElement; - const style = window.getComputedStyle(html); - const rect = html.getBoundingClientRect(); - const zIndex = Number.parseInt(style.zIndex || '0', 10); - const isFloating = style.position === 'fixed' || style.position === 'absolute' || style.position === 'sticky' || zIndex > 0; - if (!isFloating) return false; - if (rect.width < config.visibilityLimits.minOverlayWidth || rect.height < config.visibilityLimits.minOverlayHeight) return false; - if (rect.bottom < 0 || rect.right < 0 || rect.top > window.innerHeight || rect.left > window.innerWidth) return false; - if (rect.width >= window.innerWidth * config.visibilityLimits.maxViewportOverlayRatio && rect.height >= window.innerHeight * config.visibilityLimits.maxViewportOverlayRatio) return false; - const { interactiveCount, text } = getUsefulContent(element); - return interactiveCount > 0 || text.length > 0; - } - - function isFloatingOverlay(element: Element): boolean { - const style = window.getComputedStyle(element as HTMLElement); - return style.position === 'fixed' || style.position === 'absolute' || Number.parseInt(style.zIndex || '0', 10) > 0; - } - - const seen = new Set(); - const collected: Element[] = []; - for (const selector of config.overlaySelectors) { - for (const element of Array.from(document.querySelectorAll(selector))) { - if (seen.has(element)) continue; - seen.add(element); - if (!isVisible(element)) continue; - if (!element.matches(config.overlaySemanticSelector) && !isFloatingOverlay(element)) continue; - const { interactiveCount, text } = getUsefulContent(element); - if (interactiveCount === 0 && text.length === 0) continue; - collected.push(element); - } - } - - const overlays = collected.filter((element) => !collected.some((other) => other !== element && element.contains(other))).map((element) => (element as HTMLElement).outerHTML.slice(0, config.limits.overlayHtmlLength)); - - if (overlays.length === 0 && config.geometryFallback !== false) { - const floatingCandidates = Array.from(document.body.querySelectorAll('*')) - .filter((element) => !seen.has(element) && isVisible(element) && isLikelyFloatingOverlay(element)) - .sort((left, right) => { - const leftStyle = window.getComputedStyle(left as HTMLElement); - const rightStyle = window.getComputedStyle(right as HTMLElement); - const leftZ = Number.parseInt(leftStyle.zIndex || '0', 10) || 0; - const rightZ = Number.parseInt(rightStyle.zIndex || '0', 10) || 0; - if (leftZ !== rightZ) return rightZ - leftZ; - const leftRect = (left as HTMLElement).getBoundingClientRect(); - const rightRect = (right as HTMLElement).getBoundingClientRect(); - return leftRect.width * leftRect.height - rightRect.width * rightRect.height; - }); - - for (const element of floatingCandidates.slice(0, config.limits.maxOverlayCount)) { - overlays.push((element as HTMLElement).outerHTML.slice(0, config.limits.overlayHtmlLength)); - } - } - - return overlays.slice(0, config.limits.maxOverlayCount).join('\n\n--- overlay ---\n\n'); -} - export function extractComponentScopeHtml(eidx: string, config: ComponentScopeExtractionConfig): string { const element = document.querySelector(`[${config.eidxAttr}="${eidx}"]`); if (!element) return ''; @@ -544,10 +457,6 @@ export function extractComponentScopeHtml(eidx: string, config: ComponentScopeEx return ''; } -export function getVisibleOverlayHtmlExtractorSource(): string { - return extractVisibleOverlayHtml.toString(); -} - export function getComponentScopeHtmlExtractorSource(): string { return extractComponentScopeHtml.toString(); } diff --git a/src/utils/overlay.ts b/src/utils/overlay.ts index df95ec5e..cb370e0b 100644 --- a/src/utils/overlay.ts +++ b/src/utils/overlay.ts @@ -1,12 +1,6 @@ import { detectFocusArea } from './aria.js'; import { type HtmlDiffPart, pathToXPath } from './html-diff.js'; -import { HTML_EXTRACTION_LIMITS, HTML_SELECTORS, HTML_VISIBILITY_LIMITS, type VisibleOverlayExtractionConfig, extractHeadings } from './html.js'; - -export const OVERLAY_SELECTORS = { - semanticOverlays: ['[role="dialog"]', '[role="listbox"]', '[role="menu"]', '[role="tooltip"]:not([style*="display: none"]):not([style*="visibility: hidden"])', '[class*="modal"]', '[class*="dialog"]', '[class*="overlay"]', '[class*="popup"]', '[class*="drawer"]', '[class*="lightbox"]'], - modalOverlays: ['[role="dialog"]', '[role="alertdialog"]', '[aria-modal="true"]', '[class*="modal"]', '[class*="dialog"]', '[class*="overlay"]', '[class*="popup"]', '[class*="drawer"]', '[class*="lightbox"]'], - overlaySemanticSelector: '[role="dialog"], [role="alertdialog"], [aria-modal="true"], [role="listbox"], [role="menu"], [role="tooltip"]', -} as const; +import { extractHeadings } from './html.js'; export type OverlayType = 'dialog' | 'modal' | 'drawer' | 'region'; export type OverlayData = { type?: OverlayType | null; name?: string | null; root?: string | null }; @@ -30,10 +24,6 @@ export class Overlay { return this.type !== null; } - static fromHtml(html: string): Overlay { - return new Overlay({ type: 'modal', name: Overlay.nameFromHtml(html) }); - } - static fromSubRoot(subRoot: AppearedSubRoot, verdict: RegionVerdict): Overlay { let type: OverlayType = 'region'; if (verdict.overlays) { @@ -49,23 +39,11 @@ export class Overlay { return new Overlay(detectFocusArea(snapshot)); } - static resolve(data: { overlayHtml?: string; overlay?: OverlayData | null; ariaSnapshot?: string | null }): Overlay { - if (data.overlayHtml) return Overlay.fromHtml(data.overlayHtml); + static resolve(data: { overlay?: OverlayData | null; ariaSnapshot?: string | null }): Overlay { if (data.overlay) return new Overlay(data.overlay); return Overlay.fromAria(data.ariaSnapshot ?? null); } - static captureConfig(): VisibleOverlayExtractionConfig { - return { - interactiveContentSelector: HTML_SELECTORS.interactiveContent, - limits: HTML_EXTRACTION_LIMITS, - overlaySelectors: OVERLAY_SELECTORS.modalOverlays, - overlaySemanticSelector: OVERLAY_SELECTORS.overlaySemanticSelector, - visibilityLimits: HTML_VISIBILITY_LIMITS, - geometryFallback: false, - }; - } - private static nameFromHtml(html: string): string | null { const headings = extractHeadings(html); return [headings.h1, headings.h2, headings.h3, headings.h4].filter(Boolean).join(' ') || null; diff --git a/tests/unit/overlay-detection.test.ts b/tests/unit/overlay-detection.test.ts index c26437de..837b66d0 100644 --- a/tests/unit/overlay-detection.test.ts +++ b/tests/unit/overlay-detection.test.ts @@ -1,140 +1,11 @@ import 'parse5'; -import { JSDOM } from 'jsdom'; import { describe, expect, it } from 'vitest'; import { ActionResult } from '../../src/action-result.ts'; import { htmlDiff } from '../../src/utils/html-diff.ts'; -import { HTML_EXTRACTION_LIMITS, HTML_SELECTORS, HTML_VISIBILITY_LIMITS, type VisibleOverlayExtractionConfig, extractVisibleOverlayHtml } from '../../src/utils/html.ts'; -import { OVERLAY_SELECTORS, Overlay, type RegionCoverageSamples, classifyRegionCoverage, findAppearedSubRoot, getRegionCoverageProbeSource } from '../../src/utils/overlay.ts'; - -function overlayConfig(overrides: Partial = {}): VisibleOverlayExtractionConfig { - return { - interactiveContentSelector: HTML_SELECTORS.interactiveContent, - limits: HTML_EXTRACTION_LIMITS, - overlaySelectors: OVERLAY_SELECTORS.modalOverlays, - overlaySemanticSelector: OVERLAY_SELECTORS.overlaySemanticSelector, - visibilityLimits: HTML_VISIBILITY_LIMITS, - ...overrides, - }; -} - -function withDom(html: string, run: () => void) { - const dom = new JSDOM(html); - const previousWindow = globalThis.window; - const previousDocument = globalThis.document; - (globalThis as any).window = dom.window; - (globalThis as any).document = dom.window.document; - (dom.window.Element.prototype as any).getBoundingClientRect = () => ({ width: 480, height: 320, top: 40, left: 40, bottom: 360, right: 520, x: 40, y: 40, toJSON: () => ({}) }); - try { - run(); - } finally { - (globalThis as any).window = previousWindow; - (globalThis as any).document = previousDocument; - } -} - -describe('extractVisibleOverlayHtml', () => { - it('keeps only the innermost overlay when wrapper and dialog are both floating', () => { - withDom( - ` - -
-
-

Copy report

- -
-
- - `, - () => { - const html = extractVisibleOverlayHtml(overlayConfig()); - expect(html).toContain('nebula-modal-dialog'); - expect(html).not.toContain('nebula-modal-root'); - } - ); - }); - - it('collects the floating wrapper when nested elements only carry the class token', () => { - withDom( - ` - -
-
-
-

Copy report

- -
-
-
- - `, - () => { - const html = extractVisibleOverlayHtml(overlayConfig()); - expect(html).toContain('Copy report'); - expect(html.split('--- overlay ---')).toHaveLength(1); - } - ); - }); - - it('ignores a class token on a sticky header', () => { - withDom( - ` - -
- -

Reports

-
- - `, - () => { - expect(extractVisibleOverlayHtml(overlayConfig({ geometryFallback: false }))).toBe(''); - } - ); - }); - - it('trusts semantic markup without requiring floating geometry', () => { - withDom( - ` - -
-

Confirm launch

- -
- - `, - () => { - const html = extractVisibleOverlayHtml(overlayConfig({ geometryFallback: false })); - expect(html).toContain('Confirm launch'); - } - ); - }); - - it('skips the geometry fallback at capture and keeps it for driller', () => { - const markup = ` - -
Saved
- - `; - withDom(markup, () => { - expect(extractVisibleOverlayHtml(overlayConfig({ geometryFallback: false }))).toBe(''); - }); - withDom(markup, () => { - expect(extractVisibleOverlayHtml(overlayConfig())).toContain('toast-panel'); - }); - }); -}); +import { Overlay, type RegionCoverageSamples, classifyRegionCoverage, findAppearedSubRoot, getRegionCoverageProbeSource } from '../../src/utils/overlay.ts'; describe('ActionResult overlay', () => { - it('derives the modal descriptor from captured overlay html', () => { - const result = new ActionResult({ - url: '/', - overlayHtml: '

Copy report

to Nebula space

', - }); - expect(result.overlay.detected).toBe(true); - expect(result.overlay.type).toBe('modal'); - expect(result.overlay.name).toBe('Copy report to Nebula space'); - }); - - it('falls back to the aria snapshot when no overlay html was captured', () => { + it('falls back to the aria snapshot when no overlay was stored', () => { const result = new ActionResult({ url: '/', ariaSnapshot: '- dialog "Delete confirmation"' }); expect(result.overlay.detected).toBe(true); expect(result.overlay.type).toBe('dialog'); @@ -149,11 +20,14 @@ describe('ActionResult overlay', () => { }); describe('Overlay', () => { - it('resolves captured html first, stored descriptor second, aria last', () => { + it('resolve prefers stored overlay data over aria', () => { const aria = '- dialog "From aria"'; - expect(Overlay.resolve({ overlayHtml: '

From html

', overlay: { type: 'modal', name: 'Stored' }, ariaSnapshot: aria }).name).toBe('From html'); - expect(Overlay.resolve({ overlay: { type: 'modal', name: 'Stored' }, ariaSnapshot: aria }).name).toBe('Stored'); - expect(Overlay.resolve({ ariaSnapshot: aria }).name).toBe('From aria'); + const overlay = Overlay.resolve({ overlay: { type: 'modal', name: 'Stored' }, ariaSnapshot: aria }); + expect(overlay.name).toBe('Stored'); + }); + + it('resolve falls back to aria detection', () => { + expect(Overlay.resolve({ ariaSnapshot: '- dialog "From aria"' }).detected).toBe(true); }); it('rehydrates from a plain persisted descriptor', () => { From 07c4cd8a655477d559768d99bf4757498c1abd77 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 14:26:45 +0300 Subject: [PATCH 13/27] docs: changelog for region-of-interest states --- CHANGELOG.md | 40 +++++++++++++++++++ .../plans/2026-08-29-region-states.md | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 506853ae..c075968a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,45 @@ # Changelog +## 2026-08-29 + +### Changes + +- State Manager: Drawers, side panels and swapped-in subviews are now recognised as pages in their + own right. Until now only a modal that announced itself as a dialog counted as a state; a panel + built as a plain positioned element, or a wizard step that replaced half the screen without + changing the URL, was invisible — the agent kept aiming at the elements behind it, and a test that + opened and closed the same panel over and over looked like it was standing still. A large area + appearing on the page is now detected by comparing the page before and after the action and + measuring whether it covers what is behind it, so opening one is recorded as a move to a new state + and closing it as a move back. +- Action: The result of a click that opens a panel now leads with the panel. A large change used to + be written off as a whole-page redraw and replaced with a placeholder saying how many characters + were dropped, which threw away the one thing worth reading. The result now names what opened and + where it lives, and carries that area's markup instead of the placeholder. +- [Tester] Knows to keep working inside the area that just opened. For a panel that covers the page, + it is told the exact container to scope its locators to. For one that appears inline, it is told + the scenario most likely continues there while the rest of the page stays available — the stricter + "nothing outside is clickable" wording is reserved for areas actually measured as covering. +- [Pilot] Tells a covering modal apart from an inline area. The state summary now shows the + container for a modal and a separate `region:` line for an area that appeared in place, so Pilot + can steer a stuck Tester into a subview instead of assuming a dialog is blocking it. +- [Researcher] Describes drawers and subviews, not just dialogs. The extra pass that documents what + a modal contains now runs for any named area that opens, and its notes are still filed under the + page they belong to, so a panel that is open when a page is first analyzed no longer splits that + page's UI map in two. +- Experience Tracker: Steps learned while a panel was open are now scoped to it. Experience files + written for such a state record the panel's container as `root:` in their frontmatter, and are + loaded only while a matching area is open — so panel-specific recipes stop being offered as advice + on the plain page. Existing files without the key behave exactly as before. +- [Driller] Reads a nested popup or menu from what the click already reported instead of asking the + browser a second time. +- Overlay detection is now one mechanism instead of three. The old path recognised overlays by + looking for class names containing words like "modal" or "drawer", which only ever worked on sites + that happened to name their CSS that way; it has been removed in favour of accessibility roles + plus the page-comparison and geometry check above. One consequence: an overlay already on screen + at the very first capture that carries no accessibility role is no longer detected until the next + action. + ## 2026-08-28 ### Changes diff --git a/docs/superpowers/plans/2026-08-29-region-states.md b/docs/superpowers/plans/2026-08-29-region-states.md index 1ecdad41..ffe3e29c 100644 --- a/docs/superpowers/plans/2026-08-29-region-states.md +++ b/docs/superpowers/plans/2026-08-29-region-states.md @@ -1263,7 +1263,7 @@ git commit -m "refactor: remove selector-based overlay detection; overlay.ts is **Files:** - Modify: `CHANGELOG.md` (via the `/changelog` skill) -- [ ] **Step 1: Full verification** +- [x] **Step 1: Full verification** ```bash bun run format From 1422f4fbe30d5c79618c06b7f09cbdd9bb9f063a Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 14:30:43 +0300 Subject: [PATCH 14/27] docs: complete region-of-interest plan checklist --- docs/superpowers/plans/2026-08-29-region-states.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/plans/2026-08-29-region-states.md b/docs/superpowers/plans/2026-08-29-region-states.md index ffe3e29c..5960f2eb 100644 --- a/docs/superpowers/plans/2026-08-29-region-states.md +++ b/docs/superpowers/plans/2026-08-29-region-states.md @@ -1274,11 +1274,11 @@ bun test tests/integration/ Expected: everything green. Fix regressions before proceeding; do not skip failing tests. -- [ ] **Step 2: Dedup pass** +- [x] **Step 2: Dedup pass** Run the code-duplication-detector agent over the changed files (house rule after major changes). Apply only findings that touch code introduced by this plan. -- [ ] **Step 3: Changelog** +- [x] **Step 3: Changelog** Invoke the `/changelog` skill to add the entry for this feature, then commit: @@ -1287,6 +1287,6 @@ git add CHANGELOG.md git commit -m "docs: changelog for region-of-interest states" ``` -- [ ] **Step 4: Report** +- [x] **Step 4: Report** Report to the user: what was built, what was deleted (the spec's "Removed code" table), test results, and that end-to-end validation against a real app is available via the local `regression:*` bunosh commands — which only the user decides to run. Never trigger the regression CI workflow. From a6bcce95329295364924c90d6f788dff3765a6d8 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 14:36:46 +0300 Subject: [PATCH 15/27] fix: invalidate memoized diff when html is reassigned --- src/action-result.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/action-result.ts b/src/action-result.ts index 454eedaf..2ca5c771 100644 --- a/src/action-result.ts +++ b/src/action-result.ts @@ -170,6 +170,7 @@ export class ActionResult implements ActionResultData { set html(value: string) { this._html = value; this.snapshotCache.clear(); + this._diffCache = null; } get screenshot(): Buffer | undefined { From 724c6b0d1d766de468526c5fdee95695a965a632 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 14:51:44 +0300 Subject: [PATCH 16/27] refactor: Overlay value object plus OverlayPage detector as the overlay API Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01TkmQDF4DkphEYoy71twQNx --- .../specs/2026-08-29-region-states-design.md | 42 ++-- src/action-result.ts | 9 +- src/action.ts | 28 +-- src/utils/overlay.ts | 149 +++++++++----- tests/unit/action-result-diff.test.ts | 3 +- tests/unit/overlay-detection.test.ts | 193 +++++++++--------- 6 files changed, 225 insertions(+), 199 deletions(-) diff --git a/docs/superpowers/specs/2026-08-29-region-states-design.md b/docs/superpowers/specs/2026-08-29-region-states-design.md index 7295dadf..9c82754a 100644 --- a/docs/superpowers/specs/2026-08-29-region-states-design.md +++ b/docs/superpowers/specs/2026-08-29-region-states-design.md @@ -59,27 +59,31 @@ exactly **two** detection signals, both general: The selector-heuristic path is **deleted entirely** (see "Removed code"). No third approach, no class-name patterns, no priority chain. -The pipeline, run after every action inside `Action.capturePageState`, before the (sync) -`stateManager.updateState` call — `Action` orchestrates (it is the only browser mover), every -decision function lives in `overlay.ts`: +`overlay.ts` exposes exactly two classes: `Overlay`, the immutable value describing what is open, +and `OverlayPage`, which wraps the live page and owns detection. The single public entry point is +`new OverlayPage(page).detectRegion(diffParts)`; every lower-level step (subroot picking, the +browser probe, coverage classification) is a private member. The pipeline runs after every action +inside `Action.capturePageState`, before the (sync) `stateManager.updateState` call — `Action` +only hands the page over (it is the only browser mover): ``` capture html/aria └─ same URL, not iframe, html changed, no ARIA overlay already detected └─ diff vs previous state (parse5, memoized — shared with toToolResult) - └─ findAppearedSubRoot: appeared subtree ≥ 10K chars (overlay.ts) - └─ probeRegionCoverage in page.evaluate (overlay.ts source) - └─ classifyRegionCoverage (overlay.ts) - ├─ overlays the page → Overlay 'modal' | 'drawer' - └─ inline → Overlay 'region' + └─ OverlayPage.detectRegion(parts) + ├─ appeared subtree ≥ 10K chars (private) + ├─ coverage probe via page.evaluate (private) + └─ coverage classification (private) + ├─ overlays the page → Overlay 'modal' | 'drawer' + └─ inline → Overlay 'region' ``` Detection is 100% structural — size threshold, diff paths, geometry. No AI in the path. AI enters only downstream: `researchOverlay` describes the region, Tester/Pilot decide what to do in it. -### 1. Appeared-subroot detection (`overlay.ts`, over `html-diff.ts` parts) +### 1. Appeared-subroot detection (`OverlayPage`, over `html-diff.ts` parts) -`findAppearedSubRoot(parts: HtmlDiffPart[])` returns the largest part that contains an appeared +The first private step of `detectRegion` picks the largest part that contains an appeared element (`ELEMENT:` line in `part.added`) and whose minified `subtree` is ≥ `SUBROOT_MIN_HTML` (10 000 chars, unexported const — no config knob). `html-diff.ts` stays a generic diff engine; it newly exports `pathToXPath` so overlay.ts can convert appeared-element paths. @@ -96,17 +100,19 @@ both: When `container` degrades to `body` (top-level appended node — the common portal case), the `elementXPath` doubles as the root selector. -### 2. Coverage verification (`overlay.ts`) +### 2. Coverage verification (`OverlayPage`) Split into dumb browser-side collection and a pure classifier, because jsdom has no layout and -only the pure half can be unit-tested. Both halves live in `overlay.ts`: +only the pure half can be unit-tested. Both are private members of `OverlayPage`; tests reach +them through `detectRegion` with a fake page returning canned samples: -- **Browser probe** (`probeRegionCoverage`, shipped as a source string like the existing - extractor pattern): resolves the element by XPath, collects raw samples — bounding rect, +- **Browser probe** (a module-private plain function shipped as a source string into + `page.evaluate` — it must stay a plain self-contained function so `toString()` reconstruction + works in the browser): resolves the element by XPath, collects raw samples — bounding rect, viewport size, computed position and z-index, `elementFromPoint` hits at sample points **outside** the region's rect (classified as `inside` / `blocked` / `page`), sibling `inert`/`aria-hidden` flags, body scroll lock. -- **Pure classifier** (`classifyRegionCoverage(samples)`): returns `{ overlays, coverage }`. +- **Pure classifier**: turns the samples into `{ overlays, coverage }`. Overlaying = coverage ≥ 0.8, or siblings inerted, or a floating element whose outside sample points are all blocked by a scrim rather than landing on page content. @@ -127,7 +133,11 @@ Tester refuse legitimate navigation. `hasDialogAppeared`) keeps its semantics. - `get present()` — any region, inline included. New consumers that want "an area of interest exists" use this. -- `Overlay.fromSubRoot(subRoot, verdict)` — verdict `overlays: true` with coverage ≥ 0.8 → +- `html`: the region's minified subtree, carried on the overlay itself — `toToolResult` renders + it as the single diff part instead of a collapsed dump. +- `describe()` — the one-line human/model-facing summary + (`drawer "Edit User" opened, scope: aside.panel`), used for `pageDiff.areaOfInterest`. +- `OverlayPage.detectRegion` builds the Overlay: verdict `overlays: true` with coverage ≥ 0.8 → `modal`; overlaying with partial coverage → `drawer`; otherwise `region`. - `Overlay.resolve` simplifies to two sources: stored `overlay` data, else `fromAria`. diff --git a/src/action-result.ts b/src/action-result.ts index 2ca5c771..e1d02f3d 100644 --- a/src/action-result.ts +++ b/src/action-result.ts @@ -89,7 +89,6 @@ export class ActionResult implements ActionResultData { public links: Link[] = []; public verifications?: Record; public overlay: Overlay = new Overlay(); - public regionSubtree: string | undefined = undefined; private _diffCache: { previousId: number | undefined; diff: Diff } | null = null; constructor(data: ActionResultData) { @@ -546,14 +545,12 @@ export class ActionResult implements ActionResultData { } if (this.overlay.present && !previousState.overlay.present) { - let area = `${this.overlay.type} "${this.overlay.name || 'unnamed'}" opened`; - if (this.overlay.root) area += `, scope: ${this.overlay.root}`; - pageDiff.areaOfInterest = area; + pageDiff.areaOfInterest = this.overlay.describe(); } - if (pageDiff.areaOfInterest && this.regionSubtree && this.overlay.root) { + if (pageDiff.areaOfInterest && this.overlay.html && this.overlay.root) { const htmlConfig = ConfigParser.getInstance().getConfig().html; - let subtree = await minifyHtml(htmlCombinedSnapshot(this.regionSubtree, htmlConfig?.combined)); + let subtree = await minifyHtml(htmlCombinedSnapshot(this.overlay.html, htmlConfig?.combined)); if (subtree.length > HTML_PART_SUBTREE_BUDGET) { subtree = `${subtree.slice(0, HTML_PART_SUBTREE_BUDGET)}...`; } diff --git a/src/action.ts b/src/action.ts index 58f9e5ed..90bb1b1d 100644 --- a/src/action.ts +++ b/src/action.ts @@ -13,7 +13,7 @@ import type { StateManager } from './state-manager.js'; import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts'; import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js'; import { createDebug, setStepSpanParent, tag } from './utils/logger.js'; -import { Overlay, type RegionCoverageSamples, classifyRegionCoverage, findAppearedSubRoot, getRegionCoverageProbeSource } from './utils/overlay.js'; +import { OverlayPage } from './utils/overlay.js'; import { sleep, waitForPageReadiness } from './utils/page-readiness.ts'; import { safeFilename } from './utils/strings.ts'; import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts'; @@ -202,29 +202,9 @@ class Action { if (!result.isSameUrl({ url: previous.url })) return; const diff = await result.diff(previous); - const subRoot = findAppearedSubRoot(diff.htmlParts); - if (!subRoot) return; - - const samples = await this.probeRegion(subRoot.elementXPath); - const verdict = classifyRegionCoverage(samples); - result.overlay = Overlay.fromSubRoot(subRoot, verdict); - result.regionSubtree = subRoot.subtree; - debugLog(`Region of interest: ${result.overlay.type} "${result.overlay.name}" root=${result.overlay.root} coverage=${verdict.coverage.toFixed(2)}`); - } - - private async probeRegion(xpath: string): Promise { - return this.playwrightHelper.page - .evaluate( - ({ probeSource, config }: { probeSource: string; config: any }) => { - const probe = new Function(`return ${probeSource}`)() as (config: any) => any; - return probe(config); - }, - { probeSource: getRegionCoverageProbeSource(), config: { xpath } } - ) - .catch((err: Error) => { - debugLog('Region coverage probe failed:', err.message); - return null; - }); + const overlay = await new OverlayPage(this.playwrightHelper.page).detectRegion(diff.htmlParts); + if (!overlay) return; + result.overlay = overlay; } private async captureMainDocumentStatus(): Promise { diff --git a/src/utils/overlay.ts b/src/utils/overlay.ts index cb370e0b..f53f22e4 100644 --- a/src/utils/overlay.ts +++ b/src/utils/overlay.ts @@ -1,19 +1,24 @@ import { detectFocusArea } from './aria.js'; import { type HtmlDiffPart, pathToXPath } from './html-diff.js'; import { extractHeadings } from './html.js'; +import { createDebug } from './logger.js'; + +const debugLog = createDebug('explorbot:overlay'); export type OverlayType = 'dialog' | 'modal' | 'drawer' | 'region'; -export type OverlayData = { type?: OverlayType | null; name?: string | null; root?: string | null }; +export type OverlayData = { type?: OverlayType | null; name?: string | null; root?: string | null; html?: string | null }; export class Overlay { readonly type: OverlayType | null; readonly name: string | null; readonly root: string | null; + readonly html: string | null; constructor(data: OverlayData = {}) { this.type = data.type ?? null; this.name = data.name ?? null; this.root = data.root ?? null; + this.html = data.html ?? null; } get detected(): boolean { @@ -24,15 +29,11 @@ export class Overlay { return this.type !== null; } - static fromSubRoot(subRoot: AppearedSubRoot, verdict: RegionVerdict): Overlay { - let type: OverlayType = 'region'; - if (verdict.overlays) { - type = 'drawer'; - if (verdict.coverage >= FULL_COVERAGE_RATIO) type = 'modal'; - } - let root = subRoot.container; - if (root === 'body') root = subRoot.elementXPath; - return new Overlay({ type, name: Overlay.nameFromHtml(subRoot.subtree), root }); + describe(): string { + if (!this.present) return ''; + let text = `${this.type} "${this.name || 'unnamed'}" opened`; + if (this.root) text += `, scope: ${this.root}`; + return text; } static fromAria(snapshot: string | null): Overlay { @@ -43,57 +44,99 @@ export class Overlay { if (data.overlay) return new Overlay(data.overlay); return Overlay.fromAria(data.ariaSnapshot ?? null); } +} - private static nameFromHtml(html: string): string | null { - const headings = extractHeadings(html); - return [headings.h1, headings.h2, headings.h3, headings.h4].filter(Boolean).join(' ') || null; +export class OverlayPage { + constructor(private page: { evaluate(fn: any, arg: any): Promise } | null) {} + + async detectRegion(parts: HtmlDiffPart[]): Promise { + const subRoot = this.appearedSubRoot(parts); + if (!subRoot) return null; + const samples = await this.probe(subRoot.elementXPath); + const verdict = this.classify(samples); + const overlay = this.toOverlay(subRoot, verdict); + debugLog(`Region detected: ${overlay.describe()} coverage=${verdict.coverage.toFixed(2)}`); + return overlay; } -} -const SUBROOT_MIN_HTML = 10_000; -const FULL_COVERAGE_RATIO = 0.8; + private appearedSubRoot(parts: HtmlDiffPart[]): AppearedSubRoot | null { + let best: AppearedSubRoot | null = null; + for (const part of parts) { + const appeared = part.added.find((line) => line.startsWith('ELEMENT:')); + if (!appeared) continue; + if (part.subtree.length < SUBROOT_MIN_HTML) continue; + if (best && part.subtree.length <= best.size) continue; + best = { + container: part.container, + elementXPath: pathToXPath(appeared.slice('ELEMENT:'.length)), + subtree: part.subtree, + size: part.subtree.length, + }; + } + return best; + } -export function findAppearedSubRoot(parts: HtmlDiffPart[]): AppearedSubRoot | null { - let best: AppearedSubRoot | null = null; - for (const part of parts) { - const appeared = part.added.find((line) => line.startsWith('ELEMENT:')); - if (!appeared) continue; - if (part.subtree.length < SUBROOT_MIN_HTML) continue; - if (best && part.subtree.length <= best.size) continue; - best = { - container: part.container, - elementXPath: pathToXPath(appeared.slice('ELEMENT:'.length)), - subtree: part.subtree, - size: part.subtree.length, - }; + private async probe(xpath: string): Promise { + if (!this.page) return null; + return this.page + .evaluate( + ({ probeSource, config }: { probeSource: string; config: any }) => { + const probe = new Function(`return ${probeSource}`)() as (config: any) => any; + return probe(config); + }, + { probeSource: probeRegionCoverage.toString(), config: { xpath } } + ) + .catch((err: Error) => { + debugLog('Region coverage probe failed:', err.message); + return null; + }); } - return best; -} -export function classifyRegionCoverage(samples: RegionCoverageSamples | null): RegionVerdict { - if (!samples?.found) return { overlays: false, coverage: 0 }; - const viewportArea = samples.viewport.width * samples.viewport.height; - if (!viewportArea) return { overlays: false, coverage: 0 }; + private classify(samples: RegionCoverageSamples | null): RegionVerdict { + if (!samples?.found) return { overlays: false, coverage: 0 }; + const viewportArea = samples.viewport.width * samples.viewport.height; + if (!viewportArea) return { overlays: false, coverage: 0 }; + + const rect = samples.rect; + const visibleWidth = Math.min(rect.x + rect.width, samples.viewport.width) - Math.max(rect.x, 0); + const visibleHeight = Math.min(rect.y + rect.height, samples.viewport.height) - Math.max(rect.y, 0); + const coverage = (Math.max(0, visibleWidth) * Math.max(0, visibleHeight)) / viewportArea; - const rect = samples.rect; - const visibleWidth = Math.min(rect.x + rect.width, samples.viewport.width) - Math.max(rect.x, 0); - const visibleHeight = Math.min(rect.y + rect.height, samples.viewport.height) - Math.max(rect.y, 0); - const coverage = (Math.max(0, visibleWidth) * Math.max(0, visibleHeight)) / viewportArea; + if (coverage >= FULL_COVERAGE_RATIO) return { overlays: true, coverage }; + if (samples.siblingsInert) return { overlays: true, coverage }; - if (coverage >= FULL_COVERAGE_RATIO) return { overlays: true, coverage }; - if (samples.siblingsInert) return { overlays: true, coverage }; + const floating = samples.position === 'fixed' || samples.position === 'absolute' || samples.zIndex > 0; + if (!floating) return { overlays: false, coverage }; - const floating = samples.position === 'fixed' || samples.position === 'absolute' || samples.zIndex > 0; - if (!floating) return { overlays: false, coverage }; + const outside = samples.outsideHits; + if (outside.length > 0 && outside.every((hit) => hit !== 'page')) return { overlays: true, coverage }; + if (samples.bodyScrollLocked && outside.length > 0 && outside.filter((hit) => hit !== 'page').length * 2 >= outside.length) return { overlays: true, coverage }; - const outside = samples.outsideHits; - if (outside.length > 0 && outside.every((hit) => hit !== 'page')) return { overlays: true, coverage }; - if (samples.bodyScrollLocked && outside.length > 0 && outside.filter((hit) => hit !== 'page').length * 2 >= outside.length) return { overlays: true, coverage }; + return { overlays: false, coverage }; + } + + private toOverlay(subRoot: AppearedSubRoot, verdict: RegionVerdict): Overlay { + let type: OverlayType = 'region'; + if (verdict.overlays) { + type = 'drawer'; + if (verdict.coverage >= FULL_COVERAGE_RATIO) type = 'modal'; + } + let root = subRoot.container; + if (root === 'body') root = subRoot.elementXPath; + return new Overlay({ type, name: this.nameFrom(subRoot.subtree), root, html: subRoot.subtree }); + } - return { overlays: false, coverage }; + private nameFrom(html: string): string | null { + const headings = extractHeadings(html); + return [headings.h1, headings.h2, headings.h3, headings.h4].filter(Boolean).join(' ') || null; + } } -export function probeRegionCoverage(config: { xpath: string }): RegionCoverageSamples { +const SUBROOT_MIN_HTML = 10_000; +const FULL_COVERAGE_RATIO = 0.8; + +// Serialized via toString() into page.evaluate — must stay a plain function with no outer-scope references. +function probeRegionCoverage(config: { xpath: string }): RegionCoverageSamples { const samples: RegionCoverageSamples = { found: false, rect: { x: 0, y: 0, width: 0, height: 0 }, @@ -162,23 +205,19 @@ export function probeRegionCoverage(config: { xpath: string }): RegionCoverageSa return samples; } -export function getRegionCoverageProbeSource(): string { - return probeRegionCoverage.toString(); -} - -export interface AppearedSubRoot { +interface AppearedSubRoot { container: string; elementXPath: string; subtree: string; size: number; } -export interface RegionVerdict { +interface RegionVerdict { overlays: boolean; coverage: number; } -export interface RegionCoverageSamples { +interface RegionCoverageSamples { found: boolean; rect: { x: number; y: number; width: number; height: number }; viewport: { width: number; height: number }; diff --git a/tests/unit/action-result-diff.test.ts b/tests/unit/action-result-diff.test.ts index 235340af..4385151b 100644 --- a/tests/unit/action-result-diff.test.ts +++ b/tests/unit/action-result-diff.test.ts @@ -190,9 +190,8 @@ describe('diff memoization and areaOfInterest', () => { id: 2, url: 'https://app.example.com/users', html: '

Users

', - overlay: { type: 'drawer', name: 'Edit User', root: 'aside.panel' }, + overlay: { type: 'drawer', name: 'Edit User', root: 'aside.panel', html: '' }, }); - current.regionSubtree = ''; const result = await current.toToolResult(previous, 'aside.panel'); expect(result.pageDiff?.areaOfInterest).toBe('drawer "Edit User" opened, scope: aside.panel'); expect(result.pageDiff?.htmlParts).toHaveLength(1); diff --git a/tests/unit/overlay-detection.test.ts b/tests/unit/overlay-detection.test.ts index 837b66d0..137c7f1e 100644 --- a/tests/unit/overlay-detection.test.ts +++ b/tests/unit/overlay-detection.test.ts @@ -2,7 +2,7 @@ import 'parse5'; import { describe, expect, it } from 'vitest'; import { ActionResult } from '../../src/action-result.ts'; import { htmlDiff } from '../../src/utils/html-diff.ts'; -import { Overlay, type RegionCoverageSamples, classifyRegionCoverage, findAppearedSubRoot, getRegionCoverageProbeSource } from '../../src/utils/overlay.ts'; +import { Overlay, OverlayPage } from '../../src/utils/overlay.ts'; describe('ActionResult overlay', () => { it('falls back to the aria snapshot when no overlay was stored', () => { @@ -34,71 +34,19 @@ describe('Overlay', () => { expect(new Overlay({ type: 'modal', name: 'Copy report' }).detected).toBe(true); expect(new Overlay().detected).toBe(false); }); -}); - -describe('findAppearedSubRoot', () => { - const bigForm = Array.from({ length: 200 }, (_, i) => `
`).join(''); - const basePage = ''; - const pageWithDrawer = `

Edit User

${bigForm}
`; - - it('finds a large appeared element with container and element xpath', async () => { - const diff = await htmlDiff(basePage, pageWithDrawer); - const subRoot = findAppearedSubRoot(diff.parts); - expect(subRoot).not.toBeNull(); - expect(subRoot!.size).toBeGreaterThanOrEqual(10_000); - expect(subRoot!.container).toBe('body'); - expect(subRoot!.elementXPath).toBe('//body/div[2]'); - expect(subRoot!.subtree).toContain('Edit User'); - }); - - it('returns null when the appeared content is below the threshold', async () => { - const before = '

Users

'; - const after = '

Users

Saved successfully
'; - const diff = await htmlDiff(before, after); - expect(findAppearedSubRoot(diff.parts)).toBeNull(); - }); - it('returns null when nothing appeared', async () => { - const diff = await htmlDiff(basePage, basePage); - expect(findAppearedSubRoot(diff.parts)).toBeNull(); + it('describes an open region with its scope', () => { + const overlay = new Overlay({ type: 'drawer', name: 'Edit User', root: 'aside.panel' }); + expect(overlay.describe()).toBe('drawer "Edit User" opened, scope: aside.panel'); + expect(new Overlay().describe()).toBe(''); }); }); -describe('Overlay.fromSubRoot', () => { - const subRoot = { - container: 'aside.detail-panel', - elementXPath: '//body/div[2]', - subtree: '', - size: 12000, - }; - - it('overlaying with full coverage becomes a modal named by headings', () => { - const overlay = Overlay.fromSubRoot(subRoot, { overlays: true, coverage: 0.95 }); - expect(overlay.type).toBe('modal'); - expect(overlay.name).toBe('Edit User'); - expect(overlay.root).toBe('aside.detail-panel'); - expect(overlay.detected).toBe(true); - expect(overlay.present).toBe(true); - }); - - it('overlaying with partial coverage becomes a drawer', () => { - expect(Overlay.fromSubRoot(subRoot, { overlays: true, coverage: 0.3 }).type).toBe('drawer'); - }); - - it('inline verdict becomes a region: present but not detected', () => { - const overlay = Overlay.fromSubRoot(subRoot, { overlays: false, coverage: 0.3 }); - expect(overlay.type).toBe('region'); - expect(overlay.detected).toBe(false); - expect(overlay.present).toBe(true); - }); - - it('body container falls back to the element xpath as root', () => { - const overlay = Overlay.fromSubRoot({ ...subRoot, container: 'body' }, { overlays: true, coverage: 1 }); - expect(overlay.root).toBe('//body/div[2]'); - }); -}); +const bigForm = Array.from({ length: 200 }, (_, i) => `
`).join(''); +const basePage = ''; +const pageWithDrawer = `

Edit User

${bigForm}
`; -const samplesBase = (): RegionCoverageSamples => ({ +const overlaySamples = (overrides: Record = {}) => ({ found: true, rect: { x: 0, y: 0, width: 1280, height: 720 }, viewport: { width: 1280, height: 720 }, @@ -107,52 +55,105 @@ const samplesBase = (): RegionCoverageSamples => ({ outsideHits: [], siblingsInert: false, bodyScrollLocked: false, + ...overrides, }); -describe('classifyRegionCoverage', () => { - it('full viewport coverage is overlaying', () => { - const verdict = classifyRegionCoverage(samplesBase()); - expect(verdict.overlays).toBe(true); - expect(verdict.coverage).toBeCloseTo(1); +const pageProbing = (samples: unknown) => ({ evaluate: async () => samples }); + +describe('OverlayPage.detectRegion', () => { + it('classifies a large appeared element covering the viewport as a modal', async () => { + const diff = await htmlDiff(basePage, pageWithDrawer); + const overlay = await new OverlayPage(pageProbing(overlaySamples())).detectRegion(diff.parts); + expect(overlay).not.toBeNull(); + expect(overlay!.type).toBe('modal'); + expect(overlay!.name).toBe('Edit User'); + expect(overlay!.root).toBe('//body/div[2]'); + expect(overlay!.html).toContain('Edit User'); + expect(overlay!.detected).toBe(true); + expect(overlay!.present).toBe(true); + }); + + it('keeps a stable container selector as root when the region appears inside one', async () => { + const before = '

Users

'; + const after = `

Users

`; + const diff = await htmlDiff(before, after); + const overlay = await new OverlayPage(pageProbing(overlaySamples())).detectRegion(diff.parts); + expect(overlay).not.toBeNull(); + expect(overlay!.root).toBe('#record-editor'); + }); + + it('classifies a partial floating region with all outside points blocked as a drawer', async () => { + const diff = await htmlDiff(basePage, pageWithDrawer); + const samples = overlaySamples({ + rect: { x: 880, y: 0, width: 400, height: 720 }, + outsideHits: ['blocked', 'blocked', 'blocked', 'blocked'], + }); + const overlay = await new OverlayPage(pageProbing(samples)).detectRegion(diff.parts); + expect(overlay!.type).toBe('drawer'); + expect(overlay!.detected).toBe(true); }); - it('partial floating region with all outside points blocked is overlaying', () => { - const samples = samplesBase(); - samples.rect = { x: 880, y: 0, width: 400, height: 720 }; - samples.outsideHits = ['blocked', 'blocked', 'blocked', 'blocked']; - const verdict = classifyRegionCoverage(samples); - expect(verdict.overlays).toBe(true); - expect(verdict.coverage).toBeLessThan(0.8); + it('treats inert siblings as overlaying regardless of geometry', async () => { + const diff = await htmlDiff(basePage, pageWithDrawer); + const samples = overlaySamples({ rect: { x: 0, y: 0, width: 400, height: 400 }, siblingsInert: true }); + const overlay = await new OverlayPage(pageProbing(samples)).detectRegion(diff.parts); + expect(overlay!.detected).toBe(true); }); - it('inert siblings mean overlaying regardless of geometry', () => { - const samples = samplesBase(); - samples.rect = { x: 0, y: 0, width: 400, height: 400 }; - samples.siblingsInert = true; - expect(classifyRegionCoverage(samples).overlays).toBe(true); + it('classifies a static in-flow region with page hits outside as inline', async () => { + const diff = await htmlDiff(basePage, pageWithDrawer); + const samples = overlaySamples({ + rect: { x: 200, y: 100, width: 800, height: 500 }, + position: 'static', + zIndex: 0, + outsideHits: ['page', 'page', 'page'], + }); + const overlay = await new OverlayPage(pageProbing(samples)).detectRegion(diff.parts); + expect(overlay!.type).toBe('region'); + expect(overlay!.detected).toBe(false); + expect(overlay!.present).toBe(true); + }); + + it('degrades to an inline region when no page is available', async () => { + const diff = await htmlDiff(basePage, pageWithDrawer); + const overlay = await new OverlayPage(null).detectRegion(diff.parts); + expect(overlay!.type).toBe('region'); }); - it('static in-flow region with page hits outside is inline', () => { - const samples = samplesBase(); - samples.rect = { x: 200, y: 100, width: 800, height: 500 }; - samples.position = 'static'; - samples.zIndex = 0; - samples.outsideHits = ['page', 'page', 'page']; - expect(classifyRegionCoverage(samples).overlays).toBe(false); + it('degrades to an inline region when the probe fails', async () => { + const diff = await htmlDiff(basePage, pageWithDrawer); + const failing = { evaluate: async () => Promise.reject(new Error('page crashed')) }; + const overlay = await new OverlayPage(failing).detectRegion(diff.parts); + expect(overlay!.type).toBe('region'); + }); + + it('ships a self-contained probe to the page', async () => { + let captured: unknown = null; + const executing = { + evaluate: async (fn: any, arg: any) => { + try { + return fn(arg); + } catch (err) { + captured = err; + throw err; + } + }, + }; + const diff = await htmlDiff(basePage, pageWithDrawer); + const overlay = await new OverlayPage(executing).detectRegion(diff.parts); + expect(String(captured)).toContain('window'); + expect(overlay!.type).toBe('region'); }); - it('missing element or null samples is inline with zero coverage', () => { - expect(classifyRegionCoverage(null)).toEqual({ overlays: false, coverage: 0 }); - const samples = samplesBase(); - samples.found = false; - expect(classifyRegionCoverage(samples)).toEqual({ overlays: false, coverage: 0 }); + it('returns null when the appeared content is below the threshold', async () => { + const before = '

Users

'; + const after = '

Users

Saved successfully
'; + const diff = await htmlDiff(before, after); + expect(await new OverlayPage(null).detectRegion(diff.parts)).toBeNull(); }); -}); -describe('getRegionCoverageProbeSource', () => { - it('serializes to a reconstructible function', () => { - const source = getRegionCoverageProbeSource(); - const fn = new Function(`return ${source}`)(); - expect(typeof fn).toBe('function'); + it('returns null when nothing appeared', async () => { + const diff = await htmlDiff(basePage, basePage); + expect(await new OverlayPage(null).detectRegion(diff.parts)).toBeNull(); }); }); From f98f18d3a6ec6213cc9e21d796d31e834af6639f Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 14:58:47 +0300 Subject: [PATCH 17/27] refactor: verify region openness with a single center hit-test Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01TkmQDF4DkphEYoy71twQNx --- .../specs/2026-08-29-region-states-design.md | 42 +++--- src/utils/overlay.ts | 142 +++++------------- tests/unit/overlay-detection.test.ts | 58 +++---- 3 files changed, 84 insertions(+), 158 deletions(-) diff --git a/docs/superpowers/specs/2026-08-29-region-states-design.md b/docs/superpowers/specs/2026-08-29-region-states-design.md index 9c82754a..c9cc964c 100644 --- a/docs/superpowers/specs/2026-08-29-region-states-design.md +++ b/docs/superpowers/specs/2026-08-29-region-states-design.md @@ -100,25 +100,29 @@ both: When `container` degrades to `body` (top-level appended node — the common portal case), the `elementXPath` doubles as the root selector. -### 2. Coverage verification (`OverlayPage`) - -Split into dumb browser-side collection and a pure classifier, because jsdom has no layout and -only the pure half can be unit-tested. Both are private members of `OverlayPage`; tests reach -them through `detectRegion` with a fake page returning canned samples: - -- **Browser probe** (a module-private plain function shipped as a source string into - `page.evaluate` — it must stay a plain self-contained function so `toString()` reconstruction - works in the browser): resolves the element by XPath, collects raw samples — bounding rect, - viewport size, computed position and z-index, `elementFromPoint` hits at sample points - **outside** the region's rect (classified as `inside` / `blocked` / `page`), sibling - `inert`/`aria-hidden` flags, body scroll lock. -- **Pure classifier**: turns the samples into `{ overlays, coverage }`. - Overlaying = coverage ≥ 0.8, or siblings inerted, or a floating element whose outside sample - points are all blocked by a scrim rather than landing on page content. - -Probe failure (page navigating away, evaluate throws) degrades to `overlays: false` with a debug -log — a false "inline" verdict costs a softer prompt; a false "overlaying" verdict would make -Tester refuse legitimate navigation. +### 2. Openness verification (`OverlayPage`) + +One hit-test decides whether the appeared region is **actually open**: take the center of the +region's visible (viewport-intersected) rect, ask `document.elementFromPoint` what lives there, +and check the hit belongs to the region (`.modal` is the region; the input at its center belongs +to it → it is on top). A region whose own center resolves to a foreign element is hidden or +covered — verified not open, and **discarded** rather than classified. + +For an open region, two values collected in the same probe decide the kind: + +- computed position — floating (`fixed`/`absolute`/positive z-index) → overlaying; in-flow → + inline `region` (so soft navigation never triggers the strict focus scope); +- visible-rect coverage of the viewport — overlaying with coverage ≥ 0.8 → `modal`, else + `drawer`. + +The probe is a module-private plain function shipped as a source string into `page.evaluate` +(it must stay self-contained so `toString()` reconstruction works in the browser); tests drive +`detectRegion` with fake pages returning canned probe results. + +When the probe cannot run at all (no page, evaluate throws, element already gone) the region +degrades to inline `region` with a debug log — never to an overlay: a false "overlaying" verdict +would make Tester refuse legitimate navigation. An off-screen in-flow region (below the fold) +also stays inline instead of being discarded. ### 3. Overlay carries the region (`overlay.ts`) diff --git a/src/utils/overlay.ts b/src/utils/overlay.ts index f53f22e4..24a71862 100644 --- a/src/utils/overlay.ts +++ b/src/utils/overlay.ts @@ -52,10 +52,13 @@ export class OverlayPage { async detectRegion(parts: HtmlDiffPart[]): Promise { const subRoot = this.appearedSubRoot(parts); if (!subRoot) return null; - const samples = await this.probe(subRoot.elementXPath); - const verdict = this.classify(samples); - const overlay = this.toOverlay(subRoot, verdict); - debugLog(`Region detected: ${overlay.describe()} coverage=${verdict.coverage.toFixed(2)}`); + const probe = await this.probe(subRoot.elementXPath); + if (probe?.found && probe.onScreen && !probe.centerBelongs) { + debugLog('Appeared region is hidden or covered, ignoring it'); + return null; + } + const overlay = this.toOverlay(subRoot, probe); + debugLog(`Region detected: ${overlay.describe()}`); return overlay; } @@ -76,7 +79,7 @@ export class OverlayPage { return best; } - private async probe(xpath: string): Promise { + private async probe(xpath: string): Promise { if (!this.page) return null; return this.page .evaluate( @@ -84,42 +87,19 @@ export class OverlayPage { const probe = new Function(`return ${probeSource}`)() as (config: any) => any; return probe(config); }, - { probeSource: probeRegionCoverage.toString(), config: { xpath } } + { probeSource: inspectRegion.toString(), config: { xpath } } ) .catch((err: Error) => { - debugLog('Region coverage probe failed:', err.message); + debugLog('Region probe failed:', err.message); return null; }); } - private classify(samples: RegionCoverageSamples | null): RegionVerdict { - if (!samples?.found) return { overlays: false, coverage: 0 }; - const viewportArea = samples.viewport.width * samples.viewport.height; - if (!viewportArea) return { overlays: false, coverage: 0 }; - - const rect = samples.rect; - const visibleWidth = Math.min(rect.x + rect.width, samples.viewport.width) - Math.max(rect.x, 0); - const visibleHeight = Math.min(rect.y + rect.height, samples.viewport.height) - Math.max(rect.y, 0); - const coverage = (Math.max(0, visibleWidth) * Math.max(0, visibleHeight)) / viewportArea; - - if (coverage >= FULL_COVERAGE_RATIO) return { overlays: true, coverage }; - if (samples.siblingsInert) return { overlays: true, coverage }; - - const floating = samples.position === 'fixed' || samples.position === 'absolute' || samples.zIndex > 0; - if (!floating) return { overlays: false, coverage }; - - const outside = samples.outsideHits; - if (outside.length > 0 && outside.every((hit) => hit !== 'page')) return { overlays: true, coverage }; - if (samples.bodyScrollLocked && outside.length > 0 && outside.filter((hit) => hit !== 'page').length * 2 >= outside.length) return { overlays: true, coverage }; - - return { overlays: false, coverage }; - } - - private toOverlay(subRoot: AppearedSubRoot, verdict: RegionVerdict): Overlay { + private toOverlay(subRoot: AppearedSubRoot, probe: RegionProbe | null): Overlay { let type: OverlayType = 'region'; - if (verdict.overlays) { + if (probe?.centerBelongs && probe.floating) { type = 'drawer'; - if (verdict.coverage >= FULL_COVERAGE_RATIO) type = 'modal'; + if (probe.coverage >= FULL_COVERAGE_RATIO) type = 'modal'; } let root = subRoot.container; if (root === 'body') root = subRoot.elementXPath; @@ -136,73 +116,31 @@ const SUBROOT_MIN_HTML = 10_000; const FULL_COVERAGE_RATIO = 0.8; // Serialized via toString() into page.evaluate — must stay a plain function with no outer-scope references. -function probeRegionCoverage(config: { xpath: string }): RegionCoverageSamples { - const samples: RegionCoverageSamples = { - found: false, - rect: { x: 0, y: 0, width: 0, height: 0 }, - viewport: { width: window.innerWidth, height: window.innerHeight }, - position: 'static', - zIndex: 0, - outsideHits: [], - siblingsInert: false, - bodyScrollLocked: false, - }; +function inspectRegion(config: { xpath: string }): RegionProbe { + const probe: RegionProbe = { found: false, onScreen: false, floating: false, coverage: 0, centerBelongs: false }; const result = document.evaluate(config.xpath, document, null, 9, null); const node = result.singleNodeValue; - if (!node || node.nodeType !== 1) return samples; + if (!node || node.nodeType !== 1) return probe; const element = node as HTMLElement; const rect = element.getBoundingClientRect(); - if (rect.width === 0 && rect.height === 0) return samples; + if (rect.width === 0 && rect.height === 0) return probe; + probe.found = true; const style = window.getComputedStyle(element); - samples.found = true; - samples.rect = { x: rect.x, y: rect.y, width: rect.width, height: rect.height }; - samples.position = style.position; - samples.zIndex = Number.parseInt(style.zIndex || '0', 10) || 0; - - const bodyStyle = window.getComputedStyle(document.body); - samples.bodyScrollLocked = bodyStyle.overflow === 'hidden' || bodyStyle.overflowY === 'hidden'; - - for (const sibling of Array.from(element.parentElement?.children || [])) { - if (sibling === element) continue; - if (!sibling.hasAttribute('inert') && sibling.getAttribute('aria-hidden') !== 'true') continue; - samples.siblingsInert = true; - break; - } - - function classifyHit(hit: Element | null): 'inside' | 'blocked' | 'page' { - if (!hit) return 'page'; - if (element.contains(hit)) return 'inside'; - let current: Element | null = hit; - for (let depth = 0; current && depth < 4; depth++) { - const hitStyle = window.getComputedStyle(current as HTMLElement); - const hitZ = Number.parseInt(hitStyle.zIndex || '0', 10) || 0; - if ((hitStyle.position === 'fixed' || hitStyle.position === 'absolute') && hitZ > 0) return 'blocked'; - current = current.parentElement; - } - return 'page'; - } - - const inset = 10; - const width = window.innerWidth; - const height = window.innerHeight; - const points: Array<[number, number]> = [ - [inset, inset], - [width - inset, inset], - [inset, height - inset], - [width - inset, height - inset], - [width / 2, inset], - [width / 2, height - inset], - [inset, height / 2], - [width - inset, height / 2], - ]; - for (const [x, y] of points) { - if (x >= rect.left && x <= rect.right && y >= rect.top && y <= rect.bottom) continue; - samples.outsideHits.push(classifyHit(document.elementFromPoint(x, y))); - } - - return samples; + probe.floating = style.position === 'fixed' || style.position === 'absolute' || (Number.parseInt(style.zIndex || '0', 10) || 0) > 0; + + const left = Math.max(rect.left, 0); + const top = Math.max(rect.top, 0); + const right = Math.min(rect.right, window.innerWidth); + const bottom = Math.min(rect.bottom, window.innerHeight); + if (right <= left || bottom <= top) return probe; + probe.onScreen = true; + probe.coverage = ((right - left) * (bottom - top)) / (window.innerWidth * window.innerHeight); + + const hit = document.elementFromPoint((left + right) / 2, (top + bottom) / 2); + probe.centerBelongs = !!hit && (hit === element || element.contains(hit)); + return probe; } interface AppearedSubRoot { @@ -212,18 +150,10 @@ interface AppearedSubRoot { size: number; } -interface RegionVerdict { - overlays: boolean; - coverage: number; -} - -interface RegionCoverageSamples { +interface RegionProbe { found: boolean; - rect: { x: number; y: number; width: number; height: number }; - viewport: { width: number; height: number }; - position: string; - zIndex: number; - outsideHits: Array<'inside' | 'blocked' | 'page'>; - siblingsInert: boolean; - bodyScrollLocked: boolean; + onScreen: boolean; + floating: boolean; + coverage: number; + centerBelongs: boolean; } diff --git a/tests/unit/overlay-detection.test.ts b/tests/unit/overlay-detection.test.ts index 137c7f1e..149ddfe1 100644 --- a/tests/unit/overlay-detection.test.ts +++ b/tests/unit/overlay-detection.test.ts @@ -46,24 +46,21 @@ const bigForm = Array.from({ length: 200 }, (_, i) => `