From 1b8fb30fec2847e1cc6feebb186f896b2119de3d Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 14:34:26 +0300 Subject: [PATCH 1/4] Skip planning on error and not-found pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exploration spent a full round of planning on pages that had nothing to test. A sub-page that turned out to be an error page was researched and planned anyway, and a page that reports a missing record with a normal 200 response — no HTTP status to detect it by — came back with a set of invented scenarios written against a page with nothing to click, once per planning style. Sub-page exploration now checks the page it just visited and moves on to the next candidate when it is an error page, before any research runs. For pages that only their own copy identifies as dead, the planner is told to return no scenarios, and no scenarios for a page nothing has been planned for is reported as an error page — which lands in the same skip. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013TYaLWuuLjj5mb26NuYjoj --- CHANGELOG.md | 11 +++++++++++ src/ai/planner.ts | 5 ++++- src/commands/explore-command.ts | 6 ++++++ tests/integration/planner.test.ts | 33 +++++++++++++++++++++++++++++-- 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 506853ae..ab3bcf3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 2026-08-29 + +### Changes + +- [Planner] A page that reports the thing you asked for does not exist gets no test plan. A missing + record, a failed load, or a page with nothing on it still came back with a full set of invented + scenarios — every one of them written against a page that had nothing to click — and each planning + style spent another call inventing more. The planner now returns no scenarios for such a page. +- Explore: A sub-page that turns out to be an error page is skipped before any research or planning + runs on it, and exploration moves straight on to the next candidate page. + ## 2026-08-28 ### Changes diff --git a/src/ai/planner.ts b/src/ai/planner.ts index 3ad03107..e73fb1c8 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -10,6 +10,7 @@ import type { StateManager } from '../state-manager.js'; import { Stats } from '../stats.ts'; import { Suite } from '../suite.ts'; import { Plan, Test } from '../test-plan.ts'; +import { ErrorPageError } from '../utils/error-page.ts'; import { createDebug, tag } from '../utils/logger.js'; import { jsonToTable } from '../utils/markdown-parser.ts'; import { mdq } from '../utils/markdown-query.js'; @@ -195,7 +196,8 @@ export class Planner extends PlannerBase implements Agent { throw new Error('No tasks were created successfully'); } - if (aiResult.object.scenarios.length === 0 && !this.currentPlan) { + if (aiResult.object.scenarios.length === 0 && !this.currentPlan?.tests.length) { + if (!feature) throw new ErrorPageError(actionResult.url || state.url, actionResult.title, actionResult.httpStatus); throw new Error('No tasks were created successfully'); } @@ -335,6 +337,7 @@ export class Planner extends PlannerBase implements Agent { Based on the page research, create ${this.MIN_TASKS}-${this.MAX_TASKS} exploratory testing scenarios. For each scenario provide specific steps and expected outcomes. + Exception: if the page reports the requested resource is missing, shows a failure state, or holds no content and no controls, return an empty scenarios list. Never invent tests for a page with nothing to exercise. diff --git a/src/commands/explore-command.ts b/src/commands/explore-command.ts index bfdd80fd..199f9bfc 100644 --- a/src/commands/explore-command.ts +++ b/src/commands/explore-command.ts @@ -249,6 +249,12 @@ export class ExploreCommand extends BaseCommand { tag('info').log(`Exploring sub-page: ${pick.url} (${pick.reason})`); try { await this.explorBot.visit(pick.url); + const errorPage = getStateErrorPageError(this.explorBot.stateManager().getCurrentState()); + if (errorPage) { + tag('warning').log(`Skipping sub-page: ${errorPage.message}`); + this.failedSubPages.add(normalizeUrl(pick.url)); + continue; + } await this.runAllStyles(pick.url, undefined, mainPlan, this.completedPlans, styles); const subPlan = this.explorBot.getCurrentPlan(); if (subPlan && !this.completedPlans.includes(subPlan)) { diff --git a/tests/integration/planner.test.ts b/tests/integration/planner.test.ts index 3ff1af58..06cbb088 100644 --- a/tests/integration/planner.test.ts +++ b/tests/integration/planner.test.ts @@ -10,6 +10,7 @@ import { clearPlanRegistry, registerPlan } from '../../src/ai/planner/subpages.t import { Provider } from '../../src/ai/provider.ts'; import { ConfigParser } from '../../src/config.ts'; import { Plan, Test } from '../../src/test-plan.ts'; +import { ErrorPageError } from '../../src/utils/error-page.ts'; const UI_MAPS_DIR = join(process.cwd(), 'test-data', 'ui-maps'); @@ -275,10 +276,38 @@ describe('Planner with aimock', () => { expect(mock.getRequests().length).toBe(0); }); - it('throws when AI returns empty scenarios and no current plan', async () => { + it('instructs to return no scenarios for a page with nothing to exercise', async () => { + await planner.plan(); + + const prompt = extractPromptText(mock.getLastRequest()); + expect(prompt).toContain('return an empty scenarios list'); + }); + + it('reports an error page when AI returns empty scenarios', async () => { + mock.clearFixtures(); + mock.on({}, { content: JSON.stringify({ planName: 'Empty', scenarios: [] }) }); + + await expect(planner.plan()).rejects.toThrow(ErrorPageError); + }); + + it('throws a planning error when feature focus returns empty scenarios', async () => { mock.clearFixtures(); mock.on({}, { content: JSON.stringify({ planName: 'Empty', scenarios: [] }) }); - await expect(planner.plan()).rejects.toThrow('No tasks were created successfully'); + await expect(planner.plan('search')).rejects.toThrow('No tasks were created successfully'); + }); + + it('keeps an existing plan when a later style returns empty scenarios', async () => { + const existingPlan = new Plan('Task Board Testing'); + existingPlan.url = '/tasks/board'; + existingPlan.addTest(new Test('Create a new task via the Create Task modal', 'critical', ['Task appears'], '/tasks/board', ['Click Create'])); + planner.currentPlan = existingPlan; + + mock.clearFixtures(); + mock.on({}, { content: JSON.stringify({ planName: 'Empty', scenarios: [] }) }); + + const plan = await planner.plan(); + + expect(plan.tests.length).toBe(1); }); }); From 915c2b4276fdcbb491cfcc4ae68d4927cc7a686d Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 14:51:16 +0300 Subject: [PATCH 2/4] Drop the error-page inference from empty planner output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zero scenarios meant the model returned nothing, which is not the same as the page being dead — the research can be thin, the call can fail, the model can decline. Reporting that as an error page put a diagnosis on the page that nothing had established, and the !feature gate was only there to hide how often the guess would be wrong. Empty output is a planning failure again, as before. The prompt rule stands on its own: a page with nothing to exercise gets no scenarios, so nothing is invented for it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013TYaLWuuLjj5mb26NuYjoj --- CHANGELOG.md | 4 ++-- src/ai/planner.ts | 4 +--- tests/integration/planner.test.ts | 26 ++------------------------ 3 files changed, 5 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab3bcf3b..b9dd7eff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,8 @@ - [Planner] A page that reports the thing you asked for does not exist gets no test plan. A missing record, a failed load, or a page with nothing on it still came back with a full set of invented - scenarios — every one of them written against a page that had nothing to click — and each planning - style spent another call inventing more. The planner now returns no scenarios for such a page. + scenarios, every one of them written against a page that had nothing to click. The planner now + proposes nothing for such a page. - Explore: A sub-page that turns out to be an error page is skipped before any research or planning runs on it, and exploration moves straight on to the next candidate page. diff --git a/src/ai/planner.ts b/src/ai/planner.ts index e73fb1c8..901254fe 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -10,7 +10,6 @@ import type { StateManager } from '../state-manager.js'; import { Stats } from '../stats.ts'; import { Suite } from '../suite.ts'; import { Plan, Test } from '../test-plan.ts'; -import { ErrorPageError } from '../utils/error-page.ts'; import { createDebug, tag } from '../utils/logger.js'; import { jsonToTable } from '../utils/markdown-parser.ts'; import { mdq } from '../utils/markdown-query.js'; @@ -196,8 +195,7 @@ export class Planner extends PlannerBase implements Agent { throw new Error('No tasks were created successfully'); } - if (aiResult.object.scenarios.length === 0 && !this.currentPlan?.tests.length) { - if (!feature) throw new ErrorPageError(actionResult.url || state.url, actionResult.title, actionResult.httpStatus); + if (aiResult.object.scenarios.length === 0 && !this.currentPlan) { throw new Error('No tasks were created successfully'); } diff --git a/tests/integration/planner.test.ts b/tests/integration/planner.test.ts index 06cbb088..5d83b766 100644 --- a/tests/integration/planner.test.ts +++ b/tests/integration/planner.test.ts @@ -10,7 +10,6 @@ import { clearPlanRegistry, registerPlan } from '../../src/ai/planner/subpages.t import { Provider } from '../../src/ai/provider.ts'; import { ConfigParser } from '../../src/config.ts'; import { Plan, Test } from '../../src/test-plan.ts'; -import { ErrorPageError } from '../../src/utils/error-page.ts'; const UI_MAPS_DIR = join(process.cwd(), 'test-data', 'ui-maps'); @@ -283,31 +282,10 @@ describe('Planner with aimock', () => { expect(prompt).toContain('return an empty scenarios list'); }); - it('reports an error page when AI returns empty scenarios', async () => { + it('throws when AI returns empty scenarios and no current plan', async () => { mock.clearFixtures(); mock.on({}, { content: JSON.stringify({ planName: 'Empty', scenarios: [] }) }); - await expect(planner.plan()).rejects.toThrow(ErrorPageError); - }); - - it('throws a planning error when feature focus returns empty scenarios', async () => { - mock.clearFixtures(); - mock.on({}, { content: JSON.stringify({ planName: 'Empty', scenarios: [] }) }); - - await expect(planner.plan('search')).rejects.toThrow('No tasks were created successfully'); - }); - - it('keeps an existing plan when a later style returns empty scenarios', async () => { - const existingPlan = new Plan('Task Board Testing'); - existingPlan.url = '/tasks/board'; - existingPlan.addTest(new Test('Create a new task via the Create Task modal', 'critical', ['Task appears'], '/tasks/board', ['Click Create'])); - planner.currentPlan = existingPlan; - - mock.clearFixtures(); - mock.on({}, { content: JSON.stringify({ planName: 'Empty', scenarios: [] }) }); - - const plan = await planner.plan(); - - expect(plan.tests.length).toBe(1); + await expect(planner.plan()).rejects.toThrow('No tasks were created successfully'); }); }); From 32c559e81c8bb8b287623324c8a31852c9556f4c Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 1 Sep 2026 03:04:41 +0300 Subject: [PATCH 3/4] Treat an empty plan as a skip, not a planning failure Zero scenarios with no plan to expand threw the same generic planning error as a failed call, so exploration retried the page and then spent another two calls per remaining planning style before giving up on it. The planner now throws EmptyPlanError, which says only what happened to the output and nothing about the page. Explore skips straight to the next sub-page candidate on it, with no retry and no further styles. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01NrPgHXFYH6PzRksorVUGFC --- CHANGELOG.md | 6 ++++-- src/ai/planner.ts | 9 ++++++++- src/commands/explore-command.ts | 15 +++++++++++++-- tests/integration/planner.test.ts | 20 +++++++++++++++++--- 4 files changed, 42 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51444f29..24074369 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,10 @@ record, a failed load, or a page with nothing on it still came back with a full set of invented scenarios, every one of them written against a page that had nothing to click. The planner now proposes nothing for such a page. -- Explore: A sub-page that turns out to be an error page is skipped before any research or planning - runs on it, and exploration moves straight on to the next candidate page. +- Explore: A sub-page with nothing to test is skipped and exploration moves straight on to the next + candidate page. An error page is caught before any research or planning runs on it, and a page the + planner proposes no scenarios for is dropped as well, instead of being reported as a planning + failure and retried once per planning style. - [Provider] Groq prompt cache hits are counted again. Groq reports how much of a prompt it served from cache, but the pinned `@ai-sdk/groq` build read that number out of the response and then dropped it, so every Groq request was recorded as a full-price miss and the cache hit rate showed diff --git a/src/ai/planner.ts b/src/ai/planner.ts index f83054fc..6e14cec4 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -196,7 +196,7 @@ export class Planner extends PlannerBase implements Agent { } if (aiResult.object.scenarios.length === 0 && !this.currentPlan) { - throw new Error('No tasks were created successfully'); + throw new EmptyPlanError(actionResult.url || state.url); } const defaultStartUrl = this.getDefaultStartUrl(state); @@ -634,3 +634,10 @@ export class Planner extends PlannerBase implements Agent { `; } } + +export class EmptyPlanError extends Error { + constructor(public readonly url: string) { + super(`no scenarios proposed for ${url}`); + this.name = 'EmptyPlanError'; + } +} diff --git a/src/commands/explore-command.ts b/src/commands/explore-command.ts index 199f9bfc..8d1ce60d 100644 --- a/src/commands/explore-command.ts +++ b/src/commands/explore-command.ts @@ -1,4 +1,5 @@ import figureSet from 'figures'; +import { EmptyPlanError } from '../ai/planner.js'; import { getStyles } from '../ai/planner/styles.js'; import { outputPath } from '../config.js'; import { normalizeUrl } from '../state-manager.js'; @@ -100,7 +101,13 @@ export class ExploreCommand extends BaseCommand { } private async runFreshMode(mainUrl: string | undefined, feature: string | undefined, styles?: string[]): Promise { - await this.runAllStyles(mainUrl, feature, undefined, undefined, styles); + try { + await this.runAllStyles(mainUrl, feature, undefined, undefined, styles); + } catch (err) { + if (!(err instanceof EmptyPlanError)) throw err; + tag('warning').log(`Nothing to test here: ${err.message}`); + return; + } this.rememberCurrentPlan(); const mainPlan = this.explorBot.getCurrentPlan(); if (!mainPlan) return; @@ -263,6 +270,10 @@ export class ExploreCommand extends BaseCommand { knownUrls.add(normalizeUrl(pick.url)); } catch (err) { this.failedSubPages.add(normalizeUrl(pick.url)); + if (err instanceof EmptyPlanError) { + tag('info').log(`Skipping sub-page: ${err.message}`); + continue; + } tag('warning').log(`Sub-page exploration failed: ${err instanceof Error ? err.message : err}`); } } @@ -311,7 +322,7 @@ export class ExploreCommand extends BaseCommand { await this.explorBot.plan(feature, opts); if (this.explorBot.lastPlanError) { - if (this.explorBot.lastPlanError instanceof ErrorPageError) { + if (this.explorBot.lastPlanError instanceof ErrorPageError || this.explorBot.lastPlanError instanceof EmptyPlanError) { throw this.explorBot.lastPlanError; } tag('info').log(`Retrying planning style '${opts.style}'...`); diff --git a/tests/integration/planner.test.ts b/tests/integration/planner.test.ts index 5d83b766..c0a36a04 100644 --- a/tests/integration/planner.test.ts +++ b/tests/integration/planner.test.ts @@ -3,7 +3,7 @@ import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { createOpenAI } from '@ai-sdk/openai'; import { LLMock } from '@copilotkit/aimock'; -import { Planner } from '../../src/ai/planner.ts'; +import { EmptyPlanError, Planner } from '../../src/ai/planner.ts'; import { clearSessionDedup } from '../../src/ai/planner/session-dedup.ts'; import { clearStyleCache } from '../../src/ai/planner/styles.ts'; import { clearPlanRegistry, registerPlan } from '../../src/ai/planner/subpages.ts'; @@ -282,10 +282,24 @@ describe('Planner with aimock', () => { expect(prompt).toContain('return an empty scenarios list'); }); - it('throws when AI returns empty scenarios and no current plan', async () => { + it('signals an empty plan when AI returns no scenarios and there is no current plan', async () => { mock.clearFixtures(); mock.on({}, { content: JSON.stringify({ planName: 'Empty', scenarios: [] }) }); - await expect(planner.plan()).rejects.toThrow('No tasks were created successfully'); + await expect(planner.plan()).rejects.toThrow(EmptyPlanError); + }); + + it('keeps an existing plan when a later style returns empty scenarios', async () => { + const existingPlan = new Plan('Task Board Testing'); + existingPlan.url = '/tasks/board'; + existingPlan.addTest(new Test('Create a new task via the Create Task modal', 'critical', ['Task appears'], '/tasks/board', ['Click Create'])); + planner.currentPlan = existingPlan; + + mock.clearFixtures(); + mock.on({}, { content: JSON.stringify({ planName: 'Empty', scenarios: [] }) }); + + const plan = await planner.plan(); + + expect(plan.tests.length).toBe(1); }); }); From 64ad7e7e9cb17d0a1c6bdf5986afb15a1fe30dea Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 1 Sep 2026 03:24:01 +0300 Subject: [PATCH 4/4] Check the plan for tests instead of throwing on an empty one An empty scenario list is a value, not an exception. The planner returns the plan it built, empty or not, and explore reads its length: no tests means nothing to test here, so it stops trying further planning styles and the next sub-page candidate is picked. An empty plan is not carried into the saved plans either. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01NrPgHXFYH6PzRksorVUGFC --- src/ai/planner.ts | 11 ----------- src/commands/explore-command.ts | 22 ++++++++-------------- tests/integration/planner.test.ts | 8 +++++--- 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/src/ai/planner.ts b/src/ai/planner.ts index 6e14cec4..f1ff2e66 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -195,10 +195,6 @@ export class Planner extends PlannerBase implements Agent { throw new Error('No tasks were created successfully'); } - if (aiResult.object.scenarios.length === 0 && !this.currentPlan) { - throw new EmptyPlanError(actionResult.url || state.url); - } - const defaultStartUrl = this.getDefaultStartUrl(state); const fromPlanning = aiResult.object.scenarios.map((s: any) => new Test(s.scenario, s.priority, s.expectedOutcomes, s.startUrl || defaultStartUrl, s.steps || [])); @@ -634,10 +630,3 @@ export class Planner extends PlannerBase implements Agent { `; } } - -export class EmptyPlanError extends Error { - constructor(public readonly url: string) { - super(`no scenarios proposed for ${url}`); - this.name = 'EmptyPlanError'; - } -} diff --git a/src/commands/explore-command.ts b/src/commands/explore-command.ts index b44908be..f974429b 100644 --- a/src/commands/explore-command.ts +++ b/src/commands/explore-command.ts @@ -1,5 +1,4 @@ import figureSet from 'figures'; -import { EmptyPlanError } from '../ai/planner.js'; import { getStyles } from '../ai/planner/styles.js'; import { outputPath } from '../config.js'; import { normalizeUrl } from '../state-manager.js'; @@ -113,13 +112,7 @@ export class ExploreCommand extends BaseCommand { } private async runFreshMode(mainUrl: string | undefined, feature: string | undefined, styles?: string[]): Promise { - try { - await this.runAllStyles(mainUrl, feature, undefined, undefined, styles); - } catch (err) { - if (!(err instanceof EmptyPlanError)) throw err; - tag('warning').log(`Nothing to test here: ${err.message}`); - return; - } + await this.runAllStyles(mainUrl, feature, undefined, undefined, styles); this.rememberCurrentPlan(); const mainPlan = this.explorBot.getCurrentPlan(); if (!mainPlan) return; @@ -276,16 +269,12 @@ export class ExploreCommand extends BaseCommand { } await this.runAllStyles(pick.url, undefined, mainPlan, this.completedPlans, styles); const subPlan = this.explorBot.getCurrentPlan(); - if (subPlan && !this.completedPlans.includes(subPlan)) { + if (subPlan?.tests.length && !this.completedPlans.includes(subPlan)) { this.completedPlans.push(subPlan); } knownUrls.add(normalizeUrl(pick.url)); } catch (err) { this.failedSubPages.add(normalizeUrl(pick.url)); - if (err instanceof EmptyPlanError) { - tag('info').log(`Skipping sub-page: ${err.message}`); - continue; - } tag('warning').log(`Sub-page exploration failed: ${err instanceof Error ? err.message : err}`); } } @@ -315,6 +304,11 @@ export class ExploreCommand extends BaseCommand { if (fresh && parentPlan) opts.extend = parentPlan; if (this.dryRun) opts.noSave = true; await this.planWithRetry(feature, opts, pageUrl); + const plan = this.explorBot.getCurrentPlan(); + if (plan && plan.tests.length === 0) { + tag('warning').log('Nothing to test on this page, moving on'); + return; + } await this.runPendingTests(); this.rememberCurrentPlan(); fresh = false; @@ -334,7 +328,7 @@ export class ExploreCommand extends BaseCommand { await this.explorBot.plan(feature, opts); if (this.explorBot.lastPlanError) { - if (this.explorBot.lastPlanError instanceof ErrorPageError || this.explorBot.lastPlanError instanceof EmptyPlanError) { + if (this.explorBot.lastPlanError instanceof ErrorPageError) { throw this.explorBot.lastPlanError; } tag('info').log(`Retrying planning style '${opts.style}'...`); diff --git a/tests/integration/planner.test.ts b/tests/integration/planner.test.ts index c0a36a04..49f13d00 100644 --- a/tests/integration/planner.test.ts +++ b/tests/integration/planner.test.ts @@ -3,7 +3,7 @@ import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { createOpenAI } from '@ai-sdk/openai'; import { LLMock } from '@copilotkit/aimock'; -import { EmptyPlanError, Planner } from '../../src/ai/planner.ts'; +import { Planner } from '../../src/ai/planner.ts'; import { clearSessionDedup } from '../../src/ai/planner/session-dedup.ts'; import { clearStyleCache } from '../../src/ai/planner/styles.ts'; import { clearPlanRegistry, registerPlan } from '../../src/ai/planner/subpages.ts'; @@ -282,11 +282,13 @@ describe('Planner with aimock', () => { expect(prompt).toContain('return an empty scenarios list'); }); - it('signals an empty plan when AI returns no scenarios and there is no current plan', async () => { + it('returns an empty plan when AI returns no scenarios', async () => { mock.clearFixtures(); mock.on({}, { content: JSON.stringify({ planName: 'Empty', scenarios: [] }) }); - await expect(planner.plan()).rejects.toThrow(EmptyPlanError); + const plan = await planner.plan(); + + expect(plan.tests.length).toBe(0); }); it('keeps an existing plan when a later style returns empty scenarios', async () => {