diff --git a/CHANGELOG.md b/CHANGELOG.md index 40849d9..8290e0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -165,6 +165,14 @@ ### Changes +- [Planner] A page that reports the thing you asked for does not exist gets no test plan. A missing + record, a failed load, or a page with nothing on it still came back with a full set of invented + scenarios, every one of them written against a page that had nothing to click. The planner now + proposes nothing for such a page. +- Explore: A sub-page with nothing to test is skipped and exploration moves straight on to the next + candidate page. An error page is caught before any research or planning runs on it, and a page the + planner proposes no scenarios for is dropped as well, instead of being reported as a planning + failure and retried once per planning style. - State Manager: Drawers, side panels and swapped-in subviews are now recognised as pages in their own right. Until now only a modal that announced itself as a dialog counted as a state; a panel built as a plain positioned element, or a wizard step that replaced half the screen without diff --git a/src/ai/planner.ts b/src/ai/planner.ts index 63ed52e..f1ff2e6 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -195,10 +195,6 @@ export class Planner extends PlannerBase implements Agent { throw new Error('No tasks were created successfully'); } - if (aiResult.object.scenarios.length === 0 && !this.currentPlan) { - throw new Error('No tasks were created successfully'); - } - const defaultStartUrl = this.getDefaultStartUrl(state); const fromPlanning = aiResult.object.scenarios.map((s: any) => new Test(s.scenario, s.priority, s.expectedOutcomes, s.startUrl || defaultStartUrl, s.steps || [])); @@ -335,6 +331,7 @@ export class Planner extends PlannerBase implements Agent { Based on the page research, create ${this.MIN_TASKS}-${this.MAX_TASKS} exploratory testing scenarios. For each scenario provide specific steps and expected outcomes. + Exception: if the page reports the requested resource is missing, shows a failure state, or holds no content and no controls, return an empty scenarios list. Never invent tests for a page with nothing to exercise. diff --git a/src/commands/explore-command.ts b/src/commands/explore-command.ts index 628ab14..f974429 100644 --- a/src/commands/explore-command.ts +++ b/src/commands/explore-command.ts @@ -261,9 +261,15 @@ export class ExploreCommand extends BaseCommand { tag('info').log(`Exploring sub-page: ${pick.url} (${pick.reason})`); try { await this.explorBot.visit(pick.url); + const errorPage = getStateErrorPageError(this.explorBot.stateManager().getCurrentState()); + if (errorPage) { + tag('warning').log(`Skipping sub-page: ${errorPage.message}`); + this.failedSubPages.add(normalizeUrl(pick.url)); + continue; + } await this.runAllStyles(pick.url, undefined, mainPlan, this.completedPlans, styles); const subPlan = this.explorBot.getCurrentPlan(); - if (subPlan && !this.completedPlans.includes(subPlan)) { + if (subPlan?.tests.length && !this.completedPlans.includes(subPlan)) { this.completedPlans.push(subPlan); } knownUrls.add(normalizeUrl(pick.url)); @@ -298,6 +304,11 @@ export class ExploreCommand extends BaseCommand { if (fresh && parentPlan) opts.extend = parentPlan; if (this.dryRun) opts.noSave = true; await this.planWithRetry(feature, opts, pageUrl); + const plan = this.explorBot.getCurrentPlan(); + if (plan && plan.tests.length === 0) { + tag('warning').log('Nothing to test on this page, moving on'); + return; + } await this.runPendingTests(); this.rememberCurrentPlan(); fresh = false; diff --git a/tests/integration/planner.test.ts b/tests/integration/planner.test.ts index 3ff1af5..49f13d0 100644 --- a/tests/integration/planner.test.ts +++ b/tests/integration/planner.test.ts @@ -275,10 +275,33 @@ describe('Planner with aimock', () => { expect(mock.getRequests().length).toBe(0); }); - it('throws when AI returns empty scenarios and no current plan', async () => { + it('instructs to return no scenarios for a page with nothing to exercise', async () => { + await planner.plan(); + + const prompt = extractPromptText(mock.getLastRequest()); + expect(prompt).toContain('return an empty scenarios list'); + }); + + it('returns an empty plan when AI returns no scenarios', async () => { + mock.clearFixtures(); + mock.on({}, { content: JSON.stringify({ planName: 'Empty', scenarios: [] }) }); + + const plan = await planner.plan(); + + expect(plan.tests.length).toBe(0); + }); + + it('keeps an existing plan when a later style returns empty scenarios', async () => { + const existingPlan = new Plan('Task Board Testing'); + existingPlan.url = '/tasks/board'; + existingPlan.addTest(new Test('Create a new task via the Create Task modal', 'critical', ['Task appears'], '/tasks/board', ['Click Create'])); + planner.currentPlan = existingPlan; + mock.clearFixtures(); mock.on({}, { content: JSON.stringify({ planName: 'Empty', scenarios: [] }) }); - await expect(planner.plan()).rejects.toThrow('No tasks were created successfully'); + const plan = await planner.plan(); + + expect(plan.tests.length).toBe(1); }); });