Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,14 @@

### Changes

- [Planner] A page that reports the thing you asked for does not exist gets no test plan. A missing
record, a failed load, or a page with nothing on it still came back with a full set of invented
scenarios, every one of them written against a page that had nothing to click. The planner now
proposes nothing for such a page.
- Explore: A sub-page with nothing to test is skipped and exploration moves straight on to the next
candidate page. An error page is caught before any research or planning runs on it, and a page the
planner proposes no scenarios for is dropped as well, instead of being reported as a planning
failure and retried once per planning style.
- State Manager: Drawers, side panels and swapped-in subviews are now recognised as pages in their
own right. Until now only a modal that announced itself as a dialog counted as a state; a panel
built as a plain positioned element, or a wizard step that replaced half the screen without
Expand Down
5 changes: 1 addition & 4 deletions src/ai/planner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -195,10 +195,6 @@ export class Planner extends PlannerBase implements Agent {
throw new Error('No tasks were created successfully');
}

if (aiResult.object.scenarios.length === 0 && !this.currentPlan) {
throw new Error('No tasks were created successfully');
}

const defaultStartUrl = this.getDefaultStartUrl(state);
const fromPlanning = aiResult.object.scenarios.map((s: any) => new Test(s.scenario, s.priority, s.expectedOutcomes, s.startUrl || defaultStartUrl, s.steps || []));

Expand Down Expand Up @@ -335,6 +331,7 @@ export class Planner extends PlannerBase implements Agent {
<task>
Based on the page research, create ${this.MIN_TASKS}-${this.MAX_TASKS} exploratory testing scenarios.
For each scenario provide specific steps and expected outcomes.
Exception: if the page reports the requested resource is missing, shows a failure state, or holds no content and no controls, return an empty scenarios list. Never invent tests for a page with nothing to exercise.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

plan() still throws when scenarios is empty and there is no current plan. So this turns the intended skip into a planning error. Should an empty list be handled as a valid skip?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reworked in 64ad7e7 — the earlier EmptyPlanError is gone. An empty scenario list is a value, not an exception: plan() returns the plan it built and runAllStyles checks plan.tests.length === 0, which means nothing to test here, so it stops trying further planning styles and the loop picks the next sub-page candidate. An empty plan is not pushed into the saved plans either.

Expanding an existing plan is unchanged, and a genuine planning failure still retries as before.

</task>

<rules>
Expand Down
13 changes: 12 additions & 1 deletion src/commands/explore-command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -261,9 +261,15 @@ export class ExploreCommand extends BaseCommand {
tag('info').log(`Exploring sub-page: ${pick.url} (${pick.reason})`);
try {
await this.explorBot.visit(pick.url);
const errorPage = getStateErrorPageError(this.explorBot.stateManager().getCurrentState());
if (errorPage) {
tag('warning').log(`Skipping sub-page: ${errorPage.message}`);
this.failedSubPages.add(normalizeUrl(pick.url));
continue;
}
await this.runAllStyles(pick.url, undefined, mainPlan, this.completedPlans, styles);
const subPlan = this.explorBot.getCurrentPlan();
if (subPlan && !this.completedPlans.includes(subPlan)) {
if (subPlan?.tests.length && !this.completedPlans.includes(subPlan)) {
this.completedPlans.push(subPlan);
}
knownUrls.add(normalizeUrl(pick.url));
Expand Down Expand Up @@ -298,6 +304,11 @@ export class ExploreCommand extends BaseCommand {
if (fresh && parentPlan) opts.extend = parentPlan;
if (this.dryRun) opts.noSave = true;
await this.planWithRetry(feature, opts, pageUrl);
const plan = this.explorBot.getCurrentPlan();
if (plan && plan.tests.length === 0) {
tag('warning').log('Nothing to test on this page, moving on');
return;
}
await this.runPendingTests();
this.rememberCurrentPlan();
fresh = false;
Expand Down
27 changes: 25 additions & 2 deletions tests/integration/planner.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -275,10 +275,33 @@ describe('Planner with aimock', () => {
expect(mock.getRequests().length).toBe(0);
});

it('throws when AI returns empty scenarios and no current plan', async () => {
it('instructs to return no scenarios for a page with nothing to exercise', async () => {
await planner.plan();

const prompt = extractPromptText(mock.getLastRequest());
expect(prompt).toContain('return an empty scenarios list');
});

it('returns an empty plan when AI returns no scenarios', async () => {
mock.clearFixtures();
mock.on({}, { content: JSON.stringify({ planName: 'Empty', scenarios: [] }) });

const plan = await planner.plan();

expect(plan.tests.length).toBe(0);
});

it('keeps an existing plan when a later style returns empty scenarios', async () => {
const existingPlan = new Plan('Task Board Testing');
existingPlan.url = '/tasks/board';
existingPlan.addTest(new Test('Create a new task via the Create Task modal', 'critical', ['Task appears'], '/tasks/board', ['Click Create']));
planner.currentPlan = existingPlan;

mock.clearFixtures();
mock.on({}, { content: JSON.stringify({ planName: 'Empty', scenarios: [] }) });

await expect(planner.plan()).rejects.toThrow('No tasks were created successfully');
const plan = await planner.plan();

expect(plan.tests.length).toBe(1);
});
});
Loading