diff --git a/CHANGELOG.md b/CHANGELOG.md index f197487..78028a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## 2026-08-31 + +### Changes + +- Prima: every command now prints `### Artifacts` naming the files it just wrote — the ARIA tree and + the html always, plus the screenshot and network log when they were captured. Until now those files + were written on every command but only named when a `check` came back with a CONTRADICTION, so the + envelope offered a hash and nothing to suggest there was a page on disk worth reading. The ARIA tree + carries values and checked states, so one `grep` over it often answers a question that would + otherwise cost another `verify` or `ask`. +- Prima: `do` now names the per-step captures by file rather than only their directory, so the + `diff.yaml` recording what each step changed is visible from the envelope. +- Prima: `prima status ` no longer needs a browser. It only reads recorded files, but it used to + open a session first and failed with "No browser to drive" once the run it was meant to explain had + finished. It also looks the hash up across every recorded site instead of assuming the most recent + one, and lists everything else kept under the hash. A hash that really is missing now reports the + directory it looked in. + ## 2026-08-30 ### Changes diff --git a/boat/prima/src/cli.ts b/boat/prima/src/cli.ts index ed3ff6c..bac3fbb 100644 --- a/boat/prima/src/cli.ts +++ b/boat/prima/src/cli.ts @@ -31,9 +31,9 @@ const checkHelp = dedent` as false. Outcomes are settled against a screenshot of the whole page: what a user can see is the proof, and the run log only says what was done. CONTRADICTION means the two - disagree - reported with both sides rather than settled one way, and ### Artifacts - then names the html, aria and screenshot on disk so you can judge it yourself. Not - finding something in the picture is not enough on its own; that is "not verified". + disagree - reported with both sides rather than settled one way, so read the html, + aria and screenshot named under ### Artifacts and judge it yourself. Not finding + something in the picture is not enough on its own; that is "not verified". ok: follows those outcomes - false when one FAILED or CONTRADICTED, or when the run could not complete, which is reported as such rather than as an app failure. Page problems seen on the way appear under ### Answer, not as step failures. @@ -53,6 +53,13 @@ const verifyHelp = dedent` could not be expressed, which is not the same as false. `; +const statusHelp = dedent` + Reads the files a command recorded, so it needs no browser and outlives the session. + The hash is looked up across every recorded site. ### Artifacts names every file kept + under it: the aria tree, the html, the screenshot and network log when they were + captured, and the per-step captures of a do run. +`; + const reportHelp = dedent` Commands are logged as they run, so the report needs no browser and outlives the session. The most recent session is reported unless --pw-session names another. @@ -122,7 +129,7 @@ function primaFor(options: any): Prima { return new Prima(buildOptions(options)); } -async function runPrima(options: any, command: string, run: (prima: Prima) => Promise, record = true): Promise { +async function runPrima(options: any, command: string, run: (prima: Prima) => Promise): Promise { setQuietMode(!isVerboseMode()); trackActivityLine(); const prima = primaFor(options); @@ -136,7 +143,7 @@ async function runPrima(options: any, command: string, run: (prima: Prima) => Pr envelope = await prima.toolFailureEnvelope(command, error); } - if (record) prima.record(envelope, Date.now() - startedAt); + prima.record(envelope, Date.now() - startedAt); clearActivityLine(); console.log(renderEnvelope(envelope)); await prima.stop().catch(() => {}); @@ -211,9 +218,15 @@ export function createPrimaCommands(name = 'prima'): Command { process.exit(0); }); - addCommonOptions(cmd.command('status ').description('Show the artifacts and page detail recorded for an earlier command')).action(async (hash, options) => { - await runPrima(options, `status ${hash}`, (prima) => prima.status(hash), false); - }); + addCommonOptions(cmd.command('status ').description('Show the artifacts and page detail recorded for an earlier command')) + .addHelpText('after', `\n${statusHelp}`) + .action(async (hash, options) => { + setQuietMode(!isVerboseMode()); + const prima = primaFor(options); + const envelope = await prima.status(hash).catch((error: unknown) => prima.toolFailureEnvelope(`status ${hash}`, error)); + console.log(renderEnvelope(envelope)); + process.exit(envelope.ok ? 0 : 1); + }); addCommonOptions(cmd.command('report').description('Turn every command of a session into one html and markdown report')) .addHelpText('after', `\n${reportHelp}`) diff --git a/boat/prima/src/envelope.ts b/boat/prima/src/envelope.ts index 7bf8fc3..c1e6d47 100644 --- a/boat/prima/src/envelope.ts +++ b/boat/prima/src/envelope.ts @@ -1,6 +1,10 @@ -import { mkdirSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, readdirSync, writeFileSync } from 'node:fs'; import path from 'node:path'; +export const STATUS_FILE = 'status.json'; + +const ARTIFACT_FILES = { aria: 'aria.yml', html: 'page.html', screenshot: 'page.png', network: 'network.jsonl' }; + const EXPECTATION_LABELS = { passed: 'PASSED ', failed: 'FAILED ', @@ -33,7 +37,7 @@ export interface EnvelopeData { failure?: { error: string; compactAria?: string }; instance: InstanceInfo; status?: string; - artifacts?: { aria: string; html: string; screenshot?: string; network?: string }; + artifacts?: ArtifactPaths; } export function renderEnvelope(data: EnvelopeData): string { @@ -41,27 +45,40 @@ export function renderEnvelope(data: EnvelopeData): string { return sections.filter((section) => section).join('\n\n'); } -export function writeArtifacts(dir: string, snapshot: { aria: string | null; html: string | null; screenshot?: Buffer; requests: unknown[] }): { aria: string; html: string; screenshot?: string; network?: string } { +export function writeArtifacts(dir: string, snapshot: { aria: string | null; html: string | null; screenshot?: Buffer; requests: unknown[] }): ArtifactPaths { mkdirSync(dir, { recursive: true }); - const paths: { aria: string; html: string; screenshot?: string; network?: string } = { - aria: path.resolve(dir, 'aria.yml'), - html: path.resolve(dir, 'page.html'), + const paths: ArtifactPaths = { + aria: path.resolve(dir, ARTIFACT_FILES.aria), + html: path.resolve(dir, ARTIFACT_FILES.html), }; writeFileSync(paths.aria, snapshot.aria ?? '', 'utf-8'); writeFileSync(paths.html, snapshot.html ?? '', 'utf-8'); if (snapshot.screenshot) { - paths.screenshot = path.resolve(dir, 'page.png'); + paths.screenshot = path.resolve(dir, ARTIFACT_FILES.screenshot); writeFileSync(paths.screenshot, snapshot.screenshot); } if (!snapshot.requests.length) return paths; - paths.network = path.resolve(dir, 'network.jsonl'); + paths.network = path.resolve(dir, ARTIFACT_FILES.network); writeFileSync(paths.network, snapshot.requests.map((request) => `${JSON.stringify(request)}\n`).join(''), 'utf-8'); return paths; } +export function readArtifacts(dir: string): ArtifactPaths { + const paths: ArtifactPaths = { aria: path.resolve(dir, ARTIFACT_FILES.aria), html: path.resolve(dir, ARTIFACT_FILES.html) }; + if (existsSync(path.resolve(dir, ARTIFACT_FILES.screenshot))) paths.screenshot = path.resolve(dir, ARTIFACT_FILES.screenshot); + if (existsSync(path.resolve(dir, ARTIFACT_FILES.network))) paths.network = path.resolve(dir, ARTIFACT_FILES.network); + + const recorded = [...Object.values(ARTIFACT_FILES), STATUS_FILE]; + const files = readdirSync(dir) + .filter((entry) => !recorded.includes(entry)) + .sort(); + if (files.length) paths.files = files; + return paths; +} + function renderResult(data: EnvelopeData): string { const lines = [`ok: ${data.ok}`, `command: ${data.command}`]; if (data.used?.length) lines.push(`used: ${data.used.join('; ')}`); @@ -100,7 +117,7 @@ function renderSteps(data: EnvelopeData): string | null { lines.push(`${index + 1}. ${mark} ${step.label}`); for (const line of (step.proof || '').split('\n').filter(Boolean)) lines.push(` ${line}`); }); - if (data.stepFiles) lines.push('', `page after each step: ${data.stepFiles}`); + if (data.stepFiles) lines.push('', `page after each step: ${data.stepFiles}/-.{aria.yaml,html,diff.yaml}`); return section('Steps', lines.join('\n')); } @@ -178,6 +195,7 @@ export function renderArtifacts(data: EnvelopeData): string | null { const lines = [`aria: ${data.artifacts.aria}`, `html: ${data.artifacts.html}`]; if (data.artifacts.screenshot) lines.push(`screenshot: ${data.artifacts.screenshot}`); if (data.artifacts.network) lines.push(`network: ${data.artifacts.network}`); + if (data.artifacts.files?.length) lines.push(`also here: ${data.artifacts.files.join(', ')}`); return section('Artifacts', lines.join('\n')); } @@ -189,3 +207,11 @@ function align(label: string, marker: string, width: number): string { function section(title: string, body: string): string { return `### ${title}\n${body}`; } + +export interface ArtifactPaths { + aria: string; + html: string; + screenshot?: string; + network?: string; + files?: string[]; +} diff --git a/boat/prima/src/prima.ts b/boat/prima/src/prima.ts index 85ac7f0..c5cfdbd 100644 --- a/boat/prima/src/prima.ts +++ b/boat/prima/src/prima.ts @@ -25,7 +25,7 @@ import { browserErrorMessage } from '../../../src/utils/browser-errors.ts'; import { pluralize } from '../../../src/utils/logger.ts'; import { mdq } from '../../../src/utils/markdown-query.ts'; import { safeFilename } from '../../../src/utils/strings.ts'; -import { type EnvelopeData, type InstanceInfo, writeArtifacts } from './envelope.ts'; +import { type EnvelopeData, type InstanceInfo, STATUS_FILE, readArtifacts, writeArtifacts } from './envelope.ts'; import { isFunctionExpression, takePwValue, toCodeceptWrapper } from './pw-parser.ts'; import { type PwServerDescriptor, readDescriptors, selectDescriptor } from './pw-registry.ts'; import { type SessionRun, latestSessionFile, readSession, recordCommand, sessionFile, sessionsDir } from './session-log.ts'; @@ -376,7 +376,6 @@ export class Prima { const problems = [...unreached.map((expectation) => `not reached: ${expectation.text}`), ...contradicted.map((expectation) => `the picture and the run disagree about: ${expectation.text}`)]; if (problems.length) envelope.failure = { error: problems.join('\n') }; - if (contradicted.length) envelope.artifacts = this.artifacts; if (!test.hasFinished || test.isSkipped) { envelope.ok = false; @@ -982,14 +981,17 @@ export class Prima { } private async successEnvelope(command: string, used: string[], result: ActionResult, previousState: WebPageState | null): Promise { + const changes = await this.pageChanges(result, previousState, used[0]); + const status = await this.saveStatus(result); return { ok: true, command, used, page: this.pageBlock(result, previousState), - changes: await this.pageChanges(result, previousState, used[0]), + changes, instance: await this.instanceInfo(), - status: await this.saveStatus(result), + status, + artifacts: this.artifacts, }; } @@ -998,24 +1000,28 @@ export class Prima { const failure: EnvelopeData['failure'] = { error: browserErrorMessage(error) }; if (result.ariaSnapshot) failure.compactAria = compactAriaSnapshot(result.ariaSnapshot, true); + const status = await this.saveStatus(result); return { ok: false, command, page: this.pageBlock(result, previousState), failure, instance: await this.instanceInfo(), - status: await this.saveStatus(result), + status, + artifacts: this.artifacts, }; } private async reportEnvelope(command: string, result: ActionResult, previousState: WebPageState | null, outcome: Partial): Promise { + const status = await this.saveStatus(result); return { ok: true, command, page: this.pageBlock(result, previousState), ...outcome, instance: await this.instanceInfo(), - status: await this.saveStatus(result), + status, + artifacts: this.artifacts, }; } @@ -1060,9 +1066,16 @@ export class Prima { } async status(hash: string): Promise { + if (!this.artifactsDir) { + const sites = listSites(); + const site = sites.find((candidate) => existsSync(path.join(candidate.dir, 'output', 'prima', hash))) || sites[0]; + if (site && !this.configBaseUrl()) this.sessionUrl = site.url; + await this.loadConfig(); + } + const dir = this.statusDir(hash); - const statusFile = path.join(dir, 'status.json'); - if (!existsSync(statusFile)) return this.toolFailureEnvelope(`status ${hash}`, `No command was recorded under ${hash}. Every envelope prints its own hash on the Instance line.`); + const statusFile = path.join(dir, STATUS_FILE); + if (!existsSync(statusFile)) return this.toolFailureEnvelope(`status ${hash}`, `No command was recorded under ${dir}. Every envelope prints its own hash on the Instance line.`); const saved = JSON.parse(readFileSync(statusFile, 'utf-8')); return { @@ -1070,14 +1083,14 @@ export class Prima { command: `status ${hash}`, page: saved.page, instance: await this.instanceInfo(), - artifacts: { aria: path.join(dir, 'aria.yml'), html: path.join(dir, 'page.html') }, + artifacts: readArtifacts(dir), }; } private async saveStatus(result: ActionResult): Promise { const hash = this.statusHash(); await this.writeSnapshot(result); - writeFileSync(path.join(this.statusDir(hash), 'status.json'), JSON.stringify({ page: this.pageBlock(result, null) }), 'utf-8'); + writeFileSync(path.join(this.statusDir(hash), STATUS_FILE), JSON.stringify({ page: this.pageBlock(result, null) }), 'utf-8'); return hash; } diff --git a/boat/prima/tests/envelope.test.ts b/boat/prima/tests/envelope.test.ts index 92fa931..c2ae653 100644 --- a/boat/prima/tests/envelope.test.ts +++ b/boat/prima/tests/envelope.test.ts @@ -1,8 +1,8 @@ import { describe, expect, test } from 'bun:test'; -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { type EnvelopeData, renderEnvelope, writeArtifacts } from '../src/envelope.ts'; +import { type EnvelopeData, readArtifacts, renderEnvelope, writeArtifacts } from '../src/envelope.ts'; const base: EnvelopeData = { ok: true, @@ -161,6 +161,25 @@ describe('renderEnvelope', () => { expect(out).not.toContain('### Artifacts'); }); + test('a successful action names the files it recorded, so they invite reading', () => { + const out = renderEnvelope(base); + expect(out).toContain('### Artifacts'); + expect(out).toContain('aria: /tmp/x/aria.yml'); + expect(out).toContain('html: /tmp/x/page.html'); + expect(out).toContain('network: /tmp/x/network.jsonl'); + expect(out).not.toContain('screenshot:'); + }); + + test('files recorded beside the named ones are listed too', () => { + const out = renderEnvelope({ ...base, artifacts: { ...base.artifacts!, files: ['1-click-add.aria.yaml', '1-click-add.diff.yaml'] } }); + expect(out).toContain('also here: 1-click-add.aria.yaml, 1-click-add.diff.yaml'); + }); + + test('step captures name the files written for each step, not just their directory', () => { + const out = renderEnvelope({ ...base, steps: [{ label: "I.click('Add')", ok: true, proof: '' }], stepFiles: '/tmp/x' }); + expect(out).toContain('page after each step: /tmp/x/-.{aria.yaml,html,diff.yaml}'); + }); + test('open tabs alone are evidence enough for a running browser', () => { const out = renderEnvelope({ ...base, instance: { name: 'default', tabs: 2, others: [] } }); expect(out).toContain('| running'); @@ -168,6 +187,24 @@ describe('renderEnvelope', () => { }); }); +describe('readArtifacts', () => { + test('reports the files a recorded command left behind, and only those', () => { + const dir = mkdtempSync(path.join(tmpdir(), 'prima-')); + writeArtifacts(dir, { aria: '- button "Login"', html: '', requests: [] }); + writeFileSync(path.join(dir, 'status.json'), '{}', 'utf-8'); + writeFileSync(path.join(dir, '1-click-add.diff.yaml'), 'added: []', 'utf-8'); + + const found = readArtifacts(dir); + + expect(found.aria).toBe(path.join(dir, 'aria.yml')); + expect(found.html).toBe(path.join(dir, 'page.html')); + expect(found.network).toBeUndefined(); + expect(found.screenshot).toBeUndefined(); + expect(found.files).toEqual(['1-click-add.diff.yaml']); + rmSync(dir, { recursive: true, force: true }); + }); +}); + describe('writeArtifacts', () => { test('writes aria, html and network files and returns absolute paths', () => { const dir = mkdtempSync(path.join(tmpdir(), 'prima-')); diff --git a/boat/prima/tests/prima.test.ts b/boat/prima/tests/prima.test.ts index d3b3165..b2087d8 100644 --- a/boat/prima/tests/prima.test.ts +++ b/boat/prima/tests/prima.test.ts @@ -136,6 +136,8 @@ describe('Prima.pw', () => { expect(existsSync(path.join(artifactsRoot, envelope.status!, 'aria.yml'))).toBe(true); expect(existsSync(path.join(artifactsRoot, envelope.status!, 'page.html'))).toBe(true); expect(existsSync(path.join(artifactsRoot, envelope.status!, 'network.jsonl'))).toBe(false); + expect(envelope.artifacts?.aria).toBe(path.join(artifactsRoot, envelope.status!, 'aria.yml')); + expect(envelope.artifacts?.network).toBeUndefined(); expect(envelope.instance.name).toBe('default'); }); diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 7b0fe90..05c9e26 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -685,7 +685,9 @@ html: /home/you/.explorbot/sites/app.example.com/output/prima/2026-08-04T10-04-2 `used:` is code that already executed: for `go` the CodeceptJS step it ran, for `pw` the Playwright expression you passed, which a CodeceptJS test needs wrapped in `I.usePlaywrightTo(...)`. Log lines can precede the envelope, so start parsing at the first `###` line. -`### Changes` renders on every action envelope, saying `no change` when the tree is identical — so a successful command proves what it did instead of leaving you to check. `check` and `do` report per step rather than in aggregate: `### Steps` names each step with the code it ran and what proved it, and `page after each step:` points at the captures. `check` adds `### Expected outcomes`; `ask`, `research`, and `verify` add `### Answer`, `### Research`, or `### Assertions`; `pw` adds `### Value` when its expression returns one. `network:` appears under `### Artifacts` only when requests were captured, and everything else recorded for a command is behind `prima status `. +`### Changes` renders on every action envelope, saying `no change` when the tree is identical — so a successful command proves what it did instead of leaving you to check. `check` and `do` report per step rather than in aggregate: `### Steps` names each step with the code it ran and what proved it, and `page after each step:` names the per-step captures. `check` adds `### Expected outcomes`; `ask`, `research`, and `verify` add `### Answer`, `### Research`, or `### Assertions`; `pw` adds `### Value` when its expression returns one. + +`### Artifacts` names the files every command leaves on disk: the aria tree and the html always, `screenshot:` and `network:` when a screenshot was taken or requests were captured. The aria tree carries values and checked states, so one `grep` over it settles a question the envelope did not answer, without asking the page again. `prima status ` reprints those paths for an earlier command and lists whatever else was kept under the hash; it only reads recorded files, so it needs no browser and outlives the session. **A failed action is a failure.** Nothing is retried along a different route and no other element is substituted, so `ok: true` means the action you asked for is the one that landed. A failure adds `### Failure` with the error and the compact ARIA of the page, so you can retarget from the envelope itself instead of opening the artifact files. diff --git a/tests/integration/prima-smoke.test.ts b/tests/integration/prima-smoke.test.ts index fb5982b..48f099e 100644 --- a/tests/integration/prima-smoke.test.ts +++ b/tests/integration/prima-smoke.test.ts @@ -127,11 +127,12 @@ describe('Prima drives a real page', () => { expect(envelope.page.url).toContain('note=the+hinge+arrived+bent'); }); - test('an action reports a status hash instead of artifact paths, and status resolves it', async () => { + test('an action names the files it recorded, and status resolves the hash to the same ones', async () => { const envelope = await prima.pw("({ page }) => page.click('text=Submit')"); expect(envelope.status).toMatch(/^[0-9a-f]{15}$/); - expect(envelope.artifacts).toBeUndefined(); + expect(existsSync(envelope.artifacts!.aria)).toBe(true); + expect(renderEnvelope(envelope)).toContain(envelope.artifacts!.aria); expect(renderEnvelope(envelope)).toContain(`prima status ${envelope.status}`); const status = await prima.status(envelope.status!);