prima check: run a scenario end to end and report what it proved

.gitignore

@@ -9,6 +9,10 @@ bun.lockb
 # Regression harness run artifacts
 tests/regression/.runs/
 
+# Session state and browser sessions written by running the CLI in this repo
+session.json
+.playwright-cli/
+
 # Environment variables
 .env
 .env.local
@@ -148,3 +152,6 @@ Thumbs.db
 
 # Keep the temporarily vendored OpenRouter AI-SDK-7 provider (PR #511)
 !vendor/*.tgz
+
+# Reporter debug dump
+testomatio.debug.json

CHANGELOG.md

@@ -1,13 +1,187 @@
 # Changelog
 
+## 2026-08-16
+
+### Changes
+
+- Token counters include the attempts that were retried. Only the attempt that finally succeeded
+  was counted, so a run that retried its way through a flaky model reported a fraction of the
+  tokens it had actually spent.
+- Starting on a host other than the configured one now says so. Relative navigation still resolves
+  against the base URL, so the warning names the `web.url` to set to make the two agree.
+- `--session <file>` resolves against the working directory given by `-p, --path` rather than the
+  directory the command was started from, so a run pointed at another project keeps its session
+  file with that project.
+
+## 2026-08-12
+
+### Prima shows what it is doing while it does it
+
+A prima command used to sit silent until the envelope arrived. It now writes the current activity —
+the model it is asking, the browser step underway, the scenario it is testing — as a single line
+that each new activity overwrites. The line is erased before the envelope is printed, so the report is the only thing left
+on screen.
+
+It is drawn on stderr and only when that is a terminal: piped or captured output stays exactly as it
+was, byte for byte. With `DEBUG` set the log takes over and the line stays out of its way.
+
 ## 2026-08-11
 
+### Prima prints its log when `DEBUG` is set
+
+`DEBUG` in front of any prima command prints everything the run does: config and browser
+attachment, every step as it executes, and the debug stream of the agents behind it. Prima carries
+no logging flags of its own — `--verbose` and `--debug` are gone.
+
+```bash
+DEBUG='explorbot:*' prima do "open the account menu" "switch the theme to dark"
+DEBUG='explorbot:tester' prima check "a workflow can be created"
+```
+
 ### Changes
 
+- Verbose mode turns the debug log on for real. It set the `DEBUG` variable after the logging
+  library had already read it, so `explorbot --verbose` printed a fraction of what
+  `DEBUG='explorbot:*'` printed in front of the same command.
 - Page snapshots no longer carry variant-prefixed classes such as `dark:`, `hover:` or `md:`. Those
   classes describe how an element looks in some other state, not what it is, so the agents now read
   a smaller and less noisy page.
 
+## 2026-08-10
+
+### `prima report` collects a whole session into one report
+
+Every prima command is logged to `output/prima/sessions/` as it runs. `prima report` turns that
+log into an HTML and a Markdown report — each command with its steps, expected outcomes and the
+proof recorded for them. It needs no browser, so the report still comes out after the session is
+closed.
+
+```bash
+prima report                      # the session used most recently
+prima report --pw-session my-app  # a named playwright-cli session
+```
+
+The log is written in the format the Testomat.io reporter replays, so the same file can be sent
+to Testomat.io as a run:
+
+```bash
+TESTOMATIO=<apiKey> npx @testomatio/reporter replay <the path prima report prints>
+```
+
+### Changes
+
+- [Prima] Prima no longer generates a report of its own while it runs. A `check` used to end by
+  writing the session HTML and Markdown reports, which belong to an explorbot run rather than to
+  a single command. Reports are produced only when `prima report` asks for one.
+
+## 2026-08-09
+
+### `prima check` runs a whole scenario and reports what it proved
+
+`prima check` takes a behaviour, not a click path, and runs it the way a tester would: it drives
+the page, verifies the outcome itself, and reports every step with the proof for each. Use it when
+you want a verdict; use `prima do` when you already know the steps.
+
+```bash
+prima check "a workflow can be created and appears in the list" --url http://app.test
+prima check "signup rejects a duplicate email" \
+  --expected "an error names the email as taken" \
+  --expected "no second account is created"
+```
+
+- **`--expected <outcome>`** — an outcome the run must reach; repeat the flag for several. Without
+  it the scenario text is the single expected outcome. Each one comes back under
+  `### Expected outcomes` as `PASSED`, `FAILED` or `not verified` — "not verified" means the run
+  never checked it, which is not the same as false.
+- **`--url <url>`** — open that page before starting, when the browser is not already on it.
+
+### `prima status <hash>` reopens an earlier command
+
+Every envelope prints a hash on its `### Instance` line. `prima status <hash>` returns the page
+detail and artifact paths recorded for that command, so envelopes stay short and the full ARIA
+tree, HTML and network log are one command away instead of inline.
+
+```bash
+prima status 66800d6d2c8c553
+```
+
+### Changes
+
+- [Prima] `do` now ends at the last step you gave it. It used to keep acting after the sequence
+  was finished, and could report success while a step it was asked to check never held. A step it
+  could not carry out is named under `### Failure` and fails the command.
+- [Prima] `do` carries the same tools a test run does, so a single call can act, look and assert
+  across a long sequence rather than being split into one command per step.
+- [Prima] `pw` returns the value of the expression under `### Value`. A `page.title()` or
+  `locator.count()` used to run and have its answer thrown away.
+- [Prima] `verify` lists every assertion it ran with its own `PASSED` or `FAILED`, plus the
+  Playwright form of the ones that held, and gives no overall verdict — read the lines and decide.
+- [Prima] Research output drops CSS selectors, XPaths and coordinates, which were the bulk of the
+  map and are not what you act on.
+- [Prima] Prima commands print the envelope and nothing else. The banner, config line, browser
+  startup and disconnect chatter are hidden unless `--verbose` or `--debug` is passed.
+- [Prima] `click` and `fill` are removed — describe the whole sequence to `do` instead, which
+  attaches once and carries all of it.
+- [Tester] A test that stops making progress is handed to final review instead of being marked
+  failed on the spot. A run that had already done its work and gone quiet was reported as a
+  failure; the verdict now comes from reviewing the result.
+- [Tester] Console and network errors seen during a run are reported as page problems rather than
+  as failed steps, so they no longer sink a test that otherwise passed.
+- [Tester] An expected outcome counts as settled only when it is recorded back word for word,
+  and the prompt now says so — outcomes phrased differently were silently left unaccounted for.
+- Page changes report values typed into fields, under a `typed:` section, alongside what was added,
+  removed and toggled. Filling a form previously showed as no change at all.
+- Long field values in page snapshots are cut to an excerpt with a pointer to the full text, which
+  keeps a page holding a large document readable.
+- Pages are considered ready as soon as the DOM goes quiet, instead of waiting on network idle.
+  Applications with a live websocket never reached network idle, so every snapshot paid the full
+  timeout.
+- Generated tests assert element visibility, hidden state and field values through real Playwright
+  locators instead of leaving a TODO comment.
+
+## 2026-08-07
+
+### Prima never substitutes your target
+
+A failed action now fails. Previously a `pw` call on a selector that did not exist could be
+"healed" into clicking a different element and still report `ok: true` — so `ok: true` did not
+mean your own action landed. Automatic retry along a different route is gone entirely, together
+with the `--no-heal` flag that used to switch it off, and the `healed:` line and
+`### Healing attempts` block in the envelope.
+
+### Configuration
+
+- **`ai.agents.prima.researchAfterVisits`** — how many visits to a page before `prima do` works
+  from that page's stored research map instead of its accessibility tree. Default: `3`.
+
+### Changes
+
+- [Prima] `### Changes` now appears on every action, showing what the accessibility tree gained,
+  lost or toggled — or saying `no change` outright. It previously went missing whenever the
+  command also produced an answer, a research map or a verdict, so a successful click proved
+  nothing.
+- [Prima] Attaching to a running `playwright-cli` session works again. Prima was matching on a
+  field that no release of `playwright-cli` writes, so it reported no browser while one was open
+  and told you to open the session you already had.
+- [Prima] Attached sessions are driven through the browser's own Playwright build, which is what
+  makes reading the page work across versions.
+- [Prima] New `context()` step for `do`: when the element an instruction needs is missing from the
+  context it holds, it returns the page again, and drops to raw markup if asked a second time on
+  the same page.
+- [Prima] `network.jsonl` is listed only when requests were actually recorded, instead of always
+  pointing at an empty file.
+- [Navigator] `verify` now separates "this claim is false" from "no assertion can express this
+  claim". A claim it cannot phrase is no longer reported as a failing check, and is no longer
+  remembered as one.
+- [Navigator] Verification can assert whether a control is enabled, disabled, checked, selected or
+  expanded. Only presence and text could be asserted before, so state claims always failed.
+- Navigation: a redirect that only adds query parameters — a session id, a workspace flag — counts
+  as arriving. Reaching such a page previously burned minutes of retries and could still end in a
+  failure while the page sat correctly loaded.
+- Page snapshots keep every attribute on an element. A control marked `[pressed]` or `[disabled]`
+  used to lose its ref, which hid exactly the controls worth acting on.
+- Playwright updated to 1.62.
+
 ## 2026-08-06
 
 ### Global Installation

bin/explorbot-cli.ts

@@ -27,7 +27,7 @@ const pkgVersion = JSON.parse(fs.readFileSync(pkgPath, 'utf-8')).version as stri
 
 program.name(cli).description('AI-powered web exploration tool').version(pkgVersion, '-V, --version');
 
-if (!process.env.EXPLORBOT_NO_BANNER) {
+if (!process.env.EXPLORBOT_NO_BANNER && !process.argv.includes('prima')) {
   console.log(`⛵ ${chalk.yellow.bold(`Explorbot v${pkgVersion}`)} ${chalk.dim('Autonomous Testing Agent')}`);
 }
 

boat/prima/src/activity-line.ts

@@ -0,0 +1,33 @@
+import chalk from 'chalk';
+import { type ActivityEntry, addActivityListener, removeActivityListener } from '../../../src/activity.ts';
+import { isVerboseMode } from '../../../src/utils/logger.ts';
+
+const RESET_LINE = '\r\u001b[2K';
+
+const stream = process.stderr;
+let tracking = false;
+
+export function trackActivityLine(): void {
+  if (tracking) return;
+  if (!stream.isTTY) return;
+  if (isVerboseMode()) return;
+
+  tracking = true;
+  addActivityListener(writeActivityLine);
+}
+
+export function clearActivityLine(): void {
+  if (!tracking) return;
+
+  tracking = false;
+  removeActivityListener(writeActivityLine);
+  stream.write(RESET_LINE);
+}
+
+function writeActivityLine(activity: ActivityEntry | null): void {
+  if (!activity) return;
+
+  const width = (stream.columns || 80) - 2;
+  const message = Array.from(activity.message.replace(/\s+/g, ' ').trim()).slice(0, width).join('');
+  stream.write(`${RESET_LINE}${chalk.gray(message)}`);
+}