From f39a65cf73143044b3a179b35ee059e7893120df Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Sun, 15 Nov 2026 01:10:30 +0900 Subject: [PATCH 1/2] feat(uat): page assertions, a versioned export record, and replay Spec asserts (url_matches, title_contains, text_present, text_absent, selector_count) are checked by the server on the page the run ends on: when finish is called, or when the runner closes the browser. The report carries every row with what was observed, and asserts_passed as the judgement; a harness that never finishes leaves them not_evaluated. lyra-uat export writes the stable part of a run as lyra-uat-export v1, whose JSON Schema is committed under uat/schemas and pinned to the model by a test. lyra-uat replay feeds a run's own trace to the scripted brain with the run's spec and compares status, exit reason, outcome, steps and every assertion; redacted arguments make it say exact=false. --- README.md | 2 +- docs/UAT.md | 46 ++- scripts/verify_uat_e2e.py | 57 +++- src/lyra_browser/tools/uat.py | 3 + src/lyra_browser/uat/asserts.py | 103 +++++++ src/lyra_browser/uat/batch.py | 1 + src/lyra_browser/uat/cli.py | 45 ++- src/lyra_browser/uat/export.py | 130 +++++++++ src/lyra_browser/uat/recorder.py | 6 + src/lyra_browser/uat/replay.py | 76 +++++ src/lyra_browser/uat/report.py | 46 +++ src/lyra_browser/uat/runner.py | 6 + src/lyra_browser/uat/schemas/export.v1.json | 297 ++++++++++++++++++++ src/lyra_browser/uat/server.py | 4 +- src/lyra_browser/uat/spec.py | 50 ++++ tests/uat/test_uat_report.py | 116 ++++++++ tests/uat/test_uat_runner.py | 146 ++++++++++ 17 files changed, 1128 insertions(+), 6 deletions(-) create mode 100644 src/lyra_browser/uat/asserts.py create mode 100644 src/lyra_browser/uat/export.py create mode 100644 src/lyra_browser/uat/replay.py create mode 100644 src/lyra_browser/uat/schemas/export.v1.json diff --git a/README.md b/README.md index e5a57f2..7b804f0 100644 --- a/README.md +++ b/README.md @@ -1049,7 +1049,7 @@ lyra-uat run examples/uat/demo/run.yaml # one persona, Claude Code plays it lyra-uat batch examples/uat/demo/batch.yaml # two personas in parallel, one process each ``` -A run leaves a directory: `report.json` (and `report.md`), `trace.jsonl`, `events.jsonl`, `captures/`, `network.jsonl`, `console.jsonl`, and the browser's own `audit.jsonl`. Persona, target, brain, hooks and limits are YAML or JSON; `lyra-uat schema run|batch|report` prints the JSON Schemas. A run that should act without anyone to ask sets `target.policy: autonomous`; the default is `closed`, which keeps the run on its trusted sites. Details, the report format and each brain's requirements are in [docs/UAT.md](https://github.com/Intrect-io/lyra-browser/blob/master/docs/UAT.md). +A run leaves a directory: `report.json` (and `report.md`), `trace.jsonl`, `events.jsonl`, `captures/`, `network.jsonl`, `console.jsonl`, and the browser's own `audit.jsonl`. Persona, target, brain, hooks and limits are YAML or JSON; `lyra-uat schema run|batch|report|export` prints the JSON Schemas. Spec `asserts` judge the final page without the model; `lyra-uat export` writes a versioned record for a benchmark and `lyra-uat replay` re-runs a run's own calls to check the judgement holds. A run that should act without anyone to ask sets `target.policy: autonomous`; the default is `closed`, which keeps the run on its trusted sites. Details, the report format and each brain's requirements are in [docs/UAT.md](https://github.com/Intrect-io/lyra-browser/blob/master/docs/UAT.md). ## Headless mode diff --git a/docs/UAT.md b/docs/UAT.md index 569c63e..585b399 100644 --- a/docs/UAT.md +++ b/docs/UAT.md @@ -200,7 +200,7 @@ uat-runs// observations.json counts, and what the server knows of the browser browser/ the browser's data dir: audit.jsonl and a profile of its own harness/ harness brains: prompts, argv, stdout/stderr, MCP config - run.json the spec the run was started from + run.json the spec the run was started from (what replay reads) ``` `report.json` (schema version 1; `lyra-uat schema report`): @@ -223,6 +223,50 @@ uat-runs// that never clicked, an invalid profile, observers that failed to attach. - `brain` — backend, model, turns, tokens (input, output, cache read/write), cost where known, session id. `observations`, `hooks`, `artifacts`. +- `asserts` / `asserts_passed` — the spec's assertions on the final page and their + conjunction (below); `asserts_passed` is `null` when the spec has none. + +### Assertions: a judgement the model cannot make up + +`finish` is what the persona *says* happened. `asserts` in a run or batch spec say what +the page must *show*, and the server checks them itself, once: when `finish` is called, +on the page the persona finished on, or, if it never is, when the runner closes the +browser. A harness that exits without `finish` leaves them `not_evaluated`, and that +counts as not passed. + +```yaml +asserts: + - {id: on-receipt, kind: url_matches, value: "/orders/\\d+$"} # regular expression + - {id: title, kind: title_contains, value: "Order complete"} + - {id: thanks, kind: text_present, value: "Thank you", selector: main} # selector optional + - {id: no-error, kind: text_absent, value: "declined"} + - {id: one-receipt, kind: selector_count, selector: ".receipt", min: 1, max: 1} +``` + +Text is the visible text (`innerText`) of the page or of what `selector` matches. Each +row records what was observed, so a failure says what the page showed instead. + +### Export and replay + +```bash +lyra-uat export uat-runs/ [--out run.export.json] # one flat record +lyra-uat schema export # its JSON Schema +lyra-uat replay uat-runs/ [--out DIR] # same calls, same judgement? +``` + +`export` writes the stable part of a run that a benchmark or dashboard reads: task +(persona) id, `success` (the assertions' verdict, `null` without any), the outcome the +persona claimed, final URL, steps, tool calls, findings, tokens, cost and every step. +The shape is `lyra-uat-export` version 1; its schema is committed as +`src/lyra_browser/uat/schemas/export.v1.json` and a test keeps it equal to the model, so +a change of shape is a new version. A benchmark's own result format is a mapping from +this record, kept with that benchmark's adapter. + +`replay` runs the recorded tool calls again, in order, with the `scripted` brain and the +run's own spec (`run.json`), and compares status, exit reason, outcome, steps used and +every assertion. It prints the comparison and exits 0 when the judgement is the same, 2 +when it is not. Values the audit redacts (typed passwords) cannot be replayed as typed: +the result then says `"exact": false`. `lyra-uat run` exits 0 when the persona called `finish`, 2 when the run did not complete (budget, wall clock, a brain or hook failure, an invalid profile), and 130 diff --git a/scripts/verify_uat_e2e.py b/scripts/verify_uat_e2e.py index 1da1e92..f01f16e 100644 --- a/scripts/verify_uat_e2e.py +++ b/scripts/verify_uat_e2e.py @@ -32,7 +32,15 @@ def expect(condition: bool, message: str, detail: object = "") -> None: print(f"PASS {message}") -def spec_for(base: str, out: Path, *, budget: int, script: list[dict], hooks: dict | None = None): +def spec_for( + base: str, + out: Path, + *, + budget: int, + script: list[dict], + hooks: dict | None = None, + asserts: list[dict] | None = None, +): from lyra_browser.uat.spec import RunSpec origin = base.rstrip("/") @@ -54,6 +62,7 @@ def spec_for(base: str, out: Path, *, budget: int, script: list[dict], hooks: di }, "brain": {"backend": "scripted", "script": script}, "hooks": hooks or {}, + "asserts": asserts or [], "out_dir": str(out), "headless": True, } @@ -110,6 +119,7 @@ async def completed_run(base: str, alt_base: str, out: Path) -> None: budget=10, script=script, hooks={"before": "echo seeded $LYRA_UAT_PERSONA", "after": "echo checked"}, + asserts=ASSERTS, ) report = await run_spec(spec) run_dir = Path(report.artifacts.dir) @@ -173,6 +183,51 @@ async def completed_run(base: str, alt_base: str, out: Path) -> None: ) expect((run_dir / "browser" / "audit.jsonl").is_file(), "run has its own audit log") expect((run_dir / "browser" / "profile").is_dir(), "run has its own browser profile") + await judged_exported_and_replayed(report) + + +# What the pricing page shows after "Buy Studio": its click sets the title to "bought". +ASSERTS = [ + {"id": "on-pricing", "kind": "url_matches", "value": "/pricing$"}, + {"id": "bought", "kind": "title_contains", "value": "bought"}, + {"id": "button", "kind": "text_present", "value": "Buy Studio", "selector": "#buy"}, + {"id": "one-button", "kind": "selector_count", "selector": "#buy", "min": 1, "max": 1}, + {"id": "no-free-plan", "kind": "text_absent", "value": "Free forever"}, + {"id": "no-cart", "kind": "selector_count", "selector": ".cart-item", "min": 1}, +] + + +async def judged_exported_and_replayed(report) -> None: + from lyra_browser.uat.export import committed_schema, export_report + from lyra_browser.uat.replay import replay_run + + statuses = {a.id: a.status for a in report.asserts} + expect( + statuses + == { + "on-pricing": "passed", + "bought": "passed", + "button": "passed", + "one-button": "passed", + "no-free-plan": "passed", + "no-cart": "failed", + }, + "assertions judged on the finished page by the server", + [(a.id, a.status, a.observed, a.error) for a in report.asserts], + ) + expect(report.asserts_passed is False, "one failed assertion fails the judgement") + record = export_report(report) + try: + import jsonschema + except ImportError: + print("NOTE jsonschema is not installed: the export is checked by its model only") + else: + jsonschema.validate(record.model_dump(mode="json"), committed_schema()) + print("PASS export validates against schemas/export.v1.json") + expect(record.success is False and record.task_id == "e2e", "export record", record.success) + result = await replay_run(Path(report.artifacts.dir)) + expect(result["same"], "replay reaches the same judgement", result["differences"]) + print(f"NOTE replay exact={result['exact']} (typed values the audit redacts are not exact)") async def budget_run(base: str, out: Path) -> None: diff --git a/src/lyra_browser/tools/uat.py b/src/lyra_browser/tools/uat.py index 8f9b740..3ae3205 100644 --- a/src/lyra_browser/tools/uat.py +++ b/src/lyra_browser/tools/uat.py @@ -10,6 +10,7 @@ from fastmcp import FastMCP from ..context import ServerContext +from ..uat.asserts import evaluate_once from ..uat.recorder import RunRecorder SEVERITIES = ("blocker", "major", "minor") @@ -128,6 +129,8 @@ async def finish( return bad if not summary.strip(): return {"status": "error", "reason": "summary must not be empty."} + # The page the persona finished on is the page the spec's assertions judge. + await evaluate_once(ctx, recorder) recorder.set_finish( outcome=outcome, summary=summary, diff --git a/src/lyra_browser/uat/asserts.py b/src/lyra_browser/uat/asserts.py new file mode 100644 index 0000000..6f18dcc --- /dev/null +++ b/src/lyra_browser/uat/asserts.py @@ -0,0 +1,103 @@ +"""Deterministic checks on the page a run ended on. + +The persona's ``finish`` says what it believes happened; an assertion says what the page +shows. Assertions are part of the spec, evaluated by the server — not the model — once: +when ``finish`` is called, on the page the persona finished on, or, for a run that never +calls it, when the runner closes the browser. The same spec on the same page gives the +same answer, which is what makes a run comparable with its replay and with a benchmark's +expected result. +""" + +from __future__ import annotations + +import re +from typing import Any + +from .spec import Assertion + +# One call for every text/count assertion: the page's visible text (all of it, or what a +# selector matches) and how many elements a selector matches. ``innerText`` is what a +# person sees; hidden text does not count. +_PAGE_STATE_JS = """ +(selector) => { + if (!selector) { + return {text: document.body ? document.body.innerText : "", count: null, error: null}; + } + let nodes; + try { + nodes = Array.from(document.querySelectorAll(selector)); + } catch (e) { + return {text: "", count: 0, error: String(e && e.message || e)}; + } + return {text: nodes.map(n => n.innerText || n.textContent || "").join("\\n"), + count: nodes.length, error: null}; +} +""" + + +def _result(a: Assertion, passed: bool, observed: Any, error: str | None = None) -> dict: + return { + "id": a.id, + "kind": a.kind, + "status": "error" if error else ("passed" if passed else "failed"), + "expected": a.describe(), + "observed": observed, + **({"error": error} if error else {}), + } + + +async def _state(page: Any, selector: str | None) -> dict[str, Any]: + state = await page.evaluate(_PAGE_STATE_JS, selector) + if not isinstance(state, dict): + raise RuntimeError(f"the page answered {state!r}") + return state + + +async def evaluate(page: Any, assertions: list[Assertion]) -> list[dict[str, Any]]: + """Every assertion against ``page``; one that cannot be evaluated is an ``error`` row, + never an exception (the report must still be written).""" + results: list[dict[str, Any]] = [] + for a in assertions: + try: + if a.kind == "url_matches": + url = str(page.url) + results.append(_result(a, re.search(a.value, url) is not None, url)) + elif a.kind == "title_contains": + title = str(await page.title()) + results.append(_result(a, a.value in title, title)) + elif a.kind in ("text_present", "text_absent"): + state = await _state(page, a.selector) + if state.get("error"): + results.append(_result(a, False, None, f"selector: {state['error']}")) + continue + found = a.value in str(state.get("text") or "") + results.append(_result(a, found == (a.kind == "text_present"), found)) + elif a.kind == "selector_count": + state = await _state(page, a.selector) + if state.get("error"): + results.append(_result(a, False, None, f"selector: {state['error']}")) + continue + count = int(state.get("count") or 0) + ok = (a.min is None or count >= a.min) and (a.max is None or count <= a.max) + results.append(_result(a, ok, count)) + except Exception as exc: # noqa: BLE001 - a dead page is a result, not a crash + results.append(_result(a, False, None, f"{type(exc).__name__}: {exc}"[:300])) + return results + + +async def evaluate_once(ctx: Any, recorder: Any) -> None: + """Evaluate the run's assertions on the current page and record them, once per run. + + Never launches a browser: a run whose browser is gone records every assertion as an + ``error`` saying so.""" + if not recorder.assertions or recorder.asserts_done: + return + recorder.asserts_done = True + try: + page = await ctx.session.page(revive=False) + except Exception as exc: # noqa: BLE001 - no page is a result + reason = f"no page to check: {type(exc).__name__}: {exc}"[:300] + rows = [_result(a, False, None, reason) for a in recorder.assertions] + else: + rows = await evaluate(page, recorder.assertions) + recorder.add_asserts(rows) diff --git a/src/lyra_browser/uat/batch.py b/src/lyra_browser/uat/batch.py index c4be353..ca3fb42 100644 --- a/src/lyra_browser/uat/batch.py +++ b/src/lyra_browser/uat/batch.py @@ -119,6 +119,7 @@ def build_run_spec(batch: BatchSpec, persona_id: str, runs_dir: Path) -> RunSpec hooks=batch.hooks, limits=batch.limits, tools=batch.tools, + asserts=batch.asserts, out_dir=runs_dir, data_dir=batch.data_dirs.get(persona_id), headless=batch.headless, diff --git a/src/lyra_browser/uat/cli.py b/src/lyra_browser/uat/cli.py index b3eb4c5..a1d0a3e 100644 --- a/src/lyra_browser/uat/cli.py +++ b/src/lyra_browser/uat/cli.py @@ -5,8 +5,10 @@ lyra-uat run [--target FILE] [--brain NAME] [--model M] [--out DIR] ... lyra-uat batch lyra-uat validate - lyra-uat schema [run|batch|report] + lyra-uat schema [run|batch|report|export] lyra-uat render + lyra-uat export [--out FILE] + lyra-uat replay [--out DIR] Exit codes of ``run``: 0 when the run completed (findings or not), 2 when it was cut off (budget, wall clock, harness) or the brain never finished, 130 when @@ -66,11 +68,23 @@ def build_parser() -> argparse.ArgumentParser: validate.add_argument("--target", type=Path) schema = sub.add_parser("schema", help="Print a JSON Schema.") - schema.add_argument("which", nargs="?", default="report", choices=["run", "batch", "report"]) + schema.add_argument( + "which", nargs="?", default="report", choices=["run", "batch", "report", "export"] + ) render = sub.add_parser("render", help="Render report.json as Markdown to stdout.") render.add_argument("report", type=Path) + export = sub.add_parser("export", help="Write a run as a versioned export record.") + export.add_argument("report", type=Path, help="A run directory or its report.json.") + export.add_argument("--out", type=Path, help="File to write (default: stdout).") + + replay = sub.add_parser( + "replay", help="Run a recorded run again with its own tool calls and compare." + ) + replay.add_argument("run_dir", type=Path) + replay.add_argument("--out", type=Path, help="Where the replay's run goes (default: beside).") + demo = sub.add_parser( "demo-site", help="Serve the built-in demo site to run a persona against." ) @@ -209,6 +223,10 @@ def cmd_schema(args: argparse.Namespace) -> int: schema = RunSpec.model_json_schema() elif args.which == "batch": schema = BatchSpec.model_json_schema() + elif args.which == "export": + from .export import export_schema + + schema = export_schema() else: from .report import Report @@ -225,6 +243,27 @@ def cmd_render(args: argparse.Namespace) -> int: return EXIT_OK +def cmd_export(args: argparse.Namespace) -> int: + from .export import export_report, load_report + + record = export_report(load_report(args.report)).model_dump_json(indent=2) + if args.out: + args.out.write_text(record, encoding="utf-8") + print(args.out) + else: + print(record) + return EXIT_OK + + +def cmd_replay(args: argparse.Namespace) -> int: + """0 when the replay reached the same judgement, 2 when it did not.""" + from .replay import replay_run + + result = asyncio.run(replay_run(args.run_dir, out_dir=args.out)) + print(json.dumps(result, indent=2, ensure_ascii=False)) + return EXIT_OK if result["same"] else EXIT_INCOMPLETE + + def cmd_demo_site(args: argparse.Namespace) -> int: from .demo_site import serve_forever @@ -238,6 +277,8 @@ def cmd_demo_site(args: argparse.Namespace) -> int: "validate": cmd_validate, "schema": cmd_schema, "render": cmd_render, + "export": cmd_export, + "replay": cmd_replay, "demo-site": cmd_demo_site, } diff --git a/src/lyra_browser/uat/export.py b/src/lyra_browser/uat/export.py new file mode 100644 index 0000000..f4f3d0a --- /dev/null +++ b/src/lyra_browser/uat/export.py @@ -0,0 +1,130 @@ +"""A run as one flat, versioned record for a benchmark or a dashboard. + +``report.json`` is the whole evidence of a run and changes as lyra-uat grows. An export is +the stable part a scorer reads: which task, whether it succeeded by the spec's own +assertions, what the persona claimed, how long and how much it cost, and the steps it +took. The shape is ``ExportV1``; its JSON Schema is committed next to this module as +``schemas/export.v1.json`` (``lyra-uat schema export`` prints it), and a test keeps the +two the same, so a change of shape is a new version, never a silent edit. + +A benchmark's own result format is a mapping from this record, kept with the adapter +for that benchmark rather than here. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Literal + +from pydantic import Field + +from .report import AssertResult, Model, Report + +EXPORT_FORMAT = "lyra-uat-export" +EXPORT_VERSION = 1 +SCHEMA_FILE = Path(__file__).with_name("schemas") / "export.v1.json" + + +class ExportStep(Model): + n: int + tool: str + args: dict[str, Any] + status: str + url_after: str + counted: bool + duration_ms: int = 0 + screenshot: str | None = None + + +class ExportV1(Model): + format: Literal["lyra-uat-export"] = EXPORT_FORMAT + version: Literal[1] = EXPORT_VERSION + task_id: str = Field(description="the persona id: what was asked") + run_id: str + started_at: str + finished_at: str + duration_s: float + run_status: Literal["completed", "incomplete", "error"] + exit_reason: str + # The judgement a scorer should use: the spec's assertions. None when the spec has none, + # in which case only ``claimed_outcome`` is known, and that is the model's own word. + success: bool | None + asserts: list[AssertResult] + claimed_outcome: Literal["reached_goal", "partial", "blocked", "unknown"] + summary: str + final_url: str + steps_used: int + step_budget: int + tool_calls: int + findings: int + brain_backend: str + brain_model: str | None + input_tokens: int + output_tokens: int + cost_usd: float | None + lyra_browser_version: str + steps: list[ExportStep] + report_json: str | None + + +def export_report(report: Report) -> ExportV1: + final_url = next((s.url_after for s in reversed(report.trace) if s.url_after), "") + return ExportV1( + task_id=report.persona.id, + run_id=report.run.id, + started_at=report.run.started_at, + finished_at=report.run.finished_at, + duration_s=report.run.duration_s, + run_status=report.run.status, + exit_reason=report.run.exit_reason, + success=report.asserts_passed, + asserts=list(report.asserts), + claimed_outcome=report.outcome.status, + summary=report.outcome.summary, + final_url=final_url, + steps_used=report.outcome.steps_used, + step_budget=report.outcome.step_budget, + tool_calls=report.outcome.attempts, + findings=len(report.findings), + brain_backend=report.brain.backend, + brain_model=report.brain.model, + input_tokens=report.brain.input_tokens, + output_tokens=report.brain.output_tokens, + cost_usd=report.brain.cost_usd, + lyra_browser_version=report.run.lyra_browser_version, + steps=[ + ExportStep( + n=s.n, + tool=s.tool, + args=s.args, + status=s.status, + url_after=s.url_after, + counted=s.counted, + duration_ms=s.duration_ms, + screenshot=s.screenshot, + ) + for s in report.trace + ], + report_json=report.artifacts.report_json, + ) + + +def export_schema() -> dict[str, Any]: + return ExportV1.model_json_schema() + + +def load_report(path: Path) -> Report: + """``report.json`` itself, or the run directory that holds it.""" + if path.is_dir(): + path = path / "report.json" + return Report.model_validate_json(path.read_text(encoding="utf-8")) + + +def write_export(report: Report, out: Path) -> Path: + out.write_text(export_report(report).model_dump_json(indent=2), encoding="utf-8") + return out + + +def committed_schema() -> dict[str, Any]: + return json.loads(SCHEMA_FILE.read_text(encoding="utf-8")) diff --git a/src/lyra_browser/uat/recorder.py b/src/lyra_browser/uat/recorder.py index 55acc3c..bf2c47e 100644 --- a/src/lyra_browser/uat/recorder.py +++ b/src/lyra_browser/uat/recorder.py @@ -176,6 +176,9 @@ def __init__( self.finding_count = 0 self.verdicts: dict[str, str] = {} self.finished = False + # The spec's assertions (set by the server builder) and whether they were checked. + self.assertions: list[Any] = [] + self.asserts_done = False self.last_url = "" self._entry_done = False self._system_depth = 0 @@ -369,6 +372,9 @@ def add_verdict(self, *, finding_id: str, status: str, evidence: str) -> None: def add_note(self, text: str) -> None: self._event("note", text=text) + def add_asserts(self, results: list[dict[str, Any]]) -> None: + self._event("asserts", results=results) + def set_finish( self, *, diff --git a/src/lyra_browser/uat/replay.py b/src/lyra_browser/uat/replay.py new file mode 100644 index 0000000..7137b32 --- /dev/null +++ b/src/lyra_browser/uat/replay.py @@ -0,0 +1,76 @@ +"""Run a recorded run again, step for step, and compare the judgement. + +A run's ``trace.jsonl`` holds every tool call the brain made, with its arguments, in +order; ``run.json`` holds the spec. Replaying feeds those calls to the ``scripted`` +brain against the same spec, so no model is involved, and compares what decides a run: +its status and exit reason, the outcome ``finish`` reported, the steps it spent and every +assertion's status. Same judgement means the site behaved the same for the same actions. + +Arguments the audit redacts (typed values of password-like fields) cannot be replayed as +typed; a replay that contains one says so (``exact`` is False) rather than pretend. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from .report import Report +from .server import RUN_FILE, read_run_file + +REDACTED = " dict[str, Any]: + """What a replay must reproduce.""" + return { + "run_status": report.run.status, + "exit_reason": report.run.exit_reason, + "outcome": report.outcome.status, + "steps_used": report.outcome.steps_used, + "asserts": {a.id: a.status for a in report.asserts}, + "asserts_passed": report.asserts_passed, + } + + +def script_from(report: Report) -> tuple[list[dict[str, Any]], bool]: + """The trace as a scripted brain's script, and whether every argument is intact.""" + script, exact = [], True + for step in report.trace: + if any(isinstance(v, str) and v.startswith(REDACTED) for v in step.args.values()): + exact = False + script.append({"tool": step.tool, "args": dict(step.args)}) + return script, exact + + +async def replay_run(run_dir: Path, *, out_dir: Path | None = None) -> dict[str, Any]: + """Replay the run in ``run_dir`` into ``out_dir`` (default: next to it) and compare.""" + from .runner import run_spec + + run_dir = run_dir.resolve() + original = Report.model_validate_json((run_dir / "report.json").read_text(encoding="utf-8")) + spec, _ = read_run_file(run_dir / RUN_FILE) + script, exact = script_from(original) + if not script: + raise ValueError(f"{run_dir}: the trace is empty, there is nothing to replay") + replayed_spec = spec.model_copy( + update={ + "brain": spec.brain.model_copy( + update={"backend": "scripted", "script": script, "model": None} + ), + "out_dir": out_dir or run_dir.parent, + } + ) + replayed = await run_spec(replayed_spec) + before, after = judgement(original), judgement(replayed) + differences = { + k: {"original": before[k], "replay": after[k]} for k in before if before[k] != after[k] + } + return { + "original": str(run_dir / "report.json"), + "replay": replayed.artifacts.report_json, + "same": not differences, + "exact": exact, + "judgement": before, + "differences": differences, + } diff --git a/src/lyra_browser/uat/report.py b/src/lyra_browser/uat/report.py index b0e0080..77eca05 100644 --- a/src/lyra_browser/uat/report.py +++ b/src/lyra_browser/uat/report.py @@ -131,6 +131,15 @@ class Note(Model): text: str +class AssertResult(Model): + id: str + kind: str + status: Literal["passed", "failed", "error", "not_evaluated"] + expected: str + observed: Any = None + error: str | None = None + + class PolicyEvent(Model): step: int kind: str @@ -223,6 +232,10 @@ class Report(Model): side_effects: SideEffects policy: Policy observations: Observations | None = None + # The spec's assertions on the page the run ended on, and their conjunction: True when + # every one passed, False when any did not, None when the spec has none. + asserts: list[AssertResult] = Field(default_factory=list) + asserts_passed: bool | None = None metrics: Metrics = Field(default_factory=Metrics) warnings: list[str] = Field(default_factory=list) hooks: Hooks @@ -328,6 +341,23 @@ def enforced_policies(spec: RunSpec) -> list[str]: return rules +def _assert_results(spec: RunSpec, events: list[dict[str, Any]]) -> list[AssertResult]: + """What the server recorded for the spec's assertions; one it never got to (the + browser died first, a harness exited without finish) is ``not_evaluated``.""" + recorded = next((e for e in reversed(events) if e.get("type") == "asserts"), None) + by_id = {str(r.get("id")): r for r in (recorded or {}).get("results", [])} + out = [] + for a in spec.asserts: + row = by_id.get(a.id) + if row is None: + out.append( + AssertResult(id=a.id, kind=a.kind, status="not_evaluated", expected=a.describe()) + ) + else: + out.append(AssertResult.model_validate(row)) + return out + + def build_report( spec: RunSpec, run_dir: Path, @@ -390,6 +420,7 @@ def build_report( if e.get("type") == "policy" ] finish = next((e for e in reversed(events) if e.get("type") == "finish"), None) + asserts = _assert_results(spec, events) steps_used = sum(1 for s in trace if s.counted) if finish is not None: @@ -474,6 +505,8 @@ def build_report( events=policy_events, ), observations=observations_model, + asserts=asserts, + asserts_passed=all(a.status == "passed" for a in asserts) if asserts else None, metrics=metrics, warnings=warnings, hooks=hooks, @@ -533,6 +566,18 @@ def _short_args(step: TraceStep) -> str: return text if len(text) <= 90 else text[:87] + "..." +def _assert_lines(report: Report) -> list[str]: + if not report.asserts: + return [] + verdict = "all passed" if report.asserts_passed else "NOT all passed" + lines = ["", f"## Assertions ({verdict})", "", "| id | check | status | observed |"] + lines.append("|---|---|---|---|") + for a in report.asserts: + seen = a.error or json.dumps(a.observed, ensure_ascii=False, default=str)[:80] + lines.append(f"| {a.id} | {a.expected} | {a.status} | {seen} |") + return lines + + def render_markdown(report: Report) -> str: p, o, r = report.persona, report.outcome, report.run lines = [ @@ -556,6 +601,7 @@ def render_markdown(report: Report) -> str: if report.metrics.actions_by_tool: moves = ", ".join(f"{k} {v}" for k, v in report.metrics.actions_by_tool.items()) lines += ["", f"Actions: {moves}"] + lines += _assert_lines(report) lines += ["", "## Trace", ""] for s in report.trace: diff --git a/src/lyra_browser/uat/runner.py b/src/lyra_browser/uat/runner.py index 3970697..78fae54 100644 --- a/src/lyra_browser/uat/runner.py +++ b/src/lyra_browser/uat/runner.py @@ -17,6 +17,7 @@ from datetime import UTC, datetime from pathlib import Path +from .asserts import evaluate_once from .brains import BrainError, BrainSession, HarnessBrain, LoopBrain, ToolSpec, make_brain from .prompt import build_system_prompt, build_task_prompt from .report import BrainInfo, ExitReason, HookResult, Hooks, Report, build_report, write_report @@ -108,6 +109,11 @@ async def _run_loop_brain( reason = "brain_error" info = BrainInfo(backend=brain.backend, error=f"{type(exc).__name__}: {exc}") finally: + # A run that never called finish is judged on the page it was left on. + try: + await evaluate_once(ctx, recorder) + except Exception as exc: # noqa: BLE001 - the report must still be written + print(f"lyra-uat: checking the assertions failed: {exc!r}", file=sys.stderr) try: await ctx.session.stop() except Exception as exc: # noqa: BLE001 — closing is best effort at this point diff --git a/src/lyra_browser/uat/schemas/export.v1.json b/src/lyra_browser/uat/schemas/export.v1.json new file mode 100644 index 0000000..64d012d --- /dev/null +++ b/src/lyra_browser/uat/schemas/export.v1.json @@ -0,0 +1,297 @@ +{ + "$defs": { + "AssertResult": { + "additionalProperties": false, + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "kind": { + "title": "Kind", + "type": "string" + }, + "status": { + "enum": [ + "passed", + "failed", + "error", + "not_evaluated" + ], + "title": "Status", + "type": "string" + }, + "expected": { + "title": "Expected", + "type": "string" + }, + "observed": { + "default": null, + "title": "Observed" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Error" + } + }, + "required": [ + "id", + "kind", + "status", + "expected" + ], + "title": "AssertResult", + "type": "object" + }, + "ExportStep": { + "additionalProperties": false, + "properties": { + "n": { + "title": "N", + "type": "integer" + }, + "tool": { + "title": "Tool", + "type": "string" + }, + "args": { + "additionalProperties": true, + "title": "Args", + "type": "object" + }, + "status": { + "title": "Status", + "type": "string" + }, + "url_after": { + "title": "Url After", + "type": "string" + }, + "counted": { + "title": "Counted", + "type": "boolean" + }, + "duration_ms": { + "default": 0, + "title": "Duration Ms", + "type": "integer" + }, + "screenshot": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Screenshot" + } + }, + "required": [ + "n", + "tool", + "args", + "status", + "url_after", + "counted" + ], + "title": "ExportStep", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "format": { + "const": "lyra-uat-export", + "default": "lyra-uat-export", + "title": "Format", + "type": "string" + }, + "version": { + "const": 1, + "default": 1, + "title": "Version", + "type": "integer" + }, + "task_id": { + "description": "the persona id: what was asked", + "title": "Task Id", + "type": "string" + }, + "run_id": { + "title": "Run Id", + "type": "string" + }, + "started_at": { + "title": "Started At", + "type": "string" + }, + "finished_at": { + "title": "Finished At", + "type": "string" + }, + "duration_s": { + "title": "Duration S", + "type": "number" + }, + "run_status": { + "enum": [ + "completed", + "incomplete", + "error" + ], + "title": "Run Status", + "type": "string" + }, + "exit_reason": { + "title": "Exit Reason", + "type": "string" + }, + "success": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Success" + }, + "asserts": { + "items": { + "$ref": "#/$defs/AssertResult" + }, + "title": "Asserts", + "type": "array" + }, + "claimed_outcome": { + "enum": [ + "reached_goal", + "partial", + "blocked", + "unknown" + ], + "title": "Claimed Outcome", + "type": "string" + }, + "summary": { + "title": "Summary", + "type": "string" + }, + "final_url": { + "title": "Final Url", + "type": "string" + }, + "steps_used": { + "title": "Steps Used", + "type": "integer" + }, + "step_budget": { + "title": "Step Budget", + "type": "integer" + }, + "tool_calls": { + "title": "Tool Calls", + "type": "integer" + }, + "findings": { + "title": "Findings", + "type": "integer" + }, + "brain_backend": { + "title": "Brain Backend", + "type": "string" + }, + "brain_model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Brain Model" + }, + "input_tokens": { + "title": "Input Tokens", + "type": "integer" + }, + "output_tokens": { + "title": "Output Tokens", + "type": "integer" + }, + "cost_usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cost Usd" + }, + "lyra_browser_version": { + "title": "Lyra Browser Version", + "type": "string" + }, + "steps": { + "items": { + "$ref": "#/$defs/ExportStep" + }, + "title": "Steps", + "type": "array" + }, + "report_json": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Report Json" + } + }, + "required": [ + "task_id", + "run_id", + "started_at", + "finished_at", + "duration_s", + "run_status", + "exit_reason", + "success", + "asserts", + "claimed_outcome", + "summary", + "final_url", + "steps_used", + "step_budget", + "tool_calls", + "findings", + "brain_backend", + "brain_model", + "input_tokens", + "output_tokens", + "cost_usd", + "lyra_browser_version", + "steps", + "report_json" + ], + "title": "ExportV1", + "type": "object" +} diff --git a/src/lyra_browser/uat/server.py b/src/lyra_browser/uat/server.py index 84a0c79..0a4506a 100644 --- a/src/lyra_browser/uat/server.py +++ b/src/lyra_browser/uat/server.py @@ -106,7 +106,7 @@ def read_run_file(path: Path) -> tuple[RunSpec, Path]: def make_recorder(spec: RunSpec, run_dir: Path) -> RunRecorder: - return RunRecorder( + recorder = RunRecorder( run_dir, step_budget=spec.persona.step_budget, entry_url=spec.persona.entry_url, @@ -114,6 +114,8 @@ def make_recorder(spec: RunSpec, run_dir: Path) -> RunRecorder: auto_screenshot=spec.limits.auto_screenshot, prior_finding_ids=[f.id for f in spec.prior_findings], ) + recorder.assertions = list(spec.asserts) + return recorder async def build_uat_server( diff --git a/src/lyra_browser/uat/spec.py b/src/lyra_browser/uat/spec.py index e318c1a..b691f05 100644 --- a/src/lyra_browser/uat/spec.py +++ b/src/lyra_browser/uat/spec.py @@ -225,6 +225,53 @@ class PriorFinding(Strict): round: str | None = None +AssertKind = Literal[ + "url_matches", "title_contains", "text_present", "text_absent", "selector_count" +] + + +class Assertion(Strict): + """A check on the page the run ended on, evaluated by the server, never the model. + + ``url_matches``: ``value`` is a regular expression searched in the URL. + ``title_contains``: ``value`` is a substring of the title. + ``text_present`` / ``text_absent``: ``value`` in the visible text of the page, or of + what ``selector`` matches. ``selector_count``: how many elements ``selector`` matches, + within ``min``..``max``. + """ + + id: str = Field(min_length=1) + kind: AssertKind + value: str = "" + selector: str | None = None + min: int | None = Field(None, ge=0) + max: int | None = Field(None, ge=0) + + @model_validator(mode="after") + def _complete(self) -> Assertion: + if self.kind == "selector_count": + if not self.selector: + raise ValueError(f"assertion {self.id!r}: selector_count needs a selector") + if self.min is None and self.max is None: + raise ValueError(f"assertion {self.id!r}: selector_count needs min or max") + elif not self.value: + raise ValueError(f"assertion {self.id!r}: {self.kind} needs a value") + if self.kind == "url_matches": + try: + re.compile(self.value) + except re.error as exc: + raise ValueError(f"assertion {self.id!r}: bad regular expression: {exc}") from None + return self + + def describe(self) -> str: + where = f" in {self.selector}" if self.selector else "" + if self.kind == "selector_count": + low = "" if self.min is None else f">= {self.min}" + high = "" if self.max is None else f"<= {self.max}" + return f"count({self.selector}) {' and '.join(x for x in (low, high) if x)}" + return f"{self.kind} {self.value!r}{where}" + + DEFAULT_TOOLS: tuple[str, ...] = ( "navigate", "go_back", @@ -263,6 +310,8 @@ class RunSpec(Strict): hooks: Hooks = Field(default_factory=Hooks) limits: Limits = Field(default_factory=Limits) tools: ToolPolicy = Field(default_factory=ToolPolicy) + # Checks on the page the run ends on (see ``Assertion``); the report's judgement. + asserts: list[Assertion] = Field(default_factory=list) # Where run directories are created: ``//``. out_dir: Path = Path("uat-runs") # The browser profile. None: a fresh one inside the run directory. Set: an @@ -332,6 +381,7 @@ class BatchSpec(Strict): hooks: Hooks = Field(default_factory=Hooks) limits: Limits = Field(default_factory=Limits) tools: ToolPolicy = Field(default_factory=ToolPolicy) + asserts: list[Assertion] = Field(default_factory=list) out_dir: Path = Path("uat-runs") # Persona id -> browser profile (``LYRA_BROWSER_DATA_DIR`` layout) that already holds # its login. Required for ``preseeded`` personas; others get a fresh profile. diff --git a/tests/uat/test_uat_report.py b/tests/uat/test_uat_report.py index d164dca..2eae4cf 100644 --- a/tests/uat/test_uat_report.py +++ b/tests/uat/test_uat_report.py @@ -288,3 +288,119 @@ def test_markdown_flags_a_clean_run_without_a_trace(tmp_path): run_dir, audit = make_run(tmp_path, events=[FINISH], trace=[]) text = render_markdown(build(spec, run_dir, audit)) assert "none reported (and no trace: not a clean run)" in text + + +# --- assertions and the export record ------------------------------------------------ + +ASSERTS = [ + {"id": "done", "kind": "url_matches", "value": "/done"}, + {"id": "paid", "kind": "text_present", "value": "Paid"}, +] + + +def test_assertions_never_checked_are_not_evaluated_and_judge_nothing(tmp_path): + spec = spec_with(asserts=ASSERTS) + run_dir, audit = make_run(tmp_path, events=[FINISH]) + report = build(spec, run_dir, audit) + assert [a.status for a in report.asserts] == ["not_evaluated", "not_evaluated"] + assert report.asserts_passed is False + assert "NOT all passed" in render_markdown(report) + + +def test_recorded_assertions_reach_the_report_and_their_conjunction_is_the_judgement(tmp_path): + spec = spec_with(asserts=ASSERTS) + results = [ + {"id": "done", "kind": "url_matches", "status": "passed", "expected": "x", "observed": "u"}, + {"id": "paid", "kind": "text_present", "status": "passed", "expected": "y", "observed": 1}, + ] + run_dir, audit = make_run(tmp_path, events=[{"type": "asserts", "results": results}, FINISH]) + report = build(spec, run_dir, audit) + assert report.asserts_passed is True and [a.id for a in report.asserts] == ["done", "paid"] + # No assertions in the spec: no judgement either way. + run_dir2, audit2 = make_run(tmp_path / "b", events=[FINISH]) + assert build(spec_with(), run_dir2, audit2).asserts_passed is None + + +def test_an_assertion_spec_must_be_complete(): + import pytest + from pydantic import ValidationError + + from lyra_browser.uat.spec import Assertion + + for bad in ( + {"id": "a", "kind": "text_present"}, # no value + {"id": "a", "kind": "selector_count", "selector": ".x"}, # no bound + {"id": "a", "kind": "selector_count", "min": 1}, # no selector + {"id": "a", "kind": "url_matches", "value": "("}, # not a regular expression + {"id": "a", "kind": "looks_nice", "value": "x"}, + ): + with pytest.raises(ValidationError): + Assertion.model_validate(bad) + ok = Assertion.model_validate({"id": "n", "kind": "selector_count", "selector": "li", "max": 3}) + assert ok.describe() == "count(li) <= 3" + + +async def test_evaluate_reads_url_title_text_and_counts_from_the_page(): + from lyra_browser.uat.asserts import evaluate + from lyra_browser.uat.spec import Assertion + + class Page: + url = "https://shop.example/done?id=1" + + async def title(self): + return "Order complete" + + async def evaluate(self, expression, selector): + if selector == "bad[": + return {"text": "", "count": 0, "error": "not a valid selector"} + if selector == ".item": + return {"text": "a\nb", "count": 2, "error": None} + return {"text": "Thank you. Paid.", "count": None, "error": None} + + checks = [ + Assertion(id="u", kind="url_matches", value=r"/done\?id=\d+$"), + Assertion(id="t", kind="title_contains", value="complete"), + Assertion(id="p", kind="text_present", value="Paid"), + Assertion(id="e", kind="text_absent", value="Error"), + Assertion(id="c", kind="selector_count", selector=".item", min=1, max=1), + Assertion(id="s", kind="text_present", value="x", selector="bad["), + ] + rows = {r["id"]: r for r in await evaluate(Page(), checks)} + assert {k: r["status"] for k, r in rows.items()} == { + "u": "passed", + "t": "passed", + "p": "passed", + "e": "passed", + "c": "failed", + "s": "error", + } + assert rows["c"]["observed"] == 2 and "selector" in rows["s"]["error"] + + +def test_the_committed_export_schema_is_the_models_schema(): + """A change of the export's shape must be a new version file, not a silent edit.""" + from lyra_browser.uat.export import committed_schema, export_schema + + assert committed_schema() == export_schema() + + +def test_an_export_validates_and_a_broken_one_does_not(tmp_path): + import pytest + + from lyra_browser.uat.export import ExportV1, committed_schema, export_report + + spec = spec_with(asserts=ASSERTS[:1]) + results = [ + {"id": "done", "kind": "url_matches", "status": "failed", "expected": "x", "observed": "u"} + ] + run_dir, audit = make_run(tmp_path, events=[{"type": "asserts", "results": results}, FINISH]) + record = export_report(build(spec, run_dir, audit)) + assert record.success is False and record.claimed_outcome == "partial" + assert record.final_url == "https://example.com/pricing" and len(record.steps) == 3 + payload = json.loads(record.model_dump_json()) + ExportV1.model_validate(payload) + jsonschema = pytest.importorskip("jsonschema") + jsonschema.validate(payload, committed_schema()) + payload["version"] = 2 + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(payload, committed_schema()) diff --git a/tests/uat/test_uat_runner.py b/tests/uat/test_uat_runner.py index b13d12d..ddf03ed 100644 --- a/tests/uat/test_uat_runner.py +++ b/tests/uat/test_uat_runner.py @@ -290,3 +290,149 @@ async def test_a_harness_brain_is_reported_from_the_files_it_left(tmp_path): failed = await runner_mod.run_spec(spec, brain=FileWritingHarness(exit_code=1)) # The files say finish; a non-zero harness exit does not override a finished run. assert failed.run.status == "completed" and failed.brain.exit_code == 1 + + +# --- assertions, replay ------------------------------------------------------------------ + + +def _with_asserts(tmp_path: Path, script: list[dict]) -> RunSpec: + return spec_for( + tmp_path, + script, + asserts=[ + {"id": "on-done", "kind": "url_matches", "value": r"/done$"}, + {"id": "thanks", "kind": "text_present", "value": "Thank you"}, + {"id": "no-error", "kind": "text_absent", "value": "Error"}, + {"id": "one-receipt", "kind": "selector_count", "selector": ".receipt", "min": 1}, + ], + ) + + +SCRIPT_TO_DONE = [ + {"tool": "navigate", "args": {"url": ENTRY}, "expect": "ok"}, + {"tool": "navigate", "args": {"url": "https://start.example/done"}, "expect": "ok"}, + {"tool": "finish", "args": {"outcome": "reached_goal", "summary": "Bought it."}}, +] + + +def _page_shows(page: FakePage, text: str, count: int) -> None: + from lyra_browser.uat.asserts import _PAGE_STATE_JS + + page.eval_answers[_PAGE_STATE_JS] = {"text": text, "count": count, "error": None} + + +@pytest.mark.asyncio +async def test_assertions_are_judged_on_the_page_the_persona_finished_on(fake_browser, tmp_path): + _page_shows(fake_browser, "Thank you for your order", 1) + report = await runner_mod.run_spec(_with_asserts(tmp_path, SCRIPT_TO_DONE)) + assert {a.id: a.status for a in report.asserts} == { + "on-done": "passed", + "thanks": "passed", + "no-error": "passed", + "one-receipt": "passed", + } + assert report.asserts_passed is True + assert report.asserts[0].observed == "https://start.example/done" + assert "## Assertions (all passed)" in (Path(report.artifacts.dir) / "report.md").read_text() + + +@pytest.mark.asyncio +async def test_a_failed_assertion_fails_the_judgement_whatever_the_persona_claimed( + fake_browser, tmp_path +): + _page_shows(fake_browser, "Error: card declined", 0) + report = await runner_mod.run_spec(_with_asserts(tmp_path, SCRIPT_TO_DONE)) + assert report.outcome.status == "reached_goal" # what the model said + assert report.asserts_passed is False # what the page shows + statuses = {a.id: a.status for a in report.asserts} + assert statuses == { + "on-done": "passed", + "thanks": "failed", + "no-error": "failed", + "one-receipt": "failed", + } + + +@pytest.mark.asyncio +async def test_a_run_without_finish_is_judged_when_the_browser_closes(fake_browser, tmp_path): + _page_shows(fake_browser, "Thank you", 1) + report = await runner_mod.run_spec(_with_asserts(tmp_path, SCRIPT_TO_DONE[:2])) + assert report.run.exit_reason == "no_finish" + assert report.asserts_passed is True and len(report.asserts) == 4 + + +@pytest.mark.asyncio +async def test_replay_reaches_the_same_judgement_and_says_when_it_does_not(fake_browser, tmp_path): + from lyra_browser.uat.export import committed_schema, export_report + from lyra_browser.uat.replay import replay_run + + _page_shows(fake_browser, "Thank you", 1) + original = await runner_mod.run_spec(_with_asserts(tmp_path, SCRIPT_TO_DONE)) + fake_browser.url = "about:blank" + result = await replay_run(Path(original.artifacts.dir)) + assert result["same"] is True and result["exact"] is True, result + assert result["judgement"]["asserts_passed"] is True + replayed = Report.model_validate_json(Path(result["replay"]).read_text(encoding="utf-8")) + assert replayed.brain.backend == "scripted" + assert [s.tool for s in replayed.trace] == [s.tool for s in original.trace] + + # The site changed: the same actions now end on an error page. + _page_shows(fake_browser, "Error", 0) + fake_browser.url = "about:blank" + changed = await replay_run(Path(original.artifacts.dir), out_dir=tmp_path / "replays") + assert changed["same"] is False + assert changed["differences"]["asserts_passed"] == {"original": True, "replay": False} + + # An export of either validates against the committed schema. + jsonschema = pytest.importorskip("jsonschema") + for report in (original, replayed): + record = json.loads(export_report(report).model_dump_json()) + jsonschema.validate(record, committed_schema()) + assert record["task_id"] == "p1" and record["success"] is True + + +def test_replay_marks_redacted_arguments_as_inexact(tmp_path): + from lyra_browser.uat.replay import script_from + + report = Report.model_validate(_minimal_report(tmp_path)) + script, exact = script_from(report) + assert exact is False + assert script == [ + {"tool": "type_text", "args": {"selector": "#pw", "value": ""}} + ] + + +def _minimal_report(tmp_path: Path) -> dict: + from datetime import UTC, datetime + + from lyra_browser.uat.report import BrainInfo, Hooks, build_report + + spec = spec_for(tmp_path, [{"tool": "finish", "args": {}}]) + run_dir = tmp_path / "r" + run_dir.mkdir() + row = { + "n": 1, + "ts": "t", + "tool": "type_text", + "args": {"selector": "#pw", "value": ""}, + "status": "ok", + "action": True, + "counted": True, + "system": False, + "url_before": ENTRY, + "url_after": ENTRY, + "result_head": "", + } + (run_dir / "trace.jsonl").write_text(json.dumps(row) + "\n") + now = datetime.now(UTC) + report = build_report( + spec, + run_dir, + run_id="r", + started_at=now, + finished_at=now, + brain=BrainInfo(backend="scripted"), + hooks=Hooks(), + audit_path=run_dir / "audit.jsonl", + ) + return report.model_dump(mode="json") From 25f32e3e2388840a00ed2d2145048f1d1b1cd2aa Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Sun, 15 Nov 2026 01:15:42 +0900 Subject: [PATCH 2/2] fix(uat): replay stops before a redacted step; assertions never launch a browser The gate checks assertions, the export and the replay on a run of their own. --- scripts/verify_uat_e2e.py | 25 ++++++++++++++++++++----- src/lyra_browser/uat/asserts.py | 16 ++++++++++++++++ src/lyra_browser/uat/replay.py | 25 +++++++++++++++---------- tests/uat/test_uat_runner.py | 28 ++++++++++++++++++++-------- 4 files changed, 71 insertions(+), 23 deletions(-) diff --git a/scripts/verify_uat_e2e.py b/scripts/verify_uat_e2e.py index f01f16e..e81c258 100644 --- a/scripts/verify_uat_e2e.py +++ b/scripts/verify_uat_e2e.py @@ -119,7 +119,6 @@ async def completed_run(base: str, alt_base: str, out: Path) -> None: budget=10, script=script, hooks={"before": "echo seeded $LYRA_UAT_PERSONA", "after": "echo checked"}, - asserts=ASSERTS, ) report = await run_spec(spec) run_dir = Path(report.artifacts.dir) @@ -183,7 +182,6 @@ async def completed_run(base: str, alt_base: str, out: Path) -> None: ) expect((run_dir / "browser" / "audit.jsonl").is_file(), "run has its own audit log") expect((run_dir / "browser" / "profile").is_dir(), "run has its own browser profile") - await judged_exported_and_replayed(report) # What the pricing page shows after "Buy Studio": its click sets the title to "bought". @@ -197,9 +195,22 @@ async def completed_run(base: str, alt_base: str, out: Path) -> None: ] -async def judged_exported_and_replayed(report) -> None: +async def judged_exported_and_replayed(base: str, out: Path) -> None: + """Assertions, the export and a replay, on a run of their own: straight to the pricing + page by URL, then "Buy Studio", then finish.""" from lyra_browser.uat.export import committed_schema, export_report from lyra_browser.uat.replay import replay_run + from lyra_browser.uat.runner import run_spec + + script = [ + {"tool": "navigate", "args": {"url": base}, "expect": "ok"}, + {"tool": "navigate", "args": {"url": f"{base}pricing"}, "expect": "ok"}, + {"tool": "click", "args": {"selector": "#buy"}, "expect": "ok"}, + {"tool": "finish", "args": {"outcome": "reached_goal", "summary": "Bought."}}, + ] + report = await run_spec(spec_for(base, out, budget=5, script=script, asserts=ASSERTS)) + print(report.one_line()) + expect(report.run.exit_reason == "finish", "judged run finished", report.brain.error) statuses = {a.id: a.status for a in report.asserts} expect( @@ -226,8 +237,11 @@ async def judged_exported_and_replayed(report) -> None: print("PASS export validates against schemas/export.v1.json") expect(record.success is False and record.task_id == "e2e", "export record", record.success) result = await replay_run(Path(report.artifacts.dir)) - expect(result["same"], "replay reaches the same judgement", result["differences"]) - print(f"NOTE replay exact={result['exact']} (typed values the audit redacts are not exact)") + expect( + result["same"] and result["exact"], + "replay of every step reaches the same judgement", + result, + ) async def budget_run(base: str, out: Path) -> None: @@ -258,6 +272,7 @@ async def main(keep: bool) -> None: print(f"demo site {base}, runs in {out}") await completed_run(base, alt_base, out) await budget_run(base, out) + await judged_exported_and_replayed(base, out) finally: server.shutdown() if keep: diff --git a/src/lyra_browser/uat/asserts.py b/src/lyra_browser/uat/asserts.py index 6f18dcc..67236e2 100644 --- a/src/lyra_browser/uat/asserts.py +++ b/src/lyra_browser/uat/asserts.py @@ -93,6 +93,22 @@ async def evaluate_once(ctx: Any, recorder: Any) -> None: if not recorder.assertions or recorder.asserts_done: return recorder.asserts_done = True + if not getattr(ctx.session, "_launched_once", True): + # ``page()`` would open a browser that was never opened, only to judge it. + recorder.add_asserts( + [ + { + "id": a.id, + "kind": a.kind, + "status": "not_evaluated", + "expected": a.describe(), + "observed": None, + "error": "the browser was never opened", + } + for a in recorder.assertions + ] + ) + return try: page = await ctx.session.page(revive=False) except Exception as exc: # noqa: BLE001 - no page is a result diff --git a/src/lyra_browser/uat/replay.py b/src/lyra_browser/uat/replay.py index 7137b32..8a8c61a 100644 --- a/src/lyra_browser/uat/replay.py +++ b/src/lyra_browser/uat/replay.py @@ -6,8 +6,11 @@ its status and exit reason, the outcome ``finish`` reported, the steps it spent and every assertion's status. Same judgement means the site behaved the same for the same actions. -Arguments the audit redacts (typed values of password-like fields) cannot be replayed as -typed; a replay that contains one says so (``exact`` is False) rather than pretend. +The trace is written through the audit's redaction, which hides every typed value +(``type_text``'s ``value`` among them): such a step cannot be replayed as it was typed, +and typing the placeholder instead would be a different action. The replay stops before +the first one and says so (``exact`` False, ``stopped_at`` its step number); its +judgement is then that of a shorter run and is compared all the same. """ from __future__ import annotations @@ -33,14 +36,15 @@ def judgement(report: Report) -> dict[str, Any]: } -def script_from(report: Report) -> tuple[list[dict[str, Any]], bool]: - """The trace as a scripted brain's script, and whether every argument is intact.""" - script, exact = [], True +def script_from(report: Report) -> tuple[list[dict[str, Any]], int | None]: + """The trace as a scripted brain's script, up to the first step whose arguments were + redacted (whose number is returned, None when every step is intact).""" + script: list[dict[str, Any]] = [] for step in report.trace: if any(isinstance(v, str) and v.startswith(REDACTED) for v in step.args.values()): - exact = False + return script, step.n script.append({"tool": step.tool, "args": dict(step.args)}) - return script, exact + return script, None async def replay_run(run_dir: Path, *, out_dir: Path | None = None) -> dict[str, Any]: @@ -50,9 +54,9 @@ async def replay_run(run_dir: Path, *, out_dir: Path | None = None) -> dict[str, run_dir = run_dir.resolve() original = Report.model_validate_json((run_dir / "report.json").read_text(encoding="utf-8")) spec, _ = read_run_file(run_dir / RUN_FILE) - script, exact = script_from(original) + script, stopped_at = script_from(original) if not script: - raise ValueError(f"{run_dir}: the trace is empty, there is nothing to replay") + raise ValueError(f"{run_dir}: no step of the trace can be replayed") replayed_spec = spec.model_copy( update={ "brain": spec.brain.model_copy( @@ -70,7 +74,8 @@ async def replay_run(run_dir: Path, *, out_dir: Path | None = None) -> dict[str, "original": str(run_dir / "report.json"), "replay": replayed.artifacts.report_json, "same": not differences, - "exact": exact, + "exact": stopped_at is None, + "stopped_at": stopped_at, "judgement": before, "differences": differences, } diff --git a/tests/uat/test_uat_runner.py b/tests/uat/test_uat_runner.py index ddf03ed..878ea50 100644 --- a/tests/uat/test_uat_runner.py +++ b/tests/uat/test_uat_runner.py @@ -371,6 +371,7 @@ async def test_replay_reaches_the_same_judgement_and_says_when_it_does_not(fake_ fake_browser.url = "about:blank" result = await replay_run(Path(original.artifacts.dir)) assert result["same"] is True and result["exact"] is True, result + assert result["stopped_at"] is None assert result["judgement"]["asserts_passed"] is True replayed = Report.model_validate_json(Path(result["replay"]).read_text(encoding="utf-8")) assert replayed.brain.backend == "scripted" @@ -391,15 +392,13 @@ async def test_replay_reaches_the_same_judgement_and_says_when_it_does_not(fake_ assert record["task_id"] == "p1" and record["success"] is True -def test_replay_marks_redacted_arguments_as_inexact(tmp_path): +def test_replay_stops_before_a_step_whose_value_was_redacted(tmp_path): from lyra_browser.uat.replay import script_from report = Report.model_validate(_minimal_report(tmp_path)) - script, exact = script_from(report) - assert exact is False - assert script == [ - {"tool": "type_text", "args": {"selector": "#pw", "value": ""}} - ] + script, stopped_at = script_from(report) + assert stopped_at == 2 + assert script == [{"tool": "navigate", "args": {"url": ENTRY}}] def _minimal_report(tmp_path: Path) -> dict: @@ -410,9 +409,22 @@ def _minimal_report(tmp_path: Path) -> dict: spec = spec_for(tmp_path, [{"tool": "finish", "args": {}}]) run_dir = tmp_path / "r" run_dir.mkdir() - row = { + first = { "n": 1, "ts": "t", + "tool": "navigate", + "args": {"url": ENTRY}, + "status": "ok", + "action": True, + "counted": False, + "system": True, + "url_before": "", + "url_after": ENTRY, + "result_head": "", + } + row = { + "n": 2, + "ts": "t", "tool": "type_text", "args": {"selector": "#pw", "value": ""}, "status": "ok", @@ -423,7 +435,7 @@ def _minimal_report(tmp_path: Path) -> dict: "url_after": ENTRY, "result_head": "", } - (run_dir / "trace.jsonl").write_text(json.dumps(row) + "\n") + (run_dir / "trace.jsonl").write_text(json.dumps(first) + "\n" + json.dumps(row) + "\n") now = datetime.now(UTC) report = build_report( spec,