diff --git a/README.md b/README.md index 47541d7..c766d50 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ If you're writing evals, Stepfork complements them: evals measure model quality

- Install the alpha: pip install --pre stepfork + Install the current release: pip install stepfork

Product goal: @@ -127,11 +127,10 @@ pip install --pre stepfork uv pip install --pre stepfork ``` -The `--pre` flag is required while Stepfork is a pre-release. You can also -install from the tagged GitHub repository: +You can also install from source or a tagged release: ```bash -pip install "git+https://github.com/utsab345/stepfork.git@v0.1.0a6" +pip install "git+https://github.com/utsab345/stepfork.git@v0.2.0" ``` Requires Python 3.11, 3.12, or 3.13. This installs the `stepfork` CLI and the @@ -325,9 +324,12 @@ business outcome - it lives in `expected.json`, defined by a human. |---|---| | `stepfork validate [PATH] [--partial] [--verify-integrity]` | Check a `.sftrace` bundle structurally (and its integrity). | | `stepfork inspect [PATH] [--json] [--events] [--errors-only] [--step N]` | Review a bundle locally. | -| `stepfork replay [PATH] --entrypoint MODULE:FUNCTION [--mode M]` | Rerun the entrypoint with dependencies answered from the trace. | +| `stepfork replay [PATH] --entrypoint MODULE:FUNCTION [--mode M] [--allow-live-tool NAME] [--verbose]` | Rerun the entrypoint with dependencies answered from the trace. | +| `stepfork minimize [PATH] --entrypoint MODULE:FUNCTION [--expect-output FILE] [--mode M] [--output PATH] [--overwrite] [--timeout SECONDS] [--attempts N]` | Minimize a failing trace to the smallest reproducible subset. | +| `stepfork fork [PATH] [--at-step N \| --at-event ID] [--output PATH] [--overwrite]` | Fork a trace at a specific step (Python API for overrides). | +| `stepfork score [TEST] [--expect-output PATH] [--bundle PATH] [--json]` | Score an exported expectation against a mutant battery. | | `stepfork diff [BASELINE] [CANDIDATE] [--json]` | Compare the behavior of two runs. | -| `stepfork export [PATH] --pytest --entrypoint MODULE:FUNCTION [--expect-output JSON] [--expect-trajectory JSON] [--output FILE] [--overwrite]` | Generate an executable pytest regression test. | +| `stepfork export [PATH] --pytest --entrypoint MODULE:FUNCTION [--expect-output JSON] [--expect-trajectory JSON] [--output FILE] [--overwrite] [--mode M] [--allow-live-tool NAME]` | Generate an executable pytest regression test. | The full reference (arguments, examples, expected output, and common errors) is in [docs/cli.md](docs/cli.md). @@ -379,7 +381,7 @@ sources live in `docs/`. - [Release checklist](docs/release-checklist.md) - [PyPI publishing checklist](docs/pypi-publishing.md) - [Early adopter guide](docs/community/early-adopter-guide.md) -- [Latest release notes (v0.1.0a6)](docs/releases/v0.1.0a6.md) +- [Latest release notes (v0.2.0)](docs/releases/v0.2.0.md) - [Changelog](CHANGELOG.md) ## Security and Limitations @@ -387,8 +389,8 @@ sources live in `docs/`. Stepfork is experimental. The API, CLI, and `.sftrace` trace format may change before v1.0. -- **Alpha on PyPI.** `pip install --pre stepfork` installs the current - pre-release; the API and trace format may still change. +- **Beta status.** `pip install stepfork` installs the current release; the API + and trace format may still change before v1.0. - **Limited integrations.** Optional adapters cover the OpenAI Python SDK (synchronous and asynchronous, non-streaming `chat.completions.create`) and LangGraph (synchronous and asynchronous `BaseChatModel` generation and diff --git a/docs/cli.md b/docs/cli.md index b13dc11..c2ff12f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,7 +1,7 @@ # CLI Reference The `stepfork` CLI manages `.sftrace` bundles. This page documents every -command as of v0.1.0a6. Run `stepfork --help` for an up-to-date summary. +command as of v0.2.0. Run `stepfork --help` for an up-to-date summary. ```text Usage: stepfork [OPTIONS] COMMAND [ARGS]... @@ -22,6 +22,8 @@ Commands: - [inspect](#inspect) - [replay](#replay) - [minimize](#minimize) +- [fork](#fork) +- [score](#score) - [diff](#diff) - [export](#export) @@ -261,7 +263,7 @@ Options: | Option | Description | |---|---| | `--expect-output FILE` | JSON file with expected output for comparison. | -| `--mode`, `-m` | Replay mode: `frozen`, `live`, or `hybrid`. Default: `frozen`. | +| `--mode`, `-m` | Replay mode: `frozen`, `live`, `forbidden`, `manual`, or `derived`. Default: `frozen`. | | `--output`, `-o` | Path for the minimized `.sftrace` bundle. Default: `-minimized.sftrace`. | | `--overwrite` | Overwrite the output bundle if it exists. Default: `false`. | | `--timeout SECONDS` | Timeout in seconds for each replay attempt. Default: `30.0`. | @@ -437,6 +439,91 @@ Common errors: --- +## score + +Score an exported expectation against a mutant battery derived from the +recorded run, without executing anything. + +```text +Usage: stepfork score [OPTIONS] TEST +``` + +An exported test asserts `EXPECTATION`. Two things can satisfy that assertion +and pass the same green run: the claim you meant to check, or a copy of the run +the recording came from. `score` derives the battery that separates them -- the +claim-drawn row is the recorded run's own output, and the code-drawn rows +perturb the expectation's leaves -- and reports which rows the expectation +rejects. + +The test file is **read, never executed**: it is parsed with `ast`, JSON-shaped +values are read with `ast.literal_eval`, and the bundle path is read from the +expressions the exporter writes. Nothing is imported and no entrypoint runs. + +Arguments: + +| Argument | Description | +|---|---| +| `TEST` | Exported pytest regression test. Required. | + +Options: + +| Option | Description | +|---|---| +| `--expect-output ` | Score this expectation instead of the one the test embeds. | +| `--bundle ` | Bundle to read the recorded run from. Default: the `TRACE_PATH` the test names. | +| `--json` | Print the score as JSON. | +| `--help` | Show command help and exit. | + +A claim-pinned expectation rejects the recorded run: + +```text +battery: 1 claim-drawn + 6 code-drawn mutant(s), derived from test_notify_regression.py and notify.sftrace + + claim the recorded run killed (first difference at carrier) + code carrier: with a suffix killed (first difference at carrier) + code order_id: with a suffix killed (first difference at order_id) + code should_notify: flipped killed (first difference at should_notify) + +claim-pinned: rejected 7/7 mutants, including every claim-drawn mutant +pins: 1 live (carrier); 2 leaf(s) reproduce the run (order_id, should_notify) +``` + +An expectation that has drifted into the recording accepts it, which is the +finding: the generated test would pass against the code that produced the +recording. + +```text +run-pinned: the expectation accepts the claim-drawn mutant(s) the recorded run, so it encodes the recorded run rather than the claim +The generated test passes against the code that produced the recording. Treat the expectation as an unapproved oracle. +pins: 0 live (none); 3 leaf(s) reproduce the run (carrier, order_id, should_notify) +``` + +The `pins` line answers a second question the verdict does not: which leaves +carry a value the run did not produce (live pins), and which merely reproduce +the run. A leaf that reproduces the run is not a defect -- a fix does not change +every field -- but it is not evidence of anchoring either. + +Exit codes: + +| Code | Meaning | +|---|---| +| `0` | `claim_pinned`: the expectation rejects the recorded run. | +| `1` | `run_pinned`: the expectation accepts it; it is an unapproved oracle. | +| `2` | Cannot answer: no expectation to score, no recorded run to derive a claim-drawn mutant from, or a test file that cannot be read. | + +Common errors: + +- `:: EXPECTATION is not a literal value ...` -- the test was + hand-edited; `score` reads the literals the export wrote. +- `cannot read the bundle path from this expression; pass --bundle ...` -- the + `TRACE_PATH` expression is outside the shapes the exporter writes. +- `trace bundle not found: ` -- the bundle the test names is not at that + path. Copy it beside the test, or pass `--bundle`. +- `no expectation to score` -- the test asserts the recorded run without an + `--expect-output`, so there is no claim for a mutant to separate from it. + +--- + ## Error handling Command output is sanitized before display: secrets in trace-derived text diff --git a/docs/community/early-adopter-guide.md b/docs/community/early-adopter-guide.md index 3cd8d70..8211758 100644 --- a/docs/community/early-adopter-guide.md +++ b/docs/community/early-adopter-guide.md @@ -32,14 +32,14 @@ Stepfork is on PyPI. Install the current alpha: ```bash python -m venv .venv source .venv/bin/activate -pip install --pre stepfork +pip install stepfork pip install pytest stepfork --version ``` The `--pre` flag is required while Stepfork is a pre-release. Alternatively, install the tagged repository build: -`pip install "git+https://github.com/utsab345/stepfork.git@v0.1.0a6"`. +`pip install "git+https://github.com/utsab345/stepfork.git@v0.2.0"`. Requires Python 3.11, 3.12, or 3.13. diff --git a/docs/getting-started.md b/docs/getting-started.md index cc74e37..2d8437f 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -9,14 +9,14 @@ This guide installs Stepfork and walks through the full workflow with a Install the current alpha from PyPI: ```bash -pip install --pre stepfork +pip install stepfork ``` The `--pre` flag is required while Stepfork is a pre-release. To install from the tagged GitHub repository instead: ```bash -pip install "git+https://github.com/utsab345/stepfork.git@v0.1.0a6" +pip install "git+https://github.com/utsab345/stepfork.git@v0.2.0" ``` Requires Python 3.11, 3.12, or 3.13. This installs the `stepfork` CLI and the diff --git a/docs/index.md b/docs/index.md index 8113ca9..3688c24 100644 --- a/docs/index.md +++ b/docs/index.md @@ -73,4 +73,4 @@ in this repository. Stepfork is experimental. See the [changelog](https://github.com/utsab345/stepfork/blob/main/CHANGELOG.md) and -the [v0.1.0a6 release notes](releases/v0.1.0a6.md) for what shipped. +the [v0.2.0 release notes](releases/v0.2.0.md) for what shipped. diff --git a/docs/minimize.md b/docs/minimize.md index b7fea85..6a5bf7e 100644 --- a/docs/minimize.md +++ b/docs/minimize.md @@ -85,7 +85,7 @@ stepfork minimize --entrypoint MODULE:FUNCTION [OPTIONS] | Option | Description | Default | |--------|-------------|---------| | `--expect-output FILE` | JSON file with expected output for comparison | None | -| `--mode`, `-m` | Replay mode: `frozen`, `live`, or `hybrid` | `frozen` | +| `--mode`, `-m` | Replay mode: `frozen`, `live`, `forbidden`, `manual`, or `derived` | `frozen` | | `--output`, `-o` | Path for the minimized `.sftrace` bundle | `-minimized.sftrace` | | `--overwrite` | Overwrite the output bundle if it exists | `false` | | `--timeout SECONDS` | Timeout for each replay attempt | `30.0` | diff --git a/docs/pytest-export.md b/docs/pytest-export.md index 34a8398..bbf4548 100644 --- a/docs/pytest-export.md +++ b/docs/pytest-export.md @@ -9,6 +9,32 @@ A trace records what the agent *did*, including the bug. If you generate a test that asserts the recorded output, it passes against the very code that produced the recording, which is not a regression test. +## Frozen-replay contract and limitations + +Stepfork's pytest export uses frozen replay: the generated test executes your +trusted entrypoint with instrumented tool and LLM calls answered from the recorded +trace. The contract is strict to preserve determinism and prevent silent +acceptance of unexpected behavior. + +**Supported when:** the fixed implementation makes the same instrumented calls in +the same order with the same inputs as recorded in the failure trace. The +regression test can then assert the corrected output against the trace. + +**Not supported when:** the fix legitimately changes later tool arguments, +invokes a tool not present in the recorded trace, or omits/moves recorded calls. +In these cases, frozen replay will raise a divergence (e.g., `ReplayMismatchError` +for changed inputs, or `ReplayExhaustedError`/unmatched calls for sequence +changes). The test generated from the failure trace **cannot pass** on the fixed +code because replay cannot satisfy the recorded sequence. + +When a fix changes the call sequence or arguments, record the *fixed* run and +export a test from that trace instead. The exported test will fail on the buggy +code (by replay divergence) and pass on the fixed code. `stepfork score` can help +assess whether an expectation is claim-pinned (rejects the recorded run) or +run-pinned (accepts it). + +--- + The regression test needs a separate **expected outcome**: the behavior you want after the bug is fixed. Pass it with `--expect-output`. @@ -82,6 +108,62 @@ not embed or copy the fixture. If moving the test to another project or CI, copy the bundle with it and verify `TRACE_PATH` resolves there. These are test environment requirements; `pytest` is not a Stepfork runtime dependency. +## Is the expectation still anchored? (`stepfork score`) + +An expectation can be two different things and pass the same green run: the +claim you meant to assert, or a copy of the run you wrote it from. `stepfork +score` separates them, and it does it by derivation rather than by execution. + +```bash +stepfork score test_booking_regression.py +stepfork score test_booking_regression.py --expect-output expected.json --json +``` + +Two values are already at hand when the export is prepared, and both are read +back rather than re-derived: + +- the **expectation** the generated test asserts; +- the **recorded run's output** (`RunEnd.output` in the bundle), which is the + value the buggy code that produced the recording actually returned. + +The claim-drawn row is that second value verbatim: the run's own output is the +most plausible output of a variant implementation, because an implementation did +produce it. It is the row that discriminates. A claim-pinned expectation rejects +it -- the run it came from is the bug, so the expectation must fail against it. +A run-pinned expectation accepts it, and that acceptance is the finding: the +granularity of the comparison means the generated test would pass against the +code that produced the recording. + +The remaining rows are **controls**, not discriminators: perturbations of the +expectation's own leaves (an integer moved by one, a string suffixed, a value +replaced by `null`). A run-pinned expectation rejects those too, so they cannot +separate the two readings by themselves; what they do is make the contrast +legible, because a battery in which every control is killed and the claim-drawn +row survives reads differently from one in which everything is killed. + +`score` also reports **pin coverage**: which leaves of the expectation carry a +value the run did not produce (live pins), and which merely reproduce the run. +A leaf that reproduces the run is not a defect -- a fix does not change every +field -- but it is not evidence of anchoring either, which is why it is reported +rather than counted. + +Exit codes: `0` when the expectation is claim-pinned, `1` when it is run-pinned, +`2` when the question cannot be answered. + +Three limits are worth stating plainly. + +- **No recorded run, no claim-drawn row.** A trace whose run did not complete + carries no output, so there is nothing to draw the discriminating row from. + The battery is then the controls alone and the verdict is `undetermined` -- a + reading, not a failure, and the honest one for a battery that cannot separate + the two. +- **The bundle must be there.** `score` reads the recorded run from the bundle + the test names, so it has the same requirement the test has. Pass `--bundle` + to point at one elsewhere. +- **Nothing is executed.** Neither the test file nor the entrypoint runs: the + file is parsed, the literals are read, and each row is scored with the same + canonical comparison the generated test uses. + ## Generated test The generated file is produced from a fixed, reviewed template. Trace payloads diff --git a/docs/releases/v0.2.0.md b/docs/releases/v0.2.0.md new file mode 100644 index 0000000..46fc6bf --- /dev/null +++ b/docs/releases/v0.2.0.md @@ -0,0 +1,46 @@ +# Stepfork v0.2.0 - Behavioral Replay + +This current release adds trajectory assertions, async instrumented boundaries, and an +opt-in hybrid mode for selected live LLM calls. Existing `.sftrace` bundles +remain compatible; frozen replay remains the default. + +## Changes + +- Assert tool calls, absence, counts, JSON arguments, order, sequences, and a + maximum dependency-step count against the execution that actually replayed. + Generated pytest tests can include a reviewed JSON trajectory expectation. +- Record and frozen-replay async tools, LLM requests, and agent entrypoints. + Supported OpenAI and LangGraph adapters now include async paths. +- Re-execute selected LLMs in hybrid replay while recorded tools stay frozen. + The [offline model-decision example](https://github.com/utsab345/stepfork/tree/v0.2.0/examples/model_decision) + uses a deterministic fake provider and shows a failing regression before a + prompt fix and a passing one afterward. It does not establish real-model + quality improvement. +- Live tool execution now requires `allow_live_tools={"name"}` or + `--allow-live-tool name`. Previously, `mode="live"` ran every matching tool. + See [migration guidance](../replay.md#modes). + +Hybrid replay is **not deterministic**. Selected live models can incur network +calls, token costs, and latency. The tool allowlist controls instrumented tool +bodies by name; it does not sandbox arbitrary code or uninstrumented side +effects. Concurrent async calls still match one strict invocation sequence, +while task scheduling remains outside Stepfork's control. + +## Installation + +```bash +pip install stepfork +``` + +Requires Python 3.11, 3.12, or 3.13. + +## Verification + +- 444 tests passed on Python 3.11, 3.12, and 3.13. Python 3.12 combined + statement/branch coverage was 92%. +- Ruff, MyPy, strict documentation builds, six offline demos, source and wheel + builds, and an installed-wheel smoke of the new APIs passed locally. + +## Feedback + +https://github.com/utsab345/stepfork/issues diff --git a/src/stepfork/cli/export.py b/src/stepfork/cli/export.py index a04b877..d5f01f8 100644 --- a/src/stepfork/cli/export.py +++ b/src/stepfork/cli/export.py @@ -17,10 +17,9 @@ ExportExistsError, export_pytest_test, ) +from stepfork.export.mutants import recorded_output from stepfork.trace import ( JsonValue, - RunEnd, - RunStatus, Trace, TraceStorageError, validate_bundle, @@ -156,7 +155,7 @@ def export_command( expectation = _load_expectation(expect_output) has_expectation = True else: - expectation = _recorded_output(trace) + expectation = recorded_output(trace) has_expectation = expectation is not None warning = RECORDED_OUTPUT_WARNING @@ -223,13 +222,6 @@ def _load_expectation(path: Path) -> JsonValue: return redact_json(payload).value -def _recorded_output(trace: Trace) -> JsonValue | None: - for event in reversed(trace.events): - if isinstance(event, RunEnd) and event.run_status is RunStatus.COMPLETED: - return event.output - return None - - def _safe_stem(path: Path) -> str: stem = re.sub(r"\W+", "_", path.stem).strip("_") return stem or "trace" diff --git a/src/stepfork/cli/main.py b/src/stepfork/cli/main.py index 7881044..b7b06e7 100644 --- a/src/stepfork/cli/main.py +++ b/src/stepfork/cli/main.py @@ -12,6 +12,7 @@ from stepfork.cli.inspect import inspect_command from stepfork.cli.minimize import minimize_command from stepfork.cli.replay import replay_command +from stepfork.cli.score import score_command from stepfork.cli.validate import validate_command app = typer.Typer( @@ -28,6 +29,7 @@ app.command("inspect")(inspect_command) app.command("minimize")(minimize_command) app.command("replay")(replay_command) +app.command("score")(score_command) app.command("validate")(validate_command) diff --git a/src/stepfork/cli/minimize.py b/src/stepfork/cli/minimize.py index 541abf2..d03a106 100644 --- a/src/stepfork/cli/minimize.py +++ b/src/stepfork/cli/minimize.py @@ -49,7 +49,7 @@ def minimize_command( typer.Option( "--mode", "-m", - help="Replay mode: frozen (default), live, or hybrid", + help="Replay mode: frozen (default), live, forbidden, manual, or derived", ), ] = "frozen", output: Annotated[ @@ -126,6 +126,13 @@ def minimize_command( # Run minimization console.print("[yellow]Detecting failure signature...[/yellow]") + try: + from stepfork.trace.replay_policy import ReplayPolicy + + ReplayPolicy(mode) + except ValueError: + console.print(f"[red]Invalid replay mode {mode!r}.[/red]") + raise typer.Exit(2) from None engine = MinimizationEngine( trace, entrypoint, diff --git a/src/stepfork/cli/score.py b/src/stepfork/cli/score.py new file mode 100644 index 0000000..045595a --- /dev/null +++ b/src/stepfork/cli/score.py @@ -0,0 +1,250 @@ +"""`stepfork score` command.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Annotated + +import typer +from rich.console import Console + +from stepfork.export.mutants import ( + PinCoverage, + derive_battery, + pin_coverage, + recorded_output, +) +from stepfork.export.scoring import ExpectationScore, score_expectation +from stepfork.export.testfile import ExportedTestError, read_exported_test +from stepfork.trace import JsonValue, Trace, TraceStorageError, redact_json + +console = Console() + +EXIT_ANCHORED = 0 +EXIT_RUN_PINNED = 1 +EXIT_CANNOT_SCORE = 2 + + +def score_command( + test: Annotated[ + Path, + typer.Argument( + help="Exported pytest regression test (read, never executed).", + exists=False, + file_okay=True, + dir_okay=False, + ), + ], + expect_output: Annotated[ + Path | None, + typer.Option( + "--expect-output", + help="Score this expectation instead of the one the test embeds.", + ), + ] = None, + bundle: Annotated[ + Path | None, + typer.Option( + "--bundle", + help="Trace bundle to read the recorded run from " + "(default: the TRACE_PATH the test names).", + ), + ] = None, + as_json: Annotated[ + bool, + typer.Option("--json", help="Print the score as JSON."), + ] = False, +) -> None: + """Score an exported expectation against a battery derived from the run. + + The battery is derived, not executed: the claim-drawn row is the recorded + run's own output, and the code-drawn rows are perturbations of the + expectation's leaves. A claim-pinned expectation rejects the recorded run; + a run-pinned one accepts it, which means the generated test passes against + the code that produced the recording. + + Exits 0 when the expectation anchors the claim, 1 when it is run-pinned, + and 2 when the question cannot be answered (no expectation, no recorded + run to draw a claim-drawn mutant from, or a file that cannot be read). + """ + try: + exported = read_exported_test(test) + except ExportedTestError as exc: + _fail(str(exc)) + return + + if expect_output is not None: + expectation = _load_json(expect_output) + if expectation is None: + return + has_expectation = True + else: + expectation = exported.expectation + has_expectation = exported.has_expectation + + destination = bundle if bundle is not None else exported.trace_path + if destination is None: + _fail( + f"{test} names no trace bundle; pass --bundle to name the .sftrace " + "bundle to read the recorded run from" + ) + return + if not destination.exists(): + _fail(f"trace bundle not found: {destination}") + return + + try: + trace = Trace.load(destination) + except TraceStorageError as exc: + _fail(f"trace bundle unreadable: {exc}") + return + + recorded = recorded_output(trace) + if recorded is not None: + recorded = redact_json(recorded).value + battery = derive_battery(expectation, recorded, has_expectation=has_expectation) + score = score_expectation(expectation, battery, has_expectation=has_expectation) + coverage = ( + pin_coverage(expectation, recorded) + if recorded is not None and has_expectation + else None + ) + + if as_json: + # Plain stdout, not the rich console: markup would be interpreted and + # a wrapped line is no longer one JSON document. + typer.echo( + json.dumps( + _as_json(score, coverage, test, destination), + indent=2, + sort_keys=False, + ) + ) + else: + _report(score, coverage, test, destination) + raise typer.Exit(_exit_status(score)) + + +def _exit_status(score: ExpectationScore) -> int: + if score.verdict == "claim_pinned": + return EXIT_ANCHORED + if score.verdict == "run_pinned": + return EXIT_RUN_PINNED + return EXIT_CANNOT_SCORE + + +def _report( + score: ExpectationScore, + coverage: PinCoverage | None, + test: Path, + bundle: Path, +) -> None: + """Print the battery and its verdict.""" + if score.verdict == "no_expectation": + console.print("[yellow]no expectation to score[/yellow]") + console.print( + f"{test} asserts the recorded run without an --expect-output; " + "there is no claim for a mutant to separate from it." + ) + return + + claim = [mutant for mutant in score.mutants if mutant.kind == "claim"] + code = [mutant for mutant in score.mutants if mutant.kind == "code"] + console.print( + f"battery: {len(claim)} claim-drawn + {len(code)} code-drawn mutant(s), " + f"derived from {test.name} and {bundle.name}" + ) + console.print("") + for mutant in score.mutants: + outcome = "killed" if mutant.killed else "SURVIVED" + location = ( + f" (first difference at {mutant.differing_path})" + if mutant.differing_path + else "" + ) + console.print(f" {mutant.kind:<5} {mutant.name:<36} {outcome}{location}") + console.print("") + + if score.verdict == "run_pinned": + console.print(f"[red]{score.summary()}[/red]") + console.print( + "The generated test passes against the code that produced the " + "recording. Treat the expectation as an unapproved oracle." + ) + elif score.verdict == "undetermined": + console.print(f"[yellow]{score.summary()}[/yellow]") + console.print( + "No recorded run output was available, so no claim-drawn mutant " + "could be derived. Re-record the trace, or score against a bundle " + "that carries a completed run." + ) + else: + console.print(f"[green]{score.summary()}[/green]") + console.print( + "The expectation rejects the run it was drawn from, so it encodes " + "the claim rather than the recording." + ) + + if coverage is not None: + console.print( + f"pins: {len(coverage.live)} live ({_joined(coverage.live)}); " + f"{len(coverage.reproduced)} leaf(s) reproduce the run " + f"({_joined(coverage.reproduced)})" + ) + + +def _joined(paths: tuple[str, ...]) -> str: + return ", ".join(paths) if paths else "none" + + +def _as_json( + score: ExpectationScore, + coverage: PinCoverage | None, + test: Path, + bundle: Path, +) -> dict[str, JsonValue]: + """Return the score as a JSON-shaped payload.""" + return { + "test": str(test), + "bundle": str(bundle), + "verdict": score.verdict, + "killed": score.killed, + "total": score.total, + "discrimination": score.discrimination, + "mutants": [ + { + "name": mutant.name, + "kind": mutant.kind, + "killed": mutant.killed, + "differing_path": mutant.differing_path, + } + for mutant in score.mutants + ], + "pins": ( + { + "live": list(coverage.live), + "reproduced": list(coverage.reproduced), + } + if coverage is not None + else None + ), + } + + +def _load_json(path: Path) -> JsonValue | None: + """Read an expectation file, reporting a read failure as a cannot-score.""" + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except OSError as exc: + _fail(f"cannot read {path}: {exc}") + return None + except json.JSONDecodeError as exc: + _fail(f"{path} is not valid JSON: {exc.msg}") + return None + return redact_json(payload).value + + +def _fail(message: str) -> None: + console.print(f"[red]{message}[/red]") + raise typer.Exit(EXIT_CANNOT_SCORE) diff --git a/src/stepfork/export/__init__.py b/src/stepfork/export/__init__.py index b483258..52fae79 100644 --- a/src/stepfork/export/__init__.py +++ b/src/stepfork/export/__init__.py @@ -7,6 +7,13 @@ export_pytest_test, generate_pytest_source, ) +from stepfork.export.mutants import ( + RECORDED_RUN_MUTANT, + PinCoverage, + derive_battery, + pin_coverage, + recorded_output, +) from stepfork.export.runtime import run_regression_case from stepfork.export.scoring import ( ExpectationScore, @@ -17,15 +24,20 @@ ) __all__ = [ + "RECORDED_RUN_MUTANT", "EntrypointError", "ExpectationScore", "ExportError", "ExportExistsError", "MutantOutcome", "MutantScore", + "PinCoverage", "ScoringError", + "derive_battery", "export_pytest_test", "generate_pytest_source", + "pin_coverage", + "recorded_output", "resolve_entrypoint", "run_regression_case", "score_expectation", diff --git a/src/stepfork/export/mutants.py b/src/stepfork/export/mutants.py new file mode 100644 index 0000000..295dbf3 --- /dev/null +++ b/src/stepfork/export/mutants.py @@ -0,0 +1,296 @@ +"""Derive a mutant battery from an exported expectation and the recorded run. + +:mod:`stepfork.export.scoring` scores a battery the caller supplies. Building +that battery by hand is the part that does not scale, and it is the part that +is easy to skip: an expectation that has drifted into a copy of the recorded +run looks exactly like an anchored one until a claim-drawn mutant separates +them. This module derives the battery mechanically from two things the export +already has. + +Where the two mutants come from +------------------------------- + +An exported test compares the replayed output against ``EXPECTATION``. Two +values are therefore already at hand when the export is prepared: + +* the **expectation** -- the claim, as the generated test asserts it; +* the **recorded run's output** -- the ``RunEnd.output`` of the trace bundle, + which is the value the code that produced the recording actually returned. + +The claim-drawn mutant is that second value, verbatim: the run's own output is +the most plausible output of a variant implementation, because some +implementation did produce it. It is the row that discriminates. A +claim-pinned expectation rejects it -- the run it came from is the bug, so the +expectation must fail against it. A run-pinned expectation accepts it, and +that acceptance is the finding: the test would pass against the code that +produced the recording. + +The code-drawn mutants are perturbations of the *claim's* own leaves -- an +integer moved by one, a string suffixed, a value replaced by ``None``. They +are controls, not discriminators: a run-pinned expectation rejects them too, so +they cannot separate the two readings by themselves. They are what makes the +reading legible, because the contrast between a row family that is always +killed and a row that survives is the whole measurement. + +Two guards keep the families honest +----------------------------------- + +* A perturbation that would land on the recorded value at that leaf is + **skipped**. A row carrying the run's own value is a claim-drawn row wearing a + code-drawn label, and it would let a run-pinned expectation read as though the + battery had caught something. +* The claim-drawn row is the recorded output **whole**, never a projection of it + onto the expectation's shape. The generated test compares whole values, and a + comparison that ignores keys the claim does not name would accept an + expectation that is *stricter* than the run -- the one reading this module + must not produce. + +Per-leaf attribution comes from :func:`pin_coverage`, which reports which +leaves of the expectation carry a claim (they differ from the run) and which +merely reproduce it. It is a comparison of the two values, not a mutation, so +it stays exact where a mutation would only be indicative. + +Nothing here executes anything: a mutant is a value, the comparison is +canonical JSON equality, and running a variant remains the caller's business. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from dataclasses import dataclass +from typing import TypeAlias, cast + +from stepfork.diff.compare import values_equal +from stepfork.export.scoring import MutantOutcome +from stepfork.trace import JsonValue, RunEnd, RunStatus, Trace + +RECORDED_RUN_MUTANT = "the recorded run" +"""Name of the claim-drawn mutant: the run's own output, unscored for shape.""" + +LeafPath: TypeAlias = tuple[str | int, ...] +"""A leaf's location, as a walk from the root: keys as strings, indices as ints.""" + +PathValue: TypeAlias = tuple[str, JsonValue] +"""A mutant label and the value it substitutes at one leaf.""" + + +def recorded_output(trace: Trace) -> JsonValue | None: + """Return the output of the last completed run, or None when it has none. + + A trace that ended in a failure state carries no output, and neither does + one recorded before the run reached ``RunEnd``. The absence is reported + rather than worked around: with no recorded output there is nothing to + draw a claim-drawn mutant from, and the honest verdict is that the battery + cannot separate the two readings. + """ + for event in reversed(trace.events): + if isinstance(event, RunEnd) and event.run_status is RunStatus.COMPLETED: + return event.output + return None + + +def render_path(path: LeafPath) -> str: + """Render a leaf path the way :func:`first_difference` renders one.""" + rendered = "" + for step in path: + if isinstance(step, int): + rendered += f"[{step}]" + else: + rendered += f".{step}" if rendered else step + return rendered or "$" + + +@dataclass(frozen=True) +class PinCoverage: + """Which leaves of an expectation carry a claim, and which reproduce the run. + + ``reproduced`` leaves are not defects: a fix does not change every field, + and an expectation that requires an untouched field to keep its recorded + value is asserting the fix, not the run. What they are is *not evidence of + anchoring*, and a summary that reported only a count of leaves would read + them as though they were. + """ + + live: tuple[str, ...] + """Paths whose expectation value differs from the recorded run's value.""" + + reproduced: tuple[str, ...] + """Paths whose expectation value equals the recorded run's value.""" + + @property + def total(self) -> int: + """Number of leaves compared.""" + return len(self.live) + len(self.reproduced) + + @property + def anchored(self) -> bool: + """True when at least one leaf carries a value the run did not produce.""" + return bool(self.live) + + +def derive_battery( + expectation: JsonValue, + recorded: JsonValue | None, + *, + has_expectation: bool = True, +) -> tuple[MutantOutcome, ...]: + """Derive a mutant battery from an expectation and a recorded run. + + The battery is deterministic: leaves are visited in sorted key order, and + each leaf contributes its perturbations in a fixed order, so two runs over + the same inputs produce the same rows with the same names. + + Args: + expectation: The expectation the generated test asserts. + recorded: The recorded run's output, or None when the trace has none. + None is not an error: the battery is then the code-drawn controls + alone, which scores as ``undetermined`` -- the honest reading of a + battery that cannot separate the two. + has_expectation: False when the export carried no expectation at all. + The battery is then empty, because the generated test performs no + comparison for a mutant to change the outcome of. + + Returns: + One claim-drawn row (when a recorded output exists) followed by the + code-drawn control rows. + """ + if not has_expectation: + return () + + mutants: list[MutantOutcome] = [] + if recorded is not None: + mutants.append( + MutantOutcome( + name=RECORDED_RUN_MUTANT, + kind="claim", + output=recorded, + ) + ) + + for path, value in _leaves(expectation): + found, recorded_value = ( + _at(recorded, path) if recorded is not None else (False, None) + ) + for label, replacement in _perturbations(value): + if found and values_equal(replacement, recorded_value): + continue + mutants.append( + MutantOutcome( + name=f"{render_path(path)}: {label}", + kind="code", + output=_replace(expectation, path, replacement), + ) + ) + return tuple(mutants) + + +def pin_coverage(expectation: JsonValue, recorded: JsonValue) -> PinCoverage: + """Report which expectation leaves a recorded run can and cannot account for. + + A leaf the run does not carry at all counts as live: the claim states + something about a value the run never produced, which is a claim the run + cannot have pinned. + """ + live: list[str] = [] + reproduced: list[str] = [] + for path, value in _leaves(expectation): + found, recorded_value = _at(recorded, path) + if found and values_equal(value, recorded_value): + reproduced.append(render_path(path)) + else: + live.append(render_path(path)) + return PinCoverage(live=tuple(live), reproduced=tuple(reproduced)) + + +def _is_leaf(value: JsonValue) -> bool: + """True for a scalar, and for a container with nothing inside it. + + An empty container is a leaf because there is no value below it to compare: + ``{"a": {}}`` claims that ``a`` holds nothing, and a run that filled ``a`` + contradicts that claim at ``a`` itself. + """ + if isinstance(value, (dict, list)): + return not value + return True + + +def _leaves( + value: JsonValue, path: LeafPath = () +) -> Iterator[tuple[LeafPath, JsonValue]]: + """Yield every leaf of a JSON value, in sorted key order.""" + if _is_leaf(value): + yield path, value + return + if isinstance(value, dict): + for key in sorted(value): + yield from _leaves(value[key], (*path, key)) + return + items = cast("list[JsonValue]", value) + for index, item in enumerate(items): + yield from _leaves(item, (*path, index)) + + +def _at(value: JsonValue | None, path: LeafPath) -> tuple[bool, JsonValue]: + """Return whether a value sits at a path, and the value if one does.""" + current: JsonValue | None = value + for step in path: + if isinstance(step, int): + if not isinstance(current, list) or step >= len(current): + return False, None + current = cast("list[JsonValue]", current)[step] + continue + if not isinstance(current, dict) or step not in current: + return False, None + current = cast("dict[str, JsonValue]", current)[step] + return True, current + + +def _replace(value: JsonValue, path: LeafPath, replacement: JsonValue) -> JsonValue: + """Return a copy of a value with one leaf replaced.""" + if not path: + return replacement + step, *rest = path + tail: LeafPath = tuple(rest) + if isinstance(step, int): + items = cast("list[JsonValue]", value) + return [ + _replace(item, tail, replacement) if index == step else item + for index, item in enumerate(items) + ] + mapping = cast("dict[str, JsonValue]", value) + return { + key: _replace(item, tail, replacement) if key == step else item + for key, item in mapping.items() + } + + +def _perturbations(value: JsonValue) -> tuple[PathValue, ...]: + """Return the code-drawn perturbations available for one leaf value. + + Only plausible, shape-preserving variants are offered: a value that could + not have come out of a working implementation is not a mutant, it is noise. + ``bool`` is checked before ``int`` because ``bool`` is a subclass of it. + + A perturbation that would leave the value unchanged is dropped. Floating + point makes that reachable -- ``1e308 + 1.0`` is ``1e308`` -- and a row + that carries the claim's own value is not a mutant: it would survive every + expectation, including the ones the battery exists to convict. + """ + if isinstance(value, bool): + candidates: tuple[PathValue, ...] = ( + ("flipped", not value), + ("as a string", "True" if value else "False"), + ) + elif isinstance(value, int): + candidates = (("off by one", value + 1), ("as a string", str(value))) + elif isinstance(value, float): + candidates = (("off by one", value + 1.0), ("as a string", repr(value))) + elif isinstance(value, str): + candidates = (("with a suffix", f"{value}?"), ("as null", None)) + elif value is None: + candidates = (("as zero", 0), ("as an empty string", "")) + else: + return () + return tuple( + candidate for candidate in candidates if not values_equal(candidate[1], value) + ) diff --git a/src/stepfork/export/scoring.py b/src/stepfork/export/scoring.py index ea352a2..7352251 100644 --- a/src/stepfork/export/scoring.py +++ b/src/stepfork/export/scoring.py @@ -124,27 +124,27 @@ def by_kind(self, kind: MutantKind) -> tuple[MutantScore, ...]: return tuple(mutant for mutant in self.mutants if mutant.kind == kind) def summary(self) -> str: - """Return a one-line reading of the score.""" + """Return a one-line reading of the score, labelled with its verdict.""" if self.verdict == "no_expectation": return "no expectation to score: the test asserts the recorded run" if self.verdict == "undetermined": return ( - f"rejected {self.killed}/{self.total} mutants, but the battery " - "carries no claim-drawn mutant, so it cannot separate a " - "claim-pinned expectation from a run-pinned one" + f"undetermined: rejected {self.killed}/{self.total} mutants, but " + "the battery carries no claim-drawn mutant, so it cannot " + "separate a claim-pinned expectation from a run-pinned one" ) if self.verdict == "run_pinned": survived = ", ".join( mutant.name for mutant in self.by_kind("claim") if mutant.survived ) return ( - "expectation is run-pinned: it accepts the claim-drawn " - f"mutant(s) {survived}, so it encodes the recorded run rather " - "than the claim" + f"run-pinned: the expectation accepts the claim-drawn mutant(s) " + f"{survived}, so it encodes the recorded run rather than " + "the claim" ) return ( - f"expectation is claim-pinned: rejected {self.killed}/{self.total} " - "mutants, including every claim-drawn mutant" + f"claim-pinned: rejected {self.killed}/{self.total} mutants, " + "including every claim-drawn mutant" ) diff --git a/src/stepfork/export/testfile.py b/src/stepfork/export/testfile.py new file mode 100644 index 0000000..1d27ad3 --- /dev/null +++ b/src/stepfork/export/testfile.py @@ -0,0 +1,178 @@ +"""Read the literals an exported pytest test carries, without executing it. + +``stepfork export`` writes a fixed template whose module-level assignments +name the bundle, the expectation, and whether an expectation is asserted at +all. Reading those back is what lets ``stepfork score`` work on the file the +export just wrote, rather than on a second copy of the expectation that can +drift from it. + +Reading is not running. The file is parsed with :mod:`ast`, JSON-shaped values +are read with :func:`ast.literal_eval`, and the bundle path is evaluated by a +whitelist evaluator that understands exactly the two expression shapes the +generator emits -- ``Path(__file__).resolve().parent`` optionally joined with +literal parts, and a bare ``Path("")``. Anything else is refused with +a message that names the file and suggests ``--bundle``, because the +alternative to refusing is guessing, and a guess here would score the wrong +bundle silently. Nothing is imported, no code is compiled, and nothing in the +file is executed. +""" + +from __future__ import annotations + +import ast +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from stepfork.trace import JsonValue + +READ_NAMES = ( + "TRACE_PATH", + "EXPECTATION", + "HAS_EXPECTATION", + "TRAJECTORY_EXPECTATION", +) + + +class ExportedTestError(ValueError): + """Raised when an exported test file cannot be read.""" + + +@dataclass(frozen=True) +class ExportedTest: + """The literals an exported test carries. + + ``trace_path`` is None when the file assigns no ``TRACE_PATH``; the caller + may still name a bundle itself, so the absence is reported rather than + raised here. + """ + + path: Path + trace_path: Path | None + expectation: JsonValue + has_expectation: bool + trajectory_expectation: JsonValue | None + + +def read_exported_test(path: Path) -> ExportedTest: + """Read the literals of an exported regression test. + + Args: + path: The generated test file. + + Returns: + The bundle path it names (when it names one), the expectation it + asserts, whether it asserts one, and any trajectory expectation. + + Raises: + ExportedTestError: If the file cannot be read, does not parse, assigns a + value that is not a literal, or names its bundle with an + expression outside the shapes the generator emits. + """ + try: + source = path.read_text(encoding="utf-8") + except OSError as exc: + raise ExportedTestError(f"cannot read {path}: {exc}") from exc + try: + module = ast.parse(source, filename=str(path)) + except SyntaxError as exc: + raise ExportedTestError(f"{path} does not parse: {exc}") from exc + + assigned = _assigned_literals(module) + expectation = _literal(assigned, "EXPECTATION", path, default=None) + has_expectation = _literal(assigned, "HAS_EXPECTATION", path, default=True) + if not isinstance(has_expectation, bool): + raise ExportedTestError( + f"{path}: HAS_EXPECTATION is {has_expectation!r}; the export writes a bool" + ) + trajectory = _literal(assigned, "TRAJECTORY_EXPECTATION", path, default=None) + node = assigned.get("TRACE_PATH") + return ExportedTest( + path=path, + trace_path=_evaluate_path(node, path) if node is not None else None, + expectation=expectation, + has_expectation=has_expectation, + trajectory_expectation=trajectory, + ) + + +def _assigned_literals(module: ast.Module) -> dict[str, ast.expr]: + """Map each read name to the expression of its last module-level assignment.""" + found: dict[str, ast.expr] = {} + for statement in module.body: + if not isinstance(statement, ast.Assign): + continue + for target in statement.targets: + if isinstance(target, ast.Name) and target.id in READ_NAMES: + found[target.id] = statement.value + return found + + +def _literal( + assigned: dict[str, ast.expr], + name: str, + path: Path, + *, + default: JsonValue, +) -> JsonValue: + """Read one assigned value as a JSON literal.""" + node = assigned.get(name) + if node is None: + return default + try: + value = ast.literal_eval(node) + except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as exc: + raise ExportedTestError( + f"{path}:{node.lineno}: {name} is not a literal value " + f"({type(exc).__name__}); score reads the literals the export wrote" + ) from exc + return cast("JsonValue", value) + + +def _evaluate_path(node: ast.expr, test_path: Path) -> Path: + """Evaluate a bundle path expression, refusing anything outside the template. + + The accepted shapes are the ones ``stepfork.export.generator._path_expression`` + emits, so a file the generator wrote always reads and a hand-edited + expression is refused rather than misread. + """ + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return Path(node.value) + if isinstance(node, ast.Name) and node.id == "__file__": + return test_path + if isinstance(node, ast.Attribute) and node.attr == "parent": + return _evaluate_path(node.value, test_path).parent + if isinstance(node, ast.Call): + function = node.func + if ( + isinstance(function, ast.Name) + and function.id == "Path" + and len(node.args) == 1 + ): + return Path(_string_argument(node.args[0], test_path)) + if ( + isinstance(function, ast.Attribute) + and function.attr == "resolve" + and not node.args + ): + return _evaluate_path(function.value, test_path).resolve() + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div): + return _evaluate_path(node.left, test_path) / _string_argument( + node.right, test_path + ) + raise ExportedTestError( + f"{test_path}:{node.lineno}: cannot read the bundle path from this " + "expression; pass --bundle to name the .sftrace bundle directly" + ) + + +def _string_argument(node: ast.expr, test_path: Path) -> str: + """Read one path part: a string literal, or ``__file__`` inside ``Path(...)``.""" + if isinstance(node, ast.Name) and node.id == "__file__": + return str(test_path) + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + raise ExportedTestError( + f"{test_path}:{node.lineno}: a path part is not a string literal; " + "pass --bundle to name the .sftrace bundle directly" + ) diff --git a/src/stepfork/minimize/engine.py b/src/stepfork/minimize/engine.py index 77c3155..bd4722e 100644 --- a/src/stepfork/minimize/engine.py +++ b/src/stepfork/minimize/engine.py @@ -88,6 +88,20 @@ def minimize(self) -> MinimizationResult: """ # Step 1: Detect the original failure signature try: + try: + from stepfork.trace.replay_policy import ReplayPolicy + + ReplayPolicy(self.mode) + except ValueError as exc: + return MinimizationResult( + success=False, + original_event_count=len(self.trace.events), + minimized_event_count=None, + replay_attempts=self.total_replay_attempts, + preserved_failure=False, + is_minimal=False, + error_message=f"Invalid replay mode {self.mode!r}: {exc}", + ) original_signature = detect_failure_signature( self.trace, self.entrypoint, diff --git a/tests/unit/test_cli_score.py b/tests/unit/test_cli_score.py new file mode 100644 index 0000000..a074abb --- /dev/null +++ b/tests/unit/test_cli_score.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path + +from typer.testing import CliRunner + +from stepfork import record +from stepfork.cli.main import app +from stepfork.export.generator import generate_pytest_source +from stepfork.trace import JsonValue + +runner = CliRunner() + +RECORDED: JsonValue = {"status": "refunded", "amount": 0.0} +CLAIM: JsonValue = {"status": "refunded", "amount": 42.5} + + +def _flat(text: str) -> str: + """Collapse the line breaks the console inserts when it wraps.""" + return re.sub(r"\s+", " ", text) + + +def _record(bundle: Path, output: JsonValue) -> Path: + with record("refund-agent", output=bundle) as session: + session.set_output(output) + return bundle + + +def _write_case( + tmp_path: Path, + *, + recorded: JsonValue = RECORDED, + expectation: JsonValue = CLAIM, + has_expectation: bool = True, + record_run: bool = True, +) -> tuple[Path, Path]: + bundle = tmp_path / "refund.sftrace" + if record_run: + _record(bundle, recorded) + test = tmp_path / "test_refund_regression.py" + test.write_text( + generate_pytest_source( + trace_path=bundle, + output_path=test, + entrypoint="pkg.mod:run", + expectation=expectation, + has_expectation=has_expectation, + ), + encoding="utf-8", + ) + return bundle, test + + +def test_score_reports_a_claim_pinned_expectation(tmp_path: Path) -> None: + _, test = _write_case(tmp_path) + result = runner.invoke(app, ["score", str(test)]) + assert result.exit_code == 0 + assert "claim-pinned" in _flat(result.stdout) + assert "the recorded run" in _flat(result.stdout) + + +def test_score_convicts_a_run_pinned_expectation(tmp_path: Path) -> None: + _, test = _write_case(tmp_path, expectation=RECORDED) + result = runner.invoke(app, ["score", str(test)]) + assert result.exit_code == 1 + assert "run-pinned" in _flat(result.stdout) + assert "unapproved oracle" in _flat(result.stdout) + + +def test_score_prints_json(tmp_path: Path) -> None: + _, test = _write_case(tmp_path) + result = runner.invoke(app, ["score", str(test), "--json"]) + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["verdict"] == "claim_pinned" + assert payload["pins"] == {"live": ["amount"], "reproduced": ["status"]} + assert payload["total"] == payload["killed"] + assert {mutant["kind"] for mutant in payload["mutants"]} == {"claim", "code"} + + +def test_score_accepts_an_expectation_override(tmp_path: Path) -> None: + """Scoring a candidate expectation is the pre-export review step.""" + _, test = _write_case(tmp_path, expectation=RECORDED) + candidate = tmp_path / "expected.json" + candidate.write_text(json.dumps(CLAIM), encoding="utf-8") + assert runner.invoke(app, ["score", str(test)]).exit_code == 1 + result = runner.invoke(app, ["score", str(test), "--expect-output", str(candidate)]) + assert result.exit_code == 0 + assert "claim-pinned" in _flat(result.stdout) + + +def test_score_reports_a_missing_bundle(tmp_path: Path) -> None: + _, test = _write_case(tmp_path, record_run=False) + result = runner.invoke(app, ["score", str(test)]) + assert result.exit_code == 2 + assert "trace bundle not found" in _flat(result.stdout) + + +def test_score_names_a_bundle_it_was_given(tmp_path: Path) -> None: + bundle, test = _write_case(tmp_path, expectation=RECORDED) + test.write_text( + test.read_text(encoding="utf-8").replace("TRACE_PATH = ", "TRACE_PATH_OLD = "), + encoding="utf-8", + ) + result = runner.invoke(app, ["score", str(test), "--bundle", str(bundle)]) + assert result.exit_code == 1 + assert "run-pinned" in _flat(result.stdout) + + +def test_score_without_a_bundle_asks_for_one(tmp_path: Path) -> None: + _, test = _write_case(tmp_path, record_run=False) + test.write_text( + test.read_text(encoding="utf-8").replace("TRACE_PATH = ", "TRACE_PATH_OLD = "), + encoding="utf-8", + ) + result = runner.invoke(app, ["score", str(test)]) + assert result.exit_code == 2 + assert "--bundle" in _flat(result.stdout) + + +def test_score_reports_an_export_without_an_expectation(tmp_path: Path) -> None: + _, test = _write_case(tmp_path, has_expectation=False, expectation=None) + result = runner.invoke(app, ["score", str(test)]) + assert result.exit_code == 2 + assert "no expectation to score" in _flat(result.stdout) + + +def test_score_cannot_separate_readings_without_a_recorded_run(tmp_path: Path) -> None: + """A run that recorded no output leaves nothing claim-drawn to derive.""" + _, test = _write_case(tmp_path, recorded=None) + result = runner.invoke(app, ["score", str(test)]) + assert result.exit_code == 2 + assert "undetermined" in _flat(result.stdout) + + +def test_score_reports_an_unreadable_test_file(tmp_path: Path) -> None: + test = tmp_path / "test_handwritten.py" + test.write_text("EXPECTATION = build_expectation()\n", encoding="utf-8") + result = runner.invoke(app, ["score", str(test)]) + assert result.exit_code == 2 + assert "not a literal value" in _flat(result.stdout) + + +def test_score_is_registered() -> None: + result = runner.invoke(app, ["--help"]) + assert result.exit_code == 0 + assert "score" in _flat(result.stdout) diff --git a/tests/unit/test_export_mutants.py b/tests/unit/test_export_mutants.py new file mode 100644 index 0000000..0226382 --- /dev/null +++ b/tests/unit/test_export_mutants.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +from collections.abc import Sequence + +from hypothesis import given +from hypothesis import strategies as st + +from stepfork.diff.compare import first_difference +from stepfork.export.mutants import ( + RECORDED_RUN_MUTANT, + PinCoverage, + _perturbations, + derive_battery, + pin_coverage, + recorded_output, + render_path, +) +from stepfork.export.scoring import MutantKind, MutantOutcome, score_expectation +from stepfork.trace import JsonValue, RunEnd, RunStart, RunStatus, Trace + +# A recorded refund run: the agent reported the refund as done and issued +# nothing. The claim is that a refund of 42.50 reaches the order. +RECORDED_RUN: JsonValue = {"status": "refunded", "amount": 0.0} +CLAIM_PINNED: JsonValue = {"status": "refunded", "amount": 42.5} +RUN_PINNED: JsonValue = {"status": "refunded", "amount": 0.0} + + +def _derived( + expectation: JsonValue, recorded: JsonValue | None +) -> tuple[MutantOutcome, ...]: + return derive_battery(expectation, recorded) + + +def _names(battery: Sequence[MutantOutcome]) -> list[str]: + return [mutant.name for mutant in battery] + + +def _by_kind(battery: Sequence[MutantOutcome], kind: MutantKind) -> list[MutantOutcome]: + return [mutant for mutant in battery if mutant.kind == kind] + + +def test_derived_battery_separates_a_claim_pinned_expectation() -> None: + score = score_expectation(CLAIM_PINNED, _derived(CLAIM_PINNED, RECORDED_RUN)) + assert score.verdict == "claim_pinned" + assert score.total > 1 + + +def test_derived_battery_convicts_a_run_pinned_expectation() -> None: + """The finding the derivation exists to produce: the claim row survives.""" + score = score_expectation(RUN_PINNED, _derived(RUN_PINNED, RECORDED_RUN)) + assert score.verdict == "run_pinned" + survivors = [mutant.name for mutant in score.survivors] + assert survivors == [RECORDED_RUN_MUTANT] + # The code-drawn controls are killed either way, which is why the reading + # cannot come from them. + assert all(mutant.killed for mutant in score.mutants if mutant.kind == "code") + + +def test_claim_mutant_is_the_recorded_run_whole_not_a_projection() -> None: + """A projection would hide an expectation that is stricter than the run.""" + recorded: JsonValue = {"status": "refunded", "amount": 0.0, "receipt": "r-1"} + claim: JsonValue = {"status": "refunded"} + battery = _derived(claim, recorded) + claim_rows = _by_kind(battery, "claim") + assert len(claim_rows) == 1 + assert claim_rows[0].output == recorded + score = score_expectation(claim, battery) + assert score.verdict == "claim_pinned" + + +def test_expectation_stricter_than_the_run_is_never_run_pinned() -> None: + """`{}` claims an empty output; a run that produced fields contradicts it.""" + recorded: JsonValue = {"status": "refunded"} + score = score_expectation({}, _derived({}, recorded)) + assert score.verdict == "claim_pinned" + + +def test_no_recorded_output_yields_controls_only() -> None: + """Without a recorded run there is nothing claim-drawn to derive.""" + battery = _derived(CLAIM_PINNED, None) + assert _by_kind(battery, "claim") == [] + assert _by_kind(battery, "code") != [] + assert score_expectation(CLAIM_PINNED, battery).verdict == "undetermined" + + +def test_no_expectation_yields_an_empty_battery() -> None: + assert derive_battery(None, RECORDED_RUN, has_expectation=False) == () + score = score_expectation(None, (), has_expectation=False) + assert score.verdict == "no_expectation" + + +def test_a_perturbation_that_lands_on_the_recorded_value_is_skipped() -> None: + """A code-drawn label on the run's own value would be a claim row in disguise.""" + claim: JsonValue = {"amount": 42.5} + recorded: JsonValue = {"amount": 43.5} # the "off by one" perturbation + battery = _derived(claim, recorded) + names = _names(battery) + assert "amount: off by one" not in names + assert "amount: as a string" in names + # Only the perturbation that collides is dropped; the claim row is not. + assert _names(_by_kind(battery, "claim")) == [RECORDED_RUN_MUTANT] + + +def test_code_drawn_rows_carry_the_claim_with_one_leaf_changed() -> None: + battery = _by_kind(_derived(CLAIM_PINNED, RECORDED_RUN), "code") + assert battery + for mutant in battery: + difference = first_difference(CLAIM_PINNED, mutant.output) + assert difference is not None + names = _names(_derived(CLAIM_PINNED, RECORDED_RUN)) + assert "amount: as a string" in names + assert "status: with a suffix" in names + + +def test_leaf_paths_are_rendered_the_way_first_difference_renders_them() -> None: + claim: JsonValue = {"plan": ["search", "book"], "meta": {"city": "Paris"}} + recorded: JsonValue = {"plan": ["search", "cancel"], "meta": {"city": "Paris"}} + names = _names(_derived(claim, recorded)) + assert "plan[1]: with a suffix" in names + difference = first_difference(claim, recorded) + assert difference is not None + assert difference[0] == "plan[1]" + assert render_path(("plan", 1)) == difference[0] + assert render_path(()) == "$" + + +def test_battery_is_deterministic() -> None: + claim: JsonValue = {"a": 1, "b": {"c": "x"}, "d": True} + first = _derived(claim, {"a": 2, "b": {"c": "y"}, "d": False}) + second = _derived(claim, {"a": 2, "b": {"c": "y"}, "d": False}) + assert first == second + + +def test_pin_coverage_splits_live_leaves_from_reproduced_ones() -> None: + claim: JsonValue = {"amount": 42.5, "status": "refunded"} + coverage = pin_coverage(claim, RECORDED_RUN) + assert isinstance(coverage, PinCoverage) + assert coverage.live == ("amount",) + assert coverage.reproduced == ("status",) + assert coverage.total == 2 + assert coverage.anchored + + +def test_pin_coverage_counts_a_leaf_the_run_never_produced_as_live() -> None: + coverage = pin_coverage({"amount": 42.5, "receipt": "r-1"}, {"amount": 42.5}) + assert coverage.live == ("receipt",) + assert coverage.reproduced == ("amount",) + + +def test_an_empty_container_is_a_leaf_the_run_can_contradict() -> None: + claim: JsonValue = {"trace": {}} + recorded: JsonValue = {"trace": {"steps": 3}} + coverage = pin_coverage(claim, recorded) + assert coverage.live == ("trace",) + assert score_expectation(claim, _derived(claim, recorded)).verdict == "claim_pinned" + + +def test_recorded_output_reads_the_last_completed_run() -> None: + trace = Trace(agent_name="probe") + trace.add(RunStart()) + trace.add(RunEnd(run_status=RunStatus.COMPLETED, output={"ok": True})) + trace.add(RunEnd(run_status=RunStatus.FAILED)) + assert recorded_output(trace) == {"ok": True} + + +def test_recorded_output_is_none_when_no_run_completed() -> None: + trace = Trace(agent_name="probe") + trace.add(RunStart()) + trace.add(RunEnd(run_status=RunStatus.FAILED)) + assert recorded_output(trace) is None + assert recorded_output(Trace(agent_name="empty")) is None + + +@given( + st.one_of( + st.booleans(), + st.integers(), + st.floats(allow_nan=False, allow_infinity=False), + st.text(), + st.none(), + ) +) +def test_every_perturbation_changes_its_value(value: JsonValue) -> None: + """A no-op row would survive every expectation, convicting nothing.""" + for label, replacement in _perturbations(value): + assert label + assert first_difference(value, replacement) is not None diff --git a/tests/unit/test_export_testfile.py b/tests/unit/test_export_testfile.py new file mode 100644 index 0000000..9a818c5 --- /dev/null +++ b/tests/unit/test_export_testfile.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from stepfork.export.generator import generate_pytest_source +from stepfork.export.testfile import ExportedTestError, read_exported_test + +EXPECTATION = {"status": "refunded", "amount": 42.5} + + +def _write(tmp_path: Path, source: str, name: str = "test_run_regression.py") -> Path: + path = tmp_path / name + path.write_text(source, encoding="utf-8") + return path + + +def _generated(tmp_path: Path, *, depth: int = 0, **kwargs: object) -> Path: + bundle = tmp_path + for index in range(depth): + bundle = bundle / f"level{index}" + bundle = bundle / "refund.sftrace" + output = tmp_path / "test_refund_regression.py" + source = generate_pytest_source( + trace_path=bundle, + output_path=output, + entrypoint="pkg.mod:run", + expectation=EXPECTATION, + **kwargs, # type: ignore[arg-type] + ) + return _write(tmp_path, source) + + +def test_reads_the_literals_the_generator_writes(tmp_path: Path) -> None: + test = _generated(tmp_path) + exported = read_exported_test(test) + assert exported.path == test + assert exported.trace_path == tmp_path / "refund.sftrace" + assert exported.expectation == EXPECTATION + assert exported.has_expectation is True + assert exported.trajectory_expectation is None + + +def test_reads_the_absolute_path_form(tmp_path: Path) -> None: + """A bundle too far up the tree is written as an absolute Path(...).""" + test = _generated(tmp_path, depth=5) + assert "Path('/" in test.read_text(encoding="utf-8") + exported = read_exported_test(test) + assert exported.trace_path == ( + tmp_path / "level0/level1/level2/level3/level4/refund.sftrace" + ) + + +def test_reads_a_reviewed_trajectory_expectation(tmp_path: Path) -> None: + trajectory = {"called": ["search_flights"], "sequence": ["search_flights"]} + test = _generated(tmp_path, trajectory_expectation=trajectory) + assert read_exported_test(test).trajectory_expectation == trajectory + + +def test_reads_an_export_without_an_expectation(tmp_path: Path) -> None: + test = _generated(tmp_path, has_expectation=False) + exported = read_exported_test(test) + assert exported.has_expectation is False + assert exported.expectation is None + + +def test_reading_never_executes_the_file(tmp_path: Path) -> None: + """The contract is parse-and-read; an exported test must not run here.""" + marker = tmp_path / "executed.marker" + test = _write( + tmp_path, + "from pathlib import Path\n" + f"Path({str(marker)!r}).write_text('ran')\n" + "TRACE_PATH = Path('/nowhere/run.sftrace')\n" + "EXPECTATION = {'ok': True}\n", + ) + exported = read_exported_test(test) + assert exported.expectation == {"ok": True} + assert not marker.exists() + + +def test_refuses_a_non_literal_expectation(tmp_path: Path) -> None: + test = _write( + tmp_path, + "TRACE_PATH = Path('/nowhere/run.sftrace')\nEXPECTATION = dict(status='ok')\n", + ) + with pytest.raises(ExportedTestError, match="not a literal"): + read_exported_test(test) + + +def test_refuses_an_unsupported_bundle_expression(tmp_path: Path) -> None: + test = _write( + tmp_path, + "TRACE_PATH = Path(__file__).resolve().parent / TRACES[0]\n" + "EXPECTATION = {'ok': True}\n", + ) + with pytest.raises(ExportedTestError, match="--bundle"): + read_exported_test(test) + + +def test_a_file_without_trace_path_reports_none(tmp_path: Path) -> None: + test = _write(tmp_path, "EXPECTATION = {'ok': True}\n") + exported = read_exported_test(test) + assert exported.trace_path is None + assert exported.has_expectation is True + + +def test_refuses_a_non_boolean_has_expectation(tmp_path: Path) -> None: + test = _write( + tmp_path, + "TRACE_PATH = Path('/nowhere/run.sftrace')\nEXPECTATION = {'ok': True}\n" + "HAS_EXPECTATION = 'yes'\n", + ) + with pytest.raises(ExportedTestError, match="HAS_EXPECTATION"): + read_exported_test(test) + + +def test_the_last_assignment_wins(tmp_path: Path) -> None: + """The module's value is its final one, so reading agrees with running.""" + test = _write( + tmp_path, + "TRACE_PATH = Path('/first/run.sftrace')\n" + "TRACE_PATH = Path('/second/run.sftrace')\n" + "EXPECTATION = {'ok': True}\n", + ) + assert read_exported_test(test).trace_path == Path("/second/run.sftrace") + + +def test_a_missing_file_is_reported(tmp_path: Path) -> None: + with pytest.raises(ExportedTestError, match="cannot read"): + read_exported_test(tmp_path / "absent.py") + + +def test_an_unparsable_file_is_reported(tmp_path: Path) -> None: + test = _write(tmp_path, "EXPECTATION = {'ok':\n") + with pytest.raises(ExportedTestError, match="does not parse"): + read_exported_test(test) diff --git a/tests/unit/test_minimize_mode_validation.py b/tests/unit/test_minimize_mode_validation.py new file mode 100644 index 0000000..fe3a18d --- /dev/null +++ b/tests/unit/test_minimize_mode_validation.py @@ -0,0 +1,92 @@ +"""Unit tests for minimize mode validation.""" + +from __future__ import annotations + +from pathlib import Path + +from typer.testing import CliRunner + +from stepfork import record, trace_tool +from stepfork.cli.main import app + +runner = CliRunner() + + +@trace_tool(name="simple_tool") +def simple_tool(x: int) -> int: + """Simple test tool.""" + return x * 2 + + +def simple_agent() -> dict[str, int]: + """Simple test agent.""" + result = simple_tool(5) + return {"value": result} + + +def test_minimize_rejects_hybrid_mode(tmp_path: Path) -> None: + """Minimize rejects unsupported 'hybrid' mode.""" + trace_path = tmp_path / "test.sftrace" + with record("cli-test-agent", output=str(trace_path)) as session: + result = simple_agent() + session.set_output(result) + + result = runner.invoke( + app, + [ + "minimize", + str(trace_path), + "--entrypoint", + "test_minimize_mode_validation:simple_agent", + "--mode", + "hybrid", + ], + ) + assert result.exit_code == 2 + assert "Invalid replay mode" in result.stdout + + +def test_minimize_rejects_bogus_mode(tmp_path: Path) -> None: + """Minimize rejects invalid mode values.""" + trace_path = tmp_path / "test.sftrace" + with record("cli-test-agent", output=str(trace_path)) as session: + result = simple_agent() + session.set_output(result) + + result = runner.invoke( + app, + [ + "minimize", + str(trace_path), + "--entrypoint", + "test_minimize_mode_validation:simple_agent", + "--mode", + "bogus", + ], + ) + assert result.exit_code == 2 + assert "Invalid replay mode" in result.stdout + + +def test_minimize_accepts_valid_modes(tmp_path: Path) -> None: + """Valid modes are accepted by minimize.""" + trace_path = tmp_path / "test.sftrace" + with record("cli-test-agent", output=str(trace_path)) as session: + result = simple_agent() + session.set_output(result) + + for mode in ("frozen", "live", "forbidden", "manual", "derived"): + result = runner.invoke( + app, + [ + "minimize", + str(trace_path), + "--entrypoint", + "test_minimize_mode_validation:simple_agent", + "--mode", + mode, + ], + ) + # Valid modes may still fail minimization if trace doesn't fail as expected, + # but should not be rejected with exit 2 for invalid mode + assert result.exit_code != 2, f"Mode {mode} was incorrectly rejected" diff --git a/tests/unit/test_replay_contract.py b/tests/unit/test_replay_contract.py new file mode 100644 index 0000000..fac555e --- /dev/null +++ b/tests/unit/test_replay_contract.py @@ -0,0 +1,106 @@ +"""Contract tests for frozen replay behavior. + +These tests document and verify the supported replay contract: +- Original trace with unchanged calls/args replays correctly +- Changed arguments cause ReplayMismatchError (input diverged) +- Additional tool calls cause ReplayExhaustedError (unexpected tool call) +- Missing recorded calls cause ReplayMismatchError (unmatched calls) + +Generated tests from failure traces that cannot execute on the fixed code +correctly fail during replay due to contract violations. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from stepfork import record, trace_tool +from stepfork.replay import ReplayMismatchError, ReplaySession + + +@trace_tool(name="step") +def step(value: str) -> dict[str, str]: + return {"value": value} + + +@trace_tool(name="compute") +def compute(x: int, y: int) -> dict[str, int]: + return {"sum": x + y, "diff": x - y} + + +def original_trace_agent() -> dict[str, str]: + return step("a") + + +def agent_changes_arg() -> dict[str, str]: + return step("b") # different arg from recorded "a" + + +def agent_extra_call() -> dict[str, str]: + step("a") + step("extra") # extra call not in original trace + return {"extra": True} + + +def agent_missing_calls() -> dict[str, object]: + return {"done": True} # skips the recorded step call + + +def _record_step_trace(destination: Path) -> None: + with record("step-agent", output=destination): + step("a") + # Set output after recording the step call + # Better: record with session output + pass + + +def test_original_trace_replays_successfully(tmp_path: Path) -> None: + trace_path = tmp_path / "trace.sftrace" + with record("step-agent", output=trace_path) as session: + session.set_output(step("a")) + + # Replay against original agent behavior + with ReplaySession.from_trace(trace_path, mode="frozen") as replay: + result = step("a") + replay.verify_complete() + assert result == {"value": "a"} + + +def test_changed_arguments_cause_replay_mismatch(tmp_path: Path) -> None: + trace_path = tmp_path / "trace.sftrace" + with record("step-agent", output=trace_path) as session: + session.set_output(step("a")) + + # Replay with different argument - should diverge + with ( + pytest.raises(ReplayMismatchError, match="input diverged"), + ReplaySession.from_trace(trace_path, mode="frozen"), + ): + step("b") + + +def test_extra_tool_call_cause_replay_exhausted(tmp_path: Path) -> None: + trace_path = tmp_path / "trace.sftrace" + with record("step-agent", output=trace_path) as session: + session.set_output(step("a")) + + # First call matches, second is extra (beyond recorded) + with ( + pytest.raises(ReplayMismatchError), + ReplaySession.from_trace(trace_path, mode="frozen"), + ): + step("a") + step("extra") # extra call + + +def test_missing_recorded_calls_cause_unmatched(tmp_path: Path) -> None: + trace_path = tmp_path / "trace.sftrace" + with record("step-agent", output=trace_path) as session: + session.set_output(step("a")) + + # Agent makes no calls but trace expects 1 + with pytest.raises(ReplayMismatchError, match=r"unmatched|ended before consuming"): + with ReplaySession.from_trace(trace_path, mode="frozen") as replay: + replay.verify_complete() diff --git a/tests/unit/test_replay_contract_changed_args.py b/tests/unit/test_replay_contract_changed_args.py new file mode 100644 index 0000000..283f5ea --- /dev/null +++ b/tests/unit/test_replay_contract_changed_args.py @@ -0,0 +1,39 @@ +"""Contract test for changed arguments case matching issue #17.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from stepfork import record, trace_tool +from stepfork.replay import ReplayMismatchError, ReplaySession + + +@trace_tool(name="issue_refund") +def issue_refund(customer_id: str, amount_cents: int) -> dict: + return {"customer_id": customer_id, "refunded_cents": amount_cents} + + +def run_buggy() -> dict: + issue_refund("c-1", 49) # records 49 + return {"refunded": 49} + + +def run_fixed() -> dict: + issue_refund("c-1", 4999) # changes to 4999 + return {"refunded": 4999} + + +def test_changed_args_causes_divergence(tmp_path: Path) -> None: + # Record from buggy version + trace_path = tmp_path / "failure.sftrace" + with record("triage-agent", output=trace_path) as session: + session.set_output(run_buggy()) + + # Replay against fixed version - arguments changed + with ( + pytest.raises(ReplayMismatchError, match="input diverged"), + ReplaySession.from_trace(trace_path, mode="frozen"), + ): + run_fixed() diff --git a/tests/unit/test_replay_contract_extra_calls.py b/tests/unit/test_replay_contract_extra_calls.py new file mode 100644 index 0000000..720a529 --- /dev/null +++ b/tests/unit/test_replay_contract_extra_calls.py @@ -0,0 +1,47 @@ +"""Contract test for extra calls case matching issue #17.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from stepfork import record, trace_tool +from stepfork.replay import ReplayMismatchError, ReplaySession + + +@trace_tool(name="get_advice") +def get_advice(ticket: str) -> str: + return "advice" + + +@trace_tool(name="escalate") +def escalate(priority: str) -> dict: + return {"escalated": True, "priority": priority} + + +def run_crash() -> dict: + # Only first call is made before crash + get_advice("T-1") + # escalate is never called (bug/crash) + return {"ok": True} + + +def run_fixed() -> dict: + get_advice("T-1") + escalate("high") # extra call beyond recorded trace + return {"ok": True} + + +def test_extra_calls_cause_divergence(tmp_path: Path) -> None: + # Record trace from crash version + trace_path = tmp_path / "crash.sftrace" + with record("crash-agent", output=trace_path) as session: + session.set_output(run_crash()) + + # Fixed version makes an extra call not in recording + with ( + pytest.raises(ReplayMismatchError), + ReplaySession.from_trace(trace_path, mode="frozen"), + ): + run_fixed()