diff --git a/docs/cli.md b/docs/cli.md index b13dc11..086b010 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -437,6 +437,91 @@ Common errors: --- +## score + +Score an exported expectation against a mutant battery derived from the +recorded run, without executing anything. + +```text +Usage: stepfork score [OPTIONS] TEST +``` + +An exported test asserts `EXPECTATION`. Two things can satisfy that assertion +and pass the same green run: the claim you meant to check, or a copy of the run +the recording came from. `score` derives the battery that separates them -- the +claim-drawn row is the recorded run's own output, and the code-drawn rows +perturb the expectation's leaves -- and reports which rows the expectation +rejects. + +The test file is **read, never executed**: it is parsed with `ast`, JSON-shaped +values are read with `ast.literal_eval`, and the bundle path is read from the +expressions the exporter writes. Nothing is imported and no entrypoint runs. + +Arguments: + +| Argument | Description | +|---|---| +| `TEST` | Exported pytest regression test. Required. | + +Options: + +| Option | Description | +|---|---| +| `--expect-output ` | Score this expectation instead of the one the test embeds. | +| `--bundle ` | Bundle to read the recorded run from. Default: the `TRACE_PATH` the test names. | +| `--json` | Print the score as JSON. | +| `--help` | Show command help and exit. | + +A claim-pinned expectation rejects the recorded run: + +```text +battery: 1 claim-drawn + 6 code-drawn mutant(s), derived from test_notify_regression.py and notify.sftrace + + claim the recorded run killed (first difference at carrier) + code carrier: with a suffix killed (first difference at carrier) + code order_id: with a suffix killed (first difference at order_id) + code should_notify: flipped killed (first difference at should_notify) + +claim-pinned: rejected 7/7 mutants, including every claim-drawn mutant +pins: 1 live (carrier); 2 leaf(s) reproduce the run (order_id, should_notify) +``` + +An expectation that has drifted into the recording accepts it, which is the +finding: the generated test would pass against the code that produced the +recording. + +```text +run-pinned: the expectation accepts the claim-drawn mutant(s) the recorded run, so it encodes the recorded run rather than the claim +The generated test passes against the code that produced the recording. Treat the expectation as an unapproved oracle. +pins: 0 live (none); 3 leaf(s) reproduce the run (carrier, order_id, should_notify) +``` + +The `pins` line answers a second question the verdict does not: which leaves +carry a value the run did not produce (live pins), and which merely reproduce +the run. A leaf that reproduces the run is not a defect -- a fix does not change +every field -- but it is not evidence of anchoring either. + +Exit codes: + +| Code | Meaning | +|---|---| +| `0` | `claim_pinned`: the expectation rejects the recorded run. | +| `1` | `run_pinned`: the expectation accepts it; it is an unapproved oracle. | +| `2` | Cannot answer: no expectation to score, no recorded run to derive a claim-drawn mutant from, or a test file that cannot be read. | + +Common errors: + +- `:: EXPECTATION is not a literal value ...` -- the test was + hand-edited; `score` reads the literals the export wrote. +- `cannot read the bundle path from this expression; pass --bundle ...` -- the + `TRACE_PATH` expression is outside the shapes the exporter writes. +- `trace bundle not found: ` -- the bundle the test names is not at that + path. Copy it beside the test, or pass `--bundle`. +- `no expectation to score` -- the test asserts the recorded run without an + `--expect-output`, so there is no claim for a mutant to separate from it. + +--- + ## Error handling Command output is sanitized before display: secrets in trace-derived text diff --git a/docs/pytest-export.md b/docs/pytest-export.md index 34a8398..29447e4 100644 --- a/docs/pytest-export.md +++ b/docs/pytest-export.md @@ -82,6 +82,62 @@ not embed or copy the fixture. If moving the test to another project or CI, copy the bundle with it and verify `TRACE_PATH` resolves there. These are test environment requirements; `pytest` is not a Stepfork runtime dependency. +## Is the expectation still anchored? (`stepfork score`) + +An expectation can be two different things and pass the same green run: the +claim you meant to assert, or a copy of the run you wrote it from. `stepfork +score` separates them, and it does it by derivation rather than by execution. + +```bash +stepfork score test_booking_regression.py +stepfork score test_booking_regression.py --expect-output expected.json --json +``` + +Two values are already at hand when the export is prepared, and both are read +back rather than re-derived: + +- the **expectation** the generated test asserts; +- the **recorded run's output** (`RunEnd.output` in the bundle), which is the + value the buggy code that produced the recording actually returned. + +The claim-drawn row is that second value verbatim: the run's own output is the +most plausible output of a variant implementation, because an implementation did +produce it. It is the row that discriminates. A claim-pinned expectation rejects +it -- the run it came from is the bug, so the expectation must fail against it. +A run-pinned expectation accepts it, and that acceptance is the finding: the +granularity of the comparison means the generated test would pass against the +code that produced the recording. + +The remaining rows are **controls**, not discriminators: perturbations of the +expectation's own leaves (an integer moved by one, a string suffixed, a value +replaced by `null`). A run-pinned expectation rejects those too, so they cannot +separate the two readings by themselves; what they do is make the contrast +legible, because a battery in which every control is killed and the claim-drawn +row survives reads differently from one in which everything is killed. + +`score` also reports **pin coverage**: which leaves of the expectation carry a +value the run did not produce (live pins), and which merely reproduce the run. +A leaf that reproduces the run is not a defect -- a fix does not change every +field -- but it is not evidence of anchoring either, which is why it is reported +rather than counted. + +Exit codes: `0` when the expectation is claim-pinned, `1` when it is run-pinned, +`2` when the question cannot be answered. + +Three limits are worth stating plainly. + +- **No recorded run, no claim-drawn row.** A trace whose run did not complete + carries no output, so there is nothing to draw the discriminating row from. + The battery is then the controls alone and the verdict is `undetermined` -- a + reading, not a failure, and the honest one for a battery that cannot separate + the two. +- **The bundle must be there.** `score` reads the recorded run from the bundle + the test names, so it has the same requirement the test has. Pass `--bundle` + to point at one elsewhere. +- **Nothing is executed.** Neither the test file nor the entrypoint runs: the + file is parsed, the literals are read, and each row is scored with the same + canonical comparison the generated test uses. + ## Generated test The generated file is produced from a fixed, reviewed template. Trace payloads diff --git a/src/stepfork/cli/export.py b/src/stepfork/cli/export.py index a04b877..d5f01f8 100644 --- a/src/stepfork/cli/export.py +++ b/src/stepfork/cli/export.py @@ -17,10 +17,9 @@ ExportExistsError, export_pytest_test, ) +from stepfork.export.mutants import recorded_output from stepfork.trace import ( JsonValue, - RunEnd, - RunStatus, Trace, TraceStorageError, validate_bundle, @@ -156,7 +155,7 @@ def export_command( expectation = _load_expectation(expect_output) has_expectation = True else: - expectation = _recorded_output(trace) + expectation = recorded_output(trace) has_expectation = expectation is not None warning = RECORDED_OUTPUT_WARNING @@ -223,13 +222,6 @@ def _load_expectation(path: Path) -> JsonValue: return redact_json(payload).value -def _recorded_output(trace: Trace) -> JsonValue | None: - for event in reversed(trace.events): - if isinstance(event, RunEnd) and event.run_status is RunStatus.COMPLETED: - return event.output - return None - - def _safe_stem(path: Path) -> str: stem = re.sub(r"\W+", "_", path.stem).strip("_") return stem or "trace" diff --git a/src/stepfork/cli/main.py b/src/stepfork/cli/main.py index 7881044..b7b06e7 100644 --- a/src/stepfork/cli/main.py +++ b/src/stepfork/cli/main.py @@ -12,6 +12,7 @@ from stepfork.cli.inspect import inspect_command from stepfork.cli.minimize import minimize_command from stepfork.cli.replay import replay_command +from stepfork.cli.score import score_command from stepfork.cli.validate import validate_command app = typer.Typer( @@ -28,6 +29,7 @@ app.command("inspect")(inspect_command) app.command("minimize")(minimize_command) app.command("replay")(replay_command) +app.command("score")(score_command) app.command("validate")(validate_command) diff --git a/src/stepfork/cli/score.py b/src/stepfork/cli/score.py new file mode 100644 index 0000000..045595a --- /dev/null +++ b/src/stepfork/cli/score.py @@ -0,0 +1,250 @@ +"""`stepfork score` command.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Annotated + +import typer +from rich.console import Console + +from stepfork.export.mutants import ( + PinCoverage, + derive_battery, + pin_coverage, + recorded_output, +) +from stepfork.export.scoring import ExpectationScore, score_expectation +from stepfork.export.testfile import ExportedTestError, read_exported_test +from stepfork.trace import JsonValue, Trace, TraceStorageError, redact_json + +console = Console() + +EXIT_ANCHORED = 0 +EXIT_RUN_PINNED = 1 +EXIT_CANNOT_SCORE = 2 + + +def score_command( + test: Annotated[ + Path, + typer.Argument( + help="Exported pytest regression test (read, never executed).", + exists=False, + file_okay=True, + dir_okay=False, + ), + ], + expect_output: Annotated[ + Path | None, + typer.Option( + "--expect-output", + help="Score this expectation instead of the one the test embeds.", + ), + ] = None, + bundle: Annotated[ + Path | None, + typer.Option( + "--bundle", + help="Trace bundle to read the recorded run from " + "(default: the TRACE_PATH the test names).", + ), + ] = None, + as_json: Annotated[ + bool, + typer.Option("--json", help="Print the score as JSON."), + ] = False, +) -> None: + """Score an exported expectation against a battery derived from the run. + + The battery is derived, not executed: the claim-drawn row is the recorded + run's own output, and the code-drawn rows are perturbations of the + expectation's leaves. A claim-pinned expectation rejects the recorded run; + a run-pinned one accepts it, which means the generated test passes against + the code that produced the recording. + + Exits 0 when the expectation anchors the claim, 1 when it is run-pinned, + and 2 when the question cannot be answered (no expectation, no recorded + run to draw a claim-drawn mutant from, or a file that cannot be read). + """ + try: + exported = read_exported_test(test) + except ExportedTestError as exc: + _fail(str(exc)) + return + + if expect_output is not None: + expectation = _load_json(expect_output) + if expectation is None: + return + has_expectation = True + else: + expectation = exported.expectation + has_expectation = exported.has_expectation + + destination = bundle if bundle is not None else exported.trace_path + if destination is None: + _fail( + f"{test} names no trace bundle; pass --bundle to name the .sftrace " + "bundle to read the recorded run from" + ) + return + if not destination.exists(): + _fail(f"trace bundle not found: {destination}") + return + + try: + trace = Trace.load(destination) + except TraceStorageError as exc: + _fail(f"trace bundle unreadable: {exc}") + return + + recorded = recorded_output(trace) + if recorded is not None: + recorded = redact_json(recorded).value + battery = derive_battery(expectation, recorded, has_expectation=has_expectation) + score = score_expectation(expectation, battery, has_expectation=has_expectation) + coverage = ( + pin_coverage(expectation, recorded) + if recorded is not None and has_expectation + else None + ) + + if as_json: + # Plain stdout, not the rich console: markup would be interpreted and + # a wrapped line is no longer one JSON document. + typer.echo( + json.dumps( + _as_json(score, coverage, test, destination), + indent=2, + sort_keys=False, + ) + ) + else: + _report(score, coverage, test, destination) + raise typer.Exit(_exit_status(score)) + + +def _exit_status(score: ExpectationScore) -> int: + if score.verdict == "claim_pinned": + return EXIT_ANCHORED + if score.verdict == "run_pinned": + return EXIT_RUN_PINNED + return EXIT_CANNOT_SCORE + + +def _report( + score: ExpectationScore, + coverage: PinCoverage | None, + test: Path, + bundle: Path, +) -> None: + """Print the battery and its verdict.""" + if score.verdict == "no_expectation": + console.print("[yellow]no expectation to score[/yellow]") + console.print( + f"{test} asserts the recorded run without an --expect-output; " + "there is no claim for a mutant to separate from it." + ) + return + + claim = [mutant for mutant in score.mutants if mutant.kind == "claim"] + code = [mutant for mutant in score.mutants if mutant.kind == "code"] + console.print( + f"battery: {len(claim)} claim-drawn + {len(code)} code-drawn mutant(s), " + f"derived from {test.name} and {bundle.name}" + ) + console.print("") + for mutant in score.mutants: + outcome = "killed" if mutant.killed else "SURVIVED" + location = ( + f" (first difference at {mutant.differing_path})" + if mutant.differing_path + else "" + ) + console.print(f" {mutant.kind:<5} {mutant.name:<36} {outcome}{location}") + console.print("") + + if score.verdict == "run_pinned": + console.print(f"[red]{score.summary()}[/red]") + console.print( + "The generated test passes against the code that produced the " + "recording. Treat the expectation as an unapproved oracle." + ) + elif score.verdict == "undetermined": + console.print(f"[yellow]{score.summary()}[/yellow]") + console.print( + "No recorded run output was available, so no claim-drawn mutant " + "could be derived. Re-record the trace, or score against a bundle " + "that carries a completed run." + ) + else: + console.print(f"[green]{score.summary()}[/green]") + console.print( + "The expectation rejects the run it was drawn from, so it encodes " + "the claim rather than the recording." + ) + + if coverage is not None: + console.print( + f"pins: {len(coverage.live)} live ({_joined(coverage.live)}); " + f"{len(coverage.reproduced)} leaf(s) reproduce the run " + f"({_joined(coverage.reproduced)})" + ) + + +def _joined(paths: tuple[str, ...]) -> str: + return ", ".join(paths) if paths else "none" + + +def _as_json( + score: ExpectationScore, + coverage: PinCoverage | None, + test: Path, + bundle: Path, +) -> dict[str, JsonValue]: + """Return the score as a JSON-shaped payload.""" + return { + "test": str(test), + "bundle": str(bundle), + "verdict": score.verdict, + "killed": score.killed, + "total": score.total, + "discrimination": score.discrimination, + "mutants": [ + { + "name": mutant.name, + "kind": mutant.kind, + "killed": mutant.killed, + "differing_path": mutant.differing_path, + } + for mutant in score.mutants + ], + "pins": ( + { + "live": list(coverage.live), + "reproduced": list(coverage.reproduced), + } + if coverage is not None + else None + ), + } + + +def _load_json(path: Path) -> JsonValue | None: + """Read an expectation file, reporting a read failure as a cannot-score.""" + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except OSError as exc: + _fail(f"cannot read {path}: {exc}") + return None + except json.JSONDecodeError as exc: + _fail(f"{path} is not valid JSON: {exc.msg}") + return None + return redact_json(payload).value + + +def _fail(message: str) -> None: + console.print(f"[red]{message}[/red]") + raise typer.Exit(EXIT_CANNOT_SCORE) diff --git a/src/stepfork/export/__init__.py b/src/stepfork/export/__init__.py index b483258..52fae79 100644 --- a/src/stepfork/export/__init__.py +++ b/src/stepfork/export/__init__.py @@ -7,6 +7,13 @@ export_pytest_test, generate_pytest_source, ) +from stepfork.export.mutants import ( + RECORDED_RUN_MUTANT, + PinCoverage, + derive_battery, + pin_coverage, + recorded_output, +) from stepfork.export.runtime import run_regression_case from stepfork.export.scoring import ( ExpectationScore, @@ -17,15 +24,20 @@ ) __all__ = [ + "RECORDED_RUN_MUTANT", "EntrypointError", "ExpectationScore", "ExportError", "ExportExistsError", "MutantOutcome", "MutantScore", + "PinCoverage", "ScoringError", + "derive_battery", "export_pytest_test", "generate_pytest_source", + "pin_coverage", + "recorded_output", "resolve_entrypoint", "run_regression_case", "score_expectation", diff --git a/src/stepfork/export/mutants.py b/src/stepfork/export/mutants.py new file mode 100644 index 0000000..295dbf3 --- /dev/null +++ b/src/stepfork/export/mutants.py @@ -0,0 +1,296 @@ +"""Derive a mutant battery from an exported expectation and the recorded run. + +:mod:`stepfork.export.scoring` scores a battery the caller supplies. Building +that battery by hand is the part that does not scale, and it is the part that +is easy to skip: an expectation that has drifted into a copy of the recorded +run looks exactly like an anchored one until a claim-drawn mutant separates +them. This module derives the battery mechanically from two things the export +already has. + +Where the two mutants come from +------------------------------- + +An exported test compares the replayed output against ``EXPECTATION``. Two +values are therefore already at hand when the export is prepared: + +* the **expectation** -- the claim, as the generated test asserts it; +* the **recorded run's output** -- the ``RunEnd.output`` of the trace bundle, + which is the value the code that produced the recording actually returned. + +The claim-drawn mutant is that second value, verbatim: the run's own output is +the most plausible output of a variant implementation, because some +implementation did produce it. It is the row that discriminates. A +claim-pinned expectation rejects it -- the run it came from is the bug, so the +expectation must fail against it. A run-pinned expectation accepts it, and +that acceptance is the finding: the test would pass against the code that +produced the recording. + +The code-drawn mutants are perturbations of the *claim's* own leaves -- an +integer moved by one, a string suffixed, a value replaced by ``None``. They +are controls, not discriminators: a run-pinned expectation rejects them too, so +they cannot separate the two readings by themselves. They are what makes the +reading legible, because the contrast between a row family that is always +killed and a row that survives is the whole measurement. + +Two guards keep the families honest +----------------------------------- + +* A perturbation that would land on the recorded value at that leaf is + **skipped**. A row carrying the run's own value is a claim-drawn row wearing a + code-drawn label, and it would let a run-pinned expectation read as though the + battery had caught something. +* The claim-drawn row is the recorded output **whole**, never a projection of it + onto the expectation's shape. The generated test compares whole values, and a + comparison that ignores keys the claim does not name would accept an + expectation that is *stricter* than the run -- the one reading this module + must not produce. + +Per-leaf attribution comes from :func:`pin_coverage`, which reports which +leaves of the expectation carry a claim (they differ from the run) and which +merely reproduce it. It is a comparison of the two values, not a mutation, so +it stays exact where a mutation would only be indicative. + +Nothing here executes anything: a mutant is a value, the comparison is +canonical JSON equality, and running a variant remains the caller's business. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from dataclasses import dataclass +from typing import TypeAlias, cast + +from stepfork.diff.compare import values_equal +from stepfork.export.scoring import MutantOutcome +from stepfork.trace import JsonValue, RunEnd, RunStatus, Trace + +RECORDED_RUN_MUTANT = "the recorded run" +"""Name of the claim-drawn mutant: the run's own output, unscored for shape.""" + +LeafPath: TypeAlias = tuple[str | int, ...] +"""A leaf's location, as a walk from the root: keys as strings, indices as ints.""" + +PathValue: TypeAlias = tuple[str, JsonValue] +"""A mutant label and the value it substitutes at one leaf.""" + + +def recorded_output(trace: Trace) -> JsonValue | None: + """Return the output of the last completed run, or None when it has none. + + A trace that ended in a failure state carries no output, and neither does + one recorded before the run reached ``RunEnd``. The absence is reported + rather than worked around: with no recorded output there is nothing to + draw a claim-drawn mutant from, and the honest verdict is that the battery + cannot separate the two readings. + """ + for event in reversed(trace.events): + if isinstance(event, RunEnd) and event.run_status is RunStatus.COMPLETED: + return event.output + return None + + +def render_path(path: LeafPath) -> str: + """Render a leaf path the way :func:`first_difference` renders one.""" + rendered = "" + for step in path: + if isinstance(step, int): + rendered += f"[{step}]" + else: + rendered += f".{step}" if rendered else step + return rendered or "$" + + +@dataclass(frozen=True) +class PinCoverage: + """Which leaves of an expectation carry a claim, and which reproduce the run. + + ``reproduced`` leaves are not defects: a fix does not change every field, + and an expectation that requires an untouched field to keep its recorded + value is asserting the fix, not the run. What they are is *not evidence of + anchoring*, and a summary that reported only a count of leaves would read + them as though they were. + """ + + live: tuple[str, ...] + """Paths whose expectation value differs from the recorded run's value.""" + + reproduced: tuple[str, ...] + """Paths whose expectation value equals the recorded run's value.""" + + @property + def total(self) -> int: + """Number of leaves compared.""" + return len(self.live) + len(self.reproduced) + + @property + def anchored(self) -> bool: + """True when at least one leaf carries a value the run did not produce.""" + return bool(self.live) + + +def derive_battery( + expectation: JsonValue, + recorded: JsonValue | None, + *, + has_expectation: bool = True, +) -> tuple[MutantOutcome, ...]: + """Derive a mutant battery from an expectation and a recorded run. + + The battery is deterministic: leaves are visited in sorted key order, and + each leaf contributes its perturbations in a fixed order, so two runs over + the same inputs produce the same rows with the same names. + + Args: + expectation: The expectation the generated test asserts. + recorded: The recorded run's output, or None when the trace has none. + None is not an error: the battery is then the code-drawn controls + alone, which scores as ``undetermined`` -- the honest reading of a + battery that cannot separate the two. + has_expectation: False when the export carried no expectation at all. + The battery is then empty, because the generated test performs no + comparison for a mutant to change the outcome of. + + Returns: + One claim-drawn row (when a recorded output exists) followed by the + code-drawn control rows. + """ + if not has_expectation: + return () + + mutants: list[MutantOutcome] = [] + if recorded is not None: + mutants.append( + MutantOutcome( + name=RECORDED_RUN_MUTANT, + kind="claim", + output=recorded, + ) + ) + + for path, value in _leaves(expectation): + found, recorded_value = ( + _at(recorded, path) if recorded is not None else (False, None) + ) + for label, replacement in _perturbations(value): + if found and values_equal(replacement, recorded_value): + continue + mutants.append( + MutantOutcome( + name=f"{render_path(path)}: {label}", + kind="code", + output=_replace(expectation, path, replacement), + ) + ) + return tuple(mutants) + + +def pin_coverage(expectation: JsonValue, recorded: JsonValue) -> PinCoverage: + """Report which expectation leaves a recorded run can and cannot account for. + + A leaf the run does not carry at all counts as live: the claim states + something about a value the run never produced, which is a claim the run + cannot have pinned. + """ + live: list[str] = [] + reproduced: list[str] = [] + for path, value in _leaves(expectation): + found, recorded_value = _at(recorded, path) + if found and values_equal(value, recorded_value): + reproduced.append(render_path(path)) + else: + live.append(render_path(path)) + return PinCoverage(live=tuple(live), reproduced=tuple(reproduced)) + + +def _is_leaf(value: JsonValue) -> bool: + """True for a scalar, and for a container with nothing inside it. + + An empty container is a leaf because there is no value below it to compare: + ``{"a": {}}`` claims that ``a`` holds nothing, and a run that filled ``a`` + contradicts that claim at ``a`` itself. + """ + if isinstance(value, (dict, list)): + return not value + return True + + +def _leaves( + value: JsonValue, path: LeafPath = () +) -> Iterator[tuple[LeafPath, JsonValue]]: + """Yield every leaf of a JSON value, in sorted key order.""" + if _is_leaf(value): + yield path, value + return + if isinstance(value, dict): + for key in sorted(value): + yield from _leaves(value[key], (*path, key)) + return + items = cast("list[JsonValue]", value) + for index, item in enumerate(items): + yield from _leaves(item, (*path, index)) + + +def _at(value: JsonValue | None, path: LeafPath) -> tuple[bool, JsonValue]: + """Return whether a value sits at a path, and the value if one does.""" + current: JsonValue | None = value + for step in path: + if isinstance(step, int): + if not isinstance(current, list) or step >= len(current): + return False, None + current = cast("list[JsonValue]", current)[step] + continue + if not isinstance(current, dict) or step not in current: + return False, None + current = cast("dict[str, JsonValue]", current)[step] + return True, current + + +def _replace(value: JsonValue, path: LeafPath, replacement: JsonValue) -> JsonValue: + """Return a copy of a value with one leaf replaced.""" + if not path: + return replacement + step, *rest = path + tail: LeafPath = tuple(rest) + if isinstance(step, int): + items = cast("list[JsonValue]", value) + return [ + _replace(item, tail, replacement) if index == step else item + for index, item in enumerate(items) + ] + mapping = cast("dict[str, JsonValue]", value) + return { + key: _replace(item, tail, replacement) if key == step else item + for key, item in mapping.items() + } + + +def _perturbations(value: JsonValue) -> tuple[PathValue, ...]: + """Return the code-drawn perturbations available for one leaf value. + + Only plausible, shape-preserving variants are offered: a value that could + not have come out of a working implementation is not a mutant, it is noise. + ``bool`` is checked before ``int`` because ``bool`` is a subclass of it. + + A perturbation that would leave the value unchanged is dropped. Floating + point makes that reachable -- ``1e308 + 1.0`` is ``1e308`` -- and a row + that carries the claim's own value is not a mutant: it would survive every + expectation, including the ones the battery exists to convict. + """ + if isinstance(value, bool): + candidates: tuple[PathValue, ...] = ( + ("flipped", not value), + ("as a string", "True" if value else "False"), + ) + elif isinstance(value, int): + candidates = (("off by one", value + 1), ("as a string", str(value))) + elif isinstance(value, float): + candidates = (("off by one", value + 1.0), ("as a string", repr(value))) + elif isinstance(value, str): + candidates = (("with a suffix", f"{value}?"), ("as null", None)) + elif value is None: + candidates = (("as zero", 0), ("as an empty string", "")) + else: + return () + return tuple( + candidate for candidate in candidates if not values_equal(candidate[1], value) + ) diff --git a/src/stepfork/export/scoring.py b/src/stepfork/export/scoring.py index ea352a2..7352251 100644 --- a/src/stepfork/export/scoring.py +++ b/src/stepfork/export/scoring.py @@ -124,27 +124,27 @@ def by_kind(self, kind: MutantKind) -> tuple[MutantScore, ...]: return tuple(mutant for mutant in self.mutants if mutant.kind == kind) def summary(self) -> str: - """Return a one-line reading of the score.""" + """Return a one-line reading of the score, labelled with its verdict.""" if self.verdict == "no_expectation": return "no expectation to score: the test asserts the recorded run" if self.verdict == "undetermined": return ( - f"rejected {self.killed}/{self.total} mutants, but the battery " - "carries no claim-drawn mutant, so it cannot separate a " - "claim-pinned expectation from a run-pinned one" + f"undetermined: rejected {self.killed}/{self.total} mutants, but " + "the battery carries no claim-drawn mutant, so it cannot " + "separate a claim-pinned expectation from a run-pinned one" ) if self.verdict == "run_pinned": survived = ", ".join( mutant.name for mutant in self.by_kind("claim") if mutant.survived ) return ( - "expectation is run-pinned: it accepts the claim-drawn " - f"mutant(s) {survived}, so it encodes the recorded run rather " - "than the claim" + f"run-pinned: the expectation accepts the claim-drawn mutant(s) " + f"{survived}, so it encodes the recorded run rather than " + "the claim" ) return ( - f"expectation is claim-pinned: rejected {self.killed}/{self.total} " - "mutants, including every claim-drawn mutant" + f"claim-pinned: rejected {self.killed}/{self.total} mutants, " + "including every claim-drawn mutant" ) diff --git a/src/stepfork/export/testfile.py b/src/stepfork/export/testfile.py new file mode 100644 index 0000000..1d27ad3 --- /dev/null +++ b/src/stepfork/export/testfile.py @@ -0,0 +1,178 @@ +"""Read the literals an exported pytest test carries, without executing it. + +``stepfork export`` writes a fixed template whose module-level assignments +name the bundle, the expectation, and whether an expectation is asserted at +all. Reading those back is what lets ``stepfork score`` work on the file the +export just wrote, rather than on a second copy of the expectation that can +drift from it. + +Reading is not running. The file is parsed with :mod:`ast`, JSON-shaped values +are read with :func:`ast.literal_eval`, and the bundle path is evaluated by a +whitelist evaluator that understands exactly the two expression shapes the +generator emits -- ``Path(__file__).resolve().parent`` optionally joined with +literal parts, and a bare ``Path("")``. Anything else is refused with +a message that names the file and suggests ``--bundle``, because the +alternative to refusing is guessing, and a guess here would score the wrong +bundle silently. Nothing is imported, no code is compiled, and nothing in the +file is executed. +""" + +from __future__ import annotations + +import ast +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from stepfork.trace import JsonValue + +READ_NAMES = ( + "TRACE_PATH", + "EXPECTATION", + "HAS_EXPECTATION", + "TRAJECTORY_EXPECTATION", +) + + +class ExportedTestError(ValueError): + """Raised when an exported test file cannot be read.""" + + +@dataclass(frozen=True) +class ExportedTest: + """The literals an exported test carries. + + ``trace_path`` is None when the file assigns no ``TRACE_PATH``; the caller + may still name a bundle itself, so the absence is reported rather than + raised here. + """ + + path: Path + trace_path: Path | None + expectation: JsonValue + has_expectation: bool + trajectory_expectation: JsonValue | None + + +def read_exported_test(path: Path) -> ExportedTest: + """Read the literals of an exported regression test. + + Args: + path: The generated test file. + + Returns: + The bundle path it names (when it names one), the expectation it + asserts, whether it asserts one, and any trajectory expectation. + + Raises: + ExportedTestError: If the file cannot be read, does not parse, assigns a + value that is not a literal, or names its bundle with an + expression outside the shapes the generator emits. + """ + try: + source = path.read_text(encoding="utf-8") + except OSError as exc: + raise ExportedTestError(f"cannot read {path}: {exc}") from exc + try: + module = ast.parse(source, filename=str(path)) + except SyntaxError as exc: + raise ExportedTestError(f"{path} does not parse: {exc}") from exc + + assigned = _assigned_literals(module) + expectation = _literal(assigned, "EXPECTATION", path, default=None) + has_expectation = _literal(assigned, "HAS_EXPECTATION", path, default=True) + if not isinstance(has_expectation, bool): + raise ExportedTestError( + f"{path}: HAS_EXPECTATION is {has_expectation!r}; the export writes a bool" + ) + trajectory = _literal(assigned, "TRAJECTORY_EXPECTATION", path, default=None) + node = assigned.get("TRACE_PATH") + return ExportedTest( + path=path, + trace_path=_evaluate_path(node, path) if node is not None else None, + expectation=expectation, + has_expectation=has_expectation, + trajectory_expectation=trajectory, + ) + + +def _assigned_literals(module: ast.Module) -> dict[str, ast.expr]: + """Map each read name to the expression of its last module-level assignment.""" + found: dict[str, ast.expr] = {} + for statement in module.body: + if not isinstance(statement, ast.Assign): + continue + for target in statement.targets: + if isinstance(target, ast.Name) and target.id in READ_NAMES: + found[target.id] = statement.value + return found + + +def _literal( + assigned: dict[str, ast.expr], + name: str, + path: Path, + *, + default: JsonValue, +) -> JsonValue: + """Read one assigned value as a JSON literal.""" + node = assigned.get(name) + if node is None: + return default + try: + value = ast.literal_eval(node) + except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as exc: + raise ExportedTestError( + f"{path}:{node.lineno}: {name} is not a literal value " + f"({type(exc).__name__}); score reads the literals the export wrote" + ) from exc + return cast("JsonValue", value) + + +def _evaluate_path(node: ast.expr, test_path: Path) -> Path: + """Evaluate a bundle path expression, refusing anything outside the template. + + The accepted shapes are the ones ``stepfork.export.generator._path_expression`` + emits, so a file the generator wrote always reads and a hand-edited + expression is refused rather than misread. + """ + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return Path(node.value) + if isinstance(node, ast.Name) and node.id == "__file__": + return test_path + if isinstance(node, ast.Attribute) and node.attr == "parent": + return _evaluate_path(node.value, test_path).parent + if isinstance(node, ast.Call): + function = node.func + if ( + isinstance(function, ast.Name) + and function.id == "Path" + and len(node.args) == 1 + ): + return Path(_string_argument(node.args[0], test_path)) + if ( + isinstance(function, ast.Attribute) + and function.attr == "resolve" + and not node.args + ): + return _evaluate_path(function.value, test_path).resolve() + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div): + return _evaluate_path(node.left, test_path) / _string_argument( + node.right, test_path + ) + raise ExportedTestError( + f"{test_path}:{node.lineno}: cannot read the bundle path from this " + "expression; pass --bundle to name the .sftrace bundle directly" + ) + + +def _string_argument(node: ast.expr, test_path: Path) -> str: + """Read one path part: a string literal, or ``__file__`` inside ``Path(...)``.""" + if isinstance(node, ast.Name) and node.id == "__file__": + return str(test_path) + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + raise ExportedTestError( + f"{test_path}:{node.lineno}: a path part is not a string literal; " + "pass --bundle to name the .sftrace bundle directly" + ) diff --git a/tests/unit/test_cli_score.py b/tests/unit/test_cli_score.py new file mode 100644 index 0000000..a074abb --- /dev/null +++ b/tests/unit/test_cli_score.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path + +from typer.testing import CliRunner + +from stepfork import record +from stepfork.cli.main import app +from stepfork.export.generator import generate_pytest_source +from stepfork.trace import JsonValue + +runner = CliRunner() + +RECORDED: JsonValue = {"status": "refunded", "amount": 0.0} +CLAIM: JsonValue = {"status": "refunded", "amount": 42.5} + + +def _flat(text: str) -> str: + """Collapse the line breaks the console inserts when it wraps.""" + return re.sub(r"\s+", " ", text) + + +def _record(bundle: Path, output: JsonValue) -> Path: + with record("refund-agent", output=bundle) as session: + session.set_output(output) + return bundle + + +def _write_case( + tmp_path: Path, + *, + recorded: JsonValue = RECORDED, + expectation: JsonValue = CLAIM, + has_expectation: bool = True, + record_run: bool = True, +) -> tuple[Path, Path]: + bundle = tmp_path / "refund.sftrace" + if record_run: + _record(bundle, recorded) + test = tmp_path / "test_refund_regression.py" + test.write_text( + generate_pytest_source( + trace_path=bundle, + output_path=test, + entrypoint="pkg.mod:run", + expectation=expectation, + has_expectation=has_expectation, + ), + encoding="utf-8", + ) + return bundle, test + + +def test_score_reports_a_claim_pinned_expectation(tmp_path: Path) -> None: + _, test = _write_case(tmp_path) + result = runner.invoke(app, ["score", str(test)]) + assert result.exit_code == 0 + assert "claim-pinned" in _flat(result.stdout) + assert "the recorded run" in _flat(result.stdout) + + +def test_score_convicts_a_run_pinned_expectation(tmp_path: Path) -> None: + _, test = _write_case(tmp_path, expectation=RECORDED) + result = runner.invoke(app, ["score", str(test)]) + assert result.exit_code == 1 + assert "run-pinned" in _flat(result.stdout) + assert "unapproved oracle" in _flat(result.stdout) + + +def test_score_prints_json(tmp_path: Path) -> None: + _, test = _write_case(tmp_path) + result = runner.invoke(app, ["score", str(test), "--json"]) + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["verdict"] == "claim_pinned" + assert payload["pins"] == {"live": ["amount"], "reproduced": ["status"]} + assert payload["total"] == payload["killed"] + assert {mutant["kind"] for mutant in payload["mutants"]} == {"claim", "code"} + + +def test_score_accepts_an_expectation_override(tmp_path: Path) -> None: + """Scoring a candidate expectation is the pre-export review step.""" + _, test = _write_case(tmp_path, expectation=RECORDED) + candidate = tmp_path / "expected.json" + candidate.write_text(json.dumps(CLAIM), encoding="utf-8") + assert runner.invoke(app, ["score", str(test)]).exit_code == 1 + result = runner.invoke(app, ["score", str(test), "--expect-output", str(candidate)]) + assert result.exit_code == 0 + assert "claim-pinned" in _flat(result.stdout) + + +def test_score_reports_a_missing_bundle(tmp_path: Path) -> None: + _, test = _write_case(tmp_path, record_run=False) + result = runner.invoke(app, ["score", str(test)]) + assert result.exit_code == 2 + assert "trace bundle not found" in _flat(result.stdout) + + +def test_score_names_a_bundle_it_was_given(tmp_path: Path) -> None: + bundle, test = _write_case(tmp_path, expectation=RECORDED) + test.write_text( + test.read_text(encoding="utf-8").replace("TRACE_PATH = ", "TRACE_PATH_OLD = "), + encoding="utf-8", + ) + result = runner.invoke(app, ["score", str(test), "--bundle", str(bundle)]) + assert result.exit_code == 1 + assert "run-pinned" in _flat(result.stdout) + + +def test_score_without_a_bundle_asks_for_one(tmp_path: Path) -> None: + _, test = _write_case(tmp_path, record_run=False) + test.write_text( + test.read_text(encoding="utf-8").replace("TRACE_PATH = ", "TRACE_PATH_OLD = "), + encoding="utf-8", + ) + result = runner.invoke(app, ["score", str(test)]) + assert result.exit_code == 2 + assert "--bundle" in _flat(result.stdout) + + +def test_score_reports_an_export_without_an_expectation(tmp_path: Path) -> None: + _, test = _write_case(tmp_path, has_expectation=False, expectation=None) + result = runner.invoke(app, ["score", str(test)]) + assert result.exit_code == 2 + assert "no expectation to score" in _flat(result.stdout) + + +def test_score_cannot_separate_readings_without_a_recorded_run(tmp_path: Path) -> None: + """A run that recorded no output leaves nothing claim-drawn to derive.""" + _, test = _write_case(tmp_path, recorded=None) + result = runner.invoke(app, ["score", str(test)]) + assert result.exit_code == 2 + assert "undetermined" in _flat(result.stdout) + + +def test_score_reports_an_unreadable_test_file(tmp_path: Path) -> None: + test = tmp_path / "test_handwritten.py" + test.write_text("EXPECTATION = build_expectation()\n", encoding="utf-8") + result = runner.invoke(app, ["score", str(test)]) + assert result.exit_code == 2 + assert "not a literal value" in _flat(result.stdout) + + +def test_score_is_registered() -> None: + result = runner.invoke(app, ["--help"]) + assert result.exit_code == 0 + assert "score" in _flat(result.stdout) diff --git a/tests/unit/test_export_mutants.py b/tests/unit/test_export_mutants.py new file mode 100644 index 0000000..0226382 --- /dev/null +++ b/tests/unit/test_export_mutants.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +from collections.abc import Sequence + +from hypothesis import given +from hypothesis import strategies as st + +from stepfork.diff.compare import first_difference +from stepfork.export.mutants import ( + RECORDED_RUN_MUTANT, + PinCoverage, + _perturbations, + derive_battery, + pin_coverage, + recorded_output, + render_path, +) +from stepfork.export.scoring import MutantKind, MutantOutcome, score_expectation +from stepfork.trace import JsonValue, RunEnd, RunStart, RunStatus, Trace + +# A recorded refund run: the agent reported the refund as done and issued +# nothing. The claim is that a refund of 42.50 reaches the order. +RECORDED_RUN: JsonValue = {"status": "refunded", "amount": 0.0} +CLAIM_PINNED: JsonValue = {"status": "refunded", "amount": 42.5} +RUN_PINNED: JsonValue = {"status": "refunded", "amount": 0.0} + + +def _derived( + expectation: JsonValue, recorded: JsonValue | None +) -> tuple[MutantOutcome, ...]: + return derive_battery(expectation, recorded) + + +def _names(battery: Sequence[MutantOutcome]) -> list[str]: + return [mutant.name for mutant in battery] + + +def _by_kind(battery: Sequence[MutantOutcome], kind: MutantKind) -> list[MutantOutcome]: + return [mutant for mutant in battery if mutant.kind == kind] + + +def test_derived_battery_separates_a_claim_pinned_expectation() -> None: + score = score_expectation(CLAIM_PINNED, _derived(CLAIM_PINNED, RECORDED_RUN)) + assert score.verdict == "claim_pinned" + assert score.total > 1 + + +def test_derived_battery_convicts_a_run_pinned_expectation() -> None: + """The finding the derivation exists to produce: the claim row survives.""" + score = score_expectation(RUN_PINNED, _derived(RUN_PINNED, RECORDED_RUN)) + assert score.verdict == "run_pinned" + survivors = [mutant.name for mutant in score.survivors] + assert survivors == [RECORDED_RUN_MUTANT] + # The code-drawn controls are killed either way, which is why the reading + # cannot come from them. + assert all(mutant.killed for mutant in score.mutants if mutant.kind == "code") + + +def test_claim_mutant_is_the_recorded_run_whole_not_a_projection() -> None: + """A projection would hide an expectation that is stricter than the run.""" + recorded: JsonValue = {"status": "refunded", "amount": 0.0, "receipt": "r-1"} + claim: JsonValue = {"status": "refunded"} + battery = _derived(claim, recorded) + claim_rows = _by_kind(battery, "claim") + assert len(claim_rows) == 1 + assert claim_rows[0].output == recorded + score = score_expectation(claim, battery) + assert score.verdict == "claim_pinned" + + +def test_expectation_stricter_than_the_run_is_never_run_pinned() -> None: + """`{}` claims an empty output; a run that produced fields contradicts it.""" + recorded: JsonValue = {"status": "refunded"} + score = score_expectation({}, _derived({}, recorded)) + assert score.verdict == "claim_pinned" + + +def test_no_recorded_output_yields_controls_only() -> None: + """Without a recorded run there is nothing claim-drawn to derive.""" + battery = _derived(CLAIM_PINNED, None) + assert _by_kind(battery, "claim") == [] + assert _by_kind(battery, "code") != [] + assert score_expectation(CLAIM_PINNED, battery).verdict == "undetermined" + + +def test_no_expectation_yields_an_empty_battery() -> None: + assert derive_battery(None, RECORDED_RUN, has_expectation=False) == () + score = score_expectation(None, (), has_expectation=False) + assert score.verdict == "no_expectation" + + +def test_a_perturbation_that_lands_on_the_recorded_value_is_skipped() -> None: + """A code-drawn label on the run's own value would be a claim row in disguise.""" + claim: JsonValue = {"amount": 42.5} + recorded: JsonValue = {"amount": 43.5} # the "off by one" perturbation + battery = _derived(claim, recorded) + names = _names(battery) + assert "amount: off by one" not in names + assert "amount: as a string" in names + # Only the perturbation that collides is dropped; the claim row is not. + assert _names(_by_kind(battery, "claim")) == [RECORDED_RUN_MUTANT] + + +def test_code_drawn_rows_carry_the_claim_with_one_leaf_changed() -> None: + battery = _by_kind(_derived(CLAIM_PINNED, RECORDED_RUN), "code") + assert battery + for mutant in battery: + difference = first_difference(CLAIM_PINNED, mutant.output) + assert difference is not None + names = _names(_derived(CLAIM_PINNED, RECORDED_RUN)) + assert "amount: as a string" in names + assert "status: with a suffix" in names + + +def test_leaf_paths_are_rendered_the_way_first_difference_renders_them() -> None: + claim: JsonValue = {"plan": ["search", "book"], "meta": {"city": "Paris"}} + recorded: JsonValue = {"plan": ["search", "cancel"], "meta": {"city": "Paris"}} + names = _names(_derived(claim, recorded)) + assert "plan[1]: with a suffix" in names + difference = first_difference(claim, recorded) + assert difference is not None + assert difference[0] == "plan[1]" + assert render_path(("plan", 1)) == difference[0] + assert render_path(()) == "$" + + +def test_battery_is_deterministic() -> None: + claim: JsonValue = {"a": 1, "b": {"c": "x"}, "d": True} + first = _derived(claim, {"a": 2, "b": {"c": "y"}, "d": False}) + second = _derived(claim, {"a": 2, "b": {"c": "y"}, "d": False}) + assert first == second + + +def test_pin_coverage_splits_live_leaves_from_reproduced_ones() -> None: + claim: JsonValue = {"amount": 42.5, "status": "refunded"} + coverage = pin_coverage(claim, RECORDED_RUN) + assert isinstance(coverage, PinCoverage) + assert coverage.live == ("amount",) + assert coverage.reproduced == ("status",) + assert coverage.total == 2 + assert coverage.anchored + + +def test_pin_coverage_counts_a_leaf_the_run_never_produced_as_live() -> None: + coverage = pin_coverage({"amount": 42.5, "receipt": "r-1"}, {"amount": 42.5}) + assert coverage.live == ("receipt",) + assert coverage.reproduced == ("amount",) + + +def test_an_empty_container_is_a_leaf_the_run_can_contradict() -> None: + claim: JsonValue = {"trace": {}} + recorded: JsonValue = {"trace": {"steps": 3}} + coverage = pin_coverage(claim, recorded) + assert coverage.live == ("trace",) + assert score_expectation(claim, _derived(claim, recorded)).verdict == "claim_pinned" + + +def test_recorded_output_reads_the_last_completed_run() -> None: + trace = Trace(agent_name="probe") + trace.add(RunStart()) + trace.add(RunEnd(run_status=RunStatus.COMPLETED, output={"ok": True})) + trace.add(RunEnd(run_status=RunStatus.FAILED)) + assert recorded_output(trace) == {"ok": True} + + +def test_recorded_output_is_none_when_no_run_completed() -> None: + trace = Trace(agent_name="probe") + trace.add(RunStart()) + trace.add(RunEnd(run_status=RunStatus.FAILED)) + assert recorded_output(trace) is None + assert recorded_output(Trace(agent_name="empty")) is None + + +@given( + st.one_of( + st.booleans(), + st.integers(), + st.floats(allow_nan=False, allow_infinity=False), + st.text(), + st.none(), + ) +) +def test_every_perturbation_changes_its_value(value: JsonValue) -> None: + """A no-op row would survive every expectation, convicting nothing.""" + for label, replacement in _perturbations(value): + assert label + assert first_difference(value, replacement) is not None diff --git a/tests/unit/test_export_testfile.py b/tests/unit/test_export_testfile.py new file mode 100644 index 0000000..9a818c5 --- /dev/null +++ b/tests/unit/test_export_testfile.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from stepfork.export.generator import generate_pytest_source +from stepfork.export.testfile import ExportedTestError, read_exported_test + +EXPECTATION = {"status": "refunded", "amount": 42.5} + + +def _write(tmp_path: Path, source: str, name: str = "test_run_regression.py") -> Path: + path = tmp_path / name + path.write_text(source, encoding="utf-8") + return path + + +def _generated(tmp_path: Path, *, depth: int = 0, **kwargs: object) -> Path: + bundle = tmp_path + for index in range(depth): + bundle = bundle / f"level{index}" + bundle = bundle / "refund.sftrace" + output = tmp_path / "test_refund_regression.py" + source = generate_pytest_source( + trace_path=bundle, + output_path=output, + entrypoint="pkg.mod:run", + expectation=EXPECTATION, + **kwargs, # type: ignore[arg-type] + ) + return _write(tmp_path, source) + + +def test_reads_the_literals_the_generator_writes(tmp_path: Path) -> None: + test = _generated(tmp_path) + exported = read_exported_test(test) + assert exported.path == test + assert exported.trace_path == tmp_path / "refund.sftrace" + assert exported.expectation == EXPECTATION + assert exported.has_expectation is True + assert exported.trajectory_expectation is None + + +def test_reads_the_absolute_path_form(tmp_path: Path) -> None: + """A bundle too far up the tree is written as an absolute Path(...).""" + test = _generated(tmp_path, depth=5) + assert "Path('/" in test.read_text(encoding="utf-8") + exported = read_exported_test(test) + assert exported.trace_path == ( + tmp_path / "level0/level1/level2/level3/level4/refund.sftrace" + ) + + +def test_reads_a_reviewed_trajectory_expectation(tmp_path: Path) -> None: + trajectory = {"called": ["search_flights"], "sequence": ["search_flights"]} + test = _generated(tmp_path, trajectory_expectation=trajectory) + assert read_exported_test(test).trajectory_expectation == trajectory + + +def test_reads_an_export_without_an_expectation(tmp_path: Path) -> None: + test = _generated(tmp_path, has_expectation=False) + exported = read_exported_test(test) + assert exported.has_expectation is False + assert exported.expectation is None + + +def test_reading_never_executes_the_file(tmp_path: Path) -> None: + """The contract is parse-and-read; an exported test must not run here.""" + marker = tmp_path / "executed.marker" + test = _write( + tmp_path, + "from pathlib import Path\n" + f"Path({str(marker)!r}).write_text('ran')\n" + "TRACE_PATH = Path('/nowhere/run.sftrace')\n" + "EXPECTATION = {'ok': True}\n", + ) + exported = read_exported_test(test) + assert exported.expectation == {"ok": True} + assert not marker.exists() + + +def test_refuses_a_non_literal_expectation(tmp_path: Path) -> None: + test = _write( + tmp_path, + "TRACE_PATH = Path('/nowhere/run.sftrace')\nEXPECTATION = dict(status='ok')\n", + ) + with pytest.raises(ExportedTestError, match="not a literal"): + read_exported_test(test) + + +def test_refuses_an_unsupported_bundle_expression(tmp_path: Path) -> None: + test = _write( + tmp_path, + "TRACE_PATH = Path(__file__).resolve().parent / TRACES[0]\n" + "EXPECTATION = {'ok': True}\n", + ) + with pytest.raises(ExportedTestError, match="--bundle"): + read_exported_test(test) + + +def test_a_file_without_trace_path_reports_none(tmp_path: Path) -> None: + test = _write(tmp_path, "EXPECTATION = {'ok': True}\n") + exported = read_exported_test(test) + assert exported.trace_path is None + assert exported.has_expectation is True + + +def test_refuses_a_non_boolean_has_expectation(tmp_path: Path) -> None: + test = _write( + tmp_path, + "TRACE_PATH = Path('/nowhere/run.sftrace')\nEXPECTATION = {'ok': True}\n" + "HAS_EXPECTATION = 'yes'\n", + ) + with pytest.raises(ExportedTestError, match="HAS_EXPECTATION"): + read_exported_test(test) + + +def test_the_last_assignment_wins(tmp_path: Path) -> None: + """The module's value is its final one, so reading agrees with running.""" + test = _write( + tmp_path, + "TRACE_PATH = Path('/first/run.sftrace')\n" + "TRACE_PATH = Path('/second/run.sftrace')\n" + "EXPECTATION = {'ok': True}\n", + ) + assert read_exported_test(test).trace_path == Path("/second/run.sftrace") + + +def test_a_missing_file_is_reported(tmp_path: Path) -> None: + with pytest.raises(ExportedTestError, match="cannot read"): + read_exported_test(tmp_path / "absent.py") + + +def test_an_unparsable_file_is_reported(tmp_path: Path) -> None: + test = _write(tmp_path, "EXPECTATION = {'ok':\n") + with pytest.raises(ExportedTestError, match="does not parse"): + read_exported_test(test)