From a5d3f9a879a649313e1ae252c414e63e72e7e818 Mon Sep 17 00:00:00 2001 From: how2how2how2-arch Date: Mon, 19 Oct 2026 01:06:07 +0800 Subject: [PATCH 1/5] feat(export): write the receipt from the runner, and say could_not_execute --- src/stepfork/export/receipt.py | 394 +++++++++++++++++++++++++++++++++ 1 file changed, 394 insertions(+) create mode 100644 src/stepfork/export/receipt.py diff --git a/src/stepfork/export/receipt.py b/src/stepfork/export/receipt.py new file mode 100644 index 0000000..a243b01 --- /dev/null +++ b/src/stepfork/export/receipt.py @@ -0,0 +1,394 @@ +"""Runner-owned receipts for an exported regression test. + +``stepfork export`` writes a pytest file whose body calls +:func:`stepfork.export.runtime.run_regression_case`. That file is the *check*. +This module is the *runner*: it decides whether the check executed at all, runs +it as a subprocess, and writes a receipt describing what happened. + +The distinction matters because a pytest exit status cannot carry it. Measured +on pytest 9.1.1, with the test body raising rather than asserting: + +=========================== ========== ============ +case exit status JUnit kind +=========================== ========== ============ +assertion failed 1 ``failure`` +the trace bundle is missing 1 ``failure`` +a dependency is absent 1 ``failure`` +a fixture cannot be built 1 ``error`` +collection failed 2 ``error`` +no tests were collected 5 (none) +=========================== ========== ============ + +So on the artifact ``stepfork export`` actually produces, "the agent's +behaviour changed" and "the bundle this test names is not there" arrive as the +same status *and* the same JUnit kind. A CI that keys on either one reports a +missing artifact as a caught regression. + +Two things fix that, and both belong to the runner rather than to the test: + +1. **The artifacts are resolved before the check is launched.** If the bundle + the test names is absent or unreadable, the receipt says + ``could_not_execute`` and pytest is never started, so no failure can be + manufactured for it. +2. **The receipt is written by this process, after the child exits, from what + the child produced.** It records the verdict, the raw exit status, the + JUnit counts, a digest of the bundle and a hash of the test file, so the + three inputs the verdict rests on are named in the artifact itself. + +The test under measurement never writes the receipt. A file the check writes +about itself is a file the check can get wrong, and the interesting failures +here are exactly the ones where the check does not run at all. + +**What this does not catch.** The runner pre-checks the artifacts it can name +from the test's literals -- the bundle. It does not pre-check what the test +body *imports*: an entrypoint the local tree cannot resolve, or a dependency +imported inside the test, raises inside the test body, and pytest reports an +exception raised there as ``failure``, exactly as it reports a changed return +value. Those cases still read as ``failed``. Moving the check is the fix, not +widening the pattern match: resolving ``ENTRYPOINT`` before the launch, the way +the bundle is resolved, would turn them into ``could_not_execute`` too. +""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +import tempfile +import time +import xml.etree.ElementTree as ElementTree +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from stepfork import __version__ +from stepfork.export.testfile import ExportedTestError, read_exported_test + +VERDICT_PASSED = "passed" +VERDICT_FAILED = "failed" +VERDICT_COULD_NOT_EXECUTE = "could_not_execute" + +#: Exit codes `stepfork run` returns, so a CI can key on the status alone. +EXIT_PASSED = 0 +EXIT_FAILED = 1 +EXIT_COULD_NOT_EXECUTE = 2 + +EXIT_BY_VERDICT = { + VERDICT_PASSED: EXIT_PASSED, + VERDICT_FAILED: EXIT_FAILED, + VERDICT_COULD_NOT_EXECUTE: EXIT_COULD_NOT_EXECUTE, +} + +RECEIPT_VERSION = 1 + + +@dataclass(frozen=True) +class ReceiptCounts: + """The JUnit totals the verdict was derived from.""" + + tests: int = 0 + failures: int = 0 + errors: int = 0 + skipped: int = 0 + + def to_json(self) -> dict[str, int]: + return { + "tests": self.tests, + "failures": self.failures, + "errors": self.errors, + "skipped": self.skipped, + } + + +@dataclass(frozen=True) +class Receipt: + """What the runner observed, written by the runner. + + ``reason`` is set whenever the verdict is ``could_not_execute``, and names + the specific way the check failed to run: ``bundle_missing``, + ``bundle_digest_failed``, ``check_error``, ``no_tests_collected``. + """ + + verdict: str + test: str + test_sha256: str + bundle: str | None = None + bundle_digest: str | None = None + reason: str | None = None + detail: str | None = None + exit_status: int | None = None + counts: ReceiptCounts = field(default_factory=ReceiptCounts) + command: list[str] = field(default_factory=list) + duration_ms: int = 0 + created_at: str = "" + stepfork_version: str = __version__ + receipt_version: int = RECEIPT_VERSION + + def to_json(self) -> dict[str, Any]: + return { + "receipt_version": self.receipt_version, + "verdict": self.verdict, + "reason": self.reason, + "detail": self.detail, + "exit_status": self.exit_status, + "bundle": self.bundle, + "bundle_digest": self.bundle_digest, + "test": self.test, + "test_sha256": self.test_sha256, + "counts": self.counts.to_json(), + "command": self.command, + "duration_ms": self.duration_ms, + "stepfork_version": self.stepfork_version, + "created_at": self.created_at, + } + + +def file_sha256(path: Path) -> str: + """SHA-256 of one file's bytes.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def bundle_digest(path: Path) -> str: + """SHA-256 over a bundle's files, in sorted relative-path order. + + Hashing the file contents in a fixed order rather than the directory's + metadata means the digest names the data, not the checkout: copying a + bundle between machines leaves the digest unchanged, and editing any file + in it changes it. + """ + digest = hashlib.sha256() + if path.is_dir(): + root = path + entries = sorted( + (candidate for candidate in path.rglob("*") if candidate.is_file()), + key=lambda candidate: candidate.relative_to(root).as_posix(), + ) + else: + root = path.parent + entries = [path] + for candidate in entries: + digest.update(candidate.relative_to(root).as_posix().encode("utf-8")) + digest.update(b"\0") + digest.update(file_sha256(candidate).encode("ascii")) + digest.update(b"\n") + return digest.hexdigest() + + +def _now() -> str: + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _counts_from_junit(path: Path) -> ReceiptCounts | None: + """Read JUnit totals, or None when the report is absent or unreadable.""" + try: + root = ElementTree.parse(path).getroot() + except (OSError, ElementTree.ParseError): + return None + suite = root.find("testsuite") if root.tag == "testsuites" else root + if suite is None: + return None + + def total(name: str) -> int: + try: + return int(suite.get(name) or 0) + except ValueError: + return 0 + + return ReceiptCounts( + tests=total("tests"), + failures=total("failures"), + errors=total("errors"), + skipped=total("skipped"), + ) + + +def _first_error_message(path: Path) -> str | None: + """The message of the first ``error`` element, trimmed for a receipt line.""" + try: + root = ElementTree.parse(path).getroot() + except (OSError, ElementTree.ParseError): + return None + for case in root.iter("testcase"): + for child in case: + if child.tag != "error": + continue + message = (child.get("message") or "").strip().splitlines() + if message: + return message[0][:300] + return None + + +def run_exported_test( + test_path: Path, + *, + bundle: Path | None = None, + pytest_args: list[str] | None = None, + python: str = sys.executable, + timeout_seconds: float | None = None, + cwd: Path | None = None, +) -> Receipt: + """Run one exported regression test and describe what happened. + + The bundle is resolved and digested *before* pytest starts, so a missing or + unreadable bundle is reported as ``could_not_execute`` rather than as a + failure the check itself cannot be distinguished from. + """ + created = _now() + pytest_args = list(pytest_args or []) + + try: + test_sha = file_sha256(test_path) + except OSError as exc: + return Receipt( + verdict=VERDICT_COULD_NOT_EXECUTE, + reason="test_unreadable", + detail=str(exc), + test=str(test_path), + test_sha256="", + bundle=str(bundle) if bundle is not None else None, + created_at=created, + ) + + resolved_bundle = bundle + if resolved_bundle is None: + try: + resolved_bundle = read_exported_test(test_path).trace_path + except ExportedTestError: + # A hand-written test that is not an export names no bundle, so + # there is no artifact to check first. That is not a refusal: the + # verdict then rests on the JUnit counts alone, which still + # separates "the check disagreed" from "the check never ran". + resolved_bundle = None + + if resolved_bundle is not None and not resolved_bundle.exists(): + return Receipt( + verdict=VERDICT_COULD_NOT_EXECUTE, + reason="bundle_missing", + detail=( + f"the bundle this test names is not there: {resolved_bundle}; " + "the check cannot run, so this is not a regression" + ), + test=str(test_path), + test_sha256=test_sha, + bundle=str(resolved_bundle), + created_at=created, + ) + + digest: str | None = None + try: + if resolved_bundle is not None: + digest = bundle_digest(resolved_bundle) + except OSError as exc: + return Receipt( + verdict=VERDICT_COULD_NOT_EXECUTE, + reason="bundle_unreadable", + detail=str(exc), + test=str(test_path), + test_sha256=test_sha, + bundle=str(resolved_bundle), + created_at=created, + ) + + report = Path(tempfile.mkdtemp(prefix="stepfork-receipt-")) / "junit.xml" + command = [ + python, + "-m", + "pytest", + str(test_path), + "-q", + "--junit-xml", + str(report), + *pytest_args, + ] + + started = time.monotonic() + status: int | None = None + detail: str | None = None + try: + completed = subprocess.run( + command, + cwd=str(cwd) if cwd is not None else None, + capture_output=True, + text=True, + timeout=timeout_seconds, + check=False, + ) + status = completed.returncode + except subprocess.TimeoutExpired: + detail = f"pytest did not finish within {timeout_seconds:g}s" + except OSError as exc: + detail = f"could not launch pytest: {exc}" + duration_ms = int((time.monotonic() - started) * 1000) + + counts = _counts_from_junit(report) + + def receipt(verdict: str, reason: str | None, extra: str | None = None) -> Receipt: + return Receipt( + verdict=verdict, + reason=reason, + detail=extra if extra is not None else detail, + exit_status=status, + counts=counts or ReceiptCounts(), + command=command, + duration_ms=duration_ms, + test=str(test_path), + test_sha256=test_sha, + bundle=str(resolved_bundle), + bundle_digest=digest, + created_at=created, + ) + + if detail is not None: + return receipt(VERDICT_COULD_NOT_EXECUTE, "pytest_unavailable") + + if counts is None: + return receipt( + VERDICT_COULD_NOT_EXECUTE, + "report_missing", + "pytest produced no JUnit report, so the run cannot be classified", + ) + + if counts.errors: + return receipt( + VERDICT_COULD_NOT_EXECUTE, + "check_error", + _first_error_message(report) + or f"{counts.errors} test(s) errored before their assertions ran", + ) + + if counts.tests == 0: + return receipt( + VERDICT_COULD_NOT_EXECUTE, + "no_tests_collected", + "pytest collected no tests from this file", + ) + + if status == 0 and not counts.failures: + return receipt(VERDICT_PASSED, None) + + if counts.failures: + return receipt(VERDICT_FAILED, None) + + # A non-zero status with no recorded failure or error is a pytest-level + # problem (interrupted, internal error, bad arguments), not a verdict + # about the agent. + return receipt( + VERDICT_COULD_NOT_EXECUTE, + "pytest_error", + f"pytest exited {status} without recording a test outcome", + ) + + +def write_receipt(receipt: Receipt, path: Path) -> None: + """Write the receipt as JSON, creating the parent directory if needed.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(receipt.to_json(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) From fc99d9d65d25e7e29c3633a0f3e0b54e90f7c409 Mon Sep 17 00:00:00 2001 From: how2how2how2-arch Date: Mon, 19 Oct 2026 01:06:44 +0800 Subject: [PATCH 2/5] feat(run): a command that says could_not_execute --- src/stepfork/cli/run.py | 141 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 src/stepfork/cli/run.py diff --git a/src/stepfork/cli/run.py b/src/stepfork/cli/run.py new file mode 100644 index 0000000..3549e69 --- /dev/null +++ b/src/stepfork/cli/run.py @@ -0,0 +1,141 @@ +"""`stepfork run` command.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Annotated + +import typer +from rich.console import Console +from rich.table import Table + +from stepfork.export.receipt import ( + EXIT_BY_VERDICT, + VERDICT_COULD_NOT_EXECUTE, + VERDICT_FAILED, + VERDICT_PASSED, + Receipt, + run_exported_test, + write_receipt, +) + +console = Console() + +STYLE = { + VERDICT_PASSED: "[green]PASSED[/green]", + VERDICT_FAILED: "[red]FAILED[/red]", + VERDICT_COULD_NOT_EXECUTE: "[yellow]COULD NOT EXECUTE[/yellow]", +} + + +def run_command( + test: Annotated[ + Path, + typer.Argument( + help="Path to the pytest file `stepfork export` wrote.", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + ), + ], + bundle: Annotated[ + Path | None, + typer.Option( + "--bundle", + help="Name the .sftrace bundle instead of reading it from the test.", + ), + ] = None, + receipt: Annotated[ + Path | None, + typer.Option( + "--receipt", + help="Write a JSON receipt of what the runner observed.", + ), + ] = None, + json_output: Annotated[ + bool, + typer.Option("--json", help="Print the receipt to stdout as JSON."), + ] = False, + pytest_arg: Annotated[ + list[str] | None, + typer.Option( + "--pytest-arg", + help="Extra argument for the pytest subprocess; repeat per argument.", + ), + ] = None, + timeout_seconds: Annotated[ + float | None, + typer.Option( + "--timeout-seconds", + help="Stop the check after this long and report could_not_execute.", + ), + ] = None, +) -> None: + """Run an exported regression test and write the receipt yourself. + + `stepfork export` writes a test; this command runs it. The difference from + running pytest directly is the verdict: a check that could not execute -- + the bundle is missing, a fixture cannot be built, nothing was collected -- + is reported as `could_not_execute`, which is neither a pass nor a caught + regression. pytest's exit status does not carry that distinction: measured + on pytest 9.1.1 a missing bundle and a changed return value both exit 1 and + both land in JUnit as `failure`. + + The receipt is written by this process after the child exits, never by the + test, and it names the bundle digest and the test hash the verdict rests + on. + """ + found: Receipt = run_exported_test( + test, + bundle=bundle, + pytest_args=pytest_arg, + timeout_seconds=timeout_seconds, + ) + + if json_output: + console.print_json(json.dumps(found.to_json(), sort_keys=True)) + else: + _report(found) + + if receipt is not None: + write_receipt(found, receipt) + if not json_output: + console.print(f"Receipt: {receipt}") + + raise typer.Exit(EXIT_BY_VERDICT[found.verdict]) + + +def _report(found: Receipt) -> None: + table = Table.grid(padding=(0, 4)) + table.add_column(style="bold") + table.add_column() + table.add_row("Test", found.test) + table.add_row("Test sha256", found.test_sha256[:16] or "(unreadable)") + table.add_row("Bundle", found.bundle or "(none)") + table.add_row( + "Bundle digest", + (found.bundle_digest or "(none)")[:16], + ) + if found.exit_status is not None: + table.add_row("pytest exit", str(found.exit_status)) + counts = found.counts + table.add_row( + "Collected", + f"{counts.tests} test(s), {counts.failures} failure(s), " + f"{counts.errors} error(s), {counts.skipped} skipped", + ) + console.print(table) + console.print() + console.print(f"Verdict: {STYLE[found.verdict]}") + if found.reason is not None: + console.print(f"Reason: {found.reason}") + if found.detail: + console.print(f"{found.detail}") + if found.verdict == VERDICT_COULD_NOT_EXECUTE: + console.print( + "The check did not execute, so this run says nothing about the " + "agent's behaviour." + ) + console.print() From e2cd19b8c2ac519f7bb0c395423db417db3f7931 Mon Sep 17 00:00:00 2001 From: how2how2how2-arch Date: Mon, 19 Oct 2026 01:09:05 +0800 Subject: [PATCH 3/5] feat(cli): wire stepfork run into the app --- src/stepfork/cli/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/stepfork/cli/main.py b/src/stepfork/cli/main.py index b7b06e7..618d597 100644 --- a/src/stepfork/cli/main.py +++ b/src/stepfork/cli/main.py @@ -12,6 +12,7 @@ from stepfork.cli.inspect import inspect_command from stepfork.cli.minimize import minimize_command from stepfork.cli.replay import replay_command +from stepfork.cli.run import run_command from stepfork.cli.score import score_command from stepfork.cli.validate import validate_command @@ -29,6 +30,7 @@ app.command("inspect")(inspect_command) app.command("minimize")(minimize_command) app.command("replay")(replay_command) +app.command("run")(run_command) app.command("score")(score_command) app.command("validate")(validate_command) From d33137dd9861f153aec937bbe8609d639238a5ef Mon Sep 17 00:00:00 2001 From: how2how2how2-arch Date: Mon, 19 Oct 2026 01:09:33 +0800 Subject: [PATCH 4/5] test(run): cover the tri-state verdict and the receipt --- tests/unit/test_cli_run.py | 194 +++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 tests/unit/test_cli_run.py diff --git a/tests/unit/test_cli_run.py b/tests/unit/test_cli_run.py new file mode 100644 index 0000000..82c178f --- /dev/null +++ b/tests/unit/test_cli_run.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path + +from typer.testing import CliRunner + +from stepfork import record +from stepfork.cli.main import app +from stepfork.export.generator import generate_pytest_source +from stepfork.export.receipt import ( + EXIT_COULD_NOT_EXECUTE, + EXIT_FAILED, + EXIT_PASSED, + VERDICT_COULD_NOT_EXECUTE, + VERDICT_FAILED, + VERDICT_PASSED, + bundle_digest, +) +from stepfork.trace import JsonValue + +runner = CliRunner() + +RECORDED: JsonValue = {"status": "refunded", "amount": 42.5} +CLAIM: JsonValue = {"status": "refunded", "amount": 42.5} + + +def _flat(text: str) -> str: + """Collapse the line breaks the console inserts when it wraps.""" + return re.sub(r"\s+", " ", text) + + +def _record(bundle: Path, output: JsonValue) -> Path: + with record("refund-agent", output=bundle) as session: + session.set_output(output) + return bundle + + +def _export(tmp_path: Path) -> tuple[Path, Path]: + """Record a bundle and export the regression test that names it.""" + bundle = _record(tmp_path / "refund.sftrace", RECORDED) + test = tmp_path / "test_refund_regression.py" + test.write_text( + generate_pytest_source( + trace_path=bundle, + output_path=test, + entrypoint="pkg.mod:run", + expectation=CLAIM, + ), + encoding="utf-8", + ) + return bundle, test + + +def _plain_test(tmp_path: Path, body: str) -> Path: + """A test file that exercises the runner without needing a real bundle.""" + test = tmp_path / "test_plain.py" + test.write_text(body, encoding="utf-8") + return test + + +def test_run_reports_a_missing_bundle_as_could_not_execute(tmp_path: Path) -> None: + """The headline case: pytest alone reports this as a caught regression.""" + bundle, test = _export(tmp_path) + bundle.rename(tmp_path / "refund.moved") + + receipt = tmp_path / "receipt.json" + result = runner.invoke(app, ["run", str(test), "--receipt", str(receipt)]) + + assert result.exit_code == EXIT_COULD_NOT_EXECUTE + assert "COULD NOT EXECUTE" in _flat(result.stdout) + payload = json.loads(receipt.read_text(encoding="utf-8")) + assert payload["verdict"] == VERDICT_COULD_NOT_EXECUTE + assert payload["reason"] == "bundle_missing" + # pytest was never started, so no failure could be manufactured for it. + assert payload["exit_status"] is None + assert payload["counts"]["failures"] == 0 + + +def test_run_reports_a_passing_check(tmp_path: Path) -> None: + test = _plain_test(tmp_path, "def test_ok():\n assert 1 + 1 == 2\n") + receipt = tmp_path / "receipt.json" + + result = runner.invoke(app, ["run", str(test), "--receipt", str(receipt)]) + + assert result.exit_code == EXIT_PASSED + payload = json.loads(receipt.read_text(encoding="utf-8")) + assert payload["verdict"] == VERDICT_PASSED + assert payload["reason"] is None + assert payload["exit_status"] == 0 + assert payload["counts"] == {"tests": 1, "failures": 0, "errors": 0, "skipped": 0} + + +def test_run_reports_a_failing_check(tmp_path: Path) -> None: + test = _plain_test(tmp_path, "def test_bad():\n assert 1 + 1 == 3\n") + receipt = tmp_path / "receipt.json" + + result = runner.invoke(app, ["run", str(test), "--receipt", str(receipt)]) + + assert result.exit_code == EXIT_FAILED + payload = json.loads(receipt.read_text(encoding="utf-8")) + assert payload["verdict"] == VERDICT_FAILED + assert payload["counts"]["failures"] == 1 + + +def test_run_separates_a_collection_error_from_a_failure(tmp_path: Path) -> None: + """A module the check cannot import is not a statement about the agent.""" + test = _plain_test( + tmp_path, + "import absent_module_for_the_receipt_test # noqa: F401\n\n" + "def test_never_runs():\n assert True\n", + ) + receipt = tmp_path / "receipt.json" + + result = runner.invoke(app, ["run", str(test), "--receipt", str(receipt)]) + + assert result.exit_code == EXIT_COULD_NOT_EXECUTE + payload = json.loads(receipt.read_text(encoding="utf-8")) + assert payload["verdict"] == VERDICT_COULD_NOT_EXECUTE + assert payload["reason"] == "check_error" + assert payload["counts"]["errors"] >= 1 + + +def test_run_reports_an_empty_file_as_no_tests_collected(tmp_path: Path) -> None: + test = _plain_test(tmp_path, "# no tests here\n") + receipt = tmp_path / "receipt.json" + + result = runner.invoke(app, ["run", str(test), "--receipt", str(receipt)]) + + assert result.exit_code == EXIT_COULD_NOT_EXECUTE + payload = json.loads(receipt.read_text(encoding="utf-8")) + assert payload["reason"] == "no_tests_collected" + + +def test_receipt_names_the_test_and_bundle_it_rests_on(tmp_path: Path) -> None: + _, test = _export(tmp_path) + receipt = tmp_path / "nested" / "receipt.json" + + result = runner.invoke(app, ["run", str(test), "--receipt", str(receipt)]) + + payload = json.loads(receipt.read_text(encoding="utf-8")) + # a real bundle is named, and the run got far enough to have a verdict + assert payload["bundle"] is not None + assert payload["test_sha256"] + assert payload["test"].endswith("test_refund_regression.py") + assert payload["receipt_version"] == 1 + assert payload["stepfork_version"] + assert payload["created_at"].endswith("Z") + # The exported test cannot import pkg.mod in this environment, so its body + # raises. pytest reports an exception raised inside a test body as + # `failure`, so this verdict is FAILED -- the documented limit of the + # approach: the runner can pre-check the artifacts it can name (the + # bundle), not the imports the test body performs. The test asserts the + # limit rather than papering over it. + assert payload["verdict"] == VERDICT_FAILED + assert result.exit_code == EXIT_FAILED + + +def test_bundle_digest_changes_with_the_bundle_content(tmp_path: Path) -> None: + bundle = _record(tmp_path / "refund.sftrace", RECORDED) + first = bundle_digest(bundle) + + # rewrite one file inside the bundle and re-digest + target = sorted(path for path in bundle.rglob("*") if path.is_file())[0] + target.write_bytes(target.read_bytes() + b"\n") + + assert bundle_digest(bundle) != first + + +def test_run_prints_the_receipt_as_json(tmp_path: Path) -> None: + test = _plain_test(tmp_path, "def test_ok():\n assert True\n") + + result = runner.invoke(app, ["run", str(test), "--json"]) + + assert result.exit_code == EXIT_PASSED + payload = json.loads(result.stdout) + assert payload["verdict"] == VERDICT_PASSED + + +def test_run_accepts_an_explicit_bundle(tmp_path: Path) -> None: + bundle, test = _export(tmp_path) + receipt = tmp_path / "receipt.json" + + result = runner.invoke( + app, ["run", str(test), "--bundle", str(bundle), "--receipt", str(receipt)] + ) + + payload = json.loads(receipt.read_text(encoding="utf-8")) + assert payload["bundle"] == str(bundle) + assert payload["bundle_digest"] + # the bundle resolves and digests; whether the check then executes is up to + # the entrypoint it names, which this fixture does not ship + assert result.exit_code == EXIT_FAILED From 0a6b1474f4f76cdde661819769c5a5055d96cbbd Mon Sep 17 00:00:00 2001 From: how2how2how2-arch Date: Mon, 19 Oct 2026 01:10:04 +0800 Subject: [PATCH 5/5] docs(cli): document stepfork run and its exit codes --- docs/cli.md | 126 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 125 insertions(+), 1 deletion(-) diff --git a/docs/cli.md b/docs/cli.md index 086b010..c15a792 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -24,12 +24,15 @@ Commands: - [minimize](#minimize) - [diff](#diff) - [export](#export) +- [score](#score) +- [run](#run) Exit code conventions: - `0` success (or a replayed failure reproduced intentionally). - `1` the run behaved differently, or the operation could not complete safely. -- `2` invalid, unreadable, or unresolvable input. +- `2` invalid, unreadable, or unresolvable input, or (for `run`) a check that + could not execute. - `3` `validate --verify-integrity` on a legacy bundle without integrity metadata. @@ -520,6 +523,127 @@ Common errors: - `no expectation to score` -- the test asserts the recorded run without an `--expect-output`, so there is no claim for a mutant to separate from it. +## run + +Run an exported regression test and write the receipt yourself. + +```text +Usage: stepfork run [OPTIONS] TEST +``` + +`stepfork export` writes a pytest file; this command runs it and classifies the +outcome. The classification is the point: a check that **could not execute** -- +the bundle it names is not there, a fixture cannot be built, nothing was +collected -- is reported as `could_not_execute`, which is neither a pass nor a +caught regression. + +pytest's exit status does not carry that distinction. Measured on pytest 9.1.1 +against the file `stepfork export` writes: + +| Case | pytest exit | JUnit kind | +|---|---|---| +| the agent's behaviour changed | 1 | `failure` | +| the trace bundle is missing | 1 | `failure` | +| a dependency is absent | 1 | `failure` | +| a fixture cannot be built | 1 | `error` | +| collection failed | 2 | `error` | +| no tests were collected | 5 | (none) | + +The first two rows are identical, so a pipeline that keys on either one reports +a missing artifact as a caught regression. `run` resolves the bundle **before** +launching pytest, so that row becomes `could_not_execute` and no failure is +manufactured for it. + +The receipt is written by the runner after the child process exits -- never by +the test -- and it names the three inputs the verdict rests on: the raw exit +status, a digest of the bundle, and a hash of the test file. + +Arguments: + +| Argument | Description | +|---|---| +| `TEST` | The pytest file `stepfork export` wrote. Required. | + +Options: + +| Option | Description | +|---|---| +| `--bundle ` | Name the `.sftrace` bundle instead of reading `TRACE_PATH` from the test. | +| `--receipt ` | Write a JSON receipt of what the runner observed. | +| `--json` | Print the receipt to stdout. | +| `--pytest-arg ` | Extra argument for the pytest subprocess. Repeat per argument. | +| `--timeout-seconds ` | Stop the check after this long and report `could_not_execute`. | +| `--help` | Show command help and exit. | + +A passing run: + +```bash +stepfork run test_notify_regression.py --receipt receipts/notify.json +``` + +```text +Test test_notify_regression.py +Test sha256 129694b27992161b +Bundle notify.sftrace +Bundle digest 7f3eaeed6a9820ce +pytest exit 0 +Collected 1 test(s), 0 failure(s), 0 error(s), 0 skipped + +Verdict: PASSED +Receipt: receipts/notify.json +``` + +The same test with its bundle removed says what happened instead of reporting a +regression: + +```text +Verdict: COULD NOT EXECUTE +Reason: bundle_missing +the bundle this test names is not there: notify.sftrace; the check cannot run, +so this is not a regression +The check did not execute, so this run says nothing about the agent's behaviour. +``` + +The receipt: + +```json +{ + "receipt_version": 1, + "verdict": "could_not_execute", + "reason": "bundle_missing", + "exit_status": null, + "bundle": "notify.sftrace", + "bundle_digest": null, + "test": "test_notify_regression.py", + "test_sha256": "129694b2...", + "counts": {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}, + "command": [], + "duration_ms": 0, + "stepfork_version": "0.2.0", + "created_at": "2026-10-18T16:53:50Z" +} +``` + +Exit codes: + +| Code | Meaning | +|---|---| +| `0` | `passed`: the check ran and agreed. | +| `1` | `failed`: the check ran and disagreed -- a caught regression. | +| `2` | `could_not_execute`: the check never ran, or could not be built. | + +The `reason` field names which: `bundle_missing`, `bundle_unreadable`, +`check_error` (a fixture or collection error), +`no_tests_collected`, `pytest_unavailable`, `pytest_error`, `report_missing`. + +**What `run` does not catch.** The runner pre-checks the artifacts it can name +from the test's literals -- the bundle. It does not pre-check what the test body +*imports*: an entrypoint the local tree cannot resolve, or a dependency +imported inside the test, raises inside the test body, and pytest reports an +exception raised there as `failure`, exactly as it reports a changed return +value. Those still read as `failed`. Resolving `ENTRYPOINT` before the launch, +the way the bundle is resolved, is the follow-up that would move them too. + --- ## Error handling