From 267d3838b4bf9ac6006d589666d6dd981e2e3f24 Mon Sep 17 00:00:00 2001 From: Utsab Dahal Date: Sun, 18 Oct 2026 22:52:14 +0545 Subject: [PATCH] fix(cli,minimize): validate --mode upfront and reject invalid modes (exit 2) --- README.md | 20 ++-- docs/cli.md | 6 +- docs/community/early-adopter-guide.md | 4 +- docs/getting-started.md | 4 +- docs/index.md | 2 +- docs/minimize.md | 2 +- docs/pytest-export.md | 26 +++++ docs/releases/v0.2.0.md | 46 ++++++++ src/stepfork/cli/minimize.py | 9 +- src/stepfork/minimize/engine.py | 14 +++ tests/unit/test_minimize_mode_validation.py | 92 +++++++++++++++ tests/unit/test_replay_contract.py | 106 ++++++++++++++++++ .../unit/test_replay_contract_changed_args.py | 39 +++++++ .../unit/test_replay_contract_extra_calls.py | 47 ++++++++ 14 files changed, 399 insertions(+), 18 deletions(-) create mode 100644 docs/releases/v0.2.0.md create mode 100644 tests/unit/test_minimize_mode_validation.py create mode 100644 tests/unit/test_replay_contract.py create mode 100644 tests/unit/test_replay_contract_changed_args.py create mode 100644 tests/unit/test_replay_contract_extra_calls.py diff --git a/README.md b/README.md index 47541d7..c766d50 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ If you're writing evals, Stepfork complements them: evals measure model quality

- Install the alpha: pip install --pre stepfork + Install the current release: pip install stepfork

Product goal: @@ -127,11 +127,10 @@ pip install --pre stepfork uv pip install --pre stepfork ``` -The `--pre` flag is required while Stepfork is a pre-release. You can also -install from the tagged GitHub repository: +You can also install from source or a tagged release: ```bash -pip install "git+https://github.com/utsab345/stepfork.git@v0.1.0a6" +pip install "git+https://github.com/utsab345/stepfork.git@v0.2.0" ``` Requires Python 3.11, 3.12, or 3.13. This installs the `stepfork` CLI and the @@ -325,9 +324,12 @@ business outcome - it lives in `expected.json`, defined by a human. |---|---| | `stepfork validate [PATH] [--partial] [--verify-integrity]` | Check a `.sftrace` bundle structurally (and its integrity). | | `stepfork inspect [PATH] [--json] [--events] [--errors-only] [--step N]` | Review a bundle locally. | -| `stepfork replay [PATH] --entrypoint MODULE:FUNCTION [--mode M]` | Rerun the entrypoint with dependencies answered from the trace. | +| `stepfork replay [PATH] --entrypoint MODULE:FUNCTION [--mode M] [--allow-live-tool NAME] [--verbose]` | Rerun the entrypoint with dependencies answered from the trace. | +| `stepfork minimize [PATH] --entrypoint MODULE:FUNCTION [--expect-output FILE] [--mode M] [--output PATH] [--overwrite] [--timeout SECONDS] [--attempts N]` | Minimize a failing trace to the smallest reproducible subset. | +| `stepfork fork [PATH] [--at-step N \| --at-event ID] [--output PATH] [--overwrite]` | Fork a trace at a specific step (Python API for overrides). | +| `stepfork score [TEST] [--expect-output PATH] [--bundle PATH] [--json]` | Score an exported expectation against a mutant battery. | | `stepfork diff [BASELINE] [CANDIDATE] [--json]` | Compare the behavior of two runs. | -| `stepfork export [PATH] --pytest --entrypoint MODULE:FUNCTION [--expect-output JSON] [--expect-trajectory JSON] [--output FILE] [--overwrite]` | Generate an executable pytest regression test. | +| `stepfork export [PATH] --pytest --entrypoint MODULE:FUNCTION [--expect-output JSON] [--expect-trajectory JSON] [--output FILE] [--overwrite] [--mode M] [--allow-live-tool NAME]` | Generate an executable pytest regression test. | The full reference (arguments, examples, expected output, and common errors) is in [docs/cli.md](docs/cli.md). @@ -379,7 +381,7 @@ sources live in `docs/`. - [Release checklist](docs/release-checklist.md) - [PyPI publishing checklist](docs/pypi-publishing.md) - [Early adopter guide](docs/community/early-adopter-guide.md) -- [Latest release notes (v0.1.0a6)](docs/releases/v0.1.0a6.md) +- [Latest release notes (v0.2.0)](docs/releases/v0.2.0.md) - [Changelog](CHANGELOG.md) ## Security and Limitations @@ -387,8 +389,8 @@ sources live in `docs/`. Stepfork is experimental. The API, CLI, and `.sftrace` trace format may change before v1.0. -- **Alpha on PyPI.** `pip install --pre stepfork` installs the current - pre-release; the API and trace format may still change. +- **Beta status.** `pip install stepfork` installs the current release; the API + and trace format may still change before v1.0. - **Limited integrations.** Optional adapters cover the OpenAI Python SDK (synchronous and asynchronous, non-streaming `chat.completions.create`) and LangGraph (synchronous and asynchronous `BaseChatModel` generation and diff --git a/docs/cli.md b/docs/cli.md index 086b010..c2ff12f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,7 +1,7 @@ # CLI Reference The `stepfork` CLI manages `.sftrace` bundles. This page documents every -command as of v0.1.0a6. Run `stepfork --help` for an up-to-date summary. +command as of v0.2.0. Run `stepfork --help` for an up-to-date summary. ```text Usage: stepfork [OPTIONS] COMMAND [ARGS]... @@ -22,6 +22,8 @@ Commands: - [inspect](#inspect) - [replay](#replay) - [minimize](#minimize) +- [fork](#fork) +- [score](#score) - [diff](#diff) - [export](#export) @@ -261,7 +263,7 @@ Options: | Option | Description | |---|---| | `--expect-output FILE` | JSON file with expected output for comparison. | -| `--mode`, `-m` | Replay mode: `frozen`, `live`, or `hybrid`. Default: `frozen`. | +| `--mode`, `-m` | Replay mode: `frozen`, `live`, `forbidden`, `manual`, or `derived`. Default: `frozen`. | | `--output`, `-o` | Path for the minimized `.sftrace` bundle. Default: `-minimized.sftrace`. | | `--overwrite` | Overwrite the output bundle if it exists. Default: `false`. | | `--timeout SECONDS` | Timeout in seconds for each replay attempt. Default: `30.0`. | diff --git a/docs/community/early-adopter-guide.md b/docs/community/early-adopter-guide.md index 3cd8d70..8211758 100644 --- a/docs/community/early-adopter-guide.md +++ b/docs/community/early-adopter-guide.md @@ -32,14 +32,14 @@ Stepfork is on PyPI. Install the current alpha: ```bash python -m venv .venv source .venv/bin/activate -pip install --pre stepfork +pip install stepfork pip install pytest stepfork --version ``` The `--pre` flag is required while Stepfork is a pre-release. Alternatively, install the tagged repository build: -`pip install "git+https://github.com/utsab345/stepfork.git@v0.1.0a6"`. +`pip install "git+https://github.com/utsab345/stepfork.git@v0.2.0"`. Requires Python 3.11, 3.12, or 3.13. diff --git a/docs/getting-started.md b/docs/getting-started.md index cc74e37..2d8437f 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -9,14 +9,14 @@ This guide installs Stepfork and walks through the full workflow with a Install the current alpha from PyPI: ```bash -pip install --pre stepfork +pip install stepfork ``` The `--pre` flag is required while Stepfork is a pre-release. To install from the tagged GitHub repository instead: ```bash -pip install "git+https://github.com/utsab345/stepfork.git@v0.1.0a6" +pip install "git+https://github.com/utsab345/stepfork.git@v0.2.0" ``` Requires Python 3.11, 3.12, or 3.13. This installs the `stepfork` CLI and the diff --git a/docs/index.md b/docs/index.md index 8113ca9..3688c24 100644 --- a/docs/index.md +++ b/docs/index.md @@ -73,4 +73,4 @@ in this repository. Stepfork is experimental. See the [changelog](https://github.com/utsab345/stepfork/blob/main/CHANGELOG.md) and -the [v0.1.0a6 release notes](releases/v0.1.0a6.md) for what shipped. +the [v0.2.0 release notes](releases/v0.2.0.md) for what shipped. diff --git a/docs/minimize.md b/docs/minimize.md index b7fea85..6a5bf7e 100644 --- a/docs/minimize.md +++ b/docs/minimize.md @@ -85,7 +85,7 @@ stepfork minimize --entrypoint MODULE:FUNCTION [OPTIONS] | Option | Description | Default | |--------|-------------|---------| | `--expect-output FILE` | JSON file with expected output for comparison | None | -| `--mode`, `-m` | Replay mode: `frozen`, `live`, or `hybrid` | `frozen` | +| `--mode`, `-m` | Replay mode: `frozen`, `live`, `forbidden`, `manual`, or `derived` | `frozen` | | `--output`, `-o` | Path for the minimized `.sftrace` bundle | `-minimized.sftrace` | | `--overwrite` | Overwrite the output bundle if it exists | `false` | | `--timeout SECONDS` | Timeout for each replay attempt | `30.0` | diff --git a/docs/pytest-export.md b/docs/pytest-export.md index 29447e4..bbf4548 100644 --- a/docs/pytest-export.md +++ b/docs/pytest-export.md @@ -9,6 +9,32 @@ A trace records what the agent *did*, including the bug. If you generate a test that asserts the recorded output, it passes against the very code that produced the recording, which is not a regression test. +## Frozen-replay contract and limitations + +Stepfork's pytest export uses frozen replay: the generated test executes your +trusted entrypoint with instrumented tool and LLM calls answered from the recorded +trace. The contract is strict to preserve determinism and prevent silent +acceptance of unexpected behavior. + +**Supported when:** the fixed implementation makes the same instrumented calls in +the same order with the same inputs as recorded in the failure trace. The +regression test can then assert the corrected output against the trace. + +**Not supported when:** the fix legitimately changes later tool arguments, +invokes a tool not present in the recorded trace, or omits/moves recorded calls. +In these cases, frozen replay will raise a divergence (e.g., `ReplayMismatchError` +for changed inputs, or `ReplayExhaustedError`/unmatched calls for sequence +changes). The test generated from the failure trace **cannot pass** on the fixed +code because replay cannot satisfy the recorded sequence. + +When a fix changes the call sequence or arguments, record the *fixed* run and +export a test from that trace instead. The exported test will fail on the buggy +code (by replay divergence) and pass on the fixed code. `stepfork score` can help +assess whether an expectation is claim-pinned (rejects the recorded run) or +run-pinned (accepts it). + +--- + The regression test needs a separate **expected outcome**: the behavior you want after the bug is fixed. Pass it with `--expect-output`. diff --git a/docs/releases/v0.2.0.md b/docs/releases/v0.2.0.md new file mode 100644 index 0000000..46fc6bf --- /dev/null +++ b/docs/releases/v0.2.0.md @@ -0,0 +1,46 @@ +# Stepfork v0.2.0 - Behavioral Replay + +This current release adds trajectory assertions, async instrumented boundaries, and an +opt-in hybrid mode for selected live LLM calls. Existing `.sftrace` bundles +remain compatible; frozen replay remains the default. + +## Changes + +- Assert tool calls, absence, counts, JSON arguments, order, sequences, and a + maximum dependency-step count against the execution that actually replayed. + Generated pytest tests can include a reviewed JSON trajectory expectation. +- Record and frozen-replay async tools, LLM requests, and agent entrypoints. + Supported OpenAI and LangGraph adapters now include async paths. +- Re-execute selected LLMs in hybrid replay while recorded tools stay frozen. + The [offline model-decision example](https://github.com/utsab345/stepfork/tree/v0.2.0/examples/model_decision) + uses a deterministic fake provider and shows a failing regression before a + prompt fix and a passing one afterward. It does not establish real-model + quality improvement. +- Live tool execution now requires `allow_live_tools={"name"}` or + `--allow-live-tool name`. Previously, `mode="live"` ran every matching tool. + See [migration guidance](../replay.md#modes). + +Hybrid replay is **not deterministic**. Selected live models can incur network +calls, token costs, and latency. The tool allowlist controls instrumented tool +bodies by name; it does not sandbox arbitrary code or uninstrumented side +effects. Concurrent async calls still match one strict invocation sequence, +while task scheduling remains outside Stepfork's control. + +## Installation + +```bash +pip install stepfork +``` + +Requires Python 3.11, 3.12, or 3.13. + +## Verification + +- 444 tests passed on Python 3.11, 3.12, and 3.13. Python 3.12 combined + statement/branch coverage was 92%. +- Ruff, MyPy, strict documentation builds, six offline demos, source and wheel + builds, and an installed-wheel smoke of the new APIs passed locally. + +## Feedback + +https://github.com/utsab345/stepfork/issues diff --git a/src/stepfork/cli/minimize.py b/src/stepfork/cli/minimize.py index 541abf2..d03a106 100644 --- a/src/stepfork/cli/minimize.py +++ b/src/stepfork/cli/minimize.py @@ -49,7 +49,7 @@ def minimize_command( typer.Option( "--mode", "-m", - help="Replay mode: frozen (default), live, or hybrid", + help="Replay mode: frozen (default), live, forbidden, manual, or derived", ), ] = "frozen", output: Annotated[ @@ -126,6 +126,13 @@ def minimize_command( # Run minimization console.print("[yellow]Detecting failure signature...[/yellow]") + try: + from stepfork.trace.replay_policy import ReplayPolicy + + ReplayPolicy(mode) + except ValueError: + console.print(f"[red]Invalid replay mode {mode!r}.[/red]") + raise typer.Exit(2) from None engine = MinimizationEngine( trace, entrypoint, diff --git a/src/stepfork/minimize/engine.py b/src/stepfork/minimize/engine.py index 77c3155..bd4722e 100644 --- a/src/stepfork/minimize/engine.py +++ b/src/stepfork/minimize/engine.py @@ -88,6 +88,20 @@ def minimize(self) -> MinimizationResult: """ # Step 1: Detect the original failure signature try: + try: + from stepfork.trace.replay_policy import ReplayPolicy + + ReplayPolicy(self.mode) + except ValueError as exc: + return MinimizationResult( + success=False, + original_event_count=len(self.trace.events), + minimized_event_count=None, + replay_attempts=self.total_replay_attempts, + preserved_failure=False, + is_minimal=False, + error_message=f"Invalid replay mode {self.mode!r}: {exc}", + ) original_signature = detect_failure_signature( self.trace, self.entrypoint, diff --git a/tests/unit/test_minimize_mode_validation.py b/tests/unit/test_minimize_mode_validation.py new file mode 100644 index 0000000..fe3a18d --- /dev/null +++ b/tests/unit/test_minimize_mode_validation.py @@ -0,0 +1,92 @@ +"""Unit tests for minimize mode validation.""" + +from __future__ import annotations + +from pathlib import Path + +from typer.testing import CliRunner + +from stepfork import record, trace_tool +from stepfork.cli.main import app + +runner = CliRunner() + + +@trace_tool(name="simple_tool") +def simple_tool(x: int) -> int: + """Simple test tool.""" + return x * 2 + + +def simple_agent() -> dict[str, int]: + """Simple test agent.""" + result = simple_tool(5) + return {"value": result} + + +def test_minimize_rejects_hybrid_mode(tmp_path: Path) -> None: + """Minimize rejects unsupported 'hybrid' mode.""" + trace_path = tmp_path / "test.sftrace" + with record("cli-test-agent", output=str(trace_path)) as session: + result = simple_agent() + session.set_output(result) + + result = runner.invoke( + app, + [ + "minimize", + str(trace_path), + "--entrypoint", + "test_minimize_mode_validation:simple_agent", + "--mode", + "hybrid", + ], + ) + assert result.exit_code == 2 + assert "Invalid replay mode" in result.stdout + + +def test_minimize_rejects_bogus_mode(tmp_path: Path) -> None: + """Minimize rejects invalid mode values.""" + trace_path = tmp_path / "test.sftrace" + with record("cli-test-agent", output=str(trace_path)) as session: + result = simple_agent() + session.set_output(result) + + result = runner.invoke( + app, + [ + "minimize", + str(trace_path), + "--entrypoint", + "test_minimize_mode_validation:simple_agent", + "--mode", + "bogus", + ], + ) + assert result.exit_code == 2 + assert "Invalid replay mode" in result.stdout + + +def test_minimize_accepts_valid_modes(tmp_path: Path) -> None: + """Valid modes are accepted by minimize.""" + trace_path = tmp_path / "test.sftrace" + with record("cli-test-agent", output=str(trace_path)) as session: + result = simple_agent() + session.set_output(result) + + for mode in ("frozen", "live", "forbidden", "manual", "derived"): + result = runner.invoke( + app, + [ + "minimize", + str(trace_path), + "--entrypoint", + "test_minimize_mode_validation:simple_agent", + "--mode", + mode, + ], + ) + # Valid modes may still fail minimization if trace doesn't fail as expected, + # but should not be rejected with exit 2 for invalid mode + assert result.exit_code != 2, f"Mode {mode} was incorrectly rejected" diff --git a/tests/unit/test_replay_contract.py b/tests/unit/test_replay_contract.py new file mode 100644 index 0000000..fac555e --- /dev/null +++ b/tests/unit/test_replay_contract.py @@ -0,0 +1,106 @@ +"""Contract tests for frozen replay behavior. + +These tests document and verify the supported replay contract: +- Original trace with unchanged calls/args replays correctly +- Changed arguments cause ReplayMismatchError (input diverged) +- Additional tool calls cause ReplayExhaustedError (unexpected tool call) +- Missing recorded calls cause ReplayMismatchError (unmatched calls) + +Generated tests from failure traces that cannot execute on the fixed code +correctly fail during replay due to contract violations. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from stepfork import record, trace_tool +from stepfork.replay import ReplayMismatchError, ReplaySession + + +@trace_tool(name="step") +def step(value: str) -> dict[str, str]: + return {"value": value} + + +@trace_tool(name="compute") +def compute(x: int, y: int) -> dict[str, int]: + return {"sum": x + y, "diff": x - y} + + +def original_trace_agent() -> dict[str, str]: + return step("a") + + +def agent_changes_arg() -> dict[str, str]: + return step("b") # different arg from recorded "a" + + +def agent_extra_call() -> dict[str, str]: + step("a") + step("extra") # extra call not in original trace + return {"extra": True} + + +def agent_missing_calls() -> dict[str, object]: + return {"done": True} # skips the recorded step call + + +def _record_step_trace(destination: Path) -> None: + with record("step-agent", output=destination): + step("a") + # Set output after recording the step call + # Better: record with session output + pass + + +def test_original_trace_replays_successfully(tmp_path: Path) -> None: + trace_path = tmp_path / "trace.sftrace" + with record("step-agent", output=trace_path) as session: + session.set_output(step("a")) + + # Replay against original agent behavior + with ReplaySession.from_trace(trace_path, mode="frozen") as replay: + result = step("a") + replay.verify_complete() + assert result == {"value": "a"} + + +def test_changed_arguments_cause_replay_mismatch(tmp_path: Path) -> None: + trace_path = tmp_path / "trace.sftrace" + with record("step-agent", output=trace_path) as session: + session.set_output(step("a")) + + # Replay with different argument - should diverge + with ( + pytest.raises(ReplayMismatchError, match="input diverged"), + ReplaySession.from_trace(trace_path, mode="frozen"), + ): + step("b") + + +def test_extra_tool_call_cause_replay_exhausted(tmp_path: Path) -> None: + trace_path = tmp_path / "trace.sftrace" + with record("step-agent", output=trace_path) as session: + session.set_output(step("a")) + + # First call matches, second is extra (beyond recorded) + with ( + pytest.raises(ReplayMismatchError), + ReplaySession.from_trace(trace_path, mode="frozen"), + ): + step("a") + step("extra") # extra call + + +def test_missing_recorded_calls_cause_unmatched(tmp_path: Path) -> None: + trace_path = tmp_path / "trace.sftrace" + with record("step-agent", output=trace_path) as session: + session.set_output(step("a")) + + # Agent makes no calls but trace expects 1 + with pytest.raises(ReplayMismatchError, match=r"unmatched|ended before consuming"): + with ReplaySession.from_trace(trace_path, mode="frozen") as replay: + replay.verify_complete() diff --git a/tests/unit/test_replay_contract_changed_args.py b/tests/unit/test_replay_contract_changed_args.py new file mode 100644 index 0000000..283f5ea --- /dev/null +++ b/tests/unit/test_replay_contract_changed_args.py @@ -0,0 +1,39 @@ +"""Contract test for changed arguments case matching issue #17.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from stepfork import record, trace_tool +from stepfork.replay import ReplayMismatchError, ReplaySession + + +@trace_tool(name="issue_refund") +def issue_refund(customer_id: str, amount_cents: int) -> dict: + return {"customer_id": customer_id, "refunded_cents": amount_cents} + + +def run_buggy() -> dict: + issue_refund("c-1", 49) # records 49 + return {"refunded": 49} + + +def run_fixed() -> dict: + issue_refund("c-1", 4999) # changes to 4999 + return {"refunded": 4999} + + +def test_changed_args_causes_divergence(tmp_path: Path) -> None: + # Record from buggy version + trace_path = tmp_path / "failure.sftrace" + with record("triage-agent", output=trace_path) as session: + session.set_output(run_buggy()) + + # Replay against fixed version - arguments changed + with ( + pytest.raises(ReplayMismatchError, match="input diverged"), + ReplaySession.from_trace(trace_path, mode="frozen"), + ): + run_fixed() diff --git a/tests/unit/test_replay_contract_extra_calls.py b/tests/unit/test_replay_contract_extra_calls.py new file mode 100644 index 0000000..720a529 --- /dev/null +++ b/tests/unit/test_replay_contract_extra_calls.py @@ -0,0 +1,47 @@ +"""Contract test for extra calls case matching issue #17.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from stepfork import record, trace_tool +from stepfork.replay import ReplayMismatchError, ReplaySession + + +@trace_tool(name="get_advice") +def get_advice(ticket: str) -> str: + return "advice" + + +@trace_tool(name="escalate") +def escalate(priority: str) -> dict: + return {"escalated": True, "priority": priority} + + +def run_crash() -> dict: + # Only first call is made before crash + get_advice("T-1") + # escalate is never called (bug/crash) + return {"ok": True} + + +def run_fixed() -> dict: + get_advice("T-1") + escalate("high") # extra call beyond recorded trace + return {"ok": True} + + +def test_extra_calls_cause_divergence(tmp_path: Path) -> None: + # Record trace from crash version + trace_path = tmp_path / "crash.sftrace" + with record("crash-agent", output=trace_path) as session: + session.set_output(run_crash()) + + # Fixed version makes an extra call not in recording + with ( + pytest.raises(ReplayMismatchError), + ReplaySession.from_trace(trace_path, mode="frozen"), + ): + run_fixed()