[Feature]: Score the exported expectation with a claim-drawn mutant, and make "could not replay" its own verdict #14

"""r188 fixture for the Stepfork issue: two measurements, re-run at send time.

PART A  A mutant battery drawn from the code cannot tell an expectation that
        pins the claim from one that pins the observed run; one mutant drawn
        from the claim separates them.
PART B  Who writes the verdict, and whether "could not replay" is
        distinguishable from "the check ran and failed".
Stdlib + pytest. Part B shells out to real pytest.
"""
import json
import os
import subprocess
import sys
import tempfile

# ---------------------------------------------------------------- PART A
PIN = 2024        # the claim: the artefact's year is pinned by the requirement
OBSERVED = 2026   # what a re-run of the changed code produced

CORRECT = lambda: PIN  # noqa: E731  the implementation the claim is true of
MUTANTS = [
    ("real_year  (CLAIM-drawn: pin dropped, plausible year)", lambda: OBSERVED),
    ("const_2020 (code-drawn)", lambda: 2020),
    ("as_string  (code-drawn)", lambda: "2024"),
    ("off_by_one (code-drawn)", lambda: PIN + 1),
]
TESTS = [
    ("expectation = the claim  (== 2024)", lambda v: v == PIN),
    ("expectation = the run    (== 2026)", lambda v: v == OBSERVED),
]


def part_a():
    print("PART A - does the expectation still discriminate?")
    print("  %-54s %-22s %s" % ("mutant", TESTS[0][0], TESTS[1][0]))
    kills = {t[0]: 0 for t in TESTS}
    code_kills = {t[0]: 0 for t in TESTS}
    for mname, impl in MUTANTS:
        row = []
        claim_drawn = "CLAIM-drawn" in mname
        for tname, test in TESTS:
            k = not test(impl())
            row.append("killed " if k else "survive")
            if k:
                kills[tname] += 1
                if not claim_drawn:
                    code_kills[tname] += 1
        print("  %-54s %-22s %s" % (mname, row[0], row[1]))
    print()
    print("  the implementation the claim is true of:")
    for tname, test in TESTS:
        print("    %-36s %s" % (tname, "pass" if test(CORRECT()) else "FAIL"))
    print()
    n_code = len(MUTANTS) - 1
    print("  battery drawn from the CODE (%d mutants):            claim-test %d/%d  run-test %d/%d"
          % (n_code, code_kills[TESTS[0][0]], n_code, code_kills[TESTS[1][0]], n_code))
    print("  same battery + 1 mutant drawn from the CLAIM (%d mutants): claim-test %d/%d  run-test %d/%d"
          % (n_code + 1, kills[TESTS[0][0]], n_code + 1, kills[TESTS[1][0]], n_code + 1))
    print()

# ---------------------------------------------------------------- PART B
APP_OK = """
def run_agent():
    return {"destination": "Lisbon", "selected_flight": "B"}
"""

# (a) the generated regression test, roughly as Stepfork exports it: the bundle
#     is resolved when the test runs, and an inability to replay raises inside
#     the test function.
TEST_INFUNCTION = """\
import pathlib

TRACE_PATH = pathlib.Path(__file__).parent / "fixture.sftrace"
{receipt}


def test_stepfork_regression_failure():
    if not TRACE_PATH.exists():
        raise AssertionError("could not replay: bundle missing at %s" % TRACE_PATH)
    from app import run_agent
    got = run_agent()
    assert got["selected_flight"] == "A", "expected A, got %s" % got["selected_flight"]
"""

# (b) the same test with the bundle resolved at import time, so "cannot replay"
#     is a collection error rather than a failed test.
TEST_ATIMPORT = """\
import pathlib

TRACE_PATH = pathlib.Path(__file__).parent / "fixture.sftrace"
if not TRACE_PATH.exists():
    raise RuntimeError("could not replay: bundle missing at %s" % TRACE_PATH)
{receipt}


def test_stepfork_regression_failure():
    from app import run_agent
    got = run_agent()
    assert got["selected_flight"] == "A", "expected A, got %s" % got["selected_flight"]
"""

RUNNER = """\
import json, subprocess, sys, pathlib
tree = pathlib.Path(sys.argv[1])
p = subprocess.run([sys.executable, "-m", "pytest", "-q", "test_regression.py"],
                   cwd=tree, capture_output=True, text=True)
last = [l for l in p.stdout.strip().splitlines() if l.strip()][-1:] or [""]
print("      exit_status=%d   pytest says: %s" % (p.returncode, last[0][:88]))
print("      runner receipt: %s" % json.dumps(
    {"exit_status": p.returncode, "verdict": "PASS" if p.returncode == 0 else "FAIL"}))
"""

SELF_RECEIPT = ('pathlib.Path(__file__).parent.joinpath("receipt.json").write_text('
                "'{\\\"verdict\\\": \\\"PASS\\\"}')")


def arm(label, template, bundle_present, self_written_receipt):
    d = tempfile.mkdtemp(prefix="r188_")
    tree = os.path.join(d, "tree")
    os.makedirs(tree)
    with open(os.path.join(tree, "app.py"), "w") as fh:
        fh.write(APP_OK)
    if bundle_present:
        with open(os.path.join(tree, "fixture.sftrace"), "w") as fh:
            fh.write("{}")
    with open(os.path.join(tree, "test_regression.py"), "w") as fh:
        fh.write(template.format(receipt=SELF_RECEIPT if self_written_receipt else ""))
    runner = os.path.join(d, "run_regression.py")
    with open(runner, "w") as fh:
        fh.write(RUNNER)
    print("  " + label)
    subprocess.run([sys.executable, runner, tree], check=False)
    inside = os.path.join(tree, "receipt.json")
    if os.path.exists(inside):
        print("      receipt written INSIDE the tree under test: %s" % open(inside).read())


def part_b():
    print("PART B - who writes the verdict, and can 'could not replay' be told apart")
    arm("arm 1: bundle present, the code is wrong (the honest case)",
        TEST_INFUNCTION, True, False)
    arm("arm 2: same failure, but the generated test writes its own receipt",
        TEST_INFUNCTION, True, True)
    arm("arm 3: the bundle is missing; the test raises when it tries to replay",
        TEST_INFUNCTION, False, False)
    arm("arm 4: the bundle is missing; the test resolves it at import",
        TEST_ATIMPORT, False, False)
    print()


if __name__ == "__main__":
    part_a()
    part_b()