diff --git a/.github/workflows/evidence.yml b/.github/workflows/evidence.yml index b702219..1f68ae9 100644 --- a/.github/workflows/evidence.yml +++ b/.github/workflows/evidence.yml @@ -3,6 +3,16 @@ on: [push, pull_request] permissions: contents: read jobs: + dispatch: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 + with: + python-version: '3.14' + - run: python -m pip install -r examples/dispatch/requirements.txt + - run: python examples/dispatch/verify.py --check pyee-native: runs-on: ubuntu-24.04 timeout-minutes: 10 diff --git a/.gitignore b/.gitignore index 4c166b8..dd0ef98 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ __pycache__/ *.pyc .venv/ .venv-pyee/ +.venv-dispatch/ **/target/ /.artifacts/ **/node_modules/ diff --git a/CHANGELOG.md b/CHANGELOG.md index b699907..799bdef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +- Add a focused python-dispatch example with native tests, optional expected/observed + Mermaid diagrams and tables that separate property writes, new callbacks, cumulative + log and the property value. Demonstrate the first mismatch with a wrong expectation. +- Explain the existing rat-salsa case through five real API calls, adapter costs and + UI limits; link the existing pyee native-test PR without duplicating it. +- Runner behavior, protocol, CLI options and runtime dependencies are unchanged. + ## 0.1.0a7 / v0.1.0-alpha.7 Published on 27 October 2026 (Europe/Kyiv) to [PyPI](https://pypi.org/project/interaction-check/0.1.0a7/) diff --git a/MANIFEST.in b/MANIFEST.in index 4ac8e47..3be0ee5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,3 +5,5 @@ include docs/NATIVE_TESTS.md docs/RELEASING.md recursive-include examples/quickstart *.py *.json *.md include docs/CHOOSING.md docs/REPORTING.md include tools/render_quickstart.py docs/evidence/quickstart.json docs/assets/quickstart.svg +recursive-include examples/dispatch *.py *.json *.md *.txt +include docs/REAL_API_WALKTHROUGH.md docs/evidence/dispatch.json diff --git a/README.md b/README.md index e2554d9..b9e11e6 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,14 @@ checked prefix, which must be rerun; it does not automatically minimize a failur [Choosing a tool](docs/CHOOSING.md) connects these decisions to maintainer feedback. [A short useful bug report](docs/REPORTING.md) shows how to share one actionable failure. +## Read a real API trace + +Current source includes **[property writes and callbacks](examples/dispatch/README.md)**, +[expected/observed Mermaid diagrams](examples/dispatch/trace.md), a +[deliberate first mismatch](examples/dispatch/mismatch.md), and ordinary native tests. +For terminal widgets, see [five rat-salsa calls and their limits](docs/REAL_API_WALKTHROUGH.md). +These examples are newer than the published package; the runner protocol is unchanged. + ## Current prerelease **[0.1.0a7 on PyPI](https://pypi.org/project/interaction-check/0.1.0a7/) · published 27 October 2026 (Europe/Kyiv) · Python 3.10+** diff --git a/docs/CHOOSING.md b/docs/CHOOSING.md index 9d5161e..9945f7b 100644 --- a/docs/CHOOSING.md +++ b/docs/CHOOSING.md @@ -42,6 +42,11 @@ The two projects are independent: ## What the feedback changed +For a real-library walkthrough and expected/observed diagrams, see +[python-dispatch](../examples/dispatch/README.md). For the precise scope of the terminal +widget proposal, see [rat-salsa](REAL_API_WALKTHROUGH.md). The pyee native scenario is +already in [PR #228](https://github.com/jfhbrook/pyee/pull/228); no duplicate is needed. + | Question from maintainers | Concrete response in this repository | | --- | --- | | [Redux: what problem and upstream change?](https://github.com/reduxjs/redux/discussions/4962#discussioncomment-19203020) | Use the decision table above. Current core-store examples justify no change to Redux's CI. | diff --git a/docs/REAL_API_WALKTHROUGH.md b/docs/REAL_API_WALKTHROUGH.md new file mode 100644 index 0000000..7be7dc7 --- /dev/null +++ b/docs/REAL_API_WALKTHROUGH.md @@ -0,0 +1,43 @@ +# What an adapter does: rat-salsa + +The [rat-salsa question](https://github.com/thscharler/rat-salsa/issues/36#issuecomment-6461513249) +was about what we proposed: an optional external reproduction maintained by us. +It does not justify an upstream dependency or CI change. + +The existing [five-step case](https://github.com/PerchAnton/interaction-check/blob/055dfc24a4eda7478719dcbbb1f0ac123176aec9/scenarios/rat-salsa/03-empty-noop-redo.json) +calls a real `rat_text::text_input::TextInputState` through this +[Rust adapter](https://github.com/PerchAnton/interaction-check/blob/055dfc24a4eda7478719dcbbb1f0ac123176aec9/adapters/rat-salsa/src/main.rs): + +| JSON event | Actual API | Expected `value` | +| --- | --- | --- | +| `{"op":"new","text":""}` | `TextInputState::new()`, `set_text("")` | `""` | +| `{"op":"text","text":"x"}` | `insert_str("x")` | `"x"` | +| `{"op":"key","key":"undo"}` | `undo()` | `""` | +| `{"op":"key","key":"backspace"}` | `delete_prev_char()` | `""` | +| `{"op":"key","key":"redo"}` | `redo()` | `"x"` | + +The adapter returns `buffer.text()` after each action, alongside cursor/selection. +A hypothetical failure on the last step would report `state.value: expected 'x', got ''`. +This illustrates the report shape, **not a newly reproduced rat-salsa bug**. The adapter +and lockfile are on the linked historical evaluation revision; PyPI does not install them. + +For a current-source example that you can run and see fail deliberately, use +[JSON → python-dispatch → callbacks](../examples/dispatch/README.md). Its verifier +checks the first mismatch and produces expected/observed diagrams without a Rust toolchain. + +## Why not a normal Rust test? + +A native test can call those same five methods and assert `buffer.text() == "x"`. +That is preferred for a single regression. Replay adds Python, a JSONL adapter, +scenario maintenance and startup/serialization cost. It helps only if sharing a case +**within that adapter's contract** makes reproduction or revision comparison easier. + +The methods do not exercise terminal rendering, focus routing, OS keyboard events, +clipboard or an application's integration. They establish neither additional coverage +nor faster tests. All actions may already have run before the first mismatch is reported; +exporting a checked prefix does not undo effects. + +For pyee, the proposed behavior is already in the existing +[native-test PR #228](https://github.com/jfhbrook/pyee/pull/228). A new runner dependency +and a duplicate PR are unnecessary. Acceptance and adoption remain separate from +passing external examples. diff --git a/docs/evidence/dispatch.json b/docs/evidence/dispatch.json new file mode 100644 index 0000000..0ff85d4 --- /dev/null +++ b/docs/evidence/dispatch.json @@ -0,0 +1,207 @@ +{ + "package": "python-dispatch==0.2.3", + "native_tests": 3, + "synthetic_wrong_expectation": true, + "source_sha256": { + "examples/dispatch/adapter.py": "f986bea18171e9d875b2e6ed9c706a3f9a425261abb3c1bbd1a5ed9b3e0b41dd", + "examples/dispatch/native_tests.py": "50c0b900ead484bbbc7171e15385339c3eb5c14a5e267c3fb15e6e743b60db79", + "examples/dispatch/scenario.json": "84c731627683dd79093f97c6a3244c0ac4d539c97bcdf5d638b97e10155c2f07", + "examples/dispatch/render.py": "c3897660a53ee8a2505318032db25521951bcabe1201910439f40e1da5347f15", + "examples/dispatch/verify.py": "8a138d1ce86697f8a72a58db166affce1c4b5dbcfd03a70a4540b202a93bb125", + "examples/dispatch/requirements.txt": "485695df5b9bcd20665e0a74d7619dad44c469004de763390c5752ff249bd91c" + }, + "results": [ + { + "name": "Property writes are not callback counts", + "passed": true, + "steps": 5, + "stderr": "", + "trace": [ + { + "step": 1, + "event": { + "op": "write", + "values": [ + 1, + 1 + ], + "locked": false + }, + "expect": { + "count": 1, + "log": [ + 1 + ] + }, + "actual": { + "count": 1, + "log": [ + 1 + ] + }, + "passed": true + }, + { + "step": 2, + "event": { + "op": "write", + "values": [ + 2, + 3, + 4 + ], + "locked": true + }, + "expect": { + "count": 4, + "log": [ + 1, + 4 + ] + }, + "actual": { + "count": 4, + "log": [ + 1, + 4 + ] + }, + "passed": true + }, + { + "step": 3, + "event": { + "op": "write", + "values": [ + 4 + ], + "locked": false + }, + "expect": { + "count": 4, + "log": [ + 1, + 4 + ] + }, + "actual": { + "count": 4, + "log": [ + 1, + 4 + ] + }, + "passed": true + }, + { + "step": 4, + "event": { + "op": "unbind" + }, + "expect": { + "count": 4, + "log": [ + 1, + 4 + ] + }, + "actual": { + "count": 4, + "log": [ + 1, + 4 + ] + }, + "passed": true + }, + { + "step": 5, + "event": { + "op": "write", + "values": [ + 5 + ], + "locked": false + }, + "expect": { + "count": 5, + "log": [ + 1, + 4 + ] + }, + "actual": { + "count": 5, + "log": [ + 1, + 4 + ] + }, + "passed": true + } + ] + }, + { + "passed": false, + "error": "Step 2 ({'op': 'write', 'values': [2, 3, 4], 'locked': True}): state.log: expected [1, 2, 3, 4], got [1, 4]", + "trace": [ + { + "step": 1, + "event": { + "op": "write", + "values": [ + 1, + 1 + ], + "locked": false + }, + "expect": { + "count": 1, + "log": [ + 1 + ] + }, + "actual": { + "count": 1, + "log": [ + 1 + ] + }, + "passed": true + }, + { + "step": 2, + "event": { + "op": "write", + "values": [ + 2, + 3, + 4 + ], + "locked": true + }, + "expect": { + "count": 4, + "log": [ + 1, + 2, + 3, + 4 + ] + }, + "actual": { + "count": 4, + "log": [ + 1, + 4 + ] + }, + "passed": false, + "differences": [ + "state.log: expected [1, 2, 3, 4], got [1, 4]" + ] + } + ] + } + ] +} diff --git a/examples/dispatch/README.md b/examples/dispatch/README.md new file mode 100644 index 0000000..678b8ce --- /dev/null +++ b/examples/dispatch/README.md @@ -0,0 +1,74 @@ +# From JSON to real property callbacks + +This example calls **python-dispatch 0.2.3**, not a simulation. It is external +evaluation code maintained here, not an upstream integration or a discovered bug. +From current source (these files are newer than the alpha.7 tag), in a virtual environment: + +```sh +python -m pip install -r examples/dispatch/requirements.txt +python examples/dispatch/verify.py --check +``` + +The command runs 3 ordinary tests, 5 replay steps, and a deliberately incorrect +expectation that must fail at step 2. The extra package is for this example only; +the runner still has no runtime dependencies. + +| Term | Meaning | +| --- | --- | +| Write | Assignment to `model.count`; an equal assignment may emit nothing | +| Callback | One invocation of the retained receiver | +| New callbacks | The suffix added to `log` during this step | +| `log` | All callback values since the adapter started, in order | +| `count` | Current **property value**, not the number of callbacks | + +The [complete trace](trace.md) has separate expected/observed tables and Mermaid +diagrams. At step 2, three locked writes `[2, 3, 4]` produce one new callback `[4]`; +the cumulative log is `[1, 4]`. After unbinding, writing `5` changes `count` but leaves +the log unchanged. See the library's [property API](https://python-dispatch.readthedocs.io/en/latest/properties.html). + +## JSON → adapter → actual API → comparison + +```json +{"event": {"op": "write", "values": [2, 3, 4], "locked": true}, + "expect": {"count": 4, "log": [1, 4]}} +``` + +1. The runner sends only `event`, a protocol number and a step ID to [adapter.py](adapter.py). +2. The adapter opens `model.emission_lock("count")` and assigns each value to a real `Property`. +3. The receiver appends observed notifications. The adapter returns `count` and `log`. +4. The runner compares that snapshot with `expect`, which the adapter never receives. + +The verifier separately changes **only the expected log** at step 2 to `[1, 2, 3, 4]`. +The real library returns `[1, 4]`, and the runner reports the first mismatch. +[View both sides](mismatch.md). This is a mistaken expectation, not an upstream defect. +All five actions may execute before the two-step checked prefix is reported. A diagram +summarizes steps; it does not infer callback timing within a step. + +## Optional diagrams for another run + +```sh +python interaction_check.py --trace --report dispatch-report.json examples/dispatch/scenario.json -- python examples/dispatch/adapter.py +python examples/dispatch/render.py dispatch-report.json > dispatch-trace.md +``` + +The renderer lives in this source example, not the installed runner. It requires +complete integer `count`/append-only `log` snapshots and rejects incompatible or partial +states instead of inventing events. Compact reports must be rerun with `--trace`. +Only checked steps are drawn; Markdown tables work without Mermaid. + +Run the verifier without `--check` to regenerate [evidence](../../docs/evidence/dispatch.json) +and diagrams. CI reruns the real library and checks source hashes. These are behavior +checks, not coverage or speed measurements. + +## A native test may be enough + +[native_tests.py](native_tests.py) uses `Dispatcher`, `Property` and `unittest` directly: + +```sh +python -m unittest discover -s examples/dispatch -p native_tests.py -v +``` + +Use that route for one regression. Replay earns its extra files only when sharing +the actions and observations is useful. This example does not cover async delivery, +multiple-listener ordering, rendering, focus, clipboard or application side effects. +[The same explanation for rat-salsa](../../docs/REAL_API_WALKTHROUGH.md). diff --git a/examples/dispatch/adapter.py b/examples/dispatch/adapter.py new file mode 100644 index 0000000..126a1ef --- /dev/null +++ b/examples/dispatch/adapter.py @@ -0,0 +1,51 @@ +"""A deliberately narrow JSONL adapter for python-dispatch's real Property API.""" +from contextlib import nullcontext +import json +import sys + +from pydispatch import Dispatcher, Property + + +class Counter(Dispatcher): + count = Property(0) + + +class Probe: + def __init__(self): + self.model = Counter() + self.log = [] + self.model.bind(count=self.receive) + + def receive(self, instance, value, **kwargs): + self.log.append(value) + + def step(self, event): + if event.get("op") == "write" and set(event) == {"op", "values", "locked"}: + if (type(event["locked"]) is not bool or not isinstance(event["values"], list) + or not 1 <= len(event["values"]) <= 100 + or any(type(v) is not int or abs(v) > 1_000_000 for v in event["values"])): + raise ValueError("expected 1..100 bounded integers and a boolean lock") + with self.model.emission_lock("count") if event["locked"] else nullcontext(): + for value in event["values"]: + self.model.count = value + elif event == {"op": "unbind"}: + self.model.unbind(self.receive) + else: + raise ValueError("unsupported event") + return {"count": self.model.count, "log": list(self.log)} + + +def main(): + probe = Probe() + for index, line in enumerate(sys.stdin, 1): + request = json.loads(line) + if (set(request) != {"protocol", "id", "event"} + or type(request["protocol"]) is not int or request["protocol"] != 1 + or type(request["id"]) is not int or request["id"] != index): + raise ValueError("invalid protocol envelope") + state = probe.step(request["event"]) + print(json.dumps({"protocol": 1, "id": index, "state": state}), flush=True) + + +if __name__ == "__main__": + main() diff --git a/examples/dispatch/mismatch.md b/examples/dispatch/mismatch.md new file mode 100644 index 0000000..99c51ea --- /dev/null +++ b/examples/dispatch/mismatch.md @@ -0,0 +1,50 @@ +# Property notification trace + +Only checked steps are shown; the adapter may have executed the full batch. +Writes are requested assignments. Callbacks are the new suffix of the cumulative log. +`count` is the property value, **not** the number of callbacks. +The diagrams summarize each step; they do not assert intra-step timing. + +## Expected + +| Step | Requested action | New callbacks | Cumulative log | Property count | Check | +| --- | --- | --- | --- | --- | --- | +| 1 | writes [1, 1]; lock off | [1] | [1] | 1 | pass | +| 2 | writes [2, 3, 4]; lock on | [2, 3, 4] | [1, 2, 3, 4] | 4 | mismatch | + +```mermaid +sequenceDiagram + participant A as Adapter + participant M as Property count + participant L as Callback log + Note over A,L: Step 1 - pass + A->>M: writes [1, 1], lock off + Note over A,L: Callbacks during step = [1] + Note over A,L: Property count = 1, cumulative log = [1] + Note over A,L: Step 2 - MISMATCH + A->>M: writes [2, 3, 4], lock on + Note over A,L: Callbacks during step = [2, 3, 4] + Note over A,L: Property count = 4, cumulative log = [1, 2, 3, 4] +``` + +## Observed + +| Step | Requested action | New callbacks | Cumulative log | Property count | Check | +| --- | --- | --- | --- | --- | --- | +| 1 | writes [1, 1]; lock off | [1] | [1] | 1 | pass | +| 2 | writes [2, 3, 4]; lock on | [4] | [1, 4] | 4 | mismatch | + +```mermaid +sequenceDiagram + participant A as Adapter + participant M as Property count + participant L as Callback log + Note over A,L: Step 1 - pass + A->>M: writes [1, 1], lock off + Note over A,L: Callbacks during step = [1] + Note over A,L: Property count = 1, cumulative log = [1] + Note over A,L: Step 2 - MISMATCH + A->>M: writes [2, 3, 4], lock on + Note over A,L: Callbacks during step = [4] + Note over A,L: Property count = 4, cumulative log = [1, 4] +``` diff --git a/examples/dispatch/native_tests.py b/examples/dispatch/native_tests.py new file mode 100644 index 0000000..3987933 --- /dev/null +++ b/examples/dispatch/native_tests.py @@ -0,0 +1,35 @@ +"""Equivalent behavior checks without interaction-check or an adapter.""" +import unittest +from pydispatch import Dispatcher, Property + + +class Model(Dispatcher): + count = Property(0) + + +class NativeTests(unittest.TestCase): + def setUp(self): + self.model = Model() + self.log = [] + self.model.bind(count=self.receive) + + def receive(self, instance, value, **kwargs): + self.log.append(value) + + def test_equal_write(self): + self.model.count = 1 + self.model.count = 1 + self.assertEqual(self.log, [1]) + + def test_lock_coalesces(self): + with self.model.emission_lock("count"): + for value in [2, 3, 4]: + self.model.count = value + self.assertEqual(self.model.count, 4) + self.assertEqual(self.log, [4]) + + def test_unbind_keeps_writes(self): + self.model.unbind(self.receive) + self.model.count = 5 + self.assertEqual(self.model.count, 5) + self.assertEqual(self.log, []) diff --git a/examples/dispatch/render.py b/examples/dispatch/render.py new file mode 100644 index 0000000..da020b2 --- /dev/null +++ b/examples/dispatch/render.py @@ -0,0 +1,85 @@ +"""Optional, adapter-specific Markdown/Mermaid view of --trace JSON. + +Not a generic event visualizer: only complete integer count/log snapshots from +this fresh Probe are supported. Intra-step callback timing is not recorded. +""" +import argparse +import json +from pathlib import Path + + +def integers(value): + return (isinstance(value, list) and len(value) <= 1000 + and all(type(v) is int and abs(v) <= 1_000_000 for v in value)) + + +def rows(result, side): + trace = result.get("trace") + if not isinstance(trace, list) or not 1 <= len(trace) <= 100: + raise ValueError("need 1..100 checked steps from --trace") + previous = [] # This adapter always starts with an empty callback log. + output = [] + for index, entry in enumerate(trace, 1): + state, event = entry.get(side, {}), entry.get("event", {}) + log, count = state.get("log"), state.get("count") + if (entry.get("step") != index or type(entry.get("passed")) is not bool + or type(count) is not int or abs(count) > 1_000_000 or not integers(log) + or log[:len(previous)] != previous): + raise ValueError("need consecutive steps and complete append-only count/log snapshots") + if event == {"op": "unbind"}: + action = "unbind receiver" + elif (set(event) == {"op", "values", "locked"} and event["op"] == "write" + and integers(event["values"]) and 1 <= len(event["values"]) <= 100 + and type(event["locked"]) is bool): + action = f"writes {event['values']}; lock {'on' if event['locked'] else 'off'}" + else: + raise ValueError("unsupported adapter event") + output.append((index, action, log[len(previous):], log, count, entry["passed"])) + previous = log + return output + + +def render(result): + # All labels below are fixed strings or validated integers. Report names, + # paths, stderr and arbitrary JSON strings never enter Mermaid source. + lines = ["# Property notification trace", "", + "Only checked steps are shown; the adapter may have executed the full batch.", + "Writes are requested assignments. Callbacks are the new suffix of the cumulative log.", + "`count` is the property value, **not** the number of callbacks.", + "The diagrams summarize each step; they do not assert intra-step timing.", ""] + for side, title in [("expect", "Expected"), ("actual", "Observed")]: + data = rows(result, side) + lines += [f"## {title}", "", "| Step | Requested action | New callbacks | Cumulative log | Property count | Check |", + "| --- | --- | --- | --- | --- | --- |"] + for i, action, delta, log, count, passed in data: + lines.append(f"| {i} | {action} | {delta} | {log} | {count} | {'pass' if passed else 'mismatch'} |") + lines += ["", "```mermaid", "sequenceDiagram", " participant A as Adapter", + " participant M as Property count", " participant L as Callback log"] + for i, action, delta, log, count, passed in data: + # Semicolons separate Mermaid statements; replace our label delimiter. + lines += [f" Note over A,L: Step {i} - {'pass' if passed else 'MISMATCH'}", + f" A->>M: {action.replace(';', ',')}", + f" Note over A,L: Callbacks during step = {delta}", + f" Note over A,L: Property count = {count}, cumulative log = {log}"] + lines += ["```", ""] + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("report", type=Path) + parser.add_argument("--result", type=int, default=0, help="zero-based result index") + args = parser.parse_args() + if args.report.stat().st_size > 1_048_576: + parser.error("report exceeds 1 MiB") + try: + report = json.loads(args.report.read_text(encoding="utf-8")) + if args.result < 0: + raise ValueError("negative result index") + print(render(report["results"][args.result])) + except (ValueError, KeyError, IndexError, TypeError, AttributeError) as error: + parser.error(str(error)) + + +if __name__ == "__main__": + main() diff --git a/examples/dispatch/requirements.txt b/examples/dispatch/requirements.txt new file mode 100644 index 0000000..9cf849b --- /dev/null +++ b/examples/dispatch/requirements.txt @@ -0,0 +1 @@ +python-dispatch==0.2.3 diff --git a/examples/dispatch/scenario.json b/examples/dispatch/scenario.json new file mode 100644 index 0000000..271ad31 --- /dev/null +++ b/examples/dispatch/scenario.json @@ -0,0 +1,11 @@ +{ + "schema": 1, + "name": "Property writes are not callback counts", + "steps": [ + {"event": {"op": "write", "values": [1, 1], "locked": false}, "expect": {"count": 1, "log": [1]}}, + {"event": {"op": "write", "values": [2, 3, 4], "locked": true}, "expect": {"count": 4, "log": [1, 4]}}, + {"event": {"op": "write", "values": [4], "locked": false}, "expect": {"count": 4, "log": [1, 4]}}, + {"event": {"op": "unbind"}, "expect": {"count": 4, "log": [1, 4]}}, + {"event": {"op": "write", "values": [5], "locked": false}, "expect": {"count": 5, "log": [1, 4]}} + ] +} diff --git a/examples/dispatch/trace.md b/examples/dispatch/trace.md new file mode 100644 index 0000000..1652832 --- /dev/null +++ b/examples/dispatch/trace.md @@ -0,0 +1,80 @@ +# Property notification trace + +Only checked steps are shown; the adapter may have executed the full batch. +Writes are requested assignments. Callbacks are the new suffix of the cumulative log. +`count` is the property value, **not** the number of callbacks. +The diagrams summarize each step; they do not assert intra-step timing. + +## Expected + +| Step | Requested action | New callbacks | Cumulative log | Property count | Check | +| --- | --- | --- | --- | --- | --- | +| 1 | writes [1, 1]; lock off | [1] | [1] | 1 | pass | +| 2 | writes [2, 3, 4]; lock on | [4] | [1, 4] | 4 | pass | +| 3 | writes [4]; lock off | [] | [1, 4] | 4 | pass | +| 4 | unbind receiver | [] | [1, 4] | 4 | pass | +| 5 | writes [5]; lock off | [] | [1, 4] | 5 | pass | + +```mermaid +sequenceDiagram + participant A as Adapter + participant M as Property count + participant L as Callback log + Note over A,L: Step 1 - pass + A->>M: writes [1, 1], lock off + Note over A,L: Callbacks during step = [1] + Note over A,L: Property count = 1, cumulative log = [1] + Note over A,L: Step 2 - pass + A->>M: writes [2, 3, 4], lock on + Note over A,L: Callbacks during step = [4] + Note over A,L: Property count = 4, cumulative log = [1, 4] + Note over A,L: Step 3 - pass + A->>M: writes [4], lock off + Note over A,L: Callbacks during step = [] + Note over A,L: Property count = 4, cumulative log = [1, 4] + Note over A,L: Step 4 - pass + A->>M: unbind receiver + Note over A,L: Callbacks during step = [] + Note over A,L: Property count = 4, cumulative log = [1, 4] + Note over A,L: Step 5 - pass + A->>M: writes [5], lock off + Note over A,L: Callbacks during step = [] + Note over A,L: Property count = 5, cumulative log = [1, 4] +``` + +## Observed + +| Step | Requested action | New callbacks | Cumulative log | Property count | Check | +| --- | --- | --- | --- | --- | --- | +| 1 | writes [1, 1]; lock off | [1] | [1] | 1 | pass | +| 2 | writes [2, 3, 4]; lock on | [4] | [1, 4] | 4 | pass | +| 3 | writes [4]; lock off | [] | [1, 4] | 4 | pass | +| 4 | unbind receiver | [] | [1, 4] | 4 | pass | +| 5 | writes [5]; lock off | [] | [1, 4] | 5 | pass | + +```mermaid +sequenceDiagram + participant A as Adapter + participant M as Property count + participant L as Callback log + Note over A,L: Step 1 - pass + A->>M: writes [1, 1], lock off + Note over A,L: Callbacks during step = [1] + Note over A,L: Property count = 1, cumulative log = [1] + Note over A,L: Step 2 - pass + A->>M: writes [2, 3, 4], lock on + Note over A,L: Callbacks during step = [4] + Note over A,L: Property count = 4, cumulative log = [1, 4] + Note over A,L: Step 3 - pass + A->>M: writes [4], lock off + Note over A,L: Callbacks during step = [] + Note over A,L: Property count = 4, cumulative log = [1, 4] + Note over A,L: Step 4 - pass + A->>M: unbind receiver + Note over A,L: Callbacks during step = [] + Note over A,L: Property count = 4, cumulative log = [1, 4] + Note over A,L: Step 5 - pass + A->>M: writes [5], lock off + Note over A,L: Callbacks during step = [] + Note over A,L: Property count = 5, cumulative log = [1, 4] +``` diff --git a/examples/dispatch/verify.py b/examples/dispatch/verify.py new file mode 100644 index 0000000..412a797 --- /dev/null +++ b/examples/dispatch/verify.py @@ -0,0 +1,54 @@ +"""Run real API/native checks and a deliberately wrong expectation; save or check evidence.""" +import argparse +import copy +import hashlib +from importlib.metadata import version +import json +from pathlib import Path +import subprocess +import sys + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +sys.path.insert(0, str(ROOT)) +from interaction_check import CheckError, replay # noqa: E402 +from render import render # noqa: E402 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + subprocess.run([sys.executable, "-m", "unittest", "discover", "-s", str(HERE), + "-p", "native_tests.py", "-v"], check=True) + case = json.loads((HERE / "scenario.json").read_text(encoding="utf-8")) + command = [sys.executable, str(HERE / "adapter.py")] + correct = replay(case, command, trace=True) + wrong = copy.deepcopy(case) + wrong["steps"][1]["expect"]["log"] = [1, 2, 3, 4] + try: + replay(wrong, command, trace=True) + except CheckError as error: + assert error.failure_step == 2 + assert error.trace[-1]["actual"] == {"count": 4, "log": [1, 4]} + failed = {"passed": False, "error": str(error), "trace": error.trace} + else: + raise AssertionError("deliberately wrong expectation was not caught") + sources = [HERE / n for n in ["adapter.py", "native_tests.py", "scenario.json", "render.py", "verify.py", "requirements.txt"]] + evidence = {"package": f"python-dispatch=={version('python-dispatch')}", + "native_tests": 3, "synthetic_wrong_expectation": True, + "source_sha256": {str(p.relative_to(ROOT)).replace('\\', '/'): hashlib.sha256( + p.read_text(encoding='utf-8').encode()).hexdigest() for p in sources}, + "results": [correct, failed]} + outputs = {ROOT / "docs/evidence/dispatch.json": json.dumps(evidence, indent=2) + "\n", + HERE / "trace.md": render(correct), HERE / "mismatch.md": render(failed)} + for path, text in outputs.items(): + if args.check: + assert path.read_text(encoding="utf-8") == text, f"stale evidence: {path}" + else: + path.write_text(text, encoding="utf-8") + print("3 native tests; 5 replay steps; deliberate wrong expectation detected at step 2.") + + +if __name__ == "__main__": + main() diff --git a/test_dispatch_render.py b/test_dispatch_render.py new file mode 100644 index 0000000..5432588 --- /dev/null +++ b/test_dispatch_render.py @@ -0,0 +1,50 @@ +import copy +import json +from pathlib import Path +import unittest + +from examples.dispatch.render import render + + +class DispatchRenderTests(unittest.TestCase): + def setUp(self): + case = json.loads(Path("examples/dispatch/scenario.json").read_text(encoding="utf-8")) + self.result = {"trace": [dict(s, actual=copy.deepcopy(s["expect"]), step=i, passed=True) + for i, s in enumerate(case["steps"], 1)]} + + def test_distinguishes_writes_delta_log_and_property(self): + text = render(self.result) + self.assertIn("| 2 | writes [2, 3, 4]; lock on | [4] | [1, 4] | 4 | pass |", text) + self.assertIn("| 5 | writes [5]; lock off | [] | [1, 4] | 5 | pass |", text) + self.assertEqual(text.count("sequenceDiagram"), 2) + + def test_partial_expectations_and_reset_logs_are_not_invented(self): + for change in [lambda t: t[1]["expect"].pop("log"), + lambda t: t[1]["actual"].update(log=[]), + lambda t: t[1]["actual"].update(count=True)]: + result = copy.deepcopy(self.result) + change(result["trace"]) + with self.assertRaises(ValueError): + render(result) + + def test_only_checked_prefix_and_both_sides_of_mismatch(self): + self.result["trace"] = self.result["trace"][:2] + self.result["trace"][1]["passed"] = False + self.result["trace"][1]["expect"]["log"] = [1, 2, 3, 4] + text = render(self.result) + self.assertNotIn("Step 3", text) + self.assertIn("Callbacks during step = [2, 3, 4]", text) + self.assertIn("Callbacks during step = [4]", text) + self.assertIn("MISMATCH", text) + + def test_report_text_cannot_inject_mermaid_or_html(self): + self.result["name"] = "\n```\n%%{init:evil}%%" + self.assertNotIn("script", render(self.result)) + self.result["trace"][0]["event"]["values"] = ["1; participant Evil"] + with self.assertRaises(ValueError): + render(self.result) + + def test_compact_or_empty_report_needs_trace(self): + for result in [{"passed": True}, {"trace": []}]: + with self.assertRaises(ValueError): + render(result)