From e015523240397048c49d283985ea4c1711500875 Mon Sep 17 00:00:00 2001 From: Adam Wright Date: Sat, 19 Sep 2026 06:26:37 +0000 Subject: [PATCH 1/2] Two collection guards passed against no implementation at all The website session observed that absence tests can pass vacuously -- removing their implementation left two of four green, and both were the ones asserting that something is not there. Nothing set is also nothing seen. Two of mine were the same. Deleting the wiring from `generate_answer` left five of seven green, including both tests meant to guard the dangerous directions: that a reactome selection never leaks into the userguide bundle, and that a selection does not outlive the question. Each asserted `None`, and `None` is what an unimplemented feature produces. Both now establish the mechanism is live before asserting the absence -- a reactome call that must be seen, then the userguide call that must not be; set during, absent after. Deleting the wiring now fails four of seven instead of two. The three that still pass are the prompt and schema tests, which do not depend on the wiring and are not vacuous for passing. The general form is worth keeping: a test that asserts something is absent proves nothing until the same test has shown the thing can be present. Co-Authored-By: Claude Opus 5 --- tests/agent/test_collection_routing.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/agent/test_collection_routing.py b/tests/agent/test_collection_routing.py index f0a77b2..81aa082 100644 --- a/tests/agent/test_collection_routing.py +++ b/tests/agent/test_collection_routing.py @@ -82,13 +82,25 @@ def test_the_userguide_is_never_narrowed_by_a_reactome_selection() -> None: # collection; a leaked selection names collections it does not have, and # `resolve_collections` would widen it back silently -- correct, but only # by accident, and with a WARNING for every question. + # + # The reactome half is here on purpose. Asserting `None` alone passes + # against no implementation at all -- nothing set is also nothing seen -- + # so this first shows the mechanism is live, then shows it does not reach + # the userguide. Verified by deleting the implementation: without the + # first half, this test stays green. rag = _RecordingRag() + asyncio.run( + _builder(rag, "reactome").generate_answer( + _state("reactome", ["disease_variants"]), RunnableConfig() + ) + ) + assert rag.seen == [["disease_variants"]], "the mechanism is not live" asyncio.run( _builder(rag, "userguide").generate_answer( _state("userguide", ["disease_variants"]), RunnableConfig() ) ) - assert rag.seen == [None], "a reactome selection leaked into the userguide" + assert rag.seen[1] is None, "a reactome selection leaked into the userguide" def test_the_selection_does_not_outlive_the_question() -> None: @@ -101,6 +113,9 @@ def test_the_selection_does_not_outlive_the_question() -> None: _state("reactome", ["ewas"]), RunnableConfig() ) ) + # Set during, absent after. The second assertion alone would pass against + # no implementation, since nothing set is also nothing left behind. + assert rag.seen == [["ewas"]], "the mechanism is not live" assert selected_collections.get() is None From 2f57d01000bf33e9694d9c184e08addb7939d3e4 Mon Sep 17 00:00:00 2001 From: Adam Wright Date: Sat, 19 Sep 2026 07:13:29 +0000 Subject: [PATCH 2/2] A third absence test, of the same class I had just fixed two of Reviewing the fix found one it missed. `test_the_userguide_prompt_alone_ offers_no_collections` asserted only that the block is absent from the userguide-only prompt -- which passes just as well when the block never renders for anyone. Verified by deleting it: the test stayed green while a different test in the same file caught the fault. File-level coverage was therefore fine and the test itself was worth nothing. A guard that relies on a neighbour to be non-vacuous is one refactor of the neighbour away from guarding nothing, and the neighbour is not mentioned anywhere near it. Both halves are in the one test now: the block renders for reactome, and it does not reach the userguide. Deleting the block fails it. Co-Authored-By: Claude Opus 5 --- tests/agent/test_collection_routing.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/agent/test_collection_routing.py b/tests/agent/test_collection_routing.py index 81aa082..cee62bf 100644 --- a/tests/agent/test_collection_routing.py +++ b/tests/agent/test_collection_routing.py @@ -34,8 +34,15 @@ def test_every_collection_in_the_bundle_is_described_to_the_classifier() -> None def test_the_userguide_prompt_alone_offers_no_collections() -> None: - message = build_classifier_message(frozenset({"userguide"})) - assert "disease_variants" not in message + # Both halves, in one test. Asserting only the absence passes when the + # collections block never renders for anyone -- verified by deleting it, + # after which this stayed green while a different test caught the fault. + # A guard that relies on a neighbour to be non-vacuous is one refactor + # away from guarding nothing. + assert "disease_variants" in build_classifier_message( + frozenset({"reactome", "userguide"}) + ), "the collections block does not render at all" + assert "disease_variants" not in build_classifier_message(frozenset({"userguide"})) class _RecordingRag: