From 54fd01736db82ba9b062892178f8a0d5567ce05f Mon Sep 17 00:00:00 2001 From: Adam Wright Date: Fri, 18 Sep 2026 04:54:19 +0000 Subject: [PATCH] Stop the deploy sweep paying for a web search it discards The sweep calls ainvoke without enable_postprocess, so the postprocess node ran a Tavily search for each of its fifteen questions. The sweep reads `answer` and nothing else, so every one of those results was thrown away -- and this runs after each beta deploy, which is where the cost repeats. The same defect as the answer endpoint had, in the other caller. Found while checking, before deploying beta, whether the post-deploy sweep would spend money on searches nobody reads. It would have, so that deploy ran with --skip-sweep. Each question also waited for the search before completing, which is why the sweep's per-question timings were slower than the endpoint's on the same questions. Pinned by a test, because nothing fails when this regresses: the cost is invisible from the outside. The test also guards its own assumption -- if the sweep ever starts reading additional_content, the guard is no longer the right one and says so. Co-Authored-By: Claude Opus 5 --- src/evaluation/answer_sweep.py | 7 +++++++ tests/evaluation/test_answer_sweep.py | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/evaluation/answer_sweep.py b/src/evaluation/answer_sweep.py index cfd459e..1a8cf55 100644 --- a/src/evaluation/answer_sweep.py +++ b/src/evaluation/answer_sweep.py @@ -287,6 +287,13 @@ async def run(expectations: tuple[Expectation, ...], retries: int = 1) -> list[R # independent, and a shared history would make each a # follow-up of the last. thread_id=f"sweep-{index}-{attempt}", + # No web search. The sweep reads `answer` and nothing + # else, so the postprocess node's Tavily call was paid + # for and thrown away -- fifteen of them per run, and + # this runs after every beta deploy. It also slowed each + # question by the length of a web search, for a result + # no expectation has ever looked at. + enable_postprocess=False, ) result.answer = " ".join(str(out.get("answer") or "").split()) except Exception as exc: diff --git a/tests/evaluation/test_answer_sweep.py b/tests/evaluation/test_answer_sweep.py index e297168..a7d5c6c 100644 --- a/tests/evaluation/test_answer_sweep.py +++ b/tests/evaluation/test_answer_sweep.py @@ -198,3 +198,25 @@ def test_it_runs_once_the_collection_is_installed( assert not result.skipped assert not result.ok assert len(graph.asked) == 1 + + +def test_the_sweep_does_not_pay_for_a_web_search() -> None: + """The sweep reads `answer` and nothing else. + + With postprocess left at its default it ran a Tavily search per question, + discarded the result, and slowed each question by the length of that search + -- fifteen of them, after every beta deploy. Pinned here rather than trusted, + because the cost is invisible: nothing fails when it happens. + """ + import inspect + + from evaluation import answer_sweep + + source = inspect.getsource(answer_sweep) + assert ( + "enable_postprocess=False" in source + ), "the sweep is running the postprocess web search again" + assert "additional_content" not in source, ( + "the sweep now reads additional_content, so the assertion above is no " + "longer the right guard" + )