diff --git a/src/evaluation/answer_sweep.py b/src/evaluation/answer_sweep.py index cfd459e..1a8cf55 100644 --- a/src/evaluation/answer_sweep.py +++ b/src/evaluation/answer_sweep.py @@ -287,6 +287,13 @@ async def run(expectations: tuple[Expectation, ...], retries: int = 1) -> list[R # independent, and a shared history would make each a # follow-up of the last. thread_id=f"sweep-{index}-{attempt}", + # No web search. The sweep reads `answer` and nothing + # else, so the postprocess node's Tavily call was paid + # for and thrown away -- fifteen of them per run, and + # this runs after every beta deploy. It also slowed each + # question by the length of a web search, for a result + # no expectation has ever looked at. + enable_postprocess=False, ) result.answer = " ".join(str(out.get("answer") or "").split()) except Exception as exc: diff --git a/tests/evaluation/test_answer_sweep.py b/tests/evaluation/test_answer_sweep.py index e297168..a7d5c6c 100644 --- a/tests/evaluation/test_answer_sweep.py +++ b/tests/evaluation/test_answer_sweep.py @@ -198,3 +198,25 @@ def test_it_runs_once_the_collection_is_installed( assert not result.skipped assert not result.ok assert len(graph.asked) == 1 + + +def test_the_sweep_does_not_pay_for_a_web_search() -> None: + """The sweep reads `answer` and nothing else. + + With postprocess left at its default it ran a Tavily search per question, + discarded the result, and slowed each question by the length of that search + -- fifteen of them, after every beta deploy. Pinned here rather than trusted, + because the cost is invisible: nothing fails when it happens. + """ + import inspect + + from evaluation import answer_sweep + + source = inspect.getsource(answer_sweep) + assert ( + "enable_postprocess=False" in source + ), "the sweep is running the postprocess web search again" + assert "additional_content" not in source, ( + "the sweep now reads additional_content, so the assertion above is no " + "longer the right guard" + )