From 2466d0eddea2ab12f7cdf0c6c1a33672d2236aaa Mon Sep 17 00:00:00 2001 From: githubergsx <18515835396@163.com> Date: Thu, 29 Oct 2026 20:22:48 +0800 Subject: [PATCH 1/2] [Feat] add confidence-gated browser deferral --- docs/browser-front.md | 10 +++ s1a/browser/browse.py | 16 ++++ s1a/browser/decision_model.py | 101 ++++++++++++++++++++++- scripts/calibrate_browser_deferral.py | 81 ++++++++++++++++++ tests/test_browse.py | 10 +++ tests/test_browser_policy.py | 89 +++++++++++++++++++- tests/test_calibrate_browser_deferral.py | 18 ++++ 7 files changed, 320 insertions(+), 5 deletions(-) create mode 100644 scripts/calibrate_browser_deferral.py create mode 100644 tests/test_calibrate_browser_deferral.py diff --git a/docs/browser-front.md b/docs/browser-front.md index c72bcec..2766bec 100644 --- a/docs/browser-front.md +++ b/docs/browser-front.md @@ -158,6 +158,16 @@ browser-use/jev-ultrafast (MIT), whose observe-decide-act tick this policy follo `elapsed_ms` stops before the screenshot, and neither the outcome nor the release of the browser and the session depends on it. Tests: `tests/test_browse.py`, and `tests/system/test_final_screenshot.py` on a live @playwright/mcp. +21. Confidence-gated deferral (`--defer-below P`, off by default) keeps the decision model as the first-stage + selector. When its operation confidence is strictly below `P` (a finite value from 0 through 1), the configured + chat model receives the current goal, page and only the currently offered operation/target identifiers. Its JSON + response is decoded back through that same action space; an invented operation or target, malformed JSON or a + provider failure ends the task `BLOCKED` rather than executing the first-stage action. `decision_ticks.json` + records `deferred`, the original choice/confidence, fallback choice, latency and separately-known fallback + tokens. The cutoff is a workload- and backend-specific routing knob, not a calibrated probability or a general + quality claim. Compare it with decision-only and chat-only on the same bounded task before reporting outcomes. + `python scripts/calibrate_browser_deferral.py runs/.../decision_ticks.json --rate 0.15` derives such a cutoff + from saved artifacts only; ties are not split, so its output reports the actual deferred count/rate as well. ## Rejected diff --git a/s1a/browser/browse.py b/s1a/browser/browse.py index 81adfed..a7bfeda 100644 --- a/s1a/browser/browse.py +++ b/s1a/browser/browse.py @@ -6,6 +6,7 @@ import argparse import asyncio import json +import math import re import shutil import sys @@ -246,8 +247,12 @@ async def browse( """ if policy.rethink_on and model_name == "llm": raise RuntimeError("--rethink on needs a decision model; --model llm cannot use it") + if policy.defer_below is not None and model_name == "llm": + raise RuntimeError("--defer-below needs a decision model; --model llm cannot use it") if policy.rethink_on and decision_model is None: raise RuntimeError(f"--rethink on needs a decision model; --model {model_name} was given none") + if policy.defer_below is not None and decision_model is None: + raise RuntimeError(f"--defer-below needs a decision model; --model {model_name} was given none") logs_dir.mkdir(parents=True, exist_ok=True) calls: list[dict[str, Any]] = [] counted = CountingModel(chat, calls) @@ -354,6 +359,13 @@ def parser(spec: BrowserAgentSpec) -> argparse.ArgumentParser: default="off", help="decision models only: offer values extracted from the goal as a choice head", ) + build.add_argument( + "--defer-below", + type=float, + default=None, + metavar="P", + help="decision models only: ask the chat fallback to choose from the current action space when confidence is below P (0 to 1; off by default)", + ) build.add_argument( "--rethink", choices=("on", "off"), @@ -392,12 +404,16 @@ def parser(spec: BrowserAgentSpec) -> argparse.ArgumentParser: def policy_from_args(args: argparse.Namespace) -> BrowserPolicy: """The policy from the parsed flags. ``RecoveryLimits`` rejects a non-finite or non-positive budget here.""" + defer_below = args.defer_below + if defer_below is not None and (not math.isfinite(defer_below) or not 0 <= defer_below <= 1): + raise ValueError("--defer-below must be finite and between 0 and 1") return BrowserPolicy( prefetch_values=args.prefetch == "on", batch_actions=args.batch == "on", goal_value_cache=args.goal_values == "on", rethink_on=args.rethink == "on", recovery_limits=RecoveryLimits(max_attempts=args.rethink_attempts, timeout_s=float(args.rethink_timeout)), + defer_below=defer_below, ) diff --git a/s1a/browser/decision_model.py b/s1a/browser/decision_model.py index 3713c25..f3cd167 100644 --- a/s1a/browser/decision_model.py +++ b/s1a/browser/decision_model.py @@ -21,7 +21,7 @@ import statistics import time from collections.abc import Awaitable -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, AsyncIterator from urllib.parse import urlparse from uuid import uuid4 @@ -120,6 +120,8 @@ class BrowserPolicy: goal_value_cache: bool # offer values extracted from the goal to Jev as a choice head rethink_on: bool = False # bounded recovery: refresh plus plan under a per-task budget, never a wider tool set recovery_limits: RecoveryLimits = field(default_factory=RecoveryLimits) # attempts and active seconds, never reset + # An opt-in confidence gate. ``None`` retains the historic decision-model-only path. + defer_below: float | None = None @dataclass @@ -193,6 +195,7 @@ def __init__( self._batch_actions = policy.batch_actions self._rethink_on = policy.rethink_on self._recovery_limits = policy.recovery_limits + self._defer_below = policy.defer_below self._max_steps = max_steps self._language = spec.language self._decision_model = decision_model @@ -332,6 +335,18 @@ async def _decide_message(self, messages: Any) -> AssistantMessage: logger.warning("[BrowserDecisionModel] decision failed: %s", exc, exc_info=True) return await self._final(run, "BLOCKED", snapshot, f"decision failed: {exc}") move = interpret(decision, space) + deferred = self._defer_below is not None and move.confidence < self._defer_below + fallback_record: dict[str, Any] = { + "fallback_choice": None, + "fallback_ms": None, + "fallback_usage_known": False, + } + if deferred: + try: + move, fallback_record = await self._defer_move(run, space, observation, move) + except Exception as exc: # noqa: BLE001 - a failed escalation must never execute the weak choice + logger.warning("[BrowserDecisionModel] deferred choice failed", exc_info=True) + return await self._final(run, "BLOCKED", snapshot, f"deferred choice failed: {type(exc).__name__}") probabilities, candidates = top_probabilities(decision, move, space, TOP_CANDIDATES) run.tick += 1 record = { @@ -348,6 +363,13 @@ async def _decide_message(self, messages: Any) -> AssistantMessage: "operation": move.operation, "target": move.candidate.item.get("label") if move.candidate else None, "confidence": round(move.confidence, 3), + "deferred": deferred, + "decision_choice": { + "operation": decision.choice("operation").key, + "target": self._decision_target_key(decision), + }, + "decision_confidence": round(decision.choice("operation").confidence, 3), + **fallback_record, **( {"backend_confidence": round(move.backend_confidence, 3)} if move.backend_confidence is not None @@ -409,6 +431,76 @@ async def _decide_message(self, messages: Any) -> AssistantMessage: return await self._final(run, "BLOCKED", snapshot, "") return await self._act(run, move, snapshot, record) + @staticmethod + def _decision_target_key(decision: Any) -> str | None: + """The first-stage target key, retained even when a fallback replaces its move.""" + operation = decision.choice("operation").key + name = f"{operation.lower()}_target" + return decision.choice(name).key if name in decision.answers else None + + async def _defer_move( + self, run: _Run, space: Any, observation: Observation, move: Move + ) -> tuple[Move, dict[str, Any]]: + """Ask the chat fallback to choose one *existing* action-space entry. + + The model receives identifiers rather than browser tool arguments. Its response is decoded back through + ``space`` here, so an invented operation or target is a terminal failure rather than an executable action. + """ + choices = { + "operations": space.operations, + "targets": {operation: sorted(head) for operation, head in space.heads.items()}, + } + state = observation.state if isinstance(observation.state, dict) else {} + prompt = { + "goal": run.goal, + "page": state.get("page", {}), + "choices": choices, + "response": {"operation": "one item from choices.operations", "target": "target id or null"}, + } + started_at = time.perf_counter() + response = await self._fallback.invoke( + [ + { + "role": "system", + "content": "Choose exactly one offered browser action. Return only JSON; never invent an operation or target.", + }, + {"role": "user", "content": json.dumps(prompt, ensure_ascii=False)}, + ] + ) + elapsed_ms = round((time.perf_counter() - started_at) * 1000) + payload = json.loads(str(getattr(response, "content", ""))) + operation = payload.get("operation") + target = payload.get("target") + if operation not in space.operations: + raise ValueError("fallback operation is not offered") + candidate = None + if operation in space.heads: + if not isinstance(target, str) or target not in space.heads[operation]: + raise ValueError("fallback target is not offered") + candidate = space.heads[operation][target] + elif target is not None: + raise ValueError("control operation must not name a target") + # OpenJiuwen chat replies expose token accounting as ``usage_metadata``. CountingModel independently + # records the call for the run-level total, while this copies the same evidence into the deferred tick. + usage = getattr(response, "usage_metadata", None) + usage_known = usage is not None + return ( + replace(move, operation=operation, candidate=candidate), + { + "fallback_choice": {"operation": operation, "target": target}, + "fallback_ms": elapsed_ms, + "fallback_usage_known": usage_known, + **( + { + "fallback_input_tokens": int(getattr(usage, "input_tokens", 0)), + "fallback_output_tokens": int(getattr(usage, "output_tokens", 0)), + } + if usage_known + else {} + ), + }, + ) + # -- bounded recovery --------------------------------------------------- @staticmethod @@ -1127,6 +1219,9 @@ def report(self) -> dict[str, Any]: "interactions": 0, "waits": 0, "median_decision_ms": 0, + "deferred_steps": 0, + "deferred_rate": 0.0, + "fallback_ms": 0, "jev_input_tokens": 0, "median_probe_ms": 0, "settle_probes": 0, @@ -1148,12 +1243,16 @@ def report(self) -> dict[str, Any]: }, } jev = [t["decision_ms"] for t in run.ticks] + deferred = [t for t in run.ticks if t.get("deferred")] return { "elapsed_ms": round((time.perf_counter() - run.started_at) * 1000) if run.started_at else 0, "decisions": len(jev), "interactions": len([h for h in run.history if h["kind"] != "wait"]), "waits": len([h for h in run.history if h["kind"] == "wait"]), "median_decision_ms": int(statistics.median(jev)) if jev else 0, + "deferred_steps": len(deferred), + "deferred_rate": round(len(deferred) / len(jev), 3) if jev else 0.0, + "fallback_ms": sum(int(t.get("fallback_ms") or 0) for t in deferred), "jev_input_tokens": ( sum(int(t.get("input_tokens") or 0) for t in run.ticks) if self._decision_model.bills_input_tokens diff --git a/scripts/calibrate_browser_deferral.py b/scripts/calibrate_browser_deferral.py new file mode 100644 index 0000000..c7cea54 --- /dev/null +++ b/scripts/calibrate_browser_deferral.py @@ -0,0 +1,81 @@ +"""Choose empirical browser-deferral cutoffs from saved ``decision_ticks.json`` files. + +This is deliberately an offline helper: it only reads JSON artifacts and reports the threshold that defers the +requested lowest-confidence fraction of the observed steps. It does not claim that a decision confidence is a +calibrated probability. +""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from typing import Any + + +def read_confidences(paths: list[Path]) -> list[float]: + """Read finite confidence values from complete artifacts or bare tick lists.""" + values: list[float] = [] + for path in paths: + payload: Any = json.loads(path.read_text(encoding="utf-8")) + ticks = payload.get("ticks", []) if isinstance(payload, dict) else payload + if not isinstance(ticks, list): + raise ValueError(f"{path}: expected a ticks list") + for tick in ticks: + confidence = tick.get("confidence") if isinstance(tick, dict) else None + if isinstance(confidence, (int, float)) and not isinstance(confidence, bool) and math.isfinite(confidence): + values.append(float(confidence)) + if not values: + raise ValueError("no finite confidence values found") + return values + + +def calibrate(confidences: list[float], rates: list[float]) -> list[dict[str, float | int]]: + """Return the strict-below threshold and actual count for each requested fraction. + + Ties are never split: the reported actual rate may be lower than requested, which makes a command's routing + behaviour reproducible instead of depending on artifact order. + """ + ordered = sorted(confidences) + if not ordered: + raise ValueError("at least one confidence is required") + rows: list[dict[str, float | int]] = [] + for rate in rates: + if not 0 <= rate <= 1 or not math.isfinite(rate): + raise ValueError("rates must be finite values from 0 through 1") + requested = math.floor(len(ordered) * rate) + if requested == 0: + threshold = ordered[0] + else: + threshold = ordered[min(requested, len(ordered) - 1)] + count = sum(value < threshold for value in ordered) + rows.append( + { + "requested_rate": rate, + "defer_below": threshold, + "deferred_steps": count, + "total_steps": len(ordered), + "actual_rate": count / len(ordered), + } + ) + return rows + + +def main() -> None: + parser = argparse.ArgumentParser(description="Calibrate offline confidence-gated browser deferral.") + parser.add_argument("artifacts", type=Path, nargs="+", help="saved decision_ticks.json file(s)") + parser.add_argument( + "--rate", + action="append", + type=float, + default=None, + help="requested deferral fraction, repeatable (default: 0.1, 0.15, 0.25)", + ) + args = parser.parse_args() + rates = args.rate or [0.1, 0.15, 0.25] + print(json.dumps({"calibration": calibrate(read_confidences(args.artifacts), rates)}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_browse.py b/tests/test_browse.py index 37365df..9dbf795 100644 --- a/tests/test_browse.py +++ b/tests/test_browse.py @@ -39,6 +39,7 @@ def test_the_spec_and_the_policy_defaults_and_their_overrides(self) -> None: args = browse.parser(SPEC).parse_args(GOAL) self.assertEqual((args.max_steps, args.timeout, args.profile_out), (100, 180.0, None)) self.assertEqual((args.batch, args.prefetch, args.goal_values), ("off", "on", "off")) + self.assertIsNone(args.defer_below) self.assertEqual( browse.policy_from_args(args), BrowserPolicy(prefetch_values=True, batch_actions=False, goal_value_cache=False), @@ -52,6 +53,15 @@ def test_the_spec_and_the_policy_defaults_and_their_overrides(self) -> None: BrowserPolicy(prefetch_values=False, batch_actions=True, goal_value_cache=False), ) + def test_defer_below_is_opt_in_and_rejects_invalid_thresholds(self) -> None: + args = browse.parser(SPEC).parse_args([*GOAL, "--defer-below", "0.5"]) + self.assertEqual(browse.policy_from_args(args).defer_below, 0.5) + for value in ("-0.01", "1.01", "nan", "inf"): + with self.subTest(value=value): + args = browse.parser(SPEC).parse_args([*GOAL, "--defer-below", value]) + with self.assertRaises(ValueError): + browse.policy_from_args(args) + def test_a_timeout_or_max_steps_at_or_below_zero_is_a_usage_error(self) -> None: for flags in (["--timeout", "0"], ["--timeout", "-1"], ["--max-steps", "0"], ["--max-steps", "-3"]): with self.assertRaises(SystemExit) as caught: diff --git a/tests/test_browser_policy.py b/tests/test_browser_policy.py index cde7007..daf998f 100644 --- a/tests/test_browser_policy.py +++ b/tests/test_browser_policy.py @@ -11,6 +11,7 @@ import asyncio import contextlib import json +from types import SimpleNamespace from typing import Any from unittest import IsolatedAsyncioTestCase, TestCase, mock @@ -87,11 +88,11 @@ def _answer(choice: str, ids: list[str], confidence: float = 0.9) -> dict[str, A return {"choice": choice, "confidence": confidence, "probabilities": probabilities} -def _answers(operation: str, value: str) -> dict[str, Any]: +def _answers(operation: str, value: str, confidence: float = 0.9) -> dict[str, Any]: """A payload answering every head, as Jev does; ``decide_many`` validates them all, ``interpret`` uses one.""" return { "answers": { - "operation": _answer(operation, _OPERATIONS), + "operation": _answer(operation, _OPERATIONS, confidence), "type_text_target": _answer("2", ["2"]), "click_target": _answer("1", ["1", "2"]), "select_target": _answer("3:1", ["3:1", "3:2"]), @@ -377,12 +378,19 @@ def _slot_model( decision_model: DecisionModel | None = None, runtime: Any = None, batch_actions: bool = False, + defer_below: float | None = None, + fallback: Any = None, ) -> BrowserDecisionModel: with mock.patch.object(Model, "__init__", _fake_model_init): slot_model = BrowserDecisionModel( _spec(), - _policy(goal_value_cache=goal_value_cache, prefetch_values=prefetch_values, batch_actions=batch_actions), - _fallback(), + BrowserPolicy( + prefetch_values=prefetch_values, + batch_actions=batch_actions, + goal_value_cache=goal_value_cache, + defer_below=defer_below, + ), + fallback or _fallback(), decision_model=decision_model or JevModel(_Wire(scripted)), value_model=None, ) @@ -402,6 +410,76 @@ def _decision_probes(runtime: Any) -> list[dict[str, Any]]: class TestBrowserDecisionModel(IsolatedAsyncioTestCase): + async def test_low_confidence_defers_to_a_choice_in_the_current_action_space(self) -> None: + fallback = _fallback() + + async def choose(messages: list[dict[str, str]], **_kwargs: Any) -> Any: + fallback.asked.append(messages) + return SimpleNamespace( + content='{"operation": "CLICK", "target": "1"}', + usage_metadata=SimpleNamespace(input_tokens=12, output_tokens=3), + ) + + fallback.invoke = choose + slot_model = _slot_model( + [_answers("CLICK", "none", confidence=0.2)], + goal_value_cache=False, + prefetch_values=False, + defer_below=0.5, + fallback=fallback, + ) + + message = await slot_model.invoke(_MESSAGES, tools=_TOOLS) + + self.assertEqual(message.tool_calls[0].name, "browser_click") + tick = slot_model.ticks[0] + self.assertTrue(tick["deferred"]) + self.assertEqual(tick["decision_choice"]["operation"], "CLICK") + self.assertEqual(tick["fallback_choice"], {"operation": "CLICK", "target": "1"}) + self.assertTrue(tick["fallback_usage_known"]) + self.assertEqual((tick["fallback_input_tokens"], tick["fallback_output_tokens"]), (12, 3)) + self.assertEqual(len(fallback.asked), 1) + self.assertEqual(slot_model.report()["deferred_steps"], 1) + + async def test_high_confidence_does_not_call_the_fallback(self) -> None: + fallback = _fallback() + slot_model = _slot_model( + [_answers("CLICK", "none", confidence=0.9)], + goal_value_cache=False, + prefetch_values=False, + defer_below=0.5, + fallback=fallback, + ) + + await slot_model.invoke(_MESSAGES, tools=_TOOLS) + + self.assertFalse(slot_model.ticks[0]["deferred"]) + self.assertEqual(fallback.asked, []) + + async def test_invalid_or_failed_fallback_blocks_without_an_action(self) -> None: + for response in ('{"operation": "CLICK", "target": "999"}', RuntimeError("unavailable")): + with self.subTest(response=response): + fallback = _fallback() + + async def choose(_messages: list[dict[str, str]], **_kwargs: Any) -> Any: + if isinstance(response, Exception): + raise response + return mock.MagicMock(content=response) + + fallback.invoke = choose + slot_model = _slot_model( + [_answers("CLICK", "none", confidence=0.2)], + goal_value_cache=False, + prefetch_values=False, + defer_below=0.5, + fallback=fallback, + ) + terminal = await slot_model.invoke(_MESSAGES, tools=_TOOLS) + self.assertIsNone(terminal.tool_calls) + summary = json.loads(terminal.content) + self.assertEqual(summary["status"], "BLOCKED") + self.assertIn("deferred choice failed", summary["reason"]) + async def test_first_tick_types_the_prefetched_value_then_done(self) -> None: slot_model = _slot_model([_answers("TYPE_TEXT", "none"), _answers("DONE", "none")]) @@ -520,6 +598,9 @@ async def test_report_before_any_turn_returns_the_zeroed_shape(self) -> None: "interactions": 0, "waits": 0, "median_decision_ms": 0, + "deferred_steps": 0, + "deferred_rate": 0.0, + "fallback_ms": 0, "jev_input_tokens": 0, "median_probe_ms": 0, "settle_probes": 0, diff --git a/tests/test_calibrate_browser_deferral.py b/tests/test_calibrate_browser_deferral.py new file mode 100644 index 0000000..3ae01bc --- /dev/null +++ b/tests/test_calibrate_browser_deferral.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import unittest + +from scripts.calibrate_browser_deferral import calibrate + + +class TestCalibration(unittest.TestCase): + def test_ties_are_not_split(self) -> None: + row = calibrate([0.1, 0.2, 0.2, 0.9], [0.5])[0] + self.assertEqual(row["defer_below"], 0.2) + self.assertEqual((row["deferred_steps"], row["actual_rate"]), (1, 0.25)) + + def test_rejects_invalid_rate_and_empty_data(self) -> None: + with self.assertRaises(ValueError): + calibrate([], [0.1]) + with self.assertRaises(ValueError): + calibrate([0.5], [1.1]) From bfb3df767db26925f713ad9b682d998e7afe561a Mon Sep 17 00:00:00 2001 From: githubergsx <18515835396@163.com> Date: Thu, 29 Oct 2026 20:43:13 +0800 Subject: [PATCH 2/2] [Fix] preserve browser records when deferral is off --- s1a/browser/decision_model.py | 55 +++++++++++++++--------- tests/test_browser_policy.py | 23 +++++++--- tests/test_calibrate_browser_deferral.py | 18 +++++++- 3 files changed, 69 insertions(+), 27 deletions(-) diff --git a/s1a/browser/decision_model.py b/s1a/browser/decision_model.py index f3cd167..e412c7d 100644 --- a/s1a/browser/decision_model.py +++ b/s1a/browser/decision_model.py @@ -336,11 +336,13 @@ async def _decide_message(self, messages: Any) -> AssistantMessage: return await self._final(run, "BLOCKED", snapshot, f"decision failed: {exc}") move = interpret(decision, space) deferred = self._defer_below is not None and move.confidence < self._defer_below - fallback_record: dict[str, Any] = { - "fallback_choice": None, - "fallback_ms": None, - "fallback_usage_known": False, - } + fallback_record: dict[str, Any] = {} + if self._defer_below is not None: + fallback_record = { + "fallback_choice": None, + "fallback_ms": None, + "fallback_usage_known": False, + } if deferred: try: move, fallback_record = await self._defer_move(run, space, observation, move) @@ -363,13 +365,6 @@ async def _decide_message(self, messages: Any) -> AssistantMessage: "operation": move.operation, "target": move.candidate.item.get("label") if move.candidate else None, "confidence": round(move.confidence, 3), - "deferred": deferred, - "decision_choice": { - "operation": decision.choice("operation").key, - "target": self._decision_target_key(decision), - }, - "decision_confidence": round(decision.choice("operation").confidence, 3), - **fallback_record, **( {"backend_confidence": round(move.backend_confidence, 3)} if move.backend_confidence is not None @@ -387,6 +382,18 @@ async def _decide_message(self, messages: Any) -> AssistantMessage: "model": move.model, # who answered: the response's model id, else the one the request named **decision.provenance, } + if self._defer_below is not None: + record.update( + { + "deferred": deferred, + "decision_choice": { + "operation": decision.choice("operation").key, + "target": self._decision_target_key(decision), + }, + "decision_confidence": round(decision.choice("operation").confidence, 3), + **fallback_record, + } + ) if self._decision_model.name == "jev-vl-served": record["decision_raw"] = decision.raw if screenshot_ms is not None: # an image-reading model: the capture is part of the step's cost @@ -1213,15 +1220,12 @@ def report(self) -> dict[str, Any]: """ run = self._run if run is None: - return { + report = { "elapsed_ms": 0, "decisions": 0, "interactions": 0, "waits": 0, "median_decision_ms": 0, - "deferred_steps": 0, - "deferred_rate": 0.0, - "fallback_ms": 0, "jev_input_tokens": 0, "median_probe_ms": 0, "settle_probes": 0, @@ -1242,17 +1246,16 @@ def report(self) -> dict[str, Any]: "events": [], }, } + if self._defer_below is not None: + report.update({"deferred_steps": 0, "deferred_rate": 0.0, "fallback_ms": 0}) + return report jev = [t["decision_ms"] for t in run.ticks] - deferred = [t for t in run.ticks if t.get("deferred")] - return { + report = { "elapsed_ms": round((time.perf_counter() - run.started_at) * 1000) if run.started_at else 0, "decisions": len(jev), "interactions": len([h for h in run.history if h["kind"] != "wait"]), "waits": len([h for h in run.history if h["kind"] == "wait"]), "median_decision_ms": int(statistics.median(jev)) if jev else 0, - "deferred_steps": len(deferred), - "deferred_rate": round(len(deferred) / len(jev), 3) if jev else 0.0, - "fallback_ms": sum(int(t.get("fallback_ms") or 0) for t in deferred), "jev_input_tokens": ( sum(int(t.get("input_tokens") or 0) for t in run.ticks) if self._decision_model.bills_input_tokens @@ -1268,6 +1271,16 @@ def report(self) -> dict[str, Any]: "history": run.history, "recovery": self._recovery_summary(run), } + if self._defer_below is not None: + deferred = [t for t in run.ticks if t["deferred"]] + report.update( + { + "deferred_steps": len(deferred), + "deferred_rate": round(len(deferred) / len(jev), 3) if jev else 0.0, + "fallback_ms": sum(int(t["fallback_ms"] or 0) for t in deferred), + } + ) + return report def _probe_failed(snapshot: dict[str, Any]) -> bool: diff --git a/tests/test_browser_policy.py b/tests/test_browser_policy.py index daf998f..0b60dea 100644 --- a/tests/test_browser_policy.py +++ b/tests/test_browser_policy.py @@ -456,8 +456,8 @@ async def test_high_confidence_does_not_call_the_fallback(self) -> None: self.assertFalse(slot_model.ticks[0]["deferred"]) self.assertEqual(fallback.asked, []) - async def test_invalid_or_failed_fallback_blocks_without_an_action(self) -> None: - for response in ('{"operation": "CLICK", "target": "999"}', RuntimeError("unavailable")): + async def test_invalid_malformed_or_failed_fallback_blocks_without_an_action(self) -> None: + for response in ('{"operation": "CLICK", "target": "999"}', "not JSON", RuntimeError("unavailable")): with self.subTest(response=response): fallback = _fallback() @@ -511,8 +511,16 @@ async def test_the_tick_records_the_decisions_latency_tokens_and_the_answering_m self.assertEqual(tick["probabilities"], {"1": 0.9, "2": 0.1}, "the click head's probabilities, for the replay") self.assertEqual(tick["candidates"]["1"], "Search") self.assertEqual((tick["model"], tick["requested_model"]), ("jev-9", "typesafe/jev-test"), "who answered (#93)") + self.assertTrue( + {"deferred", "decision_choice", "decision_confidence", "fallback_choice", "fallback_ms"}.isdisjoint(tick), + "the opt-in gate must not alter tick records while unset", + ) report = slot_model.report() self.assertEqual((report["decisions"], report["median_decision_ms"], report["jev_input_tokens"]), (1, 7, 315)) + self.assertTrue( + {"deferred_steps", "deferred_rate", "fallback_ms"}.isdisjoint(report), + "the opt-in gate must not alter reports while unset", + ) async def test_a_tick_marks_usage_unknown_when_the_backend_did_not_report_it(self) -> None: from s1a.decision_models import ScriptedModel @@ -598,9 +606,6 @@ async def test_report_before_any_turn_returns_the_zeroed_shape(self) -> None: "interactions": 0, "waits": 0, "median_decision_ms": 0, - "deferred_steps": 0, - "deferred_rate": 0.0, - "fallback_ms": 0, "jev_input_tokens": 0, "median_probe_ms": 0, "settle_probes": 0, @@ -625,6 +630,14 @@ async def test_report_before_any_turn_returns_the_zeroed_shape(self) -> None: self.assertEqual(slot_model.ticks, []) self.assertEqual(slot_model.started_at, 0.0) + async def test_enabled_gate_adds_zeroed_report_fields_before_any_turn(self) -> None: + slot_model = _slot_model([], defer_below=0.5) + report = slot_model.report() + self.assertEqual( + (report["deferred_steps"], report["deferred_rate"], report["fallback_ms"]), + (0, 0.0, 0), + ) + class TestJevRunIsolation(IsolatedAsyncioTestCase): """B1: per-task state must not leak into the task that follows it.""" diff --git a/tests/test_calibrate_browser_deferral.py b/tests/test_calibrate_browser_deferral.py index 3ae01bc..513ea1a 100644 --- a/tests/test_calibrate_browser_deferral.py +++ b/tests/test_calibrate_browser_deferral.py @@ -1,8 +1,11 @@ from __future__ import annotations +import json +import tempfile import unittest +from pathlib import Path -from scripts.calibrate_browser_deferral import calibrate +from scripts.calibrate_browser_deferral import calibrate, read_confidences class TestCalibration(unittest.TestCase): @@ -16,3 +19,16 @@ def test_rejects_invalid_rate_and_empty_data(self) -> None: calibrate([], [0.1]) with self.assertRaises(ValueError): calibrate([0.5], [1.1]) + + def test_reads_complete_artifacts_and_rejects_artifacts_without_finite_confidence(self) -> None: + with tempfile.TemporaryDirectory() as directory: + artifact = Path(directory) / "decision_ticks.json" + artifact.write_text( + json.dumps({"ticks": [{"confidence": 0.2}, {"confidence": 0.8}, {"confidence": True}]}), + encoding="utf-8", + ) + self.assertEqual(read_confidences([artifact]), [0.2, 0.8]) + + artifact.write_text(json.dumps({"ticks": [{"confidence": None}]}), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "no finite confidence"): + read_confidences([artifact])