From d2d23886ffc0533e58c96a2475903019f7c5849c Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 4 Oct 2026 17:24:30 +0300 Subject: [PATCH 01/17] refactor: record every turn's tool effects on a per-turn ledger Plumbing for one turn-outcome rule from band-sdk-core (2.8.0): - Pin band-sdk-core==2.8.0; build the Band tool effect table and the band_no_reply definition from core, with parity tests. - band.core.turn.Turn wraps core's TurnLedger; report_unsettled_turn is the one place a missing-reply verdict becomes a room-visible failure. - @records_turn_effects records each agent tool method's effect on tools.turn, so dispatch, direct calls, the crewai/parlant catalogs and deliver_reply all count. A raise or a blank send records nothing. - send_notice / deliver_notice post an adapter's own messages without counting as the reply; settle() ends a turn the adapter answered itself. relay_reply is the one gate for a model's final text. test_reply_boundary guards direct send_message calls. - Custom tools take a required turn and record their declared effect; ACP (external band-mcp) and Letta (external mode) record Band tools from the stream; the Slack tee shares the ledger; per-message tools are bound after busy and approval checks. - Fix Letta reading tool_return_message.tool_name instead of name. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_016nwuEBtdtneuDkJHcvijDQ Signed-off-by: Alexander Zaikman --- docs/acp.md | 19 +- pyproject.toml | 2 +- src/band/adapters/anthropic.py | 4 +- src/band/adapters/claude_sdk.py | 34 ++-- src/band/adapters/codex.py | 140 +++++++-------- src/band/adapters/copilot_sdk.py | 76 +++----- src/band/adapters/crewai_flow.py | 4 +- src/band/adapters/cursor_acp.py | 15 +- src/band/adapters/gemini.py | 4 +- src/band/adapters/google_adk.py | 4 +- src/band/adapters/langgraph.py | 55 +++--- src/band/adapters/letta.py | 19 +- src/band/adapters/opencode/adapter.py | 71 +++----- src/band/adapters/opencode/approvals.py | 7 +- src/band/adapters/parlant.py | 8 +- src/band/adapters/pydantic_ai.py | 19 +- src/band/adapters/strands.py | 51 ++++-- src/band/core/delivery.py | 34 +++- src/band/core/protocols.py | 17 +- src/band/core/simple_adapter.py | 3 +- src/band/core/turn.py | 80 +++++++++ src/band/core/types.py | 36 ++++ src/band/integrations/acp/client_adapter.py | 11 +- src/band/integrations/acp/room_emitter.py | 103 ++++------- src/band/integrations/claude_sdk/tools.py | 14 +- .../integrations/copilot_sdk/room_ask_user.py | 2 +- src/band/integrations/crewai/tools.py | 4 +- .../integrations/langgraph/langchain_tools.py | 89 +++++---- src/band/integrations/mcp/engine.py | 33 ++-- src/band/integrations/slack/adapter.py | 7 +- src/band/runtime/contact_handler.py | 8 +- src/band/runtime/custom_tools.py | 19 +- src/band/runtime/execution.py | 17 +- src/band/runtime/tools/agent.py | 26 +++ src/band/runtime/tools/effects.py | 18 +- src/band/runtime/tools/inputs/chat.py | 24 +-- src/band/runtime/turn.py | 50 ++++++ src/band/runtime/types.py | 26 --- src/band/testing/fake_tools.py | 29 ++- .../approvals/test_manual_approval.py | 32 ++++ .../claude_sdk/tools/test_custom_tools.py | 29 +++ tests/adapters/copilot_sdk/test_reply.py | 2 + .../copilot_sdk/test_tool_bridging.py | 82 ++++++++- tests/adapters/langgraph/test_lifecycle.py | 3 +- tests/adapters/lettakit.py | 4 +- tests/adapters/opencode/conftest.py | 11 +- tests/adapters/opencode/helpers.py | 50 ++++-- tests/adapters/opencode/test_turns.py | 140 ++++++++++++++- tests/adapters/test_anthropic_adapter.py | 47 +++++ tests/adapters/test_codex_adapter.py | 108 ++++++++--- tests/adapters/test_cursor_acp_adapter.py | 53 +++++- tests/adapters/test_gemini_adapter.py | 42 +++++ tests/adapters/test_google_adk_adapter.py | 23 ++- tests/adapters/test_letta_adapter.py | 47 ++++- tests/adapters/test_parlant_adapter.py | 53 +++--- tests/adapters/test_pydantic_ai_adapter.py | 111 ++++++------ tests/adapters/test_strands_adapter.py | 29 ++- tests/core/test_delivery.py | 58 +++++- tests/core/test_turn_types.py | 35 ++++ .../test_reply_boundary.py | 77 ++++++++ tests/integrations/acp/test_client_adapter.py | 114 +----------- .../acp/test_client_adapter_behavior.py | 51 +++++- tests/integrations/acp/test_room_emitter.py | 170 +++++++++++++++++- .../langgraph/test_custom_tool_defs.py | 93 ++++++++-- tests/integrations/slack/test_wrapping.py | 37 +++- tests/integrations/test_crewai_tools.py | 78 +++++++- tests/integrations/test_omp.py | 4 +- tests/mcp/test_engine.py | 32 +++- tests/runtime/test_custom_tools.py | 24 +-- tests/runtime/test_execution.py | 6 +- tests/runtime/test_turn.py | 165 +++++++++++++++++ uv.lock | 20 +-- 72 files changed, 2196 insertions(+), 816 deletions(-) create mode 100644 src/band/core/turn.py create mode 100644 src/band/runtime/turn.py create mode 100644 tests/core/test_turn_types.py create mode 100644 tests/framework_conformance/test_reply_boundary.py create mode 100644 tests/runtime/test_turn.py diff --git a/docs/acp.md b/docs/acp.md index 149efdc1a..dd9af60ec 100644 --- a/docs/acp.md +++ b/docs/acp.md @@ -26,12 +26,15 @@ assert adapter.config.command == ("codex-acp",) - **Narration is live and ordered.** `ACPCollectingClient` streams finalized chunks to `RoomTurnEmitter` as they arrive, so a Band tool's own room post (a remote band-mcp posts over REST mid-turn) lands between its `tool_call` and `tool_result`. -- **Assistant text is held to turn close** and relayed only if no completed call settled - the reply (`turn_replied_in_room`, `settles_turn_reply`). Detection matches the - `tool_call` title, because tools may run out-of-process. Narrated names are canonicalized - (`canonicalize_mcp_tool_name`) so Copilot's `band-` prefix never reaches the room. -- **`emit=` never gates** chunk recording (the reply decision must see the whole turn), - the held text, or the closing `task` event. That event is resume state: +- **Assistant text is held to turn close** and relayed as one reply through `relay_reply`, + unless the turn already replied or declined. Injected Band tools record their own effect + on `tools.turn`. With `inject_band_tools=False` an external band-mcp runs out of process, + so the emitter records each completed call's `turn_effect` from the `tool_call` title + instead, and only in that mode, so no call is counted twice. Narrated names are + canonicalized (`canonicalize_mcp_tool_name`) so Copilot's `band-` prefix never reaches + the room. +- **`emit=` never gates** that recording, the held text, or the closing `task` event. + That event is resume state: `ACPClientHistoryConverter` reads `acp_client_session_id` / `acp_client_room_id` from it to `session/load` after a restart. - **Approved permissions are silent.** Only a denied request posts a synthetic @@ -64,7 +67,9 @@ failure fails that room turn visibly instead of falling back. - **Cursor:** question, plan and permission decisions default to `manual`, resolved by a room participant with `/cursor `. Cursor omits the session id on its extension notifications, so the adapter holds a turn lock and binds them to that turn's - session; Cursor turns are serialized. + session; Cursor turns are serialized. Decision prompts, timeout notices and `/cursor` + replies post through `send_notice`, so they never count as the model's reply, and a + `/cursor` message settles its own turn without reaching the agent. The adapter does not pick a plan/agent mode itself; a caller selects one through `resolve_session_config`, which reads each session's advertised catalog before the first prompt. The live `backends` lane pins the Cursor CLI and passes `E2E_CURSOR_API_KEY` diff --git a/pyproject.toml b/pyproject.toml index 553030988..443c66df4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ dependencies = [ "band-client-rest==0.0.41", - "band-sdk-core==2.7.0", + "band-sdk-core==2.8.0", "phoenix-channels-python-client>=0.2.4", "python-dotenv>=1.2.2", "pydantic>=2.0", diff --git a/src/band/adapters/anthropic.py b/src/band/adapters/anthropic.py index 2dbe48fec..9787864cd 100644 --- a/src/band/adapters/anthropic.py +++ b/src/band/adapters/anthropic.py @@ -453,7 +453,9 @@ async def _process_tool_calls( try: custom_tool = find_custom_tool(self._custom_tools, tool_name) if custom_tool: - result = await execute_custom_tool(custom_tool, tool_input) + result = await execute_custom_tool( + custom_tool, tool_input, turn=tools.turn + ) else: result = await tools.execute_tool_call(tool_name, tool_input) if is_image_passthrough_result(tool_name, result): diff --git a/src/band/adapters/claude_sdk.py b/src/band/adapters/claude_sdk.py index e064086fa..716e9ccca 100644 --- a/src/band/adapters/claude_sdk.py +++ b/src/band/adapters/claude_sdk.py @@ -777,9 +777,6 @@ async def on_message( "ClaudeSDKAdapter session manager not initialized — was on_started() called?" ) - self._room_tools[room_id] = tools - await self._bind_mcp_tools(room_id, tools) - # Approval flow: track notify target and intercept local commands if self.config.approvals is not None: self._room_last_sender[room_id] = { @@ -798,6 +795,7 @@ async def on_message( args=args, sender=sender, ) + tools.turn.settle() return elif cmd == ClaudeSDKCommand.STATUS: await self._handle_status_command( @@ -805,6 +803,7 @@ async def on_message( room_id=room_id, sender=sender, ) + tools.turn.settle() return # A prior turn's background task (see _run_turn) may still be running @@ -814,12 +813,19 @@ async def on_message( # OpencodeAdapter's own "still processing" guard). running_turn = self._turn_tasks.get(room_id) if running_turn is not None and not running_turn.done(): - await tools.send_message( + await tools.send_notice( "Still processing the previous request in this room.", mentions=[msg.sender_id] if msg.sender_id else None, ) + tools.turn.settle() return + # Bound only once this message runs a turn: a command or busy message + # must not take over the in-flight turn's tools, which its approval + # prompts and MCP tool calls still post through. + self._room_tools[room_id] = tools + await self._bind_mcp_tools(room_id, tools) + # The manager only resumes when it has to create the client: on # bootstrap, or after a retired client (see _retire_client). stored_session_id = ( @@ -1721,7 +1727,7 @@ async def _send_best_effort( whether a missing-reply guard still applies. """ try: - await tools.send_message(message, mentions=mentions) + await tools.send_notice(message, mentions=mentions) return True except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn logger.log(log_level, "Room %s: %s: %s", room_id, failure_note, e) @@ -1803,7 +1809,7 @@ async def _resolve_manual_approval( mention = [requester["id"]] if requester else None if tools: try: - await tools.send_message( + await tools.send_notice( APPROVAL_REQUESTED_TEMPLATE.format(summary=summary, token=token), mentions=mention, ) @@ -1932,7 +1938,7 @@ async def _handle_approval_command( # --- /approvals: list pending --- if command == ClaudeSDKCommand.APPROVALS: if not (open_entries := pending.unclaimed()): - await tools.send_message("No pending approvals.", mentions=mention) + await tools.send_notice("No pending approvals.", mentions=mention) return lines = ["Pending approvals:"] now = datetime.now(UTC) @@ -1941,12 +1947,12 @@ async def _handle_approval_command( lines.append( f"- `{entry.token}`: {entry.payload.summary} ({age_s}s ago)" ) - await tools.send_message("\n".join(lines), mentions=mention) + await tools.send_notice("\n".join(lines), mentions=mention) return # --- /approve [token] | /decline [token] --- if not is_authorized_sender(self._approvals.authorized_senders, sender["id"]): - await tools.send_message( + await tools.send_notice( APPROVAL_UNAUTHORIZED_MESSAGE, mentions=mention, ) @@ -1957,12 +1963,12 @@ async def _handle_approval_command( if not token: match open_tokens: case []: - await tools.send_message("No pending approvals.", mentions=mention) + await tools.send_notice("No pending approvals.", mentions=mention) return case [only]: token = only case _: - await tools.send_message( + await tools.send_notice( "Multiple pending approvals — please specify a token: " + format_tokens(open_tokens), mentions=mention, @@ -1970,7 +1976,7 @@ async def _handle_approval_command( return if (selected := pending.get(token)) is None: - await tools.send_message( + await tools.send_notice( APPROVAL_UNKNOWN_TOKEN_TEMPLATE.format( token=token, available=format_tokens(open_tokens) or "none" ), @@ -1979,7 +1985,7 @@ async def _handle_approval_command( return if pending.try_claim(token) is None: - await tools.send_message( + await tools.send_notice( f"Approval `{token}` is no longer pending.", mentions=mention ) return @@ -2015,7 +2021,7 @@ async def _handle_status_command( f"- active_sessions: {session_count}", f"- session_id: `{session_id}`", ] - await tools.send_message("\n".join(lines), mentions=[sender["id"]]) + await tools.send_notice("\n".join(lines), mentions=[sender["id"]]) # ------------------------------------------------------------------ # Approval helpers diff --git a/src/band/adapters/codex.py b/src/band/adapters/codex.py index fb5b58204..90af0af3e 100644 --- a/src/band/adapters/codex.py +++ b/src/band/adapters/codex.py @@ -30,7 +30,8 @@ from band.core.adapterconfig import EnvAdapterConfig from band.core.delivery import ( DeliveryFailedError, - deliver_reply, + deliver_notice, + relay_reply, reraise_delivery_cause, ) from band.core.protocols import ( @@ -71,7 +72,6 @@ ) from band.runtime.custom_tools import ( CustomToolDef, - custom_tool_effects, custom_tool_to_openai_schema, execute_custom_tool, find_custom_tool, @@ -84,7 +84,6 @@ image_block_placeholder, is_image_passthrough_result, redact_tool_call_args, - settles_turn_reply, ) from band.workspaces import ( WorkspaceResolver, @@ -357,7 +356,6 @@ class TurnResult: final_text: str = "" turn_status: str = "failed" turn_error: str = "" - settled_reply: bool = False @dataclass @@ -611,7 +609,6 @@ def __init__( self._custom_tools: list[CustomToolDef] = list(additional_tools or []) if self.config.enable_self_config_tools: self._custom_tools.extend(self._build_self_config_tools()) - self._custom_effects = custom_tool_effects(self._custom_tools) if self.config.cwd is not None: raise ValueError( "cwd is not supported; use workspace_for_room or the default" @@ -865,13 +862,15 @@ async def on_message( except DeliveryFailedError as e: reraise_delivery_cause(e) if handled: + tools.turn.settle() return running = self._turn_tasks.get(room_id) if running is not None and not running.done(): - await tools.send_message( + await tools.send_notice( TURN_IN_PROGRESS_MESSAGE, mentions=self._sender_mention(msg) ) + tools.turn.settle() return release: asyncio.Future[None] = asyncio.get_running_loop().create_future() @@ -935,6 +934,7 @@ async def _run_turn( args=command[1], ) if handled: + tools.turn.settle() return thread_id = await self._ensure_thread( @@ -1090,7 +1090,6 @@ async def _run_turn( turn_status=result.turn_status, turn_error=result.turn_error, final_text=result.final_text, - settled_reply=result.settled_reply, duration_s=_turn_duration_s, ) except DeliveryFailedError as e: @@ -1182,7 +1181,6 @@ async def _emit_failed_turn_outcome( turn_status=result.turn_status, turn_error=result.turn_error, final_text=result.final_text, - settled_reply=result.settled_reply, duration_s=_time.perf_counter() - turn_start, include_reply=False, ) @@ -1211,13 +1209,12 @@ async def _process_turn_events( ) event = await self._client.recv_event(timeout_s=_remaining) if event.kind == "request": - settled_reply_now = await self._handle_server_request( + await self._handle_server_request( tools=tools, msg=msg, room_id=room_id, event=event, ) - result.settled_reply = result.settled_reply or settled_reply_now continue params = event.params if isinstance(event.params, dict) else {} @@ -1902,7 +1899,7 @@ async def _handle_server_request( msg: PlatformMessage, room_id: str, event: RpcEvent, - ) -> bool: + ) -> None: """Dispatch a server-initiated request (tool call, approval). Concurrency model: this coroutine mutates room-scoped adapter state @@ -1921,7 +1918,7 @@ async def _handle_server_request( if self._client is None: raise RuntimeError("CodexAdapter client is None — was on_started() called?") if event.id is None: - return False + return params = event.params if isinstance(event.params, dict) else {} @@ -1931,7 +1928,6 @@ async def _handle_server_request( if not isinstance(arguments, dict): arguments = {} call_id = str(params.get("callId") or "") - tool_call_succeeded = False # Don't emit reporting for codex-local slash commands — they already # surface their outcome in the room themselves. @@ -1957,12 +1953,14 @@ async def _handle_server_request( try: custom_tool = find_custom_tool(self._custom_tools, tool_name) if custom_tool: - result = await execute_custom_tool(custom_tool, arguments) + result = await execute_custom_tool( + custom_tool, arguments, turn=tools.turn + ) success = True else: # Structured: a base tool (e.g. band_send_message) can fail without # raising (bad args, API error) via ok=False; the plain variant would - # report success and wrongly suppress the final-text fallback. + # report that failure to Codex as success. outcome = await tools.execute_tool_call_structured( tool_name, arguments ) @@ -1985,7 +1983,6 @@ async def _handle_server_request( "success": success, }, ) - tool_call_succeeded = success if should_report: await tools.send_event( content=json.dumps( @@ -2041,10 +2038,7 @@ async def _handle_server_request( message_type="tool_result", ) - return ( - settles_turn_reply(tool_name, custom_effects=self._custom_effects) - and tool_call_succeeded - ) + return if event.method in CODEX_APPROVAL_METHODS: await self._handle_approval_request( @@ -2054,14 +2048,13 @@ async def _handle_server_request( event=event, params=params, ) - return False + return await self._client.respond_error( event.id, code=-32601, message=f"Unhandled server request: {event.method}", ) - return False async def _handle_approval_request( self, @@ -2146,7 +2139,7 @@ async def _handle_approval_request( } ] try: - await tools.send_message( + await tools.send_notice( f"Approval requested ({summary}). Policy decision: {decision}.", mentions=mention, ) @@ -2205,7 +2198,6 @@ async def _emit_turn_outcome( turn_status: str, turn_error: str, final_text: str, - settled_reply: bool, duration_s: float = 0.0, include_reply: bool = True, ) -> None: @@ -2294,28 +2286,20 @@ async def _emit_turn_outcome( mention = self._sender_mention(msg) if turn_status == "completed": - if ( - self.config.fallback_send_agent_text - and final_text.strip() - and not settled_reply - ): - await deliver_reply(tools, final_text.strip(), mentions=mention) + if self.config.fallback_send_agent_text: + await relay_reply(tools, final_text.strip(), mentions=mention) return if turn_status == "interrupted": - await deliver_reply( - tools, - "I stopped before completing this request.", - mentions=mention, + outcome_text = "I stopped before completing this request." + elif turn_error: + outcome_text = ( + f"I couldn't complete this request ({turn_status}): {turn_error}" ) - return - - error_text = ( - f"I couldn't complete this request ({turn_status})." - if not turn_error - else f"I couldn't complete this request ({turn_status}): {turn_error}" - ) - await deliver_reply(tools, error_text, mentions=mention) + else: + outcome_text = f"I couldn't complete this request ({turn_status})." + await deliver_notice(tools, outcome_text, mentions=mention) + tools.turn.settle() async def _emit_item_completed_events( self, @@ -2683,7 +2667,7 @@ async def _resolve_manual_approval( log_level=logging.DEBUG, ) try: - await tools.send_message(approval_msg, mentions=mention) + await tools.send_notice(approval_msg, mentions=mention) except Exception: logger.exception( "Failed to notify room %s about pending approval %s", @@ -2715,7 +2699,7 @@ async def _resolve_manual_approval( return decision timeout_decision = self.config.approval_timeout_decision try: - await tools.send_message( + await tools.send_notice( APPROVAL_TIMED_OUT_TEMPLATE.format( token=token, decision=timeout_decision ), @@ -3097,7 +3081,7 @@ async def _handle_local_command( mention = self._sender_mention(msg) if command == CodexCommand.HELP: - await deliver_reply(tools, _HELP_TEXT, mentions=mention) + await deliver_notice(tools, _HELP_TEXT, mentions=mention) return True if command == CodexCommand.STATUS: @@ -3126,13 +3110,13 @@ async def _handle_local_command( f"- token_usage: {usage_line}\n" f"- turn_task_markers: {self.config.emit_turn_task_markers}" ) - await deliver_reply(tools, status_text, mentions=mention) + await deliver_notice(tools, status_text, mentions=mention) return True if command in {CodexCommand.MODEL, CodexCommand.MODELS}: model_arg = args.strip() if not model_arg: - await deliver_reply( + await deliver_notice( tools, "Current model: " f"`{self._selected_model or 'unknown'}` " @@ -3152,13 +3136,13 @@ async def _handle_local_command( preview = ", ".join(models[:10]) if len(models) > 10: preview += ", ..." - await deliver_reply( + await deliver_notice( tools, f"Available models ({len(models)}): {preview}", mentions=mention, ) else: - await deliver_reply( + await deliver_notice( tools, "No visible models returned by Codex app-server.", mentions=mention, @@ -3169,7 +3153,7 @@ async def _handle_local_command( state = self._require_active_client_state() state.model_override = model_arg self._selected_model = model_arg - await deliver_reply( + await deliver_notice( tools, f"Model override set to `{model_arg}` for subsequent turns.", mentions=mention, @@ -3182,7 +3166,7 @@ async def _handle_local_command( supported = ", ".join(efforts) if not effort_arg: hint = f"`{model_id}` supports: {supported}. " if efforts else "" - await deliver_reply( + await deliver_notice( tools, f"Current reasoning effort: `{self.config.reasoning_effort or 'default'}`. " f"Summary: `{self.config.reasoning_summary or 'default'}`. " @@ -3193,7 +3177,7 @@ async def _handle_local_command( # A person's typo would otherwise fail every later turn; the # setreasoning tool can't check this because it runs mid-turn. if efforts and effort_arg not in efforts: - await deliver_reply( + await deliver_notice( tools, f"`{model_id}` doesn't support reasoning effort `{effort_arg}`. " f"Supported: {supported}.", @@ -3201,7 +3185,7 @@ async def _handle_local_command( ) return True self._require_active_client_state().reasoning_effort = effort_arg - await deliver_reply( + await deliver_notice( tools, f"Reasoning effort set to `{effort_arg}` for subsequent turns.", mentions=mention, @@ -3211,7 +3195,7 @@ async def _handle_local_command( # --- Phase 1: /sandbox and /permissions commands --- if command == CodexCommand.SANDBOX: if self.config.sandbox_policy is not None: - await deliver_reply( + await deliver_notice( tools, "Cannot override sandbox: a `sandbox_policy` is configured. " "Remove `sandbox_policy` from config to use per-room " @@ -3222,7 +3206,7 @@ async def _handle_local_command( mode_arg = args.strip() if not mode_arg: effective = self._effective_sandbox(room_id) or "default" - await deliver_reply( + await deliver_notice( tools, f"Current sandbox: `{effective}`. " f"Use `/{CodexCommand.SANDBOX} <{_SANDBOX_MODE_CHOICES}>` to change.", @@ -3235,7 +3219,7 @@ async def _handle_local_command( confirm_flag = _SANDBOX_CONFIRM_FLAG in tokens mode_tokens = [tok for tok in tokens if tok != _SANDBOX_CONFIRM_FLAG] if len(mode_tokens) != 1: - await deliver_reply( + await deliver_notice( tools, f"Usage: `/{CodexCommand.SANDBOX} <{_SANDBOX_MODE_CHOICES}> " f"[{_SANDBOX_CONFIRM_FLAG}]`.", @@ -3245,7 +3229,7 @@ async def _handle_local_command( mode_token = mode_tokens[0] normalized = self._normalize_sandbox_mode(mode_token) if normalized is None: - await deliver_reply( + await deliver_notice( tools, f"Invalid sandbox mode `{mode_token}`. " f"Valid: {', '.join(CodexSandboxMode)}.", @@ -3254,7 +3238,7 @@ async def _handle_local_command( return True escalating = normalized == CodexSandboxMode.DANGER_FULL_ACCESS if escalating and not confirm_flag: - await deliver_reply( + await deliver_notice( tools, f"Escalating to `{normalized}` removes all sandbox " f"restrictions. Re-run with `{_SANDBOX_CONFIRM_FLAG}` to proceed:\n" @@ -3271,7 +3255,7 @@ async def _handle_local_command( msg.sender_name or msg.sender_type or "unknown", ) self._sandbox_overrides[room_id] = normalized - await deliver_reply( + await deliver_notice( tools, f"Sandbox mode set to `{normalized}` for subsequent turns in this room.", mentions=mention, @@ -3296,7 +3280,7 @@ async def _handle_local_command( f" - [{entry.timestamp}] {entry.method}: " f"{entry.decision} by {entry.decided_by}" ) - await deliver_reply(tools, "\n".join(lines), mentions=mention) + await deliver_notice(tools, "\n".join(lines), mentions=mention) return True # --- Phase 2: /threads, /thread info, /thread archive --- @@ -3305,7 +3289,7 @@ async def _handle_local_command( if command == CodexCommand.THREADS or not subcommand: # List all room->thread mappings if not self._room_threads: - await deliver_reply( + await deliver_notice( tools, "No active thread mappings.", mentions=mention ) return True @@ -3313,13 +3297,13 @@ async def _handle_local_command( for rid, tid in self._room_threads.items(): current = " (current)" if rid == room_id else "" lines.append(f"- room `{rid}` → thread `{tid}`{current}") - await deliver_reply(tools, "\n".join(lines), mentions=mention) + await deliver_notice(tools, "\n".join(lines), mentions=mention) return True if subcommand == CodexSubcommand.INFO: mapped_thread = self._room_threads.get(room_id) if not mapped_thread: - await deliver_reply( + await deliver_notice( tools, "No thread mapped for this room.", mentions=mention ) return True @@ -3335,7 +3319,7 @@ async def _handle_local_command( f"- room_id: {room_id}\n" f"- token_usage: {usage_line}" ) - await deliver_reply(tools, info_text, mentions=mention) + await deliver_notice(tools, info_text, mentions=mention) return True if subcommand == CodexSubcommand.ARCHIVE: @@ -3344,7 +3328,7 @@ async def _handle_local_command( self._token_usage.pop(mapped_thread or "", None) self._raw_history_by_room.pop(room_id, None) self._needs_history_injection.discard(room_id) - await deliver_reply( + await deliver_notice( tools, f"Thread `{mapped_thread or 'none'}` archived. " "A new thread will be created on next message.", @@ -3358,7 +3342,7 @@ async def _handle_local_command( if command == CodexCommand.USAGE: mapped_thread = self._room_threads.get(room_id) if not mapped_thread: - await deliver_reply( + await deliver_notice( tools, "No thread mapped for this room — no usage data.", mentions=mention, @@ -3366,13 +3350,13 @@ async def _handle_local_command( return True usage = self._token_usage.get(mapped_thread) if not usage or usage.total_tokens == 0: - await deliver_reply( + await deliver_notice( tools, "No token usage recorded for this thread.", mentions=mention, ) return True - await deliver_reply( + await deliver_notice( tools, f"Thread `{mapped_thread}` — {usage.format_summary()}", mentions=mention, @@ -3395,21 +3379,21 @@ async def _handle_approval_command( if command == CodexCommand.APPROVALS: if not (open_entries := pending.unclaimed()): - await deliver_reply(tools, "No pending approvals.", mentions=mention) + await deliver_notice(tools, "No pending approvals.", mentions=mention) return True lines = ["Pending approvals:"] now = datetime.now(UTC) for entry in open_entries: age_s = int((now - entry.payload.created_at).total_seconds()) lines.append(f"- {entry.token}: {entry.payload.summary} ({age_s}s)") - await deliver_reply(tools, "\n".join(lines), mentions=mention) + await deliver_notice(tools, "\n".join(lines), mentions=mention) return True if (decision_value := _APPROVAL_COMMAND_DECISIONS.get(command)) is None: return False if not pending: - await deliver_reply( + await deliver_notice( tools, NO_APPROVALS_TO_RESOLVE_MESSAGE, mentions=mention, @@ -3421,14 +3405,14 @@ async def _handle_approval_command( if not token: match open_ids: case []: - await deliver_reply( + await deliver_notice( tools, NO_APPROVALS_TO_RESOLVE_MESSAGE, mentions=mention ) return True case [only]: token = only case _: - await deliver_reply( + await deliver_notice( tools, "Multiple approvals pending. " f"Use `/{command} `. Pending: {', '.join(open_ids)}", @@ -3437,7 +3421,7 @@ async def _handle_approval_command( return True if (selected := pending.get(token)) is None: - await deliver_reply( + await deliver_notice( tools, f"Unknown approval id `{token}`. " f"Pending: {', '.join(open_ids) or 'none'}", @@ -3451,7 +3435,7 @@ async def _handle_approval_command( # an empty string in _session_approved or report a misleading # "Future `` requests will be auto-approved" message to the user. if is_session and not selected.session_key: - await deliver_reply( + await deliver_notice( tools, f"Approval `{token}` cannot be resolved as session-level: " "this request has no command signature to match against. " @@ -3461,7 +3445,7 @@ async def _handle_approval_command( return True if pending.try_claim(token) is None: - await deliver_reply( + await deliver_notice( tools, f"Approval `{token}` is no longer pending.", mentions=mention, @@ -3473,7 +3457,7 @@ async def _handle_approval_command( # Session-level: register the session key for auto-approval if is_session: self._record_session_approval(room_id, selected.session_key) - await deliver_reply( + await deliver_notice( tools, APPROVAL_RESOLVED_TEMPLATE.format(token=token, decision=decision_value) + " This session-level approval auto-approves future " @@ -3481,7 +3465,7 @@ async def _handle_approval_command( mentions=mention, ) else: - await deliver_reply( + await deliver_notice( tools, APPROVAL_RESOLVED_TEMPLATE.format(token=token, decision=decision_value), mentions=mention, diff --git a/src/band/adapters/copilot_sdk.py b/src/band/adapters/copilot_sdk.py index 833f9df4f..b6394ecd8 100644 --- a/src/band/adapters/copilot_sdk.py +++ b/src/band/adapters/copilot_sdk.py @@ -26,6 +26,7 @@ from band.core.delivery import ( DeliveryFailedError, deliver_reply, + relay_reply, reraise_delivery_cause, ) from band.core.exceptions import BandConfigError @@ -44,7 +45,6 @@ room_inactive_answer, ) from band.runtime.custom_tools import ( - custom_tool_effects, custom_tools_to_schemas, execute_custom_tool, find_custom_tool, @@ -57,7 +57,6 @@ image_block_placeholder, is_image_passthrough_result, redact_tool_call_args, - settles_turn_reply, ) try: @@ -162,10 +161,6 @@ class TurnState: # Mention target for anything this turn posts to the room: whoever # sent the message that triggered it. sender_mention: dict[str, str] - # True once the turn produced a room message (a Band messaging tool or a - # room-routed ask_user question) — the final text then must not be - # auto-sent on top of it. - replied_in_room: bool = False # Reasoning blocks keyed by reasoning_id: the CLI re-emits a block's # ``assistant.reasoning`` event several times per turn (same id), so keying # by id posts each block once, not 2-3x. Last write wins. @@ -319,9 +314,8 @@ def __init__( self._shared_client = client self._client_factory = client_factory self._custom_tools: list[CustomToolDef] = list(additional_tools or []) - self._custom_effects = custom_tool_effects(self._custom_tools) self._session_manager: CopilotSessionManager | None = None - # Refreshed every on_message; tool handlers resolve through this so + # Rebound when each turn starts; tool handlers resolve through this so # they never stay bound to a stale tools object from an earlier turn. self._room_tools: dict[str, AgentToolsProtocol] = {} self._session_ids: dict[str, RoomSessionIds] = {} @@ -434,13 +428,15 @@ async def on_message( "CopilotSDKAdapter session manager not initialized — was on_started() called?" ) - self._room_tools[room_id] = tools if is_session_bootstrap and history.session_id: ids = self._session_ids.setdefault(room_id, RoomSessionIds()) ids.persisted = ids.persisted or history.session_id # Same-session calls must not interleave; other rooms run concurrently. async with self._session_manager.turn_lock(room_id): + # Bound only once this message holds the room, so a queued + # message never redirects the running turn's tool calls. + self._room_tools[room_id] = tools try: session, inject_text = await self._obtain_session( room_id, history, tools, is_session_bootstrap=is_session_bootstrap @@ -459,8 +455,8 @@ async def on_message( inject_text=inject_text, ) - # The session's tool handler records band_send_message calls in - # this slot; safe because the turn lock serializes turns per room. + # The session's ask_user handler reads this slot; safe because the + # turn lock serializes turns per room. turn = TurnState( sender_mention={ "id": msg.sender_id, @@ -490,7 +486,7 @@ async def on_message( # Session errors raise out of send_and_wait, so a None here # with no room output means the model genuinely said nothing. - if final_text is None and not turn.replied_in_room: + if final_text is None and not tools.turn.replied: # Incidents carry session text, so they stay in this one log line # and out of the error the runtime logs and reports to the platform. logger.warning( @@ -501,15 +497,10 @@ async def on_message( await tools.send_failure(AgentFailure(_PROVIDER, "no assistant reply")) raise RuntimeError("Copilot turn produced no reply") - # The turn may already have replied into the room; sending its - # final text too would duplicate the reply. - if final_text and not turn.replied_in_room: - try: - await deliver_reply( - tools, final_text, mentions=[turn.sender_mention] - ) - except DeliveryFailedError as e: - reraise_delivery_cause(e) + try: + await relay_reply(tools, final_text, mentions=[turn.sender_mention]) + except DeliveryFailedError as e: + reraise_delivery_cause(e) await self._persist_session_id(room_id, tools) @@ -674,30 +665,19 @@ async def _deliver_question_to_room( # cancels pending asks) must degrade, not crash the RPC. return room_inactive_answer() rendered = render_room_question(request) + # The question is the turn's reply, so a "waiting for your answer" + # wrap-up is not relayed on top of it. try: - await room_tools.send_message(rendered, mentions=[turn.sender_mention]) - except Exception as exc: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn + await deliver_reply(room_tools, rendered, mentions=[turn.sender_mention]) + except DeliveryFailedError as exc: logger.warning( - "Room %s: ask_user question delivery failed: %s", room_id, exc + "Room %s: ask_user question delivery failed: %s", room_id, exc.cause ) - return delivery_failed_answer(exc) - # The question is this turn's reply; suppress the final-text - # fallback so a "waiting for your answer" wrap-up can't shadow it. - self._mark_replied_in_room(room_id, turn) + return delivery_failed_answer(exc.cause) # The ack echoes the rendered form so the model knows exactly what # the user sees — e.g. that a bare "2" means numbered choice 2. return question_delivered_answer(rendered) - def _mark_replied_in_room(self, room_id: str, turn: TurnState) -> None: - """Record that ``turn`` produced a room message. - - Guarded by identity: an operation orphaned by a turn timeout (the - SDK never cancels in-flight dispatches) must not mark a LATER turn - as having replied. - """ - if self._turn_state.get(room_id) is turn: - turn.replied_in_room = True - # --- Tool bridging ------------------------------------------------------- def _build_bridged_tools( @@ -760,10 +740,9 @@ async def _execute_bridged_tool( invocation.arguments if isinstance(invocation.arguments, dict) else {} ) # Resolve at call time: sessions outlive any single message, and a - # fresh AgentToolsProtocol arrives with every on_message. The turn - # is captured here (before any await) so a call orphaned by a turn - # timeout can never mark a LATER turn as having replied. - turn = self._turn_state.get(room_id) + # fresh AgentToolsProtocol arrives with every on_message. Captured + # before any await, so a call orphaned by a turn timeout records its + # effect on its own turn's ledger, never a LATER turn's. room_tools = self._room_tools.get(room_id) if room_tools is None: return ToolResult( @@ -779,13 +758,13 @@ async def _execute_bridged_tool( try: custom_tool = find_custom_tool(self._custom_tools, tool_name) if custom_tool: - result = await execute_custom_tool(custom_tool, arguments) + result = await execute_custom_tool( + custom_tool, arguments, turn=room_tools.turn + ) else: # Structured variant: a base tool (e.g. band_send_message) can fail # without raising (bad args, API error) — that surfaces as ok=False, - # not an exception. Treat it as a failure so the turn is NOT marked - # replied and the final-text fallback still fires (avoids a silent - # turn). Mirrors the Slack adapter. + # not an exception, and must reach the model as a failed call. outcome = await room_tools.execute_tool_call_structured( tool_name, arguments ) @@ -833,11 +812,6 @@ async def _execute_bridged_tool( text_result = ( result if isinstance(result, str) else json.dumps(result, default=str) ) - if ( - settles_turn_reply(tool_name, custom_effects=self._custom_effects) - and turn is not None - ): - self._mark_replied_in_room(room_id, turn) if should_report: await self._report_tool_result(room_tools, invocation, text_result) return ToolResult( diff --git a/src/band/adapters/crewai_flow.py b/src/band/adapters/crewai_flow.py index c44c9b756..6a5571dac 100644 --- a/src/band/adapters/crewai_flow.py +++ b/src/band/adapters/crewai_flow.py @@ -597,7 +597,9 @@ async def call_tool( input_data = dict(arguments or kwargs) await self._report_call(name, input_data) try: - result = await execute_custom_tool(tool, input_data) + result = await execute_custom_tool( + tool=tool, arguments=input_data, turn=self._tools.turn + ) except Exception as exc: await self._report_result(name, str(exc), is_error=True) raise diff --git a/src/band/adapters/cursor_acp.py b/src/band/adapters/cursor_acp.py index 435cb2b1a..87fdf70e3 100644 --- a/src/band/adapters/cursor_acp.py +++ b/src/band/adapters/cursor_acp.py @@ -199,6 +199,7 @@ async def on_message( room_id: str, ) -> None: if await self._handle_control_message(msg, tools, room_id): + tools.turn.settle() return # ext_method routing needs _turn_lock held for the whole turn (only @@ -482,7 +483,7 @@ async def _wait_for_decision( self._resolve_evicted_decision(registration.evicted) entry = registration.entry try: - await turn.tools.send_message( + await turn.tools.send_notice( prompt.replace("{token}", entry.token), mentions=_requester_mentions(turn), ) @@ -511,7 +512,7 @@ async def _notify_decision_timeout( turn: CursorTurn, kind: DecisionKind, token: str ) -> None: try: - await turn.tools.send_message( + await turn.tools.send_notice( DECISION_TIMED_OUT_TEMPLATE.format(kind=kind, token=token), mentions=_requester_mentions(turn), ) @@ -529,7 +530,7 @@ async def _handle_control_message( await self._list_decisions(tools, room_id, mentions=mentions) return True if len(words) < 3: - await tools.send_message( + await tools.send_notice( f"Use `{ROOM_COMMAND} {CursorCommandWord.DECISIONS}` to list pending " "Cursor decisions.", mentions=mentions, @@ -538,13 +539,13 @@ async def _handle_control_message( action, token = words[1].lower(), words[2] pending = self._pending_decisions.get(token) if pending is None or pending.room_id != room_id: - await tools.send_message( + await tools.send_notice( DECISION_NOT_PENDING_TEMPLATE.format(token=token), mentions=mentions ) return True result = self._command_result(action, words[3:], pending) if result is _INVALID_DECISION: - await tools.send_message( + await tools.send_notice( f"That command is not valid for Cursor {pending.kind} decision `{token}`.", mentions=mentions, ) @@ -558,7 +559,7 @@ async def _handle_control_message( else: pending.future.set_result(result) reply = DECISION_RESOLVED_TEMPLATE.format(kind=pending.kind, token=token) - await tools.send_message(reply, mentions=mentions) + await tools.send_notice(reply, mentions=mentions) return True def _command_result( @@ -678,7 +679,7 @@ async def _list_decisions( for entry in self._pending_decisions.unclaimed_in_room(room_id) ] content = "Pending Cursor decisions: " + (", ".join(pending) or "none") - await tools.send_message(content, mentions=mentions) + await tools.send_notice(content, mentions=mentions) def _requester_mentions(turn: CursorTurn) -> list[str] | None: diff --git a/src/band/adapters/gemini.py b/src/band/adapters/gemini.py index 0ca2719b5..8c9176835 100644 --- a/src/band/adapters/gemini.py +++ b/src/band/adapters/gemini.py @@ -545,7 +545,9 @@ async def _process_function_calls( try: custom_tool = find_custom_tool(self._custom_tools, tool_name) if custom_tool: - result = await execute_custom_tool(custom_tool, tool_input) + result = await execute_custom_tool( + custom_tool, tool_input, turn=tools.turn + ) else: result = await tools.execute_tool_call(tool_name, tool_input) if is_image_passthrough_result(tool_name, result): diff --git a/src/band/adapters/google_adk.py b/src/band/adapters/google_adk.py index 4921ad7ca..79727dca0 100644 --- a/src/band/adapters/google_adk.py +++ b/src/band/adapters/google_adk.py @@ -249,7 +249,9 @@ async def run_async( try: custom_tool = find_custom_tool(self._custom_tools, self.name) if custom_tool: - result = await execute_custom_tool(custom_tool, args) + result = await execute_custom_tool( + custom_tool, args, turn=self._tools.turn + ) else: result = await self._tools.execute_tool_call(self.name, args) diff --git a/src/band/adapters/langgraph.py b/src/band/adapters/langgraph.py index ac6577beb..ed029c476 100644 --- a/src/band/adapters/langgraph.py +++ b/src/band/adapters/langgraph.py @@ -19,6 +19,7 @@ from band.core.adapterconfig import BaseAdapterConfig from band.core.protocols import GENERIC_PROVIDER_FAILURE_MESSAGE, AgentToolsProtocol from band.core.simple_adapter import SimpleAdapter +from band.core.turn import Turn from band.core.types import ( Capability, Emit, @@ -175,25 +176,6 @@ def __init__( ) self.config = config or LangGraphAdapterConfig() - # Accept the SDK's portable custom-tool form: convert any CustomToolDef - # (InputModel, handler) tuples in additional_tools to LangChain tools — the - # same shape every other adapter takes — while passing ready-made LangChain - # tools through untouched. Done once here so both the simple and advanced - # patterns get a uniform tool list, and a tool written once works across - # adapters (LangChain would otherwise reject a bare tuple). - if additional_tools: - normalized: list[Any] = [] - for item in additional_tools: - if isinstance( - item, tuple - ): # a band CustomToolDef (InputModel, handler) - normalized.extend( - langchain_tools.custom_tool_defs_to_langchain([item]) - ) - else: # already a LangChain tool / callable - normalized.append(item) - additional_tools = normalized - uses_simple_pattern = ( llm is not None and graph_factory is None and graph is None ) @@ -213,19 +195,14 @@ def __init__( if checkpointer is None: checkpointer = InMemorySaver() - additional = additional_tools or [] - - def factory(band_tools: list[Any]) -> Pregel: - all_tools = band_tools + additional + def factory(turn_tools: list[Any]) -> Pregel: return create_agent( model=llm, - tools=all_tools, + tools=turn_tools, checkpointer=checkpointer, ) graph_factory = factory - # Clear additional_tools since they're now baked into the factory - additional_tools = [] if not graph_factory and not graph: raise ValueError( @@ -249,6 +226,20 @@ def factory(band_tools: list[Any]) -> Pregel: # top of the checkpointer's already-stored state. self._bootstrapped_rooms: OrderedDict[str, None] = OrderedDict() + def _additional_tools_for_turn(self, turn: Turn) -> list[Any]: + """The caller's extra tools as LangChain tools for one room turn. + + Portable ``CustomToolDef`` tuples are converted here, per turn, so each + call records its effect on that room's turn; ready-made LangChain tools + pass through untouched. + """ + return [ + langchain_tools.custom_tool_def_to_langchain(tool, turn=turn) + if isinstance(tool, tuple) + else tool + for tool in self.additional_tools + ] + async def on_started(self, agent_name: str, agent_description: str) -> None: """Render system prompt after agent metadata is fetched.""" await super().on_started(agent_name, agent_description) @@ -325,14 +316,10 @@ async def on_message( track_usage = Emit.USAGE in self.features.emit turn_usage = TurnUsage() try: - # Get LangChain tools - lc_tools = ( - langchain_tools.agent_tools_to_langchain( - tools, - features=self.features, - ) - + self.additional_tools - ) + lc_tools = langchain_tools.agent_tools_to_langchain( + tools, + features=self.features, + ) + self._additional_tools_for_turn(tools.turn) # Build or get graph if self.graph_factory: diff --git a/src/band/adapters/letta.py b/src/band/adapters/letta.py index c8c96b498..b3f27aa85 100644 --- a/src/band/adapters/letta.py +++ b/src/band/adapters/letta.py @@ -15,7 +15,7 @@ from band.converters.letta import LettaHistoryConverter, LettaSessionState from band.core.delivery import ( DeliveryFailedError, - deliver_reply, + relay_reply, reraise_delivery_cause, ) from band.core.protocols import ( @@ -49,6 +49,7 @@ iter_tool_definitions, redact_tool_call_args, settles_turn_reply, + turn_effect, ) __all__ = [ @@ -602,7 +603,8 @@ async def _process_response_messages( }, ) case "tool_return_message": - tool_name = getattr(resp_msg, "tool_name", "unknown") + tool_name = getattr(resp_msg, "name", None) or "unknown" + self._record_external_tool_return(tools, tool_name, resp_msg) await self._report_execution_event( tools, "tool_result", @@ -648,10 +650,21 @@ async def _process_response_messages( room_id, self._mcp.send_message_tool, ) - await deliver_reply(tools, final_text, mentions=mentions) + await relay_reply(tools, final_text, mentions=mentions) return final_text_parts + def _record_external_tool_return( + self, tools: AgentToolsProtocol, tool_name: str, resp_msg: Any + ) -> None: + """Record an external band-mcp call's effect on the turn. + + A self-hosted call runs on this room's own tools, which record it + themselves; an external server's call is visible only in the stream. + """ + if self.config.mcp.mode == "external" and resp_msg.status == "success": + tools.turn.record(turn_effect(tool_name)) + async def _report_execution_event( self, tools: AgentToolsProtocol, diff --git a/src/band/adapters/opencode/adapter.py b/src/band/adapters/opencode/adapter.py index 404de56bd..cbc70d5c9 100644 --- a/src/band/adapters/opencode/adapter.py +++ b/src/band/adapters/opencode/adapter.py @@ -19,6 +19,7 @@ from band.adapters.opencode.approvals import ApprovalPorts, RoomApprovals from band.adapters.opencode.config import OpencodeAdapterConfig from band.converters.opencode import OpencodeHistoryConverter +from band.core.delivery import relay_reply from band.core.exceptions import BandConnectionError from band.core.protocols import ( FAILURE_CODE_TIMEOUT, @@ -58,17 +59,9 @@ describe_error, parse_opencode_event, ) -from band.runtime.custom_tools import ( - CustomToolDef, - custom_tool_effects, - get_custom_tool_name, -) +from band.runtime.custom_tools import CustomToolDef, get_custom_tool_name from band.runtime.prompts import render_system_prompt -from band.runtime.tools import ( - CHAT_ID_FIELD_NAME, - iter_tool_definitions, - settles_turn_reply, -) +from band.runtime.tools import CHAT_ID_FIELD_NAME, iter_tool_definitions logger = logging.getLogger(__name__) @@ -105,9 +98,6 @@ class TurnState: assistant_part_types: dict[str, str] = field(default_factory=dict) reported_tool_calls: set[str] = field(default_factory=set) reported_tool_results: set[str] = field(default_factory=set) - # Set when a room-posting band tool (band_send_message) completed this turn, - # so the text fallback stays silent instead of double-posting the reply. - replied_via_room_tool: bool = False last_error_message: str | None = None # Per-assistant-message usage for the current turn (last-write-wins per id, # since message.updated streams repeatedly). Summed across messages at turn @@ -285,7 +275,6 @@ def __init__( # concurrent agents sharing one serve. self._mcp_server_name = self._config.mcp_server_name self._custom_tools: list[CustomToolDef] = list(additional_tools or []) - self._custom_effects = custom_tool_effects(self._custom_tools) # Startup reachability check only makes sense against a real server; # an injected factory fakes that boundary (tests, custom transports). self._preflight_enabled = client_factory is None @@ -303,8 +292,8 @@ def __init__( # custom tools). Computed once at construction -- both inputs are known # here -- and reused when the shared MCP backend is built. Deriving the # names eagerly keeps the "is this our own band tool?" auto-approve - # check (and room-posting detection) independent of MCP-registration - # timing, so a second room's first turn can't race an empty set. + # check independent of MCP-registration timing, so a second room's + # first turn can't race an empty set. self._refresh_tool_definitions() def _refresh_tool_definitions(self) -> None: @@ -428,7 +417,6 @@ async def on_message( room_id: str, ) -> None: room_state = await self._get_or_create_room_state(room_id) - room_state.tools = tools if self._client is None: agent_id = getattr(tools, "agent_id", None) @@ -437,6 +425,7 @@ async def on_message( ) if await room_state.approvals.try_handle_reply(msg.content, msg.sender_id): + tools.turn.settle() return if room_state.turn and ( @@ -447,8 +436,14 @@ async def on_message( "OpenCode is still processing the previous request in this room.", "error", ) + tools.turn.settle() return + # Only a message that starts a turn rebinds the room's tools: the MCP + # backend resolves band tool calls through them, so a message consumed + # above must not take over the in-flight turn's tools. + room_state.tools = tools + await self._ensure_client_started() client = self._client if client is None: @@ -908,32 +903,13 @@ async def _handle_part_update( async def _report_tool_part( self, room_state: RoomState, part: OpencodePart ) -> None: - """Note a room-posting reply and report the tool's call/result. - - Room-posting detection runs regardless of ``Emit.TOOL_CALLS`` (which only - governs the tool_call/tool_result narration); the text-fallback - suppression must hold even when execution reporting is off. - """ - if part.state is None: + """Report the tool's call/result.""" + if part.state is None or Emit.TOOL_CALLS not in self.features.emit: return state = part.state tool_name = self._canonical_tool_name(part.tool or "unknown") - # A completed room-posting band tool IS the turn's reply -- suppress the - # text fallback (codex/copilot_sdk/ACP parity). An errored call did not - # post, so it must not suppress. ``status`` is the raw wire string, so - # compare by value (the StrEnum member equals its string). - if ( - state.status == OpencodeToolStatus.COMPLETED - and settles_turn_reply(tool_name, custom_effects=self._custom_effects) - and room_state.turn is not None - ): - room_state.turn.replied_via_room_tool = True - - if Emit.TOOL_CALLS not in self.features.emit: - return - match state.status: case ( OpencodeToolStatus.PENDING @@ -1263,26 +1239,23 @@ async def _deliver_fallback_text(self, room_id: str, turn: TurnState) -> None: # (or the test's observer WebSocket) is the fault, not model completion. logger.info( "OpenCode turn: delivering fallback room=%s " - "(text=%d chars, error=%s, replied_via_tool=%s)", + "(text=%d chars, error=%s, replied=%s)", room_id, len(text), bool(turn.last_error_message), - turn.replied_via_room_tool, + turn.tools.turn.replied, ) - # A room-posting band tool already delivered the reply; don't double-post - # its plain text or a "no reply" filler. An error is still surfaced -- - # it is not a text reply. - replied = turn.replied_via_room_tool + # An error is still surfaced after a tool reply -- it is not a text reply. try: - if text and not replied: - await turn.tools.send_message(text, mentions=turn.pending_mentions) - elif turn.last_error_message: + if await relay_reply(turn.tools, text, turn.pending_mentions): + return + if turn.last_error_message: await turn.tools.send_failure( AgentFailure(_PROVIDER, turn.last_error_message) ) - elif not replied: - await turn.tools.send_message( + elif not turn.tools.turn.replied: + await turn.tools.send_notice( NO_TEXT_REPLY_MESSAGE, mentions=turn.pending_mentions, ) diff --git a/src/band/adapters/opencode/approvals.py b/src/band/adapters/opencode/approvals.py index fd39fa343..897c429a6 100644 --- a/src/band/adapters/opencode/approvals.py +++ b/src/band/adapters/opencode/approvals.py @@ -568,7 +568,10 @@ def _which_dual_reject_hint(self) -> str: ) async def _notify_room(self, text: str, mentions: list[dict[str, str]]) -> None: - """Post a room message best-effort. + """Post an adapter notice best-effort. + + A notice, never the turn's reply: it posts on the in-flight turn's + tools, and counting it would stop that turn relaying the model's answer. A send failure must never strand the turn or crash the SSE event loop: the platform requires at least one mention, so a sender-less turn (no @@ -578,7 +581,7 @@ async def _notify_room(self, text: str, mentions: list[dict[str, str]]) -> None: if (tools := self._ports.tools()) is None: return try: - await tools.send_message(text, mentions=mentions) + await tools.send_notice(text, mentions=mentions) except Exception: logger.exception( "Failed to post approval message to room %s", self._ports.room_id diff --git a/src/band/adapters/parlant.py b/src/band/adapters/parlant.py index 8f3a555a7..27e46bb56 100644 --- a/src/band/adapters/parlant.py +++ b/src/band/adapters/parlant.py @@ -22,7 +22,7 @@ from band.core.adapterconfig import BaseAdapterConfig from band.core.delivery import ( DeliveryFailedError, - deliver_reply, + relay_reply, reraise_delivery_cause, ) from band.core.protocols import GENERIC_PROVIDER_FAILURE_MESSAGE, AgentToolsProtocol @@ -823,10 +823,10 @@ async def _process_agent_response( room_id, message_content[:100], ) - await deliver_reply( + if await relay_reply( tools, message_content, mentions=[sender_name] - ) - logger.info("Room %s: Message sent successfully", room_id) + ): + logger.info("Room %s: Message sent successfully", room_id) else: logger.warning( "Room %s: Empty message content in event", diff --git a/src/band/adapters/pydantic_ai.py b/src/band/adapters/pydantic_ai.py index be2fc5ecf..4c78b5602 100644 --- a/src/band/adapters/pydantic_ai.py +++ b/src/band/adapters/pydantic_ai.py @@ -162,8 +162,9 @@ def _custom_tool_def_to_callable(tool_def: CustomToolDef) -> Callable[..., Any]: tool callable — the same custom-tool form the other adapters accept. pydantic-ai flattens a single Pydantic-model parameter into the tool's arguments, - so the wrapper keeps the ``(args: InputModel)`` registration shape. Execution is - routed through the shared ``invoke_validated_custom_tool`` so the CustomToolDef + so the wrapper keeps the ``(args: InputModel)`` registration shape after a + leading ``RunContext``, whose deps are the room's tools, so each call records + its effect on that room's turn. Execution is routed through the shared ``invoke_validated_custom_tool`` so the CustomToolDef contract matches every other adapter: async handlers are awaited and zero-argument handlers (empty InputModel) are called without args — a plain sync passthrough would hand pydantic-ai an unawaited coroutine or raise TypeError for @@ -175,12 +176,16 @@ def _custom_tool_def_to_callable(tool_def: CustomToolDef) -> Callable[..., Any]: """ input_model, handler = tool_def - async def native(args: Any) -> Any: - return await invoke_validated_custom_tool(tool_def, args) + async def native(ctx: RunContext[AgentToolsProtocol], args: Any) -> Any: + return await invoke_validated_custom_tool(tool_def, args, turn=ctx.deps.turn) native.__name__ = get_custom_tool_name(input_model) native.__doc__ = input_model.__doc__ or native.__name__ - native.__annotations__ = {"args": input_model, "return": str} + native.__annotations__ = { + "ctx": RunContext[AgentToolsProtocol], + "args": input_model, + "return": str, + } if (effect := declared_effect(handler)) is not None: declares_turn_effect(effect)(native) return native @@ -190,8 +195,8 @@ def _takes_run_context(fn: Callable[..., Any]) -> bool: """Whether ``fn`` takes pydantic-ai's ``RunContext`` as its first parameter. Decides the registration path: ``agent.tool`` handles RunContext-first - callables, while ``agent.tool_plain`` handles context-free ones — the shape - ``_custom_tool_def_to_callable`` produces. pydantic-ai injects an unannotated + callables (including what ``_custom_tool_def_to_callable`` produces), while + ``agent.tool_plain`` handles context-free ones. pydantic-ai injects an unannotated first parameter as context; a non-RunContext annotation is invalid on the former path. Annotations are resolved, so a caller using ``from __future__ import diff --git a/src/band/adapters/strands.py b/src/band/adapters/strands.py index cd84533ee..2bdb94fee 100644 --- a/src/band/adapters/strands.py +++ b/src/band/adapters/strands.py @@ -42,6 +42,7 @@ ) from band.core.simple_adapter import SimpleAdapter from band.core.tool_filter import filter_tool_schemas +from band.core.turn import Turn from band.core.types import ( Capability, Emit, @@ -179,8 +180,13 @@ def _input_schema(input_model: type[BaseModel]) -> dict[str, Any]: return schema -def _registered_name(tool: AgentTool | Callable[..., Any]) -> str: +StrandsCustomTool = Callable[..., Any] | CustomToolDef + + +def _registered_name(tool: StrandsCustomTool) -> str: """Return the name Strands registers this tool under.""" + if isinstance(tool, tuple): + return get_custom_tool_name(tool[0]) if isinstance(tool, AgentTool): return tool.tool_name name = getattr(tool, "__name__", "") @@ -192,24 +198,29 @@ def _registered_name(tool: AgentTool | Callable[..., Any]) -> str: return name -def _build_custom_tools( - additional_tools: list[Callable[..., Any] | CustomToolDef] | None, -) -> tuple[list[AgentTool | Callable[..., Any]], dict[str, TurnEffect]]: - """Adapt portable custom tools and collect the turn effects they declared.""" - raw_tools = additional_tools or [] - converted: list[AgentTool | Callable[..., Any]] = [ - CustomToolBridge(tool_def) if isinstance(tool_def, tuple) else tool_def - for tool_def in raw_tools - ] - names = [_registered_name(tool) for tool in converted] +def _custom_tool_effects( + custom_tools: list[StrandsCustomTool], +) -> dict[str, TurnEffect]: + """Validate custom tool names and collect the turn effects they declared.""" + names = [_registered_name(tool) for tool in custom_tools] # Strands' registry is last-wins, so a collision would silently replace the # platform tool the room depends on. shadowed = sorted(set(names) & ALL_TOOL_NAMES) if shadowed: raise ValueError(f"Custom tools may not shadow Band platform tools: {shadowed}") - handlers = (raw[1] if isinstance(raw, tuple) else raw for raw in raw_tools) - return converted, declared_effects(zip(names, handlers, strict=True)) + handlers = (tool[1] if isinstance(tool, tuple) else tool for tool in custom_tools) + return declared_effects(zip(names, handlers, strict=True)) + + +def _bind_custom_tools( + custom_tools: list[StrandsCustomTool], turn: Turn +) -> list[AgentTool | Callable[..., Any]]: + """Adapt portable custom tools to Strands for one room turn.""" + return [ + CustomToolBridge(tool, turn) if isinstance(tool, tuple) else tool + for tool in custom_tools + ] class StrandsToolBridge(AgentTool): @@ -245,8 +256,9 @@ def tool_type(self) -> str: class CustomToolBridge(StrandsToolBridge): """Expose a portable custom tool through Strands' native tool protocol.""" - def __init__(self, tool_def: CustomToolDef): + def __init__(self, tool_def: CustomToolDef, turn: Turn): self._tool_def = tool_def + self._turn = turn input_model, _ = tool_def name = get_custom_tool_name(input_model) super().__init__(name, input_model, input_model.__doc__ or name) @@ -260,7 +272,7 @@ async def stream( del invocation_state, kwargs try: result = await execute_custom_tool( - self._tool_def, dict(tool_use["input"] or {}) + self._tool_def, dict(tool_use["input"] or {}), turn=self._turn ) except Exception as error: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn yield _tool_result( @@ -422,7 +434,7 @@ def __init__( config: StrandsAdapterConfig | None = None, *, history_converter: StrandsHistoryConverter | None = None, - additional_tools: list[Callable[..., Any] | CustomToolDef] | None = None, + additional_tools: list[StrandsCustomTool] | None = None, llm: Model | None = None, **features: Unpack[FeatureKwargs], ) -> None: @@ -448,7 +460,8 @@ def __init__( self._model: Model | str = model self._system_prompt: str | None = None self._message_history: dict[str, StrandsMessages] = {} - self._custom_tools, self._custom_effects = _build_custom_tools(additional_tools) + self._custom_tools = additional_tools or [] + self._custom_effects = _custom_tool_effects(self._custom_tools) async def on_started(self, agent_name: str, agent_description: str) -> None: """Render the prompt after the platform supplies agent metadata.""" @@ -474,7 +487,9 @@ def _build_agent( overflow, keeping toolUse/toolResult pairs intact. The persisted room transcript is therefore capped by the framework, not unbounded. """ - framework_tools = self._build_platform_tools(tools) + self._custom_tools + framework_tools = self._build_platform_tools(tools) + _bind_custom_tools( + self._custom_tools, tools.turn + ) return Agent( model=self._model, messages=messages, diff --git a/src/band/core/delivery.py b/src/band/core/delivery.py index 11a79203f..eb2269db7 100644 --- a/src/band/core/delivery.py +++ b/src/band/core/delivery.py @@ -12,6 +12,8 @@ import logging from typing import TYPE_CHECKING, Any, NoReturn +from band.core.content import has_visible_content + if TYPE_CHECKING: from band.core.protocols import AgentToolsProtocol @@ -43,10 +45,36 @@ async def deliver_reply( content: str, mentions: list[str] | list[dict[str, str]] | None = None, ) -> Any: - """Send a reply, raising ``DeliveryFailedError`` on failure instead of - the raw exception, so the caller's except can distinguish a delivery - failure from a provider failure.""" + """Send the model's words as the turn's reply, raising + ``DeliveryFailedError`` on failure instead of the raw exception, so the + caller's except can distinguish a delivery failure from a provider + failure.""" try: return await tools.send_message(content, mentions=mentions) except Exception as exc: raise DeliveryFailedError(exc) from exc + + +async def deliver_notice( + tools: AgentToolsProtocol, + content: str, + mentions: list[str] | list[dict[str, str]] | None = None, +) -> Any: + """Post the adapter's own message via ``send_notice`` (never the turn's + reply), raising ``DeliveryFailedError`` on failure like ``deliver_reply``.""" + try: + return await tools.send_notice(content, mentions=mentions) + except Exception as exc: + raise DeliveryFailedError(exc) from exc + + +async def relay_reply( + tools: AgentToolsProtocol, + text: str | None, + mentions: list[str] | list[dict[str, str]] | None = None, +) -> bool: + """Relay the model's final text unless a tool call already replied or + declined this turn; return whether it posted.""" + if text is None or tools.turn.replied or not has_visible_content(text): + return False + return await deliver_reply(tools, text, mentions) is not None diff --git a/src/band/core/protocols.py b/src/band/core/protocols.py index 159f4ba05..aabb36348 100644 --- a/src/band/core/protocols.py +++ b/src/band/core/protocols.py @@ -29,6 +29,7 @@ TaskLifecycleState, TaskListState, ) + from band.core.turn import Turn from band.core.types import AgentInput, Capability from band.platform.event import PlatformEvent from band.runtime.execution import ExecutionContext @@ -53,6 +54,11 @@ ) +# ``AgentFailure.provider`` for a turn failure the runtime reports on the +# adapter's behalf. +TURN_FAILURE_PROVIDER = "band-runtime" + + class FailureMetadataKey(StrEnum): """Room metadata key used by the shared failure event contract.""" @@ -150,10 +156,19 @@ class AgentToolsProtocol(Protocol): Implementations: AgentTools (default), FakeAgentTools (testing) """ + turn: Turn + """This turn's ledger; every tool call records its effect here.""" + async def send_message( self, content: str, mentions: list[str] | list[dict[str, str]] | None = None ) -> Any: - """Send a message to the chat room.""" + """Send a message to the chat room. It counts as the turn's reply.""" + ... + + async def send_notice( + self, content: str, mentions: list[str] | list[dict[str, str]] | None = None + ) -> Any: + """Post the adapter's own message; it never counts as the turn's reply.""" ... async def send_event( diff --git a/src/band/core/simple_adapter.py b/src/band/core/simple_adapter.py index fc32a5d1e..cc508a795 100644 --- a/src/band/core/simple_adapter.py +++ b/src/band/core/simple_adapter.py @@ -371,8 +371,7 @@ async def on_event(self, inp: AgentInput) -> None: that calls the per-adapter ``on_message`` override), so this is where the turn's trace-context correlation window opens -- every log line emitted anywhere during this turn's processing picks it up via - ``band.logging_config``'s log filter. (``runtime/oneshot.py`` is a - separate, non-adapter delivery path and does not go through here.) + ``band.logging_config``'s log filter. """ with trace_context_scope(): # Convert history if converter is set diff --git a/src/band/core/turn.py b/src/band/core/turn.py new file mode 100644 index 000000000..3cd1cfc29 --- /dev/null +++ b/src/band/core/turn.py @@ -0,0 +1,80 @@ +"""One turn's outcome, judged by band-sdk-core's turn-outcome rule. + +A turn completes when it replied, declined via ``band_no_reply``, did real work, +was settled by the adapter itself, or already reported a failure. Anything else +is a missing reply, reported once and marked FAILED. The rule and its text live +in core so every SDK judges a turn the same way. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +import band_sdk_core +from band_sdk_core import AgentFailure, TurnLedger, TurnVerdict + +from band.core.protocols import TURN_FAILURE_PROVIDER + +if TYPE_CHECKING: + from band.core.protocols import AgentToolsProtocol + from band.runtime.tools.types import TurnEffect + +logger = logging.getLogger(__name__) + + +class Turn: + """The ledger of what one turn's tool calls did. + + ``judged`` is set by ``SimpleAdapter.on_event`` for a turn it will judge. + ``detach()`` marks a turn released early (parked on a human decision), so + the adapter judges it at its real end instead of ``on_event``. + """ + + def __init__(self) -> None: + self._ledger = TurnLedger() + self.judged = False + self._detached = False + + def record(self, effect: TurnEffect) -> None: + core_effect = band_sdk_core.TurnEffect.from_wire_name(effect) + if core_effect is None: + raise ValueError(f"band-sdk-core has no turn effect {effect!r}") + self._ledger.record(core_effect) + + def settle(self) -> None: + """The adapter ended this turn itself (a control reply, a busy notice).""" + self._ledger.settle() + + def note_reported(self) -> None: + """A failure for this turn already reached the room.""" + self._ledger.note_reported() + + def detach(self) -> None: + self._detached = True + + @property + def detached(self) -> bool: + return self._detached + + @property + def replied(self) -> bool: + """The turn replied or declined, so the model's final text is not relayed.""" + return self._ledger.reply_settled() + + @property + def complete(self) -> bool: + return self._ledger.verdict() == TurnVerdict.Complete + + +async def report_unsettled_turn(tools: AgentToolsProtocol) -> bool: + """Report a turn that ended without completing; return whether it did. + + The one place a missing-reply verdict becomes a room-visible failure. + """ + if tools.turn.complete: + return False + await tools.send_failure( + AgentFailure(TURN_FAILURE_PROVIDER, band_sdk_core.missing_reply_message()) + ) + return True diff --git a/src/band/core/types.py b/src/band/core/types.py index 758e22a86..784838310 100644 --- a/src/band/core/types.py +++ b/src/band/core/types.py @@ -456,6 +456,42 @@ def __init__( ) +# --- Constants for synthetic messages (injected by SDK, not from platform) --- +# +# These constants define the sender identity for SDK-generated messages that are +# injected into the event queue but don't originate from the platform WebSocket. +# +# Primary use case: ContactEventHandler's HUB_ROOM strategy creates synthetic +# MessageEvents to route contact events to the hub room for LLM processing. +# These messages need a consistent sender identity that: +# 1. Clearly indicates they're system-generated (not from a real user/agent) +# 2. Can be filtered by preprocessing if needed +# 3. Are recognizable in UI/logs for debugging +# + +# Sender type for synthetic messages. Matches the platform's "System" type used +# for other system-generated content. +SYNTHETIC_SENDER_TYPE = "System" + +# Sender ID for synthetic contact event messages. This is a logical identifier +# (not a UUID) that allows filtering/identification of contact-related synthetic +# messages. Used in MessageEvent.payload.sender_id for hub room injections. +SYNTHETIC_CONTACT_EVENTS_SENDER_ID = "contact-events" + +# Human-readable sender name displayed in UI/logs for synthetic contact event +# messages. Used in MessageEvent.payload.sender_name. +SYNTHETIC_CONTACT_EVENTS_SENDER_NAME = "Contact Events" + + +def is_contact_hub_turn(*, sender_type: str | None, sender_id: str | None) -> bool: + """Whether a message is a synthetic contact event the SDK injected into the + hub room, rather than a platform message addressed to the agent.""" + return ( + sender_type == SYNTHETIC_SENDER_TYPE + and sender_id == SYNTHETIC_CONTACT_EVENTS_SENDER_ID + ) + + @dataclass(frozen=True) class PlatformMessage: """Message from the platform.""" diff --git a/src/band/integrations/acp/client_adapter.py b/src/band/integrations/acp/client_adapter.py index 07621b75d..5c0d12e3c 100644 --- a/src/band/integrations/acp/client_adapter.py +++ b/src/band/integrations/acp/client_adapter.py @@ -99,7 +99,6 @@ from band.integrations.mcp.local_server import LocalMCPServer from band.runtime.custom_tools import ( CustomToolDef, - custom_tool_effects, get_custom_tool_name, ) from band.runtime.formatters import messages_before @@ -386,7 +385,6 @@ def __init__( config.cwd, workspace_for_room ) self._custom_tools: list[CustomToolDef] = list(additional_tools or []) - self._custom_effects = custom_tool_effects(self._custom_tools) self._tool_definitions, self._own_tool_names = self._registered_tools() self._profile = profile self._resolve_session_config = resolve_session_config @@ -544,8 +542,8 @@ def _registered_tools(self) -> tuple[list[ToolDefinition], frozenset[str]]: # is already covered via iter_tool_definitions). Without it, an # external band-mcp's MCP-prefixed legacy call # (band-create_agent_chat_message) would canonicalize to nothing and - # narrate under the raw prefixed name — the one case reply-suppression - # (settles_turn_reply) already tolerates. + # narrate under the raw prefixed name (turn_effect would still + # resolve its effect). | {LEGACY_SEND_MESSAGE_TOOL} ) return definitions, names @@ -668,7 +666,7 @@ async def on_message( # The emitter posts the turn's events live, in the order the ACP stream # delivers them (see RoomTurnEmitter), so narration stays interleaved with # the permission pair and any in-room tool post. On a clean turn its - # __aexit__ relays the held text (if not already posted) and the session + # __aexit__ relays the held text (unless the turn replied) and the session # bookkeeping event; on failure it posts nothing and the error is handled # below. try: @@ -678,7 +676,8 @@ async def on_message( session_id=session_id, room_id=room_id, emit=self.features.emit, - custom_effects=self._custom_effects, + # Injected Band tools record their own effects in process. + records_tool_effects=not self.config.inject_band_tools, ) as emitter: self._install_turn_handlers( runtime, diff --git a/src/band/integrations/acp/room_emitter.py b/src/band/integrations/acp/room_emitter.py index 64358602a..2472999e3 100644 --- a/src/band/integrations/acp/room_emitter.py +++ b/src/band/integrations/acp/room_emitter.py @@ -3,10 +3,9 @@ from __future__ import annotations import logging -from collections.abc import Mapping from typing import Self -from band.core.delivery import deliver_reply +from band.core.delivery import relay_reply from band.core.protocols import AgentToolsProtocol, send_event_safe from band.core.types import Emit from band.integrations.acp.types import ( @@ -16,53 +15,13 @@ CollectedChunk, ToolStatus, ) -from band.runtime.tools import TurnEffect, settles_turn_reply +from band.runtime.tools import turn_effect logger = logging.getLogger(__name__) ACP_SESSION_CLOSED_EVENT = "ACP client session" -def turn_replied_in_room( - chunks: list[CollectedChunk], - *, - custom_effects: Mapping[str, TurnEffect] | None = None, -) -> bool: - """True when the turn already settled its reply. - - A room post, band_no_reply, or a custom tool that declared either - (``custom_effects``) settles it. Unlike copilot_sdk / codex, which execute Band tools in-process and flip a flag - at execution time, ACP tool calls may run out-of-process (a remote band-mcp - server the SDK never sees execute). The ACP session-update stream is the one - record of the turn that covers both, so detection matches the collected - tool-call chunks by their reported title (ACP has no structured tool-name - field). A reply-settling call counts once it (or its result update) reports - ``completed`` — a failed post must not suppress the text fallback, or the turn - goes silent. - """ - settling_call_ids: set[str] = set() - for chunk in chunks: - metadata = chunk.metadata or {} - if isinstance(chunk.tool, ACPToolCall) and settles_turn_reply( - chunk.tool.name, custom_effects=custom_effects - ): - if metadata.get("status") == ToolStatus.COMPLETED: - return True - # Correlate with a later result only by a real id. An empty id (a - # missing tool_call_id) would match any other id-less result — e.g. a - # non-posting tool's — and falsely suppress the text fallback, - # silencing the turn. - if chunk.tool.tool_call_id: - settling_call_ids.add(chunk.tool.tool_call_id) - elif ( - isinstance(chunk.tool, ACPToolResult) - and chunk.tool.call.tool_call_id in settling_call_ids - and metadata.get("status") == ToolStatus.COMPLETED - ): - return True - return False - - class RoomTurnEmitter: """Posts one ACP turn's output to a Band room in causal order. @@ -76,19 +35,18 @@ class RoomTurnEmitter: with no special-casing needed. The ordering is enforced upstream by ``ACPCollectingClient``'s per-session lock — ``emit`` is never entered concurrently for one session. The assistant's text reply is held until close, - because whether to relay it depends on whether the whole turn already posted - via a Band tool — if so the text would duplicate the reply already in the room. + because whether to relay it depends on whether the whole turn already replied + or declined via a Band tool — if so the text would duplicate the reply already + in the room. - On a clean close the held text is relayed (unless already posted in-room), and - the session bookkeeping ``task`` event is posted last. + On a clean close the held text is relayed (unless the turn already replied), + and the session bookkeeping ``task`` event is posted last. Which narration kinds reach the room is controlled by the emit set passed - at construction (``None``: all kinds — the historical default). Chunks are - always recorded regardless, so the tool-first delivery decision keeps - working even when narration is silenced. The closing bookkeeping ``task`` - event is state, not narration, and is posted regardless of the emit set: - ``ACPClientHistoryConverter`` reads its metadata to rebuild the - room→session map, so gating it would silently disable ``session/load`` + at construction (``None``: all kinds — the historical default). The closing + bookkeeping ``task`` event is state, not narration, and is posted regardless + of the emit set: ``ACPClientHistoryConverter`` reads its metadata to rebuild + the room→session map, so gating it would silently disable ``session/load`` resume after a restart. """ @@ -100,25 +58,24 @@ def __init__( session_id: str, room_id: str, emit: frozenset[Emit] | None = None, - custom_effects: Mapping[str, TurnEffect] | None = None, + records_tool_effects: bool = False, ) -> None: + """``records_tool_effects``: the turn's tools run out of process, so + each completed call's effect is recorded on the turn from the stream.""" self._tools = tools self._mentions = mentions self._session_id = session_id self._room_id = room_id - self._custom_effects = custom_effects + self._records_tool_effects = records_tool_effects # ``None``: post every kind (the historical behavior). Adapters pass # their resolved ``features.emit`` so a caller's ``emit=`` narrowing # reaches the room sink. self._emit = frozenset(emit) if emit is not None else frozenset(Emit) - self._chunks: list[CollectedChunk] = [] self._pending_text: list[str] = [] async def emit(self, chunk: CollectedChunk) -> None: - # Record every chunk regardless of the emit set: the tool-first - # delivery decision (``turn_replied_in_room``) must see the whole - # turn, even when narration is silenced. - self._chunks.append(chunk) + if self._records_tool_effects: + self._record_tool_effect(chunk) match chunk.chunk_type: case ChunkType.TEXT: if chunk.content: @@ -153,6 +110,21 @@ async def emit(self, chunk: CollectedChunk) -> None: chunk.chunk_type, ) + def _record_tool_effect(self, chunk: CollectedChunk) -> None: + """Record a completed tool call's effect on the turn. + + An external band-mcp runs where the SDK never sees it, so the stream is + the only record of what it did. ACP has no structured tool-name field, + so the canonicalized title names the tool; a non-Band tool resolves to + ``observe``. A failed call records nothing, so a failed post never + suppresses the text relay. + """ + if chunk.metadata.get("status") != ToolStatus.COMPLETED: + return + match chunk.tool: + case ACPToolCall(name=name) | ACPToolResult(call=ACPToolCall(name=name)): + self._tools.turn.record(turn_effect(name)) + def _tool_event_content(self, chunk: CollectedChunk) -> str: """Serialize normalized tool activity for room persistence.""" if isinstance(chunk.tool, (ACPToolCall, ACPToolResult)): @@ -208,12 +180,11 @@ async def __aexit__(self, exc_type: object, exc: object, tb: object) -> bool: # neither the held text nor the bookkeeping event. if exc_type is not None: return False - # Tool-first delivery (matches copilot_sdk / codex): if the turn already - # settled its reply, relaying its plain text too would duplicate it (and - # leak the agent's narration of the call). - if not turn_replied_in_room(self._chunks, custom_effects=self._custom_effects): - for text in self._pending_text: - await deliver_reply(self._tools, text, mentions=self._mentions) + # The held runs only ever post together at close, so they relay as the + # turn's one reply. + await relay_reply( + self._tools, "\n\n".join(self._pending_text), mentions=self._mentions + ) # Posted regardless of the emit set: this is resume state read back by # ACPClientHistoryConverter, not narration (only PLAN chunks follow # Emit.TASK_EVENTS). diff --git a/src/band/integrations/claude_sdk/tools.py b/src/band/integrations/claude_sdk/tools.py index e20272ddf..a7371e87e 100644 --- a/src/band/integrations/claude_sdk/tools.py +++ b/src/band/integrations/claude_sdk/tools.py @@ -182,6 +182,11 @@ def _format_success_payload( return {"status": "success", "result": result} +def _room_id(args: dict[str, Any], *, include_room_id: bool) -> str: + """The room a tool call targets; ``""`` when its schema carries no room.""" + return args.get(CHAT_ID_FIELD_NAME, "") if include_room_id else "" + + async def _maybe_call_tool_result_hook( tool_result_hook: ToolResultHook | None, tool_name: str, @@ -212,7 +217,7 @@ def _build_builtin_sdk_tool( schema, ) async def handler(args: dict[str, Any]) -> dict[str, Any]: - room_id = args.get(CHAT_ID_FIELD_NAME, "") if include_room_id else "" + room_id = _room_id(args, include_room_id=include_room_id) raw_args = {k: v for k, v in args.items() if k != CHAT_ID_FIELD_NAME} tools = get_tools(room_id) if tools is None: @@ -258,6 +263,7 @@ async def handler(args: dict[str, Any]) -> dict[str, Any]: def _build_custom_sdk_tool( tool_def: CustomToolDef, *, + get_tools: ToolResolver, include_room_id: bool, ) -> SdkMcpTool[Any]: input_model, _ = tool_def @@ -272,7 +278,10 @@ def _build_custom_sdk_tool( async def handler(args: dict[str, Any]) -> dict[str, Any]: try: tool_args = {k: v for k, v in args.items() if k != CHAT_ID_FIELD_NAME} - result = await execute_custom_tool(tool_def, tool_args) + tools = get_tools(_room_id(args, include_room_id=include_room_id)) + result = await execute_custom_tool( + tool_def, tool_args, turn=tools.turn if tools is not None else None + ) return _make_result(result) except Exception as error: logger.exception("Custom tool %s failed", tool_name) @@ -306,6 +315,7 @@ def build_band_sdk_tools( sdk_tools.append( _build_custom_sdk_tool( custom_tool, + get_tools=get_tools, include_room_id=include_room_id, ) ) diff --git a/src/band/integrations/copilot_sdk/room_ask_user.py b/src/band/integrations/copilot_sdk/room_ask_user.py index 017fa02e9..465e87371 100644 --- a/src/band/integrations/copilot_sdk/room_ask_user.py +++ b/src/band/integrations/copilot_sdk/room_ask_user.py @@ -89,7 +89,7 @@ def question_delivered_answer(rendered_question: str) -> UserInputResponse: return freeform_answer(QUESTION_DELIVERED_ANSWER.format(rendered=rendered_question)) -def delivery_failed_answer(error: Exception) -> UserInputResponse: +def delivery_failed_answer(error: BaseException) -> UserInputResponse: """The answer when posting the question to the room failed.""" return freeform_answer(DELIVERY_FAILED_ANSWER.format(error=error)) diff --git a/src/band/integrations/crewai/tools.py b/src/band/integrations/crewai/tools.py index f18ce1063..758b7a18a 100644 --- a/src/band/integrations/crewai/tools.py +++ b/src/band/integrations/crewai/tools.py @@ -235,7 +235,9 @@ class CustomCrewAITool(BaseTool): def _run(self, *_args: Any, **kwargs: Any) -> Any: async def execute(tools: AgentToolsProtocol) -> str: await reporter.report_call(tools, tool_name, kwargs) - result = await execute_custom_tool(definition, kwargs) + result = await execute_custom_tool( + tool=definition, arguments=kwargs, turn=tools.turn + ) await reporter.report_result(tools, tool_name, result) return json.dumps({"status": "success", "result": result}, default=str) diff --git a/src/band/integrations/langgraph/langchain_tools.py b/src/band/integrations/langgraph/langchain_tools.py index d5fee34b4..97a5db983 100644 --- a/src/band/integrations/langgraph/langchain_tools.py +++ b/src/band/integrations/langgraph/langchain_tools.py @@ -17,6 +17,7 @@ from band.core.exceptions import BandToolError from band.core.protocols import AgentToolsProtocol from band.core.tool_filter import filter_tool_schemas +from band.core.turn import Turn from band.core.types import AdapterFeatures, Capability from band.runtime.capabilities import with_hub_room_contacts from band.runtime.custom_tools import ( @@ -163,54 +164,48 @@ def validation_error_message( return platform_tools -def custom_tool_defs_to_langchain(tools: list[CustomToolDef]) -> list[StructuredTool]: - """Convert band ``CustomToolDef`` tuples to LangChain ``StructuredTool``s. +def custom_tool_def_to_langchain( + tool_def: CustomToolDef, *, turn: Turn | None +) -> StructuredTool: + """Convert a band ``CustomToolDef`` to a LangChain ``StructuredTool``. Lets the LangGraph adapter accept the SDK's portable custom-tool form — ``(InputModel, handler)`` — the same shape every other adapter takes, rather - than only ready-made LangChain tools. Each tuple becomes a StructuredTool whose - schema is the ``InputModel`` and whose body validates + runs the handler via - ``execute_custom_tool`` (which handles sync and async handlers alike). + than only ready-made LangChain tools. The StructuredTool's schema is the + ``InputModel`` and its body validates + runs the handler via + ``execute_custom_tool`` (which handles sync and async handlers alike), + recording the handler's declared effect on ``turn``: the turn of the room the + tool is built for, or ``None`` for a tool not bound to a room. """ - langchain_tools: list[StructuredTool] = [] - for input_model, handler in tools: - name = get_custom_tool_name(input_model) - description = input_model.__doc__ or f"Execute {name}" - - async def execute_custom( - *, - _tool: CustomToolDef = (input_model, handler), - _name: str = name, - **kwargs: Any, - ) -> Any: - try: - return await execute_custom_tool(_tool, kwargs) - except ValueError as e: - # Bad-argument / validation errors feed back to the LLM to retry. - return str(e) - except Exception: - # Keep the message generic; the full traceback (paths, secrets) - # only lives in the agent log — see agent_tools_to_langchain. - logger.exception("Error executing custom tool %s", _name) - return f"Error executing {_name}: see agent logs." - - def on_validation_error(error: Any, *, _name: str = name) -> str: - # Schema validation runs before the coroutine; feed bad args back to the - # LLM (to retry) via the shared formatter, so the wording matches the - # band-tool path (agent_tools_to_langchain) exactly. - return format_tool_validation_error(_name, error) - - execute_custom.__name__ = f"{name}_wrapper" - execute_custom.__doc__ = description - - langchain_tools.append( - StructuredTool.from_function( - coroutine=execute_custom, - name=name, - description=description, - args_schema=input_model, - handle_validation_error=on_validation_error, - ) - ) - - return langchain_tools + input_model, _ = tool_def + name = get_custom_tool_name(input_model) + description = input_model.__doc__ or f"Execute {name}" + + async def execute_custom(**kwargs: Any) -> Any: + try: + return await execute_custom_tool(tool_def, kwargs, turn=turn) + except ValueError as e: + # Bad-argument / validation errors feed back to the LLM to retry. + return str(e) + except Exception: + # Keep the message generic; the full traceback (paths, secrets) + # only lives in the agent log — see agent_tools_to_langchain. + logger.exception("Error executing custom tool %s", name) + return f"Error executing {name}: see agent logs." + + def on_validation_error(error: Any) -> str: + # Schema validation runs before the coroutine; feed bad args back to the + # LLM (to retry) via the shared formatter, so the wording matches the + # band-tool path (agent_tools_to_langchain) exactly. + return format_tool_validation_error(name, error) + + execute_custom.__name__ = f"{name}_wrapper" + execute_custom.__doc__ = description + + return StructuredTool.from_function( + coroutine=execute_custom, + name=name, + description=description, + args_schema=input_model, + handle_validation_error=on_validation_error, + ) diff --git a/src/band/integrations/mcp/engine.py b/src/band/integrations/mcp/engine.py index ff5e59a01..10dc2daaf 100644 --- a/src/band/integrations/mcp/engine.py +++ b/src/band/integrations/mcp/engine.py @@ -43,6 +43,7 @@ from band.core.exceptions import BandToolError from band.core.protocols import AgentToolsProtocol from band.core.tool_filter import sanitize_tool_schema +from band.core.turn import Turn from band.core.types import Capability, EventMessageType, MessageType from band.runtime.custom_tools import ( CustomToolDef, @@ -502,29 +503,44 @@ async def execute(arguments: dict[str, Any]) -> Any: ) +RoomToolResolver = Callable[[str], AgentToolsProtocol | None] + + +def _room_turn(get_tools: RoomToolResolver | None, chat_id: str | None) -> Turn | None: + """The turn of the room ``chat_id`` names, if the tool is room-bound.""" + if get_tools is None or chat_id is None: + return None + tools = get_tools(chat_id) + return tools.turn if tools is not None else None + + def build_custom_tool_registration( spec: CustomToolSpec | CustomToolDef, *, - room_bound: bool = False, + get_tools: RoomToolResolver | None = None, ) -> MCPToolRegistration: """Build a registration for a user-provided custom tool. Embedded-door only (divergence-matrix row 12: not exposed on the CLI). Dispatches straight through ``execute_custom_tool`` -- there is no - ``AgentTools``/``HumanTools`` method behind a custom tool, so no - resolver is involved. + ``AgentTools``/``HumanTools`` method behind a custom tool. With + ``get_tools`` the tool is room-bound: it takes a ``chat_id``, and its + declared effect is recorded on that room's turn. """ tool_def: CustomToolDef = ( (spec.input_model, spec.handler) if isinstance(spec, CustomToolSpec) else spec ) input_model, _ = tool_def tool_name = get_custom_tool_name(input_model) - model = extend_with_chat_id(input_model, None) if room_bound else input_model + model = ( + extend_with_chat_id(input_model, None) if get_tools is not None else input_model + ) async def execute(arguments: dict[str, Any]) -> Any: kwargs = dict(arguments) - kwargs.pop(CHAT_ID_FIELD_NAME, None) - result = await execute_custom_tool(tool_def, kwargs) + chat_id = kwargs.pop(CHAT_ID_FIELD_NAME, None) + turn = _room_turn(get_tools, chat_id) + result = await execute_custom_tool(tool_def, kwargs, turn=turn) return _serialize(result) return MCPToolRegistration( @@ -535,9 +551,6 @@ async def execute(arguments: dict[str, Any]) -> Any: ) -RoomToolResolver = Callable[[str], AgentToolsProtocol | None] - - def _filter_to_agent_surface( definitions: Sequence[ToolDefinition], ) -> list[ToolDefinition]: @@ -623,7 +636,7 @@ def build_resolved_band_mcp_tool_registrations( for definition in definitions ] registrations.extend( - build_custom_tool_registration(tool_def, room_bound=True) + build_custom_tool_registration(tool_def, get_tools=get_tools) for tool_def in additional_tools or [] ) validate_unique_tool_names(registrations) diff --git a/src/band/integrations/slack/adapter.py b/src/band/integrations/slack/adapter.py index c7f47ec2d..10d412569 100644 --- a/src/band/integrations/slack/adapter.py +++ b/src/band/integrations/slack/adapter.py @@ -55,7 +55,7 @@ from band.integrations.slack.server import build_router from band.integrations.slack.types import SlackApp, SlackRoomBinding from band.platform.posting import post_event -from band.runtime.tools import AgentTools +from band.runtime.tools import AgentTools, TurnEffect if TYPE_CHECKING: from slack_sdk.web.async_client import AsyncWebClient @@ -151,8 +151,10 @@ def __init__( hub_room_id=wrap._hub_room_id, ) # Carry ExecutionContext over so any tool methods that lean on - # it (e.g. lookup_peers) keep working. + # it (e.g. lookup_peers) keep working, and share the turn so the + # adapter judging ``wrap`` sees what the brain did through this tee. self._ctx = wrap._ctx + self.turn = wrap.turn self._slack = slack self._binding = binding self._write_tool_names: frozenset[str] = ( @@ -216,6 +218,7 @@ async def slack_send_message(self, content: str) -> dict[str, Any]: self._binding.thread_ts, ) return {"ok": False, "error": str(exc)} + self.turn.record(TurnEffect.REPLY) return {"ok": True} # ── Schema injection ──────────────────────────────────────────── diff --git a/src/band/runtime/contact_handler.py b/src/band/runtime/contact_handler.py index a832e7b62..4167203b8 100644 --- a/src/band/runtime/contact_handler.py +++ b/src/band/runtime/contact_handler.py @@ -25,6 +25,11 @@ ChatRoomRequest, ) from band.client.streaming import MessageCreatedPayload, MessageMetadata +from band.core.types import ( + SYNTHETIC_CONTACT_EVENTS_SENDER_ID, + SYNTHETIC_CONTACT_EVENTS_SENDER_NAME, + SYNTHETIC_SENDER_TYPE, +) from band.platform.event import ( ContactAddedEvent, ContactEvent, @@ -36,9 +41,6 @@ from band.platform.posting import post_event from band.runtime.contact_tools import ContactTools from band.runtime.types import ( - SYNTHETIC_CONTACT_EVENTS_SENDER_ID, - SYNTHETIC_CONTACT_EVENTS_SENDER_NAME, - SYNTHETIC_SENDER_TYPE, ContactEventConfig, ContactEventStrategy, normalize_handle, diff --git a/src/band/runtime/custom_tools.py b/src/band/runtime/custom_tools.py index a3dad4805..ddccedd9b 100644 --- a/src/band/runtime/custom_tools.py +++ b/src/band/runtime/custom_tools.py @@ -14,6 +14,7 @@ from pydantic import BaseModel, ValidationError +from band.core.turn import Turn from band.runtime.tools.types import TurnEffect logger = logging.getLogger(__name__) @@ -27,10 +28,10 @@ def declares_turn_effect(effect: TurnEffect) -> Callable[[_Handler], _Handler]: """Declare what a custom tool's successful call does to the turn's reply. - ``ACT``: it did real work, so an empty final answer afterwards is benign. + ``ACT``: it did real work, so the turn is complete without a reply. ``REPLY``: it delivers the answer itself, so no fallback text is relayed. ``DECLINE``: silence is the answer, so no fallback text is relayed either. - An undeclared custom tool fails loud (see ``runtime.tools.turn_effect``). + An undeclared custom tool counts as ``OBSERVE``: the turn still owes a reply. """ def declare(handler: _Handler) -> _Handler: @@ -191,6 +192,8 @@ def _custom_tool_accepts_input(func: Callable[..., Any]) -> bool: async def execute_custom_tool( tool: CustomToolDef, arguments: dict[str, Any], + *, + turn: Turn | None, ) -> Any: """ Execute custom tool with Pydantic validation. @@ -198,6 +201,8 @@ async def execute_custom_tool( Args: tool: (InputModel, callable) tuple arguments: Raw arguments dict from LLM + turn: The turn to record the tool's declared effect on, or ``None`` + when the tool is not bound to a room, so it cannot settle a turn. Returns: Tool execution result @@ -222,12 +227,14 @@ async def execute_custom_tool( raise ValueError( f"Invalid handler for {tool_name}: zero-argument handlers require an empty InputModel and no arguments" ) - return await invoke_validated_custom_tool(tool, validated) + return await invoke_validated_custom_tool(tool, validated, turn=turn) async def invoke_validated_custom_tool( tool: CustomToolDef, validated: Any, + *, + turn: Turn | None, ) -> Any: """ Execute a custom tool whose arguments are already a validated InputModel @@ -237,7 +244,7 @@ async def invoke_validated_custom_tool( pydantic-ai validates tool args natively): re-serializing the instance to a dict and re-validating would break models using field aliases, so the instance is passed through as-is. Async/zero-argument handler semantics - match :func:`execute_custom_tool` exactly. + match :func:`execute_custom_tool` exactly, and so does ``turn``. """ model, func = tool @@ -255,5 +262,7 @@ async def invoke_validated_custom_tool( # coroutine unawaited). result = func(*args) if inspect.isawaitable(result): - return await result + result = await result + if turn is not None: + turn.record(declared_effect(func) or TurnEffect.OBSERVE) return result diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index fa8f0b453..fbf4ca62f 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -45,9 +45,10 @@ ) from band.core.protocols import ( GENERIC_PROVIDER_FAILURE_MESSAGE, + TURN_FAILURE_PROVIDER, TurnResultAlreadyReported, ) -from band.core.types import metadata_to_dict +from band.core.types import is_contact_hub_turn, metadata_to_dict from band.logging_config import TRACE_CONTEXT from band.platform.event import ( MessageEvent, @@ -61,8 +62,6 @@ from band.runtime.participants import log_roster_call, log_roster_error from band.runtime.tools.agent import AgentTools from band.runtime.types import ( - SYNTHETIC_CONTACT_EVENTS_SENDER_ID, - SYNTHETIC_SENDER_TYPE, ConversationContext, ParticipantAddedCallback, ParticipantRemovedCallback, @@ -107,11 +106,6 @@ def _error_label(e: Exception) -> str: return str(e).strip() or type(e).__name__ -# ``AgentFailure.provider`` for a turn failure the runtime reports on the -# adapter's behalf. -_TURN_FAILURE_PROVIDER = "band-runtime" - - @runtime_checkable class Execution(Protocol): """ @@ -1967,9 +1961,8 @@ async def _process_event(self, event: PlatformEvent) -> bool: # Detect synthetic messages (e.g., contact events injected into hub room) # These don't exist in the database, so skip all tracking and marking - is_synthetic = ( - payload.sender_type == SYNTHETIC_SENDER_TYPE - and payload.sender_id == SYNTHETIC_CONTACT_EVENTS_SENDER_ID + is_synthetic = is_contact_hub_turn( + sender_type=payload.sender_type, sender_id=payload.sender_id ) if is_synthetic: logger.debug("Processing synthetic contact event message") @@ -2051,7 +2044,7 @@ async def _handle_turn_failure( # The exception text can carry credentials; it stays in mark_failed # and the logs, and the room gets the generic message. await AgentTools.from_context(self).send_failure( - AgentFailure(_TURN_FAILURE_PROVIDER, GENERIC_PROVIDER_FAILURE_MESSAGE) + AgentFailure(TURN_FAILURE_PROVIDER, GENERIC_PROVIDER_FAILURE_MESSAGE) ) async def _process_event_body( diff --git a/src/band/runtime/tools/agent.py b/src/band/runtime/tools/agent.py index b142e0fa7..c97e291f5 100644 --- a/src/band/runtime/tools/agent.py +++ b/src/band/runtime/tools/agent.py @@ -51,6 +51,7 @@ validate_include, ) from band.core.tool_filter import sanitize_tool_schema +from band.core.turn import Turn from band.core.types import Capability, MessageType from band.core.validation import at_least_one_of from band.platform.posting import post_event, post_message @@ -68,6 +69,7 @@ serialize_tool_result, validate_tool_arguments, ) +from band.runtime.turn import records_turn_effects if TYPE_CHECKING: from anthropic.types import ToolParam @@ -267,6 +269,7 @@ def cache_info(self) -> Any: ... def cache_parameters(self) -> Any: ... +@records_turn_effects class AgentTools(AgentToolsProtocol): """ Room-bound tools for LLM platform interaction. @@ -325,6 +328,7 @@ def __init__( self._hub_room_id = hub_room_id self._agent_id = agent_id self._ctx: ExecutionContext | None = None + self.turn = Turn() @property def agent_id(self) -> str | None: @@ -406,6 +410,27 @@ async def send_message( stacklevel=2, ) + return await self._post_message( + content, mentions, attachment_ids=attachment_ids + ) + + async def send_notice( + self, content: str, mentions: list[str] | list[dict[str, str]] | None = None + ) -> Any: + """Post the adapter's own message (a prompt, a status reply, a notice). + + Unlike ``send_message`` it never counts as the turn's reply: a notice + posted mid-turn must not stand in for the model's answer. + """ + return await self._post_message(content, mentions) + + async def _post_message( + self, + content: str, + mentions: list[str] | list[dict[str, str]] | None, + *, + attachment_ids: list[str] | None = None, + ) -> Any: resolved_mentions = self._resolve_required_mentions(mentions) logger.debug("Sending message to room %s", self.room_id) @@ -485,6 +510,7 @@ async def send_failure(self, failure: band_sdk_core.AgentFailure) -> Any: except Exception as exc: logger.exception("send_failure could not post the failure event") return {"ok": False, "error": str(exc)} + self.turn.note_reported() if self._ctx is not None: self._ctx.note_turn_failure_reported() return response diff --git a/src/band/runtime/tools/effects.py b/src/band/runtime/tools/effects.py index 4727262ee..c02458af3 100644 --- a/src/band/runtime/tools/effects.py +++ b/src/band/runtime/tools/effects.py @@ -10,26 +10,20 @@ from collections.abc import Mapping +from band_sdk_core import band_tool_effects + from band.runtime.tools.registry import ( - ALL_TOOL_NAMES, - EVENT_TOOL_NAMES, LEGACY_SEND_MESSAGE_TOOL, - READ_ONLY_TOOL_NAMES, canonicalize_mcp_tool_name, ) from band.runtime.tools.types import BandTool, TurnEffect -# Every Band tool not listed below is a durable action (ACT). -# ``band_send_room_file`` also posts a message (the file's attaching message). +# band-sdk-core owns the effect of every Band tool. The legacy send-message +# spelling is Python's own alias for band_send_message. _BAND_EFFECTS: dict[str, TurnEffect] = { - **dict.fromkeys(ALL_TOOL_NAMES, TurnEffect.ACT), - **dict.fromkeys(READ_ONLY_TOOL_NAMES | EVENT_TOOL_NAMES, TurnEffect.OBSERVE), - **dict.fromkeys( - {BandTool.SEND_MESSAGE, BandTool.SEND_ROOM_FILE, LEGACY_SEND_MESSAGE_TOOL}, - TurnEffect.REPLY, - ), - BandTool.NO_REPLY: TurnEffect.DECLINE, + name: TurnEffect(effect.wire_name) for name, effect in band_tool_effects().items() } +_BAND_EFFECTS[LEGACY_SEND_MESSAGE_TOOL] = _BAND_EFFECTS[BandTool.SEND_MESSAGE] def turn_effect( diff --git a/src/band/runtime/tools/inputs/chat.py b/src/band/runtime/tools/inputs/chat.py index 92684cdef..61fc17727 100644 --- a/src/band/runtime/tools/inputs/chat.py +++ b/src/band/runtime/tools/inputs/chat.py @@ -8,6 +8,7 @@ from typing import Any, Literal +from band_sdk_core import no_reply_tool from pydantic import AliasChoices, BaseModel, Field, field_validator from band.core.content import BLANK_CONTENT_ERROR, has_visible_content @@ -36,8 +37,8 @@ class SendMessageInput(BaseModel): """Send a message to the chat room. Use this to respond to users or other agents. Messages require at least one @mention - in the mentions array. You MUST use this tool to communicate - plain text responses - won't reach users. + in the mentions array. When the latest message needs no answer from you, call + band_no_reply instead of sending one. """ content: str = Field(..., description="The message content to send") @@ -124,19 +125,18 @@ class GetParticipantsInput(BaseModel): # No parameters required +# band-sdk-core owns band_no_reply's definition, shared by every SDK. +_NO_REPLY_TOOL = no_reply_tool() +_NO_REPLY_REASON = next( + p["description"] for p in _NO_REPLY_TOOL["parameters"] if p["name"] == "reason" +) + + class NoReplyInput(BaseModel): - """End this turn without posting anything to the room. + reason: str | None = Field(default=None, description=_NO_REPLY_REASON) - Call this instead of band_send_message when the latest message needs no - answer from you: it was addressed to someone else, it is an FYI or an - acknowledgement, or another participant already answered it. It is a - complete turn, not an error; do not also send a message. - """ - reason: str | None = Field( - default=None, - description="Short note on why no reply is needed (kept in local logs only)", - ) +NoReplyInput.__doc__ = _NO_REPLY_TOOL["description"] class CreateChatroomInput(BaseModel): diff --git a/src/band/runtime/turn.py b/src/band/runtime/turn.py new file mode 100644 index 000000000..c99551e48 --- /dev/null +++ b/src/band/runtime/turn.py @@ -0,0 +1,50 @@ +"""Records each Band tool call's effect on the turn that made it.""" + +from __future__ import annotations + +import functools +from collections.abc import Awaitable, Callable +from typing import Any, TypeVar + +from band.core.turn import Turn +from band.runtime.tools.effects import turn_effect +from band.runtime.tools.registry import TOOL_DEFINITIONS +from band.runtime.tools.types import Surface, TurnEffect + +_ToolsClass = TypeVar("_ToolsClass", bound=type) + + +def record_tool_result(turn: Turn, tool_name: str, result: Any) -> None: + """Record a successful Band tool call's effect on ``turn``. + + A reply tool that returned ``None`` refused a blank send and posted nothing. + """ + effect = turn_effect(tool_name) + if effect is TurnEffect.REPLY and result is None: + return + turn.record(effect) + + +def _recording( + tool_name: str, method: Callable[..., Awaitable[Any]] +) -> Callable[..., Awaitable[Any]]: + @functools.wraps(method) + async def record_on_success(self: Any, *args: Any, **kwargs: Any) -> Any: + result = await method(self, *args, **kwargs) + record_tool_result(self.turn, tool_name, result) + return result + + return record_on_success + + +def records_turn_effects(cls: _ToolsClass) -> _ToolsClass: + """Record every agent tool method's effect on the instance's ``turn``. + + Wrapping the methods themselves counts every call path: registry dispatch, + a framework calling the method directly, and ``deliver_reply``. + """ + for definition in TOOL_DEFINITIONS.values(): + method = cls.__dict__.get(definition.method_name) + if definition.surface == Surface.AGENT and method is not None: + setattr(cls, definition.method_name, _recording(definition.name, method)) + return cls diff --git a/src/band/runtime/types.py b/src/band/runtime/types.py index 216f274e7..f22ad51b4 100644 --- a/src/band/runtime/types.py +++ b/src/band/runtime/types.py @@ -15,32 +15,6 @@ from band.core.types import ConflictPolicy -# --- Constants for synthetic messages (injected by SDK, not from platform) --- -# -# These constants define the sender identity for SDK-generated messages that are -# injected into the event queue but don't originate from the platform WebSocket. -# -# Primary use case: ContactEventHandler's HUB_ROOM strategy creates synthetic -# MessageEvents to route contact events to the hub room for LLM processing. -# These messages need a consistent sender identity that: -# 1. Clearly indicates they're system-generated (not from a real user/agent) -# 2. Can be filtered by preprocessing if needed -# 3. Are recognizable in UI/logs for debugging -# - -# Sender type for synthetic messages. Matches the platform's "System" type used -# for other system-generated content. -SYNTHETIC_SENDER_TYPE = "System" - -# Sender ID for synthetic contact event messages. This is a logical identifier -# (not a UUID) that allows filtering/identification of contact-related synthetic -# messages. Used in MessageEvent.payload.sender_id for hub room injections. -SYNTHETIC_CONTACT_EVENTS_SENDER_ID = "contact-events" - -# Human-readable sender name displayed in UI/logs for synthetic contact event -# messages. Used in MessageEvent.payload.sender_name. -SYNTHETIC_CONTACT_EVENTS_SENDER_NAME = "Contact Events" - def normalize_handle(handle: str | None) -> str | None: """ diff --git a/src/band/testing/fake_tools.py b/src/band/testing/fake_tools.py index 670bed72f..c7b04884a 100644 --- a/src/band/testing/fake_tools.py +++ b/src/band/testing/fake_tools.py @@ -56,6 +56,7 @@ TaskListState, task_ref, ) +from band.core.turn import Turn from band.core.types import ( Capability, ContactRequestAction, @@ -75,6 +76,7 @@ serialize_tool_result, strip_handle_prefix, ) +from band.runtime.turn import record_tool_result, records_turn_effects # Synthetic identity FakeAgentTools uses for the "joins you to the task on # first status/active_form write" semantics band_update_task documents. @@ -224,6 +226,7 @@ async def __aexit__( self.released.set() +@records_turn_effects class FakeAgentTools: """ Fake implementation of AgentToolsProtocol for testing. @@ -317,6 +320,7 @@ def __init__( self.context_calls: list[dict[str, Any]] = [] self._held_messages: list[HeldMessage] = [] self._observers: list[tuple[Callable[[], bool], asyncio.Future[None]]] = [] + self.turn = Turn() @property def agent_id(self) -> str | None: @@ -350,6 +354,18 @@ async def send_message( ``None`` without recording anything — mirroring the real send's non-throwing refusal at ``band.platform.posting.post_message``. """ + return await self._post(content, mentions) + + async def send_notice( + self, content: str, mentions: list[str] | list[dict[str, str]] | None = None + ) -> MessageSentResponse | None: + """Record the adapter's own message; like ``AgentTools.send_notice`` it + never counts as the turn's reply.""" + return await self._post(content, mentions) + + async def _post( + self, content: str, mentions: list[str] | list[dict[str, str]] | None + ) -> MessageSentResponse | None: if self.send_message_error is not None: raise self.send_message_error self._require_mentions(mentions) @@ -464,9 +480,11 @@ async def send_failure( """Same best-effort delegation as ``AgentTools.send_failure``.""" content, metadata = to_failure_event(failure) try: - return await self.send_event(content, MessageType.ERROR, metadata) + response = await self.send_event(content, MessageType.ERROR, metadata) except Exception as exc: # noqa: BLE001 -- best-effort like the real send_failure; must never raise inside a caller's own except block return {"ok": False, "error": str(exc)} + self.turn.note_reported() + return response async def add_participant( self, identifier: str, role: str = "member" @@ -1087,9 +1105,14 @@ async def execute_tool_call_structured( self, tool_name: str, arguments: dict[str, Any] ) -> ToolCallOutcome: """Record the call and report success. Override in a subclass to return - ``ok=False`` (a base tool failing without raising) for failure-path tests.""" + ``ok=False`` (a base tool failing without raising) for failure-path tests. + + The tool methods are not called, so the successful call's effect is + recorded here.""" self.tool_calls.append({"tool_name": tool_name, "arguments": arguments}) - return ToolCallOutcome(value={"status": "ok"}, ok=True) + outcome = ToolCallOutcome(value={"status": "ok"}, ok=True) + record_tool_result(self.turn, tool_name, outcome.value) + return outcome # --- Assertion helpers --- diff --git a/tests/adapters/claude_sdk/approvals/test_manual_approval.py b/tests/adapters/claude_sdk/approvals/test_manual_approval.py index cbdc84f17..d1fa4df91 100644 --- a/tests/adapters/claude_sdk/approvals/test_manual_approval.py +++ b/tests/adapters/claude_sdk/approvals/test_manual_approval.py @@ -67,6 +67,38 @@ async def test_a_room_answers_each_gated_tool_while_its_commands_stay_local( assert room.failures == [] +async def test_messages_during_a_parked_turn_settle_on_their_own_tools( + claude_room: OpenRoom, +) -> None: + """While a turn waits on an approval, a command or a second request is + answered and settled on its own message's tools; the parked turn keeps + the tools it started with, and its prompt is never taken for its reply.""" + room = await claude_room(with_approvals()) + room.claude.script([LIST_FILES, room.model_reply("Listed.")]) + status, busy, approve = room.fresh_tools(), room.fresh_tools(), room.fresh_tools() + + await room.send("List the files") + prompted_turn_replied = room.tools.turn.replied + await room.send("/status", tools=status) + await room.send("And the hidden ones?", tools=busy) + await room.send("/approve", tools=approve) + await room.settled() + + assert prompted_turn_replied is False + assert room.chat == [ + prompt("a-1", "Bash: `ls`"), + resolved("a-1", "accept"), + "Listed.", + ] + assert room.tools.turn.replied + assert [message["content"] for message in busy.messages_sent] == [ + "Still processing the previous request in this room." + ] + assert approve.messages_sent == [] + for side_turn in (status.turn, busy.turn, approve.turn): + assert (side_turn.complete, side_turn.replied) == (True, False) + + async def test_a_busy_room_evicts_the_oldest_ask_and_asks_for_a_token( claude_room: OpenRoom, ) -> None: diff --git a/tests/adapters/claude_sdk/tools/test_custom_tools.py b/tests/adapters/claude_sdk/tools/test_custom_tools.py index 366a7f74b..b047e66fa 100644 --- a/tests/adapters/claude_sdk/tools/test_custom_tools.py +++ b/tests/adapters/claude_sdk/tools/test_custom_tools.py @@ -6,6 +6,8 @@ from pydantic import BaseModel, Field from band.core.types import Capability +from band.runtime.custom_tools import declares_turn_effect +from band.runtime.tools import TurnEffect from tests.adapters.claude_sdk.helpers import ClaudeRoom, with_approvals OpenRoom = Callable[..., Awaitable[ClaudeRoom]] @@ -32,6 +34,17 @@ def add(args: CalculatorInput) -> float: return args.a + args.b +class FileTicketInput(BaseModel): + """File a ticket.""" + + title: str + + +@declares_turn_effect(TurnEffect.ACT) +async def file_ticket(args: FileTicketInput) -> str: + return f"Filed: {args.title}" + + CUSTOM_TOOLS = [(EchoInput, echo), (CalculatorInput, add)] @@ -73,3 +86,19 @@ async def test_memory_tools_are_served_only_with_the_memory_capability( output = str(room.tool_outputs["band_list_memories"]) assert ("No such tool available" not in output) is served assert room.chat == ["ok"] + + +async def test_a_custom_tool_records_its_declared_effect_on_the_rooms_turn( + claude_room: OpenRoom, +) -> None: + """A custom tool declared as real work completes the room's turn without + a reply, so nothing is reported missing.""" + room = await claude_room( + additional_tools=[*CUSTOM_TOOLS, (FileTicketInput, file_ticket)] + ) + room.claude.script([room.model_call("fileticket", title="Broken build")]) + + await room.send("file a ticket for the broken build") + + assert (room.tools.turn.complete, room.tools.turn.replied) == (True, False) + assert room.failures == [] diff --git a/tests/adapters/copilot_sdk/test_reply.py b/tests/adapters/copilot_sdk/test_reply.py index 59aa1eeb4..929aa9b75 100644 --- a/tests/adapters/copilot_sdk/test_reply.py +++ b/tests/adapters/copilot_sdk/test_reply.py @@ -51,6 +51,8 @@ async def test_reply_sent_with_sender_mention(self): sent = tools.messages_sent[0] assert sent["content"] == "Hi Alice!" assert sent["mentions"] == [{"id": "user-1", "name": "Alice"}] + # The relayed final text is the turn's reply. + assert tools.turn.complete @pytest.mark.asyncio async def test_send_message_failure_is_not_reported_as_provider_failure(self): diff --git a/tests/adapters/copilot_sdk/test_tool_bridging.py b/tests/adapters/copilot_sdk/test_tool_bridging.py index 8c9f3691e..47d8fd905 100644 --- a/tests/adapters/copilot_sdk/test_tool_bridging.py +++ b/tests/adapters/copilot_sdk/test_tool_bridging.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import json from typing import Any @@ -10,9 +11,11 @@ from band.adapters.copilot_sdk import _COPILOT_SDK_AVAILABLE from band.core.types import Emit -from band.runtime.tools import ToolCallOutcome +from band.runtime.custom_tools import declares_turn_effect +from band.runtime.tools import ToolCallOutcome, TurnEffect from tests.adapters.copilot_sdk.fakes import ( FakeCopilotClient, + FakeCopilotSession, ToolSchemaFakeTools, make_started_adapter, requires_copilot_sdk, @@ -199,6 +202,83 @@ async def test_handler_uses_current_room_tools(self): assert not first_tools.tool_calls assert second_tools.tool_calls + @pytest.mark.asyncio + async def test_queued_message_does_not_take_over_running_turn_tools(self): + """A message waiting on the room's turn lock must not redirect the + running turn's tool calls onto its own tools.""" + second_queued = asyncio.Event() + + async def call_tool_once_second_is_queued(session: FakeCopilotSession) -> None: + await second_queued.wait() + await session.find_tool("band_get_participants").handler( + ToolInvocation( + tool_call_id="c1", tool_name="band_get_participants", arguments={} + ) + ) + + client = FakeCopilotClient(turn_events=[call_tool_once_second_is_queued]) + adapter = await make_started_adapter(client) + first_tools, second_tools = ToolSchemaFakeTools(), ToolSchemaFakeTools() + + first = asyncio.create_task(run_message(adapter, first_tools)) + await asyncio.sleep(0) + second = asyncio.create_task( + run_message(adapter, second_tools, is_session_bootstrap=False) + ) + await asyncio.sleep(0) + second_queued.set() + await asyncio.gather(first, second) + + # Each turn's own call lands on its own tools. + assert len(first_tools.tool_calls) == 1 + assert len(second_tools.tool_calls) == 1 + + +class TestCustomToolTurnEffect: + """A custom tool's declared effect lands on the room turn's ledger.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("effect", "closing_text_relayed"), + [ + pytest.param(TurnEffect.REPLY, False, id="reply"), + pytest.param(TurnEffect.DECLINE, False, id="decline"), + pytest.param(TurnEffect.ACT, True, id="act"), + pytest.param(None, True, id="undeclared"), + ], + ) + async def test_declared_effect_decides_the_final_text_relay( + self, effect: TurnEffect | None, closing_text_relayed: bool + ): + class NotifyInput(BaseModel): + text: str + + async def notify(params: NotifyInput) -> str: + return "notified" + + handler = notify if effect is None else declares_turn_effect(effect)(notify) + + async def model_calls_notify(session: FakeCopilotSession) -> None: + await session.find_tool("notify").handler( + ToolInvocation( + tool_call_id="c1", tool_name="notify", arguments={"text": "hi"} + ) + ) + + client = FakeCopilotClient( + reply_content="Closing text", turn_events=[model_calls_notify] + ) + adapter = await make_started_adapter( + client, additional_tools=[(NotifyInput, handler)] + ) + tools = ToolSchemaFakeTools() + + await run_message(adapter, tools) + + relayed = [m["content"] for m in tools.messages_sent] == ["Closing text"] + assert relayed is closing_text_relayed + assert tools.turn.complete + class _ReadRoomFileFakeTools(ToolSchemaFakeTools): """ToolSchemaFakeTools plus band_read_room_file, which the base fake diff --git a/tests/adapters/langgraph/test_lifecycle.py b/tests/adapters/langgraph/test_lifecycle.py index 62b1b70d3..df51b6a78 100644 --- a/tests/adapters/langgraph/test_lifecycle.py +++ b/tests/adapters/langgraph/test_lifecycle.py @@ -44,8 +44,7 @@ def test_simple_pattern_with_additional_tools(self, mock_llm, mock_checkpointer) ) assert adapter.graph_factory is not None - # additional_tools cleared after baking into factory - assert adapter.additional_tools == [] + assert adapter.additional_tools == [mock_tool] def test_simple_pattern_creates_default_checkpointer(self, mock_llm): """The simple path should not silently become stateless.""" diff --git a/tests/adapters/lettakit.py b/tests/adapters/lettakit.py index a8024ae3b..453ab13b2 100644 --- a/tests/adapters/lettakit.py +++ b/tests/adapters/lettakit.py @@ -61,9 +61,11 @@ def make_tool_call_message( def make_tool_return_message( tool_name: str = "band_send_message", tool_return: str = '{"status": "ok"}', + status: str = "success", ) -> MagicMock: + """letta-client's ``ToolReturnMessage`` names the tool in ``name``.""" return make_letta_message( - "tool_return_message", tool_name=tool_name, tool_return=tool_return + "tool_return_message", name=tool_name, tool_return=tool_return, status=status ) diff --git a/tests/adapters/opencode/conftest.py b/tests/adapters/opencode/conftest.py index a2e30e2fe..e8b5ee474 100644 --- a/tests/adapters/opencode/conftest.py +++ b/tests/adapters/opencode/conftest.py @@ -15,6 +15,7 @@ from band.testing import FakeAgentTools from tests.adapters.opencode.helpers import ( AskFactory, + FakeMCPBackend, FakeOpencodeClient, make_fake_mcp_backend_factory, ) @@ -53,11 +54,17 @@ def build( return build +@pytest.fixture +def mcp_backend() -> FakeMCPBackend: + """The shared Band MCP backend the adapter builds, faked.""" + return FakeMCPBackend() + + @pytest.fixture(autouse=True) -def patch_mcp_backend() -> Any: +def patch_mcp_backend(mcp_backend: FakeMCPBackend) -> Any: """Patch MCP backend creation for every OpenCode adapter test.""" with patch( "band.adapters.opencode.adapter.create_band_mcp_backend", - make_fake_mcp_backend_factory(), + make_fake_mcp_backend_factory(mcp_backend), ): yield diff --git a/tests/adapters/opencode/helpers.py b/tests/adapters/opencode/helpers.py index 11fa27ecf..399518f46 100644 --- a/tests/adapters/opencode/helpers.py +++ b/tests/adapters/opencode/helpers.py @@ -3,7 +3,7 @@ from __future__ import annotations import asyncio -from collections.abc import AsyncIterator, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from datetime import UTC, datetime from typing import Any, TypeAlias, cast from unittest.mock import AsyncMock @@ -19,15 +19,20 @@ from band.core.types import ( PlatformMessage, ) +from band.integrations.mcp.engine import EmbeddedResolver from band.integrations.opencode import ( ApprovalReply, OpencodePermissionRequest, OpencodeQuestionRequest, ) from band.integrations.opencode.types import OpencodeSessionState +from band.runtime.tools import TOOL_DEFINITIONS, BandTool from band.testing import FakeAgentTools RawOpencodeEvent: TypeAlias = dict[str, Any] +# What the fake server does next: stream an SSE event, or act between events +# (e.g. the model calling a band tool over MCP). +ServerStep: TypeAlias = RawOpencodeEvent | Callable[[], Awaitable[None]] def make_platform_message( @@ -223,11 +228,11 @@ def tools_protocol(tools: FakeAgentTools) -> AgentToolsProtocol: class RaisingSendTools(FakeAgentTools): - """FakeAgentTools whose send_message always fails, to exercise the + """FakeAgentTools whose send_notice always fails, to exercise the best-effort ``_notify_room`` path: a room post that raises must be swallowed so the turn still unblocks.""" - async def send_message( + async def send_notice( self, content: str, mentions: list[str] | list[dict[str, str]] | None = None ) -> dict[str, Any]: raise BandToolError("send failed") @@ -253,10 +258,10 @@ class FakeOpencodeClient: def __init__( self, *, - prompt_event_sequences: list[list[RawOpencodeEvent]] | None = None, - reply_permission_events: dict[str, list[RawOpencodeEvent]] | None = None, - reply_question_events: dict[str, list[RawOpencodeEvent]] | None = None, - reject_question_events: dict[str, list[RawOpencodeEvent]] | None = None, + prompt_event_sequences: list[list[ServerStep]] | None = None, + reply_permission_events: dict[str, list[ServerStep]] | None = None, + reply_question_events: dict[str, list[ServerStep]] | None = None, + reject_question_events: dict[str, list[ServerStep]] | None = None, get_session_missing: set[str] | None = None, prompt_exceptions: list[Exception] | None = None, serve_registrations: dict[str, str] | None = None, @@ -277,7 +282,7 @@ def __init__( self.disconnected_mcp_servers: list[str] = [] self.closed = False self._session_counter = 0 - self._queue: asyncio.Queue[RawOpencodeEvent | None] = asyncio.Queue() + self._queue: asyncio.Queue[ServerStep | None] = asyncio.Queue() self._prompt_event_sequences = list(prompt_event_sequences or []) self._reply_permission_events = reply_permission_events or {} self._reply_question_events = reply_question_events or {} @@ -375,16 +380,19 @@ async def disconnect_mcp_server(self, name: str) -> None: self.disconnected_mcp_servers.append(name) self.serve_registrations.pop(name, None) - async def push_event(self, event: RawOpencodeEvent) -> None: - """Inject one SSE event, as the server would mid-turn.""" + async def push_event(self, event: ServerStep) -> None: + """Inject one SSE event (or server action), as the server would mid-turn.""" await self._queue.put(event) async def iter_events(self) -> AsyncIterator[RawOpencodeEvent]: while True: - event = await self._queue.get() - if event is None: + step = await self._queue.get() + if step is None: return - yield event + if callable(step): + await step() + continue + yield step async def health(self) -> None: """Fake server is always reachable.""" @@ -421,6 +429,8 @@ def __init__( self.stop_calls = 0 self._stop_started = stop_started self._stop_release = stop_release + # The adapter's room-tools resolver, captured when it builds the backend. + self.get_tools: Callable[[str], Any] | None = None async def stop(self) -> None: self.stop_calls += 1 @@ -429,6 +439,19 @@ async def stop(self) -> None: if self._stop_release is not None: await self._stop_release.wait() + def band_tool_call( + self, tool: BandTool, arguments: dict[str, Any], *, room_id: str = "room-1" + ) -> ServerStep: + """The model calling a band tool over MCP, dispatched to the room's + tools exactly as the embedded MCP server does.""" + + async def call() -> None: + assert self.get_tools is not None, "the adapter never built the backend" + resolver = EmbeddedResolver(get_tools=self.get_tools) + await resolver.invoke(TOOL_DEFINITIONS[tool], room_id, arguments) + + return call + def make_fake_mcp_backend_factory( backend: FakeMCPBackend | None = None, @@ -437,6 +460,7 @@ def make_fake_mcp_backend_factory( fake = backend or FakeMCPBackend() async def factory(**kwargs: Any) -> FakeMCPBackend: + fake.get_tools = kwargs["get_tools"] return fake mock = AsyncMock(side_effect=factory) diff --git a/tests/adapters/opencode/test_turns.py b/tests/adapters/opencode/test_turns.py index 5c01405fd..6a1761b9c 100644 --- a/tests/adapters/opencode/test_turns.py +++ b/tests/adapters/opencode/test_turns.py @@ -9,6 +9,10 @@ import pytest from band.adapters.opencode import OpencodeAdapter, OpencodeAdapterConfig +from band.adapters.opencode.approvals import ( + APPROVAL_HANDLED_TEMPLATE, + APPROVAL_REQUESTED_TEMPLATE, +) from band.core.protocols import FAILURE_CODE_TIMEOUT from band.core.types import ( Capability, @@ -16,9 +20,11 @@ ) from band.integrations.opencode import ApprovalReply, parse_opencode_event from band.integrations.opencode.types import OpencodeSessionState -from band.testing import FakeAgentTools, reported_failures +from band.runtime.tools import BandTool +from band.testing import FakeAgentTools, events_of_type, reported_failures from tests.adapters.opencode.helpers import ( AnyHTTPStatusError, + FakeMCPBackend, FakeOpencodeClient, TaskEventFailingTools, event_message_updated, @@ -1086,17 +1092,18 @@ async def test_turn_completes_when_fallback_reply_send_rejected( async def test_room_posting_tool_reply_suppresses_text_fallback( - make_adapter, tools + make_adapter, tools, mcp_backend: FakeMCPBackend ) -> None: """When the model replies via band_send_message, the adapter must not also - post the assistant's plain text (double-post). Detection holds without - execution reporting: Emit.TOOL_CALLS governs only the tool_call/tool_result - narration, not the text-fallback suppression.""" + post the assistant's plain text (double-post).""" fake_client = FakeOpencodeClient( prompt_event_sequences=[ [ event_message_updated("sess-1", "msg-1"), event_text_part("sess-1", "msg-1", "I sent it via the tool."), + mcp_backend.band_tool_call( + BandTool.SEND_MESSAGE, {"content": "hi", "mentions": ["@alice"]} + ), event_tool_part( "sess-1", "msg-1", @@ -1113,9 +1120,126 @@ async def test_room_posting_tool_reply_suppresses_text_fallback( await run_single_turn(adapter, tools) - # The tool (not executed by the fake) was the reply; the fallback stays - # silent, so the adapter posts no message of its own. - assert tools.messages_sent == [] + assert [m["content"] for m in tools.messages_sent] == ["hi"] + + +async def test_approval_prompts_never_stand_in_for_the_models_reply( + make_adapter, tools +) -> None: + """The permission prompt and its confirmation post on the turn's own tools + mid-turn. They are adapter notices, so the model's final text is still + relayed: the turn ends with exactly one model reply.""" + fake_client = FakeOpencodeClient( + prompt_event_sequences=[ + [ + event_message_updated("sess-1", "msg-1"), + event_permission("sess-1", "req-1"), + ] + ], + reply_permission_events={ + "req-1": [ + event_text_part("sess-1", "msg-1", "Approved and done."), + event_session_idle("sess-1"), + ] + }, + ) + adapter = make_adapter(fake_client) + + # Returns once the permission ask releases the turn wait. + await run_single_turn(adapter, tools, content="run it") + await adapter.on_message( + make_platform_message(content="approve req-1"), + tools_protocol(FakeAgentTools()), + OpencodeSessionState(session_id="sess-1", room_id="room-1"), + participants_msg=None, + contacts_msg=None, + is_session_bootstrap=False, + room_id="room-1", + ) + await wait_for(lambda: tools.turn.replied) + + assert [m["content"] for m in tools.messages_sent] == [ + APPROVAL_REQUESTED_TEMPLATE.format( + permission="bash", patterns="rm -rf tmp", request_id="req-1" + ), + APPROVAL_HANDLED_TEMPLATE.format(request_id="req-1", reply="once"), + "Approved and done.", + ] + assert tools.turn.complete + + +async def test_approval_reply_is_settled_and_leaves_the_turn_its_tools( + make_adapter, tools, mcp_backend: FakeMCPBackend +) -> None: + """The approve message runs no model, so its own turn is settled; and the + resumed turn's band tool calls still resolve to the turn's tools.""" + fake_client = FakeOpencodeClient( + prompt_event_sequences=[[event_permission("sess-1", "req-1")]], + reply_permission_events={ + "req-1": [ + mcp_backend.band_tool_call( + BandTool.SEND_MESSAGE, {"content": "Done.", "mentions": ["@alice"]} + ), + event_session_idle("sess-1"), + ] + }, + ) + adapter = make_adapter(fake_client) + approver_tools = FakeAgentTools() + + await run_single_turn(adapter, tools, content="run it") + await adapter.on_message( + make_platform_message(content="approve req-1"), + tools_protocol(approver_tools), + OpencodeSessionState(session_id="sess-1", room_id="room-1"), + participants_msg=None, + contacts_msg=None, + is_session_bootstrap=False, + room_id="room-1", + ) + await wait_for(lambda: tools.turn.replied) + + assert approver_tools.turn.complete + assert approver_tools.messages_sent == [] + assert tools.messages_sent[-1]["content"] == "Done." + + +async def test_busy_message_is_settled_and_leaves_the_turn_its_tools( + make_adapter, tools, mcp_backend: FakeMCPBackend +) -> None: + """A message arriving mid-turn only gets the busy notice, which settles + its own turn; the running turn's band tool calls still resolve to the + turn's tools.""" + fake_client = FakeOpencodeClient( + prompt_event_sequences=[[event_message_updated("sess-1", "msg-1")]] + ) + adapter = make_adapter(fake_client) + busy_tools = FakeAgentTools() + running_turn = asyncio.create_task(run_single_turn(adapter, tools)) + await wait_for(lambda: len(fake_client.prompt_calls) == 1) + + await adapter.on_message( + make_platform_message(content="are you done?"), + tools_protocol(busy_tools), + OpencodeSessionState(session_id="sess-1", room_id="room-1"), + participants_msg=None, + contacts_msg=None, + is_session_bootstrap=False, + room_id="room-1", + ) + await fake_client.push_event( + mcp_backend.band_tool_call( + BandTool.SEND_MESSAGE, {"content": "Done.", "mentions": ["@alice"]} + ) + ) + await fake_client.push_event(event_session_idle("sess-1")) + await running_turn + + assert busy_tools.turn.complete + assert [e["content"] for e in events_of_type(busy_tools, "error")] == [ + "OpenCode is still processing the previous request in this room." + ] + assert [m["content"] for m in tools.messages_sent] == ["Done."] async def test_non_room_posting_tool_does_not_suppress_text( diff --git a/tests/adapters/test_anthropic_adapter.py b/tests/adapters/test_anthropic_adapter.py index 60164c289..71f8da21a 100644 --- a/tests/adapters/test_anthropic_adapter.py +++ b/tests/adapters/test_anthropic_adapter.py @@ -30,6 +30,9 @@ ToolEventKey, TurnUsage, ) +from band.runtime.custom_tools import declares_turn_effect +from band.runtime.tools import TurnEffect +from band.testing import FakeAgentTools from tests.adapters.usage_events import sent_usage_payloads @@ -829,6 +832,50 @@ async def failing_tool(args: EchoInput) -> str: class TestCustomTools: """Tests for custom tool support.""" + @pytest.mark.asyncio + async def test_custom_tool_records_its_effect_on_the_room_turn( + self, sample_message + ): + @declares_turn_effect(TurnEffect.ACT) + async def file_echo(args: EchoInput) -> str: + return "filed" + + tools = FakeAgentTools(room_id="room-123") + adapter = AnthropicAdapter(additional_tools=[(EchoInput, file_echo)]) + await adapter.on_started("TestBot", "Test bot") + call_anthropic = AsyncMock( + side_effect=[ + SimpleNamespace( + stop_reason="tool_use", + content=[ + ToolUseBlock( + type="tool_use", + id="tool-1", + name="echo", + input={"message": "go"}, + ) + ], + usage=make_usage(1, 1), + ), + SimpleNamespace( + stop_reason="end_turn", content=[], usage=make_usage(1, 1) + ), + ] + ) + + with patch.object(adapter, "_call_anthropic", new=call_anthropic): + await adapter.on_message( + msg=sample_message, + tools=tools, + history=[], + participants_msg=None, + contacts_msg=None, + is_session_bootstrap=True, + room_id="room-123", + ) + + assert tools.turn.complete + def test_accepts_additional_tools_parameter(self): """Adapter should accept list of (Model, func) tuples.""" adapter = AnthropicAdapter( diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index c698d8652..5adfd1ab5 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -49,7 +49,7 @@ ) from band.runtime.custom_tools import CustomToolDef, declares_turn_effect from band.runtime.decisions import DecisionRegistry -from band.runtime.tools import ToolCallOutcome, TurnEffect +from band.runtime.tools import BandTool, ToolCallOutcome, TurnEffect from band.testing import FakeAgentTools, events_of_type, reported_failures from tests.adapters.codexturns import RecordedRequests, await_released_turn from tests.paths import host_absolute_path @@ -390,34 +390,40 @@ async def send_bootstrap( class CodexRoom: """A Codex room fed messages one at a time, as Band delivers them: each - ``send`` returns once the adapter hands the room back -- its turn done, - or parked on a human decision.""" + ``send`` gets its own tools and returns once the adapter hands the room + back -- its turn done, or parked on a human decision.""" - def __init__( - self, adapter: CodexAdapter, client: FakeCodexClient, tools: FakeAgentTools - ) -> None: + def __init__(self, adapter: CodexAdapter, client: FakeCodexClient) -> None: self.adapter = adapter self.client = client - self.tools = tools - self._bootstrapped = False + self.deliveries: list[FakeAgentTools] = [] @property def chat(self) -> list[str]: - return [message["content"] for message in self.tools.messages_sent] + """Room messages, grouped by the delivery whose tools posted them.""" + return [ + message["content"] + for tools in self.deliveries + for message in tools.messages_sent + ] + + @property + def events_sent(self) -> list[dict[str, Any]]: + return [event for tools in self.deliveries for event in tools.events_sent] @property def turn(self) -> asyncio.Task[None]: return self.adapter._turn_tasks[ROOM_ID] async def send(self, content: str) -> None: - bootstrap, self._bootstrapped = not self._bootstrapped, True + self.deliveries.append(tools := ToolSchemaFakeTools()) await self.adapter.on_message( make_platform_message(room_id=ROOM_ID, content=content), - self.tools, + tools, CodexSessionState(), participants_msg=None, contacts_msg=None, - is_session_bootstrap=bootstrap, + is_session_bootstrap=len(self.deliveries) == 1, room_id=ROOM_ID, ) @@ -442,7 +448,7 @@ async def open_room( client, config=CodexAdapterConfig(**{"approval_mode": "manual", **config}) ) await adapter.on_started("Agent", "A coding agent") - rooms.append(room := CodexRoom(adapter, client, ToolSchemaFakeTools())) + rooms.append(room := CodexRoom(adapter, client)) return room yield open_room @@ -705,7 +711,7 @@ async def test_approval_request_auto_decline(self) -> None: @pytest.mark.asyncio async def test_auto_approval_responds_even_if_notification_fails(self) -> None: class FailingNotifyTools(ToolSchemaFakeTools): - async def send_message( + async def send_notice( self, content: str, mentions: list[dict[str, str]] | None = None ) -> Any: raise RuntimeError("notification failed") @@ -748,7 +754,7 @@ async def test_manual_approval_responds_with_decline_if_notification_fails( self, ) -> None: class FailingNotifyTools(ToolSchemaFakeTools): - async def send_message( + async def send_notice( self, content: str, mentions: list[dict[str, str]] | None = None ) -> Any: raise RuntimeError("notification failed") @@ -5139,7 +5145,7 @@ async def test_network_context_included_in_approval_metadata( [approval_event] = [ e - for e in room.tools.events_sent + for e in room.events_sent if e["metadata"].get("codex_event_type") == "approval_request" ] assert approval_event["metadata"]["codex_network_context"] == { @@ -6043,20 +6049,20 @@ async def test_local_command_reply_delivery_failure_is_not_reported( self, caplog: pytest.LogCaptureFixture ) -> None: """/help's answer failing to post is Band-side delivery, not a Codex - provider failure -- deliver_reply's DeliveryFailedError must be + provider failure -- its DeliveryFailedError must be recognized and left unreported here. The original cause still propagates (the message still fails/retries at the platform level), just never misreported as a Codex AgentFailure.""" - class FailingSendMessageTools(ToolSchemaFakeTools): - async def send_message( + class FailingNoticeTools(ToolSchemaFakeTools): + async def send_notice( self, content: str, mentions: list[dict[str, str]] | None = None ) -> Any: raise RuntimeError("platform rejected the message") fake_client = FakeCodexClient() adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) - tools = FailingSendMessageTools() + tools = FailingNoticeTools() await adapter.on_started("Agent", "A coding agent") with ( @@ -6087,15 +6093,15 @@ async def test_approval_command_reply_delivery_failure_is_not_reported( the approval-command path, which runs outside on_message's main try/except and needs its own DeliveryFailedError handling.""" - class FailingSendMessageTools(ToolSchemaFakeTools): - async def send_message( + class FailingNoticeTools(ToolSchemaFakeTools): + async def send_notice( self, content: str, mentions: list[dict[str, str]] | None = None ) -> Any: raise RuntimeError("platform rejected the message") fake_client = FakeCodexClient() adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) - tools = FailingSendMessageTools() + tools = FailingNoticeTools() await adapter.on_started("Agent", "A coding agent") with ( @@ -6788,6 +6794,41 @@ async def test_no_reply_does_not_carry_into_the_next_turn( assert room.chat == ["Second answer"] +class TestFinalTextRelay: + @pytest.mark.asyncio + async def test_a_tool_reply_suppresses_the_final_text(self) -> None: + turn = await run_codex_turn( + events=[ + _tool_call_request( + 1, BandTool.SEND_MESSAGE, {"content": "Hi", "mentions": ["@a"]} + ), + _final_text("Hi, again."), + _turn_completed(), + ] + ) + + assert turn.tools.messages_sent == [] + + @pytest.mark.asyncio + async def test_a_policy_notification_never_stands_in_for_the_answer( + self, + ) -> None: + turn = await run_codex_turn( + events=[ + *parked_on_approval(_final_text("Tests pass.")), + _turn_completed(), + ], + config=CodexAdapterConfig( + approval_mode="auto_accept", approval_text_notifications=True + ), + ) + + assert [m["content"] for m in turn.tools.messages_sent] == [ + "Approval requested (command: a). Policy decision: accept.", + "Tests pass.", + ] + + class StayQuietInput(BaseModel): """Say nothing this turn.""" @@ -6823,3 +6864,24 @@ async def stay_quiet(args: StayQuietInput) -> str: ) assert [m["content"] for m in turn.tools.messages_sent] == chat + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("declare", "complete"), + [ + pytest.param(declares_turn_effect(TurnEffect.ACT), True, id="declared"), + pytest.param(_undeclared, False, id="undeclared-observes"), + ], + ) + async def test_the_tool_records_its_effect_on_the_turn( + self, declare: Callable[..., Any], complete: bool + ) -> None: + async def stay_quiet(args: StayQuietInput) -> str: + return "quiet" + + turn = await run_codex_turn( + events=[_tool_call_request(1, "stayquiet"), _turn_completed()], + additional_tools=[(StayQuietInput, declare(stay_quiet))], + ) + + assert turn.tools.turn.complete is complete diff --git a/tests/adapters/test_cursor_acp_adapter.py b/tests/adapters/test_cursor_acp_adapter.py index 2124705d9..dfd2ea6bb 100644 --- a/tests/adapters/test_cursor_acp_adapter.py +++ b/tests/adapters/test_cursor_acp_adapter.py @@ -35,17 +35,26 @@ class DecisionTools(FakeAgentTools): """Room tools that enforce the real mention contract (unlike a hand-rolled - fake, this raises on a mention-less send -- see FakeAgentTools.send_message) - and signal once a pending decision is visible in the room.""" + fake, this raises on a mention-less send -- see FakeAgentTools.send_notice) + and signal once a pending decision is visible in the room. + + Decision prompts and replies are the adapter's own posts, so they go + through ``send_notice``; it honors ``hold_message`` like ``send_message``. + """ def __init__(self) -> None: super().__init__() self.prompt_sent = asyncio.Event() - async def send_message( + async def send_notice( self, content: str, mentions: list[str] | list[dict[str, str]] | None = None ) -> object: - result = await super().send_message(content, mentions) + if (held := self._take_held_message(content)) is not None: + held.sending.set() + await held.released.wait() + if held.error is not None: + raise held.error + result = await super().send_notice(content, mentions) self.prompt_sent.set() return result @@ -59,12 +68,12 @@ def __init__(self, *, fail_after: int = 0) -> None: super().__init__() self._fail_after = fail_after - async def send_message( + async def send_notice( self, content: str, mentions: list[str] | list[dict[str, str]] | None = None ) -> object: if len(self.messages_sent) >= self._fail_after: raise RuntimeError("room delivery failed") - return await super().send_message(content, mentions) + return await super().send_notice(content, mentions) def _turn( @@ -119,11 +128,13 @@ def __init__(self, adapter: CursorACPAdapter, agent: FakeACPAgent) -> None: def chat(self) -> list[str]: return [cast(str, sent["content"]) for sent in self.tools.messages_sent] - async def send(self, content: str) -> None: + async def send(self, content: str, *, tools: FakeAgentTools | None = None) -> None: + """Deliver ``content``; ``tools`` stands in for the message's own + per-event tools, which default to the room's shared ones.""" bootstrap, self._bootstrapped = not self._bootstrapped, True await self.adapter.on_message( room_message(content), - self.tools, + tools or self.tools, ACPClientSessionState(), None, None, @@ -920,6 +931,32 @@ async def test_a_room_reply_resolves_the_decision_it_was_sent_for( class TestCursorACPAdapterControlMessages: + @pytest.mark.asyncio + async def test_a_decision_reply_settles_on_its_own_tools( + self, cursor_room: Callable[..., Awaitable[CursorRoom]] + ) -> None: + """A decision reply is a whole turn of its own: its notice settles + that message's tools, and neither it nor the decision prompt stands + in for the parked turn's answer, which still relays there.""" + room = await cursor_room( + FakeACPAgent() + .will_ask_permission(title="shell", allow_option_id="allow-once") + .will_say("ran it") + ) + await room.send("run it") + [prompt] = room.chat + reply_tools = FakeAgentTools(room_id="room-1") + + await room.send( + f"/cursor select {decision_token(prompt)} allow-once", tools=reply_tools + ) + await room.tools.until_said("ran it") + + assert reply_tools.turn.complete + assert not reply_tools.turn.replied + assert room.chat == [prompt, "ran it"] + assert room.adapter._room_tools["room-1"] is room.tools + @pytest.mark.asyncio async def test_bare_cursor_lists_pending_decisions(self) -> None: tools = DecisionTools() diff --git a/tests/adapters/test_gemini_adapter.py b/tests/adapters/test_gemini_adapter.py index 7c2d046a2..f0d9b506d 100644 --- a/tests/adapters/test_gemini_adapter.py +++ b/tests/adapters/test_gemini_adapter.py @@ -15,6 +15,9 @@ from band.adapters.gemini import GeminiAdapter, GeminiAdapterConfig from band.core.protocols import GENERIC_PROVIDER_FAILURE_MESSAGE from band.core.types import Emit, PlatformMessage, ToolEventKey +from band.runtime.custom_tools import declares_turn_effect +from band.runtime.tools import TurnEffect +from band.testing import FakeAgentTools @pytest.fixture @@ -432,6 +435,45 @@ async def echo_tool(inp: EchoInput) -> str: assert function_response.response == {"output": "hello"} mock_tools.execute_tool_call.assert_not_called() + @pytest.mark.asyncio + async def test_custom_tool_records_its_effect_on_the_room_turn( + self, sample_message + ): + class FileInput(BaseModel): + """File the report.""" + + note: str + + @declares_turn_effect(TurnEffect.ACT) + async def file_report(inp: FileInput) -> str: + return "filed" + + tools = FakeAgentTools(room_id="room-123") + adapter = GeminiAdapter(additional_tools=[(FileInput, file_report)]) + await adapter.on_started("TestBot", "Test bot") + + with patch.object( + adapter, + "_call_gemini", + AsyncMock( + side_effect=[ + _response_with_function_call("file", {"note": "go"}, "call_1"), + _response_with_text(""), + ] + ), + ): + await adapter.on_message( + msg=sample_message, + tools=tools, + history=[], + participants_msg=None, + contacts_msg=None, + is_session_bootstrap=True, + room_id="room-123", + ) + + assert tools.turn.complete + class TestReadRoomFileImagePassthrough: @pytest.mark.asyncio diff --git a/tests/adapters/test_google_adk_adapter.py b/tests/adapters/test_google_adk_adapter.py index b844a6d00..a3805a051 100644 --- a/tests/adapters/test_google_adk_adapter.py +++ b/tests/adapters/test_google_adk_adapter.py @@ -20,7 +20,9 @@ from band.core.protocols import GENERIC_PROVIDER_FAILURE_MESSAGE from band.core.types import ALL_CAPABILITIES, Capability, Emit, PlatformMessage -from band.runtime.tools import AgentTools, BandTool +from band.runtime.custom_tools import declares_turn_effect +from band.runtime.tools import AgentTools, BandTool, TurnEffect +from band.testing import FakeAgentTools pytest.importorskip("google.adk", reason="google-adk not installed") @@ -757,6 +759,25 @@ def calculate(args: CalcInput) -> str: assert "Sum: 5" in calc_result mock_tools.execute_tool_call.assert_not_called() + @pytest.mark.asyncio + async def test_custom_tool_records_its_effect_on_the_room_turn(self): + class FileInput(BaseModel): + """File the report.""" + + note: str + + @declares_turn_effect(TurnEffect.ACT) + async def file_report(args: FileInput) -> str: + return "filed" + + tools = FakeAgentTools(room_id="room-123") + adapter = GoogleADKAdapter(additional_tools=[(FileInput, file_report)]) + (bridge,) = [t for t in adapter._build_adk_tools(tools) if t.name == "file"] + + await bridge.run_async(args={"note": "go"}, tool_context=MagicMock()) + + assert tools.turn.complete + class TestContactsInjection: """Tests for contacts_msg injection.""" diff --git a/tests/adapters/test_letta_adapter.py b/tests/adapters/test_letta_adapter.py index 6e003d7dc..538d16713 100644 --- a/tests/adapters/test_letta_adapter.py +++ b/tests/adapters/test_letta_adapter.py @@ -10,7 +10,7 @@ import json import logging from datetime import UTC, datetime, timedelta -from typing import Any +from typing import Any, Literal from unittest.mock import AsyncMock, MagicMock import pytest @@ -1405,6 +1405,51 @@ async def test_relay_detection_uses_resolved_name(self) -> None: assert len(tools.messages_sent) == 0 +# ────────────────────────────────────────────────────────────────────── +# Turn recording of out-of-process tool calls +# ────────────────────────────────────────────────────────────────────── + + +class TestExternalToolRecording: + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("mcp_mode", "status", "replied"), + [ + pytest.param("external", "success", True, id="external-success"), + pytest.param("external", "error", False, id="external-error"), + # Self-hosted calls run on the room's own tools, which record them. + pytest.param("self_host", "success", False, id="self-host"), + ], + ) + async def test_streamed_tool_return_records_only_for_external_band_mcp( + self, mcp_mode: Literal["self_host", "external"], status: str, replied: bool + ) -> None: + adapter = LettaAdapter( + config=LettaAdapterConfig(mcp=LettaMCPConfig(mode=mcp_mode)) + ) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.server_id = "mcp-server-1" + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_return_message(BandTool.SEND_MESSAGE, status=status), + ) + tools = FakeAgentTools() + + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert tools.turn.replied is replied + + # ────────────────────────────────────────────────────────────────────── # Auto-relay knob # ────────────────────────────────────────────────────────────────────── diff --git a/tests/adapters/test_parlant_adapter.py b/tests/adapters/test_parlant_adapter.py index dcceabf13..af48e6ead 100644 --- a/tests/adapters/test_parlant_adapter.py +++ b/tests/adapters/test_parlant_adapter.py @@ -21,6 +21,7 @@ ) from band.core.protocols import GENERIC_PROVIDER_FAILURE_MESSAGE from band.core.types import PlatformMessage +from band.testing import FakeAgentTools, reported_failures @pytest.fixture @@ -40,16 +41,9 @@ def sample_message(): @pytest.fixture -def mock_tools(): - """Create mock AgentToolsProtocol (MagicMock base, AsyncMock methods).""" - tools = MagicMock() - tools.get_tool_schemas = MagicMock(return_value=[]) - tools.get_openai_tool_schemas = MagicMock(return_value=[]) - tools.send_message = AsyncMock(return_value={"status": "sent"}) - tools.send_event = AsyncMock(return_value={"status": "sent"}) - tools.send_failure = AsyncMock(return_value={"status": "sent"}) - tools.execute_tool_call = AsyncMock(return_value={"status": "success"}) - return tools +def mock_tools() -> FakeAgentTools: + """The room's tools, recording what the adapter posted.""" + return FakeAgentTools() @pytest.fixture @@ -828,10 +822,9 @@ async def test_reports_error_on_failure( ) # Should have tried to report the failure - mock_tools.send_failure.assert_awaited_once() - failure = mock_tools.send_failure.call_args.args[0] - assert failure.provider == "parlant" - assert failure.message == GENERIC_PROVIDER_FAILURE_MESSAGE + [failure] = reported_failures(mock_tools) + assert failure["provider"] == "parlant" + assert failure["message"] == GENERIC_PROVIDER_FAILURE_MESSAGE @pytest.mark.asyncio async def test_reports_error_on_session_init_failure( @@ -860,10 +853,9 @@ async def test_reports_error_on_session_init_failure( room_id="room-123", ) - mock_tools.send_failure.assert_awaited_once() - failure = mock_tools.send_failure.call_args.args[0] - assert failure.provider == "parlant" - assert failure.message == GENERIC_PROVIDER_FAILURE_MESSAGE + [failure] = reported_failures(mock_tools) + assert failure["provider"] == "parlant" + assert failure["message"] == GENERIC_PROVIDER_FAILURE_MESSAGE @pytest.mark.asyncio async def test_send_message_failure_is_not_reported_as_provider_failure( @@ -900,7 +892,7 @@ async def test_send_message_failure_is_not_reported_as_provider_failure( mock_app.sessions.find_events = AsyncMock(return_value=[agent_event]) adapter._app = mock_app - mock_tools.send_message.side_effect = ConnectionError("band down") + mock_tools.send_message_error = ConnectionError("band down") mock_moderation = MagicMock() mock_moderation.NONE = "none" @@ -931,7 +923,7 @@ async def test_send_message_failure_is_not_reported_as_provider_failure( room_id="room-123", ) - mock_tools.send_failure.assert_not_awaited() + assert not reported_failures(mock_tools) @pytest.mark.asyncio async def test_clears_tools_on_error( @@ -1006,11 +998,10 @@ async def test_handles_uninitialized_app( ) # No reply attempt, but the failure is reported. - mock_tools.send_message.assert_not_called() - mock_tools.send_failure.assert_awaited_once() - failure = mock_tools.send_failure.call_args.args[0] - assert failure.provider == "parlant" - assert "not initialized" in failure.message + mock_tools.assert_no_messages_sent() + [failure] = reported_failures(mock_tools) + assert failure["provider"] == "parlant" + assert "not initialized" in failure["message"] class TestResponseWaitBudget: @@ -1090,8 +1081,8 @@ async def test_retries_past_empty_windows_then_forwards_late_reply( # The reply is only returned on the 3rd wait, so forwarding it proves the loop # retried past both empty windows instead of giving up on the first. - mock_tools.send_message.assert_awaited_once_with( - "Hello there!", mentions=["Alice"] + mock_tools.assert_message_sent( + content="Hello there!", mentions=["Alice"], count=1 ) @pytest.mark.asyncio @@ -1125,7 +1116,7 @@ async def test_gives_up_after_budget_when_no_reply_ever_arrives( # It gave up within the budget (the wait_for above would raise on a hang) # without forwarding anything. - mock_tools.send_message.assert_not_awaited() + mock_tools.assert_no_messages_sent() @pytest.mark.asyncio async def test_preamble_only_times_out_without_forwarding_a_reply( @@ -1171,7 +1162,7 @@ def _preamble_once(*_args, **_kwargs): # The preamble was NOT forwarded — a stalled turn fails honestly, not silently # dressed up as an answer. - mock_tools.send_message.assert_not_awaited() + mock_tools.assert_no_messages_sent() @pytest.mark.asyncio async def test_empty_find_events_then_final_still_forwards( @@ -1202,6 +1193,6 @@ async def test_empty_find_events_then_final_still_forwards( # The final is only query-visible on the 2nd read, so forwarding it proves the # loop re-polled past the empty read instead of dropping the turn. - mock_tools.send_message.assert_awaited_once_with( - "The answer.", mentions=["Alice"] + mock_tools.assert_message_sent( + content="The answer.", mentions=["Alice"], count=1 ) diff --git a/tests/adapters/test_pydantic_ai_adapter.py b/tests/adapters/test_pydantic_ai_adapter.py index 6f82b85a8..995c530ee 100644 --- a/tests/adapters/test_pydantic_ai_adapter.py +++ b/tests/adapters/test_pydantic_ai_adapter.py @@ -72,7 +72,11 @@ PlatformMessage, TurnUsage, ) -from band.runtime.custom_tools import get_custom_tool_name +from band.runtime.custom_tools import ( + CustomToolDef, + declares_turn_effect, + get_custom_tool_name, +) from band.runtime.tools import ( ALL_TOOL_NAMES, CHAT_TOOL_NAMES, @@ -87,6 +91,7 @@ get_tool_description, platform_args_schema, ) +from band.testing import FakeAgentTools from tests.adapters.usage_events import sent_usage_payloads from tests.framework_configs.adapters import pydantic_ai_probe_tools @@ -451,7 +456,7 @@ def reply_via_tool_then_nothing( ) adapter.agent_name = "TestBot" - result = await adapter._create_agent().run("go", deps=MagicMock()) + result = await adapter._create_agent().run("go", deps=FakeAgentTools()) assert result.output is None assert posted == ["hi"] @@ -2403,6 +2408,25 @@ async def my_helper(ctx: RunContext[AgentToolsProtocol], value: str) -> str: assert "room-123" in adapter._message_history +def _portable_tool_agent( + tool_def: CustomToolDef, +) -> Agent[AgentToolsProtocol, str]: + """A bare agent running one converted ``CustomToolDef`` on pydantic-ai's own path.""" + agent = Agent(TestModel(), deps_type=AgentToolsProtocol, output_type=str) + agent.tool(_custom_tool_def_to_callable(tool_def)) + return agent + + +async def _run_portable_tool( + tool_def: CustomToolDef, tools: FakeAgentTools | None = None +) -> list[Any]: + """Run one converted ``CustomToolDef`` in a room; return what the model saw.""" + result = await _portable_tool_agent(tool_def).run( + "go", deps=tools or FakeAgentTools() + ) + return _tool_returns(result) + + class TestPortableCustomToolDef: """pydantic accepts the portable CustomToolDef (InputModel, handler) tuple form — the same custom-tool shape anthropic/crewai/claude_sdk/langgraph take.""" @@ -2424,26 +2448,6 @@ def lookup(args: LookupInput) -> str: ) # Normalized to a native callable named from the model (not the handler). assert [t.__name__ for t in adapter._custom_tools] == ["lookup"] - # ...and it still delegates to the handler (async — execution routes - # through the shared execute_custom_tool). - assert await adapter._custom_tools[0](LookupInput(key="alpha")) == "code:alpha" - - @pytest.mark.asyncio - async def test_async_handler_is_awaited(self): - """An async portable handler must be awaited (not returned as a coroutine) — - the same shared-executor path every other adapter uses.""" - - class LookupInput(BaseModel): - key: str - - async def lookup(args: LookupInput) -> str: - return f"code:{args.key}" - - adapter = PydanticAIAdapter( - PydanticAIAdapterConfig(model="openai:gpt-5.4"), - additional_tools=[(LookupInput, lookup)], - ) - assert await adapter._custom_tools[0](LookupInput(key="beta")) == "code:beta" def test_tuple_terminal_marker_is_honored(self): @@ -2473,25 +2477,14 @@ class LookupInput(BaseModel): def lookup(args: LookupInput) -> str: return f"code:{args.key}" - native = _custom_tool_def_to_callable((LookupInput, lookup)) - agent = Agent(TestModel()) - agent.tool_plain(native) - (tool,) = agent._function_toolset.tools.values() + (tool,) = _portable_tool_agent( + (LookupInput, lookup) + )._function_toolset.tools.values() schema = tool.function_schema.json_schema # pydantic-ai flattens the single model param into the tool's args. assert tool.name == "lookup" assert sorted((schema.get("properties") or {}).keys()) == ["key"] - @staticmethod - def _tool_return_contents(result) -> list: - - return [ - part.content - for message in result.all_messages() - for part in message.parts - if isinstance(part, ToolReturnPart) - ] - @pytest.mark.asyncio async def test_async_handler_tuple_is_awaited_end_to_end(self): """An async CustomToolDef handler returns its awaited value through a real @@ -2506,13 +2499,7 @@ class LookupInput(BaseModel): async def lookup(args: LookupInput) -> str: return f"code:{args.key}" - native = _custom_tool_def_to_callable((LookupInput, lookup)) - agent = Agent(TestModel(), output_type=str) - agent.tool_plain(native) - - result = await agent.run("go") - - (content,) = self._tool_return_contents(result) + (content,) = await _run_portable_tool((LookupInput, lookup)) assert isinstance(content, str) assert content.startswith("code:") @@ -2528,13 +2515,7 @@ class PingInput(BaseModel): def ping() -> str: return "pong" - native = _custom_tool_def_to_callable((PingInput, ping)) - agent = Agent(TestModel(), output_type=str) - agent.tool_plain(native) - - result = await agent.run("go") - - assert self._tool_return_contents(result) == ["pong"] + assert await _run_portable_tool((PingInput, ping)) == ["pong"] @pytest.mark.asyncio async def test_aliased_input_model_runs_end_to_end(self): @@ -2550,12 +2531,28 @@ class AliasedInput(BaseModel): def lookup(args: AliasedInput) -> str: return f"user:{args.user_id}" - native = _custom_tool_def_to_callable((AliasedInput, lookup)) - agent = Agent(TestModel(), output_type=str) - agent.tool_plain(native) - - result = await agent.run("go") - - (content,) = self._tool_return_contents(result) + (content,) = await _run_portable_tool((AliasedInput, lookup)) assert isinstance(content, str) assert content.startswith("user:") + + @pytest.mark.asyncio + async def test_declared_tool_records_its_effect_on_the_room_turn(self): + class FileInput(BaseModel): + """File the report.""" + + note: str + + @declares_turn_effect(TurnEffect.ACT) + def file_report(args: FileInput) -> str: + return "filed" + + tools = FakeAgentTools() + adapter = PydanticAIAdapter( + llm=_scripted_tool_calls(("file", {"note": "go"})), + additional_tools=[(FileInput, file_report)], + ) + adapter.agent_name = "TestBot" + + await adapter._create_agent().run("go", deps=tools) + + assert tools.turn.complete diff --git a/tests/adapters/test_strands_adapter.py b/tests/adapters/test_strands_adapter.py index 43779fec5..e4da40fc6 100644 --- a/tests/adapters/test_strands_adapter.py +++ b/tests/adapters/test_strands_adapter.py @@ -54,6 +54,7 @@ TurnUsage, is_usage_event, ) +from band.runtime.custom_tools import declares_turn_effect from band.runtime.tools import TurnEffect, get_tool_description from band.testing import ( ErrorTurn, @@ -152,7 +153,8 @@ def _alternates(history: list) -> bool: class TestCustomToolWiring: - def test_custom_tool_def_converted_to_bridge(self): + @pytest.mark.asyncio + async def test_custom_tool_def_converted_to_bridge(self): class WeatherInput(BaseModel): """Get the weather for a city.""" @@ -166,8 +168,7 @@ async def get_weather(args: WeatherInput) -> str: additional_tools=[(WeatherInput, get_weather)], ) - assert len(adapter._custom_tools) == 1 - bridge = adapter._custom_tools[0] + bridge = (await _turn_agent(adapter)).tool_registry.registry["weather"] assert isinstance(bridge, CustomToolBridge) assert bridge.tool_name == "weather" assert bridge.tool_spec["description"] == "Get the weather for a city." @@ -678,6 +679,28 @@ async def boom(args: BoomInput) -> str: assert _tool_results(adapter) == ["Error executing tool 'boom': no network"] + @pytest.mark.asyncio + async def test_custom_tool_records_its_effect_on_the_room_turn( + self, tools, scripted + ): + class FileInput(BaseModel): + """File the report.""" + + note: str + + @declares_turn_effect(TurnEffect.ACT) + async def file_report(args: FileInput) -> str: + return "filed" + + adapter = await scripted( + ToolTurn("file", {"note": "go"}), + additional_tools=[(FileInput, file_report)], + ) + + await _run_message(adapter, tools) + + assert tools.turn.complete + class TestTurnFailure: @pytest.mark.asyncio diff --git a/tests/core/test_delivery.py b/tests/core/test_delivery.py index 3426bcd34..42245d80e 100644 --- a/tests/core/test_delivery.py +++ b/tests/core/test_delivery.py @@ -4,7 +4,12 @@ import pytest -from band.core.delivery import DeliveryFailedError, deliver_reply +from band.core.delivery import ( + DeliveryFailedError, + deliver_notice, + deliver_reply, + relay_reply, +) from band.testing.fake_tools import FakeAgentTools @@ -30,3 +35,54 @@ async def test_wraps_a_send_message_failure(self) -> None: assert exc_info.value.__cause__ is exc_info.value.cause assert "mention" in str(exc_info.value.cause).lower() + + +class TestDeliverNotice: + async def test_posts_without_counting_as_the_reply(self) -> None: + tools = FakeAgentTools() + + await deliver_notice(tools, "No pending approvals.", mentions=["@alice"]) + + assert [m["content"] for m in tools.messages_sent] == ["No pending approvals."] + assert not tools.turn.complete + + async def test_wraps_a_send_notice_failure(self) -> None: + tools = FakeAgentTools() + + with pytest.raises(DeliveryFailedError): + await deliver_notice(tools, "No pending approvals.", mentions=None) + + +class TestRelayReply: + async def test_relays_the_final_text_as_the_turn_reply(self) -> None: + tools = FakeAgentTools() + + assert await relay_reply(tools, "the answer", ["@alice"]) is True + + assert [m["content"] for m in tools.messages_sent] == ["the answer"] + assert tools.turn.replied + + async def test_a_tool_reply_suppresses_the_relay(self) -> None: + tools = FakeAgentTools() + await tools.send_message("already answered", mentions=["@alice"]) + + assert await relay_reply(tools, "closing remark", ["@alice"]) is False + + assert [m["content"] for m in tools.messages_sent] == ["already answered"] + + async def test_a_decline_suppresses_the_relay(self) -> None: + tools = FakeAgentTools() + await tools.no_reply("FYI only") + + assert await relay_reply(tools, "closing remark", ["@alice"]) is False + + tools.assert_no_messages_sent() + + @pytest.mark.parametrize("text", [None, "", " \n"]) + async def test_nothing_visible_relays_nothing(self, text: str | None) -> None: + tools = FakeAgentTools() + + assert await relay_reply(tools, text, ["@alice"]) is False + + tools.assert_no_messages_sent() + assert not tools.turn.complete diff --git a/tests/core/test_turn_types.py b/tests/core/test_turn_types.py new file mode 100644 index 000000000..8ff246e92 --- /dev/null +++ b/tests/core/test_turn_types.py @@ -0,0 +1,35 @@ +"""Parity of Python's turn-effect vocabulary with band-sdk-core's. + +``TurnEffect`` stays a local ``StrEnum`` because it is public API +(``@declares_turn_effect(TurnEffect.ACT)``), and core's classes are opaque PyO3 +types. These tests keep the two equivalent. +""" + +from __future__ import annotations + +import band_sdk_core +import pytest + +from band.runtime.tools import ALL_TOOL_NAMES, TurnEffect, turn_effect + +CORE_EFFECTS = band_sdk_core.band_tool_effects() + + +def test_effect_values_match_core() -> None: + core_values = { + member.wire_name + for name in dir(band_sdk_core.TurnEffect) + if isinstance( + member := getattr(band_sdk_core.TurnEffect, name), band_sdk_core.TurnEffect + ) + } + assert {effect.value for effect in TurnEffect} == core_values + + +def test_band_tools_match_core() -> None: + assert ALL_TOOL_NAMES == set(CORE_EFFECTS) + + +@pytest.mark.parametrize("tool_name", sorted(CORE_EFFECTS)) +def test_turn_effect_matches_core(tool_name: str) -> None: + assert turn_effect(tool_name) == CORE_EFFECTS[tool_name].wire_name diff --git a/tests/framework_conformance/test_reply_boundary.py b/tests/framework_conformance/test_reply_boundary.py new file mode 100644 index 000000000..688f5fec2 --- /dev/null +++ b/tests/framework_conformance/test_reply_boundary.py @@ -0,0 +1,77 @@ +"""Reply boundary test. + +A ``send_message`` call counts as the turn's reply, so only the model's own +words may go through it: the tool methods, ``deliver_reply``/``relay_reply``, +and the framework tool implementations that call them. An adapter's own post +(an approval prompt, a busy notice, a status reply) goes through +``send_notice``, or it would stand in for the model's answer and suppress the +final-text relay. This scans ``src/band`` via AST for a direct +``x.send_message(...)`` call outside the allowlist, so a new adapter post on +``send_message`` fails here instead of silently counting as a reply. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +from tests.paths import REPO_ROOT + +_SCAN_ROOT = REPO_ROOT / "src" / "band" + +# The model's tool implementations and the relay that carries its words, plus +# two files where ``send_message`` is an unrelated client's method. +_ALLOWED_DIRS: frozenset[Path] = frozenset({_SCAN_ROOT / "runtime" / "tools"}) +_ALLOWED_FILES: frozenset[Path] = frozenset( + _SCAN_ROOT / path + for path in ( + "core/delivery.py", + "integrations/crewai/catalog.py", + "integrations/parlant/tools.py", + # Forwards to the inner tools' send_message, which records the reply. + "integrations/claude_sdk/dedup_tools.py", + # Exempt from the turn verdict: the flow posts its own outcomes. + "adapters/crewai_flow.py", + # The A2A client's own method. + "integrations/a2a/adapter.py", + # The Phoenix protocol's own method. + "testing/phoenix_server.py", + ) +) + + +def _calls_send_message(source: str) -> bool: + return any( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "send_message" + for node in ast.walk(ast.parse(source)) + ) + + +def _is_allowed(path: Path) -> bool: + return path in _ALLOWED_FILES or any(d in path.parents for d in _ALLOWED_DIRS) + + +def test_adapter_posts_never_count_as_the_model_reply() -> None: + offenders = sorted( + path.relative_to(REPO_ROOT) + for path in _SCAN_ROOT.rglob("*.py") + if not _is_allowed(path) and _calls_send_message(path.read_text("utf-8")) + ) + + assert not offenders, ( + f"Direct send_message calls outside the model's reply paths: {offenders}. " + "Post an adapter's own message with tools.send_notice (and settle the " + "turn with tools.turn.settle() when it ends the turn), or relay the " + "model's words through band.core.delivery.deliver_reply/relay_reply." + ) + + +def test_allowlist_entries_still_exist() -> None: + missing = [ + path.relative_to(REPO_ROOT) + for path in _ALLOWED_FILES | _ALLOWED_DIRS + if not path.exists() + ] + assert not missing, f"Allowlisted paths no longer exist: {missing}" diff --git a/tests/integrations/acp/test_client_adapter.py b/tests/integrations/acp/test_client_adapter.py index 163324b2b..81567262c 100644 --- a/tests/integrations/acp/test_client_adapter.py +++ b/tests/integrations/acp/test_client_adapter.py @@ -38,9 +38,7 @@ ACPClientSessionState, BandACPClient, ) -from band.integrations.acp.room_emitter import turn_replied_in_room -from band.integrations.acp.types import ACPToolCall, ACPToolResult, CollectedChunk -from band.runtime.tools import TurnEffect +from band.integrations.acp.types import ACPToolCall from band.testing import FakeAgentTools, events_of_type, reported_failures from tests.integrations.acp.acp_toolkit.harness import ( Launch, @@ -2409,116 +2407,6 @@ def test_unresolved_name_is_left_as_is(self) -> None: assert _resolve_launcher(["mystery-bin", "arg"]) == ["mystery-bin", "arg"] -class TestTurnRepliedInRoom: - """`turn_replied_in_room`: detect a room post from the ACP tool-call stream. - - ACP has no structured tool-name field and tools may run out-of-process, so the - adapter reads the collected chunk stream. These lock the id-correlation edges. - """ - - @staticmethod - def _chunk(chunk_type: str, content: str, **metadata: object) -> CollectedChunk: - tool_call_id = str(metadata.get("tool_call_id", "")) - call = ACPToolCall( - tool_call_id=tool_call_id, - name=content if chunk_type == "tool_call" else "unknown", - arguments={}, - ) - tool = ( - call - if chunk_type == "tool_call" - else ACPToolResult(call=call, output=content, status=metadata.get("status")) - ) - return CollectedChunk( - chunk_type=chunk_type, - content=content, - metadata=metadata, - tool=tool, - ) - - def test_completed_posting_tool_call_counts_as_reply(self) -> None: - chunks = [ - self._chunk( - "tool_call", - "band_send_message", - tool_call_id="tc-1", - status="completed", - ) - ] - assert turn_replied_in_room(chunks) - - def test_posting_call_correlated_to_completed_result_counts(self) -> None: - # The tool_call arrives before its terminal status; the completed result seals it. - chunks = [ - self._chunk( - "tool_call", - "band_send_message", - tool_call_id="tc-1", - status="in_progress", - ), - self._chunk("tool_result", "", tool_call_id="tc-1", status="completed"), - ] - assert turn_replied_in_room(chunks) - - def test_empty_ids_do_not_cross_match(self) -> None: - # A not-yet-completed posting call with NO id and a completed NON-posting result - # with NO id both default to "" — they must not correlate, or the text fallback - # is falsely suppressed and the turn goes silent. - chunks = [ - self._chunk("tool_call", "band_send_message", status="in_progress"), - self._chunk("tool_result", "", status="completed"), - ] - assert not turn_replied_in_room(chunks) - - def test_non_posting_tool_never_counts(self) -> None: - chunks = [ - self._chunk( - "tool_call", "get_weather", tool_call_id="tc-1", status="completed" - ) - ] - assert not turn_replied_in_room(chunks) - - @pytest.mark.parametrize("name", ["band_no_reply", "band-band_no_reply"]) - def test_completed_no_reply_settles_the_turn(self, name: str) -> None: - chunks = [ - self._chunk("tool_call", name, tool_call_id="tc-1", status="completed") - ] - assert turn_replied_in_room(chunks) - - def test_failed_no_reply_keeps_the_text_fallback(self) -> None: - chunks = [ - self._chunk( - "tool_call", "band_no_reply", tool_call_id="tc-1", status="failed" - ) - ] - assert not turn_replied_in_room(chunks) - - def test_custom_tool_declaring_silence_settles_the_turn(self) -> None: - chunks = [ - self._chunk( - "tool_call", "stayquiet", tool_call_id="tc-1", status="completed" - ) - ] - assert turn_replied_in_room( - chunks, custom_effects={"stayquiet": TurnEffect.DECLINE} - ) - assert not turn_replied_in_room(chunks) - - def test_foreign_mcp_servers_own_tool_never_counts(self) -> None: - """A non-Band MCP server's own tool that happens to end in - ``-band_send_message`` must not suppress the text fallback -- only the - Band loopback server's own ``band-`` prefix counts as a room post.""" - chunks = [ - self._chunk( - "tool_call", - "other-band_send_message", - tool_call_id="tc-1", - status="completed", - ) - ] - assert not turn_replied_in_room(chunks) - - class TestACPClientAdapterEmitSupport: """Which ``Emit`` kinds the ACP client adapter declares. diff --git a/tests/integrations/acp/test_client_adapter_behavior.py b/tests/integrations/acp/test_client_adapter_behavior.py index 74053f0f1..bc4cc37ad 100644 --- a/tests/integrations/acp/test_client_adapter_behavior.py +++ b/tests/integrations/acp/test_client_adapter_behavior.py @@ -114,12 +114,11 @@ async def test_streamed_text_deltas_become_one_message(fake_agent) -> None: @pytest.mark.asyncio async def test_band_tool_call_suppresses_text_fallback(fake_agent) -> None: - # Detection-only: the ACP stream *reports* a completed band_send_message call, - # but the fake doesn't actually post — will_call_tool only emits the frames — so - # this pins the suppression decision (tool-first delivery, matching copilot_sdk / - # codex), not the post. The end-to-end "exactly one visible reply" outcome, where - # a real band-mcp tool posts, is covered by - # test_band_mcp_reply_is_not_replayed_as_acp_tool_events (inject_band_tools=True). + # Out-of-process: an external band-mcp posts where the SDK never sees it, so + # the turn records the completed call from the ACP stream. will_call_tool only + # emits the frames, so this pins the suppression decision, not the post. The + # in-process outcome, where a real band-mcp tool posts, is covered by + # test_in_process_band_reply_suppresses_text_fallback. fake_agent.will_say("Posting the answer to the room now.").will_call_tool( "tc-1", "band_send_message", result='{"id": "msg-1"}' ) @@ -140,7 +139,7 @@ async def test_prefixed_legacy_band_tool_call_suppresses_text_fallback( # where an MCP client prefixes the server name onto the (legacy) tool name. The # in-process LocalMCPServer advertises the SDK-native names, so this prefixed # `band-create_agent_chat_message` spelling has no real-post equivalent; this - # test pins that settles_turn_reply still matches it. + # test pins that its effect still resolves to a reply. fake_agent.will_call_tool( "tc-1", "band-create_agent_chat_message", result='{"id": "msg-1"}' ).will_say("Done — posted the answer.") @@ -151,6 +150,44 @@ async def test_prefixed_legacy_band_tool_call_suppresses_text_fallback( assert reply.outline == ["tool_call", "tool_result", "task"] +@pytest.mark.asyncio +async def test_in_process_band_reply_suppresses_text_fallback(fake_agent) -> None: + """An injected Band tool's post is the turn's one reply; the closing + narration is not relayed after it.""" + fake_agent.will_call_mcp_tool( + "tc-message", + "band_send_message", + arguments={ + "room_id": "room-1", + "content": "Reply from the agent", + "mentions": ["@pat"], + }, + ).will_say("Posted the answer.") + + async with acp_adapter( + fake_agent, fake_agent_config(inject_band_tools=True) + ) as session: + reply = await session.send("question?", room="room-1") + + assert reply.texts == ["Reply from the agent"] + + +@pytest.mark.asyncio +async def test_injected_tools_never_settle_from_the_stream(fake_agent) -> None: + """With Band tools injected, only the tool that really posts settles the + reply: a call the stream merely reports keeps the text fallback.""" + fake_agent.will_call_tool( + "tc-1", "band_send_message", result='{"id": "msg-1"}' + ).will_say("The answer is 42.") + + async with acp_adapter( + fake_agent, fake_agent_config(inject_band_tools=True) + ) as session: + reply = await session.send("question?", room="room-1") + + assert reply.texts == ["The answer is 42."] + + @pytest.mark.asyncio async def test_text_relayed_when_band_post_failed(fake_agent) -> None: # A failed post must not suppress the text fallback, or the turn goes silent. diff --git a/tests/integrations/acp/test_room_emitter.py b/tests/integrations/acp/test_room_emitter.py index 503f8ffdd..9563e234f 100644 --- a/tests/integrations/acp/test_room_emitter.py +++ b/tests/integrations/acp/test_room_emitter.py @@ -25,6 +25,7 @@ ToolResultRoomEvent, ToolStatus, ) +from band.runtime.tools import BAND_MCP_SERVER_NAME, BandTool, mcp_tool_spelling from band.runtime.tools.agent import AgentTools from band.testing.fake_tools import FakeAgentTools @@ -222,9 +223,9 @@ async def test_every_emit_set_keeps_session_resume( @pytest.mark.asyncio async def test_silenced_turn_still_suppresses_duplicated_text(self) -> None: - """A turn that answered in the room via a Band messaging tool must - not also relay its held text — even with ``emit=()`` hiding that - tool call from the room.""" + """A turn that answered in the room via an out-of-process Band + messaging tool must not also relay its held text — even with + ``emit=()`` hiding that tool call from the room.""" call = ACPToolCall( tool_call_id="tc-9", name="band_send_message", @@ -238,6 +239,7 @@ async def test_silenced_turn_still_suppresses_duplicated_text(self) -> None: session_id="s1", room_id="room-1", emit=frozenset(), + records_tool_effects=True, ) async with emitter: @@ -287,3 +289,165 @@ async def test_a_denied_permission_pair_follows_the_tool_call_gate(self) -> None "tool_call", "tool_result", ] + + +def tool_call_chunk( + name: str, status: ToolStatus, *, tool_call_id: str = "tc-1" +) -> CollectedChunk: + call = ACPToolCall(tool_call_id=tool_call_id, name=name, arguments={}) + return CollectedChunk( + chunk_type=ChunkType.TOOL_CALL, + content=name, + metadata={"status": status}, + tool=call, + ) + + +def tool_result_chunk(name: str, status: ToolStatus) -> CollectedChunk: + """A result the runtime correlated to its in-progress call.""" + call = ACPToolCall(tool_call_id="tc-1", name=name, arguments={}) + return CollectedChunk( + chunk_type=ChunkType.TOOL_RESULT, + content="", + metadata={"status": status}, + tool=ACPToolResult(call=call, output="", status=status), + ) + + +async def relayed_texts( + tools: FakeAgentTools, + *chunks: CollectedChunk, + records_tool_effects: bool, +) -> list[str]: + """Run one turn of ``chunks`` and return the messages it posted.""" + emitter = RoomTurnEmitter( + tools, + mentions=[{"id": "u1", "name": "User"}], + session_id="s1", + room_id="room-1", + records_tool_effects=records_tool_effects, + ) + async with emitter: + for chunk in chunks: + await emitter.emit(chunk) + return [message["content"] for message in tools.messages_sent] + + +def text(content: str) -> CollectedChunk: + return CollectedChunk(chunk_type=ChunkType.TEXT, content=content) + + +class TestRoomTurnEmitterReplyRelay: + """The held text relays once, unless a tool call replied or declined. + + In-process Band tools record their own effect on ``tools.turn``; an + external band-mcp's calls are seen only in the stream, so the emitter + records them itself, and only in that mode. + """ + + @pytest.mark.asyncio + async def test_held_runs_relay_as_one_reply(self) -> None: + tools = FakeAgentTools() + + sent = await relayed_texts( + tools, + text("Checking."), + tool_call_chunk("shell", ToolStatus.COMPLETED), + text("Done."), + records_tool_effects=False, + ) + + assert sent == ["Checking.\n\nDone."] + assert tools.turn.replied + + @pytest.mark.asyncio + async def test_an_in_process_reply_suppresses_the_text(self) -> None: + tools = FakeAgentTools() + await tools.send_message("Posted by the tool.", mentions=["u1"]) + + sent = await relayed_texts( + tools, + tool_call_chunk(BandTool.SEND_MESSAGE, ToolStatus.COMPLETED), + text("I posted it."), + records_tool_effects=False, + ) + + assert sent == ["Posted by the tool."] + + @pytest.mark.asyncio + async def test_in_process_mode_never_records_from_the_stream(self) -> None: + """The stream reports a call the in-process tool already recorded (or + that never posted); only the tool itself may settle the reply.""" + sent = await relayed_texts( + FakeAgentTools(), + tool_call_chunk(BandTool.SEND_MESSAGE, ToolStatus.COMPLETED), + text("The answer."), + records_tool_effects=False, + ) + + assert sent == ["The answer."] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "chunks", + [ + [tool_call_chunk(BandTool.SEND_MESSAGE, ToolStatus.COMPLETED)], + [ + tool_call_chunk(BandTool.SEND_MESSAGE, ToolStatus.IN_PROGRESS), + tool_result_chunk(BandTool.SEND_MESSAGE, ToolStatus.COMPLETED), + ], + [tool_call_chunk(BandTool.NO_REPLY, ToolStatus.COMPLETED)], + [ + tool_call_chunk( + mcp_tool_spelling(BAND_MCP_SERVER_NAME, BandTool.NO_REPLY), + ToolStatus.COMPLETED, + ) + ], + ], + ids=["completed-call", "completed-result", "no-reply", "mcp-spelled-no-reply"], + ) + async def test_an_out_of_process_reply_or_decline_suppresses_the_text( + self, chunks: list[CollectedChunk] + ) -> None: + tools = FakeAgentTools() + + sent = await relayed_texts( + tools, *chunks, text("Narration."), records_tool_effects=True + ) + + assert sent == [] + assert tools.turn.replied + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "chunk", + [ + tool_call_chunk(BandTool.NO_REPLY, ToolStatus.FAILED), + tool_call_chunk(BandTool.SEND_MESSAGE, ToolStatus.IN_PROGRESS), + tool_call_chunk("get_weather", ToolStatus.COMPLETED), + tool_call_chunk( + mcp_tool_spelling("other", BandTool.SEND_MESSAGE), ToolStatus.COMPLETED + ), + ], + ids=["failed-no-reply", "unfinished-send", "non-band-tool", "foreign-server"], + ) + async def test_an_out_of_process_call_that_did_not_reply_keeps_the_text( + self, chunk: CollectedChunk + ) -> None: + sent = await relayed_texts( + FakeAgentTools(), chunk, text("The answer."), records_tool_effects=True + ) + + assert sent == ["The answer."] + + @pytest.mark.asyncio + async def test_an_out_of_process_action_completes_the_turn(self) -> None: + tools = FakeAgentTools() + + await relayed_texts( + tools, + tool_call_chunk(BandTool.CREATE_TASK, ToolStatus.COMPLETED), + records_tool_effects=True, + ) + + assert tools.turn.complete diff --git a/tests/integrations/langgraph/test_custom_tool_defs.py b/tests/integrations/langgraph/test_custom_tool_defs.py index 72463d8b0..ddd76573d 100644 --- a/tests/integrations/langgraph/test_custom_tool_defs.py +++ b/tests/integrations/langgraph/test_custom_tool_defs.py @@ -3,20 +3,28 @@ Every adapter takes custom tools as ``(InputModel, handler)`` tuples; LangGraph historically took only ready-made LangChain tools, so a bare tuple reached LangChain and raised "the first argument must be a string or a callable ... Got -". These tests cover the converter and the adapter's one-time -normalization (tuples -> StructuredTools; native LangChain tools pass through). +". These tests cover the converter and the adapter's per-turn +conversion (tuples -> StructuredTools bound to the room's turn; native LangChain +tools pass through). """ from __future__ import annotations -from unittest.mock import MagicMock +from collections.abc import AsyncIterator +from datetime import UTC, datetime +from typing import Any from langchain_core.tools import StructuredTool from langchain_core.tools import tool as lc_tool from pydantic import BaseModel, Field from band.adapters.langgraph import LangGraphAdapter -from band.integrations.langgraph.langchain_tools import custom_tool_defs_to_langchain +from band.core.turn import Turn +from band.core.types import PlatformMessage +from band.integrations.langgraph.langchain_tools import custom_tool_def_to_langchain +from band.runtime.custom_tools import declares_turn_effect +from band.runtime.tools import TurnEffect +from band.testing import FakeAgentTools, reported_failures class EchoInput(BaseModel): @@ -30,10 +38,8 @@ def echo(args: EchoInput) -> str: async def test_converter_produces_a_runnable_structured_tool() -> None: - tools = custom_tool_defs_to_langchain([(EchoInput, echo)]) + converted = custom_tool_def_to_langchain((EchoInput, echo), turn=None) - assert len(tools) == 1 - converted = tools[0] assert isinstance(converted, StructuredTool) assert converted.name == "echo" # get_custom_tool_name(EchoInput) assert converted.args_schema is EchoInput @@ -42,26 +48,81 @@ async def test_converter_produces_a_runnable_structured_tool() -> None: async def test_converter_reports_bad_args_to_the_model() -> None: - (converted,) = custom_tool_defs_to_langchain([(EchoInput, echo)]) + converted = custom_tool_def_to_langchain((EchoInput, echo), turn=None) # A validation error is returned as a string (fed back to the LLM), not raised. result = await converted.ainvoke({"wrong": "field"}) assert isinstance(result, str) and "text" in result -def test_adapter_normalizes_custom_tool_defs_and_passes_native_through() -> None: +def test_adapter_converts_custom_tool_defs_and_passes_native_through() -> None: @lc_tool def native(x: str) -> str: """A ready-made LangChain tool.""" return x - # Advanced pattern (graph_factory) keeps additional_tools on the instance, so we - # can assert the normalization. A CustomToolDef tuple + a native tool go in. adapter = LangGraphAdapter( - graph_factory=lambda tools: MagicMock(), + graph_factory=lambda tools: None, additional_tools=[(EchoInput, echo), native], ) - assert len(adapter.additional_tools) == 2 - assert not any(isinstance(t, tuple) for t in adapter.additional_tools) - names = {getattr(t, "name", None) for t in adapter.additional_tools} - assert names == {"echo", "native"} + turn_tools = adapter._additional_tools_for_turn(Turn()) + + assert not any(isinstance(t, tuple) for t in turn_tools) + assert {t.name for t in turn_tools} == {"echo", "native"} + + +class FileInput(BaseModel): + """File the report.""" + + note: str + + +@declares_turn_effect(TurnEffect.ACT) +def file_report(args: FileInput) -> str: + return "filed" + + +class CallsToolGraph: + """A graph whose one step calls the named tool, as a real agent graph would.""" + + def __init__(self, turn_tools: list[Any], tool_name: str) -> None: + (self._tool,) = [t for t in turn_tools if t.name == tool_name] + + async def astream_events( + self, graph_input: dict[str, Any], **kwargs: Any + ) -> AsyncIterator[dict[str, Any]]: + await self._tool.ainvoke({"note": "go"}) + return + yield + + +async def test_custom_tool_records_its_effect_on_the_room_turn() -> None: + tools = FakeAgentTools(room_id="room-1") + adapter = LangGraphAdapter( + graph_factory=lambda turn_tools: CallsToolGraph(turn_tools, "file"), + additional_tools=[(FileInput, file_report)], + ) + await adapter.on_started("TestBot", "Test bot") + + await adapter.on_message( + msg=PlatformMessage( + id="msg-1", + room_id="room-1", + content="file it", + sender_id="user-1", + sender_type="User", + sender_name="Alice", + message_type="text", + metadata={}, + created_at=datetime.now(UTC), + ), + tools=tools, + history=[], + participants_msg=None, + contacts_msg=None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert reported_failures(tools) == [] + assert tools.turn.complete diff --git a/tests/integrations/slack/test_wrapping.py b/tests/integrations/slack/test_wrapping.py index bac33244a..add6ab950 100644 --- a/tests/integrations/slack/test_wrapping.py +++ b/tests/integrations/slack/test_wrapping.py @@ -860,15 +860,22 @@ async def on_cleanup(self, room_id: str) -> None: # ── SlackTeeingTools — new behavior ───────────────────────────────────────── -def _make_tee_tools( - slack: AsyncMock | None = None, -) -> tuple[SlackTeeingTools, MagicMock, AsyncMock]: +def _make_band_tools() -> AgentTools: rest = MagicMock() rest.agent_api_events.create_agent_chat_event = AsyncMock() rest.agent_api_messages.create_agent_chat_message = AsyncMock( return_value=SimpleNamespace(data=SimpleNamespace(id="m")) ) - base = AgentTools(room_id="r1", rest=rest, participants=[]) + return AgentTools(room_id="r1", rest=rest, participants=[]) + + +def _make_tee_tools( + slack: AsyncMock | None = None, + *, + wrap: AgentTools | None = None, +) -> tuple[SlackTeeingTools, MagicMock, AsyncMock]: + base = wrap or _make_band_tools() + rest = base.rest if slack is None: # Only set defaults when constructing a fresh mock; a caller-provided # ``slack`` may have custom side_effects we mustn't overwrite. @@ -907,6 +914,28 @@ async def test_slack_send_message_returns_error_dict_on_failure(): assert "kaboom" in result["error"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("chat_post_message", "replied"), + [ + pytest.param(AsyncMock(return_value={"ok": True}), True, id="posted"), + pytest.param(AsyncMock(side_effect=RuntimeError("kaboom")), False, id="failed"), + ], +) +async def test_slack_reply_settles_the_wrapped_turn_only_when_posted( + chat_post_message: AsyncMock, replied: bool +): + """The adapter judges the turn on the tools it handed out, not the tee.""" + slack = AsyncMock() + slack.chat_postMessage = chat_post_message + band_tools = _make_band_tools() + tools, _, _ = _make_tee_tools(slack=slack, wrap=band_tools) + + await tools.execute_tool_call(SLACK_SEND_MESSAGE_TOOL_NAME, {"content": "hi"}) + + assert band_tools.turn.replied is replied + + def test_get_anthropic_tool_schemas_includes_slack_send_message(): tools, _, _ = _make_tee_tools() schemas = tools.get_anthropic_tool_schemas() diff --git a/tests/integrations/test_crewai_tools.py b/tests/integrations/test_crewai_tools.py index 25e28ea34..046286e92 100644 --- a/tests/integrations/test_crewai_tools.py +++ b/tests/integrations/test_crewai_tools.py @@ -21,12 +21,15 @@ from band.core.exceptions import BandToolError from band.core.memory_types import memory_type_field_description from band.core.types import AdapterFeatures, Capability, Emit -from band.runtime.custom_tools import get_custom_tool_name +from band.runtime.custom_tools import declares_turn_effect, get_custom_tool_name from band.runtime.tools import ( + BandTool, + TurnEffect, file_content_placeholder, image_block_placeholder, iter_tool_definitions, ) +from band.testing.fake_tools import FakeAgentTools class MockBaseTool: @@ -820,6 +823,79 @@ def test_tool_returns_error_json_when_get_context_returns_none(self, builder_mod assert "No room context available" in result["message"] +# --- Turn ledger: the catalog calls the recording tool methods directly --- + + +class RecordInput(BaseModel): + """Record a value.""" + + value: str + + +async def record_value(args: RecordInput) -> str: + return args.value + + +@declares_turn_effect(TurnEffect.ACT) +async def record_value_for_real(args: RecordInput) -> str: + return args.value + + +@pytest.fixture +def room_tools() -> FakeAgentTools: + return FakeAgentTools() + + +@pytest.fixture +def crew_tools(builder_mod, room_tools): + """The crew's tools by name, bound to ``room_tools`` as the room context.""" + + def build(custom_tools: list[Any] | None = None) -> dict[str, Any]: + context = builder_mod.CrewAIToolContext(room_id="room-1", tools=room_tools) + tools = builder_mod.build_band_crewai_tools( + get_context=lambda: context, + reporter=builder_mod.NoopReporter(), + custom_tools=custom_tools, + ) + return {tool.name: tool for tool in tools} + + return build + + +class TestTurnLedger: + def test_send_message_marks_the_turn_replied(self, crew_tools, room_tools): + crew_tools()[BandTool.SEND_MESSAGE]._run(content="hi", mentions=["@alice"]) + + assert room_tools.turn.replied + + def test_failed_send_then_event_leaves_the_turn_incomplete( + self, crew_tools, room_tools + ): + """A rejected reply followed by narration is still a turn owed a reply.""" + room_tools.send_message_error = BandToolError("Unknown participant 'user1'") + tools = crew_tools() + + tools[BandTool.SEND_MESSAGE]._run(content="hi", mentions=["@user1"]) + tools[BandTool.SEND_EVENT]._run(content="thinking", message_type="thought") + + assert (room_tools.messages_sent, len(room_tools.events_sent)) == ([], 1) + assert not room_tools.turn.complete + + @pytest.mark.parametrize( + ("handler", "completes"), + [(record_value, False), (record_value_for_real, True)], + ids=["undeclared-observes", "declared-act"], + ) + def test_custom_tool_records_its_declared_effect( + self, crew_tools, room_tools, handler, completes + ): + tool_name = get_custom_tool_name(RecordInput) + + crew_tools([(RecordInput, handler)])[tool_name]._run(value="x") + + assert room_tools.turn.complete is completes + + # --- run_async + nest_asyncio lazy patch --- diff --git a/tests/integrations/test_omp.py b/tests/integrations/test_omp.py index ff910fb87..cc7d602da 100644 --- a/tests/integrations/test_omp.py +++ b/tests/integrations/test_omp.py @@ -24,7 +24,7 @@ omp_provider_env, validate_omp_command, ) -from band.runtime.tools import settles_turn_reply +from band.runtime.tools import TurnEffect, turn_effect from tests.paths import CI_SCRIPTS @@ -137,7 +137,7 @@ def test_normalize_mcp_title_maps_registered_band_tool() -> None: name, args = normalize_omp_mcp_device_call(title, {"chat_id": "r1"}, own) assert name == "band_send_message" assert args == {"chat_id": "r1"} - assert settles_turn_reply(name) is True + assert turn_effect(name) is TurnEffect.REPLY def test_normalize_device_call_leaves_unknown_paths() -> None: diff --git a/tests/mcp/test_engine.py b/tests/mcp/test_engine.py index 10a861237..8e9f2c349 100644 --- a/tests/mcp/test_engine.py +++ b/tests/mcp/test_engine.py @@ -36,7 +36,8 @@ pin_existing_chat_id, validate_unique_tool_names, ) -from band.runtime.tools import TOOL_DEFINITIONS +from band.runtime.custom_tools import declares_turn_effect +from band.runtime.tools import TOOL_DEFINITIONS, TurnEffect from band.testing.fake_tools import FakeAgentTools from tests.mcp.conftest import FakeHumanTools @@ -502,9 +503,10 @@ async def handler(input_data: EchoInput) -> dict[str, str]: seen["message"] = input_data.message return {"echo": input_data.message} + fake = FakeAgentTools(room_id="room-1") registration = build_custom_tool_registration( CustomToolSpec(input_model=EchoInput, handler=handler), - room_bound=True, + get_tools=lambda _chat_id: fake, ) spec = EngineSpec(name="test-custom", tools=(registration,)) mcp = build_engine(spec) @@ -518,6 +520,32 @@ async def handler(input_data: EchoInput) -> dict[str, str]: assert seen == {"message": "hi"} +async def test_room_bound_custom_tool_records_its_effect_on_the_resolved_room_turn() -> ( + None +): + @declares_turn_effect(TurnEffect.REPLY) + async def answer(input_data: EchoInput) -> dict[str, str]: + return {"echo": input_data.message} + + rooms = { + "room-1": FakeAgentTools(room_id="room-1"), + "room-2": FakeAgentTools(room_id="room-2"), + } + registration = build_custom_tool_registration( + CustomToolSpec(input_model=EchoInput, handler=answer), + get_tools=rooms.get, + ) + mcp = build_engine(EngineSpec(name="test-custom-effect", tools=(registration,))) + + async with create_connected_server_and_client_session(mcp) as session: + await _call(session, "echo", message="hi", chat_id="room-2") + + assert {room: tools.turn.replied for room, tools in rooms.items()} == { + "room-1": False, + "room-2": True, + } + + async def test_custom_tool_accepts_bare_tuple_contract() -> None: """The bare (input_model, handler) tuple stays accepted -- the existing adapter contract, not deprecated by CustomToolSpec.""" diff --git a/tests/runtime/test_custom_tools.py b/tests/runtime/test_custom_tools.py index 59f432629..e0bcb07a7 100644 --- a/tests/runtime/test_custom_tools.py +++ b/tests/runtime/test_custom_tools.py @@ -242,7 +242,7 @@ async def test_executes_async_function(self): """Should await async tool functions.""" tool: CustomToolDef = (WeatherInput, async_weather) - result = await execute_custom_tool(tool, {"city": "NYC"}) + result = await execute_custom_tool(tool, {"city": "NYC"}, turn=None) assert result == "Weather in NYC: Sunny, 72F" @@ -252,7 +252,7 @@ async def test_executes_sync_function(self): tool: CustomToolDef = (CalculatorInput, sync_calculator) result = await execute_custom_tool( - tool, {"operation": "add", "left": 5.0, "right": 3.0} + tool, {"operation": "add", "left": 5.0, "right": 3.0}, turn=None ) assert result == "8.0" @@ -269,7 +269,7 @@ async def __call__(self, args: WeatherInput) -> str: tool: CustomToolDef = (WeatherInput, AsyncTool()) - result = await execute_custom_tool(tool, {"city": "NYC"}) + result = await execute_custom_tool(tool, {"city": "NYC"}, turn=None) assert result == "Weather in NYC: from async-callable" @@ -287,7 +287,7 @@ async def __call__(self, args: WeatherInput) -> str: tool: CustomToolDef = (WeatherInput, UnhashableTool()) - result = await execute_custom_tool(tool, {"city": "LA"}) + result = await execute_custom_tool(tool, {"city": "LA"}, turn=None) assert result == "Weather in LA: unhashable-ok" @@ -297,7 +297,9 @@ async def test_validates_input_with_pydantic(self): tool: CustomToolDef = (CalculatorInput, sync_calculator) with pytest.raises(ValueError, match="Invalid arguments for calculator"): - await execute_custom_tool(tool, {"operation": "add"}) # Missing left, right + await execute_custom_tool( + tool, {"operation": "add"}, turn=None + ) # Missing left, right @pytest.mark.asyncio async def test_validation_error_has_details(self): @@ -305,7 +307,7 @@ async def test_validation_error_has_details(self): tool: CustomToolDef = (WeatherInput, async_weather) with pytest.raises(ValueError) as exc_info: - await execute_custom_tool(tool, {}) # Missing required 'city' + await execute_custom_tool(tool, {}, turn=None) # Missing required 'city' # Should mention the missing field in formatted message error_msg = str(exc_info.value) @@ -318,7 +320,7 @@ async def test_execution_error_propagates(self): tool: CustomToolDef = (WeatherInput, failing_tool) with pytest.raises(ValueError, match="API unavailable"): - await execute_custom_tool(tool, {"city": "NYC"}) + await execute_custom_tool(tool, {"city": "NYC"}, turn=None) @pytest.mark.asyncio async def test_passes_validated_model_to_function(self): @@ -331,7 +333,9 @@ async def capture_args(args: CalculatorInput) -> str: tool: CustomToolDef = (CalculatorInput, capture_args) - await execute_custom_tool(tool, {"operation": "add", "left": 1, "right": 2}) + await execute_custom_tool( + tool, {"operation": "add", "left": 1, "right": 2}, turn=None + ) assert len(received_args) == 1 assert isinstance(received_args[0], CalculatorInput) @@ -346,7 +350,7 @@ async def test_coerces_types(self): # Pass ints instead of floats result = await execute_custom_tool( - tool, {"operation": "multiply", "left": 4, "right": 5} + tool, {"operation": "multiply", "left": 4, "right": 5}, turn=None ) assert result == "20.0" @@ -362,6 +366,6 @@ async def capture_search(args: SearchWebInput) -> str: tool: CustomToolDef = (SearchWebInput, capture_search) - await execute_custom_tool(tool, {"query": "test"}) # No max_results + await execute_custom_tool(tool, {"query": "test"}, turn=None) # No max_results assert received_args[0].max_results == 10 # Default value diff --git a/tests/runtime/test_execution.py b/tests/runtime/test_execution.py index 17e8d1bc8..731d14c1c 100644 --- a/tests/runtime/test_execution.py +++ b/tests/runtime/test_execution.py @@ -12,12 +12,12 @@ from band.client.streaming import MessageMetadata from band.core.protocols import ( GENERIC_PROVIDER_FAILURE_MESSAGE, + TURN_FAILURE_PROVIDER, TurnResultAlreadyReported, ) from band.core.types import MessageType from band.logging_config import TRACE_CONTEXT, trace_context_scope from band.runtime.execution import ( - _TURN_FAILURE_PROVIDER, BacklogProcessResult, Execution, ExecutionContext, @@ -1954,7 +1954,7 @@ async def test_websocket_final_failure_posts_an_error_event( assert await self._context(mock_link)._process_event(event) is True mock_link.mark_failed.assert_awaited_once() - assert self._reporters(error_events) == [_TURN_FAILURE_PROVIDER] + assert self._reporters(error_events) == [TURN_FAILURE_PROVIDER] async def test_report_waits_for_the_final_attempt(self, mock_link, error_events): """A failure the message will be retried from is not final; only the @@ -2039,7 +2039,7 @@ async def reporting_handler(ctx, event): assert self._reporters(error_events) == ["codex"] await ctx._process_backlog_message(self._backlog_message("msg-unreported")) - assert self._reporters(error_events) == ["codex", _TURN_FAILURE_PROVIDER] + assert self._reporters(error_events) == ["codex", TURN_FAILURE_PROVIDER] async def test_turn_result_already_reported_is_not_repeated( self, mock_link, error_events diff --git a/tests/runtime/test_turn.py b/tests/runtime/test_turn.py new file mode 100644 index 000000000..9da6813a4 --- /dev/null +++ b/tests/runtime/test_turn.py @@ -0,0 +1,165 @@ +"""The per-turn ledger: what each tool call records, and how a verdict is reported. + +Recording happens on the tool methods themselves, so these tests drive real +``AgentTools`` (over the spec'd REST client) through each call path an adapter +uses: registry dispatch, a direct method call, and ``deliver_reply``. +""" + +from __future__ import annotations + +from typing import Any + +import band_sdk_core +import pytest +from pydantic import BaseModel + +from band.core.delivery import deliver_reply +from band.core.protocols import TURN_FAILURE_PROVIDER +from band.core.turn import report_unsettled_turn +from band.integrations.claude_sdk.dedup_tools import DedupingAgentTools +from band.runtime.custom_tools import declares_turn_effect, execute_custom_tool +from band.runtime.tools import AgentTools, BandTool, TurnEffect +from band.testing.fake_tools import FakeAgentTools, reported_failures + +ALICE = {"id": "user-1", "name": "Alice", "type": "User", "handle": "alice"} + + +@pytest.fixture +def tools(mock_rest_client: Any) -> AgentTools: + return AgentTools("room-1", mock_rest_client, participants=[ALICE]) + + +class LookupInput(BaseModel): + """Look something up.""" + + topic: str + + +class TestRecordedCallPaths: + async def test_a_direct_send_message_is_the_reply(self, tools: AgentTools) -> None: + await tools.send_message("hi", mentions=["@alice"]) + + assert tools.turn.replied + assert tools.turn.complete + + async def test_a_dispatched_band_no_reply_declines(self, tools: AgentTools) -> None: + await tools.execute_tool_call(BandTool.NO_REPLY, {"reason": "FYI only"}) + + assert tools.turn.replied + assert tools.turn.complete + + async def test_deliver_reply_is_the_reply(self, tools: AgentTools) -> None: + await deliver_reply(tools, "the answer", ["@alice"]) + + assert tools.turn.replied + + async def test_real_work_completes_without_a_reply(self, tools: AgentTools) -> None: + await tools.create_chatroom() + + assert tools.turn.complete + assert not tools.turn.replied + + +class TestCallsThatRecordNothing: + async def test_a_raising_call_records_nothing(self, tools: AgentTools) -> None: + with pytest.raises(ValueError, match="Unknown participant"): + await tools.send_message("hi", mentions=["@user1"]) + + assert not tools.turn.replied + + async def test_a_blank_send_records_nothing(self, tools: AgentTools) -> None: + assert await tools.send_message(" ", mentions=["@alice"]) is None + + assert not tools.turn.replied + + async def test_an_adapter_notice_is_never_the_reply( + self, tools: AgentTools + ) -> None: + await tools.send_notice("Approve this tool call?", mentions=["@alice"]) + + assert not tools.turn.complete + + async def test_narration_leaves_the_reply_owed(self, tools: AgentTools) -> None: + """The nightly loss: a send to an unknown handle fails, the model + narrates through band_send_event, then stops.""" + with pytest.raises(ValueError, match="Unknown participant"): + await tools.send_message("hi", mentions=["@user1"]) + await tools.execute_tool_call( + BandTool.SEND_EVENT, {"content": "retrying", "message_type": "thought"} + ) + + assert not tools.turn.complete + + +class TestCustomTools: + async def test_a_declared_tool_records_its_effect(self) -> None: + @declares_turn_effect(TurnEffect.ACT) + async def post_to_ticketing(args: LookupInput) -> str: + return "filed" + + tools = FakeAgentTools() + await execute_custom_tool( + (LookupInput, post_to_ticketing), {"topic": "x"}, turn=tools.turn + ) + + assert tools.turn.complete + + async def test_an_undeclared_tool_only_observes(self) -> None: + async def lookup(args: LookupInput) -> str: + return "found" + + tools = FakeAgentTools() + await execute_custom_tool( + (LookupInput, lookup), {"topic": "x"}, turn=tools.turn + ) + + assert not tools.turn.complete + + async def test_a_failing_declared_tool_records_nothing(self) -> None: + @declares_turn_effect(TurnEffect.ACT) + async def broken(args: LookupInput) -> str: + raise RuntimeError("down") + + tools = FakeAgentTools() + with pytest.raises(RuntimeError): + await execute_custom_tool( + (LookupInput, broken), {"topic": "x"}, turn=tools.turn + ) + + assert not tools.turn.complete + + +async def test_the_dedup_wrapper_shares_the_inner_ledger() -> None: + inner = FakeAgentTools() + wrapped = DedupingAgentTools(inner) + + await wrapped.send_message("hi", mentions=["@alice"]) + + assert wrapped.turn is inner.turn + assert inner.turn.replied + + +class TestReportUnsettledTurn: + async def test_a_missing_reply_is_reported_once(self) -> None: + tools = FakeAgentTools() + + assert await report_unsettled_turn(tools) is True + assert await report_unsettled_turn(tools) is False + + assert [(f["provider"], f["message"]) for f in reported_failures(tools)] == [ + (TURN_FAILURE_PROVIDER, band_sdk_core.missing_reply_message()) + ] + + async def test_a_complete_turn_reports_nothing(self) -> None: + tools = FakeAgentTools() + await tools.no_reply() + + assert await report_unsettled_turn(tools) is False + + assert reported_failures(tools) == [] + + async def test_a_settled_turn_reports_nothing(self) -> None: + tools = FakeAgentTools() + tools.turn.settle() + + assert await report_unsettled_turn(tools) is False diff --git a/uv.lock b/uv.lock index b7bac0c84..2a19a9acb 100644 --- a/uv.lock +++ b/uv.lock @@ -799,7 +799,7 @@ requires-dist = [ { name = "anthropic", marker = "extra == 'dev-parlant'", specifier = ">=0.75.0,<1" }, { name = "async-lru", specifier = ">=2.3.0" }, { name = "band-client-rest", specifier = "==0.0.41" }, - { name = "band-sdk-core", specifier = "==2.7.0" }, + { name = "band-sdk-core", specifier = "==2.8.0" }, { name = "band-testing-python", marker = "extra == 'dev'", specifier = "==0.1.4" }, { name = "band-testing-python", marker = "extra == 'dev-crewai'", specifier = "==0.1.4" }, { name = "band-testing-python", marker = "extra == 'dev-parlant'", specifier = "==0.1.4" }, @@ -948,17 +948,17 @@ provides-extras = ["logging", "desktop", "codex", "opencode", "letta", "pydantic [[package]] name = "band-sdk-core" -version = "2.7.0" +version = "2.8.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/be/1be8b53e964d7a850988623cfa420bc8a47bba89e529b44b3797bca6dab5/band_sdk_core-2.7.0-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:307214ecce8807ab1f0248115c1c62476cd3a37eea7add227ddbf98cee3aa314", size = 507960, upload-time = "2026-09-27T08:05:28.74Z" }, - { url = "https://files.pythonhosted.org/packages/a3/c3/a0d9f13f38abc5b7b1aed234a5f48b44fb7c87756ce680c3ab926788b933/band_sdk_core-2.7.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:6510d52f9631a0b46b0437515fce0ae457b98bf66758c125fba48d87f5804fdc", size = 508093, upload-time = "2026-09-27T08:05:30.486Z" }, - { url = "https://files.pythonhosted.org/packages/bc/18/71ff5decce9417de7b28295cef17913e1d18075e4ad6e738324050f088fd/band_sdk_core-2.7.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0539f7fdf94780d9788f7c205a8bbe3804322f87a424dcbd047bc5e646151427", size = 562489, upload-time = "2026-09-27T08:05:31.983Z" }, - { url = "https://files.pythonhosted.org/packages/f2/4d/4be0e8a1305ac6c0d4138aa805d622c7b3625428ce9b54f67e68a7c70c3d/band_sdk_core-2.7.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5ff240d70794bf9c2109c7a79efa3d5e1da6b7cd566c8e7bd67e299488d0410d", size = 558128, upload-time = "2026-09-27T08:05:33.872Z" }, - { url = "https://files.pythonhosted.org/packages/2c/a8/32fe7cebb06af928b8fe956490bcb6b333fd1cf4ef5ec291515ee9c2e69e/band_sdk_core-2.7.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:96bd73608fcd89689a01966b10a2b20cbcb96f1e804e639bbc8b021a16ec0db9", size = 740133, upload-time = "2026-09-27T08:05:35.628Z" }, - { url = "https://files.pythonhosted.org/packages/4c/28/0126677ce0958526cdea01e46f0422fe5a5f139c309bf480e971d7f334bc/band_sdk_core-2.7.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4059276fa093d3bb0cb0aaaf23ef4ca9c6a05045e29e4ea78d434295fe912284", size = 777859, upload-time = "2026-09-27T08:05:37.719Z" }, - { url = "https://files.pythonhosted.org/packages/9a/65/5801ef98a4db452d8d34c1508082d468165cd0ebd4d914981e1baf5e6d99/band_sdk_core-2.7.0-cp311-abi3-win_amd64.whl", hash = "sha256:a79c4406cce136c0d73c9db345dfe7f82c8eed645b8af48a9dc69cc75446b77c", size = 374416, upload-time = "2026-09-27T08:05:39.277Z" }, - { url = "https://files.pythonhosted.org/packages/36/91/910fcc9a663a47204761257626ca4bcf5f18041152bb877527cb102b0c60/band_sdk_core-2.7.0-cp311-abi3-win_arm64.whl", hash = "sha256:c39bb72661611e72115279822ca1e1a44284e85cc3bb0e75622b534053323ffc", size = 357832, upload-time = "2026-09-27T08:05:40.99Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2b/5bdf3dca7abe80e916c643f39a2d4ed817f39aeffce92cc8c0f193f50b69/band_sdk_core-2.8.0-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:3fc2314b5e4f720e589dfd7cb9bbaa770cf981d1183d5594789b3b9892bf63ab", size = 516944, upload-time = "2026-10-04T12:56:29.276Z" }, + { url = "https://files.pythonhosted.org/packages/32/dd/2989980909a3fb1636b4f1389a269b8b50c55274478dc451fad0b3157c02/band_sdk_core-2.8.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:3f4cfc0dfcc69475f639e08c2241cea3c7f61ac837cec9adb9d3b2cb38905749", size = 521715, upload-time = "2026-10-04T12:56:31.247Z" }, + { url = "https://files.pythonhosted.org/packages/dd/04/81b623017fafd1e18105f0cf69525a64b71b71193f484bf90778a6302c79/band_sdk_core-2.8.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e68f7c75ae32baaac4f7e6b2bed3dd657d5d144433f824a0febb7d67f1663ac0", size = 570826, upload-time = "2026-10-04T12:56:32.712Z" }, + { url = "https://files.pythonhosted.org/packages/30/e3/0fbb058d0c9555afd16bba3e63404f65907f2a30aa4c0a87747f8ace7b36/band_sdk_core-2.8.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a98dad777ed251ab3d0a65f7800a9b04a72dec2c05980f73273e6f888a2f11fc", size = 565904, upload-time = "2026-10-04T12:56:34.344Z" }, + { url = "https://files.pythonhosted.org/packages/2c/25/e17dca65b86fae83ed0d74bebf5362f00c5e651cbb61f87ef432278131e3/band_sdk_core-2.8.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:10e893e7ad049ca17b6adf49a86993faff77e68165a4279f38db1d200c4b56e6", size = 749106, upload-time = "2026-10-04T12:56:36.049Z" }, + { url = "https://files.pythonhosted.org/packages/a1/b0/77e612f18774beb4b085c39b74898089ac9dfe13842f7543d9753bc18c41/band_sdk_core-2.8.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:770bf348a47469defe3c83b9769cfff706f8d53a0d166bc685a5b1349d440761", size = 786495, upload-time = "2026-10-04T12:56:37.448Z" }, + { url = "https://files.pythonhosted.org/packages/29/23/5b44118677b9ff40e148d3077f60f3dd0f10443b6cbfd22c30c03b96746d/band_sdk_core-2.8.0-cp311-abi3-win_amd64.whl", hash = "sha256:b6e46e2d975dd3d611d0c96a8dbd6c389a0ff7e557d90a29c93867a97b54ae06", size = 384499, upload-time = "2026-10-04T12:56:39.047Z" }, + { url = "https://files.pythonhosted.org/packages/67/fc/e5aed7d0371d8481ee15af26f18a52b53d82716decd2b62edd527359b772/band_sdk_core-2.8.0-cp311-abi3-win_arm64.whl", hash = "sha256:47fb449cca5baed1434abb77b89618e5cdd87857297a38c8582b801503863430", size = 367086, upload-time = "2026-10-04T12:56:40.588Z" }, ] [[package]] From 794fd5df07bb79bf6c225b36e12429a48eaf4334 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 4 Oct 2026 19:26:03 +0300 Subject: [PATCH 02/17] fix!: judge every turn with band-sdk-core's turn outcome One judge for every Python adapter. A turn that replied, declined via band_no_reply, did real work, was settled by the adapter or already reported a failure is complete. Any other turn is reported once with core's missing-reply text (provider band-runtime) and its delivery is marked FAILED. There is no nudge and no recovery. - SimpleAdapter.on_event (OneShot included) runs the verdict. Contact-hub turns are not judged. a2a, a2a_gateway, crewai_flow, BandACPServerAdapter, parlant and a static-graph LangGraph adapter opt out via judges_turns; Slack delegates to its inner adapter. - Detached turns (claude_sdk, codex, opencode, cursor_acp) are judged on their normal completion path. A cancelled, cleaned-up or interrupted turn posts nothing. - Each adapter's own reply judging is deleted: crewai ReplyTracker flags, strands terminal_fired, pydantic_ai tool_executed, claude_sdk replied_this_turn, copilot_sdk's no-reply RuntimeError, opencode's filler text, ACP turn_replied_in_room, letta's settle-on-call and auto_relay=False check, parlant's module-global sent flags, codex settled_reply. - BASE_INSTRUCTIONS and band_send_message's description teach band_no_reply. docs/turn-outcome.md describes the rule. - Correct "the platform retries" wording: the running agent skips a message once it failed max_message_retries times. - Conformance probes for every judged adapter, runtime verdict rows, detached-turn flows, and two E2E smokes (no_reply, missing reply). BREAKING CHANGE: Every SimpleAdapter turn is now judged by band-sdk-core's turn-outcome rule. Adapters that silently dropped a turn (langgraph, anthropic, gemini, google_adk, agno, letta, codex/ACP empty turns, copilot_sdk empty text) now report it and mark it FAILED. An undeclared custom tool counts as observe. settles_turn_reply, is_terminal_success and missing_reply_error are removed from band.runtime.tools. Custom SimpleAdapter subclasses are judged too: post adapter notices with send_notice and call tools.turn.settle(), because a send_message or deliver_reply post counts as the turn's reply. AgentToolsProtocol gains turn and send_notice. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_016nwuEBtdtneuDkJHcvijDQ Signed-off-by: Alexander Zaikman --- AGENTS.md | 10 + docs/acp.md | 5 +- docs/adapters/opencode.md | 4 + docs/turn-outcome.md | 244 ++++++ src/band/adapters/agno.py | 24 +- src/band/adapters/claude_sdk.py | 130 ++-- src/band/adapters/codex.py | 12 +- src/band/adapters/copilot_sdk.py | 23 +- src/band/adapters/crewai.py | 56 +- src/band/adapters/crewai_flow.py | 6 + src/band/adapters/cursor_acp.py | 13 +- src/band/adapters/langgraph.py | 5 + src/band/adapters/letta.py | 15 +- src/band/adapters/opencode/adapter.py | 45 +- src/band/adapters/parlant.py | 29 +- src/band/adapters/pydantic_ai.py | 55 +- src/band/adapters/strands.py | 25 +- src/band/client/streaming/client.py | 8 +- src/band/core/simple_adapter.py | 33 +- src/band/core/turn.py | 8 +- src/band/integrations/a2a/adapter.py | 5 + src/band/integrations/a2a/gateway/adapter.py | 5 + src/band/integrations/acp/server_adapter.py | 6 + src/band/integrations/crewai/reporting.py | 16 +- src/band/integrations/crewai/tools.py | 42 +- src/band/integrations/letta/config.py | 3 +- src/band/integrations/parlant/tools.py | 22 - src/band/integrations/pydantic_ai/tools.py | 11 +- src/band/integrations/slack/adapter.py | 5 + src/band/runtime/prompts.py | 4 +- src/band/runtime/tools/__init__.py | 6 - src/band/runtime/tools/agent.py | 4 +- src/band/runtime/tools/effects.py | 50 +- src/band/runtime/tools/registry.py | 23 +- src/band/runtime/tools/types.py | 13 +- src/band/testing/__init__.py | 14 +- src/band/testing/fake_tools.py | 15 +- tests/adapters/agno/conftest.py | 62 +- tests/adapters/agno/helpers.py | 81 +- tests/adapters/agno/test_history_guard.py | 28 +- .../adapters/agno/test_history_persistence.py | 10 +- tests/adapters/agno/test_rehydration.py | 59 +- .../approvals/test_manual_approval.py | 45 +- .../failures/test_cli_connection_error.py | 5 +- .../failures/test_turn_failure_surfacing.py | 65 +- tests/adapters/claude_sdk/helpers.py | 111 ++- .../claude_sdk/lifecycle/test_on_cleanup.py | 4 +- .../claude_sdk/lifecycle/test_on_message.py | 4 +- .../claude_sdk/test_send_message_dedup.py | 6 +- .../claude_sdk/tools/test_custom_tools.py | 25 +- tests/adapters/codexturns.py | 179 ++++- tests/adapters/copilot_sdk/fakes.py | 14 + tests/adapters/copilot_sdk/test_reply.py | 66 +- .../adapters/copilot_sdk/test_turn_failure.py | 14 +- tests/adapters/lettakit.py | 61 +- tests/adapters/opencode/helpers.py | 24 +- .../adapters/opencode/test_detached_turns.py | 102 +++ tests/adapters/opencode/test_turns.py | 53 +- tests/adapters/test_codex_adapter.py | 713 ++++++++---------- tests/adapters/test_crewai_adapter.py | 650 +++++----------- tests/adapters/test_cursor_acp_adapter.py | 164 ++-- tests/adapters/test_letta_adapter.py | 174 +++-- tests/adapters/test_pydantic_ai_adapter.py | 266 +++---- tests/adapters/test_strands_adapter.py | 75 +- tests/core/test_simple_adapter.py | 7 +- tests/e2e/baseline/README.md | 6 +- tests/e2e/baseline/expected-cells.json | 205 +++++ .../matrix/test_missing_reply_reported.py | 82 ++ .../baseline/smoke/matrix/test_no_reply.py | 49 ++ .../baseline/smoke/samples/approvalroom.py | 15 - .../baseline/smoke/samples/sample_agents.py | 13 + tests/e2e/baseline/toolkit/capture.py | 5 +- .../baseline/toolkit/observations/events.py | 9 + .../test_approval_commands.py | 6 - .../test_strands_injection_spike.py | 39 +- .../test_turn_outcome.py | 171 +++++ .../turnprobes/__init__.py | 102 +++ tests/framework_conformance/turnprobes/acp.py | 92 +++ .../turnprobes/claudesdk.py | 80 ++ .../framework_conformance/turnprobes/codex.py | 52 ++ .../turnprobes/copilotsdk.py | 49 ++ .../turnprobes/crewai.py | 54 ++ .../framework_conformance/turnprobes/letta.py | 30 + .../turnprobes/opencode.py | 102 +++ .../turnprobes/toolloop.py | 258 +++++++ tests/integrations/codex/test_adapter_e2e.py | 93 +-- tests/integrations/parlant/test_tools.py | 80 +- tests/integrations/slack/test_wrapping.py | 55 +- tests/integrations/test_crewai_tools.py | 175 ++--- tests/runtime/test_oneshot.py | 3 + tests/runtime/test_terminal_success.py | 120 --- tests/runtime/test_tools.py | 25 +- tests/runtime/test_turn.py | 47 +- tests/runtime/test_turn_effect.py | 100 +++ tests/runtime/test_turn_verdict.py | 192 +++++ tests/test_agent.py | 1 + .../websocket/test_reply_capture_delivery.py | 5 +- 97 files changed, 4088 insertions(+), 2288 deletions(-) create mode 100644 docs/turn-outcome.md create mode 100644 tests/adapters/opencode/test_detached_turns.py create mode 100644 tests/e2e/baseline/smoke/matrix/test_missing_reply_reported.py create mode 100644 tests/e2e/baseline/smoke/matrix/test_no_reply.py create mode 100644 tests/framework_conformance/test_turn_outcome.py create mode 100644 tests/framework_conformance/turnprobes/__init__.py create mode 100644 tests/framework_conformance/turnprobes/acp.py create mode 100644 tests/framework_conformance/turnprobes/claudesdk.py create mode 100644 tests/framework_conformance/turnprobes/codex.py create mode 100644 tests/framework_conformance/turnprobes/copilotsdk.py create mode 100644 tests/framework_conformance/turnprobes/crewai.py create mode 100644 tests/framework_conformance/turnprobes/letta.py create mode 100644 tests/framework_conformance/turnprobes/opencode.py create mode 100644 tests/framework_conformance/turnprobes/toolloop.py delete mode 100644 tests/runtime/test_terminal_success.py create mode 100644 tests/runtime/test_turn_effect.py create mode 100644 tests/runtime/test_turn_verdict.py diff --git a/AGENTS.md b/AGENTS.md index da74eeccd..e605c46a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,6 +19,16 @@ text. `tests/framework_conformance/test_tool_text_drift.py` is the drift guard; an adapter that builds its own tool schemas sets `advertised_arg_text` on its test config so the guard can read what the model actually sees. +## Turn Outcome + +`SimpleAdapter.on_event` judges every turn with band-sdk-core's rule: a turn +that replied, declined via `band_no_reply`, did real work, was settled by the +adapter, or already reported a failure completes; anything else is reported +once and marked FAILED. Adapters never judge turns themselves. Post an +adapter's own messages with `send_notice` (never `send_message`, which counts as +the reply) and relay a model's final text only through `relay_reply`. See +[docs/turn-outcome.md](docs/turn-outcome.md). + ## Adapter Constructor Shape Every adapter is built as `XAdapter(config: XAdapterConfig | None = None, *, diff --git a/docs/acp.md b/docs/acp.md index dd9af60ec..a84d9800f 100644 --- a/docs/acp.md +++ b/docs/acp.md @@ -69,7 +69,10 @@ failure fails that room turn visibly instead of falling back. extension notifications, so the adapter holds a turn lock and binds them to that turn's session; Cursor turns are serialized. Decision prompts, timeout notices and `/cursor` replies post through `send_notice`, so they never count as the model's reply, and a - `/cursor` message settles its own turn without reaching the agent. + `/cursor` message settles its own turn without reaching the agent. A turn parked on a + decision releases its message early (`tools.turn.detach()`), so it is judged when the + ACP turn completes normally; a failed or cancelled turn is never reported as a missing + reply. The adapter does not pick a plan/agent mode itself; a caller selects one through `resolve_session_config`, which reads each session's advertised catalog before the first prompt. The live `backends` lane pins the Cursor CLI and passes `E2E_CURSOR_API_KEY` diff --git a/docs/adapters/opencode.md b/docs/adapters/opencode.md index c29f38d0c..47db9097c 100644 --- a/docs/adapters/opencode.md +++ b/docs/adapters/opencode.md @@ -25,6 +25,10 @@ Four invariants are easy to break and expensive to rediscover: `turn_timeout_s` bounds *compute*: time parked on a manual approval is excluded, since the ask carries its own `approval_wait_timeout_s` expiry. +A manual ask also releases the turn's delivery early (it is marked processed +while the human decides), so the adapter judges that turn itself when OpenCode +finishes it: a turn that ends without a reply then posts one missing-reply +failure. A turn cancelled by cleanup or an interrupt posts nothing. `variant` is an opaque, provider- and model-specific name the running server owns: it may select reasoning effort or be a custom provider setting, so read diff --git a/docs/turn-outcome.md b/docs/turn-outcome.md new file mode 100644 index 000000000..9ba463291 --- /dev/null +++ b/docs/turn-outcome.md @@ -0,0 +1,244 @@ +# Turn outcome + +Every message pushed to an agent @mentions it, so every turn owes the room an +answer: a reply, or a deliberate decision not to reply. `SimpleAdapter.on_event` +judges each turn with one rule from `band-sdk-core`, shared with the TypeScript +SDK, and tells the platform honestly whether the delivery succeeded. + +## The rule + +A turn is **complete** when it: + +- replied (`band_send_message`, `band_send_room_file`) +- declined on purpose (`band_no_reply`) +- did real work (any Band tool whose effect is `act`, such as + `band_add_participant`) +- was settled by the adapter itself (a control reply, a busy notice) +- already reported a failure (`send_failure`) + +Anything else is a **missing reply**. Fetching state or narrating through +`band_send_event` only observes, so it never completes a turn. A missing reply +is reported once, as an `error` event from provider `band-runtime` carrying +core's text, and the delivery is marked FAILED. There is no nudge and no +automatic retry: the SDK reports what happened instead of claiming success. + +Every Band tool call records its effect on the turn's ledger, `tools.turn`, +whichever path made the call: + +```python +import asyncio + +from band.testing.fake_tools import FakeAgentTools + + +async def main() -> None: + tools = FakeAgentTools() + + await tools.send_event("Looking that up", message_type="thought") + assert not tools.turn.complete # narration only observes + + await tools.no_reply("FYI only") + assert tools.turn.complete + assert tools.turn.replied # the model's final text is not relayed + + +asyncio.run(main()) +``` + +A turn that ends without completing is reported, then raised as +`TurnResultAlreadyReported` so the runtime marks the delivery FAILED without a +second report: + +```python +import asyncio +from datetime import UTC, datetime + +import band_sdk_core + +from band.core.protocols import TurnResultAlreadyReported +from band.core.simple_adapter import SimpleAdapter +from band.core.types import AgentInput, HistoryProvider, PlatformMessage +from band.testing.fake_tools import FakeAgentTools, reported_failures + + +class Forgetful(SimpleAdapter[HistoryProvider]): + async def on_message( + self, + msg, + tools, + history, + participants_msg, + contacts_msg, + *, + is_session_bootstrap, + room_id, + ): + await tools.get_participants() + + +async def main() -> None: + tools = FakeAgentTools() + msg = PlatformMessage( + id="m1", + room_id="r1", + content="@bot hi", + sender_id="u1", + sender_type="User", + sender_name="Alice", + message_type="text", + metadata={}, + created_at=datetime.now(UTC), + ) + inp = AgentInput( + msg=msg, + tools=tools, + history=HistoryProvider(raw=[]), + participants_msg=None, + contacts_msg=None, + is_session_bootstrap=True, + room_id="r1", + ) + try: + await Forgetful().on_event(inp) + except TurnResultAlreadyReported: + pass + else: + raise AssertionError("a turn that did nothing must be reported") + + assert [f["message"] for f in reported_failures(tools)] == [ + band_sdk_core.missing_reply_message() + ] + + +asyncio.run(main()) +``` + +## Declining with `band_no_reply` + +`band_no_reply` is in every adapter's tool set, and `BASE_INSTRUCTIONS` tells +the model to call it when the latest message needs no answer: it was addressed +to someone else, it is an FYI or an acknowledgement, or someone already answered +it. The call is local only; its `reason` goes to the agent log. + +## Custom tools: `@declares_turn_effect` + +An undeclared custom tool counts as `observe`: the SDK cannot tell a lookup from +an action, so the turn still owes a reply. A tool that completes the turn says +so: + +```python +import asyncio + +from pydantic import BaseModel + +from band.runtime.custom_tools import declares_turn_effect, execute_custom_tool +from band.runtime.tools import TurnEffect +from band.testing.fake_tools import FakeAgentTools + + +class FileTicketInput(BaseModel): + """File a support ticket.""" + + title: str + + +@declares_turn_effect(TurnEffect.ACT) +async def file_ticket(args: FileTicketInput) -> str: + return f"filed: {args.title}" + + +async def main() -> None: + tools = FakeAgentTools() + await execute_custom_tool( + (FileTicketInput, file_ticket), {"title": "Login broken"}, turn=tools.turn + ) + assert tools.turn.complete + + +asyncio.run(main()) +``` + +`REPLY` declares a tool that delivers the answer itself, and `DECLINE` one +whose silence is the answer; both also stop a relaying adapter from posting the +model's final text. + +**Limits:** a custom tool only records on the turn it is handed. Where a tool +runs with `turn=None` (not bound to a room, as in a standalone MCP server), it +cannot complete any turn. A framework-native tool (rather than an +`(InputModel, handler)` pair) records its declared effect only where the +adapter observes it: Strands and Pydantic AI do; Agno tools and ready-made +LangChain tools count as `observe`. + +## Adapter posts: `send_notice` and `settle()` + +A post through `send_message` or `deliver_reply` counts as the turn's reply. An +adapter's own message (an approval prompt, a busy notice, a `/status` reply) +goes through `tools.send_notice`, which posts the same way but records nothing, +so it can never stand in for the model's answer. When the adapter consumes a +message without running the model, or ends the turn with its own text, it calls +`tools.turn.settle()`: + +```python +import asyncio + +from band.testing.fake_tools import FakeAgentTools + + +async def main() -> None: + tools = FakeAgentTools() + + await tools.send_notice( + "Still working on the previous request.", mentions=["@alice"] + ) + assert not tools.turn.complete # a notice is never the reply + + tools.turn.settle() + assert tools.turn.complete + + +asyncio.run(main()) +``` + +`relay_reply(tools, text, mentions)` from `band.core.delivery` is the one gate +for a model's final text: it posts through `deliver_reply` only when no tool +replied or declined this turn. + +`tests/framework_conformance/test_reply_boundary.py` fails if adapter code calls +`send_message` directly outside the model's reply paths. + +## Detached turns + +An adapter whose turn outlives `on_message` (one parked on a human approval) +calls `tools.turn.detach()` when it releases the turn early. `on_event` then +leaves the verdict to the adapter, which reports at the turn's normal end. A +cancelled turn reports nothing: + +```python +import asyncio + +from band.core.turn import report_unsettled_turn +from band.testing.fake_tools import FakeAgentTools, reported_failures + + +async def main() -> None: + tools = FakeAgentTools() + tools.turn.judged = True # set by SimpleAdapter.on_event + tools.turn.detach() # released while waiting for an approval + + # ...later, on the turn's normal completion path (never a ``finally``): + if tools.turn.detached: + await report_unsettled_turn(tools) + + assert len(reported_failures(tools)) == 1 + + +asyncio.run(main()) +``` + +## Exempt adapters + +`SimpleAdapter.judges_turns` is `False` where the turn is not the model's to +answer through Band tools: `A2AAdapter`, `A2AGatewayAdapter`, +`CrewAIFlowAdapter`, `BandACPServerAdapter`, `ParlantAdapter` (its engine owns +its replies) and a LangGraph adapter built from a static `graph=` (it never +gets Band tools). Synthetic contact-hub turns are never judged. diff --git a/src/band/adapters/agno.py b/src/band/adapters/agno.py index 60b03d7f9..ee0e93786 100644 --- a/src/band/adapters/agno.py +++ b/src/band/adapters/agno.py @@ -36,7 +36,6 @@ image_block_placeholder, is_image_passthrough_result, redact_tool_call_args, - settles_turn_reply, ) try: @@ -80,8 +79,9 @@ class AgnoRunError(RuntimeError): Agno catches exceptions inside ``Agent.arun`` (model/API failures included) and reports them as an error-status result rather than propagating. Raising here restores the cross-adapter contract: a failed turn must reach the - runtime so the message is marked failed and the platform retries it, - instead of being recorded as a successful turn that produced no output. + runtime so the message is marked failed (and runs again only up to the + runtime's retry budget), instead of being recorded as a successful turn + that produced no output. """ @@ -91,14 +91,6 @@ def _error_summary(detail: str | None) -> str: return text if len(text) <= 500 else f"{text[:500]}..." -def _tool_executions(response: RunOutput) -> list[Any]: - return list(getattr(response, "tools", None) or []) - - -def _tool_name(execution: Any) -> str: - return getattr(execution, "tool_name", None) or "" - - def _make_band_entrypoint(tool_name: str) -> Callable[..., Awaitable[str | ToolResult]]: async def _entrypoint(**kwargs: Any) -> str | ToolResult: active = _current_tools.get() @@ -364,16 +356,6 @@ async def on_message( self._persist_turn(room_id, response) - if not any( - settles_turn_reply(_tool_name(execution)) - for execution in _tool_executions(response) - ): - logger.debug( - "Room %s msg %s: agent did not settle its reply; nothing delivered", - room_id, - msg.id, - ) - async def on_cleanup(self, room_id: str) -> None: """Drop the room's accumulated transcript when the agent leaves.""" self._message_history.pop(room_id, None) diff --git a/src/band/adapters/claude_sdk.py b/src/band/adapters/claude_sdk.py index 716e9ccca..28d00f0bd 100644 --- a/src/band/adapters/claude_sdk.py +++ b/src/band/adapters/claude_sdk.py @@ -86,6 +86,7 @@ TurnResultAlreadyReported, ) from band.core.simple_adapter import SimpleAdapter +from band.core.turn import report_unsettled_turn from band.core.turn_lifecycle import ApprovalInterruptMixin from band.core.types import ( ApprovalMode, @@ -124,10 +125,9 @@ MCP_TOOL_PREFIX, MEMORY_TOOL_NAMES, TASK_TOOL_NAMES, - is_terminal_success, iter_tool_definitions, mcp_tool_names, - missing_reply_error, + turn_effect, ) logger = logging.getLogger(__name__) @@ -597,8 +597,8 @@ def __init__( # Custom tools (user-provided) self._custom_tools: list[CustomToolDef] = additional_tools or [] - # Only custom tools that declared an effect let a turn with no Band - # terminal tool call still count as answered — see is_terminal_success. + # A declined custom tool settles the turn only by its declared effect + # (see _declined_the_reply). self._custom_effects = custom_tool_effects(self._custom_tools) # Approval flow state @@ -617,8 +617,8 @@ def __init__( # (ToolResultBlock only carries the id). Room-scoped rather than # turn-local: a resumed session can replay a result whose tool_use # streamed in an earlier, truncated turn, and that result must still - # resolve to its name to count as terminal work. Entries are popped - # as results arrive and the room's map is dropped in on_cleanup. + # be narrated under its name. Entries are popped as results arrive + # and the room's map is dropped in on_cleanup. self._pending_tool_names: dict[str, dict[str, str]] = {} # A turn runs as a detached task so a manual approval mid-turn can @@ -1167,15 +1167,14 @@ async def _process_response( # The room's pending-call map (see __init__) — persists across turns # so a result replayed by a resumed session still resolves its name. pending_tool_names = self._pending_tool_names.setdefault(room_id, {}) - replied_this_turn = False async for sdk_message in client.receive_response(): match sdk_message: case AssistantMessage(): - replied_this_turn |= await self._on_assistant_message( + await self._on_assistant_message( sdk_message, pending_tool_names, room_id, tools ) case UserMessage(): - replied_this_turn |= await self._on_user_message( + await self._on_user_message( sdk_message, pending_tool_names, room_id, tools ) # The CLI announces every mode change in a status message; only @@ -1195,28 +1194,23 @@ async def _process_response( self.config.permission_mode, ) case ResultMessage(): - await self._on_turn_complete( - sdk_message, - room_id, - tools, - replied_this_turn=replied_this_turn, - ) + await self._on_turn_complete(sdk_message, room_id, tools) return # ``receive_response`` is documented to terminate only after its # ResultMessage. Reaching EOF first means the CLI transport died. - if replied_this_turn: - # The reply already reached the room — failing the turn would - # make the runtime redeliver the message and answer the user - # twice. Drop the dead session and treat the turn as done. + if tools.turn.complete: + # The turn already answered or did its work. Failing it would mark + # the message FAILED, which a later process or context runs again, + # repeating that work. Drop the dead session; the turn is done. logger.warning( - "Room %s: CLI stream ended without a result after the " - "reply was delivered — invalidating session", + "Room %s: CLI stream ended without a result after the turn " + "completed — invalidating session", room_id, ) await self._invalidate_session(room_id) return - # Nothing was delivered: use the normal dead-client path so the - # runtime marks this turn failed and the cached client is not reused. + # The turn is incomplete: use the normal dead-client path so the + # runtime marks it failed and the cached client is not reused. raise CLIConnectionError(self._stream_ended_without_result_error()) async def _on_assistant_message( @@ -1225,14 +1219,12 @@ async def _on_assistant_message( pending_tool_names: dict[str, str], room_id: str, tools: AgentToolsProtocol, - ) -> bool: + ) -> None: """Narrate one assistant message's blocks. - Returns True when a carried tool result was terminal work — results - normally arrive in user envelopes (see _on_user_message); + Tool results normally arrive in user envelopes (see _on_user_message); assistant-carried ones are accepted defensively. """ - replied_this_turn = False for block in message.content: match block: case TextBlock() if block.text: @@ -1240,10 +1232,9 @@ async def _on_assistant_message( case ThinkingBlock() if block.thinking: await self._narrate_thinking(block, room_id, tools) case _: - replied_this_turn |= await self._dispatch_tool_block( + await self._dispatch_tool_block( block, pending_tool_names, room_id, tools ) - return replied_this_turn async def _on_user_message( self, @@ -1251,21 +1242,16 @@ async def _on_user_message( pending_tool_names: dict[str, str], room_id: str, tools: AgentToolsProtocol, - ) -> bool: + ) -> None: """Handle the tool_use/tool_result blocks the protocol delivers in user-type envelopes. - Returns True when any result was terminal work. ``content`` may also - be a plain prompt string, which carries neither. + ``content`` may also be a plain prompt string, which carries neither. """ if not isinstance(message.content, list): - return False - replied_this_turn = False + return for block in message.content: - replied_this_turn |= await self._dispatch_tool_block( - block, pending_tool_names, room_id, tools - ) - return replied_this_turn + await self._dispatch_tool_block(block, pending_tool_names, room_id, tools) async def _dispatch_tool_block( self, @@ -1273,20 +1259,14 @@ async def _dispatch_tool_block( pending_tool_names: dict[str, str], room_id: str, tools: AgentToolsProtocol, - ) -> bool: + ) -> None: """Handle one ToolUseBlock/ToolResultBlock entry, shared by assistant- - and user-envelope message handling; any other block type is a no-op. - Returns True when the block was terminal work (a tool result).""" + and user-envelope message handling; any other block type is a no-op.""" match block: case ToolUseBlock(): await self._on_tool_use(block, pending_tool_names, room_id, tools) - return False case ToolResultBlock(): - return await self._on_tool_result( - block, pending_tool_names, room_id, tools - ) - case _: - return False + await self._on_tool_result(block, pending_tool_names, room_id, tools) async def _send_narration_event( self, @@ -1343,8 +1323,7 @@ async def _on_tool_use( room_id: str, tools: AgentToolsProtocol, ) -> None: - """Register a pending call (for the terminal-work check at turn end) - and narrate it. Shared by both envelopes a call can arrive in — the + """Register a pending call (to name its result) and narrate it. Shared by both envelopes a call can arrive in — the protocol's assistant messages, and user messages when the call is carried by a subagent/nested tool_use block.""" # Bare name for the cross-adapter tool_call record (the SDK @@ -1385,10 +1364,12 @@ async def _on_turn_complete( sdk_message: ResultMessage, room_id: str, tools: AgentToolsProtocol, - *, - replied_this_turn: bool, ) -> None: - """Close out the turn: persist session id, emit usage, surface failure.""" + """Close out the turn: persist session id, emit usage, surface failure. + + A turn released early for an approval is judged here, at its real end; + any other turn is judged by ``on_event`` once ``on_message`` returns. + """ logger.info( "Room %s: Complete - %sms, $%.4f", room_id, @@ -1415,12 +1396,10 @@ async def _on_turn_complete( AgentFailure(_PROVIDER, detail, code, sdk_message.errors) ) raise TurnResultAlreadyReported(detail) - elif not replied_this_turn and not self._declined_the_reply( - sdk_message.permission_denials, notified - ): - detail = missing_reply_error("Claude SDK") - await tools.send_failure(AgentFailure(_PROVIDER, detail)) - raise TurnResultAlreadyReported(detail) + if self._declined_the_reply(sdk_message.permission_denials, notified): + tools.turn.settle() + if tools.turn.detached: + await report_unsettled_turn(tools) def _declined_the_reply( self, permission_denials: list[Any] | None, notified: set[str] | None @@ -1434,8 +1413,8 @@ def _declined_the_reply( thing the CLI can't tell us — which of those denials actually reached the room as a decline notice (see _record_notified_decline). A denial only explains the silence when it was both notified and the declined - tool is what would have delivered the reply (is_terminal_success) — - a declined side tool like Bash still leaves the turn's question + tool's call would have done the turn's work (``turn_effect(...).did_work``) + — a declined side tool like Bash still leaves the turn's question unanswered. """ if not notified: @@ -1452,9 +1431,7 @@ def _declined_the_reply( # tool is namespaced (see _semantic_tool_name) — strip it before # comparing, same as every other tool-name check in this adapter. tool_name = self._semantic_tool_name(raw_tool_name) - if is_terminal_success( - tool_name, succeeded=True, custom_effects=self._custom_effects - ): + if turn_effect(tool_name, custom_effects=self._custom_effects).did_work: return True return False @@ -1494,17 +1471,14 @@ async def _on_tool_result( pending_tool_names: dict[str, str], room_id: str, tools: AgentToolsProtocol, - ) -> bool: - """Narrate one tool result and report whether it was terminal work. + ) -> None: + """Narrate one tool result under its call's name. Shared by both envelopes a result can arrive in: user-type messages (the protocol shape) and assistant messages (accepted defensively). - Returns True when the finished call counts as the turn's productive - work (see is_terminal_success) — i.e. the agent already answered. """ result_tool_name = pending_tool_names.pop(block.tool_use_id, None) await self._narrate_tool_result(block, result_tool_name, room_id, tools) - return self._tool_result_is_terminal(block, result_tool_name) async def _narrate_tool_result( self, @@ -1537,24 +1511,13 @@ async def _narrate_tool_result( message_type="tool_result", ) - def _tool_result_is_terminal( - self, block: ToolResultBlock, result_tool_name: str | None - ) -> bool: - """Whether this finished call counts as the turn's productive work.""" - return is_terminal_success( - result_tool_name, - succeeded=not block.is_error, - custom_effects=self._custom_effects, - ) - @staticmethod def _stream_ended_without_result_error() -> str: """Room-visible detail when the CLI stream ends without a ResultMessage. Distinct from ``_result_error_detail`` (a completed turn whose - ResultMessage reports failure) and ``missing_reply_error`` (a completed - turn that never called a reply tool): this turn never reached a - terminal message at all, e.g. the CLI subprocess exited or its stdout + ResultMessage reports failure): this turn never reached a terminal + message at all, e.g. the CLI subprocess exited or its stdout closed mid-turn. """ return ( @@ -1825,7 +1788,10 @@ async def _resolve_manual_approval( # The request has been posted (or there was nowhere to post it) -- # either way, on_message must return now so Band's room loop can - # dispatch the reply that will resolve this wait. + # dispatch the reply that will resolve this wait. The turn is then + # judged at its real end (see _on_turn_complete). + if tools: + tools.turn.detach() self._release_turn(room_id) try: diff --git a/src/band/adapters/codex.py b/src/band/adapters/codex.py index 90af0af3e..1bce5d94e 100644 --- a/src/band/adapters/codex.py +++ b/src/band/adapters/codex.py @@ -42,6 +42,7 @@ send_event_safe, ) from band.core.simple_adapter import SimpleAdapter +from band.core.turn import report_unsettled_turn from band.core.turn_lifecycle import ApprovalInterruptMixin from band.core.types import ( AgentInput, @@ -1092,6 +1093,9 @@ async def _run_turn( final_text=result.final_text, duration_s=_turn_duration_s, ) + # A turn wound down by room cleanup was aborted, not missed. + if tools.turn.detached and room_id not in self._closing_rooms: + await report_unsettled_turn(tools) except DeliveryFailedError as e: reraise_delivery_cause(e) except TurnResultAlreadyReported: @@ -1113,10 +1117,12 @@ async def _run_turn( ) raise - def _release_turn(self, room_id: str) -> None: - """Let on_message return while the room's turn waits on a human.""" + def _release_turn(self, room_id: str, tools: AgentToolsProtocol) -> None: + """Let on_message return while the room's turn waits on a human; the + turn is then judged at its real end, not when on_message returns.""" release = self._turn_release.get(room_id) if release is not None and not release.done(): + tools.turn.detach() release.set_result(None) def _forget_turn(self, room_id: str, task: asyncio.Task[None]) -> None: @@ -2691,7 +2697,7 @@ async def _resolve_manual_approval( ) raise - self._release_turn(room_id) + self._release_turn(room_id, tools) decision = await registry.wait( entry, pending.future, timeout_s=self.config.approval_wait_timeout_s ) diff --git a/src/band/adapters/copilot_sdk.py b/src/band/adapters/copilot_sdk.py index b6394ecd8..d26d9fb3f 100644 --- a/src/band/adapters/copilot_sdk.py +++ b/src/band/adapters/copilot_sdk.py @@ -167,7 +167,7 @@ class TurnState: reasonings: dict[str, str] = field(default_factory=dict) # Summed across the turn's per-call assistant.usage events; emitted once. usage: TurnUsage = field(default_factory=TurnUsage) - # Non-fatal trouble the turn hit, reported when it ends without a reply. + # Non-fatal trouble the turn hit, logged when it ends without a reply. incidents: list[str] = field(default_factory=list) @@ -484,23 +484,18 @@ async def on_message( await self._emit_thoughts(turn, tools) - # Session errors raise out of send_and_wait, so a None here - # with no room output means the model genuinely said nothing. - if final_text is None and not tools.turn.replied: - # Incidents carry session text, so they stay in this one log line - # and out of the error the runtime logs and reports to the platform. - logger.warning( - "Room %s: Copilot turn produced no reply (incidents: %s)", - room_id, - "; ".join(turn.incidents) or "none reported", - ) - await tools.send_failure(AgentFailure(_PROVIDER, "no assistant reply")) - raise RuntimeError("Copilot turn produced no reply") - try: await relay_reply(tools, final_text, mentions=[turn.sender_mention]) except DeliveryFailedError as e: reraise_delivery_cause(e) + if not tools.turn.complete: + # The runtime reports the missing reply; incidents carry session + # text, so they stay in this log line and out of that report. + logger.warning( + "Room %s: Copilot turn ended without a reply (incidents: %s)", + room_id, + "; ".join(turn.incidents) or "none reported", + ) await self._persist_session_id(room_id, tools) diff --git a/src/band/adapters/crewai.py b/src/band/adapters/crewai.py index 2e5cbd37d..06bf0c991 100644 --- a/src/band/adapters/crewai.py +++ b/src/band/adapters/crewai.py @@ -19,11 +19,7 @@ from band.converters.crewai import CrewAIHistoryConverter, CrewAIMessages from band.core.adapterconfig import BaseAdapterConfig -from band.core.protocols import ( - GENERIC_PROVIDER_FAILURE_MESSAGE, - AgentToolsProtocol, - TurnResultAlreadyReported, -) +from band.core.protocols import GENERIC_PROVIDER_FAILURE_MESSAGE, AgentToolsProtocol from band.core.simple_adapter import SimpleAdapter from band.core.types import Capability, Emit, FeatureKwargs, PlatformMessage from band.integrations.crewai import ( @@ -34,7 +30,7 @@ ) from band.runtime.custom_tools import CustomToolDef from band.runtime.prompts import render_system_prompt -from band.runtime.tools import BandTool, missing_reply_error +from band.runtime.tools import BandTool if TYPE_CHECKING: from crewai import Agent as CrewAIAgent @@ -51,10 +47,10 @@ "_current_room_context", default=None ) -# Per-turn marker for whether the agent already delivered a reply via -# band_send_message. Set in on_message, read by the tool wrappers (through -# _get_context) and the error handler. Shared by reference so the mark is -# visible across CrewAI's sync-to-async tool execution boundary. +# Per-turn tool activity: whether any tool ran and what band_send_message +# posted. Set in on_message, written by the tool wrappers (through +# _get_context). Shared by reference so the marks are visible across CrewAI's +# sync-to-async tool execution boundary. _reply_tracker_var: ContextVar[ReplyTracker | None] = ContextVar( "_crewai_reply_tracker", default=None ) @@ -416,19 +412,15 @@ async def _process_message( ) except Exception as e: - # An empty response is benign only once some tool ran this turn. - # Reaching here with no tool activity means the first-call retry - # also came back empty (or this was a different failure), which is - # indistinguishable from a genuine provider failure and must keep - # failing the delivery so the platform retries it. - if not (_is_empty_llm_response(e) and reply_tracker.any_tool_ran): + # An empty completion is how a tool-only turn ends; whether the + # turn is complete is the shared verdict's call, not this error's. + if not _is_empty_llm_response(e): logger.exception("Error processing message") await tools.send_failure( AgentFailure(_PROVIDER, GENERIC_PROVIDER_FAILURE_MESSAGE) ) raise - # Keep the exception text: it is the only record that CrewAI raised, - # and this turn is no longer marked failed for the runtime to log. + # Keep the exception text: it is the only record that CrewAI raised. logger.debug("Room %s: CrewAI returned no text: %s", room_id, e) result = None @@ -448,25 +440,6 @@ async def _process_message( } ) - if not reply_tracker.did_productive_work: - logger.warning( - "Room %s: CrewAI turn produced nothing for the room", room_id - ) - detail = missing_reply_error( - "CrewAI", - detail=( - "Repeated tool failures may also have exhausted " - f"max_iter={self.config.max_iter}." - ), - ) - await tools.send_failure(AgentFailure(_PROVIDER, detail)) - if not reply_tracker.any_tool_ran: - # Some tool activity (even read-only) means the turn did what it - # was asked and correctly had nothing left to say -- report but - # don't fail the delivery. Only true silence, no tool call at - # all, is a genuine no-response failure worth a retry. - raise TurnResultAlreadyReported(detail) - logger.info( "Room %s: CrewAI turn over for %s (output=%s chars, history=%s)", room_id, @@ -491,11 +464,10 @@ async def _kickoff_with_empty_response_retry( """Retry a first-call empty completion once before giving up. CrewAI raises the identical ValueError whether the model correctly has - nothing left to say (fine once a tool has run -- handled by the - caller) or the turn's very first call came back empty. The second case - has run no tool yet, so there is nothing to duplicate: one immediate - retry absorbs a single-call fluke instead of failing the whole - delivery and leaving CrewAI to improvise on a cold redelivery. + nothing left to say after its tool calls or the turn's very first call + came back empty. The second case has run no tool yet, so there is + nothing to duplicate: one immediate retry absorbs a single-call fluke + instead of ending the turn with nothing done. """ try: return await agent.kickoff_async(prompt) diff --git a/src/band/adapters/crewai_flow.py b/src/band/adapters/crewai_flow.py index 6a5571dac..b753b6414 100644 --- a/src/band/adapters/crewai_flow.py +++ b/src/band/adapters/crewai_flow.py @@ -1574,6 +1574,12 @@ class CrewAIFlowAdapter(SimpleAdapter[CrewAIFlowSessionState]): {Capability.MEMORY, Capability.CONTACTS, Capability.TASKS, Capability.FILES} ) + @property + def judges_turns(self) -> bool: + """The Flow owns its visible writes (reserve-send-confirm), and a turn + that only advances orchestration state owes no reply.""" + return False + def __init__( self, config: CrewAIFlowAdapterConfig | None = None, diff --git a/src/band/adapters/cursor_acp.py b/src/band/adapters/cursor_acp.py index 87fdf70e3..9cf311fc5 100644 --- a/src/band/adapters/cursor_acp.py +++ b/src/band/adapters/cursor_acp.py @@ -13,6 +13,7 @@ from band.client.streaming import ControlMode from band.core.protocols import AgentToolsProtocol +from band.core.turn import report_unsettled_turn from band.core.types import ApprovalMode, FeatureKwargs, PlatformMessage from band.integrations.acp.client_adapter import ( ACPClientAdapter, @@ -281,6 +282,10 @@ async def _run_turn( is_session_bootstrap=is_session_bootstrap, room_id=room_id, ) + # on_event already returned for a detached turn, so it is judged + # here, at its real end; a failed or cancelled turn never gets here. + if tools.turn.detached: + await report_unsettled_turn(tools) finally: self._cursor_profile.bind_session(None) if self._active_turn is turn: @@ -326,7 +331,12 @@ async def on_interrupt(self, room_id: str, mode: ControlMode) -> None: decision -- the runtime's own interrupt only cancels the task that already returned once the decision prompt was posted. Matches on_cleanup's choice not to hard-cancel the turn itself: waking its - pending decision is enough for it to wind down on its own.""" + pending decision is enough for it to wind down on its own. The + interrupt ends that turn, so it is settled and never reported as a + missing reply.""" + turn = self._active_turn + if turn is not None and turn.room_id == room_id: + turn.tools.turn.settle() self._cancel_room_decisions(room_id) async def cleanup_all(self, *, final: bool = True) -> None: @@ -497,6 +507,7 @@ async def _wait_for_decision( # the room needs its queue back the instant a decision is # outstanding, whether or not the prompt itself landed. if not turn.release.done(): + turn.tools.turn.detach() turn.release.set_result(None) result = await self._pending_decisions.wait( diff --git a/src/band/adapters/langgraph.py b/src/band/adapters/langgraph.py index ed029c476..d2640624b 100644 --- a/src/band/adapters/langgraph.py +++ b/src/band/adapters/langgraph.py @@ -226,6 +226,11 @@ def factory(turn_tools: list[Any]) -> Pregel: # top of the checkpointer's already-stored state. self._bootstrapped_rooms: OrderedDict[str, None] = OrderedDict() + @property + def judges_turns(self) -> bool: + """A static ``graph=`` never receives Band tools, so it cannot reply.""" + return self.graph_factory is not None + def _additional_tools_for_turn(self, turn: Turn) -> list[Any]: """The caller's extra tools as LangChain tools for one room turn. diff --git a/src/band/adapters/letta.py b/src/band/adapters/letta.py index b3f27aa85..f5a83c0af 100644 --- a/src/band/adapters/letta.py +++ b/src/band/adapters/letta.py @@ -48,7 +48,6 @@ CHAT_ID_FIELD_NAME, iter_tool_definitions, redact_tool_call_args, - settles_turn_reply, turn_effect, ) @@ -561,7 +560,6 @@ async def _process_response_messages( Returns the list of assistant text parts collected during the turn. """ final_text_parts: list[str] = [] - settled_reply = False # an MCP call posted the reply or declined to for resp_msg in response_messages: match getattr(resp_msg, "message_type", None): case "assistant_message": @@ -575,8 +573,6 @@ async def _process_response_messages( if tool_call else "unknown" ) - if settles_turn_reply(tool_name): - settled_reply = True # ToolCall.arguments is a JSON string (letta_client's own # wire shape); parse it so redact_tool_call_args can # replace band_send_room_file's content field -- reporting @@ -620,9 +616,8 @@ async def _process_response_messages( # to relaying the assistant text so the user still sees a response — # loudly, because a turn landing here means the MCP tool path went unused # (a dead tool path would otherwise hide behind green relays). With - # auto_relay disabled, the unused tool path fails loud as an error event - # instead. - if settled_reply: + # auto_relay disabled the text is dropped and the turn verdict decides. + if tools.turn.replied: logger.debug( "Room %s: Agent settled its reply via a Band tool, skipping auto-relay", room_id, @@ -636,12 +631,6 @@ async def _process_response_messages( room_id, self._mcp.send_message_tool, ) - detail = ( - f"Letta agent did not call {self._mcp.send_message_tool} " - "(auto-relay disabled); its reply was dropped" - ) - await tools.send_failure(AgentFailure(_PROVIDER, detail)) - raise TurnResultAlreadyReported(detail) else: final_text = "\n\n".join(final_text_parts) mentions = [reply_to_sender_id] if reply_to_sender_id else None diff --git a/src/band/adapters/opencode/adapter.py b/src/band/adapters/opencode/adapter.py index cbc70d5c9..1deb8b8ab 100644 --- a/src/band/adapters/opencode/adapter.py +++ b/src/band/adapters/opencode/adapter.py @@ -27,6 +27,7 @@ AgentToolsProtocol, ) from band.core.simple_adapter import SimpleAdapter +from band.core.turn import report_unsettled_turn from band.core.types import ( AdapterFeatures, Capability, @@ -66,7 +67,6 @@ logger = logging.getLogger(__name__) _PROVIDER = "opencode" -NO_TEXT_REPLY_MESSAGE = "OpenCode completed the turn without a text reply." _OPENCODE_SYSTEM_NOTE = """\ Responses are relayed back into the Band room by the adapter. @@ -1061,16 +1061,17 @@ async def _watch_turn_completion( logger.exception( "Failed to report the OpenCode timeout for room %s", room_id ) - await self._report_delivery_failure(room_state.room_id, turn) + await self._report_delivery_failure(turn) else: try: await self._deliver_fallback_text(room_state.room_id, turn) + await self._judge_detached_turn(turn) await self._emit_turn_usage(turn) except Exception: logger.exception( "Failed to deliver the OpenCode turn result for room %s", room_id ) - await self._report_delivery_failure(room_state.room_id, turn) + await self._report_delivery_failure(turn) finally: # Release the on_message waiter even if delivering the reply or # emitting usage raised (e.g. a sender-less turn has no one to @@ -1083,6 +1084,13 @@ async def _watch_turn_completion( expected_task=asyncio.current_task(), ) + @staticmethod + async def _judge_detached_turn(turn: TurnState) -> None: + """Judge a turn released early to wait on a human, since ``on_event`` + returned before it ended.""" + if turn.tools.turn.detached: + await report_unsettled_turn(turn.tools) + async def _abort_turn(self, turn: TurnState, reason: str) -> None: """Best-effort: tell OpenCode to stop working on this room's session.""" try: @@ -1098,23 +1106,20 @@ async def _abort_owner_turn(self, owner: TurnOwner, reason: str) -> None: if turn := owner(): await self._abort_turn(turn, reason) - async def _report_delivery_failure(self, room_id: str, turn: TurnState) -> None: + @staticmethod + async def _report_delivery_failure(turn: TurnState) -> None: """Tell the room the turn finished but its result could not be posted. - An event needs no mentions, so it still lands when the reply itself was - rejected for having none. + A failure event needs no mentions, so it still lands when the reply + itself was rejected for having none, and it marks the turn reported. """ - try: - await turn.tools.send_event( + await turn.tools.send_failure( + AgentFailure( + _PROVIDER, "OpenCode finished the turn but the result could not be posted " "to the room.", - "error", - ) - except Exception: - logger.exception( - "Failed to report the OpenCode delivery failure for room %s", - room_id, ) + ) async def _await_turn(self, turn: TurnState) -> None: """Await turn completion, but don't charge human-approval time to the @@ -1154,7 +1159,9 @@ def _release_turn_wait_for(self, turn: TurnState) -> None: self._resolve_future(turn.turn_release_future) def _release_turn_wait_for_owner(self, owner: TurnOwner) -> None: + """Release ``on_message`` early while the turn waits on a human.""" if turn := owner(): + turn.tools.turn.detach() self._release_turn_wait_for(turn) def _resolve_turn(self, turn: TurnState) -> None: @@ -1248,17 +1255,11 @@ async def _deliver_fallback_text(self, room_id: str, turn: TurnState) -> None: # An error is still surfaced after a tool reply -- it is not a text reply. try: - if await relay_reply(turn.tools, text, turn.pending_mentions): - return - if turn.last_error_message: + relayed = await relay_reply(turn.tools, text, turn.pending_mentions) + if not relayed and turn.last_error_message: await turn.tools.send_failure( AgentFailure(_PROVIDER, turn.last_error_message) ) - elif not turn.tools.turn.replied: - await turn.tools.send_notice( - NO_TEXT_REPLY_MESSAGE, - mentions=turn.pending_mentions, - ) finally: turn.pending_mentions = [] diff --git a/src/band/adapters/parlant.py b/src/band/adapters/parlant.py index 27e46bb56..972e14559 100644 --- a/src/band/adapters/parlant.py +++ b/src/band/adapters/parlant.py @@ -32,7 +32,6 @@ from band.integrations.parlant.tools import ( create_parlant_tools, set_session_tools, - was_message_sent, ) if TYPE_CHECKING: @@ -142,6 +141,11 @@ class ParlantAdapter(SimpleAdapter[ParlantMessages]): {Capability.MEMORY, Capability.CONTACTS, Capability.TASKS, Capability.FILES} ) + @property + def judges_turns(self) -> bool: + """Parlant's own engine owns its replies, so its turns are not judged.""" + return False + def __init__( self, config: ParlantAdapterConfig | None = None, @@ -650,8 +654,8 @@ async def _process_agent_response( 1. A preamble message (tagged with __preamble__) - acknowledgment before tool execution 2. Final message(s) after tool execution - If the send_message tool was called during processing, we don't need to - forward Parlant's response (it would be a duplicate or empty). + If a Band tool already replied or declined this turn, Parlant's response + is not forwarded (it would be a duplicate or empty). Waiting is bounded by a total budget, polling in shorter windows and retrying on an empty window so a slow (cold-start) turn is still answered. @@ -665,7 +669,6 @@ async def _process_agent_response( return app = self._app - session_id_str = str(session_id) from parlant.core.async_utils import ( # type: ignore[missing-import] # noqa: PLC0415 Timeout, ) @@ -708,7 +711,7 @@ async def _process_agent_response( logger.exception( "Room %s: Error waiting for update", room_id ) # Check if message was sent via tool before giving up - if was_message_sent(session_id_str): + if tools.turn.replied: logger.debug( "Room %s: Message was sent via tool, error is acceptable", room_id, @@ -718,7 +721,7 @@ async def _process_agent_response( if not has_update: # Empty poll window. If a tool already sent the reply we're done; # otherwise keep waiting until the budget — don't drop a slow turn. - if was_message_sent(session_id_str): + if tools.turn.replied: logger.debug( "Room %s: No new events but message was sent via tool, OK", room_id, @@ -803,12 +806,12 @@ async def _process_agent_response( ) continue - # Check if message was already sent via the send_message tool - # If so, don't send Parlant's response (would be duplicate/empty) - # Also don't mark as final - Parlant may still have more tool calls - if was_message_sent(session_id_str): + # The turn already replied or declined (a Band tool, or an + # earlier relay), so this response would duplicate it. Not + # final either: Parlant may still have more tool calls. + if tools.turn.replied: logger.debug( - "Room %s: Message already sent via tool, skipping Parlant response: %s...", + "Room %s: Turn already replied, skipping Parlant response: %s...", room_id, message_content[:50], ) @@ -839,7 +842,7 @@ async def _process_agent_response( return # Check if message was sent via tool (tool execution may happen without final message) - if was_message_sent(session_id_str): + if tools.turn.replied: logger.debug( "Room %s: Message sent via tool, no need to wait for final message", room_id, @@ -857,7 +860,7 @@ async def _process_agent_response( # stalls the post-preamble generation. We do NOT forward the preamble as the # reply: that would make the turn look answered when the agent actually failed # the user. Give up honestly; the turn produced no answer. - if was_message_sent(session_id_str): + if tools.turn.replied: logger.info( "Room %s: Response budget elapsed but message was sent via tool, OK", room_id, diff --git a/src/band/adapters/pydantic_ai.py b/src/band/adapters/pydantic_ai.py index 4c78b5602..5e7b89daf 100644 --- a/src/band/adapters/pydantic_ai.py +++ b/src/band/adapters/pydantic_ai.py @@ -33,6 +33,7 @@ ModelResponse, TextPart, ThinkingPart, + ToolReturnPart, UserPromptPart, ) from pydantic_ai.models import Model, ModelRequestContext @@ -46,7 +47,6 @@ from band.core.protocols import ( GENERIC_PROVIDER_FAILURE_MESSAGE, AgentToolsProtocol, - TurnResultAlreadyReported, ) from band.core.simple_adapter import SimpleAdapter from band.core.types import ( @@ -68,10 +68,7 @@ ) from band.runtime.prompts import render_system_prompt from band.runtime.tools import ( - band_tool_errored, image_block_placeholder, - is_terminal_success, - missing_reply_error, redact_tool_call_args, ) @@ -164,15 +161,16 @@ def _custom_tool_def_to_callable(tool_def: CustomToolDef) -> Callable[..., Any]: pydantic-ai flattens a single Pydantic-model parameter into the tool's arguments, so the wrapper keeps the ``(args: InputModel)`` registration shape after a leading ``RunContext``, whose deps are the room's tools, so each call records - its effect on that room's turn. Execution is routed through the shared ``invoke_validated_custom_tool`` so the CustomToolDef - contract matches every other adapter: async handlers are awaited and + its effect on that room's turn. Execution is routed through the shared + ``invoke_validated_custom_tool`` so the CustomToolDef contract matches every + other adapter: async handlers are awaited and zero-argument handlers (empty InputModel) are called without args — a plain sync passthrough would hand pydantic-ai an unawaited coroutine or raise TypeError for those. pydantic-ai has already validated ``args`` into the InputModel, so the instance is passed through directly — a dump/re-validate round-trip would break models using field aliases. The wrapper carries the stable tool name (derived from the model) and its declared turn effect, so the tool name and the - terminal-tool contract match the tuple adapters exactly. + turn effect match the tuple adapters exactly. """ input_model, handler = tool_def @@ -314,9 +312,8 @@ def __init__( _custom_tool_def_to_callable(tool) if isinstance(tool, tuple) else tool for tool in (additional_tools or []) ] - # Effects the custom tools declared on their function. Only these let an - # empty final response be treated as benign; an undeclared custom tool - # does not (fail-loud — see is_terminal_success). + # Effects the custom tools declared on their function; an undeclared + # custom tool only observes, so it cannot complete a turn. self._custom_effects = declared_effects( (fn.__name__, fn) for fn in self._custom_tools ) @@ -461,10 +458,7 @@ async def on_message( user_message[:80], ) - # Run agent with streaming to capture tool events. Track whether a - # terminal, successful tool ran (excludes read-only lookups and failed - # band tools) so we can tell a productive turn from a genuine no-op below. - tool_executed = False + # Run agent with streaming to capture tool events. # pydantic-ai's result.usage is already summed across the run's model # calls, so it's set once (on the result event), not accumulated. turn_usage = TurnUsage() @@ -518,18 +512,12 @@ async def on_message( except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn logger.warning("Failed to send tool_call event: %s", e) elif isinstance(event, FunctionToolResultEvent): - # Custom tools count as terminal only if they declared an - # effect; undeclared customs fail loud. A failed band - # tool (its wrapper returns an "Error " string) is not terminal. - result_name = event.part.tool_name - if is_terminal_success( - result_name, - succeeded=not band_tool_errored( - result_name, event.part.content - ), - custom_effects=self._custom_effects, + # Native custom tools run outside execute_custom_tool, so + # their declared effect is recorded here. + if isinstance(event.part, ToolReturnPart) and ( + effect := self._custom_effects.get(event.part.tool_name) ): - tool_executed = True + tools.turn.record(effect) if Emit.TOOL_CALLS in self.features.emit: output = event.part.content if ( @@ -583,7 +571,7 @@ async def on_message( # owes pydantic-ai. Allowing `None` — and normalizing blank text into it # — ends the ordinary nothing-left-to-say response cleanly, but some # other response the run cannot turn into output can still spend the - # refused output budget. Once a terminal tool has run (a + # refused output budget. Once the turn is complete (a # band_send_message reply, a band_store_memory, ...) the work already went # out, so that exhaustion is benign — swallow it. Every other exception — # a different UnexpectedModelBehavior, or any other type now that this @@ -593,13 +581,13 @@ async def on_message( # pydantic-ai raises the exhausted-retries case as its own distinct type, # so the isinstance check (not just the message match) is load-bearing. if ( - tool_executed + tools.turn.complete and isinstance(e, UnexpectedModelBehavior) and _is_output_retries_exhausted(e) ): logger.warning( "Room %s: Pydantic AI exhausted its output retries after " - "the agent already did productive work this turn; treating as " + "the turn was already complete; treating as " "non-fatal: %s", room_id, e, @@ -635,17 +623,6 @@ async def on_message( turn_usage = self._usage_from_messages(this_run) await self.emit_usage(tools, turn_usage) - # A clean run with no terminal work is a silently dropped reply: the model - # either answered in plain text or said nothing at all. Surface it as an - # error (mirrors the crewai adapter) instead of letting it vanish. - if not tool_executed: - logger.warning( - "Room %s: Pydantic AI turn produced nothing for the room", room_id - ) - detail = missing_reply_error("Pydantic AI") - await tools.send_failure(AgentFailure(_PROVIDER, detail)) - raise TurnResultAlreadyReported(detail) - logger.debug( "Room %s: Pydantic AI agent completed (history now has %s messages)", room_id, diff --git a/src/band/adapters/strands.py b/src/band/adapters/strands.py index 2bdb94fee..0cbb78263 100644 --- a/src/band/adapters/strands.py +++ b/src/band/adapters/strands.py @@ -38,7 +38,6 @@ from band.core.protocols import ( GENERIC_PROVIDER_FAILURE_MESSAGE, AgentToolsProtocol, - TurnResultAlreadyReported, ) from band.core.simple_adapter import SimpleAdapter from band.core.tool_filter import filter_tool_schemas @@ -69,9 +68,7 @@ get_band_tool_category, image_block_placeholder, is_image_passthrough_result, - is_terminal_success, iter_tool_definitions, - missing_reply_error, redact_tool_call_args, serialize_tool_result, validate_tool_arguments, @@ -134,7 +131,7 @@ def _tool_result(tool_use: ToolUse, *, value: object, ok: bool) -> ToolResult: def _result_text(result: ToolResult) -> str: - """Flatten a tool result for execution events and terminal-state policy.""" + """Flatten a tool result for execution events and failure detection.""" parts: list[str] = [] image_count = 0 image_index: int | None = None @@ -335,7 +332,7 @@ async def _execute(self, arguments: dict[str, Any]) -> ToolCallOutcome: class BandTurnHooks(HookProvider): - """Emit execution events and record whether a turn completed useful work.""" + """Emit execution events and record native tools' declared turn effects.""" def __init__( self, @@ -347,7 +344,6 @@ def __init__( self._tools = tools self._emit_execution = emit_execution self._custom_effects = custom_effects - self.terminal_fired = False def register_hooks(self, registry: HookRegistry, **kwargs: Any) -> None: del kwargs @@ -374,10 +370,10 @@ async def _on_after_tool(self, event: AfterToolCallEvent) -> None: succeeded = event.result.get("status") == "success" and not band_tool_errored( name, output ) - if is_terminal_success( - name, succeeded=succeeded, custom_effects=self._custom_effects - ): - self.terminal_fired = True + # Native Strands tools run outside execute_custom_tool, so their + # declared effect is recorded here. + if succeeded and (effect := self._custom_effects.get(name)): + self._tools.turn.record(effect) if not self._emit_execution: return await self._emit_event( @@ -592,7 +588,7 @@ async def on_message( is_session_bootstrap: bool, room_id: str, ) -> None: - """Run one room turn and surface a missing tool-based reply.""" + """Run one room turn through the Strands agent loop.""" room_history = self._history_for_turn( room_id, history, is_session_bootstrap=is_session_bootstrap ) @@ -618,13 +614,6 @@ async def on_message( tools=tools, hooks=hooks, ) - if not hooks.terminal_fired: - logger.warning( - "Room %s: Strands turn produced nothing for the room", room_id - ) - detail = missing_reply_error("Strands") - await tools.send_failure(AgentFailure(_PROVIDER, detail)) - raise TurnResultAlreadyReported(detail) logger.debug( "Room %s: Strands agent completed (history now has %s messages)", room_id, diff --git a/src/band/client/streaming/client.py b/src/band/client/streaming/client.py index 255724eab..72baed147 100644 --- a/src/band/client/streaming/client.py +++ b/src/band/client/streaming/client.py @@ -72,9 +72,11 @@ class DeliveryStatus(StrEnum): Mirrors the backend's allowed values. The lifecycle for a recipient is ``DELIVERED -> PROCESSING -> PROCESSED | FAILED``. ``FAILED`` is **not** - terminal: the platform retries failed messages (bounded by max retries), so - a message may cycle ``FAILED -> PROCESSING`` again before reaching - ``PROCESSED``. ``PROCESSED`` is the only success terminal. + terminal: the platform's ``/next`` serves a failed message again, so a new + agent process (or execution context) may cycle it ``FAILED -> PROCESSING`` + before ``PROCESSED``. The running agent itself skips a message once it has + failed ``max_message_retries`` times (default 1). ``PROCESSED`` is the only + success terminal. """ DELIVERED = "delivered" diff --git a/src/band/core/simple_adapter.py b/src/band/core/simple_adapter.py index cc508a795..0b9ad27ca 100644 --- a/src/band/core/simple_adapter.py +++ b/src/band/core/simple_adapter.py @@ -18,7 +18,12 @@ ModelSelection, check_model_selection, ) -from band.core.protocols import AgentToolsProtocol, HistoryConverter +from band.core.protocols import ( + AgentToolsProtocol, + HistoryConverter, + TurnResultAlreadyReported, +) +from band.core.turn import report_unsettled_turn from band.core.types import ( USAGE_EVENT_TYPE, USAGE_METADATA_KEY, @@ -30,6 +35,7 @@ PlatformConnection, PlatformMessage, TurnUsage, + is_contact_hub_turn, ) from band.core.validation import listing from band.logging_config import trace_context_scope @@ -364,6 +370,16 @@ async def on_started(self, agent_name: str, agent_description: str) -> None: # --- FrameworkAdapter protocol implementation --- + @property + def judges_turns(self) -> bool: + """Whether ``on_event`` judges each turn by core's turn-outcome rule. + + ``False`` for an adapter whose turns are not the model's to answer + through Band tools: a bridge to another agent, or a framework engine + that owns its own replies. + """ + return True + async def on_event(self, inp: AgentInput) -> None: """Implements FrameworkAdapter.on_event(). @@ -371,7 +387,10 @@ async def on_event(self, inp: AgentInput) -> None: that calls the per-adapter ``on_message`` override), so this is where the turn's trace-context correlation window opens -- every log line emitted anywhere during this turn's processing picks it up via - ``band.logging_config``'s log filter. + ``band.logging_config``'s log filter -- and where the turn is judged. + A turn that ended without completing is reported once, then raised as + ``TurnResultAlreadyReported`` so the runtime marks it FAILED without + a second report. A detached turn is judged at its real end instead. """ with trace_context_scope(): # Convert history if converter is set @@ -382,6 +401,10 @@ async def on_event(self, inp: AgentInput) -> None: # Adapters without converters should type as SimpleAdapter[HistoryProvider] converted_history = inp.history + turn = inp.tools.turn + turn.judged = self.judges_turns and not is_contact_hub_turn( + sender_type=inp.msg.sender_type, sender_id=inp.msg.sender_id + ) await self.on_message( msg=inp.msg, tools=inp.tools, @@ -391,3 +414,9 @@ async def on_event(self, inp: AgentInput) -> None: is_session_bootstrap=inp.is_session_bootstrap, room_id=inp.room_id, ) + if ( + turn.judged + and not turn.detached + and await report_unsettled_turn(inp.tools) + ): + raise TurnResultAlreadyReported("turn ended without a reply") diff --git a/src/band/core/turn.py b/src/band/core/turn.py index 3cd1cfc29..3a0bb4563 100644 --- a/src/band/core/turn.py +++ b/src/band/core/turn.py @@ -51,7 +51,13 @@ def note_reported(self) -> None: self._ledger.note_reported() def detach(self) -> None: - self._detached = True + """Move a judged turn's verdict to the adapter's real end of the turn. + + A turn ``on_event`` does not judge stays attached, so the adapter's + ``if turn.detached`` report never fires for it. + """ + if self.judged: + self._detached = True @property def detached(self) -> bool: diff --git a/src/band/integrations/a2a/adapter.py b/src/band/integrations/a2a/adapter.py index df7631097..fa6308bdf 100644 --- a/src/band/integrations/a2a/adapter.py +++ b/src/band/integrations/a2a/adapter.py @@ -94,6 +94,11 @@ class A2AAdapter(SimpleAdapter[A2ASessionState]): SUPPORTED_EMIT: ClassVar[frozenset[Emit]] = frozenset() SUPPORTED_CAPABILITIES: ClassVar[frozenset[Capability]] = frozenset() + @property + def judges_turns(self) -> bool: + """The remote A2A agent owns its replies, so its turns are not judged.""" + return False + def __init__( self, config: A2AAdapterConfig, diff --git a/src/band/integrations/a2a/gateway/adapter.py b/src/band/integrations/a2a/gateway/adapter.py index 6f557c306..90d1e6e65 100644 --- a/src/band/integrations/a2a/gateway/adapter.py +++ b/src/band/integrations/a2a/gateway/adapter.py @@ -147,6 +147,11 @@ class A2AGatewayAdapter(SimpleAdapter[GatewaySessionState]): SUPPORTED_EMIT: ClassVar[frozenset[Emit]] = frozenset() SUPPORTED_CAPABILITIES: ClassVar[frozenset[Capability]] = frozenset() + @property + def judges_turns(self) -> bool: + """Peer messages go back to an A2A caller, not the room: never judged.""" + return False + def __init__( self, config: A2AGatewayAdapterConfig | None = None, diff --git a/src/band/integrations/acp/server_adapter.py b/src/band/integrations/acp/server_adapter.py index 6859a14c7..fa26d0808 100644 --- a/src/band/integrations/acp/server_adapter.py +++ b/src/band/integrations/acp/server_adapter.py @@ -148,6 +148,12 @@ def __init__( # Push handler for unsolicited updates self._push_handler: ACPPushHandler | None = None + @property + def judges_turns(self) -> bool: + """Room messages are peers' answers relayed to the editor, not turns + this agent owes a Band reply.""" + return False + # ── Public accessors (used by ACPServer and ACPPushHandler) ── # # These accessors read/write dicts guarded by _state_lock. Since they diff --git a/src/band/integrations/crewai/reporting.py b/src/band/integrations/crewai/reporting.py index 205911293..9296925b3 100644 --- a/src/band/integrations/crewai/reporting.py +++ b/src/band/integrations/crewai/reporting.py @@ -25,23 +25,15 @@ class ReplyTracker: """Mutable per-turn markers shared (by reference) with the tool wrappers. - ``replied`` flips once ``band_send_message`` succeeds; ``tool_executed`` flips - once any terminal tool succeeds; ``any_tool_ran`` flips on any tool call at - all, success or failure, terminal or not. ``posts`` keeps what each - successful ``band_send_message`` said: CrewAI's final answer does not, so - without it the agent forgets its own messages by the next turn. + ``any_tool_ran`` flips on any tool call at all, success or failure. + ``posts`` keeps what each successful ``band_send_message`` said: CrewAI's + final answer does not, so without it the agent forgets its own messages by + the next turn. Whether the turn is complete is ``tools.turn``'s to judge. """ - replied: bool = False posts: list[str] = field(default_factory=list) - tool_executed: bool = False any_tool_ran: bool = False - @property - def did_productive_work(self) -> bool: - """Whether the turn left something behind for the room to show for it.""" - return self.replied or self.tool_executed - @dataclass(frozen=True) class CrewAIToolContext: diff --git a/src/band/integrations/crewai/tools.py b/src/band/integrations/crewai/tools.py index 758b7a18a..b52cf2782 100644 --- a/src/band/integrations/crewai/tools.py +++ b/src/band/integrations/crewai/tools.py @@ -50,7 +50,6 @@ from band.integrations.crewai.runtime import run_async from band.runtime.custom_tools import ( CustomToolDef, - custom_tool_effects, execute_custom_tool, get_custom_tool_name, ) @@ -58,11 +57,9 @@ CAPABILITY_TOOL_NAMES, EVENT_TOOL_NAMES, BandTool, - TurnEffect, append_available_mention_handles, get_band_tool_category, get_tool_description, - is_terminal_success, ) logger = logging.getLogger(__name__) @@ -79,7 +76,6 @@ def _execute_tool( reporter: CrewAIToolReporter, fallback_loop: asyncio.AbstractEventLoop | None, arguments: Mapping[str, Any], - custom_effects: Mapping[str, TurnEffect] | None = None, ) -> str: """Execute a tool with common error handling and reporting. @@ -118,45 +114,31 @@ async def _execute() -> str: result = run_async(_execute(), fallback_loop=fallback_loop) if context.reply_tracker is not None: - _mark_productive_work( - context.reply_tracker, - tool_name, - result, - arguments, - custom_effects=custom_effects, - ) + _note_tool_activity(context.reply_tracker, tool_name, result, arguments) return result -def _mark_productive_work( +def _note_tool_activity( tracker: ReplyTracker, tool_name: str, result: str, arguments: Mapping[str, Any], - *, - custom_effects: Mapping[str, TurnEffect] | None, ) -> None: - """Record that the turn did real work, so an empty final answer stays benign. - - CrewAI raises on an empty final answer; that is a genuine no-response - failure only when nothing terminal ran. ``is_terminal_success`` is the - shared rule for what counts (read-only Band tools and undeclared custom - tools do not) toward the missing-reply decision. ``any_tool_ran`` is - separate and coarser: it flips on this call alone, whether the tool - succeeded or not, so a turn that tried *something* is never mistaken for - one where the model's very first response came back empty. + """Note that a tool ran, and what a successful ``band_send_message`` said. + + ``any_tool_ran`` flips on this call alone, whether the tool succeeded or + not, so a turn that tried *something* is never mistaken for one where the + model's very first response came back empty. """ tracker.any_tool_ran = True + if tool_name != BandTool.SEND_MESSAGE: + return try: if json.loads(result).get("status") != "success": return except (json.JSONDecodeError, AttributeError, TypeError): return - if is_terminal_success(tool_name, succeeded=True, custom_effects=custom_effects): - tracker.tool_executed = True - if tool_name == BandTool.SEND_MESSAGE: - tracker.replied = True - tracker.posts.append(_describe_post(arguments)) + tracker.posts.append(_describe_post(arguments)) def _describe_post(arguments: Mapping[str, Any]) -> str: @@ -222,9 +204,6 @@ def _custom_tool( input_model, _ = definition tool_name = get_custom_tool_name(input_model) - # Only a custom tool that declared an effect lets an empty final answer be - # treated as benign; undeclared customs fail loud. - effects = custom_tool_effects([definition]) class CustomCrewAITool(BaseTool): name: str = tool_name @@ -248,7 +227,6 @@ async def execute(tools: AgentToolsProtocol) -> str: reporter=reporter, fallback_loop=fallback_loop, arguments=kwargs, - custom_effects=effects, ) return CustomCrewAITool() diff --git a/src/band/integrations/letta/config.py b/src/band/integrations/letta/config.py index e509d928d..be8e02702 100644 --- a/src/band/integrations/letta/config.py +++ b/src/band/integrations/letta/config.py @@ -138,7 +138,8 @@ class LettaAdapterConfig(EnvAdapterConfig): auto_relay: Relay the agent's plain assistant text into the room when it did not call the MCP send tool. Keeps the agent responsive when the model skips tools, but can mask a dead MCP tool path — disable - to make an unused tool path fail loudly (an error event) instead. + to drop the text instead, so a turn that neither replied nor did + real work is reported as a missing reply. delete_agents_on_cleanup: Delete the room's Letta agent on cleanup. Off by default: persisted agents are what makes resume-by-id work across restarts. Per_room mode only — the shared agent outlives diff --git a/src/band/integrations/parlant/tools.py b/src/band/integrations/parlant/tools.py index 7fd2e6676..b56d88e2d 100644 --- a/src/band/integrations/parlant/tools.py +++ b/src/band/integrations/parlant/tools.py @@ -69,10 +69,6 @@ # This approach works across async contexts (unlike ContextVar) _session_tools: dict[str, Any] = {} -# Track whether send_message was called for each session -# This helps the adapter know if it needs to forward Parlant's response -_session_message_sent: dict[str, bool] = {} - # Parlant tools take mentions as a comma-separated string, not the master # model's list[str], so the master description needs this appended — it is # genuinely Parlant-specific and not something get_tool_description() covers. @@ -164,10 +160,8 @@ def set_session_tools(session_id: str, tools: Any | None) -> None: """Set the tools for a specific Parlant session.""" if tools is None: _session_tools.pop(session_id, None) - _session_message_sent.pop(session_id, None) else: _session_tools[session_id] = tools - _session_message_sent[session_id] = False logger.debug("Set tools for session %s: %s", session_id, tools is not None) @@ -183,17 +177,6 @@ def get_session_tools(session_id: str) -> Any | None: return tools -def mark_message_sent(session_id: str) -> None: - """Mark that a message was sent via the send_message tool for this session.""" - _session_message_sent[session_id] = True - logger.debug("Marked message sent for session %s", session_id) - - -def was_message_sent(session_id: str) -> bool: - """Check if a message was sent via the send_message tool for this session.""" - return _session_message_sent.get(session_id, False) - - # Keep old API for backwards compatibility (deprecated) def set_current_tools(tools: Any | None) -> None: """Deprecated: Use set_session_tools instead.""" @@ -370,8 +353,6 @@ async def band_send_message( ) await tools.send_message(content, recipients) - # Tells the adapter its own reply would duplicate this one. - mark_message_sent(context.session_id) return ToolResult(data=f"Message sent to {', '.join(recipients)}") @band_tool("sending event") @@ -396,9 +377,6 @@ async def band_no_reply( ) -> ToolResult: tools = require_session_tools(context) await tools.no_reply(or_none(reason)) - # Same suppression as a sent message: the adapter must not post - # its own reply for this turn. - mark_message_sent(context.session_id) return ToolResult(data="No reply sent; this turn is complete") @band_tool("adding participant '{identifier}'") diff --git a/src/band/integrations/pydantic_ai/tools.py b/src/band/integrations/pydantic_ai/tools.py index 25e409fee..d0e148810 100644 --- a/src/band/integrations/pydantic_ai/tools.py +++ b/src/band/integrations/pydantic_ai/tools.py @@ -32,10 +32,10 @@ get_band_tool_category, get_tool_description, is_image_passthrough_result, - is_terminal_success, iter_tool_definitions, platform_args_schema, serialize_tool_result, + turn_effect, validate_tool_arguments, ) @@ -248,13 +248,12 @@ async def _dispatch_failed( error, exc_info=error, ) - if notify_room or not is_terminal_success(definition.name, succeeded=True): - # The "Error " prefix is load-bearing: band_tool_errored reads it to - # tell a failed Band tool from productive work. + if notify_room or not turn_effect(definition.name).did_work: message = f"Error executing {definition.name}: {error}" else: - # The method completed, so this must not look like a failed terminal - # action to the adapter even though the result could not be normalized. + # The method completed, so the model must not read it as a failure and + # redo it (re-posting a message) just because the result could not be + # normalized. message = ( f"{definition.name} executed, but its result could not be normalized: " f"{error}" diff --git a/src/band/integrations/slack/adapter.py b/src/band/integrations/slack/adapter.py index 10d412569..e319caa2d 100644 --- a/src/band/integrations/slack/adapter.py +++ b/src/band/integrations/slack/adapter.py @@ -555,6 +555,11 @@ def inner(self) -> SimpleAdapter[Any]: """The wrapped framework adapter (the brain).""" return self._inner + @property + def judges_turns(self) -> bool: + """The brain answers each turn, so whether it is judged is the brain's.""" + return self._inner.judges_turns + @property def transport(self) -> SlackTransport: """Which inbound transport this adapter was configured with.""" diff --git a/src/band/runtime/prompts.py b/src/band/runtime/prompts.py index 3945fcc0d..5c1bae77c 100644 --- a/src/band/runtime/prompts.py +++ b/src/band/runtime/prompts.py @@ -31,6 +31,7 @@ ) from band.core.types import AdapterFeatures, Capability from band.runtime.tools.inputs.chat import MENTION_IDENTIFIERS +from band.runtime.tools.types import BandTool # Base instructions appended to user's custom prompt BASE_INSTRUCTIONS = f""" @@ -38,7 +39,8 @@ Multi-participant chat. Messages show sender: [Name]: content. Messages prefixed with [System]: are platform updates (participant changes, contact updates, etc.). -Use `band_send_message(content, mentions)` to respond — a `band_send_message` call is the only way anything you say reaches the room. Any text you produce outside such a call is never delivered, and that includes a final answer you compose after using other tools. So deliver your answer by calling `band_send_message`; a turn that ends with the answer written as plain text delivers nothing. +Use `{BandTool.SEND_MESSAGE}(content, mentions)` to respond — a `{BandTool.SEND_MESSAGE}` call is the only way anything you say reaches the room. Any text you produce outside such a call is never delivered, and that includes a final answer you compose after using other tools. So deliver your answer by calling `{BandTool.SEND_MESSAGE}`. +When the latest message needs no answer from you (it was addressed to someone else, it is an FYI or an acknowledgement, or another participant already answered it), call `{BandTool.NO_REPLY}` instead to end the turn deliberately. End every turn with one of the two. {MENTION_IDENTIFIERS} ## Security diff --git a/src/band/runtime/tools/__init__.py b/src/band/runtime/tools/__init__.py index f0eca6003..306e94ba6 100644 --- a/src/band/runtime/tools/__init__.py +++ b/src/band/runtime/tools/__init__.py @@ -27,8 +27,6 @@ strip_handle_prefix, ) from band.runtime.tools.effects import ( - is_terminal_success, - settles_turn_reply, turn_effect, ) from band.runtime.tools.human import HumanTools @@ -125,7 +123,6 @@ iter_tool_definitions, mcp_tool_names, mcp_tool_spelling, - missing_reply_error, redact_tool_call_args, resolve_capabilities, ) @@ -266,20 +263,17 @@ "image_block_placeholder", "is_image_passthrough_result", "is_mcp_content_result", - "is_terminal_success", "iter_chat_pages", "iter_tool_definitions", "matches_identifier", "mcp_tool_names", "mcp_tool_spelling", - "missing_reply_error", "platform_args_schema", "platform_tool", "redact_tool_call_args", "resolve_capabilities", "resolve_tool_model", "serialize_tool_result", - "settles_turn_reply", "strip_handle_prefix", "turn_effect", "validate_tool_arguments", diff --git a/src/band/runtime/tools/agent.py b/src/band/runtime/tools/agent.py index c97e291f5..a765e94d1 100644 --- a/src/band/runtime/tools/agent.py +++ b/src/band/runtime/tools/agent.py @@ -511,7 +511,9 @@ async def send_failure(self, failure: band_sdk_core.AgentFailure) -> Any: logger.exception("send_failure could not post the failure event") return {"ok": False, "error": str(exc)} self.turn.note_reported() - if self._ctx is not None: + # A detached turn reports after its delivery was released, while the + # context may already be processing a later message. + if self._ctx is not None and not self.turn.detached: self._ctx.note_turn_failure_reported() return response diff --git a/src/band/runtime/tools/effects.py b/src/band/runtime/tools/effects.py index c02458af3..912d10f6f 100644 --- a/src/band/runtime/tools/effects.py +++ b/src/band/runtime/tools/effects.py @@ -1,9 +1,7 @@ """What a finished tool call does to the turn's reply obligation. -One ``TurnEffect`` per tool answers both questions an adapter asks after a call: -did the turn do work (``is_terminal_success``: an empty final answer is then -benign) and is its reply settled (``settles_turn_reply``: no fallback text may be -relayed). +Every tool call records its ``TurnEffect`` on the turn's ledger +(``band.core.turn.Turn``), which band-sdk-core's rule then judges. """ from __future__ import annotations @@ -42,47 +40,3 @@ def turn_effect( if band_tool in _BAND_EFFECTS: return _BAND_EFFECTS[band_tool] return (custom_effects or {}).get(tool_name, TurnEffect.OBSERVE) - - -def settles_turn_reply( - tool_name: str, *, custom_effects: Mapping[str, TurnEffect] | None = None -) -> bool: - """Whether a successful call of ``tool_name`` is the turn's reply. - - True for a room post or ``band_no_reply``'s deliberate silence, and for a - custom tool that declared either. Bridge adapters relay the model's plain text - only when no call settled the reply, so the reply is delivered exactly once. A - miss only costs a duplicate reply (the pre-suppression behavior), never a - wrong post. - """ - return turn_effect(tool_name, custom_effects=custom_effects).settles_reply - - -def is_terminal_success( - tool_name: str | None, - *, - succeeded: bool, - custom_effects: Mapping[str, TurnEffect] | None = None, -) -> bool: - """Whether a finished tool call counts as terminal productive work. - - Single source of truth shared by the tool-only adapters (claude_sdk, crewai, - pydantic-ai, strands) to decide whether an empty final model response is - *benign* (the agent already did its work this turn) or a genuine no-response - failure. Terminal work is a call that did not fail and whose ``turn_effect`` - did work. - - Observing tools never count -- fetching state (``READ_ONLY_TOOL_NAMES``) or - narrating (``EVENT_TOOL_NAMES``: band_send_event posts a thought/error/task - event) is not a chat reply or a durable requested action. Custom tools are - **not** terminal by default: the SDK cannot know whether a bare custom tool is a - lookup or a side-effecting action, so it fails loud — an empty final after only - an undeclared custom tool surfaces as a no-response error rather than being - silently swallowed. A custom tool that genuinely completes the turn declares - its effect (see ``runtime.custom_tools.declares_turn_effect``). - """ - return ( - succeeded - and tool_name is not None - and turn_effect(tool_name, custom_effects=custom_effects).did_work - ) diff --git a/src/band/runtime/tools/registry.py b/src/band/runtime/tools/registry.py index 056e593e5..e9175d4d0 100644 --- a/src/band/runtime/tools/registry.py +++ b/src/band/runtime/tools/registry.py @@ -616,10 +616,8 @@ def classify_room_binding(definition: ToolDefinition) -> tuple[bool, bool]: } ) -# Event-emitting tools are observational, not terminal work: band_send_event posts a -# thought/error/task event (narration/status) — not a chat reply or a durable requested -# action. Like read-only tools, a turn that only sends an event and then yields an empty -# final answer is a genuine no-response failure, not benign (see is_terminal_success). +# Event-emitting tools narrate (a thought/error/task event): never a chat reply or a +# durable requested action. EVENT_TOOL_NAMES: frozenset[str] = frozenset({BandTool.SEND_EVENT}) # Human-surface memory tools - parallel to MEMORY_TOOL_NAMES but on the @@ -670,23 +668,6 @@ def band_tool_errored(tool_name: str | None, content: Any) -> bool: ) -def missing_reply_error(framework: str, *, detail: str = "") -> str: - """The room-visible error for a turn that ended without a reply going out. - - Raised by every adapter that answers through tools, so the wording lives - once. Both endings are named because they look identical from the room and - are told apart only by the model's last response: a plain-text final answer - the adapter cannot post, or no output at all (empty or thinking-only), which - is what a model that considers the exchange finished actually returns. - """ - reasons = ( - f"{framework} finished a turn without calling band_send_message, so " - "nothing reached the room. The model either answered in plain text " - "instead of using the tool, or returned no output at all." - ) - return f"{reasons} {detail}" if detail else reasons - - # Fail fast on typos — catch at import time, not in a test run. # Use explicit checks instead of ``assert`` so they are not stripped by -O. if MEMORY_TOOL_NAMES - ALL_TOOL_NAMES: diff --git a/src/band/runtime/tools/types.py b/src/band/runtime/tools/types.py index d31cfba24..c4e09c2a5 100644 --- a/src/band/runtime/tools/types.py +++ b/src/band/runtime/tools/types.py @@ -77,15 +77,14 @@ class ToolCategory(StrEnum): class TurnEffect(StrEnum): - """What a successful call of a Band tool does to the turn's reply obligation. + """What a successful tool call does to the turn's reply obligation. - The one classification behind both turn-outcome questions: whether the - turn did any work (an empty final answer is then benign) and whether its - reply is settled (no fallback text may be relayed). + Mirrors band-sdk-core's ``TurnEffect`` (pinned by a parity test); it stays + a ``StrEnum`` because custom tools declare it (``declares_turn_effect``). """ OBSERVE = "observe" # fetches state or narrates; the turn still owes a reply - ACT = "act" # a durable side effect; silence afterwards is benign + ACT = "act" # a durable side effect; the turn is complete without a reply REPLY = "reply" # posts to the room DECLINE = "decline" # deliberate silence (band_no_reply) @@ -93,10 +92,6 @@ class TurnEffect(StrEnum): def did_work(self) -> bool: return self is not TurnEffect.OBSERVE - @property - def settles_reply(self) -> bool: - return self in (TurnEffect.REPLY, TurnEffect.DECLINE) - @dataclass(frozen=True) class ToolDefinition: diff --git a/src/band/testing/__init__.py b/src/band/testing/__init__.py index 633e894dd..dc09d21df 100644 --- a/src/band/testing/__init__.py +++ b/src/band/testing/__init__.py @@ -12,12 +12,18 @@ # Type-only imports for static analysis (pyrefly, mypy, etc.) if TYPE_CHECKING: + from band.testing.fake_tools import ( + MISSING_REPLY_FAILURE as MISSING_REPLY_FAILURE, + ) from band.testing.fake_tools import ( FakeAgentTools as FakeAgentTools, ) from band.testing.fake_tools import ( events_of_type as events_of_type, ) + from band.testing.fake_tools import ( + failure_reports as failure_reports, + ) from band.testing.fake_tools import ( reported_failures as reported_failures, ) @@ -55,7 +61,13 @@ __all__, __getattr__ = lazy_exports( __name__, - fake_tools=["FakeAgentTools", "events_of_type", "reported_failures"], + fake_tools=[ + "MISSING_REPLY_FAILURE", + "FakeAgentTools", + "events_of_type", + "failure_reports", + "reported_failures", + ], features=["feature_kwargs"], phoenix_server=[ "FakePhoenixServer", diff --git a/src/band/testing/fake_tools.py b/src/band/testing/fake_tools.py index c7b04884a..a55a28c0f 100644 --- a/src/band/testing/fake_tools.py +++ b/src/band/testing/fake_tools.py @@ -49,7 +49,11 @@ ) from band.core.content import has_visible_content from band.core.exceptions import BandToolError -from band.core.protocols import FailureMetadataKey, to_failure_event +from band.core.protocols import ( + TURN_FAILURE_PROVIDER, + FailureMetadataKey, + to_failure_event, +) from band.core.task_types import ( TaskAssignmentStatus, TaskLifecycleState, @@ -1192,3 +1196,12 @@ def reported_failures(tools: FakeAgentTools) -> list[dict[str, Any]]: for e in events_of_type(tools, MessageType.ERROR) if FailureMetadataKey.FAILURE in e["metadata"] ] + + +# The one failure a missing-reply verdict reports, as ``failure_reports`` lists it. +MISSING_REPLY_FAILURE = (TURN_FAILURE_PROVIDER, band_sdk_core.missing_reply_message()) + + +def failure_reports(tools: FakeAgentTools) -> list[tuple[str, str]]: + """Every failure reported via ``send_failure``, as ``(provider, message)``.""" + return [(f["provider"], f["message"]) for f in reported_failures(tools)] diff --git a/tests/adapters/agno/conftest.py b/tests/adapters/agno/conftest.py index 3ee6c1897..46520b42f 100644 --- a/tests/adapters/agno/conftest.py +++ b/tests/adapters/agno/conftest.py @@ -7,7 +7,7 @@ from collections.abc import Awaitable, Callable from typing import Any -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock import pytest from agno.agent import Agent as AgnoAgent @@ -16,8 +16,9 @@ from band.adapters.agno import AgnoAdapter from band.core.types import FeatureKwargs, PlatformMessage +from band.runtime.tools import BandTool from band.testing import FakeAgentTools -from tests.adapters.agno.helpers import CapturingModel, SchemaTools +from tests.adapters.agno.helpers import CapturingModel, SchemaTools, fake_agno_agent @pytest.fixture(autouse=True) @@ -44,46 +45,25 @@ def make_agno_agent() -> Callable[..., MagicMock]: def _make( *, - update_memory_on_run: bool = False, - enable_agentic_memory: bool = False, - add_history_to_context: bool = False, - db: object | None = None, + replies_through: FakeAgentTools | None = None, response: RunOutput | None = None, - events: list[Any] | None = None, + **config: Any, ) -> MagicMock: - agent = MagicMock(name="agno_agent") - agent.update_memory_on_run = update_memory_on_run - agent.enable_agentic_memory = enable_agentic_memory - # Explicit falsy defaults: a bare MagicMock would expose these as truthy - # auto-attributes and spuriously trip the history-management guard. - agent.add_history_to_context = add_history_to_context - agent.db = db - agent.add_tool = MagicMock() - # Real Agno agents default additional_context to None; mirror that. - agent.additional_context = None - # The adapter captures the user's tools at startup, then installs a - # callable factory. A bare MagicMock `.tools` is itself callable and would - # be mistaken for a user-supplied tools factory, so pin it to a list. - agent.tools = [] - resp = response if response is not None else RunOutput() - if events is None: - # Non-streaming path (Emit.TOOL_CALLS off): `await agent.arun(...)`. - agent.arun = AsyncMock(return_value=resp) - else: - # Streaming path (Emit.TOOL_CALLS on): the adapter iterates - # `agent.arun(stream=True, ...)`, which yields the run events then the - # final RunOutput. A bare MagicMock returns an async iterator without - # awaiting, matching how the adapter consumes the stream. - def _arun(*args: Any, **kwargs: Any) -> Any: - async def _stream() -> Any: - for event in events: - yield event - yield resp - - return _stream() - - agent.arun = MagicMock(side_effect=_arun) - return agent + async def reply() -> None: + assert replies_through is not None + await replies_through.execute_tool_call( + BandTool.SEND_MESSAGE, + { + "content": (response and response.content) or "ok", + "mentions": ["@Alice"], + }, + ) + + return fake_agno_agent( + response=response, + model_turn=reply if replies_through is not None else None, + **config, + ) return _make @@ -101,6 +81,7 @@ async def _make( add_history_to_context: bool = False, db: object | None = None, events: list[Any] | None = None, + replies_through: FakeAgentTools | None = None, **features: Unpack[FeatureKwargs], ) -> tuple[AgnoAdapter, MagicMock]: agent = make_agno_agent( @@ -108,6 +89,7 @@ async def _make( add_history_to_context=add_history_to_context, db=db, events=events, + replies_through=replies_through, ) # Most call sites here test something other than narration, and the # fake agent's arun() is a plain (non-streaming) AsyncMock unless the diff --git a/tests/adapters/agno/helpers.py b/tests/adapters/agno/helpers.py index 964a19b34..074600451 100644 --- a/tests/adapters/agno/helpers.py +++ b/tests/adapters/agno/helpers.py @@ -10,13 +10,14 @@ from __future__ import annotations +from collections.abc import Awaitable, Callable from typing import Any -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock from agno.models.base import Model from agno.models.message import Message from agno.models.response import ModelResponse, ToolExecution -from agno.run.agent import ToolCallCompletedEvent, ToolCallStartedEvent +from agno.run.agent import RunOutput, ToolCallCompletedEvent, ToolCallStartedEvent from band.core.types import ( AgentInput, @@ -24,10 +25,69 @@ HistoryProvider, PlatformMessage, ) -from band.runtime.tools import DEFAULT_CAPABILITIES +from band.runtime.tools import DEFAULT_CAPABILITIES, BandTool from band.testing import FakeAgentTools +def fake_agno_agent( + *, + update_memory_on_run: bool = False, + enable_agentic_memory: bool = False, + add_history_to_context: bool = False, + db: object | None = None, + response: RunOutput | None = None, + events: list[Any] | None = None, + model_turn: Callable[[], Awaitable[None]] | None = None, +) -> MagicMock: + """A configured Agno agent fake whose ``arun`` yields ``response``. + + The adapter runs against this instance directly, so it carries the + history/memory config the guards read. ``model_turn`` is the tool work the + model does inside the run (with the room's tools bound), before ``response``. + """ + agent = MagicMock(name="agno_agent") + agent.update_memory_on_run = update_memory_on_run + agent.enable_agentic_memory = enable_agentic_memory + # Explicit falsy defaults: a bare MagicMock would expose these as truthy + # auto-attributes and spuriously trip the history-management guard. + agent.add_history_to_context = add_history_to_context + agent.db = db + agent.add_tool = MagicMock() + # Real Agno agents default additional_context to None; mirror that. + agent.additional_context = None + # The adapter captures the user's tools at startup, then installs a + # callable factory. A bare MagicMock `.tools` is itself callable and would + # be mistaken for a user-supplied tools factory, so pin it to a list. + agent.tools = [] + resp = response if response is not None else RunOutput() + + if events is None: + # Non-streaming path (Emit.TOOL_CALLS off): `await agent.arun(...)`. + async def _arun_once(*args: Any, **kwargs: Any) -> RunOutput: + if model_turn is not None: + await model_turn() + return resp + + agent.arun = AsyncMock(side_effect=_arun_once) + else: + # Streaming path (Emit.TOOL_CALLS on): the adapter iterates + # `agent.arun(stream=True, ...)`, which yields the run events then the + # final RunOutput. A bare MagicMock returns an async iterator without + # awaiting, matching how the adapter consumes the stream. + def _arun(*args: Any, **kwargs: Any) -> Any: + async def _stream() -> Any: + if model_turn is not None: + await model_turn() + for event in events: + yield event + yield resp + + return _stream() + + agent.arun = MagicMock(side_effect=_arun) + return agent + + def tool_execution( name: str, *, @@ -67,9 +127,12 @@ class CapturingModel(Model): abstract invoke hooks are inert stubs. """ - def __init__(self, content: str = "ok") -> None: + def __init__(self, content: str = "ok", *, replies: bool = False) -> None: super().__init__(id="capturing", provider="fake") self._content = content + # Whether the model answers through the offered band_send_message, as a + # tool-only model does, rather than only returning its content. + self._replies = replies self.captured_messages: list[Message] | None = None # Tool names Agno offered the model on the most recent response call, # so tests can assert per-run tool exposure end-to-end. @@ -84,9 +147,13 @@ def _parse_provider_response_delta(self, *args: Any, **kwargs: Any) -> Any: ... async def aresponse(self, messages: list[Message], **kwargs: Any) -> ModelResponse: self.captured_messages = messages - self.captured_tool_names = [ - _tool_schema_name(t) for t in (kwargs.get("tools") or []) - ] + offered = kwargs.get("tools") or [] + self.captured_tool_names = [_tool_schema_name(t) for t in offered] + if self._replies: + (send,) = [ + t for t in offered if _tool_schema_name(t) == BandTool.SEND_MESSAGE + ] + await send.entrypoint(content=self._content, mentions=["@Alice"]) return ModelResponse(content=self._content) @property diff --git a/tests/adapters/agno/test_history_guard.py b/tests/adapters/agno/test_history_guard.py index 39bd0ad66..b97374f37 100644 --- a/tests/adapters/agno/test_history_guard.py +++ b/tests/adapters/agno/test_history_guard.py @@ -57,7 +57,7 @@ async def test_no_guard_unless_both_set( class TestRehydrationDisabled: async def test_bootstrap_run_input_omits_rehydrated_history( - self, make_started_adapter, sample_platform_message + self, make_started_adapter, sample_platform_message, tools ): raw = format_history_for_llm( [ @@ -69,7 +69,10 @@ async def test_bootstrap_run_input_omits_rehydrated_history( exclude_id=sample_platform_message.id, ) adapter, agent = await make_started_adapter( - RunOutput(content="ack"), add_history_to_context=True, db=object() + RunOutput(content="ack"), + add_history_to_context=True, + db=object(), + replies_through=tools, ) await adapter.on_event( @@ -78,6 +81,7 @@ async def test_bootstrap_run_input_omits_rehydrated_history( raw, is_session_bootstrap=True, participants_msg="Alice and Bob are here", + tools=tools, ) ) @@ -89,7 +93,7 @@ async def test_bootstrap_run_input_omits_rehydrated_history( ] async def test_second_turn_does_not_carry_over_band_transcript( - self, make_started_adapter, sample_platform_message + self, make_started_adapter, sample_platform_message, tools ): turn = RunOutput( content="a1", @@ -99,14 +103,18 @@ async def test_second_turn_does_not_carry_over_band_transcript( ], ) adapter, agent = await make_started_adapter( - turn, add_history_to_context=True, db=object() + turn, add_history_to_context=True, db=object(), replies_through=tools ) await adapter.on_event( - make_agent_input(sample_platform_message, [], is_session_bootstrap=True) + make_agent_input( + sample_platform_message, [], is_session_bootstrap=True, tools=tools + ) ) await adapter.on_event( - make_agent_input(sample_platform_message, [], is_session_bootstrap=False) + make_agent_input( + sample_platform_message, [], is_session_bootstrap=False, tools=tools + ) ) # The follow-up turn sends only the current message: Agno supplies prior @@ -117,7 +125,7 @@ async def test_second_turn_does_not_carry_over_band_transcript( class TestStorePreserved: async def test_transcript_is_still_stored_when_guard_on( - self, make_started_adapter, sample_platform_message + self, make_started_adapter, sample_platform_message, tools ): turn = RunOutput( content="a1", @@ -127,12 +135,14 @@ async def test_transcript_is_still_stored_when_guard_on( ], ) adapter, _ = await make_started_adapter( - turn, add_history_to_context=True, db=object() + turn, add_history_to_context=True, db=object(), replies_through=tools ) room_id = sample_platform_message.room_id await adapter.on_event( - make_agent_input(sample_platform_message, [], is_session_bootstrap=True) + make_agent_input( + sample_platform_message, [], is_session_bootstrap=True, tools=tools + ) ) # "Store the history, just don't rehydrate it": _persist_turn still records diff --git a/tests/adapters/agno/test_history_persistence.py b/tests/adapters/agno/test_history_persistence.py index 894ebc885..07879433e 100644 --- a/tests/adapters/agno/test_history_persistence.py +++ b/tests/adapters/agno/test_history_persistence.py @@ -27,7 +27,7 @@ from band.runtime.formatters import format_history_for_llm from tests.adapters.agno.helpers import ( CapturingModel, - SchemaTools, + ContactAwareTools, make_agent_input, platform_msg, ) @@ -66,7 +66,7 @@ def build_agent(reply: str) -> AgnoAgent: # Same db + session_id across instances models a persistent backend that # outlives a single agent process. return AgnoAgent( - model=CapturingModel(reply), + model=CapturingModel(reply, replies=True), db=db, session_id=ROOM_ID, add_history_to_context=True, @@ -84,7 +84,9 @@ def build_agent(reply: str) -> AgnoAgent: # Turn 1 — Band supplies NO history (raw=[]); only the live message is sent. first = _platform_message("m1", "remember the code is 42") await adapter.on_event( - make_agent_input(first, [], is_session_bootstrap=True, tools=SchemaTools([])) + make_agent_input( + first, [], is_session_bootstrap=True, tools=ContactAwareTools() + ) ) assert not any(m.from_history for m in _captured(adapter).captured_messages or []) @@ -102,7 +104,7 @@ def build_agent(reply: str) -> AgnoAgent: ) await adapter2.on_event( make_agent_input( - second, band_raw, is_session_bootstrap=True, tools=SchemaTools([]) + second, band_raw, is_session_bootstrap=True, tools=ContactAwareTools() ) ) diff --git a/tests/adapters/agno/test_rehydration.py b/tests/adapters/agno/test_rehydration.py index 4952c7aa4..bc7735021 100644 --- a/tests/adapters/agno/test_rehydration.py +++ b/tests/adapters/agno/test_rehydration.py @@ -23,7 +23,7 @@ class TestRehydrationPipeline: actual run input Agno received.""" async def test_all_message_kinds_become_the_right_messages( - self, make_started_adapter, sample_platform_message + self, make_started_adapter, sample_platform_message, tools ): # Authentic rehydration: build platform dicts and run them through the # real runtime formatter (which also drops the current message). @@ -50,10 +50,14 @@ async def test_all_message_kinds_become_the_right_messages( ], exclude_id=sample_platform_message.id, ) - adapter, agent = await make_started_adapter(RunOutput(content="ack")) + adapter, agent = await make_started_adapter( + RunOutput(content="ack"), replies_through=tools + ) await adapter.on_event( - make_agent_input(sample_platform_message, raw, is_session_bootstrap=True) + make_agent_input( + sample_platform_message, raw, is_session_bootstrap=True, tools=tools + ) ) msgs = run_input(agent) @@ -71,7 +75,7 @@ async def test_all_message_kinds_become_the_right_messages( assert msgs[-1].content == sample_platform_message.format_for_llm() async def test_unsupported_kinds_are_dropped( - self, make_started_adapter, sample_platform_message + self, make_started_adapter, sample_platform_message, tools ): raw = format_history_for_llm( [ @@ -87,10 +91,14 @@ async def test_unsupported_kinds_are_dropped( ], exclude_id=sample_platform_message.id, ) - adapter, agent = await make_started_adapter(RunOutput(content="ack")) + adapter, agent = await make_started_adapter( + RunOutput(content="ack"), replies_through=tools + ) await adapter.on_event( - make_agent_input(sample_platform_message, raw, is_session_bootstrap=True) + make_agent_input( + sample_platform_message, raw, is_session_bootstrap=True, tools=tools + ) ) msgs = run_input(agent) @@ -101,16 +109,20 @@ async def test_unsupported_kinds_are_dropped( ] async def test_history_is_from_history_but_current_message_is_live( - self, make_started_adapter, sample_platform_message + self, make_started_adapter, sample_platform_message, tools ): raw = format_history_for_llm( [platform_msg("h1", "hi", sender_name="Alice")], exclude_id=sample_platform_message.id, ) - adapter, agent = await make_started_adapter(RunOutput(content="ack")) + adapter, agent = await make_started_adapter( + RunOutput(content="ack"), replies_through=tools + ) await adapter.on_event( - make_agent_input(sample_platform_message, raw, is_session_bootstrap=True) + make_agent_input( + sample_platform_message, raw, is_session_bootstrap=True, tools=tools + ) ) msgs = run_input(agent) @@ -118,9 +130,11 @@ async def test_history_is_from_history_but_current_message_is_live( assert not msgs[-1].from_history # the message to actually answer async def test_participants_and_contacts_injected_before_current_message( - self, make_started_adapter, sample_platform_message + self, make_started_adapter, sample_platform_message, tools ): - adapter, agent = await make_started_adapter(RunOutput(content="ok")) + adapter, agent = await make_started_adapter( + RunOutput(content="ok"), replies_through=tools + ) await adapter.on_event( make_agent_input( @@ -129,6 +143,7 @@ async def test_participants_and_contacts_injected_before_current_message( is_session_bootstrap=True, participants_msg="Alice and Bob are here", contacts_msg="Carol is now a contact", + tools=tools, ) ) @@ -158,7 +173,7 @@ async def test_current_message_excluded_from_history_then_answered( assert all(current.content not in h["content"] for h in raw) adapter, agent = await make_started_adapter( - RunOutput(content="here is your answer") + RunOutput(content="here is your answer"), replies_through=tools ) await adapter.on_event( @@ -184,7 +199,9 @@ async def test_answers_unanswered_message_on_restart_bootstrap( ], exclude_id=sample_platform_message.id, ) - adapter, agent = await make_started_adapter(RunOutput(content="fresh answer")) + adapter, agent = await make_started_adapter( + RunOutput(content="fresh answer"), replies_through=tools + ) await adapter.on_event( make_agent_input( @@ -208,7 +225,9 @@ async def test_trailing_unanswered_user_turns_are_preserved( ], exclude_id=sample_platform_message.id, ) - adapter, agent = await make_started_adapter(RunOutput(content="answering all")) + adapter, agent = await make_started_adapter( + RunOutput(content="answering all"), replies_through=tools + ) await adapter.on_event( make_agent_input( @@ -227,7 +246,7 @@ async def test_trailing_unanswered_user_turns_are_preserved( class TestMultiTurnCarryover: async def test_persisted_transcript_feeds_the_next_turn( - self, make_started_adapter, sample_platform_message + self, make_started_adapter, sample_platform_message, tools ): # Turn 1's run produces a transcript; _persist_turn keeps it and the next # turn must build on top of it (carryover through the real on_message path). @@ -238,13 +257,17 @@ async def test_persisted_transcript_feeds_the_next_turn( Message(role="assistant", content="a1"), ], ) - adapter, agent = await make_started_adapter(turn) + adapter, agent = await make_started_adapter(turn, replies_through=tools) await adapter.on_event( - make_agent_input(sample_platform_message, [], is_session_bootstrap=True) + make_agent_input( + sample_platform_message, [], is_session_bootstrap=True, tools=tools + ) ) await adapter.on_event( - make_agent_input(sample_platform_message, [], is_session_bootstrap=False) + make_agent_input( + sample_platform_message, [], is_session_bootstrap=False, tools=tools + ) ) msgs = run_input(agent) # the second (follow-up) turn's input diff --git a/tests/adapters/claude_sdk/approvals/test_manual_approval.py b/tests/adapters/claude_sdk/approvals/test_manual_approval.py index d1fa4df91..8bf113d98 100644 --- a/tests/adapters/claude_sdk/approvals/test_manual_approval.py +++ b/tests/adapters/claude_sdk/approvals/test_manual_approval.py @@ -13,7 +13,12 @@ APPROVAL_TIMED_OUT_TEMPLATE, APPROVAL_UNAUTHORIZED_MESSAGE, ) -from tests.adapters.claude_sdk.helpers import WRITE_NOTE, ClaudeRoom, with_approvals +from band.testing import MISSING_REPLY_FAILURE, failure_reports +from tests.adapters.claude_sdk.helpers import ( + WRITE_NOTE, + ClaudeRoom, + with_approvals, +) from tests.baseline.decisions import ModelDecision, ToolCall OpenRoom = Callable[..., Awaitable[ClaudeRoom]] @@ -75,30 +80,48 @@ async def test_messages_during_a_parked_turn_settle_on_their_own_tools( the tools it started with, and its prompt is never taken for its reply.""" room = await claude_room(with_approvals()) room.claude.script([LIST_FILES, room.model_reply("Listed.")]) - status, busy, approve = room.fresh_tools(), room.fresh_tools(), room.fresh_tools() + parked, status, busy, approve = (room.fresh_tools() for _ in range(4)) - await room.send("List the files") - prompted_turn_replied = room.tools.turn.replied + await room.send("List the files", tools=parked) + prompted_turn_replied = parked.turn.replied await room.send("/status", tools=status) await room.send("And the hidden ones?", tools=busy) await room.send("/approve", tools=approve) await room.settled() - assert prompted_turn_replied is False - assert room.chat == [ + chat = room.chat + del chat[1] # the status report + assert chat == [ prompt("a-1", "Bash: `ls`"), + "Still processing the previous request in this room.", resolved("a-1", "accept"), "Listed.", ] - assert room.tools.turn.replied - assert [message["content"] for message in busy.messages_sent] == [ - "Still processing the previous request in this room." - ] - assert approve.messages_sent == [] + assert prompted_turn_replied is False + assert parked.turn.replied for side_turn in (status.turn, busy.turn, approve.turn): assert (side_turn.complete, side_turn.replied) == (True, False) +async def test_a_released_turn_that_ends_unanswered_is_reported_at_its_end( + claude_room: OpenRoom, +) -> None: + """A turn released to wait on an approval hands its message back as + handled; when it later ends without answering, the room hears one + missing-reply failure from the runtime.""" + room = await claude_room(with_approvals()) + room.claude.script([LIST_FILES, ModelDecision.text_reply("Listed, in plain text.")]) + parked = room.fresh_tools() + + await room.send("List the files", tools=parked) + reported_at_release = list(room.failures) + await room.send("/approve") + await room.settled() + + assert reported_at_release == [] + assert failure_reports(parked) == [MISSING_REPLY_FAILURE] + + async def test_a_busy_room_evicts_the_oldest_ask_and_asks_for_a_token( claude_room: OpenRoom, ) -> None: diff --git a/tests/adapters/claude_sdk/failures/test_cli_connection_error.py b/tests/adapters/claude_sdk/failures/test_cli_connection_error.py index ac2835fc7..ec3816b03 100644 --- a/tests/adapters/claude_sdk/failures/test_cli_connection_error.py +++ b/tests/adapters/claude_sdk/failures/test_cli_connection_error.py @@ -20,8 +20,9 @@ async def test_a_cli_that_dies_mid_turn_fails_only_an_unanswered_turn( claude_room: OpenRoom, ) -> None: """Dying after the reply went out completes the turn (failing it would - make the runtime redeliver and answer twice); dying before any reply - fails it. Either way the next message gets a fresh CLI process.""" + let a later process run the message again and answer twice); dying + before any reply fails it. Either way the next message gets a fresh CLI + process.""" room = await claude_room() room.claude.script( [room.model_reply("Answered before the crash."), Hangup()], diff --git a/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py b/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py index 4d74f9654..c07e7845f 100644 --- a/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py +++ b/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py @@ -15,7 +15,7 @@ TurnResultAlreadyReported, ) from band.core.types import Emit -from tests.adapters.claude_sdk.fakecli import EndTurn, Raw, Thinking +from tests.adapters.claude_sdk.fakecli import EndTurn, Thinking from tests.adapters.claude_sdk.helpers import ( MISSING_REPLY_TEXT, SEND_MESSAGE_MCP_NAME, @@ -183,69 +183,6 @@ async def test_a_declined_reply_explains_the_silence_only_if_the_room_was_told( assert room.failures == [MISSING_REPLY_TEXT] -async def test_tool_traffic_the_cli_carries_outside_assistant_calls_still_counts( - claude_room: OpenRoom, -) -> None: - """A subagent's nested reply arrives in a user envelope (with - ``is_error`` omitted, which means success), and a result can ride in the - assistant message itself; either one answers the turn.""" - room = await claude_room() - room.claude.script( - [ - Raw( - { - "type": "user", - "parent_tool_use_id": "toolu_task", - "message": { - "content": [ - { - "type": "tool_use", - "id": "toolu_nested", - "name": SEND_MESSAGE_MCP_NAME, - "input": {}, - }, - { - "type": "tool_result", - "tool_use_id": "toolu_nested", - "content": "sent", - }, - ] - }, - } - ) - ], - [ - Raw( - { - "type": "assistant", - "message": { - "model": "claude-fake", - "content": [ - { - "type": "tool_use", - "id": "toolu_inline", - "name": SEND_MESSAGE_MCP_NAME, - "input": {}, - }, - { - "type": "tool_result", - "tool_use_id": "toolu_inline", - "content": "sent", - "is_error": False, - }, - ], - }, - } - ) - ], - ) - - await room.send("delegate it") - await room.send("answer inline") - - assert room.failures == [] - - async def test_band_no_reply_ends_the_turn_quietly(claude_room: OpenRoom) -> None: """Choosing not to answer is a finished turn: nothing is posted and no missing-reply error reaches the room.""" diff --git a/tests/adapters/claude_sdk/helpers.py b/tests/adapters/claude_sdk/helpers.py index 796c415c0..afed5ae0a 100644 --- a/tests/adapters/claude_sdk/helpers.py +++ b/tests/adapters/claude_sdk/helpers.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import copy import itertools import json from datetime import UTC, datetime @@ -13,21 +14,24 @@ ClaudeSDKAdapter, ClaudeSDKAdapterConfig, ) -from band.converters.claude_sdk import ( - SESSION_ID_METADATA_KEY, - ClaudeSDKSessionState, +from band.converters.claude_sdk import SESSION_ID_METADATA_KEY +from band.core.turn import Turn +from band.core.types import ( + AgentInput, + ApprovalMode, + HistoryProvider, + MessageType, + PlatformMessage, ) -from band.core.types import ApprovalMode, MessageType, PlatformMessage -from band.runtime.tools import MCP_TOOL_PREFIX, missing_reply_error -from band.testing import FakeAgentTools +from band.runtime.tools import MCP_TOOL_PREFIX +from band.testing import MISSING_REPLY_FAILURE, FakeAgentTools from tests.adapters.claude_sdk.fakecli import FakeClaude from tests.baseline.decisions import ModelDecision # The reply tool as the SDK namespaces it (MCP_TOOL_PREFIX + bare name). SEND_MESSAGE_MCP_NAME = "mcp__band__band_send_message" -# What a turn that ended without a reply going out must say; tests assert it -# by substring rather than re-deriving it. -MISSING_REPLY_TEXT = missing_reply_error("Claude SDK") +# What the runtime reports for a turn that ended without completing. +MISSING_REPLY_TEXT = MISSING_REPLY_FAILURE[1] # A native file write, the tool call every approval path is asked about. WRITE_NOTE = ModelDecision.call("Write", file_path="notes.md", content="todo") @@ -42,6 +46,24 @@ def with_approvals( APPROVER = {"id": "u1", "name": "Bob", "handle": "@bob"} +def said(sender_name: str, content: str) -> dict[str, Any]: + """One earlier room message, as raw platform history.""" + return { + "message_type": MessageType.TEXT, + "role": "user", + "sender_name": sender_name, + "content": content, + } + + +def recorded_session(session_id: str) -> dict[str, Any]: + """The task event a room persisted for a Claude session, as raw history.""" + return { + "message_type": MessageType.TASK, + "metadata": {SESSION_ID_METADATA_KEY: session_id}, + } + + class ClaudeRoom: """A Claude SDK room driven end to end: people post messages, the scripted CLI answers them through the real SDK client, and the test reads what the @@ -53,18 +75,23 @@ def __init__( self.adapter = adapter self.claude = claude self.room_id = room_id - self.tools = self.fresh_tools() - self._message_ids = itertools.count(1) - self._bootstrapped = False - - def fresh_tools(self) -> FakeAgentTools: - """A new view of this room, like the tools the runtime builds per message.""" - return FakeAgentTools( - room_id=self.room_id, + # The room's shared log: every message's tools post into it. + self.tools = FakeAgentTools( + room_id=room_id, participants=[ {**APPROVER, "role": "member", "status": "active", "type": "User"} ], ) + self._message_ids = itertools.count(1) + self._bootstrapped = False + + def fresh_tools(self) -> FakeAgentTools: + """The tools the runtime builds for one message: their own turn + ledger, posting into the room's shared log (a shallow copy shares its + lists, holds and observers; a fault set on ``self.tools`` is copied).""" + tools = copy.copy(self.tools) + tools.turn = Turn() + return tools @property def chat(self) -> list[str]: @@ -160,34 +187,42 @@ async def send( content: str, *, sender: dict[str, str] = APPROVER, - history: str = "", + history: tuple[dict[str, Any], ...] = (), session_id: str | None = None, tools: FakeAgentTools | None = None, ) -> None: - """Deliver one room message; returns once the adapter hands the turn - back (it finished, or parked on a human). + """Deliver one room message through the runtime's entry point, so the + turn is judged as in production; returns once the adapter hands the + turn back (it finished, or parked on a human). - ``tools`` stands in for the fresh tools the runtime builds per message. + ``history`` is the room's raw history, ``session_id`` a Claude session + it persisted, and ``tools`` the message's own tools (``fresh_tools()`` + by default, as the runtime builds per message). """ bootstrap, self._bootstrapped = not self._bootstrapped, True - await self.adapter.on_message( - PlatformMessage( - id=f"msg-{next(self._message_ids)}", + raw_history = [*history] + if session_id is not None: + raw_history.append(recorded_session(session_id)) + await self.adapter.on_event( + AgentInput( + msg=PlatformMessage( + id=f"msg-{next(self._message_ids)}", + room_id=self.room_id, + content=content, + sender_id=sender["id"], + sender_type="User", + sender_name=sender["name"], + message_type="text", + metadata={}, + created_at=datetime.now(UTC), + ), + tools=tools or self.fresh_tools(), + history=HistoryProvider(raw=raw_history), + participants_msg=None, + contacts_msg=None, + is_session_bootstrap=bootstrap, room_id=self.room_id, - content=content, - sender_id=sender["id"], - sender_type="User", - sender_name=sender["name"], - message_type="text", - metadata={}, - created_at=datetime.now(UTC), - ), - tools or self.tools, - ClaudeSDKSessionState(text=history, session_id=session_id), - None, - None, - is_session_bootstrap=bootstrap, - room_id=self.room_id, + ) ) def send_in_background(self, content: str) -> asyncio.Task[None]: diff --git a/tests/adapters/claude_sdk/lifecycle/test_on_cleanup.py b/tests/adapters/claude_sdk/lifecycle/test_on_cleanup.py index b2246130c..fb2c274a7 100644 --- a/tests/adapters/claude_sdk/lifecycle/test_on_cleanup.py +++ b/tests/adapters/claude_sdk/lifecycle/test_on_cleanup.py @@ -7,7 +7,7 @@ from band.adapters.claude_sdk import ClaudeSDKAdapter from tests.adapters.claude_sdk.fakecli import Hold -from tests.adapters.claude_sdk.helpers import ClaudeRoom +from tests.adapters.claude_sdk.helpers import ClaudeRoom, said OpenRoom = Callable[..., Awaitable[ClaudeRoom]] @@ -56,7 +56,7 @@ async def test_a_room_left_mid_turn_can_be_rejoined_with_a_fresh_session( with pytest.raises(asyncio.CancelledError): await message - await room.send("I'm back", history="[Bob]: long job") + await room.send("I'm back", history=(said("Bob", "long job"),)) assert len(room.claude.sessions) == 2 assert "[Bob]: long job" in room.claude.prompts[-1] diff --git a/tests/adapters/claude_sdk/lifecycle/test_on_message.py b/tests/adapters/claude_sdk/lifecycle/test_on_message.py index 2e51faeaa..23b944f9f 100644 --- a/tests/adapters/claude_sdk/lifecycle/test_on_message.py +++ b/tests/adapters/claude_sdk/lifecycle/test_on_message.py @@ -4,7 +4,7 @@ from collections.abc import Awaitable, Callable from tests.adapters.claude_sdk.fakecli import Hold -from tests.adapters.claude_sdk.helpers import ClaudeRoom +from tests.adapters.claude_sdk.helpers import ClaudeRoom, said OpenRoom = Callable[..., Awaitable[ClaudeRoom]] @@ -19,7 +19,7 @@ async def test_a_room_bootstraps_once_then_keeps_talking_in_the_same_session( room = await claude_room() room.claude.script([room.model_reply("hi Bob")], [room.model_reply("still here")]) - await room.send("Hello, agent!", history="[Alice]: the code word is tulip") + await room.send("Hello, agent!", history=(said("Alice", "the code word is tulip"),)) await room.send("Are you there?") first, second = room.claude.prompts diff --git a/tests/adapters/claude_sdk/test_send_message_dedup.py b/tests/adapters/claude_sdk/test_send_message_dedup.py index d782c92cd..afdf65579 100644 --- a/tests/adapters/claude_sdk/test_send_message_dedup.py +++ b/tests/adapters/claude_sdk/test_send_message_dedup.py @@ -30,10 +30,8 @@ async def test_a_repeated_reply_reaches_its_room_once_even_after_the_turn_ends( await room.send("anything else?", tools=next_turn_tools) await other_room.send("say hello") - assert room.chat == ["hello"] - assert [message["content"] for message in next_turn_tools.messages_sent] == [ - "a new answer" - ] + assert room.chat == ["hello", "a new answer"] + assert next_turn_tools.turn.replied assert other_room.chat == ["hello"] assert room.failures == [] diff --git a/tests/adapters/claude_sdk/tools/test_custom_tools.py b/tests/adapters/claude_sdk/tools/test_custom_tools.py index b047e66fa..59423431d 100644 --- a/tests/adapters/claude_sdk/tools/test_custom_tools.py +++ b/tests/adapters/claude_sdk/tools/test_custom_tools.py @@ -5,10 +5,15 @@ import pytest from pydantic import BaseModel, Field +from band.core.protocols import TurnResultAlreadyReported from band.core.types import Capability from band.runtime.custom_tools import declares_turn_effect from band.runtime.tools import TurnEffect -from tests.adapters.claude_sdk.helpers import ClaudeRoom, with_approvals +from tests.adapters.claude_sdk.helpers import ( + MISSING_REPLY_TEXT, + ClaudeRoom, + with_approvals, +) OpenRoom = Callable[..., Awaitable[ClaudeRoom]] @@ -91,14 +96,20 @@ async def test_memory_tools_are_served_only_with_the_memory_capability( async def test_a_custom_tool_records_its_declared_effect_on_the_rooms_turn( claude_room: OpenRoom, ) -> None: - """A custom tool declared as real work completes the room's turn without - a reply, so nothing is reported missing.""" + """A custom tool declared as real work completes the turn without a + reply; an undeclared one only observes, so its turn is reported missing.""" room = await claude_room( additional_tools=[*CUSTOM_TOOLS, (FileTicketInput, file_ticket)] ) - room.claude.script([room.model_call("fileticket", title="Broken build")]) + room.claude.script( + [room.model_call("fileticket", title="Broken build")], + [room.model_call("echo", message="ping")], + ) + ticket, echo_only = room.fresh_tools(), room.fresh_tools() - await room.send("file a ticket for the broken build") + await room.send("file a ticket for the broken build", tools=ticket) + with pytest.raises(TurnResultAlreadyReported): + await room.send("echo ping", tools=echo_only) - assert (room.tools.turn.complete, room.tools.turn.replied) == (True, False) - assert room.failures == [] + assert (ticket.turn.complete, ticket.turn.replied) == (True, False) + assert room.failures == [MISSING_REPLY_TEXT] diff --git a/tests/adapters/codexturns.py b/tests/adapters/codexturns.py index b03dfaccf..70a628018 100644 --- a/tests/adapters/codexturns.py +++ b/tests/adapters/codexturns.py @@ -1,5 +1,5 @@ -"""Codex test helpers shared by ``tests/adapters/test_codex_adapter.py`` and -``tests/integrations/codex/test_adapter_e2e.py``. +"""Codex test helpers -- a scripted app-server and its event builders -- +shared by the codex adapter tests and the turn-outcome conformance probe. Once a manual approval releases the room, ``on_message``/``on_event`` returns early and the turn keeps running in ``adapter._turn_tasks``, out of reach of @@ -10,10 +10,11 @@ from __future__ import annotations import asyncio +from collections import deque from typing import Any -from band.adapters.codex import CodexAdapter -from band.integrations.codex import CodexRequestMethod +from band.adapters.codex import CodexAdapter, CodexAdapterConfig +from band.integrations.codex import CodexRequestMethod, RpcEvent class RecordedRequests: @@ -44,3 +45,173 @@ async def await_released_turn( await turn else: await asyncio.wait_for(turn, timeout=timeout_s) + + +class FakeCodexClient(RecordedRequests): + """Minimal fake transport client for adapter tests.""" + + def __init__( + self, + *, + events: list[RpcEvent] | None = None, + resume_error: Exception | None = None, + turn_start_error: Exception | None = None, + turn_start_error_once: bool = True, + model_list_result: dict[str, Any] | None = None, + model_list_error: Exception | None = None, + skill_roots_error: Exception | None = None, + ) -> None: + self.connected = False + self.initialized = False + self.requests: list[tuple[str, dict[str, Any]]] = [] + self.responses: list[tuple[int | str, dict[str, Any]]] = [] + self.response_errors: list[tuple[int | str, int, str]] = [] + self.closed = False + self._events = deque(events or []) + self._resume_error = resume_error + self._turn_start_error = turn_start_error + self._turn_start_error_once = turn_start_error_once + self._model_list_result = model_list_result + self._model_list_error = model_list_error + self._skill_roots_error = skill_roots_error + self._thread_counter = 0 + self._turn_counter = 0 + + async def connect(self) -> None: + self.connected = True + + async def initialize( + self, + *, + client_name: str, + client_title: str, + client_version: str, + experimental_api: bool = False, + opt_out_notification_methods: list[str] | None = None, + ) -> dict[str, Any]: + self.initialized = True + return {"userAgent": f"{client_name}/{client_version}"} + + async def request( + self, + method: str, + params: dict[str, Any] | None = None, + *, + retry_on_overload: bool = True, + ) -> dict[str, Any]: + payload = params or {} + self.requests.append((method, dict(payload))) + + match method: + case CodexRequestMethod.MODEL_LIST: + if self._model_list_error is not None: + raise self._model_list_error + if self._model_list_result is not None: + return self._model_list_result + return {"data": [{"id": "gpt-5.5", "hidden": False}]} + case CodexRequestMethod.SKILLS_EXTRA_ROOTS_SET: + if self._skill_roots_error is not None: + raise self._skill_roots_error + case CodexRequestMethod.THREAD_RESUME: + if self._resume_error is not None: + raise self._resume_error + return {"thread": {"id": payload.get("threadId", "thr-resumed")}} + case CodexRequestMethod.THREAD_START: + self._thread_counter += 1 + return {"thread": {"id": f"thr-{self._thread_counter}"}} + case CodexRequestMethod.TURN_START: + if self._turn_start_error is not None: + err = self._turn_start_error + if self._turn_start_error_once: + self._turn_start_error = None + raise err + self._turn_counter += 1 + return { + "turn": { + "id": f"turn-{self._turn_counter}", + "status": "inProgress", + "items": [], + "error": None, + } + } + + return {} + + async def recv_event(self, timeout_s: float | None = None) -> RpcEvent: + if not self._events: + raise TimeoutError + return self._events.popleft() + + async def respond(self, request_id: int | str, result: dict[str, Any]) -> None: + self.responses.append((request_id, result)) + + async def respond_error( + self, + request_id: int | str, + *, + code: int, + message: str, + data: Any | None = None, + ) -> None: + self.response_errors.append((request_id, code, message)) + + async def close(self) -> None: + self.closed = True + + +def patch_codex_client(adapter: CodexAdapter, client: FakeCodexClient) -> None: + def _build(_config: CodexAdapterConfig) -> FakeCodexClient: + return client + + adapter._build_client = _build # type: ignore[method-assign] + + +def make_codex_adapter( + client: FakeCodexClient, + config: CodexAdapterConfig | None = None, + **kwargs: Any, +) -> CodexAdapter: + adapter = CodexAdapter(config=config or CodexAdapterConfig(), **kwargs) + patch_codex_client(adapter, client) + return adapter + + +def event_notification(method: str, params: dict[str, Any]) -> RpcEvent: + return RpcEvent( + kind="notification", + method=method, + params=params, + id=None, + raw={"method": method, "params": params}, + ) + + +def event_request(request_id: int, method: str, params: dict[str, Any]) -> RpcEvent: + return RpcEvent( + kind="request", + method=method, + params=params, + id=request_id, + raw={"id": request_id, "method": method, "params": params}, + ) + + +def turn_completed(turn_id: str = "turn-1") -> RpcEvent: + """The notification that ends a scripted turn.""" + return event_notification( + "turn/completed", + {"turn": {"id": turn_id, "status": "completed", "items": [], "error": None}}, + ) + + +def final_text(text: str) -> RpcEvent: + return event_notification("item/agentMessage/delta", {"itemId": "m", "delta": text}) + + +def tool_call_request( + request_id: int, tool: str, arguments: dict[str, Any] | None = None +) -> RpcEvent: + """The server request Codex sends to invoke one Band tool.""" + return event_request( + request_id, "item/tool/call", {"tool": tool, "arguments": arguments or {}} + ) diff --git a/tests/adapters/copilot_sdk/fakes.py b/tests/adapters/copilot_sdk/fakes.py index 3a4fbcb1f..96f84ec20 100644 --- a/tests/adapters/copilot_sdk/fakes.py +++ b/tests/adapters/copilot_sdk/fakes.py @@ -23,6 +23,7 @@ from band.converters.copilot_sdk import CopilotSDKSessionState from band.core.types import PlatformMessage from band.testing import FakeAgentTools +from tests.framework_conformance.turnprobes import turn_input requires_copilot_sdk = pytest.mark.skipif( not _COPILOT_SDK_AVAILABLE, @@ -226,3 +227,16 @@ async def run_message( room_id=room_id, ) return msg + + +async def run_event( + adapter: CopilotSDKAdapter, + tools: FakeAgentTools, + *, + room_id: str = "room-1", + content: str = "hello", +) -> None: + """Run one turn through ``on_event``, where the runtime judges it.""" + await adapter.on_event( + turn_input(tools, msg=make_platform_message(room_id=room_id, content=content)) + ) diff --git a/tests/adapters/copilot_sdk/test_reply.py b/tests/adapters/copilot_sdk/test_reply.py index 929aa9b75..5eeb5c3b7 100644 --- a/tests/adapters/copilot_sdk/test_reply.py +++ b/tests/adapters/copilot_sdk/test_reply.py @@ -4,19 +4,24 @@ import logging from collections.abc import Callable +from typing import Any import pytest from band.adapters.copilot_sdk import _COPILOT_SDK_AVAILABLE -from band.core.protocols import GENERIC_PROVIDER_FAILURE_MESSAGE -from band.runtime.tools import CHAT_ID_FIELD_NAME, ToolCallOutcome -from band.testing import reported_failures +from band.core.protocols import ( + GENERIC_PROVIDER_FAILURE_MESSAGE, + TurnResultAlreadyReported, +) +from band.runtime.tools import CHAT_ID_FIELD_NAME, BandTool, ToolCallOutcome +from band.testing import MISSING_REPLY_FAILURE, failure_reports, reported_failures from tests.adapters.copilot_sdk.fakes import ( FakeCopilotClient, FakeCopilotSession, ToolSchemaFakeTools, make_started_adapter, requires_copilot_sdk, + run_event, run_message, ) @@ -38,6 +43,23 @@ ) +class CreateChatroomFakeTools(ToolSchemaFakeTools): + """Also bridges band_create_chatroom, a real-work (ACT) tool.""" + + def get_openai_tool_schemas(self, **kwargs: Any) -> list[dict[str, Any]]: + return [ + *super().get_openai_tool_schemas(**kwargs), + { + "type": "function", + "function": { + "name": BandTool.CREATE_CHATROOM, + "description": "Create a chat room", + "parameters": {"type": "object", "properties": {}}, + }, + }, + ] + + class TestReply: @pytest.mark.asyncio async def test_reply_sent_with_sender_mention(self): @@ -82,17 +104,36 @@ async def test_prompt_contains_room_context_and_message(self): assert "[Alice]: What's up?" in prompt @pytest.mark.asyncio - async def test_no_reply_raises_and_reports_error(self): + async def test_silent_turn_is_reported_by_the_shared_verdict(self): client = FakeCopilotClient(reply_content=None) adapter = await make_started_adapter(client) tools = ToolSchemaFakeTools() - with pytest.raises(RuntimeError, match="no reply"): - await run_message(adapter, tools) + with pytest.raises(TurnResultAlreadyReported): + await run_event(adapter, tools) + + assert not tools.messages_sent + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + + @pytest.mark.asyncio + async def test_act_only_turn_completes_without_a_reply(self): + async def model_creates_room(session: FakeCopilotSession) -> None: + await session.find_tool(BandTool.CREATE_CHATROOM).handler( + ToolInvocation( + tool_call_id="call-1", + tool_name=BandTool.CREATE_CHATROOM, + arguments={}, + ) + ) + + client = FakeCopilotClient(reply_content=None, turn_events=[model_creates_room]) + adapter = await make_started_adapter(client) + tools = CreateChatroomFakeTools() + + await run_event(adapter, tools) assert not tools.messages_sent - error_events = [e for e in tools.events_sent if e["message_type"] == "error"] - assert error_events + assert reported_failures(tools) == [] @pytest.mark.asyncio @pytest.mark.parametrize( @@ -156,9 +197,9 @@ async def test_no_reply_logs_the_trouble_the_turn_hit( tools = ToolSchemaFakeTools() with ( caplog.at_level(logging.WARNING), - pytest.raises(RuntimeError, match="^Copilot turn produced no reply$"), + pytest.raises(TurnResultAlreadyReported), ): - await run_message(adapter, tools) + await run_event(adapter, tools) warnings = [ record @@ -167,9 +208,8 @@ async def test_no_reply_logs_the_trouble_the_turn_hit( and f"(incidents: {account})" in record.getMessage() ] assert len(warnings) == 1 - failures = reported_failures(tools) - assert failures and failures[0]["message"] == "no assistant reply" - assert account not in failures[0]["message"] + # Session text stays in the log, out of the room-visible report. + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] @pytest.mark.asyncio async def test_session_error_raises_reports_and_evicts(self): diff --git a/tests/adapters/copilot_sdk/test_turn_failure.py b/tests/adapters/copilot_sdk/test_turn_failure.py index dd452b560..3fe6d1165 100644 --- a/tests/adapters/copilot_sdk/test_turn_failure.py +++ b/tests/adapters/copilot_sdk/test_turn_failure.py @@ -4,12 +4,14 @@ import pytest -from band.testing import reported_failures +from band.core.protocols import TurnResultAlreadyReported +from band.testing import MISSING_REPLY_FAILURE, failure_reports, reported_failures from tests.adapters.copilot_sdk.fakes import ( FakeCopilotClient, ToolSchemaFakeTools, make_started_adapter, requires_copilot_sdk, + run_event, run_message, ) @@ -62,16 +64,18 @@ async def test_crashed_runtime_revived_on_next_message(self): assert client.sessions[-1].prompts # and the turn ran @pytest.mark.asyncio - async def test_empty_final_text_raises_no_reply(self): - """An empty final assistant message is a failed turn, not a silent no-op.""" + async def test_blank_final_text_is_a_missing_reply(self): + """A blank final assistant message relays nothing, so the shared + verdict reports the turn instead of letting it pass silently.""" client = FakeCopilotClient(reply_content=" ") adapter = await make_started_adapter(client) tools = ToolSchemaFakeTools() - with pytest.raises(RuntimeError, match="no reply"): - await run_message(adapter, tools) + with pytest.raises(TurnResultAlreadyReported): + await run_event(adapter, tools) assert not tools.messages_sent + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] @pytest.mark.asyncio async def test_session_creation_failure_is_reported(self): diff --git a/tests/adapters/lettakit.py b/tests/adapters/lettakit.py index 453ab13b2..5322176b0 100644 --- a/tests/adapters/lettakit.py +++ b/tests/adapters/lettakit.py @@ -8,15 +8,21 @@ from __future__ import annotations +import json +from collections.abc import Awaitable, Callable, Sequence from datetime import UTC, datetime from typing import Any from unittest.mock import AsyncMock, MagicMock from uuid import uuid4 from pytest_httpx import HTTPXMock +from typing_extensions import Unpack -from band.core.types import PlatformMessage +from band.adapters.letta import LettaAdapter, LettaAdapterConfig, RoomContext +from band.core.types import FeatureKwargs, PlatformMessage from band.integrations.letta.prompts import render_tool_enforcement +from band.integrations.mcp.engine import build_resolved_band_mcp_tool_registrations +from band.runtime.tools import CHAT_ID_FIELD_NAME def make_platform_message( @@ -76,6 +82,59 @@ def make_letta_response(*messages: MagicMock) -> MagicMock: return resp +def ready_letta_adapter( + config: LettaAdapterConfig | None = None, + *, + room_id: str = "room-1", + **features: Unpack[FeatureKwargs], +) -> tuple[LettaAdapter, AsyncMock]: + """An adapter past startup: client set, MCP path registered, and a Letta + agent already bound to ``room_id``. Returns it with its mock client.""" + adapter = LettaAdapter(config, **features) + client = AsyncMock() + adapter._client = client + adapter._system_prompt = "Test" + adapter._mcp.server_id = "mcp-server-1" + adapter._rooms[room_id] = RoomContext(agent_id="agent-1") + return adapter, client + + +def scripted_letta_turn( + adapter: LettaAdapter, + *, + room_id: str, + tool_calls: Sequence[tuple[str, dict[str, Any]]] = (), + final_text: str = "", +) -> Callable[..., Awaitable[MagicMock]]: + """A self-hosted Letta agent's turn, as a ``messages.create`` side effect. + + Each tool call runs through the adapter's real Band MCP registrations (the + path Letta's MCP client reaches), then the turn's messages come back as + Letta's response. + """ + registrations = { + registration.name: registration + for registration in build_resolved_band_mcp_tool_registrations( + get_tools=adapter._get_room_tools, + capabilities=adapter.features.capabilities, + ) + } + + async def create(**_kwargs: Any) -> MagicMock: + messages: list[MagicMock] = [] + for name, arguments in tool_calls: + messages.append(make_tool_call_message(name, json.dumps(arguments))) + result = await registrations[name].execute( + {**arguments, CHAT_ID_FIELD_NAME: room_id} + ) + messages.append(make_tool_return_message(name, result)) + if final_text: + messages.append(make_assistant_message(final_text)) + return make_letta_response(*messages) + + return create + + def make_mock_mcp_server(server_id: str = "mcp-server-1") -> MagicMock: """Create a mock MCP server response.""" server = MagicMock() diff --git a/tests/adapters/opencode/helpers.py b/tests/adapters/opencode/helpers.py index 399518f46..ee5f13b36 100644 --- a/tests/adapters/opencode/helpers.py +++ b/tests/adapters/opencode/helpers.py @@ -16,9 +16,7 @@ from band.adapters.opencode.adapter import MCP_REGISTRATION_CONNECTED_STATUS from band.core.exceptions import BandToolError from band.core.protocols import AgentToolsProtocol -from band.core.types import ( - PlatformMessage, -) +from band.core.types import AgentInput, HistoryProvider, PlatformMessage from band.integrations.mcp.engine import EmbeddedResolver from band.integrations.opencode import ( ApprovalReply, @@ -26,7 +24,7 @@ OpencodeQuestionRequest, ) from band.integrations.opencode.types import OpencodeSessionState -from band.runtime.tools import TOOL_DEFINITIONS, BandTool +from band.runtime.tools import TOOL_DEFINITIONS from band.testing import FakeAgentTools RawOpencodeEvent: TypeAlias = dict[str, Any] @@ -227,6 +225,22 @@ def tools_protocol(tools: FakeAgentTools) -> AgentToolsProtocol: return cast(AgentToolsProtocol, tools) +def agent_input(content: str, tools: FakeAgentTools) -> AgentInput: + """What the runtime hands ``on_event`` for one room message.""" + return AgentInput( + msg=make_platform_message(content=content), + tools=tools_protocol(tools), + history=HistoryProvider(raw=[]), + participants_msg=None, + contacts_msg=None, + is_session_bootstrap=False, + room_id="room-1", + ) + + +#: The runtime's report of a turn that neither replied, declined nor worked. + + class RaisingSendTools(FakeAgentTools): """FakeAgentTools whose send_notice always fails, to exercise the best-effort ``_notify_room`` path: a room post that raises must be @@ -440,7 +454,7 @@ async def stop(self) -> None: await self._stop_release.wait() def band_tool_call( - self, tool: BandTool, arguments: dict[str, Any], *, room_id: str = "room-1" + self, tool: str, arguments: dict[str, Any], *, room_id: str = "room-1" ) -> ServerStep: """The model calling a band tool over MCP, dispatched to the room's tools exactly as the embedded MCP server does.""" diff --git a/tests/adapters/opencode/test_detached_turns.py b/tests/adapters/opencode/test_detached_turns.py new file mode 100644 index 000000000..5f6f0e850 --- /dev/null +++ b/tests/adapters/opencode/test_detached_turns.py @@ -0,0 +1,102 @@ +"""A turn released early to wait on a human is judged at its real end. + +``on_event`` returns as soon as a permission ask parks the turn, so the +delivery is PROCESSED then; the turn's own outcome is judged when OpenCode +finishes it. +""" + +from __future__ import annotations + +from band.adapters.opencode import OpencodeAdapter +from band.runtime.tools import BandTool +from band.testing import MISSING_REPLY_FAILURE, FakeAgentTools, failure_reports +from tests.adapters.opencode.helpers import ( + FakeMCPBackend, + FakeOpencodeClient, + RawOpencodeEvent, + ServerStep, + agent_input, + event_permission, + event_session_idle, + wait_for, +) + +PERMISSION_ID = "req-1" + + +def parked_turn_client(*, after_approval: list[ServerStep]) -> FakeOpencodeClient: + """OpenCode asks for a permission, then runs ``after_approval``.""" + parked: list[RawOpencodeEvent] = [event_permission("sess-1", PERMISSION_ID)] + return FakeOpencodeClient( + prompt_event_sequences=[parked], + reply_permission_events={PERMISSION_ID: after_approval}, + ) + + +async def park_turn(adapter: OpencodeAdapter, tools: FakeAgentTools) -> None: + """Run a turn's ``on_event`` until the permission ask releases it.""" + await adapter.on_started("OpenCode Agent", "A coding agent") + await adapter.on_event(agent_input("run it", tools)) + + +async def approve(adapter: OpencodeAdapter) -> FakeAgentTools: + approver_tools = FakeAgentTools() + await adapter.on_event(agent_input(f"approve {PERMISSION_ID}", approver_tools)) + return approver_tools + + +async def test_a_detached_turn_that_ends_with_nothing_is_reported_once( + make_adapter, tools +) -> None: + adapter = make_adapter( + parked_turn_client(after_approval=[event_session_idle("sess-1")]) + ) + + await park_turn(adapter, tools) + assert failure_reports(tools) == [] + + await approve(adapter) + await wait_for(lambda: bool(failure_reports(tools))) + await adapter.on_cleanup("room-1") + + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + + +async def test_a_busy_message_during_a_detached_turn_is_settled( + make_adapter, tools, mcp_backend: FakeMCPBackend +) -> None: + """The busy message's own turn is settled, and the detached turn's band + tool calls still resolve to the detached turn's tools.""" + adapter = make_adapter( + parked_turn_client( + after_approval=[ + mcp_backend.band_tool_call( + BandTool.SEND_MESSAGE, {"content": "Done.", "mentions": ["@alice"]} + ), + event_session_idle("sess-1"), + ] + ) + ) + busy_tools = FakeAgentTools() + + await park_turn(adapter, tools) + await adapter.on_event(agent_input("are you done?", busy_tools)) + await approve(adapter) + await wait_for(lambda: tools.turn.replied) + await adapter.on_cleanup("room-1") + + assert busy_tools.turn.complete + assert busy_tools.messages_sent == [] + assert tools.messages_sent[-1]["content"] == "Done." + assert failure_reports(tools) == [] + + +async def test_a_detached_turn_cancelled_by_cleanup_posts_nothing( + make_adapter, tools +) -> None: + adapter = make_adapter(parked_turn_client(after_approval=[])) + + await park_turn(adapter, tools) + await adapter.on_cleanup("room-1") + + assert failure_reports(tools) == [] diff --git a/tests/adapters/opencode/test_turns.py b/tests/adapters/opencode/test_turns.py index 6a1761b9c..234ee0a78 100644 --- a/tests/adapters/opencode/test_turns.py +++ b/tests/adapters/opencode/test_turns.py @@ -13,7 +13,7 @@ APPROVAL_HANDLED_TEMPLATE, APPROVAL_REQUESTED_TEMPLATE, ) -from band.core.protocols import FAILURE_CODE_TIMEOUT +from band.core.protocols import FAILURE_CODE_TIMEOUT, TurnResultAlreadyReported from band.core.types import ( Capability, Emit, @@ -21,12 +21,19 @@ from band.integrations.opencode import ApprovalReply, parse_opencode_event from band.integrations.opencode.types import OpencodeSessionState from band.runtime.tools import BandTool -from band.testing import FakeAgentTools, events_of_type, reported_failures +from band.testing import ( + MISSING_REPLY_FAILURE, + FakeAgentTools, + events_of_type, + failure_reports, + reported_failures, +) from tests.adapters.opencode.helpers import ( AnyHTTPStatusError, FakeMCPBackend, FakeOpencodeClient, TaskEventFailingTools, + agent_input, event_message_updated, event_message_updated_with_tokens, event_part_delta, @@ -363,26 +370,36 @@ async def test_does_not_echo_user_text_parts_as_assistant_output( ] ] ) - adapter = OpencodeAdapter( - config=OpencodeAdapterConfig(provider_id="openai", model_id="gpt-5.5"), - client_factory=lambda _config: fake_client, - ) - tools = FakeAgentTools() + adapter = make_adapter(fake_client) await adapter.on_started("OpenCode Agent", "A coding agent") - await adapter.on_message( - make_platform_message(content="user prompt text"), - tools_protocol(tools), - OpencodeSessionState(), - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-1", - ) + with pytest.raises(TurnResultAlreadyReported): + await adapter.on_event(agent_input("user prompt text", tools)) + + assert tools.messages_sent == [] + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + - assert tools.messages_sent[0]["content"] == ( - "OpenCode completed the turn without a text reply." +async def test_a_reply_that_cannot_be_posted_is_reported_once( + make_adapter, tools +) -> None: + """The delivery failure is the turn's report; the verdict adds no second.""" + tools.send_message_error = RuntimeError("no one to mention") + fake_client = FakeOpencodeClient( + prompt_event_sequences=[ + [ + event_message_updated("sess-1", "msg-assistant"), + event_text_part("sess-1", "msg-assistant", "pong"), + event_session_idle("sess-1"), + ] + ] ) + adapter = make_adapter(fake_client) + + await adapter.on_started("OpenCode Agent", "A coding agent") + await adapter.on_event(agent_input("ping", tools)) + + assert [provider for provider, _ in failure_reports(tools)] == ["opencode"] async def test_ignores_reasoning_deltas_and_relays_final_text_only( diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index 5adfd1ab5..d920d2954 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -5,7 +5,7 @@ import asyncio import json import logging -from collections import OrderedDict, deque +from collections import OrderedDict from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager from dataclasses import dataclass @@ -50,8 +50,23 @@ from band.runtime.custom_tools import CustomToolDef, declares_turn_effect from band.runtime.decisions import DecisionRegistry from band.runtime.tools import BandTool, ToolCallOutcome, TurnEffect -from band.testing import FakeAgentTools, events_of_type, reported_failures -from tests.adapters.codexturns import RecordedRequests, await_released_turn +from band.testing import ( + MISSING_REPLY_FAILURE, + FakeAgentTools, + events_of_type, + failure_reports, + reported_failures, +) +from tests.adapters.codexturns import ( + FakeCodexClient, + await_released_turn, + event_notification, + event_request, + final_text, + make_codex_adapter, + tool_call_request, + turn_completed, +) from tests.paths import host_absolute_path @@ -127,125 +142,6 @@ def get_openai_tool_schemas(self, **kwargs: Any) -> list[dict[str, Any]]: } -class FakeCodexClient(RecordedRequests): - """Minimal fake transport client for adapter tests.""" - - def __init__( - self, - *, - events: list[RpcEvent] | None = None, - resume_error: Exception | None = None, - turn_start_error: Exception | None = None, - turn_start_error_once: bool = True, - model_list_result: dict[str, Any] | None = None, - model_list_error: Exception | None = None, - skill_roots_error: Exception | None = None, - ) -> None: - self.connected = False - self.initialized = False - self.requests: list[tuple[str, dict[str, Any]]] = [] - self.responses: list[tuple[int | str, dict[str, Any]]] = [] - self.response_errors: list[tuple[int | str, int, str]] = [] - self.closed = False - self._events = deque(events or []) - self._resume_error = resume_error - self._turn_start_error = turn_start_error - self._turn_start_error_once = turn_start_error_once - self._model_list_result = model_list_result - self._model_list_error = model_list_error - self._skill_roots_error = skill_roots_error - self._thread_counter = 0 - self._turn_counter = 0 - - async def connect(self) -> None: - self.connected = True - - async def initialize( - self, - *, - client_name: str, - client_title: str, - client_version: str, - experimental_api: bool = False, - opt_out_notification_methods: list[str] | None = None, - ) -> dict[str, Any]: - self.initialized = True - return {"userAgent": f"{client_name}/{client_version}"} - - async def request( - self, - method: str, - params: dict[str, Any] | None = None, - *, - retry_on_overload: bool = True, - ) -> dict[str, Any]: - payload = params or {} - self.requests.append((method, dict(payload))) - - match method: - case CodexRequestMethod.MODEL_LIST: - if self._model_list_error is not None: - raise self._model_list_error - if self._model_list_result is not None: - return self._model_list_result - return {"data": [{"id": "gpt-5.5", "hidden": False}]} - case CodexRequestMethod.SKILLS_EXTRA_ROOTS_SET: - if self._skill_roots_error is not None: - raise self._skill_roots_error - case CodexRequestMethod.THREAD_RESUME: - if self._resume_error is not None: - raise self._resume_error - return {"thread": {"id": payload.get("threadId", "thr-resumed")}} - case CodexRequestMethod.THREAD_START: - self._thread_counter += 1 - return {"thread": {"id": f"thr-{self._thread_counter}"}} - case CodexRequestMethod.TURN_START: - if self._turn_start_error is not None: - err = self._turn_start_error - if self._turn_start_error_once: - self._turn_start_error = None - raise err - self._turn_counter += 1 - return { - "turn": { - "id": f"turn-{self._turn_counter}", - "status": "inProgress", - "items": [], - "error": None, - } - } - - return {} - - async def recv_event(self, timeout_s: float | None = None) -> RpcEvent: - if not self._events: - raise TimeoutError - return self._events.popleft() - - async def respond(self, request_id: int | str, result: dict[str, Any]) -> None: - self.responses.append((request_id, result)) - - async def respond_error( - self, - request_id: int | str, - *, - code: int, - message: str, - data: Any | None = None, - ) -> None: - self.response_errors.append((request_id, code, message)) - - async def close(self) -> None: - self.closed = True - - -def patch_codex_client(adapter: CodexAdapter, client: FakeCodexClient) -> None: - def _build(_config: CodexAdapterConfig) -> FakeCodexClient: - return client - - adapter._build_client = _build # type: ignore[method-assign] - - def patch_codex_clients_by_room( adapter: CodexAdapter, clients: dict[str, FakeCodexClient] ) -> None: @@ -259,16 +155,6 @@ def _build(_config: CodexAdapterConfig) -> FakeCodexClient: adapter._build_client = _build # type: ignore[method-assign] -def make_codex_adapter( - client: FakeCodexClient, - config: CodexAdapterConfig | None = None, - **kwargs: Any, -) -> CodexAdapter: - adapter = CodexAdapter(config=config or CodexAdapterConfig(), **kwargs) - patch_codex_client(adapter, client) - return adapter - - def wire_codex_room( adapter: CodexAdapter, client: FakeCodexClient, @@ -283,49 +169,6 @@ def wire_codex_room( adapter._initialized = True -def _event_notification(method: str, params: dict[str, Any]) -> RpcEvent: - return RpcEvent( - kind="notification", - method=method, - params=params, - id=None, - raw={"method": method, "params": params}, - ) - - -def _event_request(request_id: int, method: str, params: dict[str, Any]) -> RpcEvent: - return RpcEvent( - kind="request", - method=method, - params=params, - id=request_id, - raw={"id": request_id, "method": method, "params": params}, - ) - - -def _turn_completed(turn_id: str = "turn-1") -> RpcEvent: - """The notification that ends a scripted turn.""" - return _event_notification( - "turn/completed", - {"turn": {"id": turn_id, "status": "completed", "items": [], "error": None}}, - ) - - -def _final_text(text: str) -> RpcEvent: - return _event_notification( - "item/agentMessage/delta", {"itemId": "m", "delta": text} - ) - - -def _tool_call_request( - request_id: int, tool: str, arguments: dict[str, Any] | None = None -) -> RpcEvent: - """The server request Codex sends to invoke one Band tool.""" - return _event_request( - request_id, "item/tool/call", {"tool": tool, "arguments": arguments or {}} - ) - - @dataclass(frozen=True) class CodexTurn: """What one driven turn left behind, as the projections tests assert on.""" @@ -417,14 +260,16 @@ def turn(self) -> asyncio.Task[None]: async def send(self, content: str) -> None: self.deliveries.append(tools := ToolSchemaFakeTools()) - await self.adapter.on_message( - make_platform_message(room_id=ROOM_ID, content=content), - tools, - CodexSessionState(), - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=len(self.deliveries) == 1, - room_id=ROOM_ID, + await self.adapter.on_event( + AgentInput( + msg=make_platform_message(room_id=ROOM_ID, content=content), + tools=tools, + history=HistoryProvider(raw=[]), + participants_msg=None, + contacts_msg=None, + is_session_bootstrap=len(self.deliveries) == 1, + room_id=ROOM_ID, + ) ) async def settled(self) -> None: @@ -467,11 +312,11 @@ def test_config_defaults_are_low_noise_and_manual_approval( @pytest.mark.asyncio async def test_bootstrap_starts_thread_and_sends_fallback_message(self) -> None: events = [ - _event_notification( + event_notification( "item/agentMessage/delta", {"itemId": "msg-1", "delta": "harness-ok"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -503,7 +348,7 @@ async def test_bootstrap_starts_thread_and_sends_fallback_message(self) -> None: async def test_system_prompt_retry_after_turn_start_failure(self) -> None: """System instructions stay pending until turn/start succeeds.""" events = [ - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient( events=events, @@ -557,8 +402,8 @@ async def test_system_prompt_retry_after_turn_start_failure(self) -> None: @pytest.mark.asyncio async def test_tool_call_request_is_dispatched_and_responded(self) -> None: events = [ - _tool_call_request(42, "band_lookup_peers", {"page": 1, "page_size": 10}), - _turn_completed(), + tool_call_request(42, "band_lookup_peers", {"page": 1, "page_size": 10}), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -606,7 +451,7 @@ async def execute_tool_call_structured( return await super().execute_tool_call_structured(tool_name, arguments) events = [ - _event_request( + event_request( 77, "item/tool/call", { @@ -615,7 +460,7 @@ async def execute_tool_call_structured( "callId": "call-77", }, ), - _event_notification( + event_notification( "item/completed", { "item": { @@ -625,7 +470,7 @@ async def execute_tool_call_structured( } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -650,7 +495,7 @@ async def execute_tool_call_structured( @pytest.mark.asyncio async def test_resume_failure_falls_back_to_thread_start(self) -> None: - events = [_turn_completed()] + events = [turn_completed()] fake_client = FakeCodexClient( events=events, resume_error=CodexJsonRpcError(code=-32002, message="Not found"), @@ -676,12 +521,12 @@ async def test_resume_failure_falls_back_to_thread_start(self) -> None: @pytest.mark.asyncio async def test_approval_request_auto_decline(self) -> None: events = [ - _event_request( + event_request( 7, "item/commandExecution/requestApproval", {"command": "rm -rf tmp"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -717,12 +562,12 @@ async def send_notice( raise RuntimeError("notification failed") events = [ - _event_request( + event_request( 7, "item/commandExecution/requestApproval", {"command": "rm -rf tmp"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -760,12 +605,12 @@ async def send_notice( raise RuntimeError("notification failed") events = [ - _event_request( + event_request( 7, "item/commandExecution/requestApproval", {"command": "rm -rf tmp"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -810,7 +655,7 @@ async def send_notice( @pytest.mark.asyncio async def test_cleanup_closes_client_when_last_room_removed(self) -> None: - fake_client = FakeCodexClient(events=[_turn_completed()]) + fake_client = FakeCodexClient(events=[turn_completed()]) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) tools = ToolSchemaFakeTools() @@ -832,7 +677,7 @@ async def test_cleanup_closes_client_when_last_room_removed(self) -> None: @pytest.mark.asyncio async def test_cleanup_idempotent(self) -> None: """Calling on_cleanup twice for the same room should not raise.""" - fake_client = FakeCodexClient(events=[_turn_completed()]) + fake_client = FakeCodexClient(events=[turn_completed()]) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) tools = ToolSchemaFakeTools() @@ -856,11 +701,11 @@ async def test_cleanup_idempotent(self) -> None: async def test_cleanup_multi_room_closes_each_room_client(self) -> None: """Each room owns its Codex client; cleanup closes only that room's client.""" clients = { - "room-1": FakeCodexClient(events=[_turn_completed()]), + "room-1": FakeCodexClient(events=[turn_completed()]), # Each room owns a separate FakeCodexClient with its own turn # counter, so room-2's first turn is also "turn-1" -- matching # this room's client, not a global counter. - "room-2": FakeCodexClient(events=[_turn_completed()]), + "room-2": FakeCodexClient(events=[turn_completed()]), } adapter = CodexAdapter(config=CodexAdapterConfig()) patch_codex_clients_by_room(adapter, clients) @@ -888,15 +733,15 @@ async def test_cleanup_multi_room_closes_each_room_client(self) -> None: @pytest.mark.asyncio async def test_forwards_raw_codex_task_events(self) -> None: events = [ - _event_notification( + event_notification( "codex/event/task_started", {"taskId": "task-1", "task": {"title": "Inspect repository"}}, ), - _event_notification( + event_notification( "codex/event/task_complete", {"taskId": "task-1", "summary": "Inspection finished"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -936,11 +781,11 @@ async def test_forwards_raw_codex_task_events(self) -> None: @pytest.mark.asyncio async def test_can_disable_synthetic_turn_task_markers(self) -> None: events = [ - _event_notification( + event_notification( "codex/event/task_started", {"taskId": "task-1", "task": {"title": "Inspect repository"}}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -975,11 +820,11 @@ async def test_raw_task_event_without_explicit_task_id_does_not_emit_uuid( self, ) -> None: events = [ - _event_notification( + event_notification( "codex/event/task_started", {"id": "turn-1"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -1087,7 +932,7 @@ async def test_models_alias_lists_models_without_starting_turn(self) -> None: @pytest.mark.parametrize("effort", ["high", "max"]) async def test_reasoning_effort_passed_in_turn_overrides(self, effort: str) -> None: """Efforts newer than any list the SDK could hard-code reach Codex as-is.""" - fake_client = FakeCodexClient(events=[_turn_completed()]) + fake_client = FakeCodexClient(events=[turn_completed()]) adapter = make_codex_adapter( fake_client, config=CodexAdapterConfig( @@ -1111,7 +956,7 @@ async def test_reasoning_effort_passed_in_turn_overrides(self, effort: str) -> N @pytest.mark.asyncio async def test_reasoning_effort_omitted_when_none(self) -> None: - fake_client = FakeCodexClient(events=[_turn_completed()]) + fake_client = FakeCodexClient(events=[turn_completed()]) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) tools = ToolSchemaFakeTools() await adapter.on_started("Agent", "A coding agent") @@ -1239,7 +1084,7 @@ async def test_reasoning_command_still_answers_when_model_list_fails(self) -> No @pytest.mark.asyncio async def test_self_config_tools_registered_when_enabled(self) -> None: - fake_client = FakeCodexClient(events=[_turn_completed()]) + fake_client = FakeCodexClient(events=[turn_completed()]) adapter = make_codex_adapter( fake_client, config=CodexAdapterConfig(enable_self_config_tools=True), @@ -1263,7 +1108,7 @@ async def test_self_config_tools_registered_when_enabled(self) -> None: @pytest.mark.asyncio async def test_self_config_tools_not_registered_when_disabled(self) -> None: - fake_client = FakeCodexClient(events=[_turn_completed()]) + fake_client = FakeCodexClient(events=[turn_completed()]) adapter = make_codex_adapter( fake_client, config=CodexAdapterConfig(enable_self_config_tools=False), @@ -1287,7 +1132,7 @@ async def test_self_config_tools_not_registered_when_disabled(self) -> None: @pytest.mark.asyncio async def test_setmodel_tool_changes_model(self) -> None: events = [ - _event_request( + event_request( 99, "item/tool/call", { @@ -1296,7 +1141,7 @@ async def test_setmodel_tool_changes_model(self) -> None: "arguments": {"model": "o3"}, }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -1328,7 +1173,7 @@ async def test_setmodel_tool_changes_model(self) -> None: @pytest.mark.parametrize("effort", ["xhigh", "ultra"]) async def test_setreasoning_tool_changes_effort(self, effort: str) -> None: events = [ - _event_request( + event_request( 99, "item/tool/call", { @@ -1337,7 +1182,7 @@ async def test_setreasoning_tool_changes_effort(self, effort: str) -> None: "arguments": {"effort": effort, "summary": "detailed"}, }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -1360,7 +1205,7 @@ async def test_setreasoning_tool_changes_effort(self, effort: str) -> None: @pytest.mark.asyncio async def test_sandbox_alias_is_normalized_for_thread_and_turn(self) -> None: - events = [_turn_completed()] + events = [turn_completed()] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( fake_client, config=CodexAdapterConfig(sandbox="dangerFullAccess") @@ -1387,7 +1232,7 @@ async def test_sandbox_alias_is_normalized_for_thread_and_turn(self) -> None: @pytest.mark.asyncio async def test_external_sandbox_alias_uses_sandbox_policy(self) -> None: - events = [_turn_completed()] + events = [turn_completed()] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( fake_client, config=CodexAdapterConfig(sandbox="external-sandbox") @@ -1418,7 +1263,7 @@ async def test_external_sandbox_alias_uses_sandbox_policy(self) -> None: async def test_transport_closed_event_aborts_turn(self) -> None: """A transport/closed event should end the turn with a failed status.""" events = [ - _event_notification( + event_notification( "transport/closed", {"reason": "Codex process exited unexpectedly"}, ) @@ -1448,7 +1293,7 @@ async def test_transport_closed_resets_client_state(self) -> None: """After transport/closed, _client and _initialized should be reset so the next message rebuilds the client via _ensure_client_ready().""" events = [ - _event_notification( + event_notification( "transport/closed", {"reason": "Codex process exited unexpectedly"}, ) @@ -1480,7 +1325,7 @@ async def test_transport_closed_clears_per_room_state(self) -> None: thread/start instead of reusing a cached thread_id from the dead session.""" events = [ - _event_notification( + event_notification( "transport/closed", {"reason": "Codex process exited unexpectedly"}, ) @@ -1519,7 +1364,7 @@ async def test_transport_closed_drains_token_usage_for_dead_threads( """ events = [ - _event_notification( + event_notification( "transport/closed", {"reason": "Codex process exited unexpectedly"}, ) @@ -1557,11 +1402,11 @@ async def test_transport_closed_after_error_does_not_double_report(self) -> None the same incident must report the failure once, not twice -- matching the turn/completed branch's existing failure_reported guard.""" events = [ - _event_notification( + event_notification( "error", {"error": {"message": "Something went wrong"}, "willRetry": False}, ), - _event_notification( + event_notification( "transport/closed", {"reason": "Codex process exited unexpectedly"}, ), @@ -1587,8 +1432,8 @@ async def test_transport_closed_after_error_does_not_double_report(self) -> None @pytest.mark.asyncio async def test_turn_timeout_sends_interrupt_and_clean_error(self) -> None: """When recv_event times out, the adapter sends turn/interrupt, reports - the failure, and fails the turn so the platform retries -- same as - every sibling adapter's own turn-timeout handling.""" + the failure, and fails the turn so its delivery is marked FAILED -- + same as every sibling adapter's own turn-timeout handling.""" # No events means FakeCodexClient raises asyncio.TimeoutError immediately. fake_client = FakeCodexClient(events=[]) adapter = make_codex_adapter( @@ -1664,15 +1509,15 @@ async def test_turn_timeout_emits_failed_lifecycle_event(self) -> None: async def test_item_completed_text_overrides_accumulated_deltas(self) -> None: """item/completed text is authoritative and should replace any accumulated deltas.""" events = [ - _event_notification( + event_notification( "item/agentMessage/delta", {"itemId": "msg-1", "delta": "partial "}, ), - _event_notification( + event_notification( "item/agentMessage/delta", {"itemId": "msg-1", "delta": "garbled"}, ), - _event_notification( + event_notification( "item/completed", { "item": { @@ -1682,7 +1527,7 @@ async def test_item_completed_text_overrides_accumulated_deltas(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -1750,7 +1595,7 @@ async def calculate(inp: CalculatorInput) -> str: custom_tools: list[CustomToolDef] = [(CalculatorInput, calculate)] events = [ - _event_request( + event_request( 99, "item/tool/call", { @@ -1759,7 +1604,7 @@ async def calculate(inp: CalculatorInput) -> str: "callId": "call-99", }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -1792,7 +1637,7 @@ async def calculate(inp: CalculatorInput) -> str: async def test_execution_reporting_emits_tool_call_and_result_events(self) -> None: """With emit=Emit.TOOL_CALLS, tool_call and tool_result events are emitted.""" events = [ - _event_request( + event_request( 50, "item/tool/call", { @@ -1801,7 +1646,7 @@ async def test_execution_reporting_emits_tool_call_and_result_events(self) -> No "callId": "call-50", }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -1839,10 +1684,10 @@ async def test_send_room_file_tool_call_event_redacts_content(self) -> None: event -- report has no idea content can carry real file bytes.""" raw_content = "raw file bytes that must never reach a tool_call event" events = [ - _tool_call_request( + tool_call_request( 50, "band_send_room_file", {"content": raw_content, "filename": "f.txt"} ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -1872,7 +1717,7 @@ async def test_send_room_file_tool_call_event_redacts_content(self) -> None: async def test_execution_reporting_silenced_with_explicit_empty_emit(self) -> None: """emit=() silences tool_call/tool_result events (emit otherwise defaults on).""" events = [ - _event_request( + event_request( 50, "item/tool/call", { @@ -1881,7 +1726,7 @@ async def test_execution_reporting_silenced_with_explicit_empty_emit(self) -> No "callId": "call-50", }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig(), emit=()) @@ -1919,7 +1764,7 @@ async def fail_func(inp: FailInput) -> str: custom_tools: list[CustomToolDef] = [(FailInput, fail_func)] events = [ - _event_request( + event_request( 60, "item/tool/call", { @@ -1928,7 +1773,7 @@ async def fail_func(inp: FailInput) -> str: "callId": "call-60", }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -1971,7 +1816,7 @@ async def test_execution_reporting_emitted_for_platform_output_tools( ) -> None: """Band messaging tools are reported like any other tool — no suppression.""" events = [ - _event_request( + event_request( 70, "item/tool/call", { @@ -1980,7 +1825,7 @@ async def test_execution_reporting_emitted_for_platform_output_tools( "callId": "call-70", }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2029,7 +1874,7 @@ async def test_item_completed_mcpToolCall_send_room_file_redacts_content( tool_call event.""" raw_content = "raw file bytes that must never reach a tool_call event" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2042,7 +1887,7 @@ async def test_item_completed_mcpToolCall_send_room_file_redacts_content( } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2073,7 +1918,7 @@ async def test_item_completed_mcpToolCall_send_room_file_redacts_content( async def test_item_completed_commandExecution_emits_tool_events(self) -> None: """commandExecution item emits tool_call + tool_result with command/output.""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2087,7 +1932,7 @@ async def test_item_completed_commandExecution_emits_tool_events(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2127,7 +1972,7 @@ async def test_item_completed_commandExecution_emits_tool_events(self) -> None: async def test_item_completed_fileChange_emits_tool_events(self) -> None: """fileChange emits tool_call + tool_result with file paths.""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2141,7 +1986,7 @@ async def test_item_completed_fileChange_emits_tool_events(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2176,7 +2021,7 @@ async def test_item_completed_fileChange_emits_tool_events(self) -> None: async def test_item_completed_fileChange_missing_changes_is_safe(self) -> None: """fileChange without changes list should not crash and emits empty files.""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2187,7 +2032,7 @@ async def test_item_completed_fileChange_missing_changes_is_safe(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2216,7 +2061,7 @@ async def test_item_completed_fileChange_missing_changes_is_safe(self) -> None: async def test_item_completed_imageView_emits_tool_events(self) -> None: """imageView emits tool_call + tool_result.""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2227,7 +2072,7 @@ async def test_item_completed_imageView_emits_tool_events(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2259,7 +2104,7 @@ async def test_item_completed_imageView_emits_tool_events(self) -> None: async def test_item_completed_collabAgentToolCall_emits_tool_events(self) -> None: """collabAgentToolCall emits tool_call + tool_result preserving empty result.""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2272,7 +2117,7 @@ async def test_item_completed_collabAgentToolCall_emits_tool_events(self) -> Non } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2310,7 +2155,7 @@ async def test_item_completed_collabAgentToolCall_non_text_list_result_preserves ) -> None: """A non-text list result is dumped as JSON, not collapsed to "completed".""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2321,7 +2166,7 @@ async def test_item_completed_collabAgentToolCall_non_text_list_result_preserves } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -2347,7 +2192,7 @@ async def test_item_completed_collabAgentToolCall_non_text_list_result_preserves async def test_item_completed_mcpToolCall_emits_tool_events(self) -> None: """mcpToolCall emits tool_call + tool_result with server/tool name.""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2360,7 +2205,7 @@ async def test_item_completed_mcpToolCall_emits_tool_events(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2403,7 +2248,7 @@ async def test_item_completed_mcpToolCall_non_text_list_result_preserves_data( use its ``raw_fallback`` mode here so nothing is silently discarded. """ events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2418,7 +2263,7 @@ async def test_item_completed_mcpToolCall_non_text_list_result_preserves_data( } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -2445,8 +2290,8 @@ async def test_item_completed_mcpToolCall_non_text_list_result_preserves_data( async def test_a_dynamic_tool_call_is_reported_once(self) -> None: """Codex completes the item it already requested via item/tool/call.""" events = [ - _tool_call_request(42, "band_lookup_peers"), - _event_notification( + tool_call_request(42, "band_lookup_peers"), + event_notification( "item/completed", { "item": { @@ -2457,7 +2302,7 @@ async def test_a_dynamic_tool_call_is_reported_once(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] turn = await run_codex_turn( @@ -2471,7 +2316,7 @@ async def test_a_dynamic_tool_call_is_reported_once(self) -> None: async def test_item_completed_reasoning_emits_thought(self) -> None: """reasoning item emits thought event when emit=Emit.THOUGHTS.""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2484,7 +2329,7 @@ async def test_item_completed_reasoning_emits_thought(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2512,7 +2357,7 @@ async def test_item_completed_reasoning_emits_thought(self) -> None: async def test_item_completed_dict_summary_text_emits_thought(self) -> None: """Reasoning summary entries shaped as {text: ...} use stringify SSOT.""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2525,7 +2370,7 @@ async def test_item_completed_dict_summary_text_emits_thought(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2553,7 +2398,7 @@ async def test_item_completed_dict_summary_text_emits_thought(self) -> None: async def test_item_completed_empty_reasoning_summary_skips_thought(self) -> None: """Empty reasoning summaries must not post a '(reasoning)' placeholder.""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2563,7 +2408,7 @@ async def test_item_completed_empty_reasoning_summary_skips_thought(self) -> Non } }, ), - _event_notification( + event_notification( "item/completed", { "item": { @@ -2573,7 +2418,7 @@ async def test_item_completed_empty_reasoning_summary_skips_thought(self) -> Non } }, ), - _event_notification( + event_notification( "item/completed", { "item": { @@ -2583,7 +2428,7 @@ async def test_item_completed_empty_reasoning_summary_skips_thought(self) -> Non } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2609,15 +2454,15 @@ async def test_item_completed_empty_reasoning_summary_skips_thought(self) -> Non async def test_item_completed_empty_plan_text_skips_thought(self) -> None: """Empty plan text must not post a '(plan)' placeholder.""" events = [ - _event_notification( + event_notification( "item/completed", {"item": {"type": "plan", "id": "plan-empty", "text": ""}}, ), - _event_notification( + event_notification( "item/completed", {"item": {"type": "plan", "id": "plan-blank", "text": " "}}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2643,7 +2488,7 @@ async def test_item_completed_empty_plan_text_skips_thought(self) -> None: async def test_item_completed_skipped_when_reporting_disabled(self) -> None: """No tool events when emit narrows to Emit.TASK_EVENTS only.""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2654,7 +2499,7 @@ async def test_item_completed_skipped_when_reporting_disabled(self) -> None: } }, ), - _event_notification( + event_notification( "item/completed", { "item": { @@ -2664,7 +2509,7 @@ async def test_item_completed_skipped_when_reporting_disabled(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2694,7 +2539,7 @@ async def test_item_completed_skipped_when_reporting_disabled(self) -> None: async def test_item_completed_agentMessage_still_sets_final_text(self) -> None: """Existing agentMessage behavior preserved alongside new forwarding.""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2706,7 +2551,7 @@ async def test_item_completed_agentMessage_still_sets_final_text(self) -> None: } }, ), - _event_notification( + event_notification( "item/completed", { "item": { @@ -2716,7 +2561,7 @@ async def test_item_completed_agentMessage_still_sets_final_text(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2745,7 +2590,7 @@ async def test_item_completed_agentMessage_still_sets_final_text(self) -> None: async def test_item_completed_webSearch_emits_tool_events(self) -> None: """webSearch item emits tool_call + tool_result.""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2756,7 +2601,7 @@ async def test_item_completed_webSearch_emits_tool_events(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2787,7 +2632,7 @@ async def test_item_completed_webSearch_non_text_list_action_preserves_data( ) -> None: """A non-text list action is dumped as JSON, not collapsed to "completed".""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2798,7 +2643,7 @@ async def test_item_completed_webSearch_non_text_list_action_preserves_data( } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -2824,7 +2669,7 @@ async def test_item_completed_webSearch_non_text_list_action_preserves_data( async def test_item_completed_metadata_includes_codex_ids(self) -> None: """Forwarded events include codex_room_id, codex_thread_id, codex_turn_id.""" events = [ - _event_notification( + event_notification( "item/completed", { "item": { @@ -2835,7 +2680,7 @@ async def test_item_completed_metadata_includes_codex_ids(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -2866,7 +2711,7 @@ class TestHistoryInjection: @pytest.mark.asyncio async def test_history_injected_on_resume_failure(self) -> None: """Resume fails, fresh thread created, first turn input contains history block.""" - events = [_turn_completed()] + events = [final_text("Done."), turn_completed()] fake_client = FakeCodexClient( events=events, resume_error=CodexJsonRpcError(code=-32002, message="Thread expired"), @@ -2924,7 +2769,7 @@ async def test_history_injected_on_resume_failure(self) -> None: @pytest.mark.asyncio async def test_history_not_injected_on_successful_resume(self) -> None: """Resume succeeds, no history injection.""" - events = [_turn_completed()] + events = [final_text("Done."), turn_completed()] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) tools = ToolSchemaFakeTools() @@ -2965,7 +2810,7 @@ async def test_history_not_injected_on_successful_resume(self) -> None: @pytest.mark.asyncio async def test_history_not_injected_when_disabled(self) -> None: """inject_history_on_resume_failure=False, no injection even on failure.""" - events = [_turn_completed()] + events = [final_text("Done."), turn_completed()] fake_client = FakeCodexClient( events=events, resume_error=CodexJsonRpcError(code=-32002, message="Thread expired"), @@ -3004,7 +2849,7 @@ async def test_history_not_injected_when_disabled(self) -> None: @pytest.mark.asyncio async def test_history_filters_non_text_messages(self) -> None: """Only canonical text messages appear in injected context.""" - events = [_turn_completed()] + events = [final_text("Done."), turn_completed()] fake_client = FakeCodexClient( events=events, resume_error=CodexJsonRpcError(code=-32002, message="Not found"), @@ -3079,7 +2924,7 @@ async def test_history_filters_non_text_messages(self) -> None: @pytest.mark.asyncio async def test_history_respects_max_messages(self) -> None: """Only last max_history_messages are injected.""" - events = [_turn_completed()] + events = [final_text("Done."), turn_completed()] fake_client = FakeCodexClient( events=events, resume_error=CodexJsonRpcError(code=-32002, message="Not found"), @@ -3135,7 +2980,7 @@ async def test_history_respects_max_messages(self) -> None: @pytest.mark.asyncio async def test_history_cleared_after_injection(self) -> None: """Raw history removed from memory after first turn.""" - events = [_turn_completed()] + events = [final_text("Done."), turn_completed()] fake_client = FakeCodexClient( events=events, resume_error=CodexJsonRpcError(code=-32002, message="Not found"), @@ -3346,11 +3191,11 @@ async def test_codex_error_emits_event_unconditionally(self) -> None: """Non-retryable Codex errors always emit a structured error event.""" fake_client = FakeCodexClient( events=[ - _event_notification( + event_notification( "error", {"error": {"message": "Something went wrong"}, "willRetry": False}, ), - _event_notification( + event_notification( "turn/completed", {"turn": {"id": "turn-1", "status": "failed"}}, ), @@ -3438,8 +3283,8 @@ async def execute_tool_call_structured( ) events = [ - _tool_call_request(99, "band_send_message"), - _turn_completed(), + tool_call_request(99, BandTool.SEND_MESSAGE), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -3477,7 +3322,7 @@ class TestStructuredErrors: async def test_structured_error_from_error_event(self) -> None: """Error events with codexErrorInfo emit structured metadata.""" events = [ - _event_notification( + event_notification( "error", { "error": { @@ -3491,7 +3336,7 @@ async def test_structured_error_from_error_event(self) -> None: "willRetry": False, }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -3519,7 +3364,7 @@ async def test_structured_error_from_error_event(self) -> None: async def test_structured_error_from_failed_turn(self) -> None: """turn/completed with status=failed and codexErrorInfo emits structured error.""" events = [ - _event_notification( + event_notification( "turn/completed", { "turn": { @@ -3562,7 +3407,7 @@ async def test_structured_error_from_failed_turn_with_no_error_key(self) -> None """turn/completed with status=failed but no "error" key at all must still report a failure before raising, not just claim it did.""" events = [ - _event_notification( + event_notification( "turn/completed", {"turn": {"id": "turn-1", "status": "failed", "items": []}}, ), @@ -3629,10 +3474,10 @@ async def test_approve_session_accepts_for_the_session_and_remembers_it( self, codex_room: Callable[..., Awaitable[CodexRoom]] ) -> None: room = await codex_room( - _event_request( + event_request( 10, "item/commandExecution/requestApproval", {"command": "npm test"} ), - _turn_completed(), + turn_completed(), ) await room.send("run tests") @@ -3647,12 +3492,12 @@ async def test_approve_session_accepts_for_the_session_and_remembers_it( async def test_approval_audit_trail_emitted(self) -> None: """Approval decisions emit audit trail task events.""" events = [ - _event_request( + event_request( 7, "item/commandExecution/requestApproval", {"command": "rm -rf tmp"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -3814,7 +3659,7 @@ class TestPlanAndLifecycle: async def test_plan_steps_forwarded(self) -> None: """turn/plan/updated forwards structured plan steps.""" events = [ - _event_notification( + event_notification( "turn/plan/updated", { "plan": { @@ -3826,7 +3671,7 @@ async def test_plan_steps_forwarded(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -3861,7 +3706,7 @@ async def test_plan_steps_forwarded(self) -> None: async def test_plan_steps_not_forwarded_when_disabled(self) -> None: """turn/plan/updated is ignored when stream_plan_events=False.""" events = [ - _event_notification( + event_notification( "turn/plan/updated", { "plan": { @@ -3871,7 +3716,7 @@ async def test_plan_steps_not_forwarded_when_disabled(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -3901,7 +3746,7 @@ async def test_plan_steps_not_forwarded_when_disabled(self) -> None: async def test_turn_lifecycle_events_emitted(self) -> None: """Enriched turn lifecycle events include duration and status.""" events = [ - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -3937,7 +3782,7 @@ async def test_turn_lifecycle_events_emitted(self) -> None: async def test_threads_command_lists_mappings(self) -> None: """/threads command shows room→thread mappings.""" events = [ - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -3978,7 +3823,7 @@ async def test_threads_command_lists_mappings(self) -> None: async def test_thread_archive_clears_mapping(self) -> None: """/thread archive removes the thread mapping.""" events = [ - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -4019,11 +3864,11 @@ class TestRealtimeStreaming: async def test_reasoning_delta_streamed_as_thought(self) -> None: """item/reasoning/summaryTextDelta forwards as streaming thought.""" events = [ - _event_notification( + event_notification( "item/reasoning/summaryTextDelta", {"delta": "Analyzing the code...", "itemId": "item-1"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -4055,11 +3900,11 @@ async def test_reasoning_delta_streamed_as_thought(self) -> None: async def test_reasoning_delta_ignored_when_disabled(self) -> None: """Reasoning deltas are skipped when stream_reasoning_events=False.""" events = [ - _event_notification( + event_notification( "item/reasoning/summaryTextDelta", {"delta": "Thinking...", "itemId": "item-1"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -4087,11 +3932,11 @@ async def test_reasoning_delta_ignored_when_disabled(self) -> None: async def test_plan_delta_streamed_as_thought(self) -> None: """item/plan/delta forwards as streaming thought with plan subtype.""" events = [ - _event_notification( + event_notification( "item/plan/delta", {"delta": "Step 1: Read the test", "itemId": "plan-1"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -4122,7 +3967,7 @@ async def test_plan_delta_streamed_as_thought(self) -> None: async def test_commentary_phase_streamed_as_thought(self) -> None: """item/agentMessage/delta with phase=commentary streams as thought.""" events = [ - _event_notification( + event_notification( "item/agentMessage/delta", { "delta": "Let me think about this...", @@ -4130,7 +3975,7 @@ async def test_commentary_phase_streamed_as_thought(self) -> None: "phase": "commentary", }, ), - _event_notification( + event_notification( "item/agentMessage/delta", { "delta": "Here is the answer.", @@ -4138,7 +3983,7 @@ async def test_commentary_phase_streamed_as_thought(self) -> None: "phase": "final_answer", }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -4172,7 +4017,7 @@ async def test_commentary_excluded_from_final_text_when_streaming_enabled( ) -> None: """When stream_commentary_events=True, commentary is excluded from final_text.""" events = [ - _event_notification( + event_notification( "item/agentMessage/delta", { "delta": "thinking...", @@ -4180,7 +4025,7 @@ async def test_commentary_excluded_from_final_text_when_streaming_enabled( "phase": "commentary", }, ), - _event_notification( + event_notification( "item/agentMessage/delta", { "delta": "real answer", @@ -4188,7 +4033,7 @@ async def test_commentary_excluded_from_final_text_when_streaming_enabled( "phase": "final_answer", }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -4217,7 +4062,7 @@ async def test_commentary_included_in_final_text_when_streaming_disabled( ) -> None: """When stream_commentary_events=False (default), commentary accumulates into final_text.""" events = [ - _event_notification( + event_notification( "item/agentMessage/delta", { "delta": "thinking...", @@ -4225,7 +4070,7 @@ async def test_commentary_included_in_final_text_when_streaming_disabled( "phase": "commentary", }, ), - _event_notification( + event_notification( "item/agentMessage/delta", { "delta": "real answer", @@ -4233,7 +4078,7 @@ async def test_commentary_included_in_final_text_when_streaming_disabled( "phase": "final_answer", }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -4267,14 +4112,14 @@ class TestDiffsAndTokenUsage: async def test_diff_event_forwarded(self) -> None: """turn/diff/updated forwards as a task event when enabled.""" events = [ - _event_notification( + event_notification( "turn/diff/updated", { "diff": "--- a/src/app.py\n+++ b/src/app.py\n@@ ...", "files": ["src/app.py"], }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -4308,11 +4153,11 @@ async def test_diff_event_forwarded(self) -> None: async def test_diff_event_requires_task_events_emit(self) -> None: """Diffs are not forwarded when TASK_EVENTS is not in features.emit.""" events = [ - _event_notification( + event_notification( "turn/diff/updated", {"diff": "some diff", "files": ["f.py"]}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -4342,7 +4187,7 @@ async def test_diff_event_requires_task_events_emit(self) -> None: async def test_token_usage_tracked_and_emitted(self) -> None: """thread/tokenUsage/updated events are tracked and emitted.""" events = [ - _event_notification( + event_notification( "thread/tokenUsage/updated", { "usage": { @@ -4353,7 +4198,7 @@ async def test_token_usage_tracked_and_emitted(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -4386,7 +4231,7 @@ async def test_token_usage_tracked_and_emitted(self) -> None: async def test_token_usage_ignored_when_disabled(self) -> None: """Token usage events are tracked internally but not emitted when disabled.""" events = [ - _event_notification( + event_notification( "thread/tokenUsage/updated", { "usage": { @@ -4396,7 +4241,7 @@ async def test_token_usage_ignored_when_disabled(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -4433,7 +4278,7 @@ async def test_token_usage_ignored_when_disabled(self) -> None: async def test_usage_command_shows_token_usage(self) -> None: """/usage command shows accumulated token usage.""" events = [ - _event_notification( + event_notification( "thread/tokenUsage/updated", { "usage": { @@ -4444,7 +4289,7 @@ async def test_usage_command_shows_token_usage(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -4666,12 +4511,12 @@ async def test_session_auto_approves_matching_command_binary(self) -> None: # Two command execution requests in one turn — first will be manually # approved, second should be auto-approved by session policy. events = [ - _event_request( + event_request( 20, "item/commandExecution/requestApproval", {"command": "npm install"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -4706,12 +4551,12 @@ async def test_session_auto_approves_matching_command_binary(self) -> None: async def test_session_does_not_auto_approve_different_command(self) -> None: """Session approval for 'npm install' does NOT auto-approve 'npm publish'.""" events = [ - _event_request( + event_request( 30, "item/commandExecution/requestApproval", {"command": "npm publish"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -4743,12 +4588,12 @@ async def test_session_does_not_auto_approve_different_command(self) -> None: async def test_session_binary_granularity_approves_same_binary(self) -> None: """With binary granularity, session approval for 'npm test' auto-approves 'npm install'.""" events = [ - _event_request( + event_request( 40, "item/commandExecution/requestApproval", {"command": "npm install"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -4788,7 +4633,7 @@ class TestCleanup: async def test_on_cleanup_removes_per_room_token_usage(self) -> None: """on_cleanup for a room also removes the thread's token usage.""" events = [ - _event_notification( + event_notification( "thread/tokenUsage/updated", { "usage": { @@ -4798,7 +4643,7 @@ async def test_on_cleanup_removes_per_room_token_usage(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -4914,7 +4759,7 @@ async def test_sandbox_command_blocked_when_sandbox_policy_set(self) -> None: async def test_thread_archive_clears_raw_history(self) -> None: """/thread archive also clears raw history and injection state.""" events = [ - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -5039,7 +4884,7 @@ async def recv_event(self, timeout_s: float | None = None) -> RpcEvent: async def test_failed_turn_emits_terminal_lifecycle_event(self) -> None: """A reported failed turn still closes the lifecycle event pair.""" events = [ - _event_notification( + event_notification( "turn/completed", {"turn": {"id": "turn-1", "status": "failed", "items": []}}, ) @@ -5086,12 +4931,12 @@ class TestAcceptForSession: async def test_session_auto_approval_sends_accept_for_session(self) -> None: """Session auto-approved requests send 'acceptForSession' to Codex.""" events = [ - _event_request( + event_request( 20, "item/commandExecution/requestApproval", {"command": "npm install"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -5127,7 +4972,7 @@ async def test_network_context_included_in_approval_metadata( ) -> None: """networkContext from approval params is forwarded in metadata.""" room = await codex_room( - _event_request( + event_request( 10, "item/commandExecution/requestApproval", { @@ -5136,7 +4981,7 @@ async def test_network_context_included_in_approval_metadata( "networkContext": {"domains": ["registry.npmjs.org"]}, }, ), - _turn_completed(), + turn_completed(), ) await room.send("install lodash") @@ -5159,7 +5004,7 @@ class TestTurnStartedLifecycle: async def test_turn_started_lifecycle_event_emitted(self) -> None: """Turn started lifecycle event includes input summary.""" events = [ - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -5194,11 +5039,11 @@ class TestContextCompaction: async def test_context_compaction_event_emitted(self) -> None: """context/compacted events are forwarded as task events.""" events = [ - _event_notification( + event_notification( "context/compacted", {"threadId": "thr-1", "turnId": "turn-1"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -5228,11 +5073,11 @@ async def test_context_compaction_event_emitted(self) -> None: async def test_context_compaction_ignored_when_disabled(self) -> None: """Compaction events are not emitted when emit_turn_lifecycle_events=False.""" events = [ - _event_notification( + event_notification( "context/compacted", {"threadId": "thr-1", "turnId": "turn-1"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -5408,8 +5253,8 @@ async def test_approve_session_is_refused_for_a_change_with_no_paths( auto-approve every future file change -- and the approval stays open for a one-shot answer.""" room = await codex_room( - _event_request(15, "item/fileChange/requestApproval", {}), - _turn_completed(), + event_request(15, "item/fileChange/requestApproval", {}), + turn_completed(), ) await room.send("run") @@ -5519,12 +5364,12 @@ def test_unknown_approval_method_returns_empty_key(self) -> None: async def test_approve_session_refused_for_fileChange_without_paths(self) -> None: """/approve-session for a fileChange request with no paths is rejected.""" events = [ - _event_request( + event_request( 88, "item/fileChange/requestApproval", {"reason": "write something"}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -5569,7 +5414,7 @@ async def test_approve_session_refused_for_fileChange_without_paths(self) -> Non def parked_on_approval(*then: RpcEvent) -> tuple[RpcEvent, ...]: """A turn that asks the room to approve one command, then plays ``then``.""" return ( - _event_request(10, "item/commandExecution/requestApproval", {"command": "a"}), + event_request(10, "item/commandExecution/requestApproval", {"command": "a"}), *then, ) @@ -5596,7 +5441,7 @@ async def test_the_reply_delivered_after_the_asking_message_resolves_it( self, codex_room: Callable[..., Awaitable[CodexRoom]] ) -> None: room = await codex_room( - *parked_on_approval(_turn_completed()), approval_wait_timeout_s=3600 + *parked_on_approval(turn_completed()), approval_wait_timeout_s=3600 ) await room.send("run tests") @@ -5610,7 +5455,7 @@ async def test_a_request_while_a_turn_awaits_a_human_is_turned_away( self, codex_room: Callable[..., Awaitable[CodexRoom]] ) -> None: room = await codex_room( - *parked_on_approval(_turn_completed()), approval_wait_timeout_s=3600 + *parked_on_approval(turn_completed()), approval_wait_timeout_s=3600 ) await room.send("run tests") @@ -5632,7 +5477,7 @@ async def test_interrupt_reaches_a_turn_parked_on_a_human( as a backstop; which of the two finishes it is an asyncio-scheduling detail, so this asserts only that the turn stops running.""" room = await codex_room( - *parked_on_approval(_turn_completed()), approval_wait_timeout_s=3600 + *parked_on_approval(turn_completed()), approval_wait_timeout_s=3600 ) await room.send("run tests") @@ -5653,10 +5498,10 @@ async def test_cleanup_declines_a_parked_turn_instead_of_waiting( for the whole approval wait.""" room = await codex_room( *parked_on_approval( - _event_request( + event_request( 11, "item/commandExecution/requestApproval", {"command": "b"} ), - _turn_completed(), + turn_completed(), ), approval_wait_timeout_s=3600, ) @@ -5717,7 +5562,7 @@ def ask( tools=self.tools, msg=make_platform_message(room_id=ROOM_ID), room_id=ROOM_ID, - event=_event_request( + event=event_request( request_id, "item/commandExecution/requestApproval", params ), summary="npm test", @@ -5968,7 +5813,7 @@ async def test_thread_info_with_no_mapping(self) -> None: async def test_thread_info_includes_thread_and_usage(self) -> None: """/thread info echoes current thread id and token usage summary.""" events = [ - _event_notification( + event_notification( "thread/tokenUsage/updated", { "usage": { @@ -5978,7 +5823,7 @@ async def test_thread_info_includes_thread_and_usage(self) -> None: } }, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -6051,8 +5896,8 @@ async def test_local_command_reply_delivery_failure_is_not_reported( """/help's answer failing to post is Band-side delivery, not a Codex provider failure -- its DeliveryFailedError must be recognized and left unreported here. The original cause still - propagates (the message still fails/retries at the platform level), - just never misreported as a Codex AgentFailure.""" + propagates (the delivery is still marked FAILED), just never + misreported as a Codex AgentFailure.""" class FailingNoticeTools(ToolSchemaFakeTools): async def send_notice( @@ -6132,8 +5977,8 @@ class TestMalformedPayloadTolerance: async def test_error_event_with_non_dict_error_field(self) -> None: """`error` notification where `error` is a string must not crash the turn.""" events = [ - _event_notification("error", {"error": "oops"}), - _turn_completed(), + event_notification("error", {"error": "oops"}), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) @@ -6159,8 +6004,8 @@ async def test_failed_turn_after_error_notification_reports_once(self) -> None: """An `error` notification followed by a `turn/completed` with status=failed for the same incident must report only one failure.""" events = [ - _event_notification("error", {"error": {"message": "boom"}}), - _event_notification( + event_notification("error", {"error": {"message": "boom"}}), + event_notification( "turn/completed", { "turn": { @@ -6197,7 +6042,7 @@ async def test_failed_turn_with_falsy_scalar_error_uses_clean_fallback( """A falsy, non-dict `error` (e.g. ``False``) must not become the literal string "False" in the reported failure message.""" events = [ - _event_notification( + event_notification( "turn/completed", { "turn": { @@ -6233,7 +6078,7 @@ async def test_failed_turn_with_falsy_scalar_error_uses_clean_fallback( async def test_turn_completed_without_items_key(self) -> None: """turn/completed missing `items` is treated as an empty turn, not a crash.""" events = [ - _event_notification( + event_notification( "turn/completed", {"turn": {"id": "turn-1", "status": "completed"}}, ), @@ -6257,11 +6102,11 @@ async def test_turn_completed_without_items_key(self) -> None: async def test_turn_plan_updated_with_garbage_steps(self) -> None: """Plan deltas containing non-list `steps` must be skipped, not crash.""" events = [ - _event_notification( + event_notification( "turn/plan/updated", {"plan": {"steps": "not-a-list"}}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -6287,7 +6132,7 @@ async def test_no_lifecycle_events_when_disabled(self) -> None: """With emit_turn_lifecycle_events=False, neither started nor completed lifecycle task events are emitted.""" events = [ - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -6445,11 +6290,11 @@ async def test_multibyte_diff_respects_byte_budget(self) -> None: assert len(big_diff.encode("utf-8")) > _MAX_DIFF_METADATA_BYTES events = [ - _event_notification( + event_notification( "turn/diff/updated", {"diff": big_diff, "files": ["src/app.py"]}, ), - _turn_completed(), + turn_completed(), ] fake_client = FakeCodexClient(events=events) adapter = make_codex_adapter( @@ -6655,8 +6500,8 @@ async def execute_tool_call_structured( turn = await run_codex_turn( events=[ - _tool_call_request(42, "band_read_room_file", {"file_id": "f1"}), - _turn_completed(), + tool_call_request(42, "band_read_room_file", {"file_id": "f1"}), + turn_completed(), ], tools=ImageTools(), ) @@ -6672,8 +6517,8 @@ async def execute_tool_call_structured( async def test_non_image_result_stays_input_text(self) -> None: turn = await run_codex_turn( events=[ - _tool_call_request(42, "band_read_room_file", {"file_id": "f1"}), - _turn_completed(), + tool_call_request(42, "band_read_room_file", {"file_id": "f1"}), + turn_completed(), ] ) @@ -6691,14 +6536,14 @@ class TestSkillRoots: @pytest.mark.asyncio async def test_roots_are_sent_first_after_initialize(self) -> None: turn = await run_codex_turn( - events=[_turn_completed()], + events=[turn_completed()], config=CodexAdapterConfig(skill_roots=[SKILL_ROOT]), ) assert turn.client.requests[0] == REGISTER_SKILL_ROOT @pytest.mark.asyncio async def test_no_roots_sends_nothing(self) -> None: - turn = await run_codex_turn(events=[_turn_completed()]) + turn = await run_codex_turn(events=[turn_completed()]) assert ( CodexRequestMethod.SKILLS_EXTRA_ROOTS_SET not in turn.client.request_methods ) @@ -6706,7 +6551,7 @@ async def test_no_roots_sends_nothing(self) -> None: @pytest.mark.asyncio async def test_every_room_process_gets_the_roots(self) -> None: clients = { - room_id: FakeCodexClient(events=[_turn_completed()]) + room_id: FakeCodexClient(events=[turn_completed()]) for room_id in ("room-1", "room-2") } adapter = CodexAdapter(config=CodexAdapterConfig(skill_roots=[SKILL_ROOT])) @@ -6724,8 +6569,8 @@ async def test_every_room_process_gets_the_roots(self) -> None: async def test_a_restarted_process_gets_the_roots_again(self) -> None: client = FakeCodexClient( events=[ - _event_notification("transport/closed", {"reason": "exited"}), - _turn_completed("turn-2"), + event_notification("transport/closed", {"reason": "exited"}), + turn_completed("turn-2"), ] ) adapter = make_codex_adapter( @@ -6769,9 +6614,9 @@ class TestNoReply: async def test_no_reply_suppresses_the_fallback_text(self) -> None: turn = await run_codex_turn( events=[ - _tool_call_request(1, "band_no_reply", {"reason": "not for me"}), - _final_text("Nothing to add."), - _turn_completed(), + tool_call_request(1, BandTool.NO_REPLY, {"reason": "not for me"}), + final_text("Nothing to add."), + turn_completed(), ] ) assert turn.tools.messages_sent == [] @@ -6782,10 +6627,10 @@ async def test_no_reply_does_not_carry_into_the_next_turn( self, codex_room: Callable[..., Awaitable[CodexRoom]] ) -> None: room = await codex_room( - _tool_call_request(1, "band_no_reply"), - _turn_completed("turn-1"), - _final_text("Second answer"), - _turn_completed("turn-2"), + tool_call_request(1, BandTool.NO_REPLY), + turn_completed("turn-1"), + final_text("Second answer"), + turn_completed("turn-2"), ) await room.send("First message") @@ -6799,11 +6644,11 @@ class TestFinalTextRelay: async def test_a_tool_reply_suppresses_the_final_text(self) -> None: turn = await run_codex_turn( events=[ - _tool_call_request( + tool_call_request( 1, BandTool.SEND_MESSAGE, {"content": "Hi", "mentions": ["@a"]} ), - _final_text("Hi, again."), - _turn_completed(), + final_text("Hi, again."), + turn_completed(), ] ) @@ -6815,8 +6660,8 @@ async def test_a_policy_notification_never_stands_in_for_the_answer( ) -> None: turn = await run_codex_turn( events=[ - *parked_on_approval(_final_text("Tests pass.")), - _turn_completed(), + *parked_on_approval(final_text("Tests pass.")), + turn_completed(), ], config=CodexAdapterConfig( approval_mode="auto_accept", approval_text_notifications=True @@ -6856,9 +6701,9 @@ async def stay_quiet(args: StayQuietInput) -> str: turn = await run_codex_turn( events=[ - _tool_call_request(1, "stayquiet"), - _final_text("Nothing to add."), - _turn_completed(), + tool_call_request(1, "stayquiet"), + final_text("Nothing to add."), + turn_completed(), ], additional_tools=[(StayQuietInput, declare(stay_quiet))], ) @@ -6880,8 +6725,62 @@ async def stay_quiet(args: StayQuietInput) -> str: return "quiet" turn = await run_codex_turn( - events=[_tool_call_request(1, "stayquiet"), _turn_completed()], + events=[tool_call_request(1, "stayquiet"), turn_completed()], additional_tools=[(StayQuietInput, declare(stay_quiet))], ) assert turn.tools.turn.complete is complete + + +class TestDetachedTurnOutcome: + """A turn released to wait on a human is judged at its real end.""" + + @pytest.mark.asyncio + async def test_a_released_turn_that_ends_silently_is_reported_once( + self, codex_room: Callable[..., Awaitable[CodexRoom]] + ) -> None: + room = await codex_room(*parked_on_approval(turn_completed())) + + await room.send("run tests") + await room.send("/approve req-10") + await room.settled() + + turn, approval = room.deliveries + assert failure_reports(turn) == [MISSING_REPLY_FAILURE] + assert failure_reports(approval) == [] + + @pytest.mark.asyncio + async def test_messages_during_a_released_turn_never_take_over_its_reply( + self, codex_room: Callable[..., Awaitable[CodexRoom]] + ) -> None: + room = await codex_room( + *parked_on_approval(final_text("Tests pass."), turn_completed()) + ) + + await room.send("run tests") + await room.send("and lint too") + await room.send("/approve req-10") + await room.settled() + + turn, busy, approval = room.deliveries + assert turn.messages_sent[-1]["content"] == "Tests pass." + assert [m["content"] for m in busy.messages_sent] == [TURN_IN_PROGRESS_MESSAGE] + assert turn.turn.complete and busy.turn.complete and approval.turn.complete + assert ( + failure_reports(turn) + == failure_reports(busy) + == failure_reports(approval) + == [] + ) + + @pytest.mark.asyncio + async def test_a_released_turn_ended_by_room_cleanup_posts_nothing( + self, codex_room: Callable[..., Awaitable[CodexRoom]] + ) -> None: + room = await codex_room(*parked_on_approval(turn_completed())) + + await room.send("run tests") + await room.adapter.on_cleanup(ROOM_ID) + + [turn] = room.deliveries + assert failure_reports(turn) == [] diff --git a/tests/adapters/test_crewai_adapter.py b/tests/adapters/test_crewai_adapter.py index 699108796..4598b1a30 100644 --- a/tests/adapters/test_crewai_adapter.py +++ b/tests/adapters/test_crewai_adapter.py @@ -16,9 +16,10 @@ import json import sys import threading +from dataclasses import replace from datetime import UTC, datetime from typing import TYPE_CHECKING, Any -from unittest.mock import DEFAULT, AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock import pytest from pydantic import BaseModel, Field @@ -30,10 +31,62 @@ ) from band.core.types import Capability, Emit, PlatformMessage from band.runtime.prompts import render_system_prompt -from band.runtime.tools import BandTool, missing_reply_error +from band.runtime.tools import BandTool +from band.testing import MISSING_REPLY_FAILURE, failure_reports +from tests.framework_conformance.turnprobes import turn_input, turn_tools # The exact text CrewAI raises; the marker is the part the adapter matches on. EMPTY_LLM_RESPONSE_ERROR = f"{EMPTY_LLM_RESPONSE_MARKER} - None or empty." +EMPTY_COMPLETION = ValueError(EMPTY_LLM_RESPONSE_ERROR) + +# One scripted tool call: the crew tool's name and the arguments CrewAI passes. +ToolStep = tuple[str, dict[str, Any]] + +ANSWER = "The vault code is 4471-ECHO." +REPLY: ToolStep = (BandTool.SEND_MESSAGE, {"content": ANSWER, "mentions": ["@alice"]}) +# The platform rejects a mention-less message, and FakeAgentTools does too. +FAILED_REPLY: ToolStep = (BandTool.SEND_MESSAGE, {"content": ANSWER, "mentions": []}) +DECLINE: ToolStep = (BandTool.NO_REPLY, {"reason": "FYI only"}) +ACT: ToolStep = (BandTool.CREATE_CHATROOM, {}) +LOOKUP: ToolStep = (BandTool.LOOKUP_PEERS, {}) +NARRATION: ToolStep = ( + BandTool.SEND_EVENT, + {"content": "thinking", "message_type": "thought"}, +) + + +async def started(adapter_cls: type[CrewAIAdapterType]) -> CrewAIAdapterType: + adapter = adapter_cls() + await adapter.on_started("TestBot", "Test bot") + return adapter + + +async def call_crew_tool(adapter: CrewAIAdapterType, step: ToolStep) -> None: + """Run one of the crew's real tools as CrewAI does: synchronously, off the + event loop thread, so the bridge posts back on the adapter's loop.""" + name, arguments = step + tool = next(t for t in adapter.create_crewai_tools() if t.name == name) + await asyncio.to_thread(tool._run, **arguments) + + +def scripted_crew( + adapter: CrewAIAdapterType, + *steps: ToolStep, + ending: str | Exception | None, +) -> MagicMock: + """A crew agent whose kickoff runs ``steps`` through the real tools, then + returns ``ending`` as the final text (``None`` for a falsey result) or + raises it.""" + + async def kickoff(_prompt: str) -> Any: + for step in steps: + await call_crew_tool(adapter, step) + if isinstance(ending, Exception): + raise ending + return None if ending is None else MagicMock(raw=ending) + + return MagicMock(kickoff_async=AsyncMock(side_effect=kickoff)) + if TYPE_CHECKING: from band.adapters.crewai import CrewAIAdapter as CrewAIAdapterType @@ -179,24 +232,6 @@ def mock_crewai_agent(): return mock_agent -@pytest.fixture -def mock_crewai_agent_replied(mock_crewai_agent): - """``mock_crewai_agent`` whose ``kickoff_async`` also marks the turn as - replied, for tests that only care about kickoff/history/message - plumbing rather than the reply-tracking behavior itself (which has its - own dedicated tests under ``TestErrorHandling``).""" - module = importlib.import_module("band.adapters.crewai") - - def _mark_replied(*args, **kwargs): - tracker = module._reply_tracker_var.get() - if tracker is not None: - tracker.replied = True - return DEFAULT - - mock_crewai_agent.kickoff_async.side_effect = _mark_replied - return mock_crewai_agent - - @pytest.fixture def room_context(crewai_mocks, mock_tools): """Context manager fixture for setting up room context in tests. @@ -325,11 +360,11 @@ async def test_includes_platform_instructions_in_backstory( class TestOnMessage: @pytest.mark.asyncio async def test_initializes_history_on_bootstrap( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent_replied + self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent ): adapter = CrewAIAdapter() await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent_replied + adapter._crewai_agent = mock_crewai_agent await adapter.on_message( msg=sample_message, @@ -345,11 +380,11 @@ async def test_initializes_history_on_bootstrap( @pytest.mark.asyncio async def test_loads_existing_history( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent_replied + self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent ): adapter = CrewAIAdapter() await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent_replied + adapter._crewai_agent = mock_crewai_agent existing_history = [ {"role": "user", "content": "[Bob]: Previous message"}, @@ -370,11 +405,11 @@ async def test_loads_existing_history( @pytest.mark.asyncio async def test_calls_kickoff_async( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent_replied + self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent ): adapter = CrewAIAdapter() await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent_replied + adapter._crewai_agent = mock_crewai_agent await adapter.on_message( msg=sample_message, @@ -386,11 +421,11 @@ async def test_calls_kickoff_async( room_id="room-123", ) - mock_crewai_agent_replied.kickoff_async.assert_called_once() + mock_crewai_agent.kickoff_async.assert_called_once() @pytest.mark.asyncio async def test_replays_history_on_followup_turn( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent_replied + self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent ): """A non-bootstrap turn must replay accumulated in-session history. @@ -402,7 +437,7 @@ async def test_replays_history_on_followup_turn( """ adapter = CrewAIAdapter() await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent_replied + adapter._crewai_agent = mock_crewai_agent # Turn 1 (bootstrap): states something the agent must recall later. await adapter.on_message( @@ -439,7 +474,7 @@ async def test_replays_history_on_followup_turn( # The second kickoff must carry the prior turn as replayed context, not # just the current message. - prompt = mock_crewai_agent_replied.kickoff_async.call_args_list[1][0][0] + prompt = mock_crewai_agent.kickoff_async.call_args_list[1][0][0] assert "already handled" in prompt assert "Hello, agent!" in prompt # turn-1 content replayed to the model @@ -485,48 +520,38 @@ async def test_reports_error_on_kickoff_failure( assert failure.provider == "crewai" assert failure.message == GENERIC_PROVIDER_FAILURE_MESSAGE - @pytest.mark.asyncio - async def test_reports_error_when_crewai_completes_without_reply( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent + @pytest.mark.parametrize( + "error", + [ + RuntimeError("downstream failure after reply"), + ValueError("a different, genuine LLM problem"), + ], + ids=["non-value-error", "unrelated-value-error"], + ) + async def test_genuine_error_after_reply_still_reports_and_raises( + self, CrewAIAdapter, error ): - """A normal CrewAI return is still silent unless band_send_message ran.""" - mock_result = MagicMock() - mock_result.raw = "I could not complete the request." - mock_crewai_agent.kickoff_async.return_value = mock_result + """Only CrewAI's empty-completion ValueError ends a turn normally. Any + other exception -- even one raised after band_send_message replied -- + is a provider failure that is reported and propagates.""" + adapter = await started(CrewAIAdapter) + tools = turn_tools() + adapter._crewai_agent = scripted_crew(adapter, REPLY, ending=error) - adapter = CrewAIAdapter() - await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent - - with pytest.raises(TurnResultAlreadyReported): - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) + with pytest.raises(type(error), match=str(error)): + await adapter.on_event(turn_input(tools)) - mock_tools.send_failure.assert_awaited_once() - failure = mock_tools.send_failure.call_args.args[0] - assert failure.provider == "crewai" - assert "band_send_message" in failure.message - assert "max_iter=20" in failure.message + assert failure_reports(tools) == [("crewai", GENERIC_PROVIDER_FAILURE_MESSAGE)] @pytest.mark.asyncio - async def test_reports_error_when_crewai_returns_none_without_reply( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent + async def test_raises_error_when_agent_not_initialized( + self, CrewAIAdapter, sample_message, mock_tools ): - """A falsey CrewAI completion is still silent unless band_send_message ran.""" - mock_crewai_agent.kickoff_async.return_value = None - + """on_message raises RuntimeError if on_started was not called.""" adapter = CrewAIAdapter() - await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent + # Don't call on_started - agent remains uninitialized - with pytest.raises(TurnResultAlreadyReported): + with pytest.raises(RuntimeError, match="CrewAI agent not initialized"): await adapter.on_message( msg=sample_message, tools=mock_tools, @@ -540,431 +565,116 @@ async def test_reports_error_when_crewai_returns_none_without_reply( mock_tools.send_failure.assert_awaited_once() failure = mock_tools.send_failure.call_args.args[0] assert failure.provider == "crewai" - assert "band_send_message" in failure.message - - @pytest.mark.asyncio - async def test_does_not_report_completion_error_after_reply( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent - ): - """A turn is not silent when band_send_message already replied.""" - - module = importlib.import_module("band.adapters.crewai") - - mock_result = MagicMock() - mock_result.raw = "I already sent the user-facing reply." + assert "not initialized" in failure.message - async def _kickoff(_messages): - tracker = module._reply_tracker_var.get() - if tracker is not None: - tracker.replied = True - return mock_result - mock_crewai_agent.kickoff_async = AsyncMock(side_effect=_kickoff) +class TestTurnVerdict: + """A CrewAI turn ends on the shared verdict, not on CrewAI's exceptions.""" - adapter = CrewAIAdapter() - await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent + @pytest.mark.parametrize( + ("calls", "ending"), + [ + ((), "I could not complete the request."), + ((), None), + ((LOOKUP,), EMPTY_COMPLETION), + ((FAILED_REPLY, NARRATION), EMPTY_COMPLETION), + ], + ids=["final-text-only", "falsey-result", "read-only", "failed-send-then-event"], + ) + async def test_a_turn_that_neither_replied_nor_worked_is_reported_once( + self, CrewAIAdapter, calls, ending + ): + """The final text is discarded: this agent answers only through tools.""" + adapter = await started(CrewAIAdapter) + tools = turn_tools() + adapter._crewai_agent = scripted_crew(adapter, *calls, ending=ending) - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) + with pytest.raises(TurnResultAlreadyReported): + await adapter.on_event(turn_input(tools)) - mock_tools.send_failure.assert_not_awaited() + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + assert tools.messages_sent == [] - @pytest.mark.asyncio - async def test_suppresses_empty_final_answer_after_reply( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent + @pytest.mark.parametrize( + ("calls", "ending"), + [ + ((REPLY,), EMPTY_COMPLETION), + ((REPLY,), "I already sent the user-facing reply."), + ((DECLINE,), EMPTY_COMPLETION), + ((ACT,), EMPTY_COMPLETION), + ((ACT,), "Done."), + ], + ids=[ + "reply-then-empty", + "reply-then-text", + "decline-then-empty", + "act-then-empty", + "act-then-text", + ], + ) + async def test_a_turn_that_replied_declined_or_worked_completes( + self, CrewAIAdapter, calls, ending ): - """CrewAI raising an empty final answer AFTER the agent already replied - via band_send_message is non-fatal: no error event, no re-raise. - - This adapter replies only through band_send_message, so CrewAI's - empty-final-answer ValueError fires on essentially every turn -- - including this successful one. Routes through the real - ``_mark_productive_work`` (not hand-set flags) so the test fails if a - future change stops SEND_MESSAGE from marking a turn productive. - """ + """CrewAI's empty-completion error is how a tool-only turn ends.""" + adapter = await started(CrewAIAdapter) + tools = turn_tools() + adapter._crewai_agent = scripted_crew(adapter, *calls, ending=ending) - module = importlib.import_module("band.adapters.crewai") - tools_module = importlib.import_module("band.integrations.crewai.tools") - - async def _kickoff(_prompt): - tools_module._mark_productive_work( - module._reply_tracker_var.get(), - BandTool.SEND_MESSAGE, - json.dumps({"status": "success"}), - {"content": "Hi.", "mentions": ["@owner"]}, - custom_effects=None, - ) - raise ValueError(EMPTY_LLM_RESPONSE_ERROR) + await adapter.on_event(turn_input(tools)) - mock_crewai_agent.kickoff_async = AsyncMock(side_effect=_kickoff) + assert failure_reports(tools) == [] - adapter = CrewAIAdapter() - await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent - - # Must NOT raise — the reply already went out. - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) + async def test_an_empty_first_call_is_retried_once_then_reported( + self, CrewAIAdapter + ): + """Nothing ran before an empty first call, so one retry duplicates + nothing; a second empty call ends a turn that did nothing.""" + adapter = await started(CrewAIAdapter) + tools = turn_tools() + adapter._crewai_agent = scripted_crew(adapter, ending=EMPTY_COMPLETION) - # No failure reported to the room. - mock_tools.send_failure.assert_not_awaited() + with pytest.raises(TurnResultAlreadyReported): + await adapter.on_event(turn_input(tools)) - @pytest.mark.asyncio - async def test_suppresses_empty_final_answer_after_tool_only_turn( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent - ): - """An empty final answer after a tool-only turn (no reply) is non-fatal. - - When the user instructs the agent to run a tool and NOT send a message - (e.g. "store this memory, don't reply"), the agent does its work and has - nothing left to say — CrewAI then raises ValueError("Invalid response - from LLM call - None or empty.") on its forced final-answer step. Because - a tool already executed successfully this turn, that empty answer is - benign: no error event, no re-raise. Routes through the real - ``_mark_productive_work`` (not hand-set flags) so the test fails if a - future change stops a terminal tool from marking a turn productive. - """ + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + assert adapter._crewai_agent.kickoff_async.call_count == 2 - module = importlib.import_module("band.adapters.crewai") - tools_module = importlib.import_module("band.integrations.crewai.tools") - - async def _kickoff(_prompt): - tools_module._mark_productive_work( - module._reply_tracker_var.get(), - BandTool.STORE_MEMORY, - json.dumps({"status": "success"}), - {}, - custom_effects=None, - ) - raise ValueError(EMPTY_LLM_RESPONSE_ERROR) + async def test_an_empty_first_call_recovers_on_retry(self, CrewAIAdapter): + adapter = await started(CrewAIAdapter) + tools = turn_tools() + replying = scripted_crew(adapter, REPLY, ending=EMPTY_COMPLETION) - mock_crewai_agent.kickoff_async = AsyncMock(side_effect=_kickoff) + async def kickoff(prompt: str) -> Any: + if crew.kickoff_async.call_count == 1: + raise EMPTY_COMPLETION + return await replying.kickoff_async(prompt) - adapter = CrewAIAdapter() - await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent + crew = MagicMock(kickoff_async=AsyncMock(side_effect=kickoff)) + adapter._crewai_agent = crew - # Must NOT raise — the tool work already happened. - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) + await adapter.on_event(turn_input(tools)) - # No failure reported to the room. - mock_tools.send_failure.assert_not_awaited() + assert [m["content"] for m in tools.messages_sent] == [ANSWER] + assert failure_reports(tools) == [] + assert crew.kickoff_async.call_count == 2 - @pytest.mark.asyncio - async def test_the_next_turn_remembers_what_the_agent_posted( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent - ): + async def test_the_next_turn_remembers_what_the_agent_posted(self, CrewAIAdapter): """A turn whose work was a band_send_message must leave that message in the room's history. CrewAI's final answer rarely repeats it, and an agent that can't see it already asked a peer asks again whenever the peer's reply wakes it -- the loop a coordinator falls into.""" - module = importlib.import_module("band.adapters.crewai") - tools_module = importlib.import_module("band.integrations.crewai.tools") - prompts: list[str] = [] - - posts = ["Please check the API contract.", "Thanks, that settles it."] - - async def _kickoff(prompt): - prompts.append(prompt) - tools_module._mark_productive_work( - module._reply_tracker_var.get(), - BandTool.SEND_MESSAGE, - json.dumps({"status": "success"}), - {"content": posts[len(prompts) - 1], "mentions": ["@ops/checker"]}, - custom_effects=None, - ) - raise ValueError(EMPTY_LLM_RESPONSE_ERROR) - - mock_crewai_agent.kickoff_async = AsyncMock(side_effect=_kickoff) - adapter = CrewAIAdapter() - await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent - - for bootstrap in (True, False): - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=bootstrap, - room_id="room-123", - ) - - earlier, _, _ = prompts[1].partition("[New message -- act on this now:]") - assert "(to @ops/checker) Please check the API contract." in earlier - - @pytest.mark.asyncio - async def test_read_only_turn_with_empty_final_answer_completes( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent - ): - """A turn whose only work was read-only must finish, not fail the delivery. - - Told to run read tools and nothing else ("call band_list_tasks ... do not - call any other tool"), the agent does exactly that and stays silent: - fetching state is not terminal work, so the real marker leaves both - tracker flags down. Re-raising CrewAI's empty-response ValueError would - mark the delivery failed for a turn that did precisely what it was asked. - """ - module = importlib.import_module("band.adapters.crewai") - tools_module = importlib.import_module("band.integrations.crewai.tools") - - async def _kickoff(_prompt): - # Route through the real marker so the read-only classification -- - # not a hand-set flag -- is what keeps this turn "unproductive". - tools_module._mark_productive_work( - module._reply_tracker_var.get(), - BandTool.LIST_TASKS, - json.dumps({"status": "success", "data": []}), - {}, - custom_effects=None, - ) - raise ValueError(EMPTY_LLM_RESPONSE_ERROR) - - mock_crewai_agent.kickoff_async = AsyncMock(side_effect=_kickoff) - - adapter = CrewAIAdapter() - await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent - - # Must NOT raise — the delivery completes. - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) - - # Exactly one failure reported, carrying the shared missing-reply - # wording -- the room hears about the missing reply, not CrewAI's - # internal error. - mock_tools.send_failure.assert_awaited_once() - failure = mock_tools.send_failure.call_args.args[0] - assert failure.provider == "crewai" - assert missing_reply_error("CrewAI") in failure.message + adapter = await started(CrewAIAdapter) + crew = scripted_crew(adapter, REPLY, ending=EMPTY_COMPLETION) + adapter._crewai_agent = crew - @pytest.mark.asyncio - async def test_empty_answer_with_no_tool_call_still_raises( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent - ): - """An empty answer with zero tool activity is a genuine failure, not a - finished turn -- it must still fail the delivery so the platform retries. - - Nothing distinguishes "the model correctly stopped after read-only - work" from "the very first LLM call came back empty" by the exception - alone -- both raise CrewAI's identical ValueError. ``any_tool_ran`` is - that distinction: it is the only turn where this adapter can rule out - a genuine no-response failure, so it is the only one that must not be - silently marked processed. - """ - mock_crewai_agent.kickoff_async = AsyncMock( - side_effect=ValueError(EMPTY_LLM_RESPONSE_ERROR) + await adapter.on_event(turn_input(turn_tools())) + await adapter.on_event( + replace(turn_input(turn_tools()), is_session_bootstrap=False) ) - adapter = CrewAIAdapter() - await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent - - with pytest.raises(ValueError, match=EMPTY_LLM_RESPONSE_ERROR): - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) - - mock_tools.send_failure.assert_awaited_once() - failure = mock_tools.send_failure.call_args.args[0] - assert failure.provider == "crewai" - assert failure.message == GENERIC_PROVIDER_FAILURE_MESSAGE - # First call plus the one retry -- proves the retry actually happens - # before the final raise, not a bare pass-through of the first failure. - assert mock_crewai_agent.kickoff_async.call_count == 2 - - @pytest.mark.asyncio - async def test_empty_first_call_recovers_on_retry( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent - ): - """A turn's first LLM call coming back empty is retried once before - giving up -- a successful retry must complete the turn normally. - - Nothing ran before the empty completion, so there is no side effect - for the retry to duplicate; the retry either behaves exactly like the - first attempt would have or, as here, recovers and replies. - """ - module = importlib.import_module("band.adapters.crewai") - mock_result = MagicMock() - mock_result.raw = "Hello! I'm here to help." - - async def _kickoff(_prompt): - if mock_crewai_agent.kickoff_async.call_count == 1: - raise ValueError(EMPTY_LLM_RESPONSE_ERROR) - tracker = module._reply_tracker_var.get() - if tracker is not None: - tracker.replied = True - return mock_result - - mock_crewai_agent.kickoff_async = AsyncMock(side_effect=_kickoff) - - adapter = CrewAIAdapter() - await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent - - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) - - mock_tools.send_failure.assert_not_awaited() - # First call plus the one retry that recovered it. - assert mock_crewai_agent.kickoff_async.call_count == 2 - - @pytest.mark.asyncio - async def test_no_missing_reply_error_after_clean_tool_only_return( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent - ): - """Terminal tool work answers for a turn even when kickoff returns cleanly. - - The empty-response path is not the only ending without a reply: CrewAI - can also return normally after a tool-only turn. One rule judges both, so - neither posts a missing-reply error once terminal work has landed. - """ - module = importlib.import_module("band.adapters.crewai") - mock_result = MagicMock() - mock_result.raw = "Stored it." - - async def _kickoff(_prompt): - module._reply_tracker_var.get().tool_executed = True - return mock_result - - mock_crewai_agent.kickoff_async = AsyncMock(side_effect=_kickoff) - - adapter = CrewAIAdapter() - await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent - - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) - - mock_tools.send_failure.assert_not_awaited() - - @pytest.mark.asyncio - @pytest.mark.parametrize( - "error", - [ - RuntimeError("downstream failure after reply"), - ValueError("a different, genuine LLM problem"), - ], - ids=["non-value-error", "unrelated-value-error"], - ) - async def test_genuine_error_after_reply_still_reports_and_raises( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent, error - ): - """A genuine failure AFTER a reply went out must NOT be swallowed. - - The empty-final-answer suppression only matches CrewAI's specific - ValueError("Invalid response from LLM call ..."). Any other exception — - even one raised after band_send_message already replied — must still - post an error event and propagate, so real bugs stay visible. - """ - - module = importlib.import_module("band.adapters.crewai") - - async def _kickoff(_messages): - # Simulate band_send_message having succeeded earlier this turn. - tracker = module._reply_tracker_var.get() - if tracker is not None: - tracker.replied = True - raise error - - mock_crewai_agent.kickoff_async = AsyncMock(side_effect=_kickoff) - - adapter = CrewAIAdapter() - await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent - - # The genuine error must propagate despite the prior reply. - with pytest.raises(type(error), match=str(error)): - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) - - # And it must surface as a reported failure. - mock_tools.send_failure.assert_awaited_once() - failure = mock_tools.send_failure.call_args.args[0] - assert failure.provider == "crewai" - assert failure.message == GENERIC_PROVIDER_FAILURE_MESSAGE - - @pytest.mark.asyncio - async def test_raises_error_when_agent_not_initialized( - self, CrewAIAdapter, sample_message, mock_tools - ): - """on_message raises RuntimeError if on_started was not called.""" - adapter = CrewAIAdapter() - # Don't call on_started - agent remains uninitialized - - with pytest.raises(RuntimeError, match="CrewAI agent not initialized"): - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) - - mock_tools.send_failure.assert_awaited_once() - failure = mock_tools.send_failure.call_args.args[0] - assert failure.provider == "crewai" - assert "not initialized" in failure.message + second_prompt = crew.kickoff_async.call_args_list[1].args[0] + earlier, _, _ = second_prompt.partition("[New message -- act on this now:]") + assert f"(to @alice) {ANSWER}" in earlier class TestVerboseMode: @@ -1032,11 +742,11 @@ async def test_allow_delegation_defaults_to_false( class TestParticipantsUpdate: @pytest.mark.asyncio async def test_includes_participants_update_in_message( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent_replied + self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent ): adapter = CrewAIAdapter() await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent_replied + adapter._crewai_agent = mock_crewai_agent await adapter.on_message( msg=sample_message, @@ -1048,7 +758,7 @@ async def test_includes_participants_update_in_message( room_id="room-123", ) - call_args = mock_crewai_agent_replied.kickoff_async.call_args + call_args = mock_crewai_agent.kickoff_async.call_args prompt = call_args[0][0] assert "Alice joined" in prompt @@ -1057,11 +767,11 @@ async def test_includes_participants_update_in_message( class TestContactsUpdate: @pytest.mark.asyncio async def test_includes_contacts_update_in_message( - self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent_replied + self, CrewAIAdapter, sample_message, mock_tools, mock_crewai_agent ): adapter = CrewAIAdapter() await adapter.on_started("TestBot", "Test bot") - adapter._crewai_agent = mock_crewai_agent_replied + adapter._crewai_agent = mock_crewai_agent await adapter.on_message( msg=sample_message, @@ -1073,7 +783,7 @@ async def test_includes_contacts_update_in_message( room_id="room-123", ) - call_args = mock_crewai_agent_replied.kickoff_async.call_args + call_args = mock_crewai_agent.kickoff_async.call_args prompt = call_args[0][0] assert "@alice is now a contact" in prompt diff --git a/tests/adapters/test_cursor_acp_adapter.py b/tests/adapters/test_cursor_acp_adapter.py index dfd2ea6bb..5e56ce333 100644 --- a/tests/adapters/test_cursor_acp_adapter.py +++ b/tests/adapters/test_cursor_acp_adapter.py @@ -22,13 +22,13 @@ CursorACPAdapterConfig, CursorTurn, ) +from band.client.streaming import ControlMode from band.core.protocols import AgentToolsProtocol -from band.core.types import PlatformMessage +from band.core.types import AgentInput, HistoryProvider, PlatformMessage from band.integrations.acp.client_adapter import ACPPermissionRequest -from band.integrations.acp.client_types import ACPClientSessionState from band.integrations.acp.cursor import PLAN_REQUESTED_TEMPLATE from band.integrations.acp.types import ACPToolCall -from band.testing import FakeAgentTools +from band.testing import MISSING_REPLY_FAILURE, FakeAgentTools, failure_reports from tests.integrations.acp.acp_toolkit.agent import FakeACPAgent from tests.integrations.acp.acp_toolkit.harness import launch_for, pair_in_process @@ -39,7 +39,7 @@ class DecisionTools(FakeAgentTools): and signal once a pending decision is visible in the room. Decision prompts and replies are the adapter's own posts, so they go - through ``send_notice``; it honors ``hold_message`` like ``send_message``. + through ``send_notice``. """ def __init__(self) -> None: @@ -49,11 +49,6 @@ def __init__(self) -> None: async def send_notice( self, content: str, mentions: list[str] | list[dict[str, str]] | None = None ) -> object: - if (held := self._take_held_message(content)) is not None: - held.sending.set() - await held.released.wait() - if held.error is not None: - raise held.error result = await super().send_notice(content, mentions) self.prompt_sent.set() return result @@ -114,33 +109,40 @@ def decision_token(prompt: str) -> str: return match[1] +def said(tools: FakeAgentTools) -> list[str]: + return [cast(str, sent["content"]) for sent in tools.messages_sent] + + class CursorRoom: """A Cursor room whose ``agent acp`` peer is a scripted in-process ACP - agent: messages go through the adapter as Band delivers them.""" + agent: each message goes through ``on_event`` with its own tools, as the + runtime delivers it, so every turn is judged.""" def __init__(self, adapter: CursorACPAdapter, agent: FakeACPAgent) -> None: self.adapter = adapter self.agent = agent - self.tools = FakeAgentTools(room_id="room-1") self._bootstrapped = False - @property - def chat(self) -> list[str]: - return [cast(str, sent["content"]) for sent in self.tools.messages_sent] - - async def send(self, content: str, *, tools: FakeAgentTools | None = None) -> None: - """Deliver ``content``; ``tools`` stands in for the message's own - per-event tools, which default to the room's shared ones.""" + async def send(self, content: str) -> FakeAgentTools: + """Deliver ``content``; return the tools its turn posted through.""" + tools = FakeAgentTools(room_id="room-1") bootstrap, self._bootstrapped = not self._bootstrapped, True - await self.adapter.on_message( - room_message(content), - tools or self.tools, - ACPClientSessionState(), - None, - None, - is_session_bootstrap=bootstrap, - room_id="room-1", + await self.adapter.on_event( + AgentInput( + msg=room_message(content), + tools=tools, + history=HistoryProvider(raw=[]), + participants_msg=None, + contacts_msg=None, + is_session_bootstrap=bootstrap, + room_id="room-1", + ) ) + return tools + + async def turns_finished(self) -> None: + """Wait for every turn still running after its message returned.""" + await asyncio.gather(*self.adapter._background_tasks, return_exceptions=True) # The adapter's ACP runtime lives on the loop that started it, so setup, the @@ -916,21 +918,42 @@ async def test_a_room_reply_resolves_the_decision_it_was_sent_for( .will_ask_permission(title="shell", allow_option_id="allow-once") .will_say("ran it") ) - ran = room.tools.hold_message("ran it") - await room.send("run it") - [prompt] = room.chat - await room.send(f"/cursor select {decision_token(prompt)} allow-once") - async with ran: - pass + turn = await room.send("run it") + [prompt] = said(turn) + reply = await room.send(f"/cursor select {decision_token(prompt)} allow-once") + await turn.until_said("ran it") assert room.agent.approved is True - assert room.chat[-1] == ( + assert said(reply) == [ f"Cursor permission decision `{decision_token(prompt)}` resolved." + ] + + +class TestCursorACPAdapterDetachedTurn: + """A turn parked on a decision releases its message early, so it is + judged at its real end instead of when ``on_event`` returns.""" + + @pytest.mark.asyncio + async def test_a_detached_turn_that_ends_with_nothing_is_reported_after_release( + self, cursor_room: Callable[..., Awaitable[CursorRoom]] + ) -> None: + room = await cursor_room( + FakeACPAgent().will_ask_permission( + title="shell", allow_option_id="allow-once" + ) ) + # on_event returning normally is what keeps the delivery PROCESSED. + turn = await room.send("run it") + [prompt] = said(turn) + assert failure_reports(turn) == [] + + await room.send(f"/cursor deny {decision_token(prompt)}") + await room.turns_finished() + + assert failure_reports(turn) == [MISSING_REPLY_FAILURE] -class TestCursorACPAdapterControlMessages: @pytest.mark.asyncio async def test_a_decision_reply_settles_on_its_own_tools( self, cursor_room: Callable[..., Awaitable[CursorRoom]] @@ -943,20 +966,73 @@ async def test_a_decision_reply_settles_on_its_own_tools( .will_ask_permission(title="shell", allow_option_id="allow-once") .will_say("ran it") ) - await room.send("run it") - [prompt] = room.chat - reply_tools = FakeAgentTools(room_id="room-1") + turn = await room.send("run it") + [prompt] = said(turn) + + reply = await room.send(f"/cursor select {decision_token(prompt)} allow-once") + await room.turns_finished() + + assert reply.turn.complete + assert not reply.turn.replied + assert failure_reports(reply) == [] + assert said(turn) == [prompt, "ran it"] + assert failure_reports(turn) == [] + + @pytest.mark.asyncio + async def test_a_turn_cancelled_by_cleanup_is_not_judged( + self, cursor_room: Callable[..., Awaitable[CursorRoom]] + ) -> None: + room = await cursor_room( + FakeACPAgent().will_ask_permission( + title="shell", allow_option_id="allow-once" + ) + ) + turn = await room.send("run it") + + await room.adapter.on_cleanup("room-1") + await room.turns_finished() + + # The stopped connection fails the turn with the ACP error it raised. + assert MISSING_REPLY_FAILURE not in failure_reports(turn) + + @pytest.mark.asyncio + async def test_an_interrupted_detached_turn_is_not_reported( + self, cursor_room: Callable[..., Awaitable[CursorRoom]] + ) -> None: + """A room /stop ends the parked turn; the adapter settled it.""" + room = await cursor_room( + FakeACPAgent().will_ask_permission( + title="shell", allow_option_id="allow-once" + ) + ) + turn = await room.send("run it") + + await room.adapter.on_interrupt("room-1", ControlMode.STOP) + await room.turns_finished() - await room.send( - f"/cursor select {decision_token(prompt)} allow-once", tools=reply_tools + assert failure_reports(turn) == [] + + @pytest.mark.asyncio + async def test_a_cancelled_detached_turn_posts_nothing( + self, cursor_room: Callable[..., Awaitable[CursorRoom]] + ) -> None: + room = await cursor_room( + FakeACPAgent().will_ask_permission( + title="shell", allow_option_id="allow-once" + ) ) - await room.tools.until_said("ran it") + turn = await room.send("run it") + [prompt] = said(turn) - assert reply_tools.turn.complete - assert not reply_tools.turn.replied - assert room.chat == [prompt, "ran it"] - assert room.adapter._room_tools["room-1"] is room.tools + for task in room.adapter._background_tasks: + task.cancel() + await room.turns_finished() + assert said(turn) == [prompt] + assert failure_reports(turn) == [] + + +class TestCursorACPAdapterControlMessages: @pytest.mark.asyncio async def test_bare_cursor_lists_pending_decisions(self) -> None: tools = DecisionTools() diff --git a/tests/adapters/test_letta_adapter.py b/tests/adapters/test_letta_adapter.py index 538d16713..0d7621548 100644 --- a/tests/adapters/test_letta_adapter.py +++ b/tests/adapters/test_letta_adapter.py @@ -26,9 +26,14 @@ GENERIC_PROVIDER_FAILURE_MESSAGE, TurnResultAlreadyReported, ) -from band.core.types import Emit +from band.core.types import Capability, Emit from band.runtime.tools import BandTool -from band.testing import FakeAgentTools, reported_failures +from band.testing import ( + MISSING_REPLY_FAILURE, + FakeAgentTools, + failure_reports, + reported_failures, +) from tests.adapters.lettakit import ( default_enforcement, make_assistant_message, @@ -40,15 +45,29 @@ make_platform_message, make_tool_call_message, make_tool_return_message, + ready_letta_adapter, + scripted_letta_turn, ) - -# The tools whose call settles the turn's reply, so no assistant text is relayed. -REPLY_SETTLING_TOOLS = [ - BandTool.SEND_MESSAGE, - BandTool.SEND_ROOM_FILE, - BandTool.NO_REPLY, +from tests.framework_conformance.turnprobes import ROOM_ID, turn_input, turn_tools + +# Band tool calls that settle the turn's reply, so no assistant text is relayed. +REPLY_SETTLING_CALLS = [ + pytest.param( + BandTool.SEND_MESSAGE, {"content": "Hi", "mentions": ["@alice"]}, id="message" + ), + pytest.param( + BandTool.SEND_ROOM_FILE, + {"content": "notes", "filename": "notes.txt", "mentions": ["@alice"]}, + id="room-file", + ), + pytest.param(BandTool.NO_REPLY, {"reason": "FYI only"}, id="no-reply"), ] + +def sent_contents(tools: FakeAgentTools) -> list[str]: + return [message["content"] for message in tools.messages_sent] + + # ────────────────────────────────────────────────────────────────────── # Initialization # ────────────────────────────────────────────────────────────────────── @@ -213,36 +232,30 @@ async def _raise(*args: Any, **kwargs: Any) -> None: assert not reported_failures(tools) @pytest.mark.asyncio - @pytest.mark.parametrize("reply_tool", REPLY_SETTLING_TOOLS) + @pytest.mark.parametrize(("reply_tool", "arguments"), REPLY_SETTLING_CALLS) async def test_skip_auto_relay_when_a_reply_tool_settled_the_turn( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock], reply_tool: str + self, reply_tool: str, arguments: dict[str, Any] ) -> None: - adapter, mock_client = adapter_with_client - - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message(reply_tool), - make_tool_return_message(reply_tool), - make_assistant_message("Done!"), + adapter, mock_client = ready_letta_adapter(capabilities={Capability.FILES}) + mock_client.agents.messages.create.side_effect = scripted_letta_turn( + adapter, + room_id="room-1", + tool_calls=[(reply_tool, arguments)], + final_text="Done!", ) - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() await adapter.on_message( - msg, + make_platform_message(), tools, - history, + LettaSessionState(), None, None, is_session_bootstrap=False, room_id="room-1", ) - # No auto-relay — the agent settled its reply via an MCP tool - assert len(tools.messages_sent) == 0 + assert "Done!" not in sent_contents(tools) @pytest.mark.asyncio async def test_timeout_reports_error( @@ -1376,18 +1389,15 @@ def test_enforcement_prompt_uses_resolved_names(self) -> None: @pytest.mark.asyncio async def test_relay_detection_uses_resolved_name(self) -> None: """A send via the resolved (external) tool name suppresses auto-relay.""" - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.server_id = "mcp-server-1" + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")) + ) adapter._mcp.resolve_send_tools( ["create_agent_chat_message", "create_agent_chat_event"] ) - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - mock_client.agents.messages.create.return_value = make_letta_response( make_tool_call_message("create_agent_chat_message"), + make_tool_return_message("create_agent_chat_message"), make_assistant_message("Done!"), ) @@ -1457,53 +1467,54 @@ async def test_streamed_tool_return_records_only_for_external_band_mcp( class TestAutoRelayDisabled: @pytest.mark.asyncio - async def test_disabled_relay_fails_loud_instead_of_sending(self) -> None: - """With auto_relay off, an unused MCP send path surfaces as an error - event and the assistant text is dropped — nothing is silently relayed.""" - adapter = LettaAdapter(config=LettaAdapterConfig(auto_relay=False)) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.server_id = "mcp-server-1" - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("I'll help you!") + async def test_disabled_relay_drops_the_text_and_the_verdict_reports_it( + self, + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(auto_relay=False), room_id=ROOM_ID + ) + mock_client.agents.messages.create.side_effect = scripted_letta_turn( + adapter, room_id=ROOM_ID, tool_calls=[], final_text="I'll help you!" ) - tools = FakeAgentTools() + tools = turn_tools() with pytest.raises(TurnResultAlreadyReported): - await adapter.on_message( - make_platform_message(), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) + await adapter.on_event(turn_input(tools)) - assert len(tools.messages_sent) == 0 - failures = reported_failures(tools) - assert len(failures) == 1 - assert failures[0]["provider"] == "letta" - assert "band_send_message" in failures[0]["message"] + assert tools.messages_sent == [] + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + + @pytest.mark.asyncio + async def test_disabled_relay_completes_a_turn_that_did_real_work(self) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(auto_relay=False), room_id=ROOM_ID + ) + mock_client.agents.messages.create.side_effect = scripted_letta_turn( + adapter, + room_id=ROOM_ID, + tool_calls=[(BandTool.CREATE_CHATROOM, {})], + final_text="Created it.", + ) + + tools = turn_tools() + await adapter.on_event(turn_input(tools)) + + assert tools.messages_sent == [] + assert failure_reports(tools) == [] @pytest.mark.asyncio - @pytest.mark.parametrize("reply_tool", REPLY_SETTLING_TOOLS) + @pytest.mark.parametrize(("reply_tool", "arguments"), REPLY_SETTLING_CALLS) async def test_disabled_relay_quiet_when_a_reply_tool_settled_the_turn( - self, reply_tool: str + self, reply_tool: str, arguments: dict[str, Any] ) -> None: - adapter = LettaAdapter(config=LettaAdapterConfig(auto_relay=False)) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.server_id = "mcp-server-1" - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message(reply_tool), - make_assistant_message("Done!"), + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(auto_relay=False), capabilities={Capability.FILES} + ) + mock_client.agents.messages.create.side_effect = scripted_letta_turn( + adapter, + room_id="room-1", + tool_calls=[(reply_tool, arguments)], + final_text="Done!", ) tools = FakeAgentTools() @@ -1517,7 +1528,7 @@ async def test_disabled_relay_quiet_when_a_reply_tool_settled_the_turn( room_id="room-1", ) - assert len(tools.messages_sent) == 0 + assert "Done!" not in sent_contents(tools) assert not reported_failures(tools) @@ -1587,16 +1598,15 @@ async def test_reported_turn_consumes_delivered_seed(self) -> None: make_assistant_message("The response was not sent through the tool.") ) - with pytest.raises(TurnResultAlreadyReported): - await adapter.on_message( - make_platform_message(), - FakeAgentTools(), - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) + await adapter.on_message( + make_platform_message(), + FakeAgentTools(), + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) assert room_ctx.pending_seed == [] assert room_ctx.last_interaction is not None diff --git a/tests/adapters/test_pydantic_ai_adapter.py b/tests/adapters/test_pydantic_ai_adapter.py index 995c530ee..dc8d3c3bb 100644 --- a/tests/adapters/test_pydantic_ai_adapter.py +++ b/tests/adapters/test_pydantic_ai_adapter.py @@ -7,8 +7,9 @@ stream event handling, execution reporting, and custom tools. """ +import contextlib import json -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from contextlib import asynccontextmanager, contextmanager from datetime import UTC, datetime from pathlib import Path @@ -58,6 +59,7 @@ _is_output_retries_exhausted, _is_replayable_history_message, ) +from band.core.exceptions import BandToolError from band.core.protocols import ( GENERIC_PROVIDER_FAILURE_MESSAGE, AgentToolsProtocol, @@ -91,9 +93,15 @@ get_tool_description, platform_args_schema, ) -from band.testing import FakeAgentTools +from band.testing import ( + MISSING_REPLY_FAILURE, + FakeAgentTools, + failure_reports, + reported_failures, +) from tests.adapters.usage_events import sent_usage_payloads from tests.framework_configs.adapters import pydantic_ai_probe_tools +from tests.framework_conformance.turnprobes import ROOM_ID, turn_input def make_stream_events( @@ -766,6 +774,37 @@ def _tool_returns(result: Any) -> list[Any]: return [part.content for part in _parts(result, ToolReturnPart)] +def _streamed_tool_calls(*calls: tuple[str, dict[str, Any]]) -> FunctionModel: + """A streaming model (what ``on_message`` drives) making ``calls``, one per + request, then ending with nothing left to say.""" + pending = list(calls) + + async def stream( + messages: list[ModelMessage], info: AgentInfo + ) -> AsyncIterator[str | dict[int, DeltaToolCall]]: + if pending: + name, args = pending.pop(0) + yield {0: DeltaToolCall(name=name, json_args=json.dumps(args))} + else: + yield "" + + return FunctionModel(stream_function=stream) + + +async def _run_on_message( + adapter: PydanticAIAdapter, msg: PlatformMessage, tools: FakeAgentTools +) -> None: + await adapter.on_message( + msg=msg, + tools=tools, + history=[], + participants_msg=None, + contacts_msg=None, + is_session_bootstrap=True, + room_id=tools.room_id, + ) + + class TestAdvertisedToolSchemas: """Per-argument text fidelity is asserted in test_tool_text_drift. @@ -1388,35 +1427,23 @@ async def test_creates_agent_lazily_if_not_started( mock_create.assert_called_once() @pytest.mark.asyncio - async def test_reports_failure_when_no_terminal_tool_ran( - self, sample_message, mock_tools, mock_pydantic_agent + async def test_a_turn_that_never_replied_is_reported_by_the_verdict( + self, mock_pydantic_agent ): - """A clean run that never called a reply/terminal tool is a silently - dropped turn — must still surface as a failure, even without an - exception.""" + """A clean run that never replied, declined or acted is a silently + dropped turn — the shared verdict reports it, even without an exception.""" adapter = PydanticAIAdapter(PydanticAIAdapterConfig(model="openai:gpt-5.4")) with patch.object(adapter, "_create_agent", return_value=mock_pydantic_agent): await adapter.on_started("TestBot", "Test bot") - adapter._agent.run_stream_events = MagicMock( return_value=make_stream_events(result_messages=[]) ) + tools = FakeAgentTools(room_id=ROOM_ID) with pytest.raises(TurnResultAlreadyReported): - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) + await adapter.on_event(turn_input(tools)) - mock_tools.send_failure.assert_awaited_once() - failure = mock_tools.send_failure.call_args.args[0] - assert failure.provider == "pydantic_ai" - assert "band_send_message" in failure.message + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] class TestOnCleanup: @@ -1913,27 +1940,19 @@ async def test_event_failure_does_not_crash_run( def make_raising_stream( error: BaseException, *, - tool_result: bool, - tool_name: str = "band_send_message", - tool_content: Any = None, + work: Callable[[], Awaitable[object]] | None = None, ): - """Run stream (async CM, as 2.x returns) that fires a tool result, then raises. + """Run stream (async CM, as 2.x returns) that does ``work``, then raises. - ``tool_name``/``tool_content`` let a test pick a read-only tool or an error - result to verify those do not count as terminal productive work. + ``work`` stands in for the tool calls the run made through the room's + tools, so the turn's ledger sees exactly what those calls did. """ async def stream(): - if tool_result: - event = MagicMock(spec=FunctionToolResultEvent) - event.part = MagicMock() - event.part.tool_name = tool_name - event.part.content = ( - {"id": "msg_1"} if tool_content is None else tool_content - ) - event.tool_call_id = "call_1" - yield event + if work is not None: + await work() raise error + yield # an async generator, like the real event stream @asynccontextmanager async def events() -> AsyncIterator[AsyncIterator]: @@ -1969,24 +1988,25 @@ def test_swallow_matches_the_wording_pydantic_ai_actually_raises(self) -> None: @pytest.mark.asyncio async def test_empty_output_after_tool_is_benign( - self, sample_message, mock_tools, mock_pydantic_agent + self, sample_message, mock_pydantic_agent ): - """Output-retry exhaustion after a tool ran is swallowed.""" + """Output-retry exhaustion after the turn replied is swallowed.""" adapter = PydanticAIAdapter(PydanticAIAdapterConfig(model="openai:gpt-5.4")) with patch.object(adapter, "_create_agent", return_value=mock_pydantic_agent): await adapter.on_started("TestBot", "Test bot") + tools = FakeAgentTools(room_id="room-123") adapter._agent.run_stream_events = MagicMock( return_value=make_raising_stream( UnexpectedModelBehavior("Exceeded maximum output retries (1)"), - tool_result=True, + work=lambda: tools.send_message("hi", ["@Alice"]), ) ) # Must not raise: the reply already went out via the tool this turn. await adapter.on_message( msg=sample_message, - tools=mock_tools, + tools=tools, history=[], participants_msg=None, contacts_msg=None, @@ -2005,11 +2025,11 @@ async def test_empty_output_after_tool_is_benign( isinstance(part, UserPromptPart) and "Hello, agent!" in str(part.content) for part in preserved[-1].parts ) - mock_tools.send_failure.assert_not_awaited() + assert reported_failures(tools) == [] @pytest.mark.asyncio async def test_empty_output_preserves_full_captured_turn( - self, sample_message, mock_tools, mock_pydantic_agent + self, sample_message, mock_pydantic_agent ): """The swallow persists the whole captured turn — not just the user prompt — so a later 'what did you just say?' has the agent's reply in context.""" @@ -2017,11 +2037,12 @@ async def test_empty_output_preserves_full_captured_turn( adapter = PydanticAIAdapter(PydanticAIAdapterConfig(model="openai:gpt-5.4")) with patch.object(adapter, "_create_agent", return_value=mock_pydantic_agent): await adapter.on_started("TestBot", "Test bot") + tools = FakeAgentTools(room_id="room-xyz") adapter._agent.run_stream_events = MagicMock( return_value=make_raising_stream( UnexpectedModelBehavior("Exceeded maximum output retries (1)"), - tool_result=True, + work=lambda: tools.send_message("hi", ["@Alice"]), ) ) @@ -2039,7 +2060,7 @@ def fake_capture(): with patch("band.adapters.pydantic_ai.capture_run_messages", fake_capture): await adapter.on_message( msg=sample_message, - tools=mock_tools, + tools=tools, history=[], participants_msg=None, contacts_msg=None, @@ -2054,36 +2075,13 @@ def fake_capture(): @pytest.mark.asyncio async def test_empty_output_without_tool_propagates( - self, sample_message, mock_tools, mock_pydantic_agent + self, sample_message, mock_pydantic_agent ): """Same error with no tool executed is a real failure — propagate.""" - adapter = PydanticAIAdapter(PydanticAIAdapterConfig(model="openai:gpt-5.4")) - with patch.object(adapter, "_create_agent", return_value=mock_pydantic_agent): - await adapter.on_started("TestBot", "Test bot") - - adapter._agent.run_stream_events = MagicMock( - return_value=make_raising_stream( - UnexpectedModelBehavior("Exceeded maximum output retries (1)"), - tool_result=False, - ) + await self._assert_exhaustion_propagates( + sample_message, mock_pydantic_agent, work=None ) - with pytest.raises(UnexpectedModelBehavior): - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) - - mock_tools.send_failure.assert_awaited_once() - failure = mock_tools.send_failure.call_args.args[0] - assert failure.provider == "pydantic_ai" - assert failure.message == GENERIC_PROVIDER_FAILURE_MESSAGE - @pytest.mark.asyncio async def test_failed_run_still_emits_captured_usage( self, sample_message, mock_tools, mock_pydantic_agent @@ -2103,7 +2101,6 @@ async def test_failed_run_still_emits_captured_usage( adapter._agent.run_stream_events = MagicMock( return_value=make_raising_stream( UnexpectedModelBehavior("Invalid response, unable to find output"), - tool_result=True, ) ) @@ -2143,35 +2140,27 @@ def fake_capture(): @pytest.mark.asyncio async def test_unrelated_model_error_propagates_even_after_tool( - self, sample_message, mock_tools, mock_pydantic_agent + self, sample_message, mock_pydantic_agent ): - """The swallow is narrow: other model errors still surface after a tool.""" + """The swallow is narrow: other model errors still surface after a reply.""" adapter = PydanticAIAdapter(PydanticAIAdapterConfig(model="openai:gpt-5.4")) with patch.object(adapter, "_create_agent", return_value=mock_pydantic_agent): await adapter.on_started("TestBot", "Test bot") + tools = FakeAgentTools(room_id="room-123") adapter._agent.run_stream_events = MagicMock( return_value=make_raising_stream( UnexpectedModelBehavior("Invalid response, unable to find output"), - tool_result=True, + work=lambda: tools.send_message("hi", ["@Alice"]), ) ) with pytest.raises(UnexpectedModelBehavior): - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) + await _run_on_message(adapter, sample_message, tools) - mock_tools.send_failure.assert_awaited_once() - failure = mock_tools.send_failure.call_args.args[0] - assert failure.provider == "pydantic_ai" - assert failure.message == GENERIC_PROVIDER_FAILURE_MESSAGE + assert failure_reports(tools) == [ + ("pydantic_ai", GENERIC_PROVIDER_FAILURE_MESSAGE) + ] @pytest.mark.asyncio async def test_generic_provider_error_reports_and_propagates( @@ -2186,7 +2175,6 @@ async def test_generic_provider_error_reports_and_propagates( adapter._agent.run_stream_events = MagicMock( return_value=make_raising_stream( RuntimeError("provider connection reset"), - tool_result=False, ) ) @@ -2208,68 +2196,94 @@ async def test_generic_provider_error_reports_and_propagates( @pytest.mark.asyncio async def test_empty_output_after_read_only_tool_propagates( - self, sample_message, mock_tools, mock_pydantic_agent + self, sample_message, mock_pydantic_agent ): """A read-only lookup is not terminal work — output-validation exhaustion after only a lookup is a genuine no-response failure and must propagate.""" - adapter = PydanticAIAdapter(PydanticAIAdapterConfig(model="openai:gpt-5.4")) - with patch.object(adapter, "_create_agent", return_value=mock_pydantic_agent): - await adapter.on_started("TestBot", "Test bot") + tools = FakeAgentTools(room_id="room-123") - adapter._agent.run_stream_events = MagicMock( - return_value=make_raising_stream( - UnexpectedModelBehavior("Exceeded maximum output retries (1)"), - tool_result=True, - tool_name="band_lookup_peers", - tool_content=[{"id": "peer_1"}], - ) + await self._assert_exhaustion_propagates( + sample_message, mock_pydantic_agent, tools=tools, work=tools.lookup_peers ) - with pytest.raises(UnexpectedModelBehavior): - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) - @pytest.mark.asyncio async def test_empty_output_after_failed_band_tool_propagates( - self, sample_message, mock_tools, mock_pydantic_agent + self, sample_message, mock_pydantic_agent ): - """A band tool that returned an "Error ..." string did no work — exhausting - output validation afterward is a genuine failure and must propagate.""" + """A band tool that raised did no work — exhausting output validation + afterward is a genuine failure and must propagate.""" + tools = FakeAgentTools(room_id="room-123") + + async def rejected_send() -> None: + # The band tool wrapper turns this raise into an error string for + # the model; the turn records nothing either way. + with contextlib.suppress(BandToolError): + await tools.send_message("hi", mentions=[]) + + await self._assert_exhaustion_propagates( + sample_message, mock_pydantic_agent, tools=tools, work=rejected_send + ) + + @staticmethod + async def _assert_exhaustion_propagates( + sample_message: PlatformMessage, + mock_pydantic_agent: MagicMock, + *, + work: Callable[[], Awaitable[object]] | None, + tools: FakeAgentTools | None = None, + ) -> None: + """Output-retry exhaustion after ``work`` fails the turn as a provider error.""" + tools = tools or FakeAgentTools(room_id="room-123") adapter = PydanticAIAdapter(PydanticAIAdapterConfig(model="openai:gpt-5.4")) with patch.object(adapter, "_create_agent", return_value=mock_pydantic_agent): await adapter.on_started("TestBot", "Test bot") - adapter._agent.run_stream_events = MagicMock( return_value=make_raising_stream( UnexpectedModelBehavior("Exceeded maximum output retries (1)"), - tool_result=True, - tool_name="band_send_message", - tool_content="Error sending message: no mentions", + work=work, ) ) with pytest.raises(UnexpectedModelBehavior): - await adapter.on_message( - msg=sample_message, - tools=mock_tools, - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id="room-123", - ) + await _run_on_message(adapter, sample_message, tools) + + assert failure_reports(tools) == [ + ("pydantic_ai", GENERIC_PROVIDER_FAILURE_MESSAGE) + ] class TestCustomTools: """Tests for custom tool support (PydanticAI-native functions).""" + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("declared", "reported"), [(True, []), (False, [MISSING_REPLY_FAILURE])] + ) + async def test_native_tool_counts_only_the_effect_it_declared( + self, declared: bool, reported: list + ): + """A native tool runs outside the portable custom-tool path, so its + declared effect still has to reach the turn; an undeclared one only + observes.""" + + async def finish(ctx: RunContext[AgentToolsProtocol], note: str) -> str: + """Finish the task.""" + return "done" + + if declared: + declares_turn_effect(TurnEffect.ACT)(finish) + adapter = PydanticAIAdapter( + PydanticAIAdapterConfig(model="test"), additional_tools=[finish] + ) + await adapter.on_started("Probe", "probe") + adapter._agent.model = _streamed_tool_calls(("finish", {"note": "go"})) + tools = FakeAgentTools(room_id=ROOM_ID) + + with contextlib.suppress(TurnResultAlreadyReported): + await adapter.on_event(turn_input(tools)) + + assert failure_reports(tools) == reported + def test_accepts_additional_tools_parameter(self): """Adapter accepts list of callables.""" diff --git a/tests/adapters/test_strands_adapter.py b/tests/adapters/test_strands_adapter.py index e4da40fc6..7bed53e75 100644 --- a/tests/adapters/test_strands_adapter.py +++ b/tests/adapters/test_strands_adapter.py @@ -17,6 +17,7 @@ import pytest from pydantic import BaseModel +from tests.framework_conformance.turnprobes import turn_input from tests.strandskit import text, tool_call, tool_result pytest.importorskip("strands", reason="strands extra not installed") @@ -57,11 +58,13 @@ from band.runtime.custom_tools import declares_turn_effect from band.runtime.tools import TurnEffect, get_tool_description from band.testing import ( + MISSING_REPLY_FAILURE, ErrorTurn, FakeAgentTools, ScriptedStrandsModel, ScriptedTurn, ToolTurn, + failure_reports, reported_failures, ) @@ -134,6 +137,11 @@ async def _run_message( ) +async def _run_turn(adapter: StrandsAdapter, tools: FakeAgentTools) -> None: + """Run one turn through ``on_event``, where the shared turn verdict lives.""" + await adapter.on_event(turn_input(tools, msg=_make_msg(ROOM))) + + def _tool_results(adapter: StrandsAdapter, room_id: str = ROOM) -> list[str]: """The tool outputs the model saw this turn, in order.""" return [ @@ -507,12 +515,7 @@ async def test_later_turns_keep_the_transcript_the_adapter_owns( await _run_message(adapter, tools, history=[]) after_first = list(adapter._message_history[ROOM]) - # The scripted model has no turn left for a second reply, so this - # turn ends without calling band_send_message -- irrelevant to what - # this test checks (the transcript isn't re-seeded), so only the - # failure is asserted here, not suppressed. - with pytest.raises(TurnResultAlreadyReported): - await _run_message(adapter, tools, history=[], is_session_bootstrap=False) + await _run_message(adapter, tools, history=[], is_session_bootstrap=False) assert adapter._message_history[ROOM][: len(after_first)] == after_first @@ -588,13 +591,10 @@ async def send_message(self, content, mentions=None): adapter = await scripted(SEND_TURN) with pytest.raises(TurnResultAlreadyReported): - await _run_message(adapter, tools) + await _run_turn(adapter, tools) assert tools.messages_sent == [] - failures = reported_failures(tools) - assert len(failures) == 1 - assert failures[0]["provider"] == "strands" - assert "band_send_message" in failures[0]["message"] + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] # The shared bridge returns a normalized, model-visible tool failure. assert any( text.startswith("Error executing band_send_message:") @@ -613,8 +613,7 @@ async def send_message(self, content, mentions=None): tools = FailingTools(room_id=ROOM) adapter = await scripted(SEND_TURN, emit=Emit.TOOL_CALLS) - with pytest.raises(TurnResultAlreadyReported): - await _run_message(adapter, tools) + await _run_message(adapter, tools) rehydrated = StrandsHistoryConverter(agent_name="Bot").convert( [ @@ -637,13 +636,46 @@ async def test_read_only_tool_alone_does_not_end_the_turn(self, tools, scripted) adapter = await scripted(ToolTurn("band_lookup_peers", {})) with pytest.raises(TurnResultAlreadyReported): - await _run_message(adapter, tools) + await _run_turn(adapter, tools) assert _tool_results(adapter) # the lookup did run and succeed assert tools.messages_sent == [] - failure = reported_failures(tools)[0] - assert failure["provider"] == "strands" - assert "band_send_message" in failure["message"] + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + + @pytest.mark.asyncio + async def test_declared_native_tool_completes_the_turn(self, tools, scripted): + """A native Strands tool runs outside the portable custom-tool path, so its + declared effect still has to reach the turn.""" + + @declares_turn_effect(TurnEffect.ACT) + @strands_tool + def native_finish(note: str) -> str: + """Finish the task natively.""" + return "done" + + adapter = await scripted( + ToolTurn("native_finish", {"note": "go"}), additional_tools=[native_finish] + ) + + await _run_turn(adapter, tools) + + assert failure_reports(tools) == [] + + @pytest.mark.asyncio + async def test_undeclared_native_tool_only_observes(self, tools, scripted): + @strands_tool + def native_peek(note: str) -> str: + """Peek at something.""" + return "seen" + + adapter = await scripted( + ToolTurn("native_peek", {"note": "go"}), additional_tools=[native_peek] + ) + + with pytest.raises(TurnResultAlreadyReported): + await _run_turn(adapter, tools) + + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] @pytest.mark.asyncio async def test_invalid_tool_arguments_are_answered_not_raised( @@ -652,8 +684,7 @@ async def test_invalid_tool_arguments_are_answered_not_raised( """A malformed call is the model's mistake to correct, not a turn-ending crash.""" adapter = await scripted(ToolTurn("band_send_message", {"mentions": ["@x"]})) - with pytest.raises(TurnResultAlreadyReported): - await _run_message(adapter, tools) + await _run_message(adapter, tools) assert tools.messages_sent == [] assert _tool_results(adapter) == [ @@ -674,8 +705,7 @@ async def boom(args: BoomInput) -> str: ToolTurn("boom", {"note": "go"}), additional_tools=[(BoomInput, boom)] ) - with pytest.raises(TurnResultAlreadyReported): - await _run_message(adapter, tools) + await _run_message(adapter, tools) assert _tool_results(adapter) == ["Error executing tool 'boom': no network"] @@ -875,8 +905,7 @@ async def test_tool_call_event_redacts_content_not_raw_bytes(self, tools): ) await adapter.on_started("Bot", "A bot") - with pytest.raises(TurnResultAlreadyReported): - await _run_message(adapter, tools) + await _run_message(adapter, tools) tool_calls = [ json.loads(e["content"]) diff --git a/tests/core/test_simple_adapter.py b/tests/core/test_simple_adapter.py index 0aeacd9d7..e1b606907 100644 --- a/tests/core/test_simple_adapter.py +++ b/tests/core/test_simple_adapter.py @@ -30,7 +30,8 @@ def convert(self, raw: list[dict[str, Any]]) -> list[str]: class RecordingAdapter(SimpleAdapter[str]): - """Test adapter that records calls for verification.""" + """Test adapter that records calls for verification, then declines the + turn so the verdict stays out of these dispatch tests.""" def __init__(self, *, history_converter: HistoryConverter[str] | None = None): super().__init__(history_converter=history_converter) @@ -59,6 +60,7 @@ async def on_message( "room_id": room_id, } ) + await tools.no_reply() async def on_cleanup(self, room_id: str) -> None: self.cleanup_calls.append(room_id) @@ -162,6 +164,7 @@ async def on_message( room_id, ): seen.append(TRACE_CONTEXT.get()) + await tools.no_reply() adapter = TracingAdapter() assert TRACE_CONTEXT.get() is None @@ -338,6 +341,7 @@ async def on_message( room_id, ): self.received_history = history + await tools.no_reply() adapter = ListAdapter() inp = make_agent_input( @@ -368,6 +372,7 @@ async def on_message( room_id, ): self.received_history = history + await tools.no_reply() adapter = NoConverterAdapter() raw = [{"content": "Test"}] diff --git a/tests/e2e/baseline/README.md b/tests/e2e/baseline/README.md index 8d4060c6e..6c939c67e 100644 --- a/tests/e2e/baseline/README.md +++ b/tests/e2e/baseline/README.md @@ -467,8 +467,10 @@ Each message carries a per-recipient delivery state, exposed as DELIVERED -> PROCESSING -> PROCESSED | FAILED ``` -`FAILED` is **not** terminal — the platform retries (bounded by max retries), so a -message may cycle `FAILED -> PROCESSING` again before reaching `PROCESSED`. +`FAILED` is **not** terminal — the platform's `/next` serves a failed message again, +so a new agent process (or execution context) may cycle it `FAILED -> PROCESSING` +before `PROCESSED`. The running agent skips a message once it has failed +`max_message_retries` times (default 1), so it does not retry the message itself. `PROCESSED` is the only success terminal. ```python notest diff --git a/tests/e2e/baseline/expected-cells.json b/tests/e2e/baseline/expected-cells.json index 428502f99..9d7c144c6 100644 --- a/tests/e2e/baseline/expected-cells.json +++ b/tests/e2e/baseline/expected-cells.json @@ -4341,6 +4341,211 @@ "status": "pass", "reason": null }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_missing_reply_reported.py::test_a_turn_that_did_nothing_is_reported_and_failed", + "adapter": "agno", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_missing_reply_reported.py::test_a_turn_that_did_nothing_is_reported_and_failed", + "adapter": "anthropic", + "lanes": [ + "backends", + "core", + "crewai", + "google", + "letta", + "parlant" + ], + "status": "na", + "reason": "ignores exclude_tools, so band_no_reply stays callable" + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_missing_reply_reported.py::test_a_turn_that_did_nothing_is_reported_and_failed", + "adapter": "claude_sdk", + "lanes": [ + "backends", + "core", + "crewai", + "google", + "letta", + "parlant" + ], + "status": "na", + "reason": "ignores exclude_tools, so band_no_reply stays callable" + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_missing_reply_reported.py::test_a_turn_that_did_nothing_is_reported_and_failed", + "adapter": "copilot_sdk", + "lanes": [ + "backends", + "core", + "crewai", + "google", + "letta", + "parlant" + ], + "status": "na", + "reason": "relays the model's final text, so an empty turn can't be scripted reliably" + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_missing_reply_reported.py::test_a_turn_that_did_nothing_is_reported_and_failed", + "adapter": "crewai", + "lanes": [ + "crewai" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_missing_reply_reported.py::test_a_turn_that_did_nothing_is_reported_and_failed", + "adapter": "gemini", + "lanes": [ + "backends", + "core", + "crewai", + "google", + "letta", + "parlant" + ], + "status": "na", + "reason": "ignores exclude_tools, so band_no_reply stays callable" + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_missing_reply_reported.py::test_a_turn_that_did_nothing_is_reported_and_failed", + "adapter": "google_adk", + "lanes": [ + "backends", + "core", + "crewai", + "google", + "letta", + "parlant" + ], + "status": "na", + "reason": "ignores exclude_tools, so band_no_reply stays callable" + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_missing_reply_reported.py::test_a_turn_that_did_nothing_is_reported_and_failed", + "adapter": "langgraph", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_missing_reply_reported.py::test_a_turn_that_did_nothing_is_reported_and_failed", + "adapter": "pydantic_ai", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_missing_reply_reported.py::test_a_turn_that_did_nothing_is_reported_and_failed", + "adapter": "strands", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_no_reply.py::test_an_fyi_ends_the_turn_through_band_no_reply", + "adapter": "agno", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_no_reply.py::test_an_fyi_ends_the_turn_through_band_no_reply", + "adapter": "anthropic", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_no_reply.py::test_an_fyi_ends_the_turn_through_band_no_reply", + "adapter": "claude_sdk", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_no_reply.py::test_an_fyi_ends_the_turn_through_band_no_reply", + "adapter": "copilot_sdk", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_no_reply.py::test_an_fyi_ends_the_turn_through_band_no_reply", + "adapter": "crewai", + "lanes": [ + "crewai" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_no_reply.py::test_an_fyi_ends_the_turn_through_band_no_reply", + "adapter": "gemini", + "lanes": [ + "google" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_no_reply.py::test_an_fyi_ends_the_turn_through_band_no_reply", + "adapter": "google_adk", + "lanes": [ + "google" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_no_reply.py::test_an_fyi_ends_the_turn_through_band_no_reply", + "adapter": "langgraph", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_no_reply.py::test_an_fyi_ends_the_turn_through_band_no_reply", + "adapter": "pydantic_ai", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/matrix/test_no_reply.py::test_an_fyi_ends_the_turn_through_band_no_reply", + "adapter": "strands", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, { "test": "tests/e2e/baseline/smoke/matrix/test_noisy_room.py::test_recall_and_ignore_crosstalk_in_busy_room", "adapter": "agno", diff --git a/tests/e2e/baseline/smoke/matrix/test_missing_reply_reported.py b/tests/e2e/baseline/smoke/matrix/test_missing_reply_reported.py new file mode 100644 index 000000000..b82c0feb0 --- /dev/null +++ b/tests/e2e/baseline/smoke/matrix/test_missing_reply_reported.py @@ -0,0 +1,82 @@ +"""Matrix scenario: a turn that did nothing is reported once and marked FAILED. + +The agent has no ``band_send_message`` or ``band_no_reply`` and a prompt that +makes it read the roster and stop, so its turn can neither reply nor decline. +The SDK reports the missing reply as exactly one ``error`` event carrying +band-sdk-core's text, and the delivery ends FAILED instead of a false +PROCESSED. +""" + +from __future__ import annotations + +import band_sdk_core +import pytest + +from band.client.streaming import DeliveryStatus +from band.core.types import AdapterFeatures, MessageType +from band.runtime.tools import BandTool +from tests.e2e.baseline.agents import Adapter, ExcludedAdapter, per_adapter +from tests.e2e.baseline.smoke.samples.sample_agents import silent_turn_prompt +from tests.e2e.baseline.toolkit.capture import CaptureFactory +from tests.e2e.baseline.toolkit.provisioning import ProvisionedAgent, ResourceManager +from tests.e2e.baseline.toolkit.user_ops import UserOps + +# The model could still decline through band_no_reply, so the turn can't be +# made silent; the verdict rows in test_turn_outcome.py cover these adapters. +IGNORES_EXCLUDE_TOOLS = "ignores exclude_tools, so band_no_reply stays callable" + + +@per_adapter( + runs_tool_loop=True, + prompt=silent_turn_prompt(), + features=AdapterFeatures(exclude_tools={BandTool.SEND_MESSAGE, BandTool.NO_REPLY}), + exclude=[ + ExcludedAdapter( + Adapter.COPILOT_SDK, + "relays the model's final text, so an empty turn can't be scripted reliably", + ), + *( + ExcludedAdapter(adapter, IGNORES_EXCLUDE_TOOLS) + for adapter in ( + Adapter.ANTHROPIC, + Adapter.CLAUDE_SDK, + Adapter.GEMINI, + Adapter.GOOGLE_ADK, + ) + ), + ], +) +@pytest.mark.asyncio(loop_scope="session") +async def test_a_turn_that_did_nothing_is_reported_and_failed( + agent: ProvisionedAgent, + resource_manager: ResourceManager, + user_ops: UserOps, + reply_capture: CaptureFactory, +) -> None: + room_id = await resource_manager.provision_room( + title=f"e2e-missing-reply-{agent.adapter_id}", participants=[agent.id] + ) + + async with reply_capture(room_id) as capture: + mid = await user_ops.send_message( + room_id, + "Run the silent-turn check.", + mention_id=agent.id, + mention_name=agent.name, + ) + reached = await capture.wait_for_delivery( + mid, agent.id, until={DeliveryStatus.FAILED, DeliveryStatus.PROCESSED} + ) + errors = await capture.errors(sender_id=agent.id) + messages = await capture.events(MessageType.TEXT, sender_id=agent.id) + calls = await capture.tool_calls(sender_id=agent.id) + + assert reached is DeliveryStatus.FAILED, ( + "expected a silent turn, but the agent completed it with " + f"tools {[call.name for call in calls]} and messages " + f"{[message.content for message in messages]}" + ) + assert [error.content for error in errors] == [ + band_sdk_core.missing_reply_message() + ] + messages.assert_none() diff --git a/tests/e2e/baseline/smoke/matrix/test_no_reply.py b/tests/e2e/baseline/smoke/matrix/test_no_reply.py new file mode 100644 index 000000000..2ced8c0a1 --- /dev/null +++ b/tests/e2e/baseline/smoke/matrix/test_no_reply.py @@ -0,0 +1,49 @@ +"""Matrix scenario: a message that needs no answer ends the turn through band_no_reply. + +The user @mentions the agent with an FYI that says no answer is needed. The agent +declines with ``band_no_reply``: the delivery is PROCESSED, the call is recorded +as a ``tool_call`` event, and nothing reaches the room -- no message and no +missing-reply error. +""" + +from __future__ import annotations + +import pytest + +from band.core.types import AdapterFeatures, Emit, MessageType +from band.runtime.tools import BandTool +from tests.e2e.baseline.agents import per_adapter +from tests.e2e.baseline.toolkit.capture import CaptureFactory +from tests.e2e.baseline.toolkit.provisioning import ProvisionedAgent, ResourceManager +from tests.e2e.baseline.toolkit.user_ops import UserOps + +FYI = ( + "FYI only: the nightly deploy finished cleanly. No reply is needed, " + "please don't answer this." +) + + +@per_adapter(runs_tool_loop=True, features=AdapterFeatures(emit={Emit.TOOL_CALLS})) +@pytest.mark.asyncio(loop_scope="session") +async def test_an_fyi_ends_the_turn_through_band_no_reply( + agent: ProvisionedAgent, + resource_manager: ResourceManager, + user_ops: UserOps, + reply_capture: CaptureFactory, +) -> None: + room_id = await resource_manager.provision_room( + title=f"e2e-no-reply-{agent.adapter_id}", participants=[agent.id] + ) + + async with reply_capture(room_id) as capture: + mid = await user_ops.send_message( + room_id, FYI, mention_id=agent.id, mention_name=agent.name + ) + await capture.wait_for_processed(mid, agent.id) + calls = await capture.tool_calls(sender_id=agent.id) + messages = await capture.events(MessageType.TEXT, sender_id=agent.id) + errors = await capture.errors(sender_id=agent.id) + + calls.assert_fired(BandTool.NO_REPLY) + messages.assert_none() + errors.assert_none() diff --git a/tests/e2e/baseline/smoke/samples/approvalroom.py b/tests/e2e/baseline/smoke/samples/approvalroom.py index b1105ba06..b764a4ce6 100644 --- a/tests/e2e/baseline/smoke/samples/approvalroom.py +++ b/tests/e2e/baseline/smoke/samples/approvalroom.py @@ -10,7 +10,6 @@ import pytest from band_rest import ChatMessage -from band.adapters.opencode.adapter import NO_TEXT_REPLY_MESSAGE from band.client.streaming import MessageCreatedPayload from band.core.types import MessageType from tests.e2e.baseline.agents import Adapter @@ -168,14 +167,6 @@ async def closed( pytest.fail( f"Unexpected follow-up approvals: {unexpected_requests}" ) - if self._opencode_missing_text_reply(since): - logger.log( - APPROVAL_LOG_LEVEL, - "Approval no-text fallback adapter=%s requests=%s", - self.adapter_id, - sorted(self.handled_requests), - ) - pytest.fail("OpenCode ended the approval turn without a text reply") logger.log( APPROVAL_LOG_LEVEL, "Approval turn closed adapter=%s requests=%s final_reply_length=%s", @@ -236,15 +227,9 @@ async def _wait_for_reply_or_request( closing_reply=closing_reply, ) or bool(self._unhandled_requests(self.capture.messages.since(since))) - or self._opencode_missing_text_reply(since) ), deadline_s=self.budget.deadline_s, ) - def _opencode_missing_text_reply(self, since: int) -> bool: - return self.adapter_id is Adapter.OPENCODE and any( - NO_TEXT_REPLY_MESSAGE in reply for reply in self.said_since(since) - ) - def said_since(self, since: int) -> list[str]: return [m.content or "" for m in self.capture.messages.since(since)] diff --git a/tests/e2e/baseline/smoke/samples/sample_agents.py b/tests/e2e/baseline/smoke/samples/sample_agents.py index b652deb0d..396326251 100644 --- a/tests/e2e/baseline/smoke/samples/sample_agents.py +++ b/tests/e2e/baseline/smoke/samples/sample_agents.py @@ -31,6 +31,7 @@ ) from band.core.task_types import TaskAssignmentStatus from band.core.types import AdapterFeatures, Capability, Emit, MessageType +from band.runtime.tools import BandTool from tests.e2e.baseline.agents import Adapter, ExcludedAdapter from tests.e2e.baseline.smoke.samples.sample_tools import LOOKUP_PROMPT from tests.e2e.baseline.toolkit.observations import ContactTool, MemoryTool, TaskTool @@ -535,6 +536,18 @@ def custom_prompt_with_marker(marker: str) -> str: ) +def silent_turn_prompt() -> str: + """A prompt that makes the agent end its turn having done nothing: one + read-only call, then stop -- no message and no deliberate decline.""" + return ( + "You are a fixture in an automated test of how a silent turn is reported. " + f"Whatever the message says, call {BandTool.GET_PARTICIPANTS} once, then " + "end your turn without any text. Never call " + f"{BandTool.SEND_MESSAGE} and never call {BandTool.NO_REPLY}. This " + "overrides every other instruction about how to end a turn." + ) + + # Roster probe: drives the agent to state its own name and use its platform tools # (band_get_participants / band_lookup_peers) to report who is present and who is # invitable — the identity + roster read. diff --git a/tests/e2e/baseline/toolkit/capture.py b/tests/e2e/baseline/toolkit/capture.py index 2f75cfaa2..99230cd8a 100644 --- a/tests/e2e/baseline/toolkit/capture.py +++ b/tests/e2e/baseline/toolkit/capture.py @@ -262,8 +262,9 @@ async def wait_for_processed( been captured — that frame is an independent, unordered platform event, so to assert on reply text wait on ``wait_for_reply`` instead. - ``PROCESSED`` is the only success terminal: ``FAILED`` is transient (the - platform retries), so we wait through it rather than giving up. On + ``PROCESSED`` is the only success terminal: ``FAILED`` is not terminal + (``/next`` serves it again, so a new agent context can run it), so we + wait through it rather than giving up. On timeout the error reports the last status seen and any attempt error, so a permanently-failing message is diagnosable instead of opaque. """ diff --git a/tests/e2e/baseline/toolkit/observations/events.py b/tests/e2e/baseline/toolkit/observations/events.py index 296920882..4b777b80a 100644 --- a/tests/e2e/baseline/toolkit/observations/events.py +++ b/tests/e2e/baseline/toolkit/observations/events.py @@ -106,6 +106,15 @@ def assert_present(self, *, what: str | None = None) -> None: ) assert_nonempty(len(self), label=label) + def assert_none(self) -> None: + """Assert no event of this type was emitted (e.g. no error on a clean turn).""" + if self: + label = self.MESSAGE_TYPE.value if self.MESSAGE_TYPE else "matching" + observed = [event.content for event in self] + raise AssertionError( + f"expected no {label} event, but found {len(self)}: {observed}" + ) + class Thoughts(Events): """Captured ``thought`` events.""" diff --git a/tests/framework_conformance/test_approval_commands.py b/tests/framework_conformance/test_approval_commands.py index 8a2da176f..4662193c3 100644 --- a/tests/framework_conformance/test_approval_commands.py +++ b/tests/framework_conformance/test_approval_commands.py @@ -14,7 +14,6 @@ from band.adapters.claude_sdk import ClaudeSDKAdapter from band.adapters.codex import APPROVAL_REQUESTED_TEMPLATE as CODEX_REQUESTED from band.adapters.codex import CodexAdapter -from band.adapters.opencode.adapter import NO_TEXT_REPLY_MESSAGE from band.adapters.opencode.approvals import APPROVAL_REQUESTED_TEMPLATE from band.client.streaming import MessageCreatedPayload from band.integrations.acp.cursor import PERMISSION_REQUESTED_TEMPLATE @@ -139,11 +138,6 @@ def test_approval_closure_requires_reply_after_the_last_request_and_notice() -> notice, closing_reply=closing_reply, ) - assert not dialect.settled( - [*before_close, _message(NO_TEXT_REPLY_MESSAGE)], - notice, - closing_reply=closing_reply, - ) for closing in ( f"@[[agent]] {closing_reply}", f"`{closing_reply}`", diff --git a/tests/framework_conformance/test_strands_injection_spike.py b/tests/framework_conformance/test_strands_injection_spike.py index 240fdcb63..b9cdf7ccc 100644 --- a/tests/framework_conformance/test_strands_injection_spike.py +++ b/tests/framework_conformance/test_strands_injection_spike.py @@ -66,17 +66,15 @@ pytest.importorskip("strands", reason="strands extra not installed") from band.adapters.strands import StrandsAdapter -from band.core.protocols import ( - AgentToolsProtocol, - TurnResultAlreadyReported, -) -from band.core.types import Emit, PlatformMessage +from band.core.protocols import AgentToolsProtocol, TurnResultAlreadyReported +from band.core.types import AgentInput, Emit, HistoryProvider, PlatformMessage from band.testing import ( + MISSING_REPLY_FAILURE, FakeAgentTools, ScriptedStrandsModel, TextTurn, ToolTurn, - reported_failures, + failure_reports, ) _SEND_CONTENT = "Injected reply: PINEAPPLE" @@ -101,16 +99,18 @@ def _make_msg(room_id: str) -> PlatformMessage: async def _run(adapter: StrandsAdapter, tools: FakeAgentTools, room_id: str) -> None: - """Drive the adapter through its real lifecycle: on_started -> on_message.""" + """Drive the adapter through its real lifecycle: on_started -> on_event.""" await adapter.on_started("StrandsSpikeBot", "Tier-1 Strands injection spike bot.") - await adapter.on_message( - msg=_make_msg(room_id), - tools=cast("AgentToolsProtocol", tools), - history=[], - participants_msg=None, - contacts_msg=None, - is_session_bootstrap=True, - room_id=room_id, + await adapter.on_event( + AgentInput( + msg=_make_msg(room_id), + tools=cast("AgentToolsProtocol", tools), + history=HistoryProvider(raw=[]), + participants_msg=None, + contacts_msg=None, + is_session_bootstrap=True, + room_id=room_id, + ) ) @@ -215,10 +215,5 @@ async def test_negative_control_text_only_sends_no_message() -> None: f"expected no send for a text-only decision, got: {tools.messages_sent}" ) assert tools.tool_calls == [] - # The plain-text answer was silently dropped — the adapter must surface it. - failures = reported_failures(tools) - assert len(failures) == 1, ( - f"expected one reported failure, got: {tools.events_sent}" - ) - assert failures[0]["provider"] == "strands" - assert "band_send_message" in failures[0]["message"] + # The plain-text answer was silently dropped — the turn verdict surfaces it. + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] diff --git a/tests/framework_conformance/test_turn_outcome.py b/tests/framework_conformance/test_turn_outcome.py new file mode 100644 index 000000000..4511bb634 --- /dev/null +++ b/tests/framework_conformance/test_turn_outcome.py @@ -0,0 +1,171 @@ +"""Every judged adapter's turn ends with core's verdict, not its own. + +Each probe (see ``turnprobes``) drives one scripted turn through the adapter's +real ``on_event``. A turn that replied, declined, did real work or was settled +by the adapter completes silently; a turn that did nothing is reported once +with core's missing-reply text and raised as ``TurnResultAlreadyReported``, so +the runtime marks it FAILED. The relaying adapters post the model's final text +unless a tool already replied; the rest discard it. + +To add an adapter: write its probe in ``turnprobes/.py`` and register it +in ``TURN_OUTCOME_PROBES``. ``test_every_judged_adapter_has_a_probe`` fails +when a judged adapter has none. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum + +import pytest + +from band.core.protocols import TurnResultAlreadyReported +from band.runtime.tools import BandTool +from band.testing import MISSING_REPLY_FAILURE, failure_reports +from band.testing.fake_tools import FakeAgentTools, reported_failures +from tests.framework_configs.adapters import ADAPTER_CONFIGS +from tests.framework_conformance.turnprobes import ( + ToolCall, + TurnOutcomeProbe, + TurnScript, + acp, + claudesdk, + codex, + copilotsdk, + crewai, + letta, + opencode, + toolloop, + turn_tools, +) + +TURN_OUTCOME_PROBES: dict[str, TurnOutcomeProbe] = { + **acp.PROBES, + **claudesdk.PROBES, + **codex.PROBES, + **copilotsdk.PROBES, + **crewai.PROBES, + **letta.PROBES, + **opencode.PROBES, + **toolloop.PROBES, +} + +# Adapters that post the model's final text when no tool replied. +RELAYING_FRAMEWORK_IDS = frozenset( + {"codex", "copilot_sdk", "opencode", "letta", "acp", "cursor_acp"} +) + +ANSWER = "The vault code is 4471-ECHO." +REPLY = ToolCall(BandTool.SEND_MESSAGE, {"content": ANSWER, "mentions": ["@alice"]}) + + +class Verdict(StrEnum): + COMPLETE = "complete" + MISSING_REPLY = "missing_reply" + + +@dataclass(frozen=True) +class TurnResult: + verdict: Verdict + messages: list[str] + failures: list[tuple[str, str]] + + +async def run_turn(probe: TurnOutcomeProbe, script: TurnScript) -> TurnResult: + tools = turn_tools() + try: + await probe.run(script, tools) + verdict = Verdict.COMPLETE + except TurnResultAlreadyReported: + verdict = Verdict.MISSING_REPLY + return observed(tools, verdict) + + +def observed(tools: FakeAgentTools, verdict: Verdict) -> TurnResult: + return TurnResult( + verdict=verdict, + messages=[m["content"] for m in tools.messages_sent], + failures=failure_reports(tools), + ) + + +COMPLETING_SCRIPTS = { + "decline": TurnScript((ToolCall(BandTool.NO_REPLY, {"reason": "FYI only"}),)), + "reply": TurnScript((REPLY,)), + "act": TurnScript((ToolCall(BandTool.CREATE_CHATROOM),)), +} + + +def probe_params(framework_ids: frozenset[str] | None = None) -> list[str]: + return sorted( + name + for name in TURN_OUTCOME_PROBES + if framework_ids is None or name in framework_ids + ) + + +@pytest.mark.parametrize("framework_id", probe_params()) +@pytest.mark.parametrize("script_name", sorted(COMPLETING_SCRIPTS)) +async def test_a_turn_that_answered_or_worked_completes( + framework_id: str, script_name: str +) -> None: + result = await run_turn( + TURN_OUTCOME_PROBES[framework_id], COMPLETING_SCRIPTS[script_name] + ) + + assert result.verdict is Verdict.COMPLETE + assert result.failures == [] + + +@pytest.mark.parametrize("framework_id", probe_params()) +async def test_a_turn_that_did_nothing_is_reported_once(framework_id: str) -> None: + result = await run_turn(TURN_OUTCOME_PROBES[framework_id], TurnScript()) + + assert result.verdict is Verdict.MISSING_REPLY + assert result.failures == [MISSING_REPLY_FAILURE] + assert result.messages == [] + + +@pytest.mark.parametrize( + "framework_id", + [name for name in probe_params() if TURN_OUTCOME_PROBES[name].settle], +) +async def test_an_adapter_settled_turn_completes(framework_id: str) -> None: + settle = TURN_OUTCOME_PROBES[framework_id].settle + assert settle is not None + tools = turn_tools() + + await settle(tools) + + assert tools.turn.complete + assert reported_failures(tools) == [] + + +@pytest.mark.parametrize("framework_id", probe_params(RELAYING_FRAMEWORK_IDS)) +async def test_a_relaying_adapter_posts_the_final_text_once(framework_id: str) -> None: + result = await run_turn( + TURN_OUTCOME_PROBES[framework_id], TurnScript(final_text=ANSWER) + ) + + assert result == TurnResult(Verdict.COMPLETE, messages=[ANSWER], failures=[]) + + +@pytest.mark.parametrize("framework_id", probe_params(RELAYING_FRAMEWORK_IDS)) +async def test_a_tool_reply_suppresses_the_closing_text(framework_id: str) -> None: + result = await run_turn( + TURN_OUTCOME_PROBES[framework_id], + TurnScript((REPLY,), final_text="Anything else?"), + ) + + assert result == TurnResult(Verdict.COMPLETE, messages=[ANSWER], failures=[]) + + +def test_every_judged_adapter_has_a_probe() -> None: + unprobed = sorted( + cfg.framework_id + for cfg in ADAPTER_CONFIGS + if cfg.adapter_factory().judges_turns + and cfg.framework_id not in TURN_OUTCOME_PROBES + ) + + assert unprobed == [] diff --git a/tests/framework_conformance/turnprobes/__init__.py b/tests/framework_conformance/turnprobes/__init__.py new file mode 100644 index 000000000..d7a3ea7b2 --- /dev/null +++ b/tests/framework_conformance/turnprobes/__init__.py @@ -0,0 +1,102 @@ +"""The shape every turn-outcome probe shares. + +A probe runs one turn through an adapter's real ``on_event`` with that +adapter's existing scripted model following a ``TurnScript``, against the +``FakeAgentTools`` it is handed. It lets the verdict's exception propagate, so +``test_turn_outcome`` sees exactly what the runtime would. + +Framework imports stay inside each probe function: not every lane's venv has +every framework installed. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Any + +from band.core.types import AgentInput, HistoryProvider, PlatformMessage +from band.runtime.tools import TOOL_DEFINITIONS, ToolCallOutcome, serialize_tool_result +from band.testing.fake_tools import FakeAgentTools + +ROOM_ID = "room-turn" +ALICE: dict[str, Any] = { + "id": "user-alice", + "name": "Alice", + "type": "User", + "handle": "alice", + "role": "member", + "status": "active", +} + + +@dataclass(frozen=True) +class ToolCall: + name: str + arguments: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class TurnScript: + """What the scripted model does in one turn: these Band tool calls in + order, then this final text (empty for a tool-only ending).""" + + tool_calls: tuple[ToolCall, ...] = () + final_text: str = "" + + +@dataclass(frozen=True) +class TurnOutcomeProbe: + """``run`` drives one scripted turn. ``settle``, for an adapter with a + control path, sends a message the adapter answers itself (a status or + busy reply) without running the model.""" + + run: Callable[[TurnScript, FakeAgentTools], Awaitable[None]] + settle: Callable[[FakeAgentTools], Awaitable[None]] | None = None + + +class DispatchingFakeTools(FakeAgentTools): + """Runs each dispatched Band tool through the fake's own method, as + ``AgentTools`` does, so a scripted ``band_send_message`` really posts.""" + + async def execute_tool_call_structured( + self, tool_name: str, arguments: dict[str, Any] + ) -> ToolCallOutcome: + self.tool_calls.append({"tool_name": tool_name, "arguments": arguments}) + method = getattr(self, TOOL_DEFINITIONS[tool_name].method_name) + return ToolCallOutcome( + value=serialize_tool_result(await method(**arguments)), ok=True + ) + + +def turn_tools() -> FakeAgentTools: + return DispatchingFakeTools(room_id=ROOM_ID, participants=[ALICE]) + + +def user_message(content: str = "@agent what is the vault code?") -> PlatformMessage: + return PlatformMessage( + id="msg-turn", + room_id=ROOM_ID, + content=content, + sender_id=ALICE["id"], + sender_type=ALICE["type"], + sender_name=ALICE["name"], + message_type="text", + metadata={}, + created_at=datetime.now(UTC), + ) + + +def turn_input(tools: FakeAgentTools, msg: PlatformMessage | None = None) -> AgentInput: + """The ``AgentInput`` the runtime hands ``on_event`` for one user message.""" + msg = msg or user_message() + return AgentInput( + msg=msg, + tools=tools, + history=HistoryProvider(raw=[]), + participants_msg=None, + contacts_msg=None, + is_session_bootstrap=True, + room_id=msg.room_id, + ) diff --git a/tests/framework_conformance/turnprobes/acp.py b/tests/framework_conformance/turnprobes/acp.py new file mode 100644 index 000000000..50d520c39 --- /dev/null +++ b/tests/framework_conformance/turnprobes/acp.py @@ -0,0 +1,92 @@ +"""Turn-outcome probes for the ACP client adapters; see ``turnprobes``. + +The scripted ACP agent calls each Band tool over the adapter's injected +loopback MCP server, so every call runs through the real tool dispatch into +the turn's tools. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from band.runtime.tools import CHAT_ID_FIELD_NAME +from band.testing.fake_tools import FakeAgentTools +from tests.framework_conformance.turnprobes import ( + ROOM_ID, + TurnOutcomeProbe, + TurnScript, + turn_input, + user_message, +) + +if TYPE_CHECKING: + from band.core.types import AgentInput + from band.integrations.acp.client_adapter import ACPClientAdapter + from tests.integrations.acp.acp_toolkit import FakeACPAgent + + +def _scripted_agent(script: TurnScript) -> FakeACPAgent: + from tests.integrations.acp.acp_toolkit import ( # noqa: PLC0415 -- the ACP toolkit imports the acp (agent-client-protocol) extra at its own top level; not installed in every lane's venv + FakeACPAgent, + ) + + agent = FakeACPAgent() + for index, call in enumerate(script.tool_calls): + agent.will_call_mcp_tool( + f"tc-{index}", + call.name, + arguments={CHAT_ID_FIELD_NAME: ROOM_ID, **call.arguments}, + ) + if script.final_text: + agent.will_say(script.final_text) + return agent + + +async def _drive( + adapter: ACPClientAdapter, agent: FakeACPAgent, inp: AgentInput +) -> None: + from tests.integrations.acp.acp_toolkit import ( # noqa: PLC0415 -- the ACP toolkit imports the acp (agent-client-protocol) extra at its own top level; not installed in every lane's venv + started_acp_adapter, + ) + + async with started_acp_adapter(adapter, agent): + await adapter.on_event(inp) + + +async def run_acp(script: TurnScript, tools: FakeAgentTools) -> None: + from band.integrations.acp.client_adapter import ( # noqa: PLC0415 -- client_adapter imports the acp (agent-client-protocol) extra at its own top level; not installed in every lane's venv + ACPClientAdapter, + ) + from tests.integrations.acp.acp_toolkit import ( # noqa: PLC0415 -- the ACP toolkit imports the acp (agent-client-protocol) extra at its own top level; not installed in every lane's venv + fake_agent_config, + ) + + adapter = ACPClientAdapter(fake_agent_config(inject_band_tools=True)) + await _drive(adapter, _scripted_agent(script), turn_input(tools)) + + +def _cursor_adapter() -> ACPClientAdapter: + from band.adapters.cursor_acp import ( # noqa: PLC0415 -- cursor_acp imports the acp (agent-client-protocol) extra at its own top level; not installed in every lane's venv + CursorACPAdapter, + ) + + return CursorACPAdapter() + + +async def run_cursor_acp(script: TurnScript, tools: FakeAgentTools) -> None: + await _drive(_cursor_adapter(), _scripted_agent(script), turn_input(tools)) + + +async def settle_cursor_acp(tools: FakeAgentTools) -> None: + """``/cursor decisions`` is answered by the adapter, never the agent.""" + await _drive( + _cursor_adapter(), + _scripted_agent(TurnScript()), + turn_input(tools, user_message("/cursor decisions")), + ) + + +PROBES: dict[str, TurnOutcomeProbe] = { + "acp": TurnOutcomeProbe(run=run_acp), + "cursor_acp": TurnOutcomeProbe(run=run_cursor_acp, settle=settle_cursor_acp), +} diff --git a/tests/framework_conformance/turnprobes/claudesdk.py b/tests/framework_conformance/turnprobes/claudesdk.py new file mode 100644 index 000000000..35e4d3353 --- /dev/null +++ b/tests/framework_conformance/turnprobes/claudesdk.py @@ -0,0 +1,80 @@ +"""Turn-outcome probes for the claude_sdk adapter; see ``turnprobes``. + +The scripted Claude CLI (``tests/adapters/claude_sdk/fakecli.py``) makes each +Band tool call through the adapter's real in-process MCP server. +""" + +from __future__ import annotations + +from unittest.mock import patch + +from band.runtime.tools import MCP_TOOL_PREFIX +from band.testing.fake_tools import FakeAgentTools +from tests.baseline.decisions import ModelDecision +from tests.framework_conformance.turnprobes import ( + ROOM_ID, + TurnOutcomeProbe, + TurnScript, + turn_input, + user_message, +) + + +def scripted_turn(script: TurnScript) -> list[ModelDecision]: + """The CLI turn following ``script``: each Band call, then the final text.""" + decisions = [ + ModelDecision.call( + f"{MCP_TOOL_PREFIX}{call.name}", chat_id=ROOM_ID, **call.arguments + ) + for call in script.tool_calls + ] + if script.final_text: + decisions.append(ModelDecision.text_reply(script.final_text)) + return decisions + + +async def run_claude_sdk(script: TurnScript, tools: FakeAgentTools) -> None: + from band.adapters.claude_sdk import ( # noqa: PLC0415 -- claude_sdk extra, absent from the standard dev-crewai/dev-parlant lane venvs + ClaudeSDKAdapter, + ) + from tests.adapters.claude_sdk.fakecli import ( # noqa: PLC0415 -- claude_sdk extra, absent from the standard dev-crewai/dev-parlant lane venvs + FakeClaude, + ) + + claude = FakeClaude() + claude.script(scripted_turn(script)) + adapter = ClaudeSDKAdapter() + with patch( + "band.integrations.claude_sdk.session_manager.ClaudeSDKClient", claude.client + ): + await adapter.on_started("Agent", "An agent under test") + try: + await adapter.on_event(turn_input(tools)) + finally: + await adapter.cleanup_all() + claude.assert_done() + + +async def ask_claude_sdk_status(tools: FakeAgentTools) -> None: + from band.adapters.claude_sdk import ( # noqa: PLC0415 -- claude_sdk extra, absent from the standard dev-crewai/dev-parlant lane venvs + ClaudeApprovalOptions, + ClaudeSDKAdapter, + ClaudeSDKAdapterConfig, + ClaudeSDKCommand, + ) + + adapter = ClaudeSDKAdapter( + ClaudeSDKAdapterConfig(approvals=ClaudeApprovalOptions(mode="manual")) + ) + await adapter.on_started("Agent", "An agent under test") + try: + await adapter.on_event( + turn_input(tools, user_message(f"/{ClaudeSDKCommand.STATUS}")) + ) + finally: + await adapter.cleanup_all() + + +PROBES: dict[str, TurnOutcomeProbe] = { + "claude_sdk": TurnOutcomeProbe(run=run_claude_sdk, settle=ask_claude_sdk_status), +} diff --git a/tests/framework_conformance/turnprobes/codex.py b/tests/framework_conformance/turnprobes/codex.py new file mode 100644 index 000000000..cfc6e9a64 --- /dev/null +++ b/tests/framework_conformance/turnprobes/codex.py @@ -0,0 +1,52 @@ +"""Turn-outcome probes for the codex adapter(s); see ``turnprobes``.""" + +from __future__ import annotations + +from band.adapters.codex import CodexAdapter, CodexCommand +from band.integrations.codex import RpcEvent +from band.testing.fake_tools import FakeAgentTools +from tests.adapters.codexturns import ( + FakeCodexClient, + final_text, + make_codex_adapter, + tool_call_request, + turn_completed, +) +from tests.framework_conformance.turnprobes import ( + TurnOutcomeProbe, + TurnScript, + turn_input, + user_message, +) + + +def scripted_events(script: TurnScript) -> list[RpcEvent]: + """The app-server events of one Codex turn following ``script``.""" + events = [ + tool_call_request(request_id, call.name, call.arguments) + for request_id, call in enumerate(script.tool_calls, start=1) + ] + if script.final_text: + events.append(final_text(script.final_text)) + return [*events, turn_completed()] + + +async def started_adapter(*events: RpcEvent) -> CodexAdapter: + adapter = make_codex_adapter(FakeCodexClient(events=list(events))) + await adapter.on_started("Agent", "A coding agent") + return adapter + + +async def run_codex(script: TurnScript, tools: FakeAgentTools) -> None: + adapter = await started_adapter(*scripted_events(script)) + await adapter.on_event(turn_input(tools)) + + +async def ask_codex_status(tools: FakeAgentTools) -> None: + adapter = await started_adapter() + await adapter.on_event(turn_input(tools, user_message(f"/{CodexCommand.STATUS}"))) + + +PROBES: dict[str, TurnOutcomeProbe] = { + "codex": TurnOutcomeProbe(run=run_codex, settle=ask_codex_status), +} diff --git a/tests/framework_conformance/turnprobes/copilotsdk.py b/tests/framework_conformance/turnprobes/copilotsdk.py new file mode 100644 index 000000000..6a7c8ef58 --- /dev/null +++ b/tests/framework_conformance/turnprobes/copilotsdk.py @@ -0,0 +1,49 @@ +"""Turn-outcome probes for the copilotsdk adapter(s); see ``turnprobes``.""" + +from __future__ import annotations + +from band.adapters.copilot_sdk import _COPILOT_SDK_AVAILABLE +from band.testing.fake_tools import FakeAgentTools +from tests.adapters.copilot_sdk.fakes import ( + FakeCopilotClient, + FakeCopilotSession, + make_started_adapter, +) +from tests.framework_conformance.turnprobes import ( + ROOM_ID, + TurnOutcomeProbe, + TurnScript, + turn_input, +) + + +async def run_copilot_sdk_turn(script: TurnScript, tools: FakeAgentTools) -> None: + from copilot import ( # noqa: PLC0415 -- copilot_sdk extra, not in every lane's venv + ToolInvocation, + ) + + async def perform_tool_calls(session: FakeCopilotSession) -> None: + # The fake tools advertise no schemas, so the scripted model calls the + # adapter's own bridged-tool handler, which every Band tool shares. + handler = adapter._make_tool_handler(ROOM_ID) + for index, call in enumerate(script.tool_calls): + await handler( + ToolInvocation( + tool_call_id=f"call-{index}", + tool_name=call.name, + arguments=call.arguments, + ) + ) + + client = FakeCopilotClient( + reply_content=script.final_text or None, turn_events=[perform_tool_calls] + ) + adapter = await make_started_adapter(client) + await adapter.on_event(turn_input(tools)) + + +PROBES: dict[str, TurnOutcomeProbe] = ( + {"copilot_sdk": TurnOutcomeProbe(run=run_copilot_sdk_turn)} + if _COPILOT_SDK_AVAILABLE + else {} +) diff --git a/tests/framework_conformance/turnprobes/crewai.py b/tests/framework_conformance/turnprobes/crewai.py new file mode 100644 index 000000000..da2172722 --- /dev/null +++ b/tests/framework_conformance/turnprobes/crewai.py @@ -0,0 +1,54 @@ +"""Turn-outcome probes for the crewai adapter(s); see ``turnprobes``. + +crewai_flow is exempt from judging (``judges_turns`` is false), so it has no +probe. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +import pytest + +from band.adapters.crewai import EMPTY_LLM_RESPONSE_MARKER, CrewAIAdapter +from band.testing.fake_tools import FakeAgentTools +from tests.framework_conformance.turnprobes import ( + TurnOutcomeProbe, + TurnScript, + turn_input, +) + + +class ScriptedCrewAgent: + """Stands in for the crew's LLM loop: runs the script's tool calls through + the crew's real BaseTools, then ends the way CrewAI does -- the final text + as ``raw``, or CrewAI's empty-completion error for a tool-only ending.""" + + def __init__(self, script: TurnScript, crew_tools: list[Any]) -> None: + self._script = script + self._tools_by_name = {tool.name: tool for tool in crew_tools} + + async def kickoff_async(self, _prompt: str) -> SimpleNamespace: + # CrewAI runs its synchronous tools off the event loop thread. + await asyncio.to_thread(self._call_tools) + if not self._script.final_text: + raise ValueError(f"{EMPTY_LLM_RESPONSE_MARKER} - None or empty.") + return SimpleNamespace(raw=self._script.final_text) + + def _call_tools(self) -> None: + for call in self._script.tool_calls: + self._tools_by_name[call.name].run(**call.arguments) + + +async def run_crewai(script: TurnScript, tools: FakeAgentTools) -> None: + pytest.importorskip("crewai") + adapter = CrewAIAdapter() + await adapter.on_started("Agent", "A test agent") + assert adapter._crewai_agent is not None + adapter._crewai_agent = ScriptedCrewAgent(script, adapter._crewai_agent.tools) + await adapter.on_event(turn_input(tools)) + + +PROBES: dict[str, TurnOutcomeProbe] = {"crewai": TurnOutcomeProbe(run=run_crewai)} diff --git a/tests/framework_conformance/turnprobes/letta.py b/tests/framework_conformance/turnprobes/letta.py new file mode 100644 index 000000000..87bb0da85 --- /dev/null +++ b/tests/framework_conformance/turnprobes/letta.py @@ -0,0 +1,30 @@ +"""Turn-outcome probes for the letta adapter(s); see ``turnprobes``.""" + +from __future__ import annotations + +from band.testing.fake_tools import FakeAgentTools +from tests.framework_conformance.turnprobes import ( + ROOM_ID, + TurnOutcomeProbe, + TurnScript, + turn_input, +) + + +async def run_letta_turn(script: TurnScript, tools: FakeAgentTools) -> None: + from tests.adapters.lettakit import ( # noqa: PLC0415 -- letta extra, absent from some lane venvs + ready_letta_adapter, + scripted_letta_turn, + ) + + adapter, client = ready_letta_adapter(room_id=ROOM_ID) + client.agents.messages.create.side_effect = scripted_letta_turn( + adapter, + room_id=ROOM_ID, + tool_calls=[(call.name, call.arguments) for call in script.tool_calls], + final_text=script.final_text, + ) + await adapter.on_event(turn_input(tools)) + + +PROBES: dict[str, TurnOutcomeProbe] = {"letta": TurnOutcomeProbe(run=run_letta_turn)} diff --git a/tests/framework_conformance/turnprobes/opencode.py b/tests/framework_conformance/turnprobes/opencode.py new file mode 100644 index 000000000..0f10b375b --- /dev/null +++ b/tests/framework_conformance/turnprobes/opencode.py @@ -0,0 +1,102 @@ +"""Turn-outcome probes for the opencode adapter(s); see ``turnprobes``.""" + +from __future__ import annotations + +import asyncio +import contextlib +from collections.abc import AsyncIterator +from typing import TYPE_CHECKING +from unittest.mock import patch + +from band.adapters.opencode import OpencodeAdapter +from band.testing.fake_tools import FakeAgentTools +from tests.framework_conformance.turnprobes import ( + ROOM_ID, + TurnOutcomeProbe, + TurnScript, + turn_input, + turn_tools, + user_message, +) + +if TYPE_CHECKING: + from tests.adapters.opencode.helpers import FakeMCPBackend, FakeOpencodeClient + +SESSION_ID = "sess-1" +MESSAGE_ID = "msg-1" + + +@contextlib.asynccontextmanager +async def started_adapter( + client: FakeOpencodeClient, backend: FakeMCPBackend +) -> AsyncIterator[OpencodeAdapter]: + from tests.adapters.opencode.helpers import ( # noqa: PLC0415 -- the helpers import the mcp package (opencode extra) at their own top level; not installed in every lane's venv + make_fake_mcp_backend_factory, + ) + + with patch( + "band.adapters.opencode.adapter.create_band_mcp_backend", + make_fake_mcp_backend_factory(backend), + ): + adapter = OpencodeAdapter(client_factory=lambda _config: client) + await adapter.on_started("OpenCode Agent", "A coding agent") + try: + yield adapter + finally: + await adapter.on_cleanup(ROOM_ID) + + +async def run_opencode(script: TurnScript, tools: FakeAgentTools) -> None: + from tests.adapters.opencode.helpers import ( # noqa: PLC0415 -- the helpers import the mcp package (opencode extra) at their own top level; not installed in every lane's venv + FakeMCPBackend, + FakeOpencodeClient, + event_message_updated, + event_session_idle, + event_text_part, + ) + + backend = FakeMCPBackend() + steps = [ + event_message_updated(SESSION_ID, MESSAGE_ID), + *( + backend.band_tool_call(call.name, call.arguments, room_id=ROOM_ID) + for call in script.tool_calls + ), + ] + if script.final_text: + steps.append(event_text_part(SESSION_ID, MESSAGE_ID, script.final_text)) + steps.append(event_session_idle(SESSION_ID)) + client = FakeOpencodeClient(prompt_event_sequences=[steps]) + + async with started_adapter(client, backend) as adapter: + await adapter.on_event(turn_input(tools)) + + +async def settle_opencode(tools: FakeAgentTools) -> None: + """A message arriving mid-turn gets only the busy notice.""" + from tests.adapters.opencode.helpers import ( # noqa: PLC0415 -- the helpers import the mcp package (opencode extra) at their own top level; not installed in every lane's venv + FakeMCPBackend, + FakeOpencodeClient, + event_message_updated, + wait_for, + ) + + client = FakeOpencodeClient( + prompt_event_sequences=[[event_message_updated(SESSION_ID, MESSAGE_ID)]] + ) + async with started_adapter(client, FakeMCPBackend()) as adapter: + running = asyncio.create_task(adapter.on_event(turn_input(turn_tools()))) + try: + await wait_for(lambda: bool(client.prompt_calls)) + await adapter.on_event( + turn_input(tools, user_message("@agent are you done?")) + ) + finally: + running.cancel() + with contextlib.suppress(asyncio.CancelledError): + await running + + +PROBES: dict[str, TurnOutcomeProbe] = { + "opencode": TurnOutcomeProbe(run=run_opencode, settle=settle_opencode), +} diff --git a/tests/framework_conformance/turnprobes/toolloop.py b/tests/framework_conformance/turnprobes/toolloop.py new file mode 100644 index 000000000..3739c1bf6 --- /dev/null +++ b/tests/framework_conformance/turnprobes/toolloop.py @@ -0,0 +1,258 @@ +"""Turn-outcome probes for the in-process tool-loop adapters; see ``turnprobes``. + +Each runs the script through the adapter's own Band tool surface on its +existing scripted model. None of these adapters relay the model's final text. +""" + +from __future__ import annotations + +import json +from collections.abc import AsyncIterator +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from band.testing.fake_tools import FakeAgentTools +from tests.framework_conformance.turnprobes import ( + TurnOutcomeProbe, + TurnScript, + turn_input, +) + + +async def run_strands(script: TurnScript, tools: FakeAgentTools) -> None: + pytest.importorskip("strands") + from band.adapters.strands import ( # noqa: PLC0415 -- strands extra, absent from some lane venvs + StrandsAdapter, + ) + from band.testing import ( # noqa: PLC0415 -- strands extra, absent from some lane venvs + ScriptedStrandsModel, + TextTurn, + ToolTurn, + ) + + turns: list[Any] = [ + ToolTurn(call.name, call.arguments) for call in script.tool_calls + ] + if script.final_text: + turns.append(TextTurn(script.final_text)) + adapter = StrandsAdapter(llm=ScriptedStrandsModel(turns)) + await adapter.on_started("Agent", "A test agent") + await adapter.on_event(turn_input(tools)) + + +async def run_pydantic_ai(script: TurnScript, tools: FakeAgentTools) -> None: + pytest.importorskip("pydantic_ai") + from pydantic_ai.models.function import ( # noqa: PLC0415 -- pydantic_ai extra, absent from some lane venvs + AgentInfo, + DeltaToolCall, + FunctionModel, + ) + + from band.adapters.pydantic_ai import ( # noqa: PLC0415 -- pydantic_ai extra, absent from some lane venvs + PydanticAIAdapter, + PydanticAIAdapterConfig, + ) + + pending = list(script.tool_calls) + + async def stream( + messages: list[Any], info: AgentInfo + ) -> AsyncIterator[str | dict[int, DeltaToolCall]]: + if pending: + call = pending.pop(0) + yield { + 0: DeltaToolCall(name=call.name, json_args=json.dumps(call.arguments)) + } + else: + yield script.final_text + + adapter = PydanticAIAdapter(PydanticAIAdapterConfig(model="test")) + await adapter.on_started("Agent", "A test agent") + assert adapter._agent is not None + with adapter._agent.override(model=FunctionModel(stream_function=stream)): + await adapter.on_event(turn_input(tools)) + + +async def run_agno(script: TurnScript, tools: FakeAgentTools) -> None: + pytest.importorskip("agno") + from agno.run.agent import ( # noqa: PLC0415 -- agno extra, absent from some lane venvs + RunOutput, + ) + + from band.adapters.agno import ( # noqa: PLC0415 -- agno extra, absent from some lane venvs + AgnoAdapter, + ) + from tests.adapters.agno.helpers import ( # noqa: PLC0415 -- agno extra, absent from some lane venvs + fake_agno_agent, + ) + + async def model_turn() -> None: + # The adapter's Agno entrypoint for a Band tool is exactly this call + # on the room's bound tools. + for call in script.tool_calls: + await tools.execute_tool_call(call.name, call.arguments) + + agent = fake_agno_agent( + response=RunOutput(content=script.final_text), model_turn=model_turn + ) + adapter = AgnoAdapter(agent=agent, emit=()) + await adapter.on_started("Agent", "A test agent") + await adapter.on_event(turn_input(tools)) + + +class ScriptedGraph: + """A graph whose run calls the script's Band tools, as an agent graph would.""" + + def __init__(self, script: TurnScript, turn_tools: list[Any]) -> None: + self._script = script + self._tools_by_name = {tool.name: tool for tool in turn_tools} + + async def astream_events( + self, graph_input: dict[str, Any], **kwargs: Any + ) -> AsyncIterator[dict[str, Any]]: + for call in self._script.tool_calls: + await self._tools_by_name[call.name].ainvoke(call.arguments) + return + yield + + +async def run_langgraph(script: TurnScript, tools: FakeAgentTools) -> None: + pytest.importorskip("langgraph") + from band.adapters.langgraph import ( # noqa: PLC0415 -- langgraph extra, absent from some lane venvs + LangGraphAdapter, + ) + + adapter = LangGraphAdapter( + graph_factory=lambda turn_tools: ScriptedGraph(script, turn_tools) + ) + await adapter.on_started("Agent", "A test agent") + await adapter.on_event(turn_input(tools)) + + +async def run_anthropic(script: TurnScript, tools: FakeAgentTools) -> None: + pytest.importorskip("anthropic") + from anthropic.types import ( # noqa: PLC0415 -- anthropic extra, absent from some lane venvs + Message, + TextBlock, + ToolUseBlock, + Usage, + ) + + from band.adapters.anthropic import ( # noqa: PLC0415 -- anthropic extra, absent from some lane venvs + AnthropicAdapter, + ) + + def response(content: list[Any], stop_reason: str) -> Message: + return Message( + id="msg", + type="message", + role="assistant", + model="scripted", + content=content, + stop_reason=stop_reason, + usage=Usage(input_tokens=0, output_tokens=0), + ) + + responses = [ + response( + [ + ToolUseBlock( + type="tool_use", + id=f"tool-{index}", + name=call.name, + input=call.arguments, + ) + ], + "tool_use", + ) + for index, call in enumerate(script.tool_calls) + ] + final = ( + [TextBlock(type="text", text=script.final_text)] if script.final_text else [] + ) + responses.append(response(final, "end_turn")) + + adapter = AnthropicAdapter() + await adapter.on_started("Agent", "A test agent") + with patch.object(adapter, "_call_anthropic", AsyncMock(side_effect=responses)): + await adapter.on_event(turn_input(tools)) + + +async def run_gemini(script: TurnScript, tools: FakeAgentTools) -> None: + pytest.importorskip("google.genai") + from google.genai import ( # noqa: PLC0415 -- gemini extra, absent from some lane venvs + types, + ) + + from band.adapters.gemini import ( # noqa: PLC0415 -- gemini extra, absent from some lane venvs + GeminiAdapter, + ) + + def response(part: types.Part) -> types.GenerateContentResponse: + return types.GenerateContentResponse( + candidates=[ + types.Candidate(content=types.Content(role="model", parts=[part])) + ] + ) + + responses = [ + response(types.Part.from_function_call(name=call.name, args=call.arguments)) + for call in script.tool_calls + ] + responses.append(response(types.Part.from_text(text=script.final_text))) + + adapter = GeminiAdapter() + await adapter.on_started("Agent", "A test agent") + with patch.object(adapter, "_call_gemini", AsyncMock(side_effect=responses)): + await adapter.on_event(turn_input(tools)) + + +async def run_google_adk(script: TurnScript, tools: FakeAgentTools) -> None: + pytest.importorskip("google.adk") + from band.adapters.google_adk import ( # noqa: PLC0415 -- google_adk extra, absent from some lane venvs + GoogleADKAdapter, + _get_tool_bridge_class, + ) + + tool_bridge = _get_tool_bridge_class() + + def scripted_runner(turn_tools: Any) -> MagicMock: + """An ADK runner whose tool loop calls the script through the adapter's + own tool bridge.""" + + async def run_async(**kwargs: Any) -> AsyncIterator[Any]: + for call in script.tool_calls: + bridge = tool_bridge( + tool_name=call.name, + tool_description=call.name, + parameters_schema={}, + tools=turn_tools, + custom_tools=[], + ) + await bridge.run_async(args=call.arguments, tool_context=MagicMock()) + return + yield + + runner = MagicMock() + runner.session_service.create_session = AsyncMock() + runner.run_async = run_async + runner.close = AsyncMock() + return runner + + adapter = GoogleADKAdapter() + await adapter.on_started("Agent", "A test agent") + with patch.object(adapter, "_create_runner", side_effect=scripted_runner): + await adapter.on_event(turn_input(tools)) + + +PROBES: dict[str, TurnOutcomeProbe] = { + "strands": TurnOutcomeProbe(run=run_strands), + "pydantic_ai": TurnOutcomeProbe(run=run_pydantic_ai), + "agno": TurnOutcomeProbe(run=run_agno), + "langgraph": TurnOutcomeProbe(run=run_langgraph), + "anthropic": TurnOutcomeProbe(run=run_anthropic), + "gemini": TurnOutcomeProbe(run=run_gemini), + "google_adk": TurnOutcomeProbe(run=run_google_adk), +} diff --git a/tests/integrations/codex/test_adapter_e2e.py b/tests/integrations/codex/test_adapter_e2e.py index 97722f7d0..dac399ddc 100644 --- a/tests/integrations/codex/test_adapter_e2e.py +++ b/tests/integrations/codex/test_adapter_e2e.py @@ -199,24 +199,25 @@ def _request(request_id: int, method: str, params: dict[str, Any]) -> RpcEvent: ) +def _turn_completed() -> RpcEvent: + return _notify( + "turn/completed", + {"turn": {"id": "turn-1", "status": "completed", "items": [], "error": None}}, + ) + + +def _answered_turn() -> list[RpcEvent]: + """A turn whose final text the adapter relays as its reply.""" + return [ + _notify("item/agentMessage/delta", {"itemId": "m", "delta": "Done."}), + _turn_completed(), + ] + + @pytest.mark.asyncio async def test_on_event_uses_converter_history_to_resume_thread() -> None: tools = _ToolSchemaFakeTools() - fake_client = _FakeCodexClient( - events=[ - _notify( - "turn/completed", - { - "turn": { - "id": "turn-1", - "status": "completed", - "items": [], - "error": None, - } - }, - ) - ] - ) + fake_client = _FakeCodexClient(events=_answered_turn()) adapter = make_codex_adapter(fake_client, config=CodexAdapterConfig()) await adapter.on_started("Codex Agent", "Integration test agent") @@ -317,21 +318,7 @@ async def test_manual_approval_resolved_by_out_of_band_approve_command() -> None @pytest.mark.asyncio async def test_restart_rehydrates_mapping_from_previous_task_events() -> None: tools_first = _ToolSchemaFakeTools() - fake_client_first = _FakeCodexClient( - events=[ - _notify( - "turn/completed", - { - "turn": { - "id": "turn-1", - "status": "completed", - "items": [], - "error": None, - } - }, - ) - ] - ) + fake_client_first = _FakeCodexClient(events=_answered_turn()) adapter_first = make_codex_adapter(fake_client_first, config=CodexAdapterConfig()) await adapter_first.on_started("Codex Agent", "Integration test agent") await adapter_first.on_event( @@ -346,21 +333,7 @@ async def test_restart_rehydrates_mapping_from_previous_task_events() -> None: persisted_history = list(tools_first.events_sent) tools_second = _ToolSchemaFakeTools() - fake_client_second = _FakeCodexClient( - events=[ - _notify( - "turn/completed", - { - "turn": { - "id": "turn-1", - "status": "completed", - "items": [], - "error": None, - } - }, - ) - ] - ) + fake_client_second = _FakeCodexClient(events=_answered_turn()) adapter_second = make_codex_adapter(fake_client_second, config=CodexAdapterConfig()) await adapter_second.on_started("Codex Agent", "Integration test agent") await adapter_second.on_event( @@ -384,21 +357,7 @@ async def test_resume_failure_injects_conversation_history() -> None: verify turn input contains history context.""" # --- First session: produce some history --- tools_first = _ToolSchemaFakeTools() - fake_client_first = _FakeCodexClient( - events=[ - _notify( - "turn/completed", - { - "turn": { - "id": "turn-1", - "status": "completed", - "items": [], - "error": None, - } - }, - ) - ] - ) + fake_client_first = _FakeCodexClient(events=_answered_turn()) adapter_first = make_codex_adapter(fake_client_first, config=CodexAdapterConfig()) await adapter_first.on_started("Codex Agent", "Integration test agent") await adapter_first.on_event( @@ -430,19 +389,7 @@ async def test_resume_failure_injects_conversation_history() -> None: # --- Second session: resume fails, should inject history --- tools_second = _ToolSchemaFakeTools() fake_client_second = _FakeCodexClient( - events=[ - _notify( - "turn/completed", - { - "turn": { - "id": "turn-1", - "status": "completed", - "items": [], - "error": None, - } - }, - ) - ], + events=_answered_turn(), resume_error=CodexJsonRpcError(code=-32002, message="Thread expired"), ) adapter_second = make_codex_adapter(fake_client_second, config=CodexAdapterConfig()) diff --git a/tests/integrations/parlant/test_tools.py b/tests/integrations/parlant/test_tools.py index 9a2ce99af..010bd4b74 100644 --- a/tests/integrations/parlant/test_tools.py +++ b/tests/integrations/parlant/test_tools.py @@ -11,17 +11,15 @@ from band.core.task_types import TaskAssignmentStatus, TaskLifecycleState from band.core.types import AdapterFeatures, Capability from band.integrations.parlant.tools import ( - _session_message_sent, _session_tools, create_parlant_tools, get_current_tools, get_session_tools, - mark_message_sent, set_current_tools, set_session_tools, - was_message_sent, ) from band.runtime.tools import TASK_TOOL_NAMES, TOOL_MODELS, ListContactRequestsInput +from band.testing import FakeAgentTools try: import parlant.sdk # noqa: F401 @@ -37,7 +35,6 @@ class TestSessionToolsRegistry: def setup_method(self): """Clear registry before each test.""" _session_tools.clear() - _session_message_sent.clear() def test_set_session_tools_stores_tools(self): """Should store tools for a session.""" @@ -48,14 +45,6 @@ def test_set_session_tools_stores_tools(self): assert "session-123" in _session_tools assert _session_tools["session-123"] is mock_tools - def test_set_session_tools_initializes_message_sent_flag(self): - """Should initialize message_sent flag to False.""" - mock_tools = MagicMock() - - set_session_tools("session-123", mock_tools) - - assert _session_message_sent["session-123"] is False - def test_set_session_tools_clears_on_none(self): """Should clear tools when setting None.""" mock_tools = MagicMock() @@ -65,7 +54,6 @@ def test_set_session_tools_clears_on_none(self): set_session_tools("session-123", None) assert "session-123" not in _session_tools - assert "session-123" not in _session_message_sent def test_get_session_tools_returns_stored_tools(self): """Should return stored tools for session.""" @@ -83,45 +71,6 @@ def test_get_session_tools_returns_none_for_unknown_session(self): assert result is None -class TestMessageSentFlag: - """Tests for message sent tracking.""" - - def setup_method(self): - """Clear registry before each test.""" - _session_tools.clear() - _session_message_sent.clear() - - def test_mark_message_sent_sets_flag(self): - """Should set message_sent flag to True.""" - _session_message_sent["session-123"] = False - - mark_message_sent("session-123") - - assert _session_message_sent["session-123"] is True - - def test_was_message_sent_returns_true_when_sent(self): - """Should return True when message was sent.""" - _session_message_sent["session-123"] = True - - result = was_message_sent("session-123") - - assert result is True - - def test_was_message_sent_returns_false_when_not_sent(self): - """Should return False when message was not sent.""" - _session_message_sent["session-123"] = False - - result = was_message_sent("session-123") - - assert result is False - - def test_was_message_sent_returns_false_for_unknown_session(self): - """Should return False for unknown session.""" - result = was_message_sent("unknown-session") - - assert result is False - - class TestDeprecatedFunctions: """Tests for deprecated compatibility functions.""" @@ -493,7 +442,6 @@ class TestParlantToolFunctions: def setup_method(self): """Clear registry and set up mocks before each test.""" _session_tools.clear() - _session_message_sent.clear() @pytest.fixture def mock_tools(self): @@ -579,16 +527,17 @@ async def test_send_message_calls_tools_send_message( assert "Message sent to Alice, Bob" in result.data @pytest.mark.asyncio - async def test_send_message_marks_message_sent( - self, parlant_tools, mock_tools, mock_context + async def test_send_message_settles_the_turn_reply( + self, parlant_tools, mock_context ): - """Should mark message as sent after successful send.""" - set_session_tools(mock_context.session_id, mock_tools) + """The adapter reads the turn, so it won't duplicate the tool's reply.""" + tools = FakeAgentTools() + set_session_tools(mock_context.session_id, tools) send_message = parlant_tools["band_send_message"] await send_message(mock_context, "Hello", "Alice") - assert was_message_sent(mock_context.session_id) is True + assert tools.turn.replied @pytest.mark.asyncio async def test_no_reply_calls_tools_no_reply( @@ -604,17 +553,26 @@ async def test_no_reply_calls_tools_no_reply( assert "No reply sent" in result.data @pytest.mark.asyncio - async def test_no_reply_marks_message_sent( + async def test_no_reply_omits_a_blank_reason( self, parlant_tools, mock_tools, mock_context ): - """Should mark message as sent so the adapter does not duplicate-reply.""" set_session_tools(mock_context.session_id, mock_tools) no_reply = parlant_tools["band_no_reply"] await no_reply(mock_context, reason="") mock_tools.no_reply.assert_awaited_once_with(None) - assert was_message_sent(mock_context.session_id) is True + + @pytest.mark.asyncio + async def test_no_reply_settles_the_turn_reply(self, parlant_tools, mock_context): + """A decline settles the turn, so the adapter posts nothing of its own.""" + tools = FakeAgentTools() + set_session_tools(mock_context.session_id, tools) + + no_reply = parlant_tools["band_no_reply"] + await no_reply(mock_context, reason="") + + assert tools.turn.replied @pytest.mark.asyncio async def test_send_message_returns_error_without_tools( diff --git a/tests/integrations/slack/test_wrapping.py b/tests/integrations/slack/test_wrapping.py index add6ab950..222c1e68d 100644 --- a/tests/integrations/slack/test_wrapping.py +++ b/tests/integrations/slack/test_wrapping.py @@ -22,6 +22,7 @@ import json import time import warnings +from contextlib import AbstractContextManager, nullcontext from datetime import UTC, datetime from types import SimpleNamespace from typing import Any @@ -32,6 +33,7 @@ from httpx import ASGITransport from band.core.exceptions import BandToolError +from band.core.protocols import TurnResultAlreadyReported from band.core.simple_adapter import SimpleAdapter from band.core.types import ( AdapterFeatures, @@ -51,6 +53,7 @@ from band.integrations.slack.signature import SLACK_SIGNATURE_VERSION from band.integrations.slack.types import SlackApp, SlackRoomBinding from band.runtime.tools import AgentTools, ToolCallOutcome +from band.testing import MISSING_REPLY_FAILURE, FakeAgentTools, failure_reports from band.testing.platform import platform_connection_stub # ── Test doubles ───────────────────────────────────────────────────────────── @@ -71,7 +74,8 @@ class _SlackReplyBrain(SimpleAdapter[Any]): """Brain that replies to Slack via the new ``slack_send_message`` tool. Calls ``await tools.slack_send_message(reply)`` directly to mimic how - a real framework adapter would dispatch the tool the LLM picked. + a real framework adapter would dispatch the tool the LLM picked; with + ``reply=None`` it declines the turn instead. """ def __init__( @@ -111,7 +115,9 @@ async def on_message( "room_id": room_id, } ) - if self.reply is not None and hasattr(tools, "slack_send_message"): + if self.reply is None: + await tools.no_reply("nothing to add") + elif hasattr(tools, "slack_send_message"): await tools.slack_send_message(self.reply) async def on_cleanup(self, room_id: str) -> None: @@ -1423,6 +1429,51 @@ def _agent_input_with_history( ) +class _IdleBrain(_SlackReplyBrain): + """A brain whose turn does nothing at all.""" + + def __init__(self, *, judges_turns: bool) -> None: + super().__init__() + self._judges_turns = judges_turns + + @property + def judges_turns(self) -> bool: + return self._judges_turns + + async def on_message(self, *args: Any, **kwargs: Any) -> None: + return None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("brain_judges_turns", "verdict", "failures"), + [ + pytest.param( + True, + pytest.raises(TurnResultAlreadyReported), + [MISSING_REPLY_FAILURE], + id="judged-brain", + ), + pytest.param(False, nullcontext(), [], id="exempt-brain"), + ], +) +async def test_an_idle_turn_is_judged_by_the_brains_rule( + brain_judges_turns: bool, + verdict: AbstractContextManager[Any], + failures: list[tuple[str, str]], +): + adapter, _, _, _ = _make_adapter(inner=_IdleBrain(judges_turns=brain_judges_turns)) + tools = FakeAgentTools(room_id="room-1") + inp = _agent_input_with_history( + room_id="room-1", raw_history=[], bootstrap=False, tools=tools + ) + + with verdict: + await adapter.on_event(inp) + + assert failure_reports(tools) == failures + + @pytest.mark.asyncio async def test_on_event_rehydrates_room_binding_on_bootstrap(): """First WS-delivered message in a previously-bridged room restores state.""" diff --git a/tests/integrations/test_crewai_tools.py b/tests/integrations/test_crewai_tools.py index 046286e92..a8e716208 100644 --- a/tests/integrations/test_crewai_tools.py +++ b/tests/integrations/test_crewai_tools.py @@ -289,122 +289,6 @@ class FakePeersResponse(BaseModel): assert reported["output"] == {"peers": [{"id": "p1"}]} assert reported["is_error"] is False - def test_send_message_marks_reply_tracker(self, builder_mod): - """A successful band_send_message flips both ReplyTracker markers so the - adapter can treat a later empty final answer as benign.""" - tools_obj = MagicMock() - tools_obj.send_message = AsyncMock(return_value={"status": "sent"}) - tracker = builder_mod.ReplyTracker() - context = builder_mod.CrewAIToolContext( - room_id="room-1", tools=tools_obj, reply_tracker=tracker - ) - tools = builder_mod.build_band_crewai_tools( - get_context=lambda: context, - reporter=builder_mod.NoopReporter(), - capabilities=frozenset(), - ) - send_message = next(t for t in tools if t.name == "band_send_message") - - result = json.loads(send_message._run(content="hello", mentions=[])) - - assert result["status"] == "success" - tools_obj.send_message.assert_awaited_once() - assert tracker.replied is True - assert tracker.tool_executed is True - - def test_no_reply_marks_reply_tracker_terminal_work(self, builder_mod): - """A successful band_no_reply flips tool_executed (DECLINE) so an empty - final answer is benign, without treating the turn as a room post.""" - tools_obj = MagicMock() - tools_obj.no_reply = AsyncMock(return_value={"status": "no_reply"}) - tracker = builder_mod.ReplyTracker() - context = builder_mod.CrewAIToolContext( - room_id="room-1", tools=tools_obj, reply_tracker=tracker - ) - tools = builder_mod.build_band_crewai_tools( - get_context=lambda: context, - reporter=builder_mod.NoopReporter(), - capabilities=frozenset(), - ) - no_reply = next(t for t in tools if t.name == "band_no_reply") - - result = json.loads(no_reply._run(reason="not for me")) - - assert result["status"] == "success" - assert result["result_status"] == "no_reply" - tools_obj.no_reply.assert_awaited_once() - assert tracker.replied is False - assert tracker.tool_executed is True - - def test_send_event_is_not_terminal_work(self, builder_mod): - """band_send_event emits an observational event (thought/error/task), not - terminal work — it must NOT flip tool_executed. So a turn that only sends an - event and then yields an empty final answer is a genuine no-response failure - the adapter still surfaces, not benign noise (see is_terminal_success).""" - tools_obj = MagicMock() - tools_obj.send_event = AsyncMock(return_value=None) - tracker = builder_mod.ReplyTracker() - context = builder_mod.CrewAIToolContext( - room_id="room-1", tools=tools_obj, reply_tracker=tracker - ) - tools = builder_mod.build_band_crewai_tools( - get_context=lambda: context, - reporter=builder_mod.NoopReporter(), - capabilities=frozenset(), - ) - send_event = next(t for t in tools if t.name == "band_send_event") - - result = json.loads(send_event._run(content="thinking", message_type="task")) - - assert result["status"] == "success" - assert tracker.tool_executed is False - assert tracker.replied is False - - def test_read_only_tool_does_not_mark_tool_executed(self, builder_mod): - """A successful read-only tool (lookup/listing) must NOT flip either - marker. Fetching state is not a terminal action, so a turn that runs only - a lookup and then yields an empty final answer is a genuine no-response - failure the adapter must still surface — not benign noise.""" - tools_obj = MagicMock() - tools_obj.lookup_peers = AsyncMock(return_value={"peers": []}) - tracker = builder_mod.ReplyTracker() - context = builder_mod.CrewAIToolContext( - room_id="room-1", tools=tools_obj, reply_tracker=tracker - ) - tools = builder_mod.build_band_crewai_tools( - get_context=lambda: context, - reporter=builder_mod.NoopReporter(), - capabilities=frozenset(), - ) - lookup_peers = next(t for t in tools if t.name == "band_lookup_peers") - - result = json.loads(lookup_peers._run()) - - assert result["status"] == "success" - assert tracker.tool_executed is False - assert tracker.replied is False - - def test_reply_tracker_not_marked_on_send_failure(self, builder_mod): - """A failed send must NOT mark either tracker — the turn produced nothing.""" - tools_obj = MagicMock() - tools_obj.send_message = AsyncMock(side_effect=RuntimeError("boom")) - tracker = builder_mod.ReplyTracker() - context = builder_mod.CrewAIToolContext( - room_id="room-1", tools=tools_obj, reply_tracker=tracker - ) - tools = builder_mod.build_band_crewai_tools( - get_context=lambda: context, - reporter=builder_mod.NoopReporter(), - capabilities=frozenset(), - ) - send_message = next(t for t in tools if t.name == "band_send_message") - - result = json.loads(send_message._run(content="hello", mentions=[])) - - assert result["status"] == "error" - assert tracker.replied is False - assert tracker.tool_executed is False - def test_send_failure_appends_available_handles(self, builder_mod): """The real empty-mentions error already lists the room's handles, so the CrewAI enricher must surface them once — not append a second copy.""" @@ -823,7 +707,7 @@ def test_tool_returns_error_json_when_get_context_returns_none(self, builder_mod assert "No room context available" in result["message"] -# --- Turn ledger: the catalog calls the recording tool methods directly --- +# --- Turn ledger and tracker: the catalog calls the recording tool methods --- class RecordInput(BaseModel): @@ -847,11 +731,18 @@ def room_tools() -> FakeAgentTools: @pytest.fixture -def crew_tools(builder_mod, room_tools): +def tracker(builder_mod): + return builder_mod.ReplyTracker() + + +@pytest.fixture +def crew_tools(builder_mod, room_tools, tracker): """The crew's tools by name, bound to ``room_tools`` as the room context.""" def build(custom_tools: list[Any] | None = None) -> dict[str, Any]: - context = builder_mod.CrewAIToolContext(room_id="room-1", tools=room_tools) + context = builder_mod.CrewAIToolContext( + room_id="room-1", tools=room_tools, reply_tracker=tracker + ) tools = builder_mod.build_band_crewai_tools( get_context=lambda: context, reporter=builder_mod.NoopReporter(), @@ -863,10 +754,36 @@ def build(custom_tools: list[Any] | None = None) -> dict[str, Any]: class TestTurnLedger: - def test_send_message_marks_the_turn_replied(self, crew_tools, room_tools): - crew_tools()[BandTool.SEND_MESSAGE]._run(content="hi", mentions=["@alice"]) + @pytest.mark.parametrize( + ("tool", "arguments", "replied", "complete"), + [ + ( + BandTool.SEND_MESSAGE, + {"content": "hi", "mentions": ["@alice"]}, + True, + True, + ), + (BandTool.NO_REPLY, {"reason": "FYI only"}, True, True), + (BandTool.CREATE_CHATROOM, {}, False, True), + (BandTool.LOOKUP_PEERS, {}, False, False), + ( + BandTool.SEND_EVENT, + {"content": "hm", "message_type": "thought"}, + False, + False, + ), + ], + ids=["reply", "decline", "act", "observe", "narrate"], + ) + def test_a_tool_call_records_its_effect_on_the_turn( + self, crew_tools, room_tools, tool, arguments, replied, complete + ): + crew_tools()[tool]._run(**arguments) - assert room_tools.turn.replied + assert (room_tools.turn.replied, room_tools.turn.complete) == ( + replied, + complete, + ) def test_failed_send_then_event_leaves_the_turn_incomplete( self, crew_tools, room_tools @@ -881,6 +798,20 @@ def test_failed_send_then_event_leaves_the_turn_incomplete( assert (room_tools.messages_sent, len(room_tools.events_sent)) == ([], 1) assert not room_tools.turn.complete + def test_tracker_keeps_what_a_successful_send_said(self, crew_tools, tracker): + crew_tools()[BandTool.SEND_MESSAGE]._run(content="hi", mentions=["@alice"]) + + assert (tracker.posts, tracker.any_tool_ran) == (["(to @alice) hi"], True) + + def test_a_failed_send_still_counts_as_a_tool_that_ran( + self, crew_tools, room_tools, tracker + ): + room_tools.send_message_error = BandToolError("boom") + + crew_tools()[BandTool.SEND_MESSAGE]._run(content="hi", mentions=["@alice"]) + + assert (tracker.posts, tracker.any_tool_ran) == ([], True) + @pytest.mark.parametrize( ("handler", "completes"), [(record_value, False), (record_value_for_real, True)], diff --git a/tests/runtime/test_oneshot.py b/tests/runtime/test_oneshot.py index 9029966e7..a2020e124 100644 --- a/tests/runtime/test_oneshot.py +++ b/tests/runtime/test_oneshot.py @@ -27,6 +27,7 @@ from band.core.simple_adapter import SimpleAdapter from band.core.types import Capability from band.runtime.capabilities import FeatureFlag +from band.runtime.custom_tools import declares_turn_effect from band.runtime.formatters import build_participants_message from band.runtime.oneshot import ( OneShotEnvelopeError, @@ -35,6 +36,7 @@ _lookup_sender_name, _parse_inserted_at, ) +from band.runtime.tools import TurnEffect from tests.catalogs import CatalogAdapter from tests.runtime.conftest import ctx_item, make_link_mock, platform_msg @@ -875,6 +877,7 @@ class TestToolCallReplayAcrossCrash: async def test_crash_before_mark_processed_replays_the_tool(self) -> None: sent_emails: list[str] = [] + @declares_turn_effect(TurnEffect.ACT) async def send_email(args: SendEmailInput) -> str: sent_emails.append(args.to) return "sent" diff --git a/tests/runtime/test_terminal_success.py b/tests/runtime/test_terminal_success.py deleted file mode 100644 index 9cd963686..000000000 --- a/tests/runtime/test_terminal_success.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Unit tests for the shared "terminal productive work" predicate. - -``is_terminal_success`` is the single source of truth the tool-only adapters use -to decide whether an empty final model response is *benign* (the agent already -did its work) or a genuine no-response failure. The fail-loud policy: read-only -Band tools and *undeclared* custom tools never count; a custom tool must declare -its effect (``declares_turn_effect``, or the ``band_terminal`` shorthand). -""" - -from __future__ import annotations - -from unittest.mock import MagicMock - -import pytest - -from band.runtime.custom_tools import declared_effect, declares_turn_effect -from band.runtime.tools import ( - ALL_TOOL_NAMES, - AgentTools, - BandTool, - TurnEffect, - band_tool_errored, - is_terminal_success, - settles_turn_reply, -) - - -def test_failed_tool_never_counts() -> None: - assert is_terminal_success(BandTool.SEND_MESSAGE, succeeded=False) is False - - -def test_band_tool_errored_detects_error_prefix() -> None: - assert band_tool_errored("band_send_message", "Error sending: nope") is True - assert band_tool_errored("band_send_message", "sent") is False - # Only known Band tools follow the "Error " convention; custom tools do not. - assert band_tool_errored("lookup", "Error whatever") is False - # Non-string content (e.g. a dict result) is not an error string. - assert band_tool_errored("band_store_memory", {"id": "m1"}) is False - assert band_tool_errored(None, "Error x") is False - - -_POST_TO_SLACK = {"post_to_slack": TurnEffect.ACT} -_STAY_QUIET = {"stay_quiet": TurnEffect.DECLINE} - - -@pytest.mark.parametrize( - ("tool", "custom_effects", "did_work", "settles_reply"), - [ - pytest.param(BandTool.SEND_MESSAGE, None, True, True, id="post"), - pytest.param(BandTool.NO_REPLY, None, True, True, id="no-reply"), - # Silence after an action is benign, yet the model's plain text is still owed. - pytest.param(BandTool.ADD_PARTICIPANT, None, True, False, id="action"), - pytest.param(BandTool.GET_MEMORY, None, False, False, id="read-only"), - pytest.param(BandTool.SEND_EVENT, None, False, False, id="narration"), - # An undeclared custom tool fails loud: no work, the reply still owed. - pytest.param("weather", _POST_TO_SLACK, False, False, id="undeclared"), - pytest.param("post_to_slack", _POST_TO_SLACK, True, False, id="declared-act"), - pytest.param("stay_quiet", _STAY_QUIET, True, True, id="declared-silence"), - ], -) -def test_the_two_views_of_a_finished_tool_call( - tool: str, - custom_effects: dict[str, TurnEffect] | None, - did_work: bool, - settles_reply: bool, -) -> None: - assert ( - is_terminal_success(tool, succeeded=True, custom_effects=custom_effects) - is did_work - ) - assert settles_turn_reply(tool, custom_effects=custom_effects) is settles_reply - - -def test_declared_effect_reads_the_declaration_and_the_terminal_shorthand() -> None: - @declares_turn_effect(TurnEffect.DECLINE) - def stay_quiet() -> None: ... - - def terminal() -> None: ... - - terminal.band_terminal = True # type: ignore[attr-defined] - - def plain() -> None: ... - - class TerminalModel: - band_terminal = True - - assert declared_effect(stay_quiet) is TurnEffect.DECLINE - assert declared_effect(terminal) is TurnEffect.ACT - assert declared_effect(TerminalModel) is TurnEffect.ACT - assert declared_effect(plain) is None - - -def test_a_custom_tool_cannot_redefine_a_band_tool() -> None: - hijack = {BandTool.SEND_MESSAGE: TurnEffect.OBSERVE} - assert settles_turn_reply(BandTool.SEND_MESSAGE, custom_effects=hijack) is True - - -def test_no_reply_is_recognised_under_the_band_mcp_prefix_only() -> None: - assert settles_turn_reply("band-band_no_reply") is True - assert settles_turn_reply("other-band_no_reply") is False - - -def test_only_posts_and_no_reply_settle_the_reply() -> None: - settling = {name for name in ALL_TOOL_NAMES if settles_turn_reply(name)} - assert settling == { - BandTool.SEND_MESSAGE, - BandTool.SEND_ROOM_FILE, - BandTool.NO_REPLY, - } - - -async def test_no_reply_is_local_only(mock_rest_client: MagicMock) -> None: - tools = AgentTools("room-1", mock_rest_client) - - outcome = await tools.execute_tool_call_structured( - BandTool.NO_REPLY, {"reason": "addressed to another agent"} - ) - - assert (outcome.ok, outcome.value) == (True, {"status": "no_reply"}) - assert mock_rest_client.mock_calls == [] diff --git a/tests/runtime/test_tools.py b/tests/runtime/test_tools.py index d0e574740..92b5ba323 100644 --- a/tests/runtime/test_tools.py +++ b/tests/runtime/test_tools.py @@ -46,13 +46,14 @@ SendMessageInput, SendRoomFileInput, StoreMemoryInput, + TurnEffect, append_mention_handles_hint, available_mention_handles, canonicalize_mcp_tool_name, format_tool_validation_error, is_mcp_content_result, matches_identifier, - settles_turn_reply, + turn_effect, ) from tests.conftest import make_participant_mock from tests.content import BLANK_CONTENT_CASES @@ -2421,33 +2422,33 @@ def test_create_chatroom_input_no_task_id(self): assert model.task_id is None -class TestSettlesTurnReply: - """Which tool calls count as having settled the turn's reply.""" +class TestReplyEffectSpellings: + """Which tool names resolve to Band's reply effect.""" def test_sdk_injected_tool(self): - assert settles_turn_reply("band_send_message") is True + assert turn_effect("band_send_message") is TurnEffect.REPLY def test_standalone_band_mcp_tool(self): - assert settles_turn_reply("create_agent_chat_message") is True + assert turn_effect("create_agent_chat_message") is TurnEffect.REPLY def test_mcp_server_prefixed_names(self): """MCP clients may prefix the server name onto the tool name.""" - assert settles_turn_reply("band-band_send_message") is True - assert settles_turn_reply("band-create_agent_chat_message") is True + assert turn_effect("band-band_send_message") is TurnEffect.REPLY + assert turn_effect("band-create_agent_chat_message") is TurnEffect.REPLY def test_non_settling_tools(self): - assert settles_turn_reply("band_send_event") is False - assert settles_turn_reply("band_lookup_peers") is False - assert settles_turn_reply("get_weather") is False + assert turn_effect("band_send_event") is TurnEffect.OBSERVE + assert turn_effect("band_lookup_peers") is TurnEffect.OBSERVE + assert turn_effect("get_weather") is TurnEffect.OBSERVE def test_no_substring_false_positive(self): """Only an exact or server-prefixed match counts, not any substring.""" - assert settles_turn_reply("band_send_message_draft") is False + assert turn_effect("band_send_message_draft") is TurnEffect.OBSERVE def test_non_band_server_prefix_does_not_resolve(self): """An unrelated MCP server's own tool must never be treated as a Band reply just because it ends in ``-band_send_message``.""" - assert settles_turn_reply("other-band_send_message") is False + assert turn_effect("other-band_send_message") is TurnEffect.OBSERVE class TestCanonicalizeMcpToolName: diff --git a/tests/runtime/test_turn.py b/tests/runtime/test_turn.py index 9da6813a4..81e059cdb 100644 --- a/tests/runtime/test_turn.py +++ b/tests/runtime/test_turn.py @@ -8,18 +8,22 @@ from __future__ import annotations from typing import Any +from unittest.mock import MagicMock -import band_sdk_core import pytest from pydantic import BaseModel from band.core.delivery import deliver_reply -from band.core.protocols import TURN_FAILURE_PROVIDER from band.core.turn import report_unsettled_turn from band.integrations.claude_sdk.dedup_tools import DedupingAgentTools from band.runtime.custom_tools import declares_turn_effect, execute_custom_tool from band.runtime.tools import AgentTools, BandTool, TurnEffect -from band.testing.fake_tools import FakeAgentTools, reported_failures +from band.testing.fake_tools import ( + MISSING_REPLY_FAILURE, + FakeAgentTools, + failure_reports, + reported_failures, +) ALICE = {"id": "user-1", "name": "Alice", "type": "User", "handle": "alice"} @@ -146,9 +150,7 @@ async def test_a_missing_reply_is_reported_once(self) -> None: assert await report_unsettled_turn(tools) is True assert await report_unsettled_turn(tools) is False - assert [(f["provider"], f["message"]) for f in reported_failures(tools)] == [ - (TURN_FAILURE_PROVIDER, band_sdk_core.missing_reply_message()) - ] + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] async def test_a_complete_turn_reports_nothing(self) -> None: tools = FakeAgentTools() @@ -163,3 +165,36 @@ async def test_a_settled_turn_reports_nothing(self) -> None: tools.turn.settle() assert await report_unsettled_turn(tools) is False + + +class TestDetach: + def test_a_judged_turn_detaches(self) -> None: + tools = FakeAgentTools() + tools.turn.judged = True + + tools.turn.detach() + + assert tools.turn.detached + + def test_an_unjudged_turn_never_detaches(self) -> None: + """A contact-hub turn parked on a decision is never reported later.""" + tools = FakeAgentTools() + + tools.turn.detach() + + assert not tools.turn.detached + + +async def test_a_detached_report_never_marks_the_contexts_next_message( + mock_rest_client: Any, +) -> None: + ctx = MagicMock(participants=[ALICE], agent_id="agent-1", hub_room_id=None) + ctx.link.rest = mock_rest_client + tools = AgentTools.from_context(ctx) + tools.turn.judged = True + tools.turn.detach() + + assert await report_unsettled_turn(tools) is True + + mock_rest_client.agent_api_events.create_agent_chat_event.assert_awaited_once() + ctx.note_turn_failure_reported.assert_not_called() diff --git a/tests/runtime/test_turn_effect.py b/tests/runtime/test_turn_effect.py new file mode 100644 index 000000000..e6b248944 --- /dev/null +++ b/tests/runtime/test_turn_effect.py @@ -0,0 +1,100 @@ +"""What each tool's successful call does to the turn. + +``turn_effect`` resolves a Band tool from band-sdk-core's table and any other +tool from the effects the caller's custom tools declared. Read-only Band tools, +narration and *undeclared* custom tools only observe; a custom tool that does +real work or answers declares it (``declares_turn_effect``, or the +``band_terminal`` shorthand). +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from band.runtime.custom_tools import declared_effect, declares_turn_effect +from band.runtime.tools import ( + AgentTools, + BandTool, + TurnEffect, + band_tool_errored, + turn_effect, +) + + +def test_band_tool_errored_detects_error_prefix() -> None: + assert band_tool_errored("band_send_message", "Error sending: nope") is True + assert band_tool_errored("band_send_message", "sent") is False + # Only known Band tools follow the "Error " convention; custom tools do not. + assert band_tool_errored("lookup", "Error whatever") is False + # Non-string content (e.g. a dict result) is not an error string. + assert band_tool_errored("band_store_memory", {"id": "m1"}) is False + assert band_tool_errored(None, "Error x") is False + + +_POST_TO_SLACK = {"post_to_slack": TurnEffect.ACT} +_STAY_QUIET = {"stay_quiet": TurnEffect.DECLINE} + + +@pytest.mark.parametrize( + ("tool", "custom_effects", "effect"), + [ + pytest.param(BandTool.SEND_MESSAGE, None, TurnEffect.REPLY, id="post"), + pytest.param(BandTool.NO_REPLY, None, TurnEffect.DECLINE, id="no-reply"), + pytest.param(BandTool.ADD_PARTICIPANT, None, TurnEffect.ACT, id="action"), + pytest.param(BandTool.GET_MEMORY, None, TurnEffect.OBSERVE, id="read-only"), + pytest.param(BandTool.SEND_EVENT, None, TurnEffect.OBSERVE, id="narration"), + pytest.param("weather", _POST_TO_SLACK, TurnEffect.OBSERVE, id="undeclared"), + pytest.param( + "post_to_slack", _POST_TO_SLACK, TurnEffect.ACT, id="declared-act" + ), + pytest.param( + "stay_quiet", _STAY_QUIET, TurnEffect.DECLINE, id="declared-silence" + ), + ], +) +def test_turn_effect_of_a_finished_tool_call( + tool: str, custom_effects: dict[str, TurnEffect] | None, effect: TurnEffect +) -> None: + assert turn_effect(tool, custom_effects=custom_effects) is effect + + +def test_declared_effect_reads_the_declaration_and_the_terminal_shorthand() -> None: + @declares_turn_effect(TurnEffect.DECLINE) + def stay_quiet() -> None: ... + + def terminal() -> None: ... + + terminal.band_terminal = True # type: ignore[attr-defined] + + def plain() -> None: ... + + class TerminalModel: + band_terminal = True + + assert declared_effect(stay_quiet) is TurnEffect.DECLINE + assert declared_effect(terminal) is TurnEffect.ACT + assert declared_effect(TerminalModel) is TurnEffect.ACT + assert declared_effect(plain) is None + + +def test_a_custom_tool_cannot_redefine_a_band_tool() -> None: + hijack = {BandTool.SEND_MESSAGE: TurnEffect.OBSERVE} + assert turn_effect(BandTool.SEND_MESSAGE, custom_effects=hijack) is TurnEffect.REPLY + + +def test_no_reply_is_recognised_under_the_band_mcp_prefix_only() -> None: + assert turn_effect("band-band_no_reply") is TurnEffect.DECLINE + assert turn_effect("other-band_no_reply") is TurnEffect.OBSERVE + + +async def test_no_reply_is_local_only(mock_rest_client: MagicMock) -> None: + tools = AgentTools("room-1", mock_rest_client) + + outcome = await tools.execute_tool_call_structured( + BandTool.NO_REPLY, {"reason": "addressed to another agent"} + ) + + assert (outcome.ok, outcome.value) == (True, {"status": "no_reply"}) + assert mock_rest_client.mock_calls == [] diff --git a/tests/runtime/test_turn_verdict.py b/tests/runtime/test_turn_verdict.py new file mode 100644 index 000000000..bfb0c9f29 --- /dev/null +++ b/tests/runtime/test_turn_verdict.py @@ -0,0 +1,192 @@ +"""The turn verdict through the real runtime: ExecutionContext -> preprocessor -> +``SimpleAdapter.on_event`` -> real ``AgentTools``. + +One row per band-sdk-core turn-outcome fixture, plus the nightly loss (a send to +an unknown handle, then narration). A complete turn is marked PROCESSED with no +report; a missing reply is reported once (provider ``band-runtime``, core's +text) and marked FAILED. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import band_sdk_core +import pytest + +from band.core.delivery import relay_reply +from band.core.protocols import TURN_FAILURE_PROVIDER, AgentToolsProtocol +from band.core.simple_adapter import SimpleAdapter +from band.core.types import ( + SYNTHETIC_CONTACT_EVENTS_SENDER_ID, + SYNTHETIC_SENDER_TYPE, + HistoryProvider, + PlatformMessage, +) +from band.preprocessing.default import DefaultPreprocessor +from band.runtime.execution import ExecutionContext +from band.runtime.tools import BandTool +from band.runtime.types import SessionConfig +from band.testing import MISSING_REPLY_FAILURE +from tests.conftest import make_message_event, make_participant_mock + +AGENT_ID = "agent-123" +ROOM_ID = "room-123" +USER = ["@user-1"] + +Step = Callable[[AgentToolsProtocol], Awaitable[Any]] + + +async def unknown_handle_send(tools: AgentToolsProtocol) -> None: + try: + await tools.send_message("hi", mentions=["@user1"]) + except ValueError: + pass # the model sees the error and moves on + + +ROWS: dict[str, tuple[list[Step], bool]] = { + "nothing": ([], False), + "observe-only": ([lambda t: t.get_participants()], False), + "act": ([lambda t: t.create_chatroom()], True), + "reply": ([lambda t: t.send_message("the answer", mentions=USER)], True), + "decline": ([lambda t: t.execute_tool_call(BandTool.NO_REPLY, {})], True), + "relay": ([lambda t: relay_reply(t, "the answer", USER)], True), + "settled": ([lambda t: _settle(t)], True), + "reported": ( + [lambda t: t.send_failure(band_sdk_core.AgentFailure("codex", "boom"))], + True, + ), + "nightly": ( + [ + unknown_handle_send, + lambda t: t.execute_tool_call( + BandTool.SEND_EVENT, {"content": "retrying", "message_type": "thought"} + ), + ], + False, + ), + "blank-send": ([lambda t: t.send_message(" ", mentions=USER)], False), +} + + +async def _settle(tools: AgentToolsProtocol) -> None: + tools.turn.settle() + + +class ScriptedAdapter(SimpleAdapter[HistoryProvider]): + def __init__(self, steps: list[Step], *, judges: bool = True) -> None: + super().__init__() + self._steps = steps + self._judges = judges + + @property + def judges_turns(self) -> bool: + return self._judges + + async def on_message( + self, + msg: PlatformMessage, + tools: AgentToolsProtocol, + history: HistoryProvider, + participants_msg: str | None, + contacts_msg: str | None, + *, + is_session_bootstrap: bool, + room_id: str, + ) -> None: + for step in self._steps: + await step(tools) + + +@pytest.fixture +def link(mock_rest_client: MagicMock) -> MagicMock: + link = MagicMock() + link.agent_id = AGENT_ID + link.rest = mock_rest_client + link.rest.agent_api_participants.list_agent_chat_participants = AsyncMock( + return_value=MagicMock( + data=[make_participant_mock("user-1", "User One", "User")] + ) + ) + link.mark_processing = AsyncMock(return_value=True) + link.mark_processed = AsyncMock(return_value=True) + link.mark_failed = AsyncMock(return_value=True) + link.get_next_message = AsyncMock(return_value=None) + link.get_stale_processing_messages = AsyncMock(return_value=[]) + return link + + +def run_through(adapter: SimpleAdapter[Any], link: MagicMock) -> ExecutionContext: + preprocessor = DefaultPreprocessor() + + async def handler(ctx: ExecutionContext, event: Any) -> None: + inp = await preprocessor.process(ctx=ctx, event=event, agent_id=AGENT_ID) + if inp is not None: + await adapter.on_event(inp) + + return ExecutionContext( + ROOM_ID, + link, + handler, + config=SessionConfig(enable_context_hydration=False), + agent_id=AGENT_ID, + ) + + +def runtime_failures(link: MagicMock) -> list[tuple[str, str]]: + return [ + ( + call.kwargs["event"].metadata["failure"]["provider"], + call.kwargs["event"].content, + ) + for call in link.rest.agent_api_events.create_agent_chat_event.call_args_list + if call.kwargs["event"].metadata + and "failure" in call.kwargs["event"].metadata + and call.kwargs["event"].metadata["failure"]["provider"] + == TURN_FAILURE_PROVIDER + ] + + +@pytest.mark.parametrize("row", sorted(ROWS)) +async def test_each_turn_outcome_is_reported_honestly( + row: str, link: MagicMock +) -> None: + steps, completes = ROWS[row] + ctx = run_through(ScriptedAdapter(steps), link) + + await ctx._process_event(make_message_event(room_id=ROOM_ID, sender_id="user-1")) + + if completes: + link.mark_processed.assert_awaited_once() + link.mark_failed.assert_not_awaited() + assert runtime_failures(link) == [] + else: + link.mark_failed.assert_awaited_once() + link.mark_processed.assert_not_awaited() + assert runtime_failures(link) == [MISSING_REPLY_FAILURE] + + +async def test_an_exempt_adapter_is_never_judged(link: MagicMock) -> None: + ctx = run_through(ScriptedAdapter([], judges=False), link) + + await ctx._process_event(make_message_event(room_id=ROOM_ID, sender_id="user-1")) + + link.mark_processed.assert_awaited_once() + assert runtime_failures(link) == [] + + +async def test_a_contact_hub_turn_is_never_judged(link: MagicMock) -> None: + ctx = run_through(ScriptedAdapter([]), link) + + await ctx._process_event( + make_message_event( + room_id=ROOM_ID, + sender_id=SYNTHETIC_CONTACT_EVENTS_SENDER_ID, + sender_type=SYNTHETIC_SENDER_TYPE, + ) + ) + + assert runtime_failures(link) == [] + link.mark_failed.assert_not_awaited() diff --git a/tests/test_agent.py b/tests/test_agent.py index 23e62d78b..e92890614 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -607,6 +607,7 @@ async def on_message( "System prompt was empty during message processing! " "adapter.on_started() was not called before message arrived." ) + await tools.no_reply() async def on_cleanup(self, room_id: str) -> None: pass diff --git a/tests/websocket/test_reply_capture_delivery.py b/tests/websocket/test_reply_capture_delivery.py index 15ac67e12..97bac99e2 100644 --- a/tests/websocket/test_reply_capture_delivery.py +++ b/tests/websocket/test_reply_capture_delivery.py @@ -112,8 +112,9 @@ async def test_wait_for_processed_resolves_on_processed() -> None: async def test_wait_for_processed_waits_through_failed_then_succeeds() -> None: - """FAILED is transient (the platform retries), so the barrier must not give - up on it — it resolves only once PROCESSED arrives.""" + """FAILED is not terminal (a new agent context can run the message again), + so the barrier must not give up on it — it resolves only once PROCESSED + arrives.""" capture = ReplyCapture("room-1", deadline_s=2) waiter = asyncio.create_task(capture.wait_for_processed(MSG, AGENT)) await asyncio.sleep(0) From 89fa7a17b8708b65dd63df25713798c479159aa8 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 4 Oct 2026 19:49:37 +0300 Subject: [PATCH 03/17] fix: let a cancel reach a turn whose decision resolves in the same tick DecisionRegistry.wait used asyncio.wait_for, which on Python 3.11 returns an answer that lands in the same tick as a cancel of the waiting task and drops the cancel. Room cleanup does exactly that: it resolves a parked decision, then the ACP connection shutdown cancels the request task. The task carried on into a sender loop that had already exited, and cleanup waited on it forever. asyncio.timeout() delivers the cancel, matching ExecutionContext's event wait. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_016nwuEBtdtneuDkJHcvijDQ Signed-off-by: Alexander Zaikman --- src/band/runtime/decisions.py | 6 +++++- tests/runtime/test_decisions.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/band/runtime/decisions.py b/src/band/runtime/decisions.py index ee9e975f2..f3b3092f5 100644 --- a/src/band/runtime/decisions.py +++ b/src/band/runtime/decisions.py @@ -176,7 +176,11 @@ async def wait( first or it was replaced; a reply that claimed it first is always waited for.""" try: - return await asyncio.wait_for(asyncio.shield(future), timeout_s) + # asyncio.timeout(), not wait_for: on Python 3.11 wait_for returns + # an answer that lands in the same tick as a cancel of this task + # and drops the cancel, so a cleaned-up turn carries on. + async with asyncio.timeout(timeout_s): + return await asyncio.shield(future) except TimeoutError: # Whoever removed the ask in the deadline's own tick resolved it. if future.done(): diff --git a/tests/runtime/test_decisions.py b/tests/runtime/test_decisions.py index 7256ec09a..ee37d0844 100644 --- a/tests/runtime/test_decisions.py +++ b/tests/runtime/test_decisions.py @@ -282,3 +282,21 @@ async def test_a_failed_prompt_withdraws_the_ask_unless_a_reply_claimed_it() -> "accept" ) assert not registry + + +async def test_a_turn_cancelled_as_its_ask_resolves_still_unwinds( + open_room: Callable[..., Room], +) -> None: + """Room cleanup resolves an open ask, then cancels the turn parked on it + in the same tick. The turn must still end cancelled: an answer that + swallows the cancel lets it carry on into a connection already closing, + and the shutdown awaiting it never returns.""" + room = open_room() + await room.ask("parked", room_id="room-1") + asker = room.askers.pop("parked") + + room.tear_down("room-1") + asker.cancel() + await asyncio.wait([asker]) + + assert asker.cancelled() From db6d010ea1ae8625846da1a8cb884ff4e0279c54 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 4 Oct 2026 19:49:37 +0300 Subject: [PATCH 04/17] test: have the delegating peer ask a question, not forward a bare token A bare token with a mention reads as an FYI, which the peer may now decline with band_no_reply, so the delegation smoke asked for a reply it gave the peer no reason to send. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_016nwuEBtdtneuDkJHcvijDQ Signed-off-by: Alexander Zaikman --- tests/e2e/baseline/smoke/samples/sample_agents.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/e2e/baseline/smoke/samples/sample_agents.py b/tests/e2e/baseline/smoke/samples/sample_agents.py index 396326251..541d63658 100644 --- a/tests/e2e/baseline/smoke/samples/sample_agents.py +++ b/tests/e2e/baseline/smoke/samples/sample_agents.py @@ -639,11 +639,13 @@ def delegate_to_peer_instruction(peer_name: str, peer_id: str) -> str: """Peer-initiated delegation: drive one agent to ask peer ``peer_name`` to confirm the value it just remembered, then report that reply — so it emits a real routing mention of the peer whose body carries the value it recalled from its own context, - and the peer responds.""" + and the peer responds. The message must ask a question: a bare token reads as an + FYI, which the peer may rightly decline with band_no_reply.""" return ( "Recall the complete value token from my previous message. " - f"{peer_name} did not receive that message. First send the exact token " - f"to {peer_name} with band_send_message(content containing the token, " - f"mentions=['{peer_id}']). Do not address that first message to me. " + f"{peer_name} did not receive that message. First ask {peer_name} to " + f"confirm the exact token with {BandTool.SEND_MESSAGE}(content containing " + f"the token and asking {peer_name} to confirm it, mentions=['{peer_id}']). " + "Do not address that first message to me. " "Wait for the peer's response, then report it back to me." ) From 87c56e085e94b4a743cc313f29c9a3c70031dade Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 4 Oct 2026 20:50:37 +0300 Subject: [PATCH 05/17] test: end event-only and remember-only smoke turns deliberately An event is not a reply, so a turn that only emits band_send_event, or says nothing to an FYI-shaped "remember this fact", is now reported as a missing reply and marked FAILED. The event smokes end their turn with band_no_reply, and each burst-recall turn asks for a short acknowledgement. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_016nwuEBtdtneuDkJHcvijDQ Signed-off-by: Alexander Zaikman --- .../baseline/smoke/inspection/test_events.py | 3 ++- .../baseline/smoke/samples/sample_agents.py | 26 +++++++++++++------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/tests/e2e/baseline/smoke/inspection/test_events.py b/tests/e2e/baseline/smoke/inspection/test_events.py index 524f5d3c2..f94bad5f6 100644 --- a/tests/e2e/baseline/smoke/inspection/test_events.py +++ b/tests/e2e/baseline/smoke/inspection/test_events.py @@ -20,6 +20,7 @@ from band.core.types import MessageType from tests.e2e.baseline.agents import Adapter, with_adapters from tests.e2e.baseline.smoke.samples.sample_agents import ( + END_WITHOUT_REPLY, TOOL_AGENT, emit_event_instruction, emit_thoughts_instruction, @@ -84,7 +85,7 @@ async def test_event_subclasses_one_turn( f"(1) message_type='thought' content including {thought}; " f"(2) message_type='task' content including {task}; " f"(3) message_type='error' content including {error}. " - "Each token verbatim. Do not call any other tool." + f"Each token verbatim. {END_WITHOUT_REPLY}" ) room_id = await resource_manager.provision_room( title="e2e-events-all", participants=[agent.id] diff --git a/tests/e2e/baseline/smoke/samples/sample_agents.py b/tests/e2e/baseline/smoke/samples/sample_agents.py index 541d63658..781fb3d2e 100644 --- a/tests/e2e/baseline/smoke/samples/sample_agents.py +++ b/tests/e2e/baseline/smoke/samples/sample_agents.py @@ -294,15 +294,22 @@ def image_round_trip_instruction() -> str: ) +# An event is not a reply, so a turn that only emits events must still end +# deliberately, or it is reported as a missing reply. +END_WITHOUT_REPLY = ( + f"Then end your turn by calling {BandTool.NO_REPLY}. Do not reply with a chat " + "message and do not call any other tool." +) + + def emit_event_instruction(event_type: MessageType, marker: str) -> str: """User message forcing exactly one ``band_send_event`` of ``event_type`` whose content carries ``marker`` verbatim.""" return ( f"Call the tool band_send_event exactly once with " f"message_type='{event_type.value}' and content that includes the exact " - f"token {marker} (verbatim). That tool call is your ONLY action -- do not " - "reply with a chat message and do not call any other tool. A plain-text " - "reply does not satisfy this; you must call band_send_event." + f"token {marker} (verbatim). A plain-text reply does not satisfy this; " + f"you must call band_send_event. {END_WITHOUT_REPLY}" ) @@ -313,9 +320,8 @@ def emit_thoughts_instruction(markers: list[str]) -> str: return ( f"Call the tool band_send_event once for each of these tokens: {tokens}. " f"Each call uses message_type='{MessageType.THOUGHT.value}' with content " - "containing that exact token verbatim. Those tool calls are your ONLY " - "action -- do not reply with a chat message and do not call any other " - "tool. A plain-text reply does not satisfy this." + "containing that exact token verbatim. A plain-text reply does not " + f"satisfy this. {END_WITHOUT_REPLY}" ) @@ -620,8 +626,12 @@ def remove_participant_instruction(peer_name: str, peer_id: str) -> str: def remember_fact_instruction(fact: str) -> str: """One burst turn: ask the agent to remember ``fact`` (a unique marker). Terse so a - burst of these is cheap; the later spanning recall is what's under test.""" - return f"Remember this fact for later: {fact}." + burst of these is cheap; the later spanning recall is what's under test. It asks + for the acknowledgement outright: a turn that says nothing is reported FAILED.""" + return ( + f"Remember this fact for later: {fact}. Acknowledge it in one short " + f"{BandTool.SEND_MESSAGE} reply." + ) # Recall probe for the spanning-recall step: asks for the whole set so an early, a From 388e828acdb494b52be35e5b7f8a2c2a36919d21 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 4 Oct 2026 21:20:04 +0300 Subject: [PATCH 06/17] fix: keep the plain-text warning in the base prompt; vary the copilot repeat Teaching band_no_reply dropped the base prompt's "a turn that ends with the answer written as plain text delivers nothing", which no change meant to remove. The copilot shared-client smoke resent "Please say hello." into a session that had already answered it, a repeat the model may now decline; the successor adapter asks something new instead. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_016nwuEBtdtneuDkJHcvijDQ Signed-off-by: Alexander Zaikman --- src/band/runtime/prompts.py | 2 +- tests/e2e/baseline/smoke/adapters/test_copilot_sdk.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/band/runtime/prompts.py b/src/band/runtime/prompts.py index 5c1bae77c..e05e26b2b 100644 --- a/src/band/runtime/prompts.py +++ b/src/band/runtime/prompts.py @@ -39,7 +39,7 @@ Multi-participant chat. Messages show sender: [Name]: content. Messages prefixed with [System]: are platform updates (participant changes, contact updates, etc.). -Use `{BandTool.SEND_MESSAGE}(content, mentions)` to respond — a `{BandTool.SEND_MESSAGE}` call is the only way anything you say reaches the room. Any text you produce outside such a call is never delivered, and that includes a final answer you compose after using other tools. So deliver your answer by calling `{BandTool.SEND_MESSAGE}`. +Use `{BandTool.SEND_MESSAGE}(content, mentions)` to respond — a `{BandTool.SEND_MESSAGE}` call is the only way anything you say reaches the room. Any text you produce outside such a call is never delivered, and that includes a final answer you compose after using other tools. So deliver your answer by calling `{BandTool.SEND_MESSAGE}`; a turn that ends with the answer written as plain text delivers nothing. When the latest message needs no answer from you (it was addressed to someone else, it is an FYI or an acknowledgement, or another participant already answered it), call `{BandTool.NO_REPLY}` instead to end the turn deliberately. End every turn with one of the two. {MENTION_IDENTIFIERS} diff --git a/tests/e2e/baseline/smoke/adapters/test_copilot_sdk.py b/tests/e2e/baseline/smoke/adapters/test_copilot_sdk.py index 0e21ff45e..0651ade05 100644 --- a/tests/e2e/baseline/smoke/adapters/test_copilot_sdk.py +++ b/tests/e2e/baseline/smoke/adapters/test_copilot_sdk.py @@ -353,11 +353,11 @@ async def test_copilot_shared_client_across_adapter_lifecycles( title="e2e-copilot-shared-b", participants=[identity.id] ) - async def smoke(room_id: str) -> None: + async def smoke(room_id: str, *, prompt: str = "Please say hello.") -> None: async with reply_capture(room_id) as capture: mid = await user_ops.send_message( room_id, - "Please say hello.", + prompt, mention_id=identity.id, mention_name=identity.name, ) @@ -384,6 +384,8 @@ def make_shared_adapter(client: Any) -> CopilotSDKAdapter: async with running_agent( identity, make_shared_adapter(client), baseline_settings ): - await smoke(room_a) + # A new request: the resumed session already answered the first, + # and a repeat of it may rightly be declined with band_no_reply. + await smoke(room_a, prompt="Please say goodbye.") finally: await client.stop() From f667f5f65e1991beb846d47d416ce8c8ff75b50f Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 4 Oct 2026 23:59:56 +0300 Subject: [PATCH 07/17] fix: answer a rejected OpenCode question so the model can still reply OpenCode ends the turn on its question-reject route, which carries no feedback, so a question the room rejected, auto_reject dismissed, or a timeout expired left the turn without a reply and the turn rule reported it as a missing reply. The adapter now sends the rejection as a declining answer, the way a permission reject already carries feedback, so the model sees the refusal and answers the room. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016nwuEBtdtneuDkJHcvijDQ Signed-off-by: Alexander Zaikman --- AGENTS.md | 2 +- docs/adapters/opencode.md | 8 ++- src/band/adapters/opencode/approvals.py | 19 ++++++- tests/adapters/opencode/helpers.py | 19 +++++++ tests/adapters/opencode/test_approvals.py | 56 ++++++++++--------- .../adapters/opencode/test_detached_turns.py | 32 +++++++++++ 6 files changed, 106 insertions(+), 30 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e605c46a4..dee8b7f32 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -104,7 +104,7 @@ confined to an explicit AST-enforced allowlist `OpencodeAdapter` maps each Band room to a session on a running `opencode serve`. Band tools are never gated behind approval, unlike other tool calls — see [docs/adapters/opencode.md](docs/adapters/opencode.md) for this -and three more invariants that are easy to break. +and four more invariants that are easy to break. ## ACP (Agent Client Protocol) Integration diff --git a/docs/adapters/opencode.md b/docs/adapters/opencode.md index 47db9097c..54377f62a 100644 --- a/docs/adapters/opencode.md +++ b/docs/adapters/opencode.md @@ -4,7 +4,7 @@ `opencode serve`: room messages become prompts, and the server's SSE stream is relayed back as room messages, tool narration, and error events. -Four invariants are easy to break and expensive to rediscover: +Five invariants are easy to break and expensive to rediscover: - **Band tools are never gated.** A `permission.asked` naming one of the adapter's own registered tools is auto-approved with `always` in *every* @@ -22,6 +22,12 @@ Four invariants are easy to break and expensive to rediscover: - **The model is told the current `chat_id` every turn.** The band MCP tools' schemas require it, so without the per-turn Room Context block the platform tools are uncallable. +- **A declined ask must hand the turn back to the model.** OpenCode ends the + turn on a bare permission reject and on its question-reject route, so the model + never replies and the turn is reported as a missing reply. A permission reject + carries feedback, and a rejected question (by the room, `auto_reject`, or a + timeout) is answered with a decline instead of rejected; either way the model + sees the refusal and answers the room. `turn_timeout_s` bounds *compute*: time parked on a manual approval is excluded, since the ask carries its own `approval_wait_timeout_s` expiry. diff --git a/src/band/adapters/opencode/approvals.py b/src/band/adapters/opencode/approvals.py index 897c429a6..d4d9b1ae8 100644 --- a/src/band/adapters/opencode/approvals.py +++ b/src/band/adapters/opencode/approvals.py @@ -155,6 +155,13 @@ def parse_question_answers( "This request was declined. Do not retry it or try another way to do the " "same thing; reply to the user instead." ) +# A rejected question is answered with this, one line per question, rather +# than sent to POST /question/{id}/reject: that route carries no feedback, so +# OpenCode ends the turn on it and the model never replies to the room. +DECLINED_QUESTION_ANSWER = ( + "The user declined to answer this question. Do not ask it again; continue " + "and reply to the user without it." +) def format_question_prompt(questions: list[OpencodeQuestion], request_id: str) -> str: @@ -670,10 +677,18 @@ async def _reply_question( async def _send_question_reject( self, entry: DecisionEntry[PendingQuestion] ) -> bool: - """Perform the reject I/O for an already-claimed question.""" + """Perform the reject I/O for an already-claimed question. + + A question with nothing to answer (malformed) is the only one sent to + OpenCode's reject route; see ``DECLINED_QUESTION_ANSWER``. + """ + declined = [[DECLINED_QUESTION_ANSWER] for _ in entry.payload.questions] try: async with self._client_reply("reject question", entry.token) as client: - await client.reject_question(entry.token) + if declined: + await client.reply_question(entry.token, answers=declined) + else: + await client.reject_question(entry.token) except ApprovalReplyError: return False self._forget(self._questions, entry) diff --git a/tests/adapters/opencode/helpers.py b/tests/adapters/opencode/helpers.py index ee5f13b36..f05189b46 100644 --- a/tests/adapters/opencode/helpers.py +++ b/tests/adapters/opencode/helpers.py @@ -14,6 +14,7 @@ from band.adapters.opencode import OpencodeAdapter from band.adapters.opencode.adapter import MCP_REGISTRATION_CONNECTED_STATUS +from band.adapters.opencode.approvals import DECLINED_QUESTION_ANSWER from band.core.exceptions import BandToolError from band.core.protocols import AgentToolsProtocol from band.core.types import AgentInput, HistoryProvider, PlatformMessage @@ -540,3 +541,21 @@ def question( def _unique_id(prefix: str) -> str: return f"{prefix}-{uuid4().hex[:8]}" + + +def _is_decline(reply: dict[str, Any]) -> bool: + return all(answer == [DECLINED_QUESTION_ANSWER] for answer in reply["answers"]) + + +def declined_questions(client: FakeOpencodeClient) -> list[str]: + """Ids of the questions the adapter rejected by answering with the decline.""" + return [r["request_id"] for r in client.question_replies if _is_decline(r)] + + +def answered_questions(client: FakeOpencodeClient) -> list[tuple[str, list[list[str]]]]: + """The room's real answers, as (question id, answers), declines left out.""" + return [ + (r["request_id"], r["answers"]) + for r in client.question_replies + if not _is_decline(r) + ] diff --git a/tests/adapters/opencode/test_approvals.py b/tests/adapters/opencode/test_approvals.py index 19c204e7e..dedde2df1 100644 --- a/tests/adapters/opencode/test_approvals.py +++ b/tests/adapters/opencode/test_approvals.py @@ -13,6 +13,7 @@ from band.adapters.opencode import OpencodeAdapter, OpencodeAdapterConfig from band.adapters.opencode.approvals import ( + DECLINED_QUESTION_ANSWER, REJECTED_PERMISSION_FEEDBACK, ApprovalPorts, RoomApprovals, @@ -32,6 +33,8 @@ AskFactory, FakeOpencodeClient, RaisingSendTools, + answered_questions, + declined_questions, event_message_updated, event_permission, event_question, @@ -104,13 +107,10 @@ async def reply_permission( async def reply_question( self, request_id: str, *, answers: list[list[str]] ) -> None: - await self._block("question") + declined = all(answer == [DECLINED_QUESTION_ANSWER] for answer in answers) + await self._block("reject" if declined else "question") await super().reply_question(request_id, answers=answers) - async def reject_question(self, request_id: str) -> None: - await self._block("reject") - await super().reject_question(request_id) - class FailingReplyClient(FakeOpencodeClient): async def reply_permission( @@ -118,8 +118,10 @@ async def reply_permission( ) -> None: raise RuntimeError("permission reply failed") - async def reject_question(self, request_id: str) -> None: - raise RuntimeError("question rejection failed") + async def reply_question( + self, request_id: str, *, answers: list[list[str]] + ) -> None: + raise RuntimeError("question reply failed") async def _ignore_abort() -> None: @@ -418,7 +420,7 @@ async def test_reject_by_id_targets_a_pending_question_not_a_stale_permission( await approvals.on_question_asked(asks.question("q-1")) assert await approvals.try_handle_reply("reject q-1", "user-1") - assert client.question_rejections == ["q-1"] + assert declined_questions(client) == ["q-1"] async def test_reject_question_id_is_not_stolen_by_a_pending_permission( @@ -434,7 +436,7 @@ async def test_reject_question_id_is_not_stolen_by_a_pending_permission( await approvals.on_question_asked(asks.question("q-1")) assert await approvals.try_handle_reply("reject q-1", "user-1") - assert client.question_rejections == ["q-1"] + assert declined_questions(client) == ["q-1"] assert client.permission_replies == [] assert approvals.awaiting_human() @@ -507,7 +509,7 @@ async def test_bare_reject_with_both_permission_and_question_pending( assert await approvals.try_handle_reply("reject", "user-1") assert client.permission_replies == [] - assert client.question_rejections == [] + assert declined_questions(client) == [] assert ( "Both an approval and a question are pending" in tools.messages_sent[-1]["content"] @@ -522,7 +524,7 @@ async def test_reject_shared_id_rejects_the_question(asks: AskFactory) -> None: await approvals.on_question_asked(asks.question(shared)) assert await approvals.try_handle_reply(f"reject {shared}", "user-1") - assert client.question_rejections == [shared] + assert declined_questions(client) == [shared] assert client.permission_replies == [] @@ -531,7 +533,7 @@ async def test_named_reject_of_a_never_asked_question_is_not_consumed() -> None: approvals = make_room_approvals(client) assert not await approvals.try_handle_reply("reject never-asked-q", "user-1") - assert client.question_rejections == [] + assert declined_questions(client) == [] async def test_named_reject_of_an_already_resolved_question_is_consumed( @@ -542,10 +544,10 @@ async def test_named_reject_of_an_already_resolved_question_is_consumed( approvals = make_room_approvals(client, tools=tools) await approvals.on_question_asked(asks.question("q-1")) assert await approvals.try_handle_reply("reject q-1", "user-1") - client.question_rejections.clear() + client.question_replies.clear() assert await approvals.try_handle_reply("reject q-1", "user-1") - assert client.question_rejections == [] + assert declined_questions(client) == [] assert "no longer pending" in tools.messages_sent[-1]["content"] @@ -576,7 +578,7 @@ async def test_room_traffic_while_a_permission_reply_is_in_flight( assert [ (reply["permission_id"], reply["reply"]) for reply in client.permission_replies ] == [("req-1", "once"), ("req-2", "reject")] - assert client.question_rejections == ["q-1"] + assert declined_questions(client) == ["q-1"] assert not approvals.awaiting_human() @@ -607,10 +609,12 @@ async def test_room_traffic_while_question_answers_are_in_flight( assert await approvals.try_handle_reply("Now", "user-1") assert first is not None and first.result() - assert [ - (reply["request_id"], reply["answers"]) for reply in client.question_replies - ] == [("q-2", [["Bob"]]), ("q-1", [["Alice"]]), ("q-4", [["Now"]])] - assert client.question_rejections == ["q-3"] + assert answered_questions(client) == [ + ("q-2", [["Bob"]]), + ("q-1", [["Alice"]]), + ("q-4", [["Now"]]), + ] + assert declined_questions(client) == ["q-3"] assert not approvals.awaiting_human() @@ -636,7 +640,7 @@ async def test_a_redelivered_ask_waits_out_its_own_deadline(asks: AskFactory) -> await approvals.on_question_asked(asks.question("q-1")) await time_passes(ASK_DEADLINE_S * 2 / 3) - assert client.permission_replies == client.question_rejections == [] + assert client.permission_replies == [] and declined_questions(client) == [] assert await approvals.try_handle_reply("approve req-1", "user-1") assert await approvals.try_handle_reply("Alice", "user-1") assert [reply["reply"] for reply in client.permission_replies] == ["once"] @@ -877,7 +881,7 @@ async def test_auto_decline_approval_mode() -> None: async def test_auto_reject_question_mode() -> None: fake_client = FakeOpencodeClient( prompt_event_sequences=[[event_question("sess-1", "q-1", "What to do?")]], - reject_question_events={"q-1": [event_session_idle("sess-1")]}, + reply_question_events={"q-1": [event_session_idle("sess-1")]}, ) adapter = OpencodeAdapter( config=OpencodeAdapterConfig(question_mode="auto_reject"), @@ -896,7 +900,7 @@ async def test_auto_reject_question_mode() -> None: room_id="room-1", ) - assert fake_client.question_rejections == ["q-1"] + assert declined_questions(fake_client) == ["q-1"] assert not any( "asked question" in m["content"].lower() for m in tools.messages_sent ) @@ -914,7 +918,7 @@ async def test_a_turn_nobody_answers_expires_into_its_timeout_replies() -> None: event_question("sess-1", "q-1", "Pick a color"), ] ], - reject_question_events={"q-1": [event_session_idle("sess-1")]}, + reply_question_events={"q-1": [event_session_idle("sess-1")]}, ) adapter = OpencodeAdapter( config=OpencodeAdapterConfig( @@ -941,7 +945,7 @@ async def test_a_turn_nobody_answers_expires_into_its_timeout_replies() -> None: await tools.until(lambda: len(events_of_type(tools, "error")) == 2) assert fake_client.permission_replies == [{**DECLINED, "permission_id": "perm-1"}] - assert fake_client.question_rejections == ["q-1"] + assert declined_questions(fake_client) == ["q-1"] notices = events_of_type(tools, "error") assert all("timed out" in notice["content"].lower() for notice in notices) assert all("failure" not in notice["metadata"] for notice in notices) @@ -969,7 +973,7 @@ class ExpiryCase: "question": ExpiryCase( operation="reject", raise_ask=lambda room, asks: room.on_question_asked(asks.question("ask-1")), - sent=lambda client: client.question_rejections, + sent=declined_questions, expired_reply="ask-1", ), } @@ -1218,7 +1222,7 @@ async def test_cleanup_with_pending_question() -> None: # No question reply should have been sent, but the abandoned session # must still be told to stop working. assert fake_client.question_replies == [] - assert fake_client.question_rejections == [] + assert declined_questions(fake_client) == [] assert fake_client.aborted_sessions == ["sess-1"] diff --git a/tests/adapters/opencode/test_detached_turns.py b/tests/adapters/opencode/test_detached_turns.py index 5f6f0e850..d65022bb8 100644 --- a/tests/adapters/opencode/test_detached_turns.py +++ b/tests/adapters/opencode/test_detached_turns.py @@ -16,8 +16,11 @@ RawOpencodeEvent, ServerStep, agent_input, + event_message_updated, event_permission, + event_question, event_session_idle, + event_text_part, wait_for, ) @@ -100,3 +103,32 @@ async def test_a_detached_turn_cancelled_by_cleanup_posts_nothing( await adapter.on_cleanup("room-1") assert failure_reports(tools) == [] + + +async def test_a_rejected_question_hands_the_turn_back_to_the_model( + make_adapter, tools +) -> None: + """OpenCode ends a turn on its question-reject route, so the room's + `reject` goes back as a declining answer and the model's reply then + completes the parked turn.""" + adapter = make_adapter( + FakeOpencodeClient( + prompt_event_sequences=[[event_question("sess-1", "q-1", "Which word?")]], + reply_question_events={ + "q-1": [ + event_message_updated("sess-1", "msg-1"), + event_text_part("sess-1", "msg-1", "OK, I'll go without one."), + event_session_idle("sess-1"), + ] + }, + ) + ) + await adapter.on_started("OpenCode Agent", "A coding agent") + await adapter.on_event(agent_input("ask me which word", tools)) + + await adapter.on_event(agent_input("reject", FakeAgentTools())) + await wait_for(lambda: tools.turn.replied) + await adapter.on_cleanup("room-1") + + assert tools.messages_sent[-1]["content"] == "OK, I'll go without one." + assert failure_reports(tools) == [] From b6b86a7ea4627404e570c8428b8bfe82539447cd Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Mon, 5 Oct 2026 01:38:04 +0300 Subject: [PATCH 08/17] fix: judge Slack-inbound turns and deduped replies; address review - Slack-originated turns now run through SimpleAdapter.run_judged_turn, the same judge on_event uses, so a silent Slack turn is reported again; its already-reported failure is not re-logged as a background-task crash. - A band_send_message the claude_sdk dedup wrapper suppresses (cache hit or joined in-flight send) records REPLY on the current turn, so a CLI retry of the previous reply is no longer a false missing reply. - report_unsettled_turn logs a WARNING naming the room; judge_detached_turn replaces four open-coded detached checks; MISSING_REPLY is the one failure definition the fakes reuse. - strands/pydantic_ai record a declared effect once: the hook covers native tools only, CustomToolDef tools record through execute_custom_tool. - Tests: judging opt-outs pinned per adapter, Codex interrupted-turn notice, undeclared custom-tool rows for six adapters, dedup and Slack regressions; FakeAgentTools.chat replaces repeated content projections. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016nwuEBtdtneuDkJHcvijDQ Signed-off-by: Alexander Zaikman --- docs/turn-outcome.md | 15 ++--- src/band/adapters/claude_sdk.py | 13 ++-- src/band/adapters/codex.py | 6 +- src/band/adapters/cursor_acp.py | 8 +-- src/band/adapters/opencode/adapter.py | 11 +--- src/band/adapters/pydantic_ai.py | 34 ++++------- src/band/adapters/strands.py | 13 +++- src/band/core/simple_adapter.py | 61 +++++++++++++------ src/band/core/turn.py | 29 +++++++-- src/band/core/types.py | 24 +------- .../integrations/claude_sdk/dedup_tools.py | 9 +++ src/band/integrations/claude_sdk/tools.py | 8 +-- src/band/integrations/mcp/engine.py | 4 +- src/band/integrations/slack/adapter.py | 24 +++++--- src/band/testing/fake_tools.py | 10 ++- .../failures/test_turn_failure_surfacing.py | 15 +++++ tests/adapters/codexturns.py | 4 +- .../copilot_sdk/test_ask_user_room.py | 6 +- tests/adapters/copilot_sdk/test_reply.py | 2 +- .../copilot_sdk/test_tool_bridging.py | 2 +- tests/adapters/langgraph/test_lifecycle.py | 14 +++++ tests/adapters/opencode/helpers.py | 7 --- tests/adapters/opencode/test_lifecycle.py | 2 +- tests/adapters/opencode/test_turns.py | 12 ++-- tests/adapters/test_anthropic_adapter.py | 12 ++-- tests/adapters/test_codex_adapter.py | 24 +++++--- tests/adapters/test_crewai_adapter.py | 2 +- tests/adapters/test_crewai_flow_phase4.py | 2 +- tests/adapters/test_crewai_flow_phase5.py | 4 +- tests/adapters/test_gemini_adapter.py | 12 ++-- tests/adapters/test_google_adk_adapter.py | 15 +++-- tests/adapters/test_letta_adapter.py | 9 +-- tests/adapters/test_pydantic_ai_adapter.py | 33 ++++------ tests/adapters/test_strands_adapter.py | 25 ++------ tests/core/test_delivery.py | 6 +- .../test_turn_outcome.py | 16 ++++- .../turnprobes/__init__.py | 23 ++++++- .../integrations/a2a/gateway/test_adapter.py | 6 ++ tests/integrations/a2a/test_adapter.py | 7 +++ tests/integrations/acp/test_server_adapter.py | 5 ++ .../claude_sdk/test_dedup_tools.py | 49 +++++++++++++++ tests/integrations/codex/test_adapter_e2e.py | 20 +++--- .../langgraph/test_custom_tool_defs.py | 22 ++++--- .../slack/test_retry_idempotency.py | 5 +- tests/integrations/slack/test_wrapping.py | 34 +++++++++-- tests/integrations/test_omp.py | 2 - tests/runtime/test_turn.py | 21 +++++-- tests/runtime/test_turn_effect.py | 7 +-- 48 files changed, 426 insertions(+), 268 deletions(-) diff --git a/docs/turn-outcome.md b/docs/turn-outcome.md index 9ba463291..8f14e30ce 100644 --- a/docs/turn-outcome.md +++ b/docs/turn-outcome.md @@ -1,9 +1,11 @@ # Turn outcome Every message pushed to an agent @mentions it, so every turn owes the room an -answer: a reply, or a deliberate decision not to reply. `SimpleAdapter.on_event` -judges each turn with one rule from `band-sdk-core`, shared with the TypeScript -SDK, and tells the platform honestly whether the delivery succeeded. +answer: a reply, or a deliberate decision not to reply. +`SimpleAdapter.run_judged_turn`, which `on_event` and the Slack adapter's +inbound path both call, judges each turn with one rule from `band-sdk-core`, +shared with the TypeScript SDK, and tells the platform honestly whether the +delivery succeeded. ## The rule @@ -216,18 +218,17 @@ cancelled turn reports nothing: ```python import asyncio -from band.core.turn import report_unsettled_turn +from band.core.turn import judge_detached_turn from band.testing.fake_tools import FakeAgentTools, reported_failures async def main() -> None: tools = FakeAgentTools() - tools.turn.judged = True # set by SimpleAdapter.on_event + tools.turn.judged = True # set by SimpleAdapter.run_judged_turn tools.turn.detach() # released while waiting for an approval # ...later, on the turn's normal completion path (never a ``finally``): - if tools.turn.detached: - await report_unsettled_turn(tools) + await judge_detached_turn(tools, room_id="room-1") assert len(reported_failures(tools)) == 1 diff --git a/src/band/adapters/claude_sdk.py b/src/band/adapters/claude_sdk.py index 28d00f0bd..ffef1f8b0 100644 --- a/src/band/adapters/claude_sdk.py +++ b/src/band/adapters/claude_sdk.py @@ -86,7 +86,7 @@ TurnResultAlreadyReported, ) from band.core.simple_adapter import SimpleAdapter -from band.core.turn import report_unsettled_turn +from band.core.turn import judge_detached_turn from band.core.turn_lifecycle import ApprovalInterruptMixin from band.core.types import ( ApprovalMode, @@ -1323,9 +1323,11 @@ async def _on_tool_use( room_id: str, tools: AgentToolsProtocol, ) -> None: - """Register a pending call (to name its result) and narrate it. Shared by both envelopes a call can arrive in — the - protocol's assistant messages, and user messages when the call is - carried by a subagent/nested tool_use block.""" + """Register a pending call (to name its result) and narrate it. + + Shared by both envelopes a call can arrive in: the protocol's assistant + messages, and user messages carrying a subagent's nested tool_use block. + """ # Bare name for the cross-adapter tool_call record (the SDK # namespaces our tools mcp__band__*; see _semantic_tool_name). tool_name = self._semantic_tool_name(block.name) @@ -1398,8 +1400,7 @@ async def _on_turn_complete( raise TurnResultAlreadyReported(detail) if self._declined_the_reply(sdk_message.permission_denials, notified): tools.turn.settle() - if tools.turn.detached: - await report_unsettled_turn(tools) + await judge_detached_turn(tools, room_id=room_id) def _declined_the_reply( self, permission_denials: list[Any] | None, notified: set[str] | None diff --git a/src/band/adapters/codex.py b/src/band/adapters/codex.py index 1bce5d94e..ea8dc5bbe 100644 --- a/src/band/adapters/codex.py +++ b/src/band/adapters/codex.py @@ -42,7 +42,7 @@ send_event_safe, ) from band.core.simple_adapter import SimpleAdapter -from band.core.turn import report_unsettled_turn +from band.core.turn import judge_detached_turn from band.core.turn_lifecycle import ApprovalInterruptMixin from band.core.types import ( AgentInput, @@ -1094,8 +1094,8 @@ async def _run_turn( duration_s=_turn_duration_s, ) # A turn wound down by room cleanup was aborted, not missed. - if tools.turn.detached and room_id not in self._closing_rooms: - await report_unsettled_turn(tools) + if room_id not in self._closing_rooms: + await judge_detached_turn(tools, room_id=room_id) except DeliveryFailedError as e: reraise_delivery_cause(e) except TurnResultAlreadyReported: diff --git a/src/band/adapters/cursor_acp.py b/src/band/adapters/cursor_acp.py index 9cf311fc5..bf71e0550 100644 --- a/src/band/adapters/cursor_acp.py +++ b/src/band/adapters/cursor_acp.py @@ -13,7 +13,7 @@ from band.client.streaming import ControlMode from band.core.protocols import AgentToolsProtocol -from band.core.turn import report_unsettled_turn +from band.core.turn import judge_detached_turn from band.core.types import ApprovalMode, FeatureKwargs, PlatformMessage from band.integrations.acp.client_adapter import ( ACPClientAdapter, @@ -282,10 +282,8 @@ async def _run_turn( is_session_bootstrap=is_session_bootstrap, room_id=room_id, ) - # on_event already returned for a detached turn, so it is judged - # here, at its real end; a failed or cancelled turn never gets here. - if tools.turn.detached: - await report_unsettled_turn(tools) + # A failed or cancelled turn never gets here. + await judge_detached_turn(tools, room_id=room_id) finally: self._cursor_profile.bind_session(None) if self._active_turn is turn: diff --git a/src/band/adapters/opencode/adapter.py b/src/band/adapters/opencode/adapter.py index 1deb8b8ab..1528b919f 100644 --- a/src/band/adapters/opencode/adapter.py +++ b/src/band/adapters/opencode/adapter.py @@ -27,7 +27,7 @@ AgentToolsProtocol, ) from band.core.simple_adapter import SimpleAdapter -from band.core.turn import report_unsettled_turn +from band.core.turn import judge_detached_turn from band.core.types import ( AdapterFeatures, Capability, @@ -1065,7 +1065,7 @@ async def _watch_turn_completion( else: try: await self._deliver_fallback_text(room_state.room_id, turn) - await self._judge_detached_turn(turn) + await judge_detached_turn(turn.tools, room_id=room_state.room_id) await self._emit_turn_usage(turn) except Exception: logger.exception( @@ -1084,13 +1084,6 @@ async def _watch_turn_completion( expected_task=asyncio.current_task(), ) - @staticmethod - async def _judge_detached_turn(turn: TurnState) -> None: - """Judge a turn released early to wait on a human, since ``on_event`` - returned before it ended.""" - if turn.tools.turn.detached: - await report_unsettled_turn(turn.tools) - async def _abort_turn(self, turn: TurnState, reason: str) -> None: """Best-effort: tell OpenCode to stop working on this room's session.""" try: diff --git a/src/band/adapters/pydantic_ai.py b/src/band/adapters/pydantic_ai.py index 5e7b89daf..049f0c386 100644 --- a/src/band/adapters/pydantic_ai.py +++ b/src/band/adapters/pydantic_ai.py @@ -60,9 +60,7 @@ from band.integrations.pydantic_ai.tools import build_band_pydantic_ai_tools from band.runtime.custom_tools import ( CustomToolDef, - declared_effect, declared_effects, - declares_turn_effect, get_custom_tool_name, invoke_validated_custom_tool, ) @@ -155,24 +153,16 @@ def _drop_blank_text( def _custom_tool_def_to_callable(tool_def: CustomToolDef) -> Callable[..., Any]: - """Adapt a portable ``CustomToolDef`` (InputModel, handler) to a native pydantic-ai - tool callable — the same custom-tool form the other adapters accept. - - pydantic-ai flattens a single Pydantic-model parameter into the tool's arguments, - so the wrapper keeps the ``(args: InputModel)`` registration shape after a - leading ``RunContext``, whose deps are the room's tools, so each call records - its effect on that room's turn. Execution is routed through the shared - ``invoke_validated_custom_tool`` so the CustomToolDef contract matches every - other adapter: async handlers are awaited and - zero-argument handlers (empty InputModel) are called without args — a plain sync - passthrough would hand pydantic-ai an unawaited coroutine or raise TypeError for - those. pydantic-ai has already validated ``args`` into the InputModel, so the - instance is passed through directly — a dump/re-validate round-trip would break - models using field aliases. The wrapper carries the stable tool name (derived - from the model) and its declared turn effect, so the tool name and the - turn effect match the tuple adapters exactly. + """Adapt a portable ``CustomToolDef`` to a native pydantic-ai tool callable. + + The wrapper takes ``RunContext`` (whose deps are the room's tools), then + ``args: InputModel``, which pydantic-ai flattens into the tool's arguments + and validates. ``invoke_validated_custom_tool`` runs the handler exactly as + every other adapter does and records its declared effect on the room's turn. + The validated instance is passed through as is, since a dump/re-validate + round-trip would break aliased fields. """ - input_model, handler = tool_def + input_model, _ = tool_def async def native(ctx: RunContext[AgentToolsProtocol], args: Any) -> Any: return await invoke_validated_custom_tool(tool_def, args, turn=ctx.deps.turn) @@ -184,8 +174,6 @@ async def native(ctx: RunContext[AgentToolsProtocol], args: Any) -> Any: "args": input_model, "return": str, } - if (effect := declared_effect(handler)) is not None: - declares_turn_effect(effect)(native) return native @@ -312,8 +300,8 @@ def __init__( _custom_tool_def_to_callable(tool) if isinstance(tool, tuple) else tool for tool in (additional_tools or []) ] - # Effects the custom tools declared on their function; an undeclared - # custom tool only observes, so it cannot complete a turn. + # Effects the native callables declared; a converted CustomToolDef + # declares none, since invoke_validated_custom_tool records its effect. self._custom_effects = declared_effects( (fn.__name__, fn) for fn in self._custom_tools ) diff --git a/src/band/adapters/strands.py b/src/band/adapters/strands.py index 0cbb78263..a472e794d 100644 --- a/src/band/adapters/strands.py +++ b/src/band/adapters/strands.py @@ -198,7 +198,11 @@ def _registered_name(tool: StrandsCustomTool) -> str: def _custom_tool_effects( custom_tools: list[StrandsCustomTool], ) -> dict[str, TurnEffect]: - """Validate custom tool names and collect the turn effects they declared.""" + """Validate custom tool names and collect the native tools' declared effects. + + A portable ``(InputModel, handler)`` tool records its own effect through + ``execute_custom_tool``. + """ names = [_registered_name(tool) for tool in custom_tools] # Strands' registry is last-wins, so a collision would silently replace the # platform tool the room depends on. @@ -206,8 +210,11 @@ def _custom_tool_effects( if shadowed: raise ValueError(f"Custom tools may not shadow Band platform tools: {shadowed}") - handlers = (tool[1] if isinstance(tool, tuple) else tool for tool in custom_tools) - return declared_effects(zip(names, handlers, strict=True)) + return declared_effects( + (name, tool) + for name, tool in zip(names, custom_tools, strict=True) + if not isinstance(tool, tuple) + ) def _bind_custom_tools( diff --git a/src/band/core/simple_adapter.py b/src/band/core/simple_adapter.py index 0b9ad27ca..a45143654 100644 --- a/src/band/core/simple_adapter.py +++ b/src/band/core/simple_adapter.py @@ -383,14 +383,10 @@ def judges_turns(self) -> bool: async def on_event(self, inp: AgentInput) -> None: """Implements FrameworkAdapter.on_event(). - Every framework adapter's turn passes through here (the one place - that calls the per-adapter ``on_message`` override), so this is where + Every platform-delivered turn passes through here, so this is where the turn's trace-context correlation window opens -- every log line emitted anywhere during this turn's processing picks it up via - ``band.logging_config``'s log filter -- and where the turn is judged. - A turn that ended without completing is reported once, then raised as - ``TurnResultAlreadyReported`` so the runtime marks it FAILED without - a second report. A detached turn is judged at its real end instead. + ``band.logging_config``'s log filter. """ with trace_context_scope(): # Convert history if converter is set @@ -401,11 +397,7 @@ async def on_event(self, inp: AgentInput) -> None: # Adapters without converters should type as SimpleAdapter[HistoryProvider] converted_history = inp.history - turn = inp.tools.turn - turn.judged = self.judges_turns and not is_contact_hub_turn( - sender_type=inp.msg.sender_type, sender_id=inp.msg.sender_id - ) - await self.on_message( + await self.run_judged_turn( msg=inp.msg, tools=inp.tools, history=cast("H", converted_history), @@ -414,9 +406,44 @@ async def on_event(self, inp: AgentInput) -> None: is_session_bootstrap=inp.is_session_bootstrap, room_id=inp.room_id, ) - if ( - turn.judged - and not turn.detached - and await report_unsettled_turn(inp.tools) - ): - raise TurnResultAlreadyReported("turn ended without a reply") + + async def run_judged_turn( + self, + *, + msg: PlatformMessage, + tools: AgentToolsProtocol, + history: H, + participants_msg: str | None, + contacts_msg: str | None, + is_session_bootstrap: bool, + room_id: str, + ) -> None: + """Run one turn through ``on_message`` and judge it. + + The entry point for every turn, including those a wrapper like the + Slack adapter receives outside ``on_event``. A turn that ended without + completing is reported once, then raised as ``TurnResultAlreadyReported`` + so the runtime marks it FAILED without a second report. A detached turn + is judged at its real end instead. + """ + turn = tools.turn + turn.judged = self.judges_turns and not is_contact_hub_turn( + sender_type=msg.sender_type, sender_id=msg.sender_id + ) + # Positional, as ``on_message`` declares them: an override may rename + # its positional parameters. + await self.on_message( + msg, + tools, + history, + participants_msg, + contacts_msg, + is_session_bootstrap=is_session_bootstrap, + room_id=room_id, + ) + if ( + turn.judged + and not turn.detached + and await report_unsettled_turn(tools, room_id=room_id) + ): + raise TurnResultAlreadyReported("turn ended without a reply") diff --git a/src/band/core/turn.py b/src/band/core/turn.py index 3a0bb4563..580de9363 100644 --- a/src/band/core/turn.py +++ b/src/band/core/turn.py @@ -53,8 +53,8 @@ def note_reported(self) -> None: def detach(self) -> None: """Move a judged turn's verdict to the adapter's real end of the turn. - A turn ``on_event`` does not judge stays attached, so the adapter's - ``if turn.detached`` report never fires for it. + A turn ``on_event`` does not judge stays attached, so + ``judge_detached_turn`` never reports it. """ if self.judged: self._detached = True @@ -73,14 +73,33 @@ def complete(self) -> bool: return self._ledger.verdict() == TurnVerdict.Complete -async def report_unsettled_turn(tools: AgentToolsProtocol) -> bool: +#: What a turn that ended without completing reports to the room. +MISSING_REPLY = AgentFailure( + TURN_FAILURE_PROVIDER, band_sdk_core.missing_reply_message() +) + + +async def report_unsettled_turn(tools: AgentToolsProtocol, *, room_id: str) -> bool: """Report a turn that ended without completing; return whether it did. The one place a missing-reply verdict becomes a room-visible failure. """ if tools.turn.complete: return False - await tools.send_failure( - AgentFailure(TURN_FAILURE_PROVIDER, band_sdk_core.missing_reply_message()) + logger.warning( + "Room %s: turn ended without a reply (detached=%s)", + room_id, + tools.turn.detached, ) + await tools.send_failure(MISSING_REPLY) return True + + +async def judge_detached_turn(tools: AgentToolsProtocol, *, room_id: str) -> None: + """Judge a detached turn at the adapter's real end of it. + + ``on_event`` returned before such a turn ended; an attached turn was + already judged there. + """ + if tools.turn.detached: + await report_unsettled_turn(tools, room_id=room_id) diff --git a/src/band/core/types.py b/src/band/core/types.py index 784838310..1addf53d0 100644 --- a/src/band/core/types.py +++ b/src/band/core/types.py @@ -456,30 +456,10 @@ def __init__( ) -# --- Constants for synthetic messages (injected by SDK, not from platform) --- -# -# These constants define the sender identity for SDK-generated messages that are -# injected into the event queue but don't originate from the platform WebSocket. -# -# Primary use case: ContactEventHandler's HUB_ROOM strategy creates synthetic -# MessageEvents to route contact events to the hub room for LLM processing. -# These messages need a consistent sender identity that: -# 1. Clearly indicates they're system-generated (not from a real user/agent) -# 2. Can be filtered by preprocessing if needed -# 3. Are recognizable in UI/logs for debugging -# - -# Sender type for synthetic messages. Matches the platform's "System" type used -# for other system-generated content. +# Sender identity of the messages ContactEventHandler's HUB_ROOM strategy +# injects into the hub room; they never come from the platform. SYNTHETIC_SENDER_TYPE = "System" - -# Sender ID for synthetic contact event messages. This is a logical identifier -# (not a UUID) that allows filtering/identification of contact-related synthetic -# messages. Used in MessageEvent.payload.sender_id for hub room injections. SYNTHETIC_CONTACT_EVENTS_SENDER_ID = "contact-events" - -# Human-readable sender name displayed in UI/logs for synthetic contact event -# messages. Used in MessageEvent.payload.sender_name. SYNTHETIC_CONTACT_EVENTS_SENDER_NAME = "Contact Events" diff --git a/src/band/integrations/claude_sdk/dedup_tools.py b/src/band/integrations/claude_sdk/dedup_tools.py index 4b90ea623..6ad7102a7 100644 --- a/src/band/integrations/claude_sdk/dedup_tools.py +++ b/src/band/integrations/claude_sdk/dedup_tools.py @@ -42,6 +42,9 @@ from collections import OrderedDict from typing import TYPE_CHECKING, Any +from band.runtime.tools.types import BandTool +from band.runtime.turn import record_tool_result + if TYPE_CHECKING: from band.core.protocols import AgentToolsProtocol @@ -157,6 +160,9 @@ async def send_message( async with self._lock: self._evict_expired_locked(now) key = (content, _normalize_mentions(mentions)) + # A deduped send is this turn's reply too: the original already + # reached the room, but recorded it on its own turn. + turn = self._inner.turn cached = self._recent_sends.get(key) if cached is not None: @@ -172,6 +178,7 @@ async def send_message( len(content), len(key[1]), ) + record_tool_result(turn, BandTool.SEND_MESSAGE, cached_result) return cached_result existing = self._in_flight.get(key) @@ -195,6 +202,8 @@ async def send_message( raise await self._finalize_in_flight_send(key, task, now) + if existing is not None: + record_tool_result(turn, BandTool.SEND_MESSAGE, result) return result # --- transparent passthrough -------------------------------------- diff --git a/src/band/integrations/claude_sdk/tools.py b/src/band/integrations/claude_sdk/tools.py index a7371e87e..5451261f8 100644 --- a/src/band/integrations/claude_sdk/tools.py +++ b/src/band/integrations/claude_sdk/tools.py @@ -31,7 +31,7 @@ from band.core.exceptions import BandToolError from band.core.protocols import AgentToolsProtocol from band.core.types import Capability -from band.integrations.mcp.engine import extend_with_chat_id +from band.integrations.mcp.engine import extend_with_chat_id, room_turn from band.runtime.custom_tools import ( CustomToolDef, execute_custom_tool, @@ -278,10 +278,8 @@ def _build_custom_sdk_tool( async def handler(args: dict[str, Any]) -> dict[str, Any]: try: tool_args = {k: v for k, v in args.items() if k != CHAT_ID_FIELD_NAME} - tools = get_tools(_room_id(args, include_room_id=include_room_id)) - result = await execute_custom_tool( - tool_def, tool_args, turn=tools.turn if tools is not None else None - ) + turn = room_turn(get_tools, _room_id(args, include_room_id=include_room_id)) + result = await execute_custom_tool(tool_def, tool_args, turn=turn) return _make_result(result) except Exception as error: logger.exception("Custom tool %s failed", tool_name) diff --git a/src/band/integrations/mcp/engine.py b/src/band/integrations/mcp/engine.py index 10dc2daaf..830642145 100644 --- a/src/band/integrations/mcp/engine.py +++ b/src/band/integrations/mcp/engine.py @@ -506,7 +506,7 @@ async def execute(arguments: dict[str, Any]) -> Any: RoomToolResolver = Callable[[str], AgentToolsProtocol | None] -def _room_turn(get_tools: RoomToolResolver | None, chat_id: str | None) -> Turn | None: +def room_turn(get_tools: RoomToolResolver | None, chat_id: str | None) -> Turn | None: """The turn of the room ``chat_id`` names, if the tool is room-bound.""" if get_tools is None or chat_id is None: return None @@ -539,7 +539,7 @@ def build_custom_tool_registration( async def execute(arguments: dict[str, Any]) -> Any: kwargs = dict(arguments) chat_id = kwargs.pop(CHAT_ID_FIELD_NAME, None) - turn = _room_turn(get_tools, chat_id) + turn = room_turn(get_tools, chat_id) result = await execute_custom_tool(tool_def, kwargs, turn=turn) return _serialize(result) diff --git a/src/band/integrations/slack/adapter.py b/src/band/integrations/slack/adapter.py index e319caa2d..7651bec24 100644 --- a/src/band/integrations/slack/adapter.py +++ b/src/band/integrations/slack/adapter.py @@ -8,8 +8,8 @@ room is bound to a Slack thread, the tools are wrapped so the brain's outgoing ``send_message`` is also posted to Slack. - **Slack webhook path**: ``SlackAdapter`` synthesises a - ``PlatformMessage`` from the Slack event and invokes - ``inner.on_message`` directly with REST-backed tools that tee replies + ``PlatformMessage`` from the Slack event and runs it as a judged turn + (``inner.run_judged_turn``) with REST-backed tools that tee replies back to the originating Slack thread. """ @@ -33,7 +33,7 @@ ) from band.converters.slack import SlackHistoryConverter from band.core.adapterconfig import BaseAdapterConfig -from band.core.protocols import AgentToolsProtocol +from band.core.protocols import AgentToolsProtocol, TurnResultAlreadyReported from band.core.simple_adapter import SimpleAdapter from band.core.types import ( AdapterFeatures, @@ -805,7 +805,7 @@ async def _handle_event(self, app: SlackApp, payload: dict[str, Any]) -> None: async def _invoke_brain_for_slack_event( self, app: SlackApp, event: dict[str, Any] ) -> None: - """Synthesize a ``PlatformMessage`` and call ``inner.on_message``.""" + """Synthesize a ``PlatformMessage`` and run it as the brain's judged turn.""" text = event.get("text", "") channel = event.get("channel") slack_user = event.get("user", "") @@ -895,15 +895,19 @@ async def _invoke_brain_for_slack_event( await self._set_status(slack_client, channel, thread_ts, STATUS_THINKING) try: - await self._inner.on_message( - synthesized, - tools, - history, - SLACK_CONTEXT_NOTE, - None, + await self._inner.run_judged_turn( + msg=synthesized, + tools=tools, + history=history, + participants_msg=SLACK_CONTEXT_NOTE, + contacts_msg=None, is_session_bootstrap=is_new_room, room_id=room_id, ) + except TurnResultAlreadyReported: + # The room already has the failure, and a Slack event has no + # delivery to mark FAILED. + logger.debug("Room %s: Slack turn failure already reported", room_id) finally: await self._set_status(slack_client, channel, thread_ts, "") diff --git a/src/band/testing/fake_tools.py b/src/band/testing/fake_tools.py index a55a28c0f..4ab28d045 100644 --- a/src/band/testing/fake_tools.py +++ b/src/band/testing/fake_tools.py @@ -50,7 +50,6 @@ from band.core.content import has_visible_content from band.core.exceptions import BandToolError from band.core.protocols import ( - TURN_FAILURE_PROVIDER, FailureMetadataKey, to_failure_event, ) @@ -60,7 +59,7 @@ TaskListState, task_ref, ) -from band.core.turn import Turn +from band.core.turn import MISSING_REPLY, Turn from band.core.types import ( Capability, ContactRequestAction, @@ -1120,6 +1119,11 @@ async def execute_tool_call_structured( # --- Assertion helpers --- + @property + def chat(self) -> list[str]: + """The content of every message posted to the room, in order.""" + return [m["content"] for m in self.messages_sent] + def assert_message_sent( self, *, @@ -1199,7 +1203,7 @@ def reported_failures(tools: FakeAgentTools) -> list[dict[str, Any]]: # The one failure a missing-reply verdict reports, as ``failure_reports`` lists it. -MISSING_REPLY_FAILURE = (TURN_FAILURE_PROVIDER, band_sdk_core.missing_reply_message()) +MISSING_REPLY_FAILURE = (MISSING_REPLY.provider, MISSING_REPLY.message) def failure_reports(tools: FakeAgentTools) -> list[tuple[str, str]]: diff --git a/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py b/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py index c07e7845f..0ce344318 100644 --- a/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py +++ b/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py @@ -208,3 +208,18 @@ async def test_band_no_reply_does_not_excuse_a_later_silent_turn( await room.send("Are you there?") assert room.failures == [MISSING_REPLY_TEXT] + + +async def test_a_turn_whose_only_reply_was_deduped_is_not_a_missing_reply( + claude_room: OpenRoom, +) -> None: + """The CLI re-sending the previous turn's reply is suppressed as a + duplicate, and it still answers the turn that sent it.""" + room = await claude_room() + room.claude.script([room.model_reply("Done.")], [room.model_reply("Done.")]) + + await room.send("do it") + await room.send("do it again", tools=room.fresh_tools()) + + assert room.chat == ["Done."] + assert room.failures == [] diff --git a/tests/adapters/codexturns.py b/tests/adapters/codexturns.py index 70a628018..630e85bfd 100644 --- a/tests/adapters/codexturns.py +++ b/tests/adapters/codexturns.py @@ -196,11 +196,11 @@ def event_request(request_id: int, method: str, params: dict[str, Any]) -> RpcEv ) -def turn_completed(turn_id: str = "turn-1") -> RpcEvent: +def turn_completed(turn_id: str = "turn-1", *, status: str = "completed") -> RpcEvent: """The notification that ends a scripted turn.""" return event_notification( "turn/completed", - {"turn": {"id": turn_id, "status": "completed", "items": [], "error": None}}, + {"turn": {"id": turn_id, "status": status, "items": [], "error": None}}, ) diff --git a/tests/adapters/copilot_sdk/test_ask_user_room.py b/tests/adapters/copilot_sdk/test_ask_user_room.py index c03f19c05..87c6fc2c3 100644 --- a/tests/adapters/copilot_sdk/test_ask_user_room.py +++ b/tests/adapters/copilot_sdk/test_ask_user_room.py @@ -78,7 +78,7 @@ async def test_question_is_the_turn_reply_final_text_suppressed(self): await run_message(adapter, tools) - contents = [m["content"] for m in tools.messages_sent] + contents = tools.chat assert contents == [render_room_question(QUESTION)] @pytest.mark.asyncio @@ -108,9 +108,7 @@ async def send_message(self, content, mentions=None): assert answers == [delivery_failed_answer(RuntimeError("platform down"))] # The turn itself survives: the model's reply reaches the room. - assert [m["content"] for m in tools.messages_sent] == [ - "Proceeding without input." - ] + assert tools.chat == ["Proceeding without input."] @pytest.mark.asyncio async def test_late_dispatch_after_turn_end_degrades_gracefully(self): diff --git a/tests/adapters/copilot_sdk/test_reply.py b/tests/adapters/copilot_sdk/test_reply.py index 5eeb5c3b7..5774000fd 100644 --- a/tests/adapters/copilot_sdk/test_reply.py +++ b/tests/adapters/copilot_sdk/test_reply.py @@ -294,4 +294,4 @@ async def model_sends_message(session: FakeCopilotSession) -> None: await run_message(adapter, tools) # The tool failed, so the fallback text must reach the room (no silent turn). - assert [m["content"] for m in tools.messages_sent] == ["Fallback reply"] + assert tools.chat == ["Fallback reply"] diff --git a/tests/adapters/copilot_sdk/test_tool_bridging.py b/tests/adapters/copilot_sdk/test_tool_bridging.py index 47d8fd905..4646d24ec 100644 --- a/tests/adapters/copilot_sdk/test_tool_bridging.py +++ b/tests/adapters/copilot_sdk/test_tool_bridging.py @@ -275,7 +275,7 @@ async def model_calls_notify(session: FakeCopilotSession) -> None: await run_message(adapter, tools) - relayed = [m["content"] for m in tools.messages_sent] == ["Closing text"] + relayed = tools.chat == ["Closing text"] assert relayed is closing_text_relayed assert tools.turn.complete diff --git a/tests/adapters/langgraph/test_lifecycle.py b/tests/adapters/langgraph/test_lifecycle.py index df51b6a78..f82d2599b 100644 --- a/tests/adapters/langgraph/test_lifecycle.py +++ b/tests/adapters/langgraph/test_lifecycle.py @@ -66,6 +66,20 @@ def test_advanced_pattern_with_static_graph(self): assert adapter._static_graph is mock_graph + @pytest.mark.parametrize( + ("pattern", "judged"), + [ + pytest.param({"graph_factory": MagicMock()}, True, id="graph-factory"), + pytest.param({"graph": MagicMock()}, False, id="static-graph"), + ], + ) + def test_only_a_graph_given_band_tools_is_judged( + self, pattern: dict[str, Any], judged: bool + ): + """A static graph never receives Band tools, so judging it would + report every turn as a missing reply.""" + assert LangGraphAdapter(**pattern).judges_turns is judged + def test_raises_without_llm_or_graph(self): """Should raise if neither llm nor graph_factory/graph provided.""" with pytest.raises(ValueError, match="Must provide either llm"): diff --git a/tests/adapters/opencode/helpers.py b/tests/adapters/opencode/helpers.py index f05189b46..3f25dc720 100644 --- a/tests/adapters/opencode/helpers.py +++ b/tests/adapters/opencode/helpers.py @@ -239,9 +239,6 @@ def agent_input(content: str, tools: FakeAgentTools) -> AgentInput: ) -#: The runtime's report of a turn that neither replied, declined nor worked. - - class RaisingSendTools(FakeAgentTools): """FakeAgentTools whose send_notice always fails, to exercise the best-effort ``_notify_room`` path: a room post that raises must be @@ -276,7 +273,6 @@ def __init__( prompt_event_sequences: list[list[ServerStep]] | None = None, reply_permission_events: dict[str, list[ServerStep]] | None = None, reply_question_events: dict[str, list[ServerStep]] | None = None, - reject_question_events: dict[str, list[ServerStep]] | None = None, get_session_missing: set[str] | None = None, prompt_exceptions: list[Exception] | None = None, serve_registrations: dict[str, str] | None = None, @@ -301,7 +297,6 @@ def __init__( self._prompt_event_sequences = list(prompt_event_sequences or []) self._reply_permission_events = reply_permission_events or {} self._reply_question_events = reply_question_events or {} - self._reject_question_events = reject_question_events or {} self._get_session_missing = get_session_missing or set() self._prompt_exceptions = list(prompt_exceptions or []) # Popped one at a time per register_mcp_server call; once exhausted, @@ -374,8 +369,6 @@ async def reply_question( async def reject_question(self, request_id: str) -> None: self.question_rejections.append(request_id) - for event in self._reject_question_events.get(request_id, []): - await self._queue.put(event) async def abort_session(self, session_id: str) -> None: self.aborted_sessions.append(session_id) diff --git a/tests/adapters/opencode/test_lifecycle.py b/tests/adapters/opencode/test_lifecycle.py index f20af9c79..518cfbda7 100644 --- a/tests/adapters/opencode/test_lifecycle.py +++ b/tests/adapters/opencode/test_lifecycle.py @@ -505,7 +505,7 @@ async def test_interrupting_a_turn_stops_the_reply_and_frees_the_room(tools) -> room_id="room-1", ) - assert [m["content"] for m in tools.messages_sent] == ["after the interrupt"] + assert tools.chat == ["after the interrupt"] await adapter.on_cleanup("room-1") diff --git a/tests/adapters/opencode/test_turns.py b/tests/adapters/opencode/test_turns.py index 234ee0a78..f724fb18c 100644 --- a/tests/adapters/opencode/test_turns.py +++ b/tests/adapters/opencode/test_turns.py @@ -1137,7 +1137,7 @@ async def test_room_posting_tool_reply_suppresses_text_fallback( await run_single_turn(adapter, tools) - assert [m["content"] for m in tools.messages_sent] == ["hi"] + assert tools.chat == ["hi"] async def test_approval_prompts_never_stand_in_for_the_models_reply( @@ -1175,7 +1175,7 @@ async def test_approval_prompts_never_stand_in_for_the_models_reply( ) await wait_for(lambda: tools.turn.replied) - assert [m["content"] for m in tools.messages_sent] == [ + assert tools.chat == [ APPROVAL_REQUESTED_TEMPLATE.format( permission="bash", patterns="rm -rf tmp", request_id="req-1" ), @@ -1256,7 +1256,7 @@ async def test_busy_message_is_settled_and_leaves_the_turn_its_tools( assert [e["content"] for e in events_of_type(busy_tools, "error")] == [ "OpenCode is still processing the previous request in this room." ] - assert [m["content"] for m in tools.messages_sent] == ["Done."] + assert tools.chat == ["Done."] async def test_non_room_posting_tool_does_not_suppress_text( @@ -1285,7 +1285,7 @@ async def test_non_room_posting_tool_does_not_suppress_text( await run_single_turn(adapter, tools) - assert [m["content"] for m in tools.messages_sent] == ["Ran the command."] + assert tools.chat == ["Ran the command."] async def test_task_event_post_failure_does_not_drop_the_turn(make_adapter) -> None: @@ -1310,9 +1310,7 @@ async def test_task_event_post_failure_does_not_drop_the_turn(make_adapter) -> N await run_single_turn(adapter, tools) assert len(fake_client.prompt_calls) == 1 - assert [m["content"] for m in tools.messages_sent] == [ - "Handled despite the event failure." - ] + assert tools.chat == ["Handled despite the event failure."] assert not any( "failed while processing" in f["message"].lower() for f in reported_failures(tools) diff --git a/tests/adapters/test_anthropic_adapter.py b/tests/adapters/test_anthropic_adapter.py index 71f8da21a..904ecdafc 100644 --- a/tests/adapters/test_anthropic_adapter.py +++ b/tests/adapters/test_anthropic_adapter.py @@ -10,8 +10,10 @@ import asyncio import json import logging +from collections.abc import Callable from datetime import UTC, datetime from types import SimpleNamespace +from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -30,10 +32,9 @@ ToolEventKey, TurnUsage, ) -from band.runtime.custom_tools import declares_turn_effect -from band.runtime.tools import TurnEffect from band.testing import FakeAgentTools from tests.adapters.usage_events import sent_usage_payloads +from tests.framework_conformance.turnprobes import CUSTOM_TOOL_DECLARATIONS def make_usage(inp: int, out: int) -> SimpleNamespace: @@ -833,10 +834,11 @@ class TestCustomTools: """Tests for custom tool support.""" @pytest.mark.asyncio + @pytest.mark.parametrize(("declare", "complete"), CUSTOM_TOOL_DECLARATIONS) async def test_custom_tool_records_its_effect_on_the_room_turn( - self, sample_message + self, sample_message, declare: Callable[..., Any], complete: bool ): - @declares_turn_effect(TurnEffect.ACT) + @declare async def file_echo(args: EchoInput) -> str: return "filed" @@ -874,7 +876,7 @@ async def file_echo(args: EchoInput) -> str: room_id="room-123", ) - assert tools.turn.complete + assert tools.turn.complete is complete def test_accepts_additional_tools_parameter(self): """Adapter should accept list of (Model, func) tuples.""" diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index d920d2954..0933e1af3 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -67,6 +67,7 @@ tool_call_request, turn_completed, ) +from tests.framework_conformance.turnprobes import undeclared from tests.paths import host_absolute_path @@ -6668,7 +6669,7 @@ async def test_a_policy_notification_never_stands_in_for_the_answer( ), ) - assert [m["content"] for m in turn.tools.messages_sent] == [ + assert turn.tools.chat == [ "Approval requested (command: a). Policy decision: accept.", "Tests pass.", ] @@ -6678,10 +6679,6 @@ class StayQuietInput(BaseModel): """Say nothing this turn.""" -def _undeclared(handler: Callable[..., Any]) -> Callable[..., Any]: - return handler - - class TestCustomToolEffect: @pytest.mark.asyncio @pytest.mark.parametrize( @@ -6690,7 +6687,7 @@ class TestCustomToolEffect: pytest.param( declares_turn_effect(TurnEffect.DECLINE), [], id="declared-silence" ), - pytest.param(_undeclared, ["Nothing to add."], id="undeclared"), + pytest.param(undeclared, ["Nothing to add."], id="undeclared"), ], ) async def test_only_a_declared_tool_settles_the_reply( @@ -6708,14 +6705,14 @@ async def stay_quiet(args: StayQuietInput) -> str: additional_tools=[(StayQuietInput, declare(stay_quiet))], ) - assert [m["content"] for m in turn.tools.messages_sent] == chat + assert turn.tools.chat == chat @pytest.mark.asyncio @pytest.mark.parametrize( ("declare", "complete"), [ pytest.param(declares_turn_effect(TurnEffect.ACT), True, id="declared"), - pytest.param(_undeclared, False, id="undeclared-observes"), + pytest.param(undeclared, False, id="undeclared-observes"), ], ) async def test_the_tool_records_its_effect_on_the_turn( @@ -6732,6 +6729,15 @@ async def stay_quiet(args: StayQuietInput) -> str: assert turn.tools.turn.complete is complete +@pytest.mark.asyncio +async def test_an_interrupted_turn_is_settled_by_its_notice() -> None: + """The notice answers the room, so the turn is not also a missing reply.""" + turn = await run_codex_turn(events=[turn_completed(status="interrupted")]) + + assert turn.tools.chat == ["I stopped before completing this request."] + assert turn.tools.turn.complete + + class TestDetachedTurnOutcome: """A turn released to wait on a human is judged at its real end.""" @@ -6764,7 +6770,7 @@ async def test_messages_during_a_released_turn_never_take_over_its_reply( turn, busy, approval = room.deliveries assert turn.messages_sent[-1]["content"] == "Tests pass." - assert [m["content"] for m in busy.messages_sent] == [TURN_IN_PROGRESS_MESSAGE] + assert busy.chat == [TURN_IN_PROGRESS_MESSAGE] assert turn.turn.complete and busy.turn.complete and approval.turn.complete assert ( failure_reports(turn) diff --git a/tests/adapters/test_crewai_adapter.py b/tests/adapters/test_crewai_adapter.py index 4598b1a30..7797b5869 100644 --- a/tests/adapters/test_crewai_adapter.py +++ b/tests/adapters/test_crewai_adapter.py @@ -654,7 +654,7 @@ async def kickoff(prompt: str) -> Any: await adapter.on_event(turn_input(tools)) - assert [m["content"] for m in tools.messages_sent] == [ANSWER] + assert tools.chat == [ANSWER] assert failure_reports(tools) == [] assert crew.kickoff_async.call_count == 2 diff --git a/tests/adapters/test_crewai_flow_phase4.py b/tests/adapters/test_crewai_flow_phase4.py index 6ead790b2..53a623e8e 100644 --- a/tests/adapters/test_crewai_flow_phase4.py +++ b/tests/adapters/test_crewai_flow_phase4.py @@ -867,7 +867,7 @@ async def test_existing_run_uses_persisted_join_policy_after_config_change( is_session_bootstrap=False, ) - assert [m["content"] for m in tools.messages_sent] == ["first answer"] + assert tools.chat == ["first answer"] statuses = [ e["metadata"].get(ns, {}).get("status") for e in tools.events_sent diff --git a/tests/adapters/test_crewai_flow_phase5.py b/tests/adapters/test_crewai_flow_phase5.py index 5f8408db8..9825f1457 100644 --- a/tests/adapters/test_crewai_flow_phase5.py +++ b/tests/adapters/test_crewai_flow_phase5.py @@ -140,7 +140,7 @@ async def test_off_policy_finalizes_without_delegating_to_tagged_peer( is_session_bootstrap=True, room_id="room-1", ) - assert [m["content"] for m in tools.messages_sent] == ["direct final"] + assert tools.chat == ["direct final"] # --------------------------------------------------------------------------- @@ -673,7 +673,7 @@ def append_task_events_to_context(tools: FakeAgentTools, start: int) -> int: ) expected = json.loads(fixture_path.read_text()) actual = { - "visible_messages": [m["content"] for m in tools.messages_sent], + "visible_messages": tools.chat, "task_statuses": [ e["metadata"].get(ns, {}).get("status") for e in tools.events_sent diff --git a/tests/adapters/test_gemini_adapter.py b/tests/adapters/test_gemini_adapter.py index f0d9b506d..8c49ce1ee 100644 --- a/tests/adapters/test_gemini_adapter.py +++ b/tests/adapters/test_gemini_adapter.py @@ -3,7 +3,9 @@ from __future__ import annotations import json +from collections.abc import Callable from datetime import UTC, datetime +from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -15,9 +17,8 @@ from band.adapters.gemini import GeminiAdapter, GeminiAdapterConfig from band.core.protocols import GENERIC_PROVIDER_FAILURE_MESSAGE from band.core.types import Emit, PlatformMessage, ToolEventKey -from band.runtime.custom_tools import declares_turn_effect -from band.runtime.tools import TurnEffect from band.testing import FakeAgentTools +from tests.framework_conformance.turnprobes import CUSTOM_TOOL_DECLARATIONS @pytest.fixture @@ -436,15 +437,16 @@ async def echo_tool(inp: EchoInput) -> str: mock_tools.execute_tool_call.assert_not_called() @pytest.mark.asyncio + @pytest.mark.parametrize(("declare", "complete"), CUSTOM_TOOL_DECLARATIONS) async def test_custom_tool_records_its_effect_on_the_room_turn( - self, sample_message + self, sample_message, declare: Callable[..., Any], complete: bool ): class FileInput(BaseModel): """File the report.""" note: str - @declares_turn_effect(TurnEffect.ACT) + @declare async def file_report(inp: FileInput) -> str: return "filed" @@ -472,7 +474,7 @@ async def file_report(inp: FileInput) -> str: room_id="room-123", ) - assert tools.turn.complete + assert tools.turn.complete is complete class TestReadRoomFileImagePassthrough: diff --git a/tests/adapters/test_google_adk_adapter.py b/tests/adapters/test_google_adk_adapter.py index a3805a051..fc1f62791 100644 --- a/tests/adapters/test_google_adk_adapter.py +++ b/tests/adapters/test_google_adk_adapter.py @@ -12,7 +12,9 @@ import asyncio import importlib import json +from collections.abc import Callable from datetime import UTC, datetime +from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -20,9 +22,9 @@ from band.core.protocols import GENERIC_PROVIDER_FAILURE_MESSAGE from band.core.types import ALL_CAPABILITIES, Capability, Emit, PlatformMessage -from band.runtime.custom_tools import declares_turn_effect -from band.runtime.tools import AgentTools, BandTool, TurnEffect +from band.runtime.tools import AgentTools, BandTool from band.testing import FakeAgentTools +from tests.framework_conformance.turnprobes import CUSTOM_TOOL_DECLARATIONS pytest.importorskip("google.adk", reason="google-adk not installed") @@ -760,13 +762,16 @@ def calculate(args: CalcInput) -> str: mock_tools.execute_tool_call.assert_not_called() @pytest.mark.asyncio - async def test_custom_tool_records_its_effect_on_the_room_turn(self): + @pytest.mark.parametrize(("declare", "complete"), CUSTOM_TOOL_DECLARATIONS) + async def test_custom_tool_records_its_effect_on_the_room_turn( + self, declare: Callable[..., Any], complete: bool + ): class FileInput(BaseModel): """File the report.""" note: str - @declares_turn_effect(TurnEffect.ACT) + @declare async def file_report(args: FileInput) -> str: return "filed" @@ -776,7 +781,7 @@ async def file_report(args: FileInput) -> str: await bridge.run_async(args={"note": "go"}, tool_context=MagicMock()) - assert tools.turn.complete + assert tools.turn.complete is complete class TestContactsInjection: diff --git a/tests/adapters/test_letta_adapter.py b/tests/adapters/test_letta_adapter.py index 0d7621548..fb987708e 100644 --- a/tests/adapters/test_letta_adapter.py +++ b/tests/adapters/test_letta_adapter.py @@ -1434,14 +1434,9 @@ class TestExternalToolRecording: async def test_streamed_tool_return_records_only_for_external_band_mcp( self, mcp_mode: Literal["self_host", "external"], status: str, replied: bool ) -> None: - adapter = LettaAdapter( - config=LettaAdapterConfig(mcp=LettaMCPConfig(mode=mcp_mode)) + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode=mcp_mode)) ) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.server_id = "mcp-server-1" - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") mock_client.agents.messages.create.return_value = make_letta_response( make_tool_return_message(BandTool.SEND_MESSAGE, status=status), ) diff --git a/tests/adapters/test_pydantic_ai_adapter.py b/tests/adapters/test_pydantic_ai_adapter.py index dc8d3c3bb..28f0ac78a 100644 --- a/tests/adapters/test_pydantic_ai_adapter.py +++ b/tests/adapters/test_pydantic_ai_adapter.py @@ -101,7 +101,11 @@ ) from tests.adapters.usage_events import sent_usage_payloads from tests.framework_configs.adapters import pydantic_ai_probe_tools -from tests.framework_conformance.turnprobes import ROOM_ID, turn_input +from tests.framework_conformance.turnprobes import ( + CUSTOM_TOOL_DECLARATIONS, + ROOM_ID, + turn_input, +) def make_stream_events( @@ -2463,24 +2467,6 @@ def lookup(args: LookupInput) -> str: # Normalized to a native callable named from the model (not the handler). assert [t.__name__ for t in adapter._custom_tools] == ["lookup"] - def test_tuple_terminal_marker_is_honored(self): - - class DeployInput(BaseModel): - """deploy.""" - - target: str - - def deploy(args: DeployInput) -> str: - return "done" - - deploy.band_terminal = True # opt in as a terminal action - - adapter = PydanticAIAdapter( - PydanticAIAdapterConfig(model="openai:gpt-5.4"), - additional_tools=[(DeployInput, deploy)], - ) - assert adapter._custom_effects == {"deploy": TurnEffect.ACT} - def test_converted_tuple_flattens_in_pydantic_ai(self): class LookupInput(BaseModel): @@ -2550,13 +2536,16 @@ def lookup(args: AliasedInput) -> str: assert content.startswith("user:") @pytest.mark.asyncio - async def test_declared_tool_records_its_effect_on_the_room_turn(self): + @pytest.mark.parametrize(("declare", "complete"), CUSTOM_TOOL_DECLARATIONS) + async def test_custom_tool_records_its_effect_on_the_room_turn( + self, declare: Callable[..., Any], complete: bool + ): class FileInput(BaseModel): """File the report.""" note: str - @declares_turn_effect(TurnEffect.ACT) + @declare def file_report(args: FileInput) -> str: return "filed" @@ -2569,4 +2558,4 @@ def file_report(args: FileInput) -> str: await adapter._create_agent().run("go", deps=tools) - assert tools.turn.complete + assert tools.turn.complete is complete diff --git a/tests/adapters/test_strands_adapter.py b/tests/adapters/test_strands_adapter.py index 7bed53e75..bf252349d 100644 --- a/tests/adapters/test_strands_adapter.py +++ b/tests/adapters/test_strands_adapter.py @@ -67,6 +67,7 @@ failure_reports, reported_failures, ) +from tests.framework_conformance.turnprobes import CUSTOM_TOOL_DECLARATIONS _INPUT_TOKENS_PER_CALL = 7 _OUTPUT_TOKENS_PER_CALL = 3 @@ -187,23 +188,6 @@ async def get_weather(args: WeatherInput) -> str: # Declared no effect -> not a terminal action. assert adapter._custom_effects == {} - def test_terminal_marker_captured_from_tuple_handler(self): - class DoneInput(BaseModel): - """Finish the task.""" - - note: str - - async def finish(args: DoneInput) -> str: - return "done" - - finish.band_terminal = True # type: ignore[attr-defined] - - adapter = StrandsAdapter( - StrandsAdapterConfig(model="m"), additional_tools=[(DoneInput, finish)] - ) - - assert adapter._custom_effects == {"done": TurnEffect.ACT} - def test_custom_tool_may_not_shadow_a_platform_tool(self): """Strands' registry is last-wins, so a collision must fail at construction.""" @@ -710,15 +694,16 @@ async def boom(args: BoomInput) -> str: assert _tool_results(adapter) == ["Error executing tool 'boom': no network"] @pytest.mark.asyncio + @pytest.mark.parametrize(("declare", "complete"), CUSTOM_TOOL_DECLARATIONS) async def test_custom_tool_records_its_effect_on_the_room_turn( - self, tools, scripted + self, tools, scripted, declare: Callable[..., Any], complete: bool ): class FileInput(BaseModel): """File the report.""" note: str - @declares_turn_effect(TurnEffect.ACT) + @declare async def file_report(args: FileInput) -> str: return "filed" @@ -729,7 +714,7 @@ async def file_report(args: FileInput) -> str: await _run_message(adapter, tools) - assert tools.turn.complete + assert tools.turn.complete is complete class TestTurnFailure: diff --git a/tests/core/test_delivery.py b/tests/core/test_delivery.py index 42245d80e..8b67c9890 100644 --- a/tests/core/test_delivery.py +++ b/tests/core/test_delivery.py @@ -43,7 +43,7 @@ async def test_posts_without_counting_as_the_reply(self) -> None: await deliver_notice(tools, "No pending approvals.", mentions=["@alice"]) - assert [m["content"] for m in tools.messages_sent] == ["No pending approvals."] + assert tools.chat == ["No pending approvals."] assert not tools.turn.complete async def test_wraps_a_send_notice_failure(self) -> None: @@ -59,7 +59,7 @@ async def test_relays_the_final_text_as_the_turn_reply(self) -> None: assert await relay_reply(tools, "the answer", ["@alice"]) is True - assert [m["content"] for m in tools.messages_sent] == ["the answer"] + assert tools.chat == ["the answer"] assert tools.turn.replied async def test_a_tool_reply_suppresses_the_relay(self) -> None: @@ -68,7 +68,7 @@ async def test_a_tool_reply_suppresses_the_relay(self) -> None: assert await relay_reply(tools, "closing remark", ["@alice"]) is False - assert [m["content"] for m in tools.messages_sent] == ["already answered"] + assert tools.chat == ["already answered"] async def test_a_decline_suppresses_the_relay(self) -> None: tools = FakeAgentTools() diff --git a/tests/framework_conformance/test_turn_outcome.py b/tests/framework_conformance/test_turn_outcome.py index 4511bb634..025dde0de 100644 --- a/tests/framework_conformance/test_turn_outcome.py +++ b/tests/framework_conformance/test_turn_outcome.py @@ -84,7 +84,7 @@ async def run_turn(probe: TurnOutcomeProbe, script: TurnScript) -> TurnResult: def observed(tools: FakeAgentTools, verdict: Verdict) -> TurnResult: return TurnResult( verdict=verdict, - messages=[m["content"] for m in tools.messages_sent], + messages=tools.chat, failures=failure_reports(tools), ) @@ -160,6 +160,20 @@ async def test_a_tool_reply_suppresses_the_closing_text(framework_id: str) -> No assert result == TurnResult(Verdict.COMPLETE, messages=[ANSWER], failures=[]) +#: Registered adapters whose turns are not the model's to answer through Band tools. +UNJUDGED_FRAMEWORK_IDS = frozenset({"crewai_flow", "parlant"}) + + +def test_only_the_declared_adapters_go_unjudged() -> None: + unjudged = { + cfg.framework_id + for cfg in ADAPTER_CONFIGS + if not cfg.adapter_factory().judges_turns + } + + assert unjudged == UNJUDGED_FRAMEWORK_IDS + + def test_every_judged_adapter_has_a_probe() -> None: unprobed = sorted( cfg.framework_id diff --git a/tests/framework_conformance/turnprobes/__init__.py b/tests/framework_conformance/turnprobes/__init__.py index d7a3ea7b2..45d4452d8 100644 --- a/tests/framework_conformance/turnprobes/__init__.py +++ b/tests/framework_conformance/turnprobes/__init__.py @@ -16,11 +16,32 @@ from datetime import UTC, datetime from typing import Any +import pytest + from band.core.types import AgentInput, HistoryProvider, PlatformMessage -from band.runtime.tools import TOOL_DEFINITIONS, ToolCallOutcome, serialize_tool_result +from band.runtime.custom_tools import declares_turn_effect +from band.runtime.tools import ( + TOOL_DEFINITIONS, + ToolCallOutcome, + TurnEffect, + serialize_tool_result, +) from band.testing.fake_tools import FakeAgentTools ROOM_ID = "room-turn" + + +def undeclared(handler: Callable[..., Any]) -> Callable[..., Any]: + """Leave a custom tool's effect undeclared, so its call only observes.""" + return handler + + +#: A custom tool declared as real work completes the turn; an undeclared one +#: only observes, so the turn still owes a reply. +CUSTOM_TOOL_DECLARATIONS = [ + pytest.param(declares_turn_effect(TurnEffect.ACT), True, id="declared-act"), + pytest.param(undeclared, False, id="undeclared-observes"), +] ALICE: dict[str, Any] = { "id": "user-alice", "name": "Alice", diff --git a/tests/integrations/a2a/gateway/test_adapter.py b/tests/integrations/a2a/gateway/test_adapter.py index 14e466404..7ff99cc82 100644 --- a/tests/integrations/a2a/gateway/test_adapter.py +++ b/tests/integrations/a2a/gateway/test_adapter.py @@ -108,6 +108,12 @@ def test_explicit_gateway_url_wins(self) -> None: assert config.public_url == "https://gw.example.com" +def test_peer_messages_are_not_judged_as_turns() -> None: + """Peer messages answer an A2A caller, not the room; judging them would + report each one as a missing reply.""" + assert not A2AGatewayAdapter(rest_client=MagicMock()).judges_turns + + class TestGatewayStartup: @pytest.mark.asyncio async def test_discovers_peers_and_starts_server(self) -> None: diff --git a/tests/integrations/a2a/test_adapter.py b/tests/integrations/a2a/test_adapter.py index fef54b4e2..9bdf98a44 100644 --- a/tests/integrations/a2a/test_adapter.py +++ b/tests/integrations/a2a/test_adapter.py @@ -183,6 +183,13 @@ async def test_non_streaming_config_reaches_the_client(self) -> None: "http://localhost:10000" ) + def test_turns_are_not_judged(self) -> None: + """The remote agent owns its replies; judging its turns would report + a remote that answers nothing as this agent's missing reply.""" + adapter = A2AAdapter(A2AAdapterConfig(remote_url="http://localhost:10000")) + + assert not adapter.judges_turns + @pytest.mark.asyncio async def test_owned_http_client_has_a_generous_bounded_read_timeout(self) -> None: """A real remote turn (a live LLM call, a tool loop) routinely leaves diff --git a/tests/integrations/acp/test_server_adapter.py b/tests/integrations/acp/test_server_adapter.py index 5ba51b586..7d449ae63 100644 --- a/tests/integrations/acp/test_server_adapter.py +++ b/tests/integrations/acp/test_server_adapter.py @@ -49,6 +49,11 @@ def test_init_sets_history_converter(self) -> None: assert adapter.history_converter is not None + def test_room_messages_are_not_judged_as_turns(self) -> None: + """Room messages are peers' answers relayed to the editor; judging + them would report each one as a missing reply.""" + assert not BandACPServerAdapter().judges_turns + def test_feature_requests_are_checked_against_what_it_supports(self) -> None: with pytest.raises(BandConfigError, match="tool_calls"): BandACPServerAdapter(emit=Emit.TOOL_CALLS) diff --git a/tests/integrations/claude_sdk/test_dedup_tools.py b/tests/integrations/claude_sdk/test_dedup_tools.py index fcc60e300..ba32c5922 100644 --- a/tests/integrations/claude_sdk/test_dedup_tools.py +++ b/tests/integrations/claude_sdk/test_dedup_tools.py @@ -14,6 +14,7 @@ DEFAULT_DEDUP_TTL_SECONDS, DedupingAgentTools, ) +from band.testing import FakeAgentTools def _make_inner() -> MagicMock: @@ -519,3 +520,51 @@ async def flaky(content, mentions=None): await wrapper.send_message("hi", ["alice"]) # dedup hit assert calls[0] == 2 + + +class GatedSendTools(FakeAgentTools): + """Tools whose send stays in flight until ``gate`` is set.""" + + def __init__(self) -> None: + super().__init__() + self.gate = asyncio.Event() + + async def send_message( + self, content: str, mentions: list[str] | list[dict[str, str]] | None = None + ) -> Any: + await self.gate.wait() + return await super().send_message(content, mentions) + + +class TestDedupedSendRepliesForItsTurn: + """A suppressed duplicate still answers the turn that sent it, since the + original already reached the room.""" + + @pytest.mark.asyncio + async def test_a_cached_duplicate_replies_for_the_next_turn(self): + wrapper = DedupingAgentTools(FakeAgentTools()) + await wrapper.send_message("hi", ["alice"]) + + next_turn = FakeAgentTools() + await wrapper.update_inner(next_turn) + await wrapper.send_message("hi", ["alice"]) + + assert next_turn.messages_sent == [] + assert next_turn.turn.replied + + @pytest.mark.asyncio + async def test_a_duplicate_joining_an_in_flight_send_replies_for_its_turn(self): + first_turn = GatedSendTools() + wrapper = DedupingAgentTools(first_turn) + original = asyncio.create_task(wrapper.send_message("hi", ["alice"])) + await asyncio.sleep(0) + + next_turn = FakeAgentTools() + await wrapper.update_inner(next_turn) + duplicate = asyncio.create_task(wrapper.send_message("hi", ["alice"])) + await asyncio.sleep(0) + first_turn.gate.set() + await asyncio.gather(original, duplicate) + + assert next_turn.messages_sent == [] + assert next_turn.turn.replied diff --git a/tests/integrations/codex/test_adapter_e2e.py b/tests/integrations/codex/test_adapter_e2e.py index dac399ddc..e1c0f457e 100644 --- a/tests/integrations/codex/test_adapter_e2e.py +++ b/tests/integrations/codex/test_adapter_e2e.py @@ -21,7 +21,12 @@ ) from band.integrations.codex import CodexJsonRpcError, CodexRequestMethod, RpcEvent from band.testing import FakeAgentTools -from tests.adapters.codexturns import RecordedRequests, await_released_turn +from tests.adapters.codexturns import ( + RecordedRequests, + await_released_turn, + final_text, + turn_completed, +) def _platform_message(content: str, *, room_id: str = "room-1") -> PlatformMessage: @@ -199,22 +204,11 @@ def _request(request_id: int, method: str, params: dict[str, Any]) -> RpcEvent: ) -def _turn_completed() -> RpcEvent: - return _notify( - "turn/completed", - {"turn": {"id": "turn-1", "status": "completed", "items": [], "error": None}}, - ) - - def _answered_turn() -> list[RpcEvent]: """A turn whose final text the adapter relays as its reply.""" - return [ - _notify("item/agentMessage/delta", {"itemId": "m", "delta": "Done."}), - _turn_completed(), - ] + return [final_text("Done."), turn_completed()] -@pytest.mark.asyncio async def test_on_event_uses_converter_history_to_resume_thread() -> None: tools = _ToolSchemaFakeTools() fake_client = _FakeCodexClient(events=_answered_turn()) diff --git a/tests/integrations/langgraph/test_custom_tool_defs.py b/tests/integrations/langgraph/test_custom_tool_defs.py index ddd76573d..a7f8a61af 100644 --- a/tests/integrations/langgraph/test_custom_tool_defs.py +++ b/tests/integrations/langgraph/test_custom_tool_defs.py @@ -10,10 +10,11 @@ from __future__ import annotations -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable from datetime import UTC, datetime from typing import Any +import pytest from langchain_core.tools import StructuredTool from langchain_core.tools import tool as lc_tool from pydantic import BaseModel, Field @@ -22,9 +23,8 @@ from band.core.turn import Turn from band.core.types import PlatformMessage from band.integrations.langgraph.langchain_tools import custom_tool_def_to_langchain -from band.runtime.custom_tools import declares_turn_effect -from band.runtime.tools import TurnEffect from band.testing import FakeAgentTools, reported_failures +from tests.framework_conformance.turnprobes import CUSTOM_TOOL_DECLARATIONS class EchoInput(BaseModel): @@ -77,11 +77,6 @@ class FileInput(BaseModel): note: str -@declares_turn_effect(TurnEffect.ACT) -def file_report(args: FileInput) -> str: - return "filed" - - class CallsToolGraph: """A graph whose one step calls the named tool, as a real agent graph would.""" @@ -96,7 +91,14 @@ async def astream_events( yield -async def test_custom_tool_records_its_effect_on_the_room_turn() -> None: +@pytest.mark.parametrize(("declare", "complete"), CUSTOM_TOOL_DECLARATIONS) +async def test_custom_tool_records_its_effect_on_the_room_turn( + declare: Callable[..., Any], complete: bool +) -> None: + @declare + def file_report(args: FileInput) -> str: + return "filed" + tools = FakeAgentTools(room_id="room-1") adapter = LangGraphAdapter( graph_factory=lambda turn_tools: CallsToolGraph(turn_tools, "file"), @@ -125,4 +127,4 @@ async def test_custom_tool_records_its_effect_on_the_room_turn() -> None: ) assert reported_failures(tools) == [] - assert tools.turn.complete + assert tools.turn.complete is complete diff --git a/tests/integrations/slack/test_retry_idempotency.py b/tests/integrations/slack/test_retry_idempotency.py index d4669fc0d..a63f36b10 100644 --- a/tests/integrations/slack/test_retry_idempotency.py +++ b/tests/integrations/slack/test_retry_idempotency.py @@ -295,8 +295,11 @@ def __init__(self) -> None: super().__init__(history_converter=None) self.count = 0 - async def on_message(self, *args: Any, **kwargs: Any) -> None: + async def on_message( + self, msg: Any, tools: Any, *args: Any, **kwargs: Any + ) -> None: self.count += 1 + await tools.no_reply() async def on_cleanup(self, room_id: str) -> None: return None diff --git a/tests/integrations/slack/test_wrapping.py b/tests/integrations/slack/test_wrapping.py index 222c1e68d..051dcbf56 100644 --- a/tests/integrations/slack/test_wrapping.py +++ b/tests/integrations/slack/test_wrapping.py @@ -20,6 +20,7 @@ import hashlib import hmac import json +import logging import time import warnings from contextlib import AbstractContextManager, nullcontext @@ -41,6 +42,7 @@ Capability, Emit, HistoryProvider, + MessageType, PlatformMessage, ) from band.integrations.slack.adapter import ( @@ -1429,7 +1431,7 @@ def _agent_input_with_history( ) -class _IdleBrain(_SlackReplyBrain): +class IdleBrain(_SlackReplyBrain): """A brain whose turn does nothing at all.""" def __init__(self, *, judges_turns: bool) -> None: @@ -1462,7 +1464,7 @@ async def test_an_idle_turn_is_judged_by_the_brains_rule( verdict: AbstractContextManager[Any], failures: list[tuple[str, str]], ): - adapter, _, _, _ = _make_adapter(inner=_IdleBrain(judges_turns=brain_judges_turns)) + adapter, _, _, _ = _make_adapter(inner=IdleBrain(judges_turns=brain_judges_turns)) tools = FakeAgentTools(room_id="room-1") inp = _agent_input_with_history( room_id="room-1", raw_history=[], bootstrap=False, tools=tools @@ -1474,6 +1476,26 @@ async def test_an_idle_turn_is_judged_by_the_brains_rule( assert failure_reports(tools) == failures +@pytest.mark.asyncio +async def test_an_idle_slack_originated_turn_is_reported( + caplog: pytest.LogCaptureFixture, +): + """A turn that arrives from Slack rather than the platform is judged too, + and its reported failure is not logged again as a crash.""" + adapter, _, _, rest = _make_adapter( + inner=IdleBrain(judges_turns=True), room_ids=["room-1"] + ) + await adapter.on_started("MyBot", "") + + with caplog.at_level(logging.WARNING): + await _post_slack_event(adapter, adapter.config.apps[0], _mention_event()) + await adapter.wait_idle() + + (error,) = _posted_events(rest, MessageType.ERROR) + assert error.content == MISSING_REPLY_FAILURE[1] + assert max(r.levelno for r in caplog.records) == logging.WARNING + + @pytest.mark.asyncio async def test_on_event_rehydrates_room_binding_on_bootstrap(): """First WS-delivered message in a previously-bridged room restores state.""" @@ -1682,11 +1704,11 @@ async def test_context_event_metadata_includes_slack_room_id(): # ── Slack context mirroring (audit timeline) ──────────────────────────────── -def _thought_events(rest: MagicMock) -> list[Any]: +def _posted_events(rest: MagicMock, message_type: MessageType) -> list[Any]: return [ c.kwargs["event"] for c in rest.agent_api_events.create_agent_chat_event.await_args_list - if c.kwargs["event"].message_type == "thought" + if c.kwargs["event"].message_type == message_type ] @@ -1715,7 +1737,7 @@ async def test_user_turn_mirrored_as_thought_with_thread_id(): ) await adapter.wait_idle() - thoughts = _thought_events(rest) + thoughts = _posted_events(rest, MessageType.THOUGHT) assert len(thoughts) == 1 mirror = thoughts[0] assert mirror.message_type == "thought" @@ -1745,7 +1767,7 @@ async def test_mirroring_disabled_emits_no_thought(): ) await adapter.wait_idle() - assert _thought_events(rest) == [] + assert _posted_events(rest, MessageType.THOUGHT) == [] # Only the bootstrap context (task) event is emitted. assert rest.agent_api_events.create_agent_chat_event.await_count == 1 assert ( diff --git a/tests/integrations/test_omp.py b/tests/integrations/test_omp.py index cc7d602da..fedfa2a47 100644 --- a/tests/integrations/test_omp.py +++ b/tests/integrations/test_omp.py @@ -24,7 +24,6 @@ omp_provider_env, validate_omp_command, ) -from band.runtime.tools import TurnEffect, turn_effect from tests.paths import CI_SCRIPTS @@ -137,7 +136,6 @@ def test_normalize_mcp_title_maps_registered_band_tool() -> None: name, args = normalize_omp_mcp_device_call(title, {"chat_id": "r1"}, own) assert name == "band_send_message" assert args == {"chat_id": "r1"} - assert turn_effect(name) is TurnEffect.REPLY def test_normalize_device_call_leaves_unknown_paths() -> None: diff --git a/tests/runtime/test_turn.py b/tests/runtime/test_turn.py index 81e059cdb..6010d93d9 100644 --- a/tests/runtime/test_turn.py +++ b/tests/runtime/test_turn.py @@ -7,6 +7,7 @@ from __future__ import annotations +import logging from typing import Any from unittest.mock import MagicMock @@ -147,16 +148,26 @@ class TestReportUnsettledTurn: async def test_a_missing_reply_is_reported_once(self) -> None: tools = FakeAgentTools() - assert await report_unsettled_turn(tools) is True - assert await report_unsettled_turn(tools) is False + assert await report_unsettled_turn(tools, room_id="room-1") is True + assert await report_unsettled_turn(tools, room_id="room-1") is False assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + async def test_a_missing_reply_is_logged_for_its_room( + self, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="band.core.turn"): + await report_unsettled_turn(FakeAgentTools(), room_id="room-1") + + (record,) = caplog.records + assert record.levelno == logging.WARNING + assert "room-1" in record.getMessage() + async def test_a_complete_turn_reports_nothing(self) -> None: tools = FakeAgentTools() await tools.no_reply() - assert await report_unsettled_turn(tools) is False + assert await report_unsettled_turn(tools, room_id="room-1") is False assert reported_failures(tools) == [] @@ -164,7 +175,7 @@ async def test_a_settled_turn_reports_nothing(self) -> None: tools = FakeAgentTools() tools.turn.settle() - assert await report_unsettled_turn(tools) is False + assert await report_unsettled_turn(tools, room_id="room-1") is False class TestDetach: @@ -194,7 +205,7 @@ async def test_a_detached_report_never_marks_the_contexts_next_message( tools.turn.judged = True tools.turn.detach() - assert await report_unsettled_turn(tools) is True + assert await report_unsettled_turn(tools, room_id="room-1") is True mock_rest_client.agent_api_events.create_agent_chat_event.assert_awaited_once() ctx.note_turn_failure_reported.assert_not_called() diff --git a/tests/runtime/test_turn_effect.py b/tests/runtime/test_turn_effect.py index e6b248944..a07fd2a17 100644 --- a/tests/runtime/test_turn_effect.py +++ b/tests/runtime/test_turn_effect.py @@ -40,11 +40,6 @@ def test_band_tool_errored_detects_error_prefix() -> None: @pytest.mark.parametrize( ("tool", "custom_effects", "effect"), [ - pytest.param(BandTool.SEND_MESSAGE, None, TurnEffect.REPLY, id="post"), - pytest.param(BandTool.NO_REPLY, None, TurnEffect.DECLINE, id="no-reply"), - pytest.param(BandTool.ADD_PARTICIPANT, None, TurnEffect.ACT, id="action"), - pytest.param(BandTool.GET_MEMORY, None, TurnEffect.OBSERVE, id="read-only"), - pytest.param(BandTool.SEND_EVENT, None, TurnEffect.OBSERVE, id="narration"), pytest.param("weather", _POST_TO_SLACK, TurnEffect.OBSERVE, id="undeclared"), pytest.param( "post_to_slack", _POST_TO_SLACK, TurnEffect.ACT, id="declared-act" @@ -54,7 +49,7 @@ def test_band_tool_errored_detects_error_prefix() -> None: ), ], ) -def test_turn_effect_of_a_finished_tool_call( +def test_a_custom_tool_has_only_the_effect_it_declared( tool: str, custom_effects: dict[str, TurnEffect] | None, effect: TurnEffect ) -> None: assert turn_effect(tool, custom_effects=custom_effects) is effect From 10d4e900f26b870ef58f74feb80191a24557016d Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Mon, 5 Oct 2026 07:07:07 +0300 Subject: [PATCH 09/17] test: align turn-outcome coverage with the TypeScript SDK - Guard that every SimpleAdapter in band is registered in ADAPTER_CONFIGS or named with what covers its verdict (probe, brain, or unjudged). - Runtime verdict rows for core's send_event-only and reply-then-observe fixtures. - docs/turn-outcome.md says why the relay is only a fallback and the prompt keeps its plain-text line. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016nwuEBtdtneuDkJHcvijDQ Signed-off-by: Alexander Zaikman --- docs/turn-outcome.md | 5 ++- .../test_turn_outcome.py | 42 +++++++++++++++++++ tests/runtime/test_turn_verdict.py | 11 +++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/docs/turn-outcome.md b/docs/turn-outcome.md index 8f14e30ce..9b792ff5b 100644 --- a/docs/turn-outcome.md +++ b/docs/turn-outcome.md @@ -203,7 +203,10 @@ asyncio.run(main()) `relay_reply(tools, text, mentions)` from `band.core.delivery` is the one gate for a model's final text: it posts through `deliver_reply` only when no tool -replied or declined this turn. +replied or declined this turn. It is a fallback, not a channel to steer +toward: the base prompt still says plain text is never delivered, because +without that line models more often send their closing narration as a second +`band_send_message` (measured live on gemini-2.5-flash). `tests/framework_conformance/test_reply_boundary.py` fails if adapter code calls `send_message` directly outside the model's reply paths. diff --git a/tests/framework_conformance/test_turn_outcome.py b/tests/framework_conformance/test_turn_outcome.py index 025dde0de..c910049ff 100644 --- a/tests/framework_conformance/test_turn_outcome.py +++ b/tests/framework_conformance/test_turn_outcome.py @@ -14,12 +14,17 @@ from __future__ import annotations +import importlib +import inspect +import pkgutil from dataclasses import dataclass from enum import StrEnum import pytest +import band from band.core.protocols import TurnResultAlreadyReported +from band.core.simple_adapter import SimpleAdapter from band.runtime.tools import BandTool from band.testing import MISSING_REPLY_FAILURE, failure_reports from band.testing.fake_tools import FakeAgentTools, reported_failures @@ -174,6 +179,43 @@ def test_only_the_declared_adapters_go_unjudged() -> None: assert unjudged == UNJUDGED_FRAMEWORK_IDS +#: Every SimpleAdapter in band outside ADAPTER_CONFIGS, and what covers its verdict. +COVERED_OUTSIDE_CONFIGS = { + "ACPClientAdapter": "the acp probe", + "CopilotACPAdapter": "ACPClientAdapter's on_message, run by the acp probe", + "OmpACPAdapter": "ACPClientAdapter's on_message, run by the acp probe", + "CursorACPAdapter": "the cursor_acp probe", + "SlackAdapter": "its brain's verdict (tests/integrations/slack)", + "A2AAdapter": "unjudged (tests/integrations/a2a)", + "A2AGatewayAdapter": "unjudged (tests/integrations/a2a/gateway)", + "BandACPServerAdapter": "unjudged (tests/integrations/acp)", +} + + +def band_adapter_names() -> set[str]: + """Every concrete SimpleAdapter defined in band whose module imports here.""" + for module in pkgutil.walk_packages(band.__path__, "band."): + if module.name.endswith("__main__"): + continue + try: + importlib.import_module(module.name) + except ImportError: + continue # an extra this venv lacks; its own lane covers it + pending, names = [SimpleAdapter], set() + while pending: + for cls in pending.pop().__subclasses__(): + pending.append(cls) + if cls.__module__.startswith("band.") and not inspect.isabstract(cls): + names.add(cls.__name__) + return names + + +def test_every_band_adapter_is_registered_or_covered() -> None: + registered = {type(cfg.adapter_factory()).__name__ for cfg in ADAPTER_CONFIGS} + + assert band_adapter_names() - registered == set(COVERED_OUTSIDE_CONFIGS) + + def test_every_judged_adapter_has_a_probe() -> None: unprobed = sorted( cfg.framework_id diff --git a/tests/runtime/test_turn_verdict.py b/tests/runtime/test_turn_verdict.py index bfb0c9f29..1f928ba60 100644 --- a/tests/runtime/test_turn_verdict.py +++ b/tests/runtime/test_turn_verdict.py @@ -49,8 +49,19 @@ async def unknown_handle_send(tools: AgentToolsProtocol) -> None: ROWS: dict[str, tuple[list[Step], bool]] = { "nothing": ([], False), "observe-only": ([lambda t: t.get_participants()], False), + "send-event-only": ( + [lambda t: t.send_event("thinking", message_type="thought")], + False, + ), "act": ([lambda t: t.create_chatroom()], True), "reply": ([lambda t: t.send_message("the answer", mentions=USER)], True), + "reply-then-observe": ( + [ + lambda t: t.send_message("the answer", mentions=USER), + lambda t: t.get_participants(), + ], + True, + ), "decline": ([lambda t: t.execute_tool_call(BandTool.NO_REPLY, {})], True), "relay": ([lambda t: relay_reply(t, "the answer", USER)], True), "settled": ([lambda t: _settle(t)], True), From 9cafdc25526de0fe372ccaf5ed6364b3eac54e5c Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Mon, 5 Oct 2026 09:04:07 +0300 Subject: [PATCH 10/17] fix: address the cross-SDK and ACP-boundary reviews - OpenCode answers a malformed, question-less ask with an empty decline instead of the reject route, which blocks the model's loop; the reject route is no longer used. The decline text is true for auto-reject and expiry too. - Turn.detach() always marks the turn released; judge_detached_turn checks judged. A released contact-hub turn reporting later no longer marks the context's next message as already reported. - A send the claude_sdk dedup suppresses records nothing: it is an earlier turn's send re-issued, so it no longer marks a later, silent turn replied. - RoomTurnEmitter stages out-of-process tool effects and commits them only when the prompt closes successfully, so a rejected attempt's streamed work cannot suppress a retry's answer. - A turn failure already reported to the room logs at DEBUG in the runtime instead of ERROR with a traceback. - Guards: covered adapters must map to a real probe; every send_message, deliver_reply and relay_reply call is pinned per file with its reason. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016nwuEBtdtneuDkJHcvijDQ Signed-off-by: Alexander Zaikman --- src/band/adapters/opencode/approvals.py | 23 ++-- src/band/core/turn.py | 21 ++- src/band/integrations/acp/room_emitter.py | 16 ++- .../integrations/claude_sdk/dedup_tools.py | 9 -- src/band/integrations/opencode/client.py | 9 -- src/band/runtime/execution.py | 17 ++- .../failures/test_turn_failure_surfacing.py | 11 +- tests/adapters/opencode/helpers.py | 4 - tests/adapters/opencode/test_approvals.py | 7 +- .../test_reply_boundary.py | 126 ++++++++++-------- .../test_turn_outcome.py | 42 ++++-- tests/integrations/acp/test_room_emitter.py | 22 +++ .../claude_sdk/test_dedup_tools.py | 23 ++-- tests/integrations/opencode/test_client.py | 20 --- tests/runtime/test_turn.py | 28 ++-- tests/runtime/test_turn_verdict.py | 18 +++ 16 files changed, 222 insertions(+), 174 deletions(-) diff --git a/src/band/adapters/opencode/approvals.py b/src/band/adapters/opencode/approvals.py index d4d9b1ae8..644e97801 100644 --- a/src/band/adapters/opencode/approvals.py +++ b/src/band/adapters/opencode/approvals.py @@ -155,12 +155,12 @@ def parse_question_answers( "This request was declined. Do not retry it or try another way to do the " "same thing; reply to the user instead." ) -# A rejected question is answered with this, one line per question, rather -# than sent to POST /question/{id}/reject: that route carries no feedback, so -# OpenCode ends the turn on it and the model never replies to the room. +# A declined question is answered with this, one line per question, never sent +# to POST /question/{id}/reject: that route carries no feedback, so OpenCode +# ends the turn on it and the model never replies to the room. DECLINED_QUESTION_ANSWER = ( - "The user declined to answer this question. Do not ask it again; continue " - "and reply to the user without it." + "This question was declined. Do not ask it again; continue and reply to " + "the user without it." ) @@ -353,7 +353,7 @@ async def on_question_asked(self, request: OpencodeQuestionRequest) -> None: if not request.questions: logger.warning( - "Rejecting malformed OpenCode question.asked with no questions " + "Declining malformed OpenCode question.asked with no questions " "(request_id=%s room=%s)", request_id, self._ports.room_id, @@ -677,18 +677,15 @@ async def _reply_question( async def _send_question_reject( self, entry: DecisionEntry[PendingQuestion] ) -> bool: - """Perform the reject I/O for an already-claimed question. + """Decline an already-claimed question; see ``DECLINED_QUESTION_ANSWER``. - A question with nothing to answer (malformed) is the only one sent to - OpenCode's reject route; see ``DECLINED_QUESTION_ANSWER``. + A malformed question with no questions gets an empty answer list, which + OpenCode accepts and hands back to the model like any other decline. """ declined = [[DECLINED_QUESTION_ANSWER] for _ in entry.payload.questions] try: async with self._client_reply("reject question", entry.token) as client: - if declined: - await client.reply_question(entry.token, answers=declined) - else: - await client.reject_question(entry.token) + await client.reply_question(entry.token, answers=declined) except ApprovalReplyError: return False self._forget(self._questions, entry) diff --git a/src/band/core/turn.py b/src/band/core/turn.py index 580de9363..4019b9925 100644 --- a/src/band/core/turn.py +++ b/src/band/core/turn.py @@ -26,9 +26,9 @@ class Turn: """The ledger of what one turn's tool calls did. - ``judged`` is set by ``SimpleAdapter.on_event`` for a turn it will judge. - ``detach()`` marks a turn released early (parked on a human decision), so - the adapter judges it at its real end instead of ``on_event``. + ``judged`` is set by ``SimpleAdapter.run_judged_turn`` for a turn it will + judge. ``detach()`` marks a turn released early (parked on a human + decision); a judged one is then judged at the adapter's real end of it. """ def __init__(self) -> None: @@ -51,13 +51,8 @@ def note_reported(self) -> None: self._ledger.note_reported() def detach(self) -> None: - """Move a judged turn's verdict to the adapter's real end of the turn. - - A turn ``on_event`` does not judge stays attached, so - ``judge_detached_turn`` never reports it. - """ - if self.judged: - self._detached = True + """Release the turn before it ends; its delivery is already settled.""" + self._detached = True @property def detached(self) -> bool: @@ -96,10 +91,10 @@ async def report_unsettled_turn(tools: AgentToolsProtocol, *, room_id: str) -> b async def judge_detached_turn(tools: AgentToolsProtocol, *, room_id: str) -> None: - """Judge a detached turn at the adapter's real end of it. + """Judge a judged, detached turn at the adapter's real end of it. ``on_event`` returned before such a turn ended; an attached turn was - already judged there. + already judged there, and an unjudged one is never reported. """ - if tools.turn.detached: + if tools.turn.judged and tools.turn.detached: await report_unsettled_turn(tools, room_id=room_id) diff --git a/src/band/integrations/acp/room_emitter.py b/src/band/integrations/acp/room_emitter.py index 2472999e3..7138f7147 100644 --- a/src/band/integrations/acp/room_emitter.py +++ b/src/band/integrations/acp/room_emitter.py @@ -15,7 +15,7 @@ CollectedChunk, ToolStatus, ) -from band.runtime.tools import turn_effect +from band.runtime.tools import TurnEffect, turn_effect logger = logging.getLogger(__name__) @@ -61,7 +61,12 @@ def __init__( records_tool_effects: bool = False, ) -> None: """``records_tool_effects``: the turn's tools run out of process, so - each completed call's effect is recorded on the turn from the stream.""" + each completed call's effect is recorded on the turn from the stream. + + Those effects are staged and reach the turn only when the prompt closes + successfully: a rejected, timed-out or cancelled prompt owns none of + the work streamed during it, which may belong to another turn. + """ self._tools = tools self._mentions = mentions self._session_id = session_id @@ -72,6 +77,7 @@ def __init__( # reaches the room sink. self._emit = frozenset(emit) if emit is not None else frozenset(Emit) self._pending_text: list[str] = [] + self._staged_effects: list[TurnEffect] = [] async def emit(self, chunk: CollectedChunk) -> None: if self._records_tool_effects: @@ -111,7 +117,7 @@ async def emit(self, chunk: CollectedChunk) -> None: ) def _record_tool_effect(self, chunk: CollectedChunk) -> None: - """Record a completed tool call's effect on the turn. + """Stage a completed tool call's effect for the turn. An external band-mcp runs where the SDK never sees it, so the stream is the only record of what it did. ACP has no structured tool-name field, @@ -123,7 +129,7 @@ def _record_tool_effect(self, chunk: CollectedChunk) -> None: return match chunk.tool: case ACPToolCall(name=name) | ACPToolResult(call=ACPToolCall(name=name)): - self._tools.turn.record(turn_effect(name)) + self._staged_effects.append(turn_effect(name)) def _tool_event_content(self, chunk: CollectedChunk) -> str: """Serialize normalized tool activity for room persistence.""" @@ -180,6 +186,8 @@ async def __aexit__(self, exc_type: object, exc: object, tb: object) -> bool: # neither the held text nor the bookkeeping event. if exc_type is not None: return False + for effect in self._staged_effects: + self._tools.turn.record(effect) # The held runs only ever post together at close, so they relay as the # turn's one reply. await relay_reply( diff --git a/src/band/integrations/claude_sdk/dedup_tools.py b/src/band/integrations/claude_sdk/dedup_tools.py index 6ad7102a7..4b90ea623 100644 --- a/src/band/integrations/claude_sdk/dedup_tools.py +++ b/src/band/integrations/claude_sdk/dedup_tools.py @@ -42,9 +42,6 @@ from collections import OrderedDict from typing import TYPE_CHECKING, Any -from band.runtime.tools.types import BandTool -from band.runtime.turn import record_tool_result - if TYPE_CHECKING: from band.core.protocols import AgentToolsProtocol @@ -160,9 +157,6 @@ async def send_message( async with self._lock: self._evict_expired_locked(now) key = (content, _normalize_mentions(mentions)) - # A deduped send is this turn's reply too: the original already - # reached the room, but recorded it on its own turn. - turn = self._inner.turn cached = self._recent_sends.get(key) if cached is not None: @@ -178,7 +172,6 @@ async def send_message( len(content), len(key[1]), ) - record_tool_result(turn, BandTool.SEND_MESSAGE, cached_result) return cached_result existing = self._in_flight.get(key) @@ -202,8 +195,6 @@ async def send_message( raise await self._finalize_in_flight_send(key, task, now) - if existing is not None: - record_tool_result(turn, BandTool.SEND_MESSAGE, result) return result # --- transparent passthrough -------------------------------------- diff --git a/src/band/integrations/opencode/client.py b/src/band/integrations/opencode/client.py index 82a086d1e..107927c72 100644 --- a/src/band/integrations/opencode/client.py +++ b/src/band/integrations/opencode/client.py @@ -51,8 +51,6 @@ async def reply_question( self, request_id: str, *, answers: list[list[str]] ) -> None: ... - async def reject_question(self, request_id: str) -> None: ... - async def abort_session(self, session_id: str) -> None: ... async def register_mcp_server(self, *, name: str, url: str) -> dict[str, Any]: ... @@ -189,13 +187,6 @@ async def reply_question( ) response.raise_for_status() - async def reject_question(self, request_id: str) -> None: - response = await self._client.post( - f"/question/{request_id}/reject", - params=self._query_params(), - ) - response.raise_for_status() - async def abort_session(self, session_id: str) -> None: response = await self._client.post( f"/session/{session_id}/abort", diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index fbf4ca62f..c346a38a5 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -75,6 +75,15 @@ logger = logging.getLogger(__name__) + +def _log_turn_error(error: Exception, message: str, *args: object) -> None: + """Log a failed turn; one whose failure already reached the room was logged + where it was reported, so it stays out of ERROR alerting.""" + reported = isinstance(error, TurnResultAlreadyReported) + level = logging.DEBUG if reported else logging.ERROR + logger.log(level, message, *args, exc_info=not reported) + + CYCLE_CANCEL_GRACE_SECONDS = 1.0 @@ -1567,8 +1576,8 @@ async def _process_claimed_backlog_message( logger.debug("Message %s processed successfully", msg_id) return BacklogProcessResult.ADVANCED - except Exception as e: - logger.exception("Error processing backlog message %s", msg_id) + except Exception as e: # noqa: BLE001 -- the turn boundary: logged by _log_turn_error, then marked failed + _log_turn_error(e, "Error processing backlog message %s", msg_id) await self._handle_turn_failure(msg_id, attempts, e) return BacklogProcessResult.ADVANCED @@ -2167,8 +2176,8 @@ async def _process_event_body( logger.debug("Event %s processed successfully", event.type) return True - except Exception as e: - logger.exception("Error processing %s", event.type) + except Exception as e: # noqa: BLE001 -- the turn boundary: logged by _log_turn_error, then marked failed + _log_turn_error(e, "Error processing %s", event.type) if isinstance(event, MessageEvent) and msg_id: await self._handle_turn_failure(msg_id, attempts, e) return True diff --git a/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py b/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py index 0ce344318..5a951a208 100644 --- a/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py +++ b/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py @@ -210,16 +210,17 @@ async def test_band_no_reply_does_not_excuse_a_later_silent_turn( assert room.failures == [MISSING_REPLY_TEXT] -async def test_a_turn_whose_only_reply_was_deduped_is_not_a_missing_reply( +async def test_a_turn_whose_only_send_was_a_suppressed_duplicate_is_reported( claude_room: OpenRoom, ) -> None: - """The CLI re-sending the previous turn's reply is suppressed as a - duplicate, and it still answers the turn that sent it.""" + """A send the dedup suppresses posts nothing new, so the turn that made it + did not reply; the room hears that instead of silence.""" room = await claude_room() room.claude.script([room.model_reply("Done.")], [room.model_reply("Done.")]) await room.send("do it") - await room.send("do it again", tools=room.fresh_tools()) + with pytest.raises(TurnResultAlreadyReported): + await room.send("do it again", tools=room.fresh_tools()) assert room.chat == ["Done."] - assert room.failures == [] + assert room.failures == [MISSING_REPLY_TEXT] diff --git a/tests/adapters/opencode/helpers.py b/tests/adapters/opencode/helpers.py index 3f25dc720..6becda7f8 100644 --- a/tests/adapters/opencode/helpers.py +++ b/tests/adapters/opencode/helpers.py @@ -287,7 +287,6 @@ def __init__( self.prompt_calls: list[dict[str, Any]] = [] self.permission_replies: list[dict[str, Any]] = [] self.question_replies: list[dict[str, Any]] = [] - self.question_rejections: list[str] = [] self.aborted_sessions: list[str] = [] self.registered_mcp_servers: list[dict[str, str]] = [] self.disconnected_mcp_servers: list[str] = [] @@ -367,9 +366,6 @@ async def reply_question( for event in self._reply_question_events.get(request_id, []): await self._queue.put(event) - async def reject_question(self, request_id: str) -> None: - self.question_rejections.append(request_id) - async def abort_session(self, session_id: str) -> None: self.aborted_sessions.append(session_id) diff --git a/tests/adapters/opencode/test_approvals.py b/tests/adapters/opencode/test_approvals.py index dedde2df1..ecde0540d 100644 --- a/tests/adapters/opencode/test_approvals.py +++ b/tests/adapters/opencode/test_approvals.py @@ -392,17 +392,18 @@ async def test_mention_only_question_reply_requests_a_real_answer( assert "waiting for answers" in tools.messages_sent[-1]["content"].lower() -async def test_malformed_question_with_no_questions_is_rejected( +async def test_malformed_question_with_no_questions_is_declined( asks: AskFactory, ) -> None: - """A malformed question with an id must not leave OpenCode blocked.""" + """A malformed question is answered with an empty decline, which hands the + turn back to the model instead of blocking it.""" client = FakeOpencodeClient() tools = FakeAgentTools() approvals = make_room_approvals(client, tools=tools) await approvals.on_question_asked(asks.question("q-empty", questions=[])) - assert client.question_rejections == ["q-empty"] + assert client.question_replies == [{"request_id": "q-empty", "answers": []}] assert not approvals.awaiting_human() assert tools.messages_sent == [] diff --git a/tests/framework_conformance/test_reply_boundary.py b/tests/framework_conformance/test_reply_boundary.py index 688f5fec2..6c3e021bb 100644 --- a/tests/framework_conformance/test_reply_boundary.py +++ b/tests/framework_conformance/test_reply_boundary.py @@ -1,77 +1,89 @@ """Reply boundary test. -A ``send_message`` call counts as the turn's reply, so only the model's own -words may go through it: the tool methods, ``deliver_reply``/``relay_reply``, -and the framework tool implementations that call them. An adapter's own post -(an approval prompt, a busy notice, a status reply) goes through +A ``send_message``, ``deliver_reply`` or ``relay_reply`` call counts as the +turn's reply, so only the model's own words may go through one. An adapter's +own post (an approval prompt, a busy notice, a status reply) goes through ``send_notice``, or it would stand in for the model's answer and suppress the -final-text relay. This scans ``src/band`` via AST for a direct -``x.send_message(...)`` call outside the allowlist, so a new adapter post on -``send_message`` fails here instead of silently counting as a reply. +final-text relay. This scans ``src/band`` via AST and pins every such call +outside the tool implementations, per file with the reason it carries the +model's words, so a new reply call anywhere fails here until it is justified. """ from __future__ import annotations import ast -from pathlib import Path +from collections import Counter from tests.paths import REPO_ROOT -_SCAN_ROOT = REPO_ROOT / "src" / "band" +SCAN_ROOT = REPO_ROOT / "src" / "band" -# The model's tool implementations and the relay that carries its words, plus -# two files where ``send_message`` is an unrelated client's method. -_ALLOWED_DIRS: frozenset[Path] = frozenset({_SCAN_ROOT / "runtime" / "tools"}) -_ALLOWED_FILES: frozenset[Path] = frozenset( - _SCAN_ROOT / path - for path in ( - "core/delivery.py", - "integrations/crewai/catalog.py", - "integrations/parlant/tools.py", - # Forwards to the inner tools' send_message, which records the reply. - "integrations/claude_sdk/dedup_tools.py", - # Exempt from the turn verdict: the flow posts its own outcomes. - "adapters/crewai_flow.py", - # The A2A client's own method. - "integrations/a2a/adapter.py", - # The Phoenix protocol's own method. - "testing/phoenix_server.py", - ) -) - - -def _calls_send_message(source: str) -> bool: - return any( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Attribute) - and node.func.attr == "send_message" - for node in ast.walk(ast.parse(source)) - ) +#: The model's tool implementations, where every reply call is the model's. +TOOLS_DIR = SCAN_ROOT / "runtime" / "tools" +REPLY_CALLS = frozenset({"send_message", "deliver_reply", "relay_reply"}) -def _is_allowed(path: Path) -> bool: - return path in _ALLOWED_FILES or any(d in path.parents for d in _ALLOWED_DIRS) +#: Reply calls per file outside ``TOOLS_DIR``, and why each carries the model's words. +ALLOWED_REPLY_CALLS: dict[str, tuple[Counter[str], str]] = { + "core/delivery.py": ( + Counter(send_message=1, deliver_reply=1), + "deliver_reply posts the reply; relay_reply delegates to it", + ), + "adapters/codex.py": (Counter(relay_reply=1), "the model's final text"), + "adapters/copilot_sdk.py": ( + Counter(relay_reply=1, deliver_reply=1), + "the model's final text, and its ask_user question as the turn's reply", + ), + "adapters/letta.py": (Counter(relay_reply=1), "the model's final text"), + "adapters/opencode/adapter.py": (Counter(relay_reply=1), "the model's final text"), + "adapters/parlant.py": (Counter(relay_reply=1), "the engine's message"), + "integrations/acp/room_emitter.py": ( + Counter(relay_reply=1), + "the agent's held text runs", + ), + "integrations/crewai/catalog.py": (Counter(send_message=1), "the crew's tool"), + "integrations/parlant/tools.py": (Counter(send_message=1), "the engine's tool"), + "integrations/claude_sdk/dedup_tools.py": ( + Counter(send_message=1), + "forwards the model's tool call to the inner tools", + ), + "integrations/a2a/adapter.py": ( + Counter(send_message=1, deliver_reply=3), + "the remote agent's answer; send_message is the A2A client's method", + ), + "adapters/crewai_flow.py": ( + Counter(send_message=3), + "exempt from the verdict: the flow posts its own outcomes", + ), + "testing/phoenix_server.py": ( + Counter(send_message=2), + "the Phoenix protocol's own method", + ), +} -def test_adapter_posts_never_count_as_the_model_reply() -> None: - offenders = sorted( - path.relative_to(REPO_ROOT) - for path in _SCAN_ROOT.rglob("*.py") - if not _is_allowed(path) and _calls_send_message(path.read_text("utf-8")) +def reply_calls(source: str) -> Counter[str]: + """How many times ``source`` calls each reply path.""" + names = ( + node.func.attr if isinstance(node.func, ast.Attribute) else node.func.id + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute | ast.Name) ) + return Counter(name for name in names if name in REPLY_CALLS) - assert not offenders, ( - f"Direct send_message calls outside the model's reply paths: {offenders}. " - "Post an adapter's own message with tools.send_notice (and settle the " - "turn with tools.turn.settle() when it ends the turn), or relay the " - "model's words through band.core.delivery.deliver_reply/relay_reply." - ) +def test_every_reply_call_carries_the_model_words() -> None: + found = { + path.relative_to(SCAN_ROOT).as_posix(): calls + for path in SCAN_ROOT.rglob("*.py") + if TOOLS_DIR not in path.parents + and (calls := reply_calls(path.read_text("utf-8"))) + } -def test_allowlist_entries_still_exist() -> None: - missing = [ - path.relative_to(REPO_ROOT) - for path in _ALLOWED_FILES | _ALLOWED_DIRS - if not path.exists() - ] - assert not missing, f"Allowlisted paths no longer exist: {missing}" + assert found == {path: calls for path, (calls, _) in ALLOWED_REPLY_CALLS.items()}, ( + "A reply call changed outside the model's reply paths. Post an adapter's " + "own message with tools.send_notice (and tools.turn.settle() when it ends " + "the turn); relay the model's words through relay_reply and record why " + "in ALLOWED_REPLY_CALLS." + ) diff --git a/tests/framework_conformance/test_turn_outcome.py b/tests/framework_conformance/test_turn_outcome.py index c910049ff..66f5f2c12 100644 --- a/tests/framework_conformance/test_turn_outcome.py +++ b/tests/framework_conformance/test_turn_outcome.py @@ -179,12 +179,12 @@ def test_only_the_declared_adapters_go_unjudged() -> None: assert unjudged == UNJUDGED_FRAMEWORK_IDS -#: Every SimpleAdapter in band outside ADAPTER_CONFIGS, and what covers its verdict. -COVERED_OUTSIDE_CONFIGS = { - "ACPClientAdapter": "the acp probe", - "CopilotACPAdapter": "ACPClientAdapter's on_message, run by the acp probe", - "OmpACPAdapter": "ACPClientAdapter's on_message, run by the acp probe", - "CursorACPAdapter": "the cursor_acp probe", +#: Judged adapters outside ADAPTER_CONFIGS, by the probe that runs their rows. +#: A subclass that inherits one of these ``on_message`` methods is covered too. +PROBED_OUTSIDE_CONFIGS = {"ACPClientAdapter": "acp", "CursorACPAdapter": "cursor_acp"} + +#: Adapters outside ADAPTER_CONFIGS whose verdict their own tests pin. +PINNED_OUTSIDE_CONFIGS = { "SlackAdapter": "its brain's verdict (tests/integrations/slack)", "A2AAdapter": "unjudged (tests/integrations/a2a)", "A2AGatewayAdapter": "unjudged (tests/integrations/a2a/gateway)", @@ -192,7 +192,7 @@ def test_only_the_declared_adapters_go_unjudged() -> None: } -def band_adapter_names() -> set[str]: +def band_adapters() -> dict[str, type[SimpleAdapter]]: """Every concrete SimpleAdapter defined in band whose module imports here.""" for module in pkgutil.walk_packages(band.__path__, "band."): if module.name.endswith("__main__"): @@ -201,19 +201,37 @@ def band_adapter_names() -> set[str]: importlib.import_module(module.name) except ImportError: continue # an extra this venv lacks; its own lane covers it - pending, names = [SimpleAdapter], set() + pending, adapters = [SimpleAdapter], {} while pending: for cls in pending.pop().__subclasses__(): pending.append(cls) if cls.__module__.startswith("band.") and not inspect.isabstract(cls): - names.add(cls.__name__) - return names + adapters[cls.__name__] = cls + return adapters + + +def probe_running( + cls: type[SimpleAdapter], adapters: dict[str, type[SimpleAdapter]] +) -> str | None: + """The probe that runs ``cls``'s turn, via the ``on_message`` it uses.""" + for name, probe in PROBED_OUTSIDE_CONFIGS.items(): + owner = adapters.get(name) + if owner and issubclass(cls, owner) and cls.on_message is owner.on_message: + return probe + return None -def test_every_band_adapter_is_registered_or_covered() -> None: +def test_every_band_adapter_is_registered_probed_or_pinned() -> None: registered = {type(cfg.adapter_factory()).__name__ for cfg in ADAPTER_CONFIGS} + adapters = band_adapters() + unprobed = { + name + for name, cls in adapters.items() + if name not in registered + and probe_running(cls, adapters) not in TURN_OUTCOME_PROBES + } - assert band_adapter_names() - registered == set(COVERED_OUTSIDE_CONFIGS) + assert unprobed == set(PINNED_OUTSIDE_CONFIGS) def test_every_judged_adapter_has_a_probe() -> None: diff --git a/tests/integrations/acp/test_room_emitter.py b/tests/integrations/acp/test_room_emitter.py index 9563e234f..327557e28 100644 --- a/tests/integrations/acp/test_room_emitter.py +++ b/tests/integrations/acp/test_room_emitter.py @@ -440,6 +440,28 @@ async def test_an_out_of_process_call_that_did_not_reply_keeps_the_text( assert sent == ["The answer."] + @pytest.mark.asyncio + async def test_a_failed_prompt_records_nothing_streamed_during_it(self) -> None: + """A rejected prompt (e.g. busy) owns no work: a reply streamed during + it belongs to another turn and must not suppress the retry's answer.""" + tools = FakeAgentTools() + emitter = RoomTurnEmitter( + tools, + mentions=[{"id": "u1", "name": "User"}], + session_id="s1", + room_id="room-1", + records_tool_effects=True, + ) + + with pytest.raises(RuntimeError): + async with emitter: + await emitter.emit( + tool_call_chunk(BandTool.SEND_MESSAGE, ToolStatus.COMPLETED) + ) + raise RuntimeError("session busy") + + assert not tools.turn.replied + @pytest.mark.asyncio async def test_an_out_of_process_action_completes_the_turn(self) -> None: tools = FakeAgentTools() diff --git a/tests/integrations/claude_sdk/test_dedup_tools.py b/tests/integrations/claude_sdk/test_dedup_tools.py index ba32c5922..5645d8493 100644 --- a/tests/integrations/claude_sdk/test_dedup_tools.py +++ b/tests/integrations/claude_sdk/test_dedup_tools.py @@ -536,24 +536,27 @@ async def send_message( return await super().send_message(content, mentions) -class TestDedupedSendRepliesForItsTurn: - """A suppressed duplicate still answers the turn that sent it, since the - original already reached the room.""" +class TestDedupedSendBelongsToItsTurn: + """A suppressed duplicate is the earlier turn's send re-issued: it posts + nothing new, so it never counts as a later turn's reply.""" @pytest.mark.asyncio - async def test_a_cached_duplicate_replies_for_the_next_turn(self): - wrapper = DedupingAgentTools(FakeAgentTools()) + async def test_a_cached_duplicate_does_not_answer_the_next_turn(self): + first_turn = FakeAgentTools() + wrapper = DedupingAgentTools(first_turn) await wrapper.send_message("hi", ["alice"]) next_turn = FakeAgentTools() await wrapper.update_inner(next_turn) await wrapper.send_message("hi", ["alice"]) - assert next_turn.messages_sent == [] - assert next_turn.turn.replied + assert first_turn.turn.replied + assert not next_turn.turn.replied @pytest.mark.asyncio - async def test_a_duplicate_joining_an_in_flight_send_replies_for_its_turn(self): + async def test_a_duplicate_joining_an_in_flight_send_does_not_answer_its_turn( + self, + ): first_turn = GatedSendTools() wrapper = DedupingAgentTools(first_turn) original = asyncio.create_task(wrapper.send_message("hi", ["alice"])) @@ -566,5 +569,5 @@ async def test_a_duplicate_joining_an_in_flight_send_replies_for_its_turn(self): first_turn.gate.set() await asyncio.gather(original, duplicate) - assert next_turn.messages_sent == [] - assert next_turn.turn.replied + assert first_turn.turn.replied + assert not next_turn.turn.replied diff --git a/tests/integrations/opencode/test_client.py b/tests/integrations/opencode/test_client.py index 7aaa23da4..d66df2050 100644 --- a/tests/integrations/opencode/test_client.py +++ b/tests/integrations/opencode/test_client.py @@ -50,11 +50,6 @@ def __init__(self) -> None: self._reply_question, methods=["POST"], ), - Route( - "/question/{request_id}/reject", - self._reject_question, - methods=["POST"], - ), Route( "/session/{session_id}/abort", self._abort_session, methods=["POST"] ), @@ -99,10 +94,6 @@ async def _reply_question(self, request: Request) -> Response: await self._record(request) return Response(status_code=200) - async def _reject_question(self, request: Request) -> Response: - await self._record(request) - return Response(status_code=200) - async def _abort_session(self, request: Request) -> Response: await self._record(request) return Response(status_code=200) @@ -317,17 +308,6 @@ async def test_reply_question_posts_answers(fake_server: FakeOpencodeServer) -> await client.close() -async def test_reject_question_posts_to_reject_path( - fake_server: FakeOpencodeServer, -) -> None: - client = make_client(fake_server) - try: - await client.reject_question("req-1") - assert fake_server.requests[-1]["path"] == "/question/req-1/reject" - finally: - await client.close() - - async def test_abort_session_posts_to_abort_path( fake_server: FakeOpencodeServer, ) -> None: diff --git a/tests/runtime/test_turn.py b/tests/runtime/test_turn.py index 6010d93d9..6e6ab23bc 100644 --- a/tests/runtime/test_turn.py +++ b/tests/runtime/test_turn.py @@ -11,11 +11,12 @@ from typing import Any from unittest.mock import MagicMock +import band_sdk_core import pytest from pydantic import BaseModel from band.core.delivery import deliver_reply -from band.core.turn import report_unsettled_turn +from band.core.turn import judge_detached_turn, report_unsettled_turn from band.integrations.claude_sdk.dedup_tools import DedupingAgentTools from band.runtime.custom_tools import declares_turn_effect, execute_custom_tool from band.runtime.tools import AgentTools, BandTool, TurnEffect @@ -178,34 +179,39 @@ async def test_a_settled_turn_reports_nothing(self) -> None: assert await report_unsettled_turn(tools, room_id="room-1") is False -class TestDetach: - def test_a_judged_turn_detaches(self) -> None: +class TestJudgeDetachedTurn: + async def test_a_judged_detached_turn_is_reported(self) -> None: tools = FakeAgentTools() tools.turn.judged = True - tools.turn.detach() - assert tools.turn.detached + await judge_detached_turn(tools, room_id="room-1") + + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] - def test_an_unjudged_turn_never_detaches(self) -> None: + async def test_an_unjudged_detached_turn_is_never_reported(self) -> None: """A contact-hub turn parked on a decision is never reported later.""" tools = FakeAgentTools() - tools.turn.detach() - assert not tools.turn.detached + await judge_detached_turn(tools, room_id="room-1") + + assert reported_failures(tools) == [] +@pytest.mark.parametrize("judged", [True, False], ids=["judged", "contact-hub"]) async def test_a_detached_report_never_marks_the_contexts_next_message( - mock_rest_client: Any, + mock_rest_client: Any, judged: bool ) -> None: + """A released turn reports after its delivery settled, while the context + may already be processing a later message, judged or not.""" ctx = MagicMock(participants=[ALICE], agent_id="agent-1", hub_room_id=None) ctx.link.rest = mock_rest_client tools = AgentTools.from_context(ctx) - tools.turn.judged = True + tools.turn.judged = judged tools.turn.detach() - assert await report_unsettled_turn(tools, room_id="room-1") is True + await tools.send_failure(band_sdk_core.AgentFailure("codex", "timed out")) mock_rest_client.agent_api_events.create_agent_chat_event.assert_awaited_once() ctx.note_turn_failure_reported.assert_not_called() diff --git a/tests/runtime/test_turn_verdict.py b/tests/runtime/test_turn_verdict.py index 1f928ba60..053acf39d 100644 --- a/tests/runtime/test_turn_verdict.py +++ b/tests/runtime/test_turn_verdict.py @@ -9,6 +9,7 @@ from __future__ import annotations +import logging from collections.abc import Awaitable, Callable from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -33,6 +34,7 @@ from tests.conftest import make_message_event, make_participant_mock AGENT_ID = "agent-123" +execution_logger = ExecutionContext.__module__ ROOM_ID = "room-123" USER = ["@user-1"] @@ -179,6 +181,22 @@ async def test_each_turn_outcome_is_reported_honestly( assert runtime_failures(link) == [MISSING_REPLY_FAILURE] +async def test_a_missing_reply_is_not_logged_as_a_runtime_error( + link: MagicMock, caplog: pytest.LogCaptureFixture +) -> None: + """A silent turn is an expected verdict, already logged as a WARNING where + it was reported, so the runtime adds nothing at ERROR to trip alerting.""" + ctx = run_through(ScriptedAdapter([]), link) + + with caplog.at_level(logging.DEBUG): + await ctx._process_event( + make_message_event(room_id=ROOM_ID, sender_id="user-1") + ) + + runtime_levels = {r.levelno for r in caplog.records if r.name == execution_logger} + assert logging.ERROR not in runtime_levels + + async def test_an_exempt_adapter_is_never_judged(link: MagicMock) -> None: ctx = run_through(ScriptedAdapter([], judges=False), link) From 27410485e34d184a44110c750197f4bd1dc581c1 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Mon, 5 Oct 2026 09:57:10 +0300 Subject: [PATCH 11/17] fix: fall back when a missing-reply report fails and honor the report opt-out A missing-reply post that failed still raised TurnResultAlreadyReported, so the runtime skipped its own fallback report and logged the turn at DEBUG. It now raises a plain error in that case. The report also honors SessionConfig.report_turn_failures_to_room, as the runtime's own failure reports do. Claude SDK and Codex log a WARNING for the failures they report, since the runtime now logs reported failures at DEBUG. Also adds tests for both log levels on the live and backlog paths, pins TurnEffect.did_work to core's ledger, moves the relaying flag onto each turn-outcome probe, and uses core's TurnVerdict in the conformance suite. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016nwuEBtdtneuDkJHcvijDQ Signed-off-by: Alexander Zaikman --- AGENTS.md | 2 +- docs/turn-outcome.md | 8 +- src/band/adapters/claude_sdk.py | 3 +- src/band/adapters/codex.py | 7 ++ src/band/core/simple_adapter.py | 7 +- src/band/core/turn.py | 24 +++- src/band/integrations/acp/room_emitter.py | 4 +- src/band/runtime/execution.py | 6 +- src/band/runtime/tools/agent.py | 1 + .../failures/test_turn_failure_surfacing.py | 19 ++- tests/adapters/test_codex_adapter.py | 9 +- tests/core/test_turn_types.py | 7 ++ .../test_reply_boundary.py | 10 +- .../test_turn_outcome.py | 48 ++++---- .../turnprobes/__init__.py | 4 +- tests/framework_conformance/turnprobes/acp.py | 6 +- .../framework_conformance/turnprobes/codex.py | 2 +- .../turnprobes/copilotsdk.py | 2 +- .../framework_conformance/turnprobes/letta.py | 4 +- .../turnprobes/opencode.py | 2 +- tests/runtime/test_turn.py | 12 +- tests/runtime/test_turn_verdict.py | 108 +++++++++++++++--- 22 files changed, 215 insertions(+), 80 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index dee8b7f32..66169ffea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,7 @@ on its test config so the guard can read what the model actually sees. ## Turn Outcome -`SimpleAdapter.on_event` judges every turn with band-sdk-core's rule: a turn +`SimpleAdapter.run_judged_turn` (called by `on_event`) judges every turn with band-sdk-core's rule: a turn that replied, declined via `band_no_reply`, did real work, was settled by the adapter, or already reported a failure completes; anything else is reported once and marked FAILED. Adapters never judge turns themselves. Post an diff --git a/docs/turn-outcome.md b/docs/turn-outcome.md index 9b792ff5b..d62e59b23 100644 --- a/docs/turn-outcome.md +++ b/docs/turn-outcome.md @@ -208,13 +208,15 @@ toward: the base prompt still says plain text is never delivered, because without that line models more often send their closing narration as a second `band_send_message` (measured live on gemini-2.5-flash). -`tests/framework_conformance/test_reply_boundary.py` fails if adapter code calls -`send_message` directly outside the model's reply paths. +`tests/framework_conformance/test_reply_boundary.py` pins every `send_message`, +`deliver_reply` and `relay_reply` call outside the tool implementations, per +file with the reason it carries the model's words, so a new one fails until it +is justified. ## Detached turns An adapter whose turn outlives `on_message` (one parked on a human approval) -calls `tools.turn.detach()` when it releases the turn early. `on_event` then +calls `tools.turn.detach()` when it releases the turn early. `run_judged_turn` then leaves the verdict to the adapter, which reports at the turn's normal end. A cancelled turn reports nothing: diff --git a/src/band/adapters/claude_sdk.py b/src/band/adapters/claude_sdk.py index ffef1f8b0..7cb444a65 100644 --- a/src/band/adapters/claude_sdk.py +++ b/src/band/adapters/claude_sdk.py @@ -1370,7 +1370,7 @@ async def _on_turn_complete( """Close out the turn: persist session id, emit usage, surface failure. A turn released early for an approval is judged here, at its real end; - any other turn is judged by ``on_event`` once ``on_message`` returns. + any other turn is judged by ``run_judged_turn`` once ``on_message`` returns. """ logger.info( "Room %s: Complete - %sms, $%.4f", @@ -1394,6 +1394,7 @@ async def _on_turn_complete( else None ) detail = self._result_error_detail(sdk_message) + logger.warning("Room %s: %s", room_id, detail) await tools.send_failure( AgentFailure(_PROVIDER, detail, code, sdk_message.errors) ) diff --git a/src/band/adapters/codex.py b/src/band/adapters/codex.py index ea8dc5bbe..47e3311f7 100644 --- a/src/band/adapters/codex.py +++ b/src/band/adapters/codex.py @@ -1413,6 +1413,13 @@ async def _process_turn_events( if event.method == "transport/closed": result.turn_status = "failed" result.turn_error = "Codex transport closed unexpectedly" + logger.warning( + "Room %s: %s (thread=%s turn=%s)", + room_id, + result.turn_error, + thread_id, + turn_id, + ) # Reset client state so _ensure_client_ready() rebuilds # on the next message instead of reusing a dead client. self._client = None diff --git a/src/band/core/simple_adapter.py b/src/band/core/simple_adapter.py index a45143654..2f4ba6252 100644 --- a/src/band/core/simple_adapter.py +++ b/src/band/core/simple_adapter.py @@ -21,9 +21,8 @@ from band.core.protocols import ( AgentToolsProtocol, HistoryConverter, - TurnResultAlreadyReported, ) -from band.core.turn import report_unsettled_turn +from band.core.turn import report_unsettled_turn, unsettled_turn_error from band.core.types import ( USAGE_EVENT_TYPE, USAGE_METADATA_KEY, @@ -372,7 +371,7 @@ async def on_started(self, agent_name: str, agent_description: str) -> None: @property def judges_turns(self) -> bool: - """Whether ``on_event`` judges each turn by core's turn-outcome rule. + """Whether ``run_judged_turn`` judges each turn by core's turn-outcome rule. ``False`` for an adapter whose turns are not the model's to answer through Band tools: a bridge to another agent, or a framework engine @@ -446,4 +445,4 @@ async def run_judged_turn( and not turn.detached and await report_unsettled_turn(tools, room_id=room_id) ): - raise TurnResultAlreadyReported("turn ended without a reply") + raise unsettled_turn_error(turn) diff --git a/src/band/core/turn.py b/src/band/core/turn.py index 4019b9925..29d86add7 100644 --- a/src/band/core/turn.py +++ b/src/band/core/turn.py @@ -14,7 +14,7 @@ import band_sdk_core from band_sdk_core import AgentFailure, TurnLedger, TurnVerdict -from band.core.protocols import TURN_FAILURE_PROVIDER +from band.core.protocols import TURN_FAILURE_PROVIDER, TurnResultAlreadyReported if TYPE_CHECKING: from band.core.protocols import AgentToolsProtocol @@ -31,8 +31,11 @@ class Turn: decision); a judged one is then judged at the adapter's real end of it. """ - def __init__(self) -> None: + def __init__(self, *, posts_missing_reply: bool = True) -> None: self._ledger = TurnLedger() + # SessionConfig.report_turn_failures_to_room: the missing reply is a + # failure the runtime detects, so the session's opt-out covers it. + self.posts_missing_reply = posts_missing_reply self.judged = False self._detached = False @@ -86,14 +89,27 @@ async def report_unsettled_turn(tools: AgentToolsProtocol, *, room_id: str) -> b room_id, tools.turn.detached, ) - await tools.send_failure(MISSING_REPLY) + if tools.turn.posts_missing_reply: + await tools.send_failure(MISSING_REPLY) return True +def unsettled_turn_error(turn: Turn) -> Exception: + """What a reported, unsettled turn raises to the runtime. + + ``TurnResultAlreadyReported`` when the room has the report or the session + posts none, so the runtime adds nothing. A report that failed to post + raises a plain error instead, so the runtime's own failure report runs. + """ + if turn.complete or not turn.posts_missing_reply: + return TurnResultAlreadyReported("turn ended without a reply") + return RuntimeError("turn ended without a reply, and its report did not post") + + async def judge_detached_turn(tools: AgentToolsProtocol, *, room_id: str) -> None: """Judge a judged, detached turn at the adapter's real end of it. - ``on_event`` returned before such a turn ended; an attached turn was + ``run_judged_turn`` returned before such a turn ended; an attached turn was already judged there, and an unjudged one is never reported. """ if tools.turn.judged and tools.turn.detached: diff --git a/src/band/integrations/acp/room_emitter.py b/src/band/integrations/acp/room_emitter.py index 7138f7147..11a2c33d2 100644 --- a/src/band/integrations/acp/room_emitter.py +++ b/src/band/integrations/acp/room_emitter.py @@ -81,7 +81,7 @@ def __init__( async def emit(self, chunk: CollectedChunk) -> None: if self._records_tool_effects: - self._record_tool_effect(chunk) + self._stage_tool_effect(chunk) match chunk.chunk_type: case ChunkType.TEXT: if chunk.content: @@ -116,7 +116,7 @@ async def emit(self, chunk: CollectedChunk) -> None: chunk.chunk_type, ) - def _record_tool_effect(self, chunk: CollectedChunk) -> None: + def _stage_tool_effect(self, chunk: CollectedChunk) -> None: """Stage a completed tool call's effect for the turn. An external band-mcp runs where the SDK never sees it, so the stream is diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index c346a38a5..ac4ead8e8 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -76,6 +76,9 @@ logger = logging.getLogger(__name__) +CYCLE_CANCEL_GRACE_SECONDS = 1.0 + + def _log_turn_error(error: Exception, message: str, *args: object) -> None: """Log a failed turn; one whose failure already reached the room was logged where it was reported, so it stays out of ERROR alerting.""" @@ -84,9 +87,6 @@ def _log_turn_error(error: Exception, message: str, *args: object) -> None: logger.log(level, message, *args, exc_info=not reported) -CYCLE_CANCEL_GRACE_SECONDS = 1.0 - - class ResyncRequest: """Sentinel pushed into the execution queue to trigger an immediate /next resync. diff --git a/src/band/runtime/tools/agent.py b/src/band/runtime/tools/agent.py index a765e94d1..49c8f0ee1 100644 --- a/src/band/runtime/tools/agent.py +++ b/src/band/runtime/tools/agent.py @@ -365,6 +365,7 @@ def from_context(cls, ctx: ExecutionContext) -> AgentTools: agent_id=ctx.agent_id, ) tools._ctx = ctx + tools.turn = Turn(posts_missing_reply=ctx.config.report_turn_failures_to_room) return tools # --- Tool methods --- diff --git a/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py b/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py index 5a951a208..44954d194 100644 --- a/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py +++ b/tests/adapters/claude_sdk/failures/test_turn_failure_surfacing.py @@ -3,6 +3,7 @@ from __future__ import annotations +import logging from collections.abc import Awaitable, Callable import pytest @@ -11,6 +12,7 @@ APPROVAL_REQUESTED_TEMPLATE, APPROVAL_RESOLVED_TEMPLATE, ClaudeApprovalOptions, + ClaudeSDKAdapter, ClaudeSDKAdapterConfig, TurnResultAlreadyReported, ) @@ -89,10 +91,11 @@ async def test_the_room_hears_whenever_a_turn_left_it_unanswered( async def test_a_failure_the_cli_reports_reaches_the_room_with_its_status( - claude_room: OpenRoom, + claude_room: OpenRoom, caplog: pytest.LogCaptureFixture ) -> None: """``is_error`` wins even when ``subtype`` says success and even after a - reply went out; the room gets the CLI's own detail and API status.""" + reply went out; the room gets the CLI's own detail and API status, and the + operator a WARNING for each (the runtime logs it only at DEBUG).""" room = await claude_room() room.claude.script( [EndTurn(is_error=True, result="Not logged in · Please run /login")], @@ -106,9 +109,15 @@ async def test_a_failure_the_cli_reports_reaches_the_room_with_its_status( ], ) - for question in ("first", "second"): - with pytest.raises(TurnResultAlreadyReported): - await room.send(question) + with caplog.at_level(logging.WARNING, logger=ClaudeSDKAdapter.__module__): + for question in ("first", "second"): + with pytest.raises(TurnResultAlreadyReported): + await room.send(question) + + warnings = [r.getMessage() for r in caplog.records if r.levelno >= logging.WARNING] + assert warnings == [ + f"Room room-1: {failure['message']}" for failure in room.reported_failures + ] assert room.reported_failures == [ { diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index 0933e1af3..f74aa089e 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -1261,8 +1261,11 @@ async def test_external_sandbox_alias_uses_sandbox_policy(self) -> None: assert turn_start["sandboxPolicy"]["type"] == "externalSandbox" @pytest.mark.asyncio - async def test_transport_closed_event_aborts_turn(self) -> None: - """A transport/closed event should end the turn with a failed status.""" + async def test_transport_closed_event_aborts_turn( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """A transport/closed event ends the turn with a failure the room sees + and a WARNING the operator sees (the runtime logs it only at DEBUG).""" events = [ event_notification( "transport/closed", @@ -1288,6 +1291,8 @@ async def test_transport_closed_event_aborts_turn(self) -> None: # Adapter should report a failure mentioning the disconnect. failures = reported_failures(tools) assert any("transport closed" in f["message"].lower() for f in failures) + (warning,) = [r for r in caplog.records if r.levelno == logging.WARNING] + assert "transport closed" in warning.getMessage() @pytest.mark.asyncio async def test_transport_closed_resets_client_state(self) -> None: diff --git a/tests/core/test_turn_types.py b/tests/core/test_turn_types.py index 8ff246e92..64fe51135 100644 --- a/tests/core/test_turn_types.py +++ b/tests/core/test_turn_types.py @@ -33,3 +33,10 @@ def test_band_tools_match_core() -> None: @pytest.mark.parametrize("tool_name", sorted(CORE_EFFECTS)) def test_turn_effect_matches_core(tool_name: str) -> None: assert turn_effect(tool_name) == CORE_EFFECTS[tool_name].wire_name + + +@pytest.mark.parametrize("effect", list(TurnEffect)) +def test_did_work_matches_core_verdict(effect: TurnEffect) -> None: + ledger = band_sdk_core.TurnLedger() + ledger.record(band_sdk_core.TurnEffect.from_wire_name(effect.value)) + assert effect.did_work == (ledger.verdict() == band_sdk_core.TurnVerdict.Complete) diff --git a/tests/framework_conformance/test_reply_boundary.py b/tests/framework_conformance/test_reply_boundary.py index 6c3e021bb..28f1e956b 100644 --- a/tests/framework_conformance/test_reply_boundary.py +++ b/tests/framework_conformance/test_reply_boundary.py @@ -14,12 +14,10 @@ import ast from collections import Counter -from tests.paths import REPO_ROOT - -SCAN_ROOT = REPO_ROOT / "src" / "band" +from tests.paths import SRC_ROOT #: The model's tool implementations, where every reply call is the model's. -TOOLS_DIR = SCAN_ROOT / "runtime" / "tools" +TOOLS_DIR = SRC_ROOT / "runtime" / "tools" REPLY_CALLS = frozenset({"send_message", "deliver_reply", "relay_reply"}) @@ -75,8 +73,8 @@ def reply_calls(source: str) -> Counter[str]: def test_every_reply_call_carries_the_model_words() -> None: found = { - path.relative_to(SCAN_ROOT).as_posix(): calls - for path in SCAN_ROOT.rglob("*.py") + path.relative_to(SRC_ROOT).as_posix(): calls + for path in SRC_ROOT.rglob("*.py") if TOOLS_DIR not in path.parents and (calls := reply_calls(path.read_text("utf-8"))) } diff --git a/tests/framework_conformance/test_turn_outcome.py b/tests/framework_conformance/test_turn_outcome.py index 66f5f2c12..888a8cca6 100644 --- a/tests/framework_conformance/test_turn_outcome.py +++ b/tests/framework_conformance/test_turn_outcome.py @@ -18,9 +18,9 @@ import inspect import pkgutil from dataclasses import dataclass -from enum import StrEnum import pytest +from band_sdk_core import TurnVerdict import band from band.core.protocols import TurnResultAlreadyReported @@ -55,23 +55,13 @@ **toolloop.PROBES, } -# Adapters that post the model's final text when no tool replied. -RELAYING_FRAMEWORK_IDS = frozenset( - {"codex", "copilot_sdk", "opencode", "letta", "acp", "cursor_acp"} -) - ANSWER = "The vault code is 4471-ECHO." REPLY = ToolCall(BandTool.SEND_MESSAGE, {"content": ANSWER, "mentions": ["@alice"]}) -class Verdict(StrEnum): - COMPLETE = "complete" - MISSING_REPLY = "missing_reply" - - @dataclass(frozen=True) class TurnResult: - verdict: Verdict + verdict: TurnVerdict messages: list[str] failures: list[tuple[str, str]] @@ -80,13 +70,13 @@ async def run_turn(probe: TurnOutcomeProbe, script: TurnScript) -> TurnResult: tools = turn_tools() try: await probe.run(script, tools) - verdict = Verdict.COMPLETE + verdict = TurnVerdict.Complete except TurnResultAlreadyReported: - verdict = Verdict.MISSING_REPLY + verdict = TurnVerdict.MissingReply return observed(tools, verdict) -def observed(tools: FakeAgentTools, verdict: Verdict) -> TurnResult: +def observed(tools: FakeAgentTools, verdict: TurnVerdict) -> TurnResult: return TurnResult( verdict=verdict, messages=tools.chat, @@ -101,12 +91,12 @@ def observed(tools: FakeAgentTools, verdict: Verdict) -> TurnResult: } -def probe_params(framework_ids: frozenset[str] | None = None) -> list[str]: - return sorted( - name - for name in TURN_OUTCOME_PROBES - if framework_ids is None or name in framework_ids - ) +def probe_params() -> list[str]: + return sorted(TURN_OUTCOME_PROBES) + + +def relaying_probe_params() -> list[str]: + return [name for name in probe_params() if TURN_OUTCOME_PROBES[name].relays] @pytest.mark.parametrize("framework_id", probe_params()) @@ -118,7 +108,7 @@ async def test_a_turn_that_answered_or_worked_completes( TURN_OUTCOME_PROBES[framework_id], COMPLETING_SCRIPTS[script_name] ) - assert result.verdict is Verdict.COMPLETE + assert result.verdict == TurnVerdict.Complete assert result.failures == [] @@ -126,7 +116,7 @@ async def test_a_turn_that_answered_or_worked_completes( async def test_a_turn_that_did_nothing_is_reported_once(framework_id: str) -> None: result = await run_turn(TURN_OUTCOME_PROBES[framework_id], TurnScript()) - assert result.verdict is Verdict.MISSING_REPLY + assert result.verdict == TurnVerdict.MissingReply assert result.failures == [MISSING_REPLY_FAILURE] assert result.messages == [] @@ -146,23 +136,23 @@ async def test_an_adapter_settled_turn_completes(framework_id: str) -> None: assert reported_failures(tools) == [] -@pytest.mark.parametrize("framework_id", probe_params(RELAYING_FRAMEWORK_IDS)) +@pytest.mark.parametrize("framework_id", relaying_probe_params()) async def test_a_relaying_adapter_posts_the_final_text_once(framework_id: str) -> None: result = await run_turn( TURN_OUTCOME_PROBES[framework_id], TurnScript(final_text=ANSWER) ) - assert result == TurnResult(Verdict.COMPLETE, messages=[ANSWER], failures=[]) + assert result == TurnResult(TurnVerdict.Complete, messages=[ANSWER], failures=[]) -@pytest.mark.parametrize("framework_id", probe_params(RELAYING_FRAMEWORK_IDS)) +@pytest.mark.parametrize("framework_id", relaying_probe_params()) async def test_a_tool_reply_suppresses_the_closing_text(framework_id: str) -> None: result = await run_turn( TURN_OUTCOME_PROBES[framework_id], TurnScript((REPLY,), final_text="Anything else?"), ) - assert result == TurnResult(Verdict.COMPLETE, messages=[ANSWER], failures=[]) + assert result == TurnResult(TurnVerdict.Complete, messages=[ANSWER], failures=[]) #: Registered adapters whose turns are not the model's to answer through Band tools. @@ -199,7 +189,9 @@ def band_adapters() -> dict[str, type[SimpleAdapter]]: continue try: importlib.import_module(module.name) - except ImportError: + except ModuleNotFoundError as missing: + if (missing.name or "").startswith("band"): + raise continue # an extra this venv lacks; its own lane covers it pending, adapters = [SimpleAdapter], {} while pending: diff --git a/tests/framework_conformance/turnprobes/__init__.py b/tests/framework_conformance/turnprobes/__init__.py index 45d4452d8..e116b5fe8 100644 --- a/tests/framework_conformance/turnprobes/__init__.py +++ b/tests/framework_conformance/turnprobes/__init__.py @@ -71,10 +71,12 @@ class TurnScript: class TurnOutcomeProbe: """``run`` drives one scripted turn. ``settle``, for an adapter with a control path, sends a message the adapter answers itself (a status or - busy reply) without running the model.""" + busy reply) without running the model. ``relays`` marks an adapter that + posts the model's final text when no tool replied.""" run: Callable[[TurnScript, FakeAgentTools], Awaitable[None]] settle: Callable[[FakeAgentTools], Awaitable[None]] | None = None + relays: bool = False class DispatchingFakeTools(FakeAgentTools): diff --git a/tests/framework_conformance/turnprobes/acp.py b/tests/framework_conformance/turnprobes/acp.py index 50d520c39..9c0f1314b 100644 --- a/tests/framework_conformance/turnprobes/acp.py +++ b/tests/framework_conformance/turnprobes/acp.py @@ -87,6 +87,8 @@ async def settle_cursor_acp(tools: FakeAgentTools) -> None: PROBES: dict[str, TurnOutcomeProbe] = { - "acp": TurnOutcomeProbe(run=run_acp), - "cursor_acp": TurnOutcomeProbe(run=run_cursor_acp, settle=settle_cursor_acp), + "acp": TurnOutcomeProbe(run=run_acp, relays=True), + "cursor_acp": TurnOutcomeProbe( + run=run_cursor_acp, settle=settle_cursor_acp, relays=True + ), } diff --git a/tests/framework_conformance/turnprobes/codex.py b/tests/framework_conformance/turnprobes/codex.py index cfc6e9a64..ebb3c9c91 100644 --- a/tests/framework_conformance/turnprobes/codex.py +++ b/tests/framework_conformance/turnprobes/codex.py @@ -48,5 +48,5 @@ async def ask_codex_status(tools: FakeAgentTools) -> None: PROBES: dict[str, TurnOutcomeProbe] = { - "codex": TurnOutcomeProbe(run=run_codex, settle=ask_codex_status), + "codex": TurnOutcomeProbe(run=run_codex, settle=ask_codex_status, relays=True), } diff --git a/tests/framework_conformance/turnprobes/copilotsdk.py b/tests/framework_conformance/turnprobes/copilotsdk.py index 6a7c8ef58..cf026291a 100644 --- a/tests/framework_conformance/turnprobes/copilotsdk.py +++ b/tests/framework_conformance/turnprobes/copilotsdk.py @@ -43,7 +43,7 @@ async def perform_tool_calls(session: FakeCopilotSession) -> None: PROBES: dict[str, TurnOutcomeProbe] = ( - {"copilot_sdk": TurnOutcomeProbe(run=run_copilot_sdk_turn)} + {"copilot_sdk": TurnOutcomeProbe(run=run_copilot_sdk_turn, relays=True)} if _COPILOT_SDK_AVAILABLE else {} ) diff --git a/tests/framework_conformance/turnprobes/letta.py b/tests/framework_conformance/turnprobes/letta.py index 87bb0da85..6124de75e 100644 --- a/tests/framework_conformance/turnprobes/letta.py +++ b/tests/framework_conformance/turnprobes/letta.py @@ -27,4 +27,6 @@ async def run_letta_turn(script: TurnScript, tools: FakeAgentTools) -> None: await adapter.on_event(turn_input(tools)) -PROBES: dict[str, TurnOutcomeProbe] = {"letta": TurnOutcomeProbe(run=run_letta_turn)} +PROBES: dict[str, TurnOutcomeProbe] = { + "letta": TurnOutcomeProbe(run=run_letta_turn, relays=True) +} diff --git a/tests/framework_conformance/turnprobes/opencode.py b/tests/framework_conformance/turnprobes/opencode.py index 0f10b375b..3429a9447 100644 --- a/tests/framework_conformance/turnprobes/opencode.py +++ b/tests/framework_conformance/turnprobes/opencode.py @@ -98,5 +98,5 @@ async def settle_opencode(tools: FakeAgentTools) -> None: PROBES: dict[str, TurnOutcomeProbe] = { - "opencode": TurnOutcomeProbe(run=run_opencode, settle=settle_opencode), + "opencode": TurnOutcomeProbe(run=run_opencode, settle=settle_opencode, relays=True), } diff --git a/tests/runtime/test_turn.py b/tests/runtime/test_turn.py index 6e6ab23bc..f86a1da30 100644 --- a/tests/runtime/test_turn.py +++ b/tests/runtime/test_turn.py @@ -16,7 +16,7 @@ from pydantic import BaseModel from band.core.delivery import deliver_reply -from band.core.turn import judge_detached_turn, report_unsettled_turn +from band.core.turn import Turn, judge_detached_turn, report_unsettled_turn from band.integrations.claude_sdk.dedup_tools import DedupingAgentTools from band.runtime.custom_tools import declares_turn_effect, execute_custom_tool from band.runtime.tools import AgentTools, BandTool, TurnEffect @@ -164,6 +164,16 @@ async def test_a_missing_reply_is_logged_for_its_room( assert record.levelno == logging.WARNING assert "room-1" in record.getMessage() + async def test_a_session_that_reports_no_failures_posts_nothing(self) -> None: + """A detached turn reports outside the runtime, so the opt-out is the + turn's own to honor.""" + tools = FakeAgentTools() + tools.turn = Turn(posts_missing_reply=False) + + assert await report_unsettled_turn(tools, room_id="room-1") is True + + assert reported_failures(tools) == [] + async def test_a_complete_turn_reports_nothing(self) -> None: tools = FakeAgentTools() await tools.no_reply() diff --git a/tests/runtime/test_turn_verdict.py b/tests/runtime/test_turn_verdict.py index 053acf39d..69e2ac879 100644 --- a/tests/runtime/test_turn_verdict.py +++ b/tests/runtime/test_turn_verdict.py @@ -11,6 +11,7 @@ import logging from collections.abc import Awaitable, Callable +from datetime import UTC, datetime from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -18,7 +19,11 @@ import pytest from band.core.delivery import relay_reply -from band.core.protocols import TURN_FAILURE_PROVIDER, AgentToolsProtocol +from band.core.protocols import ( + GENERIC_PROVIDER_FAILURE_MESSAGE, + TURN_FAILURE_PROVIDER, + AgentToolsProtocol, +) from band.core.simple_adapter import SimpleAdapter from band.core.types import ( SYNTHETIC_CONTACT_EVENTS_SENDER_ID, @@ -131,7 +136,12 @@ def link(mock_rest_client: MagicMock) -> MagicMock: return link -def run_through(adapter: SimpleAdapter[Any], link: MagicMock) -> ExecutionContext: +def run_through( + adapter: SimpleAdapter[Any], + link: MagicMock, + *, + config: SessionConfig | None = None, +) -> ExecutionContext: preprocessor = DefaultPreprocessor() async def handler(ctx: ExecutionContext, event: Any) -> None: @@ -143,11 +153,38 @@ async def handler(ctx: ExecutionContext, event: Any) -> None: ROOM_ID, link, handler, - config=SessionConfig(enable_context_hydration=False), + config=config or SessionConfig(enable_context_hydration=False), agent_id=AGENT_ID, ) +async def deliver(ctx: ExecutionContext, path: str) -> None: + """Deliver one user message live over the socket, or from the backlog.""" + match path: + case "live": + await ctx._process_event( + make_message_event(room_id=ROOM_ID, sender_id="user-1") + ) + case "backlog": + await ctx._process_backlog_message( + PlatformMessage( + id="msg-backlog", + room_id=ROOM_ID, + content="@agent hi", + sender_id="user-1", + sender_type="User", + sender_name="User One", + message_type="text", + metadata={}, + created_at=datetime.now(UTC), + ) + ) + + +async def crash(tools: AgentToolsProtocol) -> None: + raise RuntimeError("provider exploded") + + def runtime_failures(link: MagicMock) -> list[tuple[str, str]]: return [ ( @@ -181,20 +218,65 @@ async def test_each_turn_outcome_is_reported_honestly( assert runtime_failures(link) == [MISSING_REPLY_FAILURE] -async def test_a_missing_reply_is_not_logged_as_a_runtime_error( - link: MagicMock, caplog: pytest.LogCaptureFixture +@pytest.mark.parametrize("path", ["live", "backlog"]) +@pytest.mark.parametrize( + ("steps", "level", "traceback"), + [ + pytest.param([], logging.DEBUG, False, id="missing-reply"), + pytest.param([crash], logging.ERROR, True, id="crash"), + ], +) +async def test_only_an_unreported_failure_is_logged_as_a_runtime_error( + link: MagicMock, + caplog: pytest.LogCaptureFixture, + path: str, + steps: list[Step], + level: int, + traceback: bool, +) -> None: + """A missing reply was logged as a WARNING where it was reported, so the + runtime keeps it out of ERROR alerting; a crash still alerts with its + traceback.""" + ctx = run_through(ScriptedAdapter(steps), link) + + with caplog.at_level(logging.DEBUG, logger=execution_logger): + await deliver(ctx, path) + + (record,) = [r for r in caplog.records if r.message.startswith("Error processing")] + assert record.levelno == level + assert bool(record.exc_info) is traceback + + +async def test_a_missing_reply_that_failed_to_post_falls_back_to_the_runtime_report( + link: MagicMock, ) -> None: - """A silent turn is an expected verdict, already logged as a WARNING where - it was reported, so the runtime adds nothing at ERROR to trip alerting.""" + """The room still hears about the turn when the verdict's own post fails.""" + create = link.rest.agent_api_events.create_agent_chat_event + create.side_effect = [RuntimeError("transient 503"), create.return_value] ctx = run_through(ScriptedAdapter([]), link) - with caplog.at_level(logging.DEBUG): - await ctx._process_event( - make_message_event(room_id=ROOM_ID, sender_id="user-1") - ) + await deliver(ctx, "live") + + link.mark_failed.assert_awaited_once() + # Attempted posts, in order: the verdict's (which failed), then the fallback. + assert runtime_failures(link) == [ + MISSING_REPLY_FAILURE, + (TURN_FAILURE_PROVIDER, GENERIC_PROVIDER_FAILURE_MESSAGE), + ] - runtime_levels = {r.levelno for r in caplog.records if r.name == execution_logger} - assert logging.ERROR not in runtime_levels + +async def test_a_session_that_reports_no_failures_posts_no_missing_reply( + link: MagicMock, +) -> None: + config = SessionConfig( + enable_context_hydration=False, report_turn_failures_to_room=False + ) + ctx = run_through(ScriptedAdapter([]), link, config=config) + + await deliver(ctx, "live") + + link.mark_failed.assert_awaited_once() + assert runtime_failures(link) == [] async def test_an_exempt_adapter_is_never_judged(link: MagicMock) -> None: From f091057300ad291d09c4f8a843b31c023d35f6b0 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Mon, 5 Oct 2026 10:13:35 +0300 Subject: [PATCH 12/17] fix: report a turn whenever its own failure post did not land The runtime skipped its fallback report for any TurnResultAlreadyReported, but the adapters raise it right after send_failure without checking that the post landed (claude_sdk, codex, letta, a2a and the missing-reply verdict). It now trusts only the context's record of a posted failure. That covers every raise site with one rule, so the missing-reply-only unsettled_turn_error is gone. That also stops a Slack turn whose report failed from logging as a background crash. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016nwuEBtdtneuDkJHcvijDQ Signed-off-by: Alexander Zaikman --- src/band/core/protocols.py | 3 +- src/band/core/simple_adapter.py | 5 +- src/band/core/turn.py | 14 +----- src/band/runtime/execution.py | 6 +-- tests/integrations/slack/test_wrapping.py | 28 +++++++++++ tests/runtime/test_execution.py | 6 ++- tests/runtime/test_turn_verdict.py | 59 +++++++++++++++++------ 7 files changed, 84 insertions(+), 37 deletions(-) diff --git a/src/band/core/protocols.py b/src/band/core/protocols.py index aabb36348..602983521 100644 --- a/src/band/core/protocols.py +++ b/src/band/core/protocols.py @@ -68,7 +68,8 @@ class FailureMetadataKey(StrEnum): class TurnResultAlreadyReported(Exception): """A terminal turn failure that a nested handler already reported via ``send_failure``. An adapter's outer ``except`` re-raises this without - reporting the same failure a second time.""" + reporting the same failure a second time. The runtime reports the turn + anyway when that ``send_failure`` did not post.""" def to_failure_event(failure: AgentFailure) -> tuple[str, dict[str, Any]]: diff --git a/src/band/core/simple_adapter.py b/src/band/core/simple_adapter.py index 2f4ba6252..e72d49488 100644 --- a/src/band/core/simple_adapter.py +++ b/src/band/core/simple_adapter.py @@ -21,8 +21,9 @@ from band.core.protocols import ( AgentToolsProtocol, HistoryConverter, + TurnResultAlreadyReported, ) -from band.core.turn import report_unsettled_turn, unsettled_turn_error +from band.core.turn import report_unsettled_turn from band.core.types import ( USAGE_EVENT_TYPE, USAGE_METADATA_KEY, @@ -445,4 +446,4 @@ async def run_judged_turn( and not turn.detached and await report_unsettled_turn(tools, room_id=room_id) ): - raise unsettled_turn_error(turn) + raise TurnResultAlreadyReported("turn ended without a reply") diff --git a/src/band/core/turn.py b/src/band/core/turn.py index 29d86add7..ff98e1743 100644 --- a/src/band/core/turn.py +++ b/src/band/core/turn.py @@ -14,7 +14,7 @@ import band_sdk_core from band_sdk_core import AgentFailure, TurnLedger, TurnVerdict -from band.core.protocols import TURN_FAILURE_PROVIDER, TurnResultAlreadyReported +from band.core.protocols import TURN_FAILURE_PROVIDER if TYPE_CHECKING: from band.core.protocols import AgentToolsProtocol @@ -94,18 +94,6 @@ async def report_unsettled_turn(tools: AgentToolsProtocol, *, room_id: str) -> b return True -def unsettled_turn_error(turn: Turn) -> Exception: - """What a reported, unsettled turn raises to the runtime. - - ``TurnResultAlreadyReported`` when the room has the report or the session - posts none, so the runtime adds nothing. A report that failed to post - raises a plain error instead, so the runtime's own failure report runs. - """ - if turn.complete or not turn.posts_missing_reply: - return TurnResultAlreadyReported("turn ended without a reply") - return RuntimeError("turn ended without a reply, and its report did not post") - - async def judge_detached_turn(tools: AgentToolsProtocol, *, room_id: str) -> None: """Judge a judged, detached turn at the adapter's real end of it. diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index ac4ead8e8..a6234c13d 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -2031,9 +2031,9 @@ async def _handle_turn_failure( ) is_final = attempts is not None and attempts >= self._retry_tracker.max_retries - already_reported = self._turn_failure_reported or isinstance( - error, TurnResultAlreadyReported - ) + # Only a post that landed counts: an adapter raises + # TurnResultAlreadyReported even when its send_failure did not post. + already_reported = self._turn_failure_reported if not ( self.config.report_turn_failures_to_room and is_final diff --git a/tests/integrations/slack/test_wrapping.py b/tests/integrations/slack/test_wrapping.py index 051dcbf56..3ac1e3b6f 100644 --- a/tests/integrations/slack/test_wrapping.py +++ b/tests/integrations/slack/test_wrapping.py @@ -1496,6 +1496,34 @@ async def test_an_idle_slack_originated_turn_is_reported( assert max(r.levelno for r in caplog.records) == logging.WARNING +@pytest.mark.asyncio +async def test_a_slack_turn_whose_report_failed_is_not_logged_as_a_crash( + caplog: pytest.LogCaptureFixture, +): + """A Slack turn has no runtime to fall back to; the failed post is logged + where it failed, and the handler does not log the turn again as a crash.""" + adapter, _, _, rest = _make_adapter( + inner=IdleBrain(judges_turns=True), room_ids=["room-1"] + ) + await adapter.on_started("MyBot", "") + create = rest.agent_api_events.create_agent_chat_event + + async def fail_error_events(**kwargs: Any) -> Any: + if kwargs["event"].message_type == MessageType.ERROR: + raise RuntimeError("503") + return create.return_value + + create.side_effect = fail_error_events + + with caplog.at_level(logging.WARNING): + await _post_slack_event(adapter, adapter.config.apps[0], _mention_event()) + await adapter.wait_idle() + + assert [r.getMessage() for r in caplog.records if r.levelno >= logging.ERROR] == [ + "send_failure could not post the failure event" + ] + + @pytest.mark.asyncio async def test_on_event_rehydrates_room_binding_on_bootstrap(): """First WS-delivered message in a previously-bridged room restores state.""" diff --git a/tests/runtime/test_execution.py b/tests/runtime/test_execution.py index 731d14c1c..0f2ec070c 100644 --- a/tests/runtime/test_execution.py +++ b/tests/runtime/test_execution.py @@ -2041,9 +2041,11 @@ async def reporting_handler(ctx, event): await ctx._process_backlog_message(self._backlog_message("msg-unreported")) assert self._reporters(error_events) == ["codex", TURN_FAILURE_PROVIDER] - async def test_turn_result_already_reported_is_not_repeated( + async def test_an_already_reported_claim_without_a_post_is_still_reported( self, mock_link, error_events ): + """Only a failure post that landed suppresses the runtime's report.""" + async def handler(ctx, event): raise TurnResultAlreadyReported("reported by a nested handler") @@ -2052,7 +2054,7 @@ async def handler(ctx, event): ) mock_link.mark_failed.assert_awaited_once() - error_events.assert_not_awaited() + assert self._reporters(error_events) == [TURN_FAILURE_PROVIDER] async def test_report_post_failure_is_swallowed(self, mock_link, error_events): """The turn is already marked failed; a failed report must not stop diff --git a/tests/runtime/test_turn_verdict.py b/tests/runtime/test_turn_verdict.py index 69e2ac879..13f6d8433 100644 --- a/tests/runtime/test_turn_verdict.py +++ b/tests/runtime/test_turn_verdict.py @@ -23,6 +23,7 @@ GENERIC_PROVIDER_FAILURE_MESSAGE, TURN_FAILURE_PROVIDER, AgentToolsProtocol, + TurnResultAlreadyReported, ) from band.core.simple_adapter import SimpleAdapter from band.core.types import ( @@ -185,20 +186,31 @@ async def crash(tools: AgentToolsProtocol) -> None: raise RuntimeError("provider exploded") -def runtime_failures(link: MagicMock) -> list[tuple[str, str]]: +ADAPTER_FAILURE = ("scripted", "Provider is down") + + +async def report_and_stop(tools: AgentToolsProtocol) -> None: + """An adapter's own failure path: report it, then end the turn.""" + await tools.send_failure(band_sdk_core.AgentFailure(*ADAPTER_FAILURE)) + raise TurnResultAlreadyReported(ADAPTER_FAILURE[1]) + + +def failure_posts(link: MagicMock) -> list[tuple[str, str]]: + """Every attempted failure post, as (provider, text), in order.""" return [ ( call.kwargs["event"].metadata["failure"]["provider"], call.kwargs["event"].content, ) for call in link.rest.agent_api_events.create_agent_chat_event.call_args_list - if call.kwargs["event"].metadata - and "failure" in call.kwargs["event"].metadata - and call.kwargs["event"].metadata["failure"]["provider"] - == TURN_FAILURE_PROVIDER + if call.kwargs["event"].metadata and "failure" in call.kwargs["event"].metadata ] +def runtime_failures(link: MagicMock) -> list[tuple[str, str]]: + return [post for post in failure_posts(link) if post[0] == TURN_FAILURE_PROVIDER] + + @pytest.mark.parametrize("row", sorted(ROWS)) async def test_each_turn_outcome_is_reported_honestly( row: str, link: MagicMock @@ -247,22 +259,37 @@ async def test_only_an_unreported_failure_is_logged_as_a_runtime_error( assert bool(record.exc_info) is traceback -async def test_a_missing_reply_that_failed_to_post_falls_back_to_the_runtime_report( +GENERIC_FAILURE = (TURN_FAILURE_PROVIDER, GENERIC_PROVIDER_FAILURE_MESSAGE) + + +@pytest.mark.parametrize("path", ["live", "backlog"]) +@pytest.mark.parametrize("report_posts", [True, False], ids=["posted", "post-failed"]) +@pytest.mark.parametrize( + ("steps", "report"), + [ + pytest.param([], MISSING_REPLY_FAILURE, id="missing-reply"), + pytest.param([report_and_stop], ADAPTER_FAILURE, id="adapter-report"), + ], +) +async def test_the_runtime_reports_a_turn_only_when_its_report_did_not_post( link: MagicMock, + path: str, + steps: list[Step], + report: tuple[str, str], + report_posts: bool, ) -> None: - """The room still hears about the turn when the verdict's own post fails.""" - create = link.rest.agent_api_events.create_agent_chat_event - create.side_effect = [RuntimeError("transient 503"), create.return_value] - ctx = run_through(ScriptedAdapter([]), link) + """Raising ``TurnResultAlreadyReported`` is not enough: the runtime's + fallback runs unless the turn's own report actually reached the room.""" + if not report_posts: + create = link.rest.agent_api_events.create_agent_chat_event + create.side_effect = [RuntimeError("transient 503"), create.return_value] + ctx = run_through(ScriptedAdapter(steps), link) - await deliver(ctx, "live") + await deliver(ctx, path) link.mark_failed.assert_awaited_once() - # Attempted posts, in order: the verdict's (which failed), then the fallback. - assert runtime_failures(link) == [ - MISSING_REPLY_FAILURE, - (TURN_FAILURE_PROVIDER, GENERIC_PROVIDER_FAILURE_MESSAGE), - ] + fallback = [] if report_posts else [GENERIC_FAILURE] + assert failure_posts(link) == [report, *fallback] async def test_a_session_that_reports_no_failures_posts_no_missing_reply( From 37cef255cc425e79af40fe0f3a1c5ebf643445d9 Mon Sep 17 00:00:00 2001 From: Natan Zalkin Date: Mon, 5 Oct 2026 14:40:47 +0300 Subject: [PATCH 13/17] fix: preserve successful turn effects and complete adapter replies --- docs/turn-outcome.md | 15 ++ src/band/adapters/letta.py | 27 ++- src/band/adapters/parlant.py | 29 ++-- src/band/adapters/pydantic_ai.py | 9 +- src/band/adapters/strands.py | 25 ++- src/band/runtime/custom_tools.py | 3 +- src/band/runtime/tools/schema.py | 11 +- tests/adapters/test_letta_adapter.py | 181 ++++++++++++++++++++- tests/adapters/test_parlant_adapter.py | 82 ++++++++++ tests/adapters/test_pydantic_ai_adapter.py | 21 +++ tests/adapters/test_strands_adapter.py | 27 +++ tests/runtime/test_turn.py | 51 +++++- 12 files changed, 455 insertions(+), 26 deletions(-) diff --git a/docs/turn-outcome.md b/docs/turn-outcome.md index d62e59b23..4ac20cdf6 100644 --- a/docs/turn-outcome.md +++ b/docs/turn-outcome.md @@ -164,6 +164,13 @@ asyncio.run(main()) whose silence is the answer; both also stop a relaying adapter from posting the model's final text. +A declared effect is recorded only on success. A handler that raises or returns +an explicit failure (`{"ok": False}`, or a string starting with `Error:` or +`Error executing `, case-insensitively) records nothing: the turn still owes an +answer, and a failed reply or decline does not suppress the final-text relay. +Other return values, including `None` from a side-effect-only handler, count as +successful. + **Limits:** a custom tool only records on the turn it is handed. Where a tool runs with `turn=None` (not bound to a room, as in a standalone MCP server), it cannot complete any turn. A framework-native tool (rather than an @@ -208,6 +215,14 @@ toward: the base prompt still says plain text is never delivered, because without that line models more often send their closing narration as a second `band_send_message` (measured live on gemini-2.5-flash). +Parlant joins the non-preamble final segments from one event batch before +relaying, so recording the fallback reply cannot suppress a later segment of +that same answer. + +For external Letta MCP servers, each successful grouped tool return is matched +to its call by `tool_call_id` and records that core tool's turn effect. Failed +returns record nothing; self-hosted tools record their own effects. + `tests/framework_conformance/test_reply_boundary.py` pins every `send_message`, `deliver_reply` and `relay_reply` call outside the tool implementations, per file with the reason it carries the model's words, so a new one fails until it diff --git a/src/band/adapters/letta.py b/src/band/adapters/letta.py index f5a83c0af..20cd21f73 100644 --- a/src/band/adapters/letta.py +++ b/src/band/adapters/letta.py @@ -560,6 +560,7 @@ async def _process_response_messages( Returns the list of assistant text parts collected during the turn. """ final_text_parts: list[str] = [] + tool_names_by_id: dict[str, str] = {} for resp_msg in response_messages: match getattr(resp_msg, "message_type", None): case "assistant_message": @@ -568,6 +569,11 @@ async def _process_response_messages( case "tool_call_message": # MCP tool call executed server-side — observe only tool_call = getattr(resp_msg, "tool_call", None) + tool_calls = getattr(resp_msg, "tool_calls", None) + if isinstance(tool_calls, list): + tool_names_by_id.update( + (call.tool_call_id, call.name) for call in tool_calls + ) tool_name = ( getattr(tool_call, "name", "unknown") if tool_call @@ -600,7 +606,9 @@ async def _process_response_messages( ) case "tool_return_message": tool_name = getattr(resp_msg, "name", None) or "unknown" - self._record_external_tool_return(tools, tool_name, resp_msg) + self._record_external_tool_return( + tools, tool_name, resp_msg, tool_names_by_id + ) await self._report_execution_event( tools, "tool_result", @@ -644,14 +652,27 @@ async def _process_response_messages( return final_text_parts def _record_external_tool_return( - self, tools: AgentToolsProtocol, tool_name: str, resp_msg: Any + self, + tools: AgentToolsProtocol, + tool_name: str, + resp_msg: Any, + tool_names_by_id: dict[str, str], ) -> None: """Record an external band-mcp call's effect on the turn. A self-hosted call runs on this room's own tools, which record it themselves; an external server's call is visible only in the stream. """ - if self.config.mcp.mode == "external" and resp_msg.status == "success": + if self.config.mcp.mode != "external": + return + tool_returns = getattr(resp_msg, "tool_returns", None) + if isinstance(tool_returns, list): + # Letta's legacy name/status describe only the first parallel call. + for result in tool_returns: + if result.status == "success": + name = tool_names_by_id.get(result.tool_call_id, "unknown") + tools.turn.record(turn_effect(name)) + elif resp_msg.status == "success": tools.turn.record(turn_effect(tool_name)) async def _report_execution_event( diff --git a/src/band/adapters/parlant.py b/src/band/adapters/parlant.py index 972e14559..b8a5fb3e5 100644 --- a/src/band/adapters/parlant.py +++ b/src/band/adapters/parlant.py @@ -656,6 +656,8 @@ async def _process_agent_response( If a Band tool already replied or declined this turn, Parlant's response is not forwarded (it would be a duplicate or empty). + Final messages in the same event batch are joined into one fallback reply; + preambles are excluded. Waiting is bounded by a total budget, polling in shorter windows and retrying on an empty window so a slow (cold-start) turn is still answered. @@ -765,6 +767,7 @@ async def _process_agent_response( # Process events and track if we got a non-preamble message got_final_message = False + final_segments: list[str] = [] for event in events: logger.debug( @@ -806,9 +809,9 @@ async def _process_agent_response( ) continue - # The turn already replied or declined (a Band tool, or an - # earlier relay), so this response would duplicate it. Not - # final either: Parlant may still have more tool calls. + # A Band tool already replied or declined, so forwarding this + # response would duplicate it. Parlant may still have more + # tool calls. if tools.turn.replied: logger.debug( "Room %s: Turn already replied, skipping Parlant response: %s...", @@ -821,21 +824,23 @@ async def _process_agent_response( got_final_message = True if message_content: - logger.debug( - "Room %s: Sending agent response to platform: %s...", - room_id, - message_content[:100], - ) - if await relay_reply( - tools, message_content, mentions=[sender_name] - ): - logger.info("Room %s: Message sent successfully", room_id) + final_segments.append(message_content) else: logger.warning( "Room %s: Empty message content in event", room_id, ) + if final_segments: + message_content = "\n\n".join(final_segments) + logger.debug( + "Room %s: Sending agent response to platform: %s...", + room_id, + message_content[:100], + ) + if await relay_reply(tools, message_content, mentions=[sender_name]): + logger.info("Room %s: Message sent successfully", room_id) + # If we got a final (non-preamble) message, we're done if got_final_message: logger.debug("Room %s: Got final message, processing complete", room_id) diff --git a/src/band/adapters/pydantic_ai.py b/src/band/adapters/pydantic_ai.py index 049f0c386..35ded9b9c 100644 --- a/src/band/adapters/pydantic_ai.py +++ b/src/band/adapters/pydantic_ai.py @@ -69,6 +69,7 @@ image_block_placeholder, redact_tool_call_args, ) +from band.runtime.tools.schema import is_failed_tool_output logger = logging.getLogger(__name__) @@ -502,8 +503,12 @@ async def on_message( elif isinstance(event, FunctionToolResultEvent): # Native custom tools run outside execute_custom_tool, so # their declared effect is recorded here. - if isinstance(event.part, ToolReturnPart) and ( - effect := self._custom_effects.get(event.part.tool_name) + if ( + isinstance(event.part, ToolReturnPart) + and not is_failed_tool_output(event.part.content) + and ( + effect := self._custom_effects.get(event.part.tool_name) + ) ): tools.turn.record(effect) if Emit.TOOL_CALLS in self.features.emit: diff --git a/src/band/adapters/strands.py b/src/band/adapters/strands.py index a472e794d..6aef518d2 100644 --- a/src/band/adapters/strands.py +++ b/src/band/adapters/strands.py @@ -73,6 +73,7 @@ serialize_tool_result, validate_tool_arguments, ) +from band.runtime.tools.schema import is_failed_tool_output logger = logging.getLogger(__name__) @@ -151,6 +152,24 @@ def _result_text(result: ToolResult) -> str: return "\n".join(parts) +def _custom_tool_failed(result: ToolResult) -> bool: + for block in result.get("content", []): + match block: + case {"json": value}: + pass + case {"text": str() as text}: + # Native Strands tools serialize ordinary dict returns as text. + try: + value = json.loads(text) + except ValueError: + value = text + case _: + continue + if is_failed_tool_output(value): + return True + return False + + def _openai_history(messages: StrandsMessages) -> StrandsMessages: """Keep tool results ahead of text in Strands' OpenAI serialization.""" normalized: StrandsMessages = [] @@ -379,7 +398,11 @@ async def _on_after_tool(self, event: AfterToolCallEvent) -> None: ) # Native Strands tools run outside execute_custom_tool, so their # declared effect is recorded here. - if succeeded and (effect := self._custom_effects.get(name)): + if ( + succeeded + and (effect := self._custom_effects.get(name)) + and not _custom_tool_failed(event.result) + ): self._tools.turn.record(effect) if not self._emit_execution: return diff --git a/src/band/runtime/custom_tools.py b/src/band/runtime/custom_tools.py index ddccedd9b..0f0e9257e 100644 --- a/src/band/runtime/custom_tools.py +++ b/src/band/runtime/custom_tools.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, ValidationError from band.core.turn import Turn +from band.runtime.tools.schema import is_failed_tool_output from band.runtime.tools.types import TurnEffect logger = logging.getLogger(__name__) @@ -263,6 +264,6 @@ async def invoke_validated_custom_tool( result = func(*args) if inspect.isawaitable(result): result = await result - if turn is not None: + if turn is not None and not is_failed_tool_output(result): turn.record(declared_effect(func) or TurnEffect.OBSERVE) return result diff --git a/src/band/runtime/tools/schema.py b/src/band/runtime/tools/schema.py index dae8d81ee..921b39d71 100644 --- a/src/band/runtime/tools/schema.py +++ b/src/band/runtime/tools/schema.py @@ -7,7 +7,7 @@ from __future__ import annotations import warnings -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from functools import wraps from typing import Any, TypeVar, cast @@ -205,6 +205,15 @@ class ToolCallOutcome: error_message: str | None = None +def is_failed_tool_output(output: Any) -> bool: + """Recognize structured ``ok=False`` results and legacy error strings.""" + if isinstance(output, Mapping): + return output.get("ok") is False + if isinstance(output, str): + return output.lower().startswith(("error:", "error executing ")) + return False + + def serialize_tool_result(result: Any) -> Any: """Serialize Pydantic tool results to dicts at the adapter boundary. diff --git a/tests/adapters/test_letta_adapter.py b/tests/adapters/test_letta_adapter.py index fb987708e..b64adc372 100644 --- a/tests/adapters/test_letta_adapter.py +++ b/tests/adapters/test_letta_adapter.py @@ -14,6 +14,10 @@ from unittest.mock import AsyncMock, MagicMock import pytest +from letta_client.types.agents.tool_call import ToolCall +from letta_client.types.agents.tool_call_message import ToolCallMessage +from letta_client.types.agents.tool_return import ToolReturn +from letta_client.types.tool_return_message import ToolReturnMessage from band.adapters.letta import ( LettaAdapter, @@ -1420,6 +1424,48 @@ async def test_relay_detection_uses_resolved_name(self) -> None: # ────────────────────────────────────────────────────────────────────── +def make_parallel_tool_messages( + results: list[tuple[str, Literal["success", "error"]]], + *, + reverse_returns: bool, +) -> tuple[ToolCallMessage, ToolReturnMessage]: + calls = [ + ToolCall(name=name, arguments="{}", tool_call_id=f"call-{index}") + for index, (name, _status) in enumerate(results) + ] + returns = [ + ToolReturn( + type="tool", + tool_return="result", + status=status, + tool_call_id=call.tool_call_id, + ) + for call, (_name, status) in zip(calls, results, strict=True) + ] + first_call, first_return = calls[0], returns[0] + if reverse_returns: + returns.reverse() + return ( + ToolCallMessage( + id="message-calls", + date=datetime.now(UTC), + message_type="tool_call_message", + tool_call=first_call, + tool_calls=calls, + ), + ToolReturnMessage( + id="message-returns", + date=datetime.now(UTC), + message_type="tool_return_message", + name=first_call.name, + status=first_return.status, + tool_call_id=first_call.tool_call_id, + tool_return="result", + tool_returns=returns, + ), + ) + + class TestExternalToolRecording: @pytest.mark.asyncio @pytest.mark.parametrize( @@ -1432,13 +1478,24 @@ class TestExternalToolRecording: ], ) async def test_streamed_tool_return_records_only_for_external_band_mcp( - self, mcp_mode: Literal["self_host", "external"], status: str, replied: bool + self, + mcp_mode: Literal["self_host", "external"], + status: Literal["success", "error"], + replied: bool, ) -> None: adapter, mock_client = ready_letta_adapter( LettaAdapterConfig(mcp=LettaMCPConfig(mode=mcp_mode)) ) mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_return_message(BandTool.SEND_MESSAGE, status=status), + ToolReturnMessage( + id="message-return", + date=datetime.now(UTC), + message_type="tool_return_message", + name=BandTool.SEND_MESSAGE, + status=status, + tool_call_id="call-1", + tool_return="result", + ), ) tools = FakeAgentTools() @@ -1454,6 +1511,126 @@ async def test_streamed_tool_return_records_only_for_external_band_mcp( assert tools.turn.replied is replied + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("effect_tool", "replied"), + [ + pytest.param(BandTool.ADD_PARTICIPANT, False, id="act"), + pytest.param(BandTool.SEND_MESSAGE, True, id="reply"), + pytest.param(BandTool.NO_REPLY, True, id="decline"), + ], + ) + @pytest.mark.parametrize("reverse_calls", [False, True]) + async def test_parallel_success_completes_turn_independent_of_order( + self, + effect_tool: str, + replied: bool, + reverse_calls: bool, + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), + room_id=ROOM_ID, + ) + results: list[tuple[str, Literal["success", "error"]]] = [ + (BandTool.GET_PARTICIPANTS, "success"), + (effect_tool, "success"), + ] + if reverse_calls: + results.reverse() + mock_client.agents.messages.create.return_value = make_letta_response( + *make_parallel_tool_messages(results, reverse_returns=False) + ) + tools = turn_tools() + + await adapter.on_event(turn_input(tools)) + + assert tools.turn.complete + assert tools.turn.replied is replied + assert tools.messages_sent == [] + assert failure_reports(tools) == [] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "effect_tool", + [BandTool.ADD_PARTICIPANT, BandTool.SEND_MESSAGE, BandTool.NO_REPLY], + ) + @pytest.mark.parametrize("reverse_returns", [False, True]) + async def test_parallel_failed_effect_does_not_complete_turn( + self, effect_tool: str, reverse_returns: bool + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), + room_id=ROOM_ID, + ) + mock_client.agents.messages.create.return_value = make_letta_response( + *make_parallel_tool_messages( + [ + (BandTool.GET_PARTICIPANTS, "success"), + (effect_tool, "error"), + ], + reverse_returns=reverse_returns, + ) + ) + tools = turn_tools() + + with pytest.raises(TurnResultAlreadyReported): + await adapter.on_event(turn_input(tools)) + + assert not tools.turn.replied + assert tools.messages_sent == [] + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + + @pytest.mark.asyncio + @pytest.mark.parametrize("effect_tool", [BandTool.SEND_MESSAGE, BandTool.NO_REPLY]) + @pytest.mark.parametrize("reverse_returns", [False, True]) + async def test_parallel_failed_reply_does_not_suppress_fallback( + self, effect_tool: str, reverse_returns: bool + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")), + room_id=ROOM_ID, + ) + mock_client.agents.messages.create.return_value = make_letta_response( + *make_parallel_tool_messages( + [(effect_tool, "error"), (BandTool.GET_PARTICIPANTS, "success")], + reverse_returns=reverse_returns, + ), + make_assistant_message("Fallback reply"), + ) + tools = turn_tools() + + await adapter.on_event(turn_input(tools)) + + assert sent_contents(tools) == ["Fallback reply"] + assert failure_reports(tools) == [] + + @pytest.mark.asyncio + @pytest.mark.parametrize("reverse_returns", [False, True]) + async def test_parallel_results_match_call_ids_not_positions( + self, reverse_returns: bool + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), + room_id=ROOM_ID, + ) + mock_client.agents.messages.create.return_value = make_letta_response( + *make_parallel_tool_messages( + [ + (BandTool.SEND_MESSAGE, "error"), + (BandTool.ADD_PARTICIPANT, "success"), + ], + reverse_returns=reverse_returns, + ) + ) + tools = turn_tools() + + await adapter.on_event(turn_input(tools)) + + assert tools.turn.complete + assert not tools.turn.replied + assert tools.messages_sent == [] + assert failure_reports(tools) == [] + # ────────────────────────────────────────────────────────────────────── # Auto-relay knob diff --git a/tests/adapters/test_parlant_adapter.py b/tests/adapters/test_parlant_adapter.py index af48e6ead..b477845f9 100644 --- a/tests/adapters/test_parlant_adapter.py +++ b/tests/adapters/test_parlant_adapter.py @@ -1085,6 +1085,88 @@ async def test_retries_past_empty_windows_then_forwards_late_reply( content="Hello there!", mentions=["Alice"], count=1 ) + @pytest.mark.asyncio + async def test_relays_all_final_parts_in_one_batch_without_preamble( + self, + mock_parlant_server: MagicMock, + mock_parlant_agent: MagicMock, + mock_tools: FakeAgentTools, + ) -> None: + adapter = ParlantAdapter( + server=mock_parlant_server, parlant_agent=mock_parlant_agent + ) + adapter._app = self._app_with_waits( + wait_results=[True], + events=[ + self._agent_event("One moment…", offset=1, tags=[PARLANT_PREAMBLE_TAG]), + self._agent_event("Your table has been booked!", offset=2), + self._agent_event( + "Please note that our kitchen contains peanuts.", offset=3 + ), + ], + ) + + await adapter._process_agent_response( + session_id="multipart-session", + room_id="room-1", + min_offset=0, + tools=mock_tools, + sender_name="Alice", + ) + + mock_tools.assert_message_sent( + content=( + "Your table has been booked!\n\n" + "Please note that our kitchen contains peanuts." + ), + mentions=["Alice"], + count=1, + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "declined", [False, True], ids=["tool-reply", "tool-decline"] + ) + async def test_tool_reply_or_decline_suppresses_entire_final_batch( + self, + mock_parlant_server: MagicMock, + mock_parlant_agent: MagicMock, + mock_tools: FakeAgentTools, + declined: bool, + ) -> None: + if declined: + await mock_tools.no_reply("No response needed.") + else: + await mock_tools.send_message("Already answered.", mentions=["Alice"]) + adapter = ParlantAdapter( + server=mock_parlant_server, parlant_agent=mock_parlant_agent + ) + adapter._app = self._app_with_waits( + wait_results=[True], + events=[ + self._agent_event("One moment…", offset=1, tags=[PARLANT_PREAMBLE_TAG]), + self._agent_event("Your table has been booked!", offset=2), + self._agent_event( + "Please note that our kitchen contains peanuts.", offset=3 + ), + ], + ) + + await adapter._process_agent_response( + session_id="tool-completed-session", + room_id="room-1", + min_offset=0, + tools=mock_tools, + sender_name="Alice", + ) + + if declined: + mock_tools.assert_no_messages_sent() + else: + mock_tools.assert_message_sent( + content="Already answered.", mentions=["Alice"], count=1 + ) + @pytest.mark.asyncio async def test_gives_up_after_budget_when_no_reply_ever_arrives( self, mock_parlant_server, mock_parlant_agent, mock_tools diff --git a/tests/adapters/test_pydantic_ai_adapter.py b/tests/adapters/test_pydantic_ai_adapter.py index 28f0ac78a..346b8c2b5 100644 --- a/tests/adapters/test_pydantic_ai_adapter.py +++ b/tests/adapters/test_pydantic_ai_adapter.py @@ -2288,6 +2288,27 @@ async def finish(ctx: RunContext[AgentToolsProtocol], note: str) -> str: assert failure_reports(tools) == reported + @pytest.mark.asyncio + async def test_a_native_tool_failure_value_leaves_the_turn_unanswered(self) -> None: + @declares_turn_effect(TurnEffect.ACT) + async def finish( + ctx: RunContext[AgentToolsProtocol], note: str + ) -> dict[str, Any]: + """Finish the task.""" + return {"ok": False, "error": "upstream refused"} + + adapter = PydanticAIAdapter( + PydanticAIAdapterConfig(model="test"), additional_tools=[finish] + ) + await adapter.on_started("Probe", "probe") + adapter._agent.model = _streamed_tool_calls(("finish", {"note": "go"})) + tools = FakeAgentTools(room_id=ROOM_ID) + + with pytest.raises(TurnResultAlreadyReported): + await adapter.on_event(turn_input(tools)) + + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + def test_accepts_additional_tools_parameter(self): """Adapter accepts list of callables.""" diff --git a/tests/adapters/test_strands_adapter.py b/tests/adapters/test_strands_adapter.py index bf252349d..3a1b35cc9 100644 --- a/tests/adapters/test_strands_adapter.py +++ b/tests/adapters/test_strands_adapter.py @@ -645,6 +645,33 @@ def native_finish(note: str) -> str: assert failure_reports(tools) == [] + @pytest.mark.asyncio + @pytest.mark.parametrize( + "output", + [ + {"ok": False, "error": "upstream refused"}, + "Error: upstream refused", + "Error executing ticket: upstream refused", + ], + ) + async def test_a_native_tool_failure_value_leaves_the_turn_unanswered( + self, tools, scripted, output: Any + ) -> None: + @declares_turn_effect(TurnEffect.ACT) + @strands_tool + def native_finish(note: str) -> Any: + """Finish the task natively.""" + return output + + adapter = await scripted( + ToolTurn("native_finish", {"note": "go"}), additional_tools=[native_finish] + ) + + with pytest.raises(TurnResultAlreadyReported): + await _run_turn(adapter, tools) + + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + @pytest.mark.asyncio async def test_undeclared_native_tool_only_observes(self, tools, scripted): @strands_tool diff --git a/tests/runtime/test_turn.py b/tests/runtime/test_turn.py index f86a1da30..0a265a31a 100644 --- a/tests/runtime/test_turn.py +++ b/tests/runtime/test_turn.py @@ -15,7 +15,7 @@ import pytest from pydantic import BaseModel -from band.core.delivery import deliver_reply +from band.core.delivery import deliver_reply, relay_reply from band.core.turn import Turn, judge_detached_turn, report_unsettled_turn from band.integrations.claude_sdk.dedup_tools import DedupingAgentTools from band.runtime.custom_tools import declares_turn_effect, execute_custom_tool @@ -98,10 +98,11 @@ async def test_narration_leaves_the_reply_owed(self, tools: AgentTools) -> None: class TestCustomTools: - async def test_a_declared_tool_records_its_effect(self) -> None: + @pytest.mark.parametrize("output", ["filed", None]) + async def test_a_declared_tool_records_its_effect(self, output: str | None) -> None: @declares_turn_effect(TurnEffect.ACT) - async def post_to_ticketing(args: LookupInput) -> str: - return "filed" + async def post_to_ticketing(args: LookupInput) -> str | None: + return output tools = FakeAgentTools() await execute_custom_tool( @@ -134,6 +135,48 @@ async def broken(args: LookupInput) -> str: assert not tools.turn.complete + @pytest.mark.parametrize( + "effect", [TurnEffect.ACT, TurnEffect.REPLY, TurnEffect.DECLINE] + ) + @pytest.mark.parametrize( + "output", + [ + {"ok": False, "error": "upstream refused"}, + "Error: upstream refused", + "Error executing ticket: upstream refused", + ], + ) + async def test_a_failure_value_does_not_complete_the_turn( + self, effect: TurnEffect, output: Any + ) -> None: + @declares_turn_effect(effect) + async def refused(args: LookupInput) -> Any: + return output + + tools = FakeAgentTools() + await execute_custom_tool( + (LookupInput, refused), {"topic": "x"}, turn=tools.turn + ) + + assert await report_unsettled_turn(tools, room_id="room-1") + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + + @pytest.mark.parametrize("effect", [TurnEffect.REPLY, TurnEffect.DECLINE]) + async def test_a_failure_value_does_not_suppress_the_fallback( + self, effect: TurnEffect + ) -> None: + @declares_turn_effect(effect) + async def refused(args: LookupInput) -> dict[str, Any]: + return {"ok": False, "error": "upstream refused"} + + tools = FakeAgentTools() + await execute_custom_tool( + (LookupInput, refused), {"topic": "x"}, turn=tools.turn + ) + await relay_reply(tools, "The operation failed.", ["@alice"]) + + assert tools.chat == ["The operation failed."] + async def test_the_dedup_wrapper_shares_the_inner_ledger() -> None: inner = FakeAgentTools() From 7eb311c173d735ee0754160a07b4d8938ed2c980 Mon Sep 17 00:00:00 2001 From: Natan Zalkin Date: Mon, 5 Oct 2026 15:13:33 +0300 Subject: [PATCH 14/17] refactor: judge Letta tool effects from complete responses --- docs/turn-outcome.md | 8 ++-- src/band/adapters/letta.py | 57 +++++++++++++++------------- tests/adapters/test_letta_adapter.py | 26 +++++++++++++ 3 files changed, 61 insertions(+), 30 deletions(-) diff --git a/docs/turn-outcome.md b/docs/turn-outcome.md index 4ac20cdf6..d1acf59a3 100644 --- a/docs/turn-outcome.md +++ b/docs/turn-outcome.md @@ -219,9 +219,11 @@ Parlant joins the non-preamble final segments from one event batch before relaying, so recording the fallback reply cannot suppress a later segment of that same answer. -For external Letta MCP servers, each successful grouped tool return is matched -to its call by `tool_call_id` and records that core tool's turn effect. Failed -returns record nothing; self-hosted tools record their own effects. +For external Letta MCP servers, the SDK examines the fully buffered response: +it collects successful grouped return IDs, then records the core turn effect +of each named call whose `tool_call_id` succeeded. Message order does not +determine the outcome. Failed returns record nothing; self-hosted tools record +their own effects. `tests/framework_conformance/test_reply_boundary.py` pins every `send_message`, `deliver_reply` and `relay_reply` call outside the tool implementations, per diff --git a/src/band/adapters/letta.py b/src/band/adapters/letta.py index 20cd21f73..21448206f 100644 --- a/src/band/adapters/letta.py +++ b/src/band/adapters/letta.py @@ -559,8 +559,8 @@ async def _process_response_messages( Returns the list of assistant text parts collected during the turn. """ + self._record_external_tool_effects(tools, response_messages) final_text_parts: list[str] = [] - tool_names_by_id: dict[str, str] = {} for resp_msg in response_messages: match getattr(resp_msg, "message_type", None): case "assistant_message": @@ -569,11 +569,6 @@ async def _process_response_messages( case "tool_call_message": # MCP tool call executed server-side — observe only tool_call = getattr(resp_msg, "tool_call", None) - tool_calls = getattr(resp_msg, "tool_calls", None) - if isinstance(tool_calls, list): - tool_names_by_id.update( - (call.tool_call_id, call.name) for call in tool_calls - ) tool_name = ( getattr(tool_call, "name", "unknown") if tool_call @@ -606,9 +601,6 @@ async def _process_response_messages( ) case "tool_return_message": tool_name = getattr(resp_msg, "name", None) or "unknown" - self._record_external_tool_return( - tools, tool_name, resp_msg, tool_names_by_id - ) await self._report_execution_event( tools, "tool_result", @@ -651,29 +643,40 @@ async def _process_response_messages( return final_text_parts - def _record_external_tool_return( - self, - tools: AgentToolsProtocol, - tool_name: str, - resp_msg: Any, - tool_names_by_id: dict[str, str], + def _record_external_tool_effects( + self, tools: AgentToolsProtocol, response_messages: list[Any] ) -> None: - """Record an external band-mcp call's effect on the turn. + """Record successful external calls from the fully buffered response. - A self-hosted call runs on this room's own tools, which record it - themselves; an external server's call is visible only in the stream. + Self-hosted tools record their own effects during execution. """ if self.config.mcp.mode != "external": return - tool_returns = getattr(resp_msg, "tool_returns", None) - if isinstance(tool_returns, list): - # Letta's legacy name/status describe only the first parallel call. - for result in tool_returns: - if result.status == "success": - name = tool_names_by_id.get(result.tool_call_id, "unknown") - tools.turn.record(turn_effect(name)) - elif resp_msg.status == "success": - tools.turn.record(turn_effect(tool_name)) + + successful_call_ids: set[str] = set() + for message in response_messages: + if getattr(message, "message_type", None) != "tool_return_message": + continue + tool_returns = getattr(message, "tool_returns", None) + if isinstance(tool_returns, list): + successful_call_ids.update( + result.tool_call_id + for result in tool_returns + if result.status == "success" + ) + elif message.status == "success": + tools.turn.record( + turn_effect(getattr(message, "name", None) or "unknown") + ) + + for message in response_messages: + if getattr(message, "message_type", None) != "tool_call_message": + continue + tool_calls = getattr(message, "tool_calls", None) + if isinstance(tool_calls, list): + for call in tool_calls: + if call.tool_call_id in successful_call_ids: + tools.turn.record(turn_effect(call.name)) async def _report_execution_event( self, diff --git a/tests/adapters/test_letta_adapter.py b/tests/adapters/test_letta_adapter.py index b64adc372..bfd81788c 100644 --- a/tests/adapters/test_letta_adapter.py +++ b/tests/adapters/test_letta_adapter.py @@ -1631,6 +1631,32 @@ async def test_parallel_results_match_call_ids_not_positions( assert tools.messages_sent == [] assert failure_reports(tools) == [] + @pytest.mark.asyncio + async def test_buffered_results_are_judged_independent_of_message_order( + self, + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), + room_id=ROOM_ID, + ) + calls, results = make_parallel_tool_messages( + [ + (BandTool.SEND_MESSAGE, "error"), + (BandTool.ADD_PARTICIPANT, "success"), + ], + reverse_returns=True, + ) + mock_client.agents.messages.create.return_value = make_letta_response( + results, calls + ) + tools = turn_tools() + + await adapter.on_event(turn_input(tools)) + + assert tools.turn.complete + assert not tools.turn.replied + assert failure_reports(tools) == [] + # ────────────────────────────────────────────────────────────────────── # Auto-relay knob From 4f3e718e137883abf7be04404dd28fa4419fb447 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Mon, 5 Oct 2026 15:52:21 +0300 Subject: [PATCH 15/17] revert: judge Letta tool effects in stream order This reverts commit 7eb311c173d735ee0754160a07b4d8938ed2c980. Letta always emits a tool_call_message before its tool_return_message: create_parallel_tool_messages_from_llm_response appends the assistant (calls) message before the grouped tool message, and both the non-streamed LettaResponse and the stream_tokens=False conversation stream convert them in list order. The single-pass id->name lookup was already correct; the order-independent two-pass scan guarded an order the server never produces. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WW717aA3yKxzE31y18jB4r --- docs/turn-outcome.md | 8 ++-- src/band/adapters/letta.py | 57 +++++++++++++--------------- tests/adapters/test_letta_adapter.py | 26 ------------- 3 files changed, 30 insertions(+), 61 deletions(-) diff --git a/docs/turn-outcome.md b/docs/turn-outcome.md index d1acf59a3..4ac20cdf6 100644 --- a/docs/turn-outcome.md +++ b/docs/turn-outcome.md @@ -219,11 +219,9 @@ Parlant joins the non-preamble final segments from one event batch before relaying, so recording the fallback reply cannot suppress a later segment of that same answer. -For external Letta MCP servers, the SDK examines the fully buffered response: -it collects successful grouped return IDs, then records the core turn effect -of each named call whose `tool_call_id` succeeded. Message order does not -determine the outcome. Failed returns record nothing; self-hosted tools record -their own effects. +For external Letta MCP servers, each successful grouped tool return is matched +to its call by `tool_call_id` and records that core tool's turn effect. Failed +returns record nothing; self-hosted tools record their own effects. `tests/framework_conformance/test_reply_boundary.py` pins every `send_message`, `deliver_reply` and `relay_reply` call outside the tool implementations, per diff --git a/src/band/adapters/letta.py b/src/band/adapters/letta.py index 21448206f..20cd21f73 100644 --- a/src/band/adapters/letta.py +++ b/src/band/adapters/letta.py @@ -559,8 +559,8 @@ async def _process_response_messages( Returns the list of assistant text parts collected during the turn. """ - self._record_external_tool_effects(tools, response_messages) final_text_parts: list[str] = [] + tool_names_by_id: dict[str, str] = {} for resp_msg in response_messages: match getattr(resp_msg, "message_type", None): case "assistant_message": @@ -569,6 +569,11 @@ async def _process_response_messages( case "tool_call_message": # MCP tool call executed server-side — observe only tool_call = getattr(resp_msg, "tool_call", None) + tool_calls = getattr(resp_msg, "tool_calls", None) + if isinstance(tool_calls, list): + tool_names_by_id.update( + (call.tool_call_id, call.name) for call in tool_calls + ) tool_name = ( getattr(tool_call, "name", "unknown") if tool_call @@ -601,6 +606,9 @@ async def _process_response_messages( ) case "tool_return_message": tool_name = getattr(resp_msg, "name", None) or "unknown" + self._record_external_tool_return( + tools, tool_name, resp_msg, tool_names_by_id + ) await self._report_execution_event( tools, "tool_result", @@ -643,40 +651,29 @@ async def _process_response_messages( return final_text_parts - def _record_external_tool_effects( - self, tools: AgentToolsProtocol, response_messages: list[Any] + def _record_external_tool_return( + self, + tools: AgentToolsProtocol, + tool_name: str, + resp_msg: Any, + tool_names_by_id: dict[str, str], ) -> None: - """Record successful external calls from the fully buffered response. + """Record an external band-mcp call's effect on the turn. - Self-hosted tools record their own effects during execution. + A self-hosted call runs on this room's own tools, which record it + themselves; an external server's call is visible only in the stream. """ if self.config.mcp.mode != "external": return - - successful_call_ids: set[str] = set() - for message in response_messages: - if getattr(message, "message_type", None) != "tool_return_message": - continue - tool_returns = getattr(message, "tool_returns", None) - if isinstance(tool_returns, list): - successful_call_ids.update( - result.tool_call_id - for result in tool_returns - if result.status == "success" - ) - elif message.status == "success": - tools.turn.record( - turn_effect(getattr(message, "name", None) or "unknown") - ) - - for message in response_messages: - if getattr(message, "message_type", None) != "tool_call_message": - continue - tool_calls = getattr(message, "tool_calls", None) - if isinstance(tool_calls, list): - for call in tool_calls: - if call.tool_call_id in successful_call_ids: - tools.turn.record(turn_effect(call.name)) + tool_returns = getattr(resp_msg, "tool_returns", None) + if isinstance(tool_returns, list): + # Letta's legacy name/status describe only the first parallel call. + for result in tool_returns: + if result.status == "success": + name = tool_names_by_id.get(result.tool_call_id, "unknown") + tools.turn.record(turn_effect(name)) + elif resp_msg.status == "success": + tools.turn.record(turn_effect(tool_name)) async def _report_execution_event( self, diff --git a/tests/adapters/test_letta_adapter.py b/tests/adapters/test_letta_adapter.py index bfd81788c..b64adc372 100644 --- a/tests/adapters/test_letta_adapter.py +++ b/tests/adapters/test_letta_adapter.py @@ -1631,32 +1631,6 @@ async def test_parallel_results_match_call_ids_not_positions( assert tools.messages_sent == [] assert failure_reports(tools) == [] - @pytest.mark.asyncio - async def test_buffered_results_are_judged_independent_of_message_order( - self, - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), - room_id=ROOM_ID, - ) - calls, results = make_parallel_tool_messages( - [ - (BandTool.SEND_MESSAGE, "error"), - (BandTool.ADD_PARTICIPANT, "success"), - ], - reverse_returns=True, - ) - mock_client.agents.messages.create.return_value = make_letta_response( - results, calls - ) - tools = turn_tools() - - await adapter.on_event(turn_input(tools)) - - assert tools.turn.complete - assert not tools.turn.replied - assert failure_reports(tools) == [] - # ────────────────────────────────────────────────────────────────────── # Auto-relay knob From 97a525060d5de2134e3daaa6995e2efd0be4ae28 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Mon, 5 Oct 2026 15:59:09 +0300 Subject: [PATCH 16/17] refactor: match each Letta tool return to its pending call Letta's grouped tool_returns carry no tool name, so a return is joined to its call by tool_call_id. Track the calls awaiting a return and pop each one as its result lands, with one path for both the grouped and the single-return shapes, instead of a name map plus a legacy branch. Every parallel call and result is now reported, not only the first, and a multimodal result serializes. Test fakes are real letta-client models, so the adapter no longer guards against MagicMock attributes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WW717aA3yKxzE31y18jB4r --- src/band/adapters/letta.py | 112 +++++++++------------- tests/adapters/lettakit.py | 95 ++++++++++++++++--- tests/adapters/test_letta_adapter.py | 136 +++++++++++++++++---------- 3 files changed, 215 insertions(+), 128 deletions(-) diff --git a/src/band/adapters/letta.py b/src/band/adapters/letta.py index 20cd21f73..ff5843f97 100644 --- a/src/band/adapters/letta.py +++ b/src/band/adapters/letta.py @@ -10,6 +10,7 @@ from typing import Any, ClassVar from band_sdk_core import AgentFailure +from pydantic_core import to_jsonable_python from typing_extensions import Unpack from band.converters.letta import LettaHistoryConverter, LettaSessionState @@ -560,64 +561,34 @@ async def _process_response_messages( Returns the list of assistant text parts collected during the turn. """ final_text_parts: list[str] = [] - tool_names_by_id: dict[str, str] = {} + calls_awaiting_return: dict[str, Any] = {} for resp_msg in response_messages: match getattr(resp_msg, "message_type", None): case "assistant_message": if text := (getattr(resp_msg, "content", "") or ""): final_text_parts.append(text) case "tool_call_message": - # MCP tool call executed server-side — observe only - tool_call = getattr(resp_msg, "tool_call", None) - tool_calls = getattr(resp_msg, "tool_calls", None) - if isinstance(tool_calls, list): - tool_names_by_id.update( - (call.tool_call_id, call.name) for call in tool_calls - ) - tool_name = ( - getattr(tool_call, "name", "unknown") - if tool_call - else "unknown" - ) - # ToolCall.arguments is a JSON string (letta_client's own - # wire shape); parse it so redact_tool_call_args can - # replace band_send_room_file's content field -- reporting - # the raw string would put real file bytes in this event. - raw_arguments = ( - getattr(tool_call, "arguments", "{}") if tool_call else "{}" - ) - try: - parsed_arguments = json.loads(raw_arguments) - except (TypeError, ValueError): - parsed_arguments = raw_arguments - reported_args = ( - redact_tool_call_args(tool_name, parsed_arguments) - if isinstance(parsed_arguments, dict) - else parsed_arguments - ) - await self._report_execution_event( - tools, - "tool_call", - tool_name, - { - ToolEventKey.NAME: tool_name, - ToolEventKey.ARGS: reported_args, - }, - ) + # MCP tool calls executed server-side — observe only. + # tool_call alone holds only the first parallel call. + for call in resp_msg.tool_calls or [resp_msg.tool_call]: + calls_awaiting_return[call.tool_call_id] = call + await self._report_tool_call(tools, call) case "tool_return_message": - tool_name = getattr(resp_msg, "name", None) or "unknown" - self._record_external_tool_return( - tools, tool_name, resp_msg, tool_names_by_id - ) - await self._report_execution_event( - tools, - "tool_result", - tool_name, - { - ToolEventKey.NAME: tool_name, - ToolEventKey.OUTPUT: getattr(resp_msg, "tool_return", ""), - }, - ) + # A server that sends no tool_returns puts its one + # result on the message itself. + for result in resp_msg.tool_returns or [resp_msg]: + call = calls_awaiting_return.pop(result.tool_call_id, None) + tool_name = call.name if call else "unknown" + self._record_external_tool_return(tools, tool_name, result) + await self._report_execution_event( + tools, + "tool_result", + tool_name, + { + ToolEventKey.NAME: tool_name, + ToolEventKey.OUTPUT: result.tool_return, + }, + ) # If the agent already settled its reply through an MCP tool (a message on # the platform, or band_no_reply) — nothing to relay. Otherwise fall back @@ -652,29 +623,34 @@ async def _process_response_messages( return final_text_parts def _record_external_tool_return( - self, - tools: AgentToolsProtocol, - tool_name: str, - resp_msg: Any, - tool_names_by_id: dict[str, str], + self, tools: AgentToolsProtocol, tool_name: str, result: Any ) -> None: """Record an external band-mcp call's effect on the turn. A self-hosted call runs on this room's own tools, which record it themselves; an external server's call is visible only in the stream. """ - if self.config.mcp.mode != "external": - return - tool_returns = getattr(resp_msg, "tool_returns", None) - if isinstance(tool_returns, list): - # Letta's legacy name/status describe only the first parallel call. - for result in tool_returns: - if result.status == "success": - name = tool_names_by_id.get(result.tool_call_id, "unknown") - tools.turn.record(turn_effect(name)) - elif resp_msg.status == "success": + if self.config.mcp.mode == "external" and result.status == "success": tools.turn.record(turn_effect(tool_name)) + async def _report_tool_call(self, tools: AgentToolsProtocol, call: Any) -> None: + # ToolCall.arguments is a JSON string (letta_client's own wire shape); + # parse it so redact_tool_call_args can replace band_send_room_file's + # content field -- reporting the raw string would put real file bytes + # in this event. + try: + arguments = json.loads(call.arguments) + except (TypeError, ValueError): + arguments = call.arguments + if isinstance(arguments, dict): + arguments = redact_tool_call_args(call.name, arguments) + await self._report_execution_event( + tools, + "tool_call", + call.name, + {ToolEventKey.NAME: call.name, ToolEventKey.ARGS: arguments}, + ) + async def _report_execution_event( self, tools: AgentToolsProtocol, @@ -691,7 +667,9 @@ async def _report_execution_event( return if tool_name in self._mcp.silent_reporting_tools: return - await tools.send_event(content=json.dumps(payload), message_type=event_type) + # A multimodal tool return is a list of letta_client content models. + content = json.dumps(payload, default=to_jsonable_python) + await tools.send_event(content=content, message_type=event_type) # ------------------------------------------------------------------ # Letta agent lifecycle diff --git a/tests/adapters/lettakit.py b/tests/adapters/lettakit.py index 5322176b0..07ff751eb 100644 --- a/tests/adapters/lettakit.py +++ b/tests/adapters/lettakit.py @@ -11,10 +11,14 @@ import json from collections.abc import Awaitable, Callable, Sequence from datetime import UTC, datetime -from typing import Any +from typing import Any, Literal from unittest.mock import AsyncMock, MagicMock from uuid import uuid4 +from letta_client.types.agents.tool_call import ToolCall +from letta_client.types.agents.tool_call_message import ToolCallMessage +from letta_client.types.agents.tool_return import ToolReturn +from letta_client.types.tool_return_message import ToolReturnMessage from pytest_httpx import HTTPXMock from typing_extensions import Unpack @@ -57,25 +61,90 @@ def make_assistant_message(content: str = "Hello!") -> MagicMock: def make_tool_call_message( tool_name: str = "band_send_message", arguments: str = '{"content": "Hi", "mentions": ["@alice"]}', -) -> MagicMock: - tool_call = MagicMock() - tool_call.name = tool_name - tool_call.arguments = arguments - return make_letta_message("tool_call_message", tool_call=tool_call) +) -> ToolCallMessage: + return make_grouped_call_message( + [ToolCall(name=tool_name, arguments=arguments, tool_call_id=call_id(tool_name))] + ) def make_tool_return_message( tool_name: str = "band_send_message", tool_return: str = '{"status": "ok"}', - status: str = "success", -) -> MagicMock: - """letta-client's ``ToolReturnMessage`` names the tool in ``name``.""" - return make_letta_message( - "tool_return_message", name=tool_name, tool_return=tool_return, status=status + status: Literal["success", "error"] = "success", +) -> ToolReturnMessage: + return make_grouped_return_message( + [ + ToolReturn( + type="tool", + tool_return=tool_return, + status=status, + tool_call_id=call_id(tool_name), + ) + ], + name=tool_name, + ) + + +def call_id(tool_name: str) -> str: + return f"call-{tool_name}" + + +def make_grouped_call_message(calls: list[ToolCall]) -> ToolCallMessage: + """Letta's shape: ``tool_call`` repeats only the first of ``tool_calls``.""" + return ToolCallMessage( + id=f"message-{calls[0].tool_call_id}", + date=datetime.now(UTC), + message_type="tool_call_message", + tool_call=calls[0], + tool_calls=calls, + ) + + +def make_grouped_return_message( + returns: list[ToolReturn], *, name: str +) -> ToolReturnMessage: + """Letta's shape: ``name``/``status``/``tool_call_id`` describe only the + first of ``tool_returns``.""" + first = returns[0] + return ToolReturnMessage( + id=f"message-return-{first.tool_call_id}", + date=datetime.now(UTC), + message_type="tool_return_message", + name=name, + status=first.status, + tool_call_id=first.tool_call_id, + tool_return=str(first.tool_return), + tool_returns=returns, + ) + + +def make_parallel_tool_messages( + results: list[tuple[str, Literal["success", "error"]]], + *, + reverse_returns: bool, +) -> tuple[ToolCallMessage, ToolReturnMessage]: + calls = [ + ToolCall(name=name, arguments="{}", tool_call_id=f"call-{index}") + for index, (name, _status) in enumerate(results) + ] + returns = [ + ToolReturn( + type="tool", + tool_return="result", + status=status, + tool_call_id=call.tool_call_id, + ) + for call, (_name, status) in zip(calls, results, strict=True) + ] + if reverse_returns: + returns.reverse() + return ( + make_grouped_call_message(calls), + make_grouped_return_message(returns, name=calls[0].name), ) -def make_letta_response(*messages: MagicMock) -> MagicMock: +def make_letta_response(*messages: Any) -> MagicMock: """Create a fake Letta API response.""" resp = MagicMock() resp.messages = list(messages) @@ -121,7 +190,7 @@ def scripted_letta_turn( } async def create(**_kwargs: Any) -> MagicMock: - messages: list[MagicMock] = [] + messages: list[Any] = [] for name, arguments in tool_calls: messages.append(make_tool_call_message(name, json.dumps(arguments))) result = await registrations[name].execute( diff --git a/tests/adapters/test_letta_adapter.py b/tests/adapters/test_letta_adapter.py index b64adc372..5b0c07cf0 100644 --- a/tests/adapters/test_letta_adapter.py +++ b/tests/adapters/test_letta_adapter.py @@ -14,8 +14,8 @@ from unittest.mock import AsyncMock, MagicMock import pytest +from letta_client.types.agents.text_content import TextContent from letta_client.types.agents.tool_call import ToolCall -from letta_client.types.agents.tool_call_message import ToolCallMessage from letta_client.types.agents.tool_return import ToolReturn from letta_client.types.tool_return_message import ToolReturnMessage @@ -30,22 +30,27 @@ GENERIC_PROVIDER_FAILURE_MESSAGE, TurnResultAlreadyReported, ) -from band.core.types import Capability, Emit +from band.core.types import Capability, Emit, ToolEventKey from band.runtime.tools import BandTool from band.testing import ( MISSING_REPLY_FAILURE, FakeAgentTools, + events_of_type, failure_reports, reported_failures, ) from tests.adapters.lettakit import ( + call_id, default_enforcement, make_assistant_message, + make_grouped_call_message, + make_grouped_return_message, make_letta_response, make_mock_agent, make_mock_async_stream, make_mock_conversation, make_mock_tool_page, + make_parallel_tool_messages, make_platform_message, make_tool_call_message, make_tool_return_message, @@ -72,6 +77,13 @@ def sent_contents(tools: FakeAgentTools) -> list[str]: return [message["content"] for message in tools.messages_sent] +def reported_tool_names(tools: FakeAgentTools, event_type: str) -> list[str]: + return [ + json.loads(event["content"])[ToolEventKey.NAME] + for event in events_of_type(tools, event_type) + ] + + # ────────────────────────────────────────────────────────────────────── # Initialization # ────────────────────────────────────────────────────────────────────── @@ -835,6 +847,51 @@ async def test_reports_non_silent_tool_calls(self) -> None: assert len(tool_call_events) == 1 assert len(tool_result_events) == 1 + @pytest.mark.asyncio + async def test_reports_every_parallel_call_and_its_result(self) -> None: + adapter, mock_client = ready_letta_adapter(emit=Emit.TOOL_CALLS) + calls = [ + ToolCall(name=BandTool.GET_PARTICIPANTS, arguments="{}", tool_call_id="a"), + ToolCall(name=BandTool.LOOKUP_PEERS, arguments="{}", tool_call_id="b"), + ] + multimodal_output = [TextContent(type="text", text="no peers")] + returns = [ + ToolReturn( + type="tool", + status="success", + tool_call_id="b", + tool_return=multimodal_output, + ), + ToolReturn( + type="tool", status="success", tool_call_id="a", tool_return="[]" + ), + ] + mock_client.agents.messages.create.return_value = make_letta_response( + make_grouped_call_message(calls), + make_grouped_return_message(returns, name=calls[0].name), + make_assistant_message("Done"), + ) + tools = FakeAgentTools() + + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert reported_tool_names(tools, "tool_call") == [ + BandTool.GET_PARTICIPANTS, + BandTool.LOOKUP_PEERS, + ] + assert reported_tool_names(tools, "tool_result") == [ + BandTool.LOOKUP_PEERS, + BandTool.GET_PARTICIPANTS, + ] + @pytest.mark.asyncio async def test_silent_tools_not_reported(self) -> None: config = LettaAdapterConfig() @@ -1424,48 +1481,6 @@ async def test_relay_detection_uses_resolved_name(self) -> None: # ────────────────────────────────────────────────────────────────────── -def make_parallel_tool_messages( - results: list[tuple[str, Literal["success", "error"]]], - *, - reverse_returns: bool, -) -> tuple[ToolCallMessage, ToolReturnMessage]: - calls = [ - ToolCall(name=name, arguments="{}", tool_call_id=f"call-{index}") - for index, (name, _status) in enumerate(results) - ] - returns = [ - ToolReturn( - type="tool", - tool_return="result", - status=status, - tool_call_id=call.tool_call_id, - ) - for call, (_name, status) in zip(calls, results, strict=True) - ] - first_call, first_return = calls[0], returns[0] - if reverse_returns: - returns.reverse() - return ( - ToolCallMessage( - id="message-calls", - date=datetime.now(UTC), - message_type="tool_call_message", - tool_call=first_call, - tool_calls=calls, - ), - ToolReturnMessage( - id="message-returns", - date=datetime.now(UTC), - message_type="tool_return_message", - name=first_call.name, - status=first_return.status, - tool_call_id=first_call.tool_call_id, - tool_return="result", - tool_returns=returns, - ), - ) - - class TestExternalToolRecording: @pytest.mark.asyncio @pytest.mark.parametrize( @@ -1487,13 +1502,38 @@ async def test_streamed_tool_return_records_only_for_external_band_mcp( LettaAdapterConfig(mcp=LettaMCPConfig(mode=mcp_mode)) ) mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message(BandTool.SEND_MESSAGE), + make_tool_return_message(BandTool.SEND_MESSAGE, status=status), + ) + tools = FakeAgentTools() + + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert tools.turn.replied is replied + + @pytest.mark.asyncio + async def test_ungrouped_tool_return_is_matched_to_its_call(self) -> None: + """A server that sends no ``tool_returns`` puts the one result on the + message itself.""" + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")) + ) + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message(BandTool.SEND_MESSAGE), ToolReturnMessage( id="message-return", date=datetime.now(UTC), message_type="tool_return_message", - name=BandTool.SEND_MESSAGE, - status=status, - tool_call_id="call-1", + status="success", + tool_call_id=call_id(BandTool.SEND_MESSAGE), tool_return="result", ), ) @@ -1509,7 +1549,7 @@ async def test_streamed_tool_return_records_only_for_external_band_mcp( room_id="room-1", ) - assert tools.turn.replied is replied + assert tools.turn.replied @pytest.mark.asyncio @pytest.mark.parametrize( From 3d2727733b55acf921fb58c68e5dc6397914b649 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Mon, 5 Oct 2026 16:06:24 +0300 Subject: [PATCH 17/17] test: custom tools record their effect on a room-bound endpoint's turn Also imports the opencode probe's helpers at module level: the module already imports the opencode adapter (and so mcp), which every lane's venv carries. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WW717aA3yKxzE31y18jB4r --- .../mcp/test_room_bound_server.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/integrations/mcp/test_room_bound_server.py b/tests/integrations/mcp/test_room_bound_server.py index de9a11d75..058ff9ac4 100644 --- a/tests/integrations/mcp/test_room_bound_server.py +++ b/tests/integrations/mcp/test_room_bound_server.py @@ -20,7 +20,8 @@ build_engine, build_resolved_band_mcp_tool_registrations, ) -from band.runtime.tools import BandTool, iter_tool_definitions +from band.runtime.custom_tools import declares_turn_effect +from band.runtime.tools import BandTool, TurnEffect, iter_tool_definitions from band.testing import FakeAgentTools from tests.mcpclient import ( STORE_MEMORY_ARGS, @@ -42,6 +43,7 @@ class LookupInput(BaseModel): query: str +@declares_turn_effect(TurnEffect.ACT) async def lookup(input_data: LookupInput) -> str: return input_data.query @@ -101,6 +103,27 @@ async def test_call_lands_in_its_endpoints_room( assert rooms[ROOM_B].memories == [] +@pytest.mark.timeout(90) +@pytest.mark.asyncio +@pytest.mark.parametrize("room_bound", [True, False]) +async def test_custom_tool_records_its_effect_on_its_rooms_turn( + rooms: dict[str, FakeAgentTools], room_bound: bool +) -> None: + room_args = {} if room_bound else {"chat_id": ROOM_B} + async with ( + room_backend(rooms, room_bound=room_bound) as backend, + mcp_session( + backend.endpoint(BandMCPTransport.HTTP, ROOM_B if room_bound else None) + ) as session, + ): + await session.call_tool("lookup", {"query": "q", **room_args}) + + assert {room: tools.turn.complete for room, tools in rooms.items()} == { + ROOM_A: False, + ROOM_B: True, + } + + @pytest.mark.timeout(90) @pytest.mark.asyncio async def test_rooms_share_one_server_without_crossing(