From 8f326c0060db45f24d39d34d57fc665b42feca69 Mon Sep 17 00:00:00 2001 From: Clement Dumas Date: Thu, 15 Oct 2026 12:38:51 -0700 Subject: [PATCH 1/2] codex: load Claude Code's deferred tools incrementally, like Codex's incremental tools With tool search on (ENABLE_TOOL_SEARCH), Claude Code sends deferred MCP tools with defer_loading: true once ToolSearch has loaded them, and marks the load with a tool_reference block in the tool_result. On the Responses Lite lane the proxy used to put every tool, deferred or not, in the leading additional_tools item, so each load rewrote the prefix and the next request missed the prompt cache. tool_reference blocks also reached the model as '[unsupported content block omitted: tool_reference]'. Now, on the lite lane, a deferred tool that a tool_reference loaded is appended as its own additional_tools item right after that tool_result, the way Codex's IncrementalTools feature appends catalog changes without rewriting history (core/tests/suite/scenarios_incremental_tools.rs). Deferred tools with no visible load (e.g. loaded before a compaction) stay in the prefix. The Anthropic-only DeferredToolPlaceholder stub is dropped on both lanes, and tool_reference renders as 'Tool loaded: '. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xmx9RE7u4ZLBzSUjpjqyBM --- src/providers/codex/translate/request.rs | 287 ++++++++++++++++++++++- 1 file changed, 276 insertions(+), 11 deletions(-) diff --git a/src/providers/codex/translate/request.rs b/src/providers/codex/translate/request.rs index fea9a779..158f7375 100644 --- a/src/providers/codex/translate/request.rs +++ b/src/providers/codex/translate/request.rs @@ -1,4 +1,4 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use base64::Engine; use serde::{Deserialize, Serialize}; @@ -533,15 +533,16 @@ fn translate_request_inner( if let Some(ref tools) = tools && !tools.is_empty() { - let tools = tools - .iter() - .map(serde_json::to_value) - .collect::, _>>()?; - prefix.push(ResponsesInputItem::AdditionalTools { - id: None, - role: "developer".to_string(), - tools, - }); + let loads = tool_reference_loads(req); + let (prefix_tools, loaded_tools) = split_loaded_deferred_tools(req, tools, &loads)?; + if !prefix_tools.is_empty() { + prefix.push(ResponsesInputItem::AdditionalTools { + id: None, + role: "developer".to_string(), + tools: prefix_tools, + }); + } + append_loaded_tools(&mut out.input, &loads, loaded_tools); } if let Some(instructions) = out.instructions.take() && !instructions.is_empty() @@ -558,7 +559,14 @@ fn translate_request_inner( } else if let Some(tools) = tools && !tools.is_empty() { - out.tools = Some(tools); + let deferred = deferred_tool_names(req); + let tools: Vec = tools + .into_iter() + .filter(|tool| !is_deferred_placeholder(tool, &deferred)) + .collect(); + if !tools.is_empty() { + out.tools = Some(tools); + } } // Never force a web_search tool_choice the request didn't register — @@ -648,6 +656,127 @@ fn read_output_format(req: &MessagesRequest) -> Option { } } +/// Claude Code sends this deferred stub only to keep Anthropic's deferred loading on. +const DEFERRED_TOOL_PLACEHOLDER: &str = "DeferredToolPlaceholder"; + +fn responses_tool_name(tool: &ResponsesTool) -> Option<&str> { + match tool { + ResponsesTool::Function(function) => Some(function.name.as_str()), + ResponsesTool::WebSearch(_) => None, + } +} + +fn deferred_tool_names(req: &MessagesRequest) -> HashSet { + req.extra + .get("tools") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter(|tool| tool.get("defer_loading").and_then(Value::as_bool) == Some(true)) + .filter_map(|tool| tool.get("name").and_then(Value::as_str).map(str::to_string)) + .collect() +} + +fn is_deferred_placeholder(tool: &ResponsesTool, deferred: &HashSet) -> bool { + responses_tool_name(tool) + .is_some_and(|name| name == DEFERRED_TOOL_PLACEHOLDER && deferred.contains(name)) +} + +/// Tool names each `tool_result` loaded through `tool_reference` blocks, keyed by tool_use_id. +fn tool_reference_loads(req: &MessagesRequest) -> HashMap> { + let mut loads: HashMap> = HashMap::new(); + for msg in req.messages.iter().filter(|msg| msg.role == "user") { + let Some(blocks) = msg.content.as_array() else { + continue; + }; + for block in blocks { + if block.get("type").and_then(Value::as_str) != Some("tool_result") { + continue; + } + let Some(id) = block.get("tool_use_id").and_then(Value::as_str) else { + continue; + }; + let names = block + .get("content") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter(|child| child.get("type").and_then(Value::as_str) == Some("tool_reference")) + .filter_map(|child| child.get("tool_name").and_then(Value::as_str)); + loads + .entry(id.to_string()) + .or_default() + .extend(names.map(str::to_string)); + } + } + loads.retain(|_, names| !names.is_empty()); + loads +} + +/// Splits the catalog for the Responses Lite lane, the way Codex's incremental tools do: +/// deferred tools that a `tool_reference` in the history loaded are taken out of the +/// prefix so they can be appended at the point they were loaded, which keeps the cached +/// prefix byte-identical across loads. Deferred tools with no visible load (e.g. loaded +/// before a compaction) stay in the prefix so they remain callable. +fn split_loaded_deferred_tools( + req: &MessagesRequest, + tools: &[ResponsesTool], + loads: &HashMap>, +) -> Result<(Vec, HashMap), anyhow::Error> { + let deferred = deferred_tool_names(req); + let loaded: HashSet<&str> = loads.values().flatten().map(String::as_str).collect(); + let mut prefix = Vec::new(); + let mut appended = HashMap::new(); + for tool in tools { + if is_deferred_placeholder(tool, &deferred) { + continue; + } + let value = serde_json::to_value(tool)?; + match responses_tool_name(tool) { + Some(name) if deferred.contains(name) && loaded.contains(name) => { + appended.insert(name.to_string(), value); + } + _ => prefix.push(value), + } + } + Ok((prefix, appended)) +} + +fn append_loaded_tools( + input: &mut Vec, + loads: &HashMap>, + mut loaded_tools: HashMap, +) { + if loaded_tools.is_empty() { + return; + } + let mut out = Vec::with_capacity(input.len() + loads.len()); + for item in input.drain(..) { + let names = match &item { + ResponsesInputItem::FunctionCallOutput { call_id, .. } => loads.get(call_id), + _ => None, + }; + out.push(item); + let tools: Vec = names + .into_iter() + .flatten() + .filter_map(|name| loaded_tools.remove(name)) + .collect(); + if !tools.is_empty() { + out.push(ResponsesInputItem::AdditionalTools { + id: None, + role: "developer".to_string(), + tools, + }); + } + } + debug_assert!( + loaded_tools.is_empty(), + "every loaded tool should follow the tool_result that loaded it" + ); + *input = out; +} + fn read_tools(req: &MessagesRequest) -> Result>, anyhow::Error> { let Some(tools) = req.extra.get("tools") else { return Ok(None); @@ -1122,6 +1251,10 @@ fn render_tool_result_block(block: &Value) -> RenderedToolResultPart { RenderedToolResultPart::Text(unsupported_tool_result_block_to_string(block)) }), Some("image") => render_tool_result_image(block), + Some("tool_reference") => match block.get("tool_name").and_then(Value::as_str) { + Some(name) => RenderedToolResultPart::Text(format!("Tool loaded: {name}")), + None => RenderedToolResultPart::Text(unsupported_tool_result_block_to_string(block)), + }, Some(other) => { RenderedToolResultPart::Text(format!("[unsupported content block omitted: {other}]")) } @@ -2190,6 +2323,138 @@ mod tests { } } + fn tool_search_request(load: bool, extra_turn: bool) -> MessagesRequest { + let mut tools = vec![ + json!({"name":"Bash","description":"run","input_schema":{"type":"object","properties":{}}}), + json!({"name":"ToolSearch","description":"search","input_schema":{"type":"object","properties":{}}}), + json!({"name":"DeferredToolPlaceholder","description":"never call","defer_loading":true, + "input_schema":{"type":"object","properties":{}}}), + ]; + let mut messages = vec![json!({"role":"user","content":"find the mail tool"})]; + if load { + tools.push(json!({"name":"mcp__mail__search","description":"search mail","defer_loading":true, + "input_schema":{"type":"object","properties":{"q":{"type":"string"}}}})); + messages.push( + json!({"role":"assistant","content":[{"type":"tool_use","id":"tu_search", + "name":"ToolSearch","input":{"query":"select:mcp__mail__search"}}]}), + ); + messages.push(json!({"role":"user","content":[ + {"type":"tool_result","tool_use_id":"tu_search", + "content":[{"type":"tool_reference","tool_name":"mcp__mail__search"}]}, + {"type":"text","text":"Tool loaded."}]})); + } + if extra_turn { + messages.push( + json!({"role":"assistant","content":[{"type":"tool_use","id":"tu_bash", + "name":"Bash","input":{"command":"ls"}}]}), + ); + messages.push(json!({"role":"user","content":[ + {"type":"tool_result","tool_use_id":"tu_bash","content":"a.txt"}]})); + } + serde_json::from_value(json!({"model":"gpt-6-astra","messages":messages,"tools":tools})) + .unwrap() + } + + fn lite_wire(req: &MessagesRequest) -> Value { + serde_json::to_value( + translate_request( + req, + TranslateOptions { + model: "gpt-6-astra".to_string(), + use_responses_lite: true, + ..opts() + }, + ) + .unwrap(), + ) + .unwrap() + } + + fn additional_tool_names(item: &Value) -> Vec { + item["tools"] + .as_array() + .unwrap() + .iter() + .map(|tool| tool["name"].as_str().unwrap().to_string()) + .collect() + } + + #[test] + fn loaded_deferred_tool_is_appended_after_its_tool_search_result() { + let wire = lite_wire(&tool_search_request(true, false)); + let input = wire["input"].as_array().unwrap(); + assert_eq!(input[0]["type"], "additional_tools"); + assert_eq!(additional_tool_names(&input[0]), vec!["Bash", "ToolSearch"]); + let output_at = input + .iter() + .position(|item| { + item["type"] == "function_call_output" && item["call_id"] == "tu_search" + }) + .unwrap(); + assert_eq!(input[output_at]["output"], "Tool loaded: mcp__mail__search"); + assert_eq!(input[output_at + 1]["type"], "additional_tools"); + assert_eq!(input[output_at + 1]["role"], "developer"); + assert_eq!( + additional_tool_names(&input[output_at + 1]), + vec!["mcp__mail__search"] + ); + assert!(!wire.to_string().contains("DeferredToolPlaceholder")); + } + + #[test] + fn tool_load_keeps_the_earlier_request_as_a_prefix() { + let before = lite_wire(&tool_search_request(false, false)); + let after = lite_wire(&tool_search_request(true, false)); + let later = lite_wire(&tool_search_request(true, true)); + let before_input = before["input"].as_array().unwrap(); + let after_input = after["input"].as_array().unwrap(); + let later_input = later["input"].as_array().unwrap(); + assert_eq!(&after_input[..before_input.len()], &before_input[..]); + assert_eq!(&later_input[..after_input.len()], &after_input[..]); + } + + #[test] + fn deferred_tool_without_a_visible_load_stays_in_the_prefix() { + let mut req = tool_search_request(false, false); + req.extra["tools"].as_array_mut().unwrap().push(json!({ + "name":"mcp__mail__search","description":"search mail","defer_loading":true, + "input_schema":{"type":"object","properties":{}} + })); + let wire = lite_wire(&req); + let input = wire["input"].as_array().unwrap(); + assert_eq!( + additional_tool_names(&input[0]), + vec!["Bash", "ToolSearch", "mcp__mail__search"] + ); + assert_eq!( + input + .iter() + .filter(|item| item["type"] == "additional_tools") + .count(), + 1 + ); + } + + #[test] + fn full_lane_keeps_loaded_tools_and_drops_the_placeholder() { + let out = translate_request(&tool_search_request(true, false), opts()).unwrap(); + let wire = serde_json::to_value(&out).unwrap(); + let names: Vec<&str> = wire["tools"] + .as_array() + .unwrap() + .iter() + .map(|tool| tool["name"].as_str().unwrap()) + .collect(); + assert_eq!(names, vec!["Bash", "ToolSearch", "mcp__mail__search"]); + assert!( + !wire["input"] + .as_array() + .unwrap() + .iter() + .any(|item| item["type"] == "additional_tools") + ); + } + #[test] fn translate_tool_result_content() { let req: MessagesRequest = serde_json::from_value(json!({ From 40515bd1d6b37477827f31c52718c5407341111a Mon Sep 17 00:00:00 2001 From: Clement Dumas Date: Thu, 15 Oct 2026 12:40:05 -0700 Subject: [PATCH 2/2] docs: document ENABLE_TOOL_SEARCH and incremental deferred tools on Codex Co-Authored-By: Claude Opus 5.5 --- docs/src/content/docs/providers/codex.md | 9 +++++++++ docs/src/content/docs/using/configure-claude-code.md | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/docs/src/content/docs/providers/codex.md b/docs/src/content/docs/providers/codex.md index cca0fc42..227152ae 100644 --- a/docs/src/content/docs/providers/codex.md +++ b/docs/src/content/docs/providers/codex.md @@ -92,6 +92,15 @@ and quota accounting depend on the upstream subscription; this setting is opt-in ## Tools and multimodal input - Claude function tools and tool results map to Responses API function calls and outputs. +- With Claude Code's tool search on (`ENABLE_TOOL_SEARCH=true`, see [Configure Claude Code](/using/configure-claude-code/#mcp-tool-search)), + a deferred tool reaches the model once `ToolSearch` loads it. On the Responses + Lite lane, the proxy appends each loaded tool as its own `additional_tools` item + right after the result that loaded it, as Codex's incremental tools do, so + earlier input is never rewritten and stays in the prompt cache. Deferred tools + with no load in the visible history, such as tools loaded before a compaction, + stay in the leading tool list. `tool_reference` blocks render as + `Tool loaded: `, and Claude Code's `DeferredToolPlaceholder` stub is not + forwarded. - Claude Code's forced `web_search_20250305` subrequest uses Codex's standalone `/alpha/search` endpoint. It keeps the resolved model, omits search reasoning, and preserves non-empty domain filters, so Luna searches do not require a Sol diff --git a/docs/src/content/docs/using/configure-claude-code.md b/docs/src/content/docs/using/configure-claude-code.md index 031d392f..2f36412d 100644 --- a/docs/src/content/docs/using/configure-claude-code.md +++ b/docs/src/content/docs/using/configure-claude-code.md @@ -28,6 +28,12 @@ CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK=1 \ The proxy always makes streaming upstream requests. It can still accumulate a non-streaming Anthropic response when the client requests one. +## MCP tool search + +Claude Code turns tool search off when `ANTHROPIC_BASE_URL` is not an Anthropic host. Every MCP tool schema is then sent with every request, which can add tens of thousands of tokens per request when many MCP servers or claude.ai connectors are connected. + +Set `ENABLE_TOOL_SEARCH=true` to keep tool search on behind the proxy. Claude Code then lists deferred tools by name and loads a tool's schema only after the model calls `ToolSearch` for it. On Codex, the proxy appends each loaded tool where it was loaded, so the request stays append-only for the prompt cache (see [Codex tools](/providers/codex/#tools-and-multimodal-input)). + ## Compaction settings A trailing `[1m]` tells Claude Code to use its larger local context policy. The proxy strips the suffix before upstream routing. It does not enlarge the provider's context window.