diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md index c2c948150..59d3162c2 100644 --- a/crates/libsy-llm-client/README.md +++ b/crates/libsy-llm-client/README.md @@ -269,6 +269,12 @@ fn build_multi_format_client( transport failures, timeouts, HTTP 408/429, and 5xx responses. Buffered body transport failures are retried; streaming body failures are not replayed after the response has been returned. +- `ModelConfig::with_responses_reasoning` controls Responses reasoning replay. + Every Responses model defaults to `PreserveEncrypted`: plaintext is removed, + encrypted provider state is retained. Stored reasoning IDs are retained unless + the request sets `store: false`. Switchyard's translated Anthropic reasoning is + removed. Set `Drop` for + a backend that cannot consume provider reasoning. Messages and tool history are retained. - `HttpBackendConfig::failure_cooldown` skips completion calls to a backend briefly after an exhausted transient failure. Zero disables it. State is shared per model within the client. Calls resume when the cooldown expires. Deployment TOML defaults diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 0265faf99..ecf8fa71f 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -65,6 +65,7 @@ pub struct ModelConfig { model_name: ModelId, default_backend: Backend, other_backends: Option>, + responses_reasoning: crate::ResponsesReasoningPolicy, } impl ModelConfig { @@ -79,8 +80,16 @@ impl ModelConfig { model_name: model_name.into(), default_backend, other_backends, + responses_reasoning: crate::ResponsesReasoningPolicy::default(), } } + + /// Sets how Responses reasoning items are replayed to this model. + #[must_use] + pub fn with_responses_reasoning(mut self, policy: crate::ResponsesReasoningPolicy) -> Self { + self.responses_reasoning = policy; + self + } } /// A model-bearing provider operation outside the normal completion endpoint. @@ -343,6 +352,13 @@ impl TranslatingLlmClient { } omit_configured_body_fields(&mut body, backend.omit_body_fields()); merge_extra_body(&mut body, backend.extra_body()); + if matches!(backend, Backend::OpenAiResponses(_)) { + self.model_to_config + .get(model) + .map(|config| config.responses_reasoning) + .unwrap_or_default() + .normalize(&mut body); + } // After the merge on purpose: the effort override must win over both the caller's // value and any `reasoning` default a target set through `extra_body`. apply_reasoning_effort(&mut body, backend); @@ -1030,25 +1046,20 @@ const CODEX_NAMESPACE_SEPARATOR: &str = "__"; /// Rewrites a Responses body into the shape OpenAI-compatible providers accept. /// /// Codex extends Responses with namespace containers and namespaced function calls, and -/// OpenAI-compatible providers expect a flat Responses tool namespace. The input also loses -/// reasoning items that the provider cannot read. +/// OpenAI-compatible providers expect a flat Responses tool namespace. fn sanitize_openai_responses_provider_body(body: &mut Value) { let Value::Object(object) = body else { return; }; - let store_off = object.get("store") == Some(&Value::Bool(false)); - sanitize_openai_responses_input_for_provider(object.get_mut("input"), store_off); + sanitize_openai_responses_input_for_provider(object.get_mut("input")); sanitize_openai_responses_tools_for_provider(object.get_mut("tools")); sanitize_openai_responses_tool_choice_for_provider(object.get_mut("tool_choice")); } -/// Removes reasoning items that the provider cannot read and flattens namespaced function -/// calls in the Responses `input` array. -fn sanitize_openai_responses_input_for_provider(input: Option<&mut Value>, store_off: bool) { +fn sanitize_openai_responses_input_for_provider(input: Option<&mut Value>) { let Some(Value::Array(items)) = input else { return; }; - items.retain(|item| !is_unreadable_reasoning_item(item, store_off)); for item in items { let Some(object) = item.as_object_mut() else { continue; @@ -1059,28 +1070,6 @@ fn sanitize_openai_responses_input_for_provider(input: Option<&mut Value>, store } } -/// Returns true for a reasoning item that an OpenAI provider would reject with 400. -/// -/// A router can serve earlier steps of a turn from Claude and a later step from an OpenAI -/// model, and the client replays Claude's reasoning items on every step. Two kinds fail: -/// -/// - An item without `encrypted_content` (absent, null, or empty) when the request sets -/// `store: false`. OpenAI treats the item's ID as a reference to stored reasoning and finds -/// nothing. With `store` on, the ID can refer to stored reasoning, so the item is kept. -/// - An item whose `encrypted_content` holds Anthropic thinking that Switchyard packed for a -/// Responses client. OpenAI cannot decrypt it, whatever `store` is set to. -fn is_unreadable_reasoning_item(item: &Value, store_off: bool) -> bool { - if item.get("type").and_then(Value::as_str) != Some("reasoning") { - return false; - } - match item.get("encrypted_content").and_then(Value::as_str) { - Some(encrypted) if !encrypted.is_empty() => { - switchyard_translation::is_anthropic_thinking_envelope(encrypted) - } - _ => store_off, - } -} - fn sanitize_openai_responses_tools_for_provider(tools: Option<&mut Value>) { let Some(Value::Array(tools)) = tools else { return; @@ -1606,6 +1595,13 @@ mod tests { )] } + fn responses_map_with_drop_policy(base_url: &str) -> Vec { + vec![ + ModelConfig::new("gpt", Backend::OpenAiResponses(config(base_url)), None) + .with_responses_reasoning(crate::ResponsesReasoningPolicy::Drop), + ] + } + #[tokio::test] async fn model_properties_uses_configured_auth_and_backend_path() -> std::result::Result<(), Box> { @@ -2401,6 +2397,88 @@ mod tests { Ok(()) } + #[tokio::test] + async fn responses_reasoning_policy_preserves_replay_history() + -> std::result::Result<(), Box> { + let history = json!([ + {"type": "message", "role": "user", "content": "continue"}, + {"type": "reasoning", "encrypted_content": "opaque", "content": [{"type": "reasoning_text", "text": "private"}]}, + {"type": "reasoning", "id": "rs_stored", "status": "completed", "summary": []}, + {"type": "reasoning", "encrypted_content": null}, + {"type": "reasoning", "encrypted_content": ""}, + {"type": "reasoning"}, + {"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"} + ]); + for store in [Some(true), Some(false), None] { + for is_drop in [false, true] { + for has_input_override in [false, true] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", "object": "response", "model": "gpt", + "status": "completed", "output": [], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }))) + .expect(1) + .mount(&server) + .await; + let base_url = format!("{}/v1", server.uri()); + let mut models = if is_drop { + responses_map_with_drop_policy(&base_url) + } else { + responses_map(&base_url) + }; + if has_input_override { + let Backend::OpenAiResponses(backend) = &mut models[0].default_backend + else { + return Err("expected Responses backend".into()); + }; + backend.omit_body_fields.insert("input".to_string()); + backend + .extra_body + .insert("input".to_string(), history.clone()); + } + let client = TranslatingLlmClient::new(&models)?; + let mut request = json!({"model": "gpt", "input": history}); + if let Some(store) = store { + request["store"] = json!(store); + } + client + .call_rewrite_model_raw( + request, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await?; + let requests = server.received_requests().await.ok_or("missing requests")?; + let body: Value = serde_json::from_slice(&requests[0].body)?; + let mut expected = vec![history[0].clone()]; + if !is_drop { + let mut encrypted = history[1].clone(); + encrypted["content"] = json!([]); + expected.push(encrypted); + if store != Some(false) { + let mut stored = history[2].clone(); + stored["content"] = json!([]); + expected.push(stored); + } + } + expected.extend([history[6].clone(), history[7].clone()]); + assert_eq!( + body["input"], + json!(expected), + "store={store:?}, drop={is_drop}, replace={has_input_override}" + ); + } + } + } + + Ok(()) + } + #[tokio::test] async fn nvidia_anthropic_requests_strip_strict_from_extra_body() -> std::result::Result<(), Box> { @@ -3470,8 +3548,7 @@ mod tests { .mount(&server) .await; - let client = TranslatingLlmClient::new(&responses_map(&format!("{}/v1", server.uri())))?; - let claude_reasoning = json!({ + let mut claude_reasoning = json!({ "type": "reasoning", "id": "rs_b99feb9a-763d-4f7f-b6b2-2c0199b47ded", "summary": [{"type": "summary_text", "text": "Read the files first."}], @@ -3508,13 +3585,13 @@ mod tests { .as_str() .unwrap_or_default() )); - let gpt_reasoning = json!({ + let mut gpt_reasoning = json!({ "type": "reasoning", "id": "rs_gpt", "summary": [], "encrypted_content": "opaque-encrypted-reasoning" }); - let empty_reasoning = + let mut empty_reasoning = json!({"type": "reasoning", "id": "rs_empty", "summary": [], "encrypted_content": ""}); let user = json!({"type": "message", "role": "user", "content": "fix the build"}); let call = json!({"type": "function_call", "call_id": "call_1", "name": "exec", "arguments": "{}"}); @@ -3529,16 +3606,29 @@ mod tests { empty_reasoning ]); - for store in [json!(false), json!(true)] { - let raw = json!({"model": "client-facing", "store": store, "input": input.clone()}); - client - .call_rewrite_model_raw( - raw, - None, - Some(&ModelId::from("gpt")), - WireFormat::OpenAiResponses, - ) - .await?; + for has_input_override in [false, true] { + let mut models = responses_map(&format!("{}/v1", server.uri())); + if has_input_override { + let Backend::OpenAiResponses(backend) = &mut models[0].default_backend else { + return Err("expected Responses backend".into()); + }; + backend.omit_body_fields.insert("input".to_string()); + backend + .extra_body + .insert("input".to_string(), input.clone()); + } + let client = TranslatingLlmClient::new(&models)?; + for store in [json!(false), json!(true)] { + let raw = json!({"model": "client-facing", "store": store, "input": input.clone()}); + client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await?; + } } let received = server @@ -3549,21 +3639,46 @@ mod tests { .iter() .map(|request| serde_json::from_slice::(&request.body)) .collect::>()?; - // With `store: false`, only GPT's own encrypted reasoning survives. - assert_eq!(sent[0]["input"], json!([user, call, output, gpt_reasoning])); - // With `store: true`, IDs can refer to stored reasoning, but OpenAI still cannot - // decrypt the Switchyard item. - assert_eq!( - sent[1]["input"], - json!([ - user, - claude_reasoning, - call, - output, - gpt_reasoning, - empty_reasoning - ]) - ); + assert_eq!(sent.len(), 4); + for request in &sent { + assert!( + request["input"] + .as_array() + .ok_or("expected input array")? + .iter() + .all(|item| !item["encrypted_content"] + .as_str() + .is_some_and(switchyard_translation::is_anthropic_thinking_envelope)), + "provider cannot read translated Anthropic reasoning" + ); + } + for item in [ + &mut claude_reasoning, + &mut gpt_reasoning, + &mut empty_reasoning, + ] { + item["content"] = json!([]); + } + for requests in sent.as_chunks::<2>().0 { + // With `store: false`, only GPT's own encrypted reasoning survives. + assert_eq!( + requests[0]["input"], + json!([user, call, output, gpt_reasoning]) + ); + // With `store: true`, IDs can refer to stored reasoning, but OpenAI still cannot + // decrypt the Switchyard item. + assert_eq!( + requests[1]["input"], + json!([ + user, + claude_reasoning, + call, + output, + gpt_reasoning, + empty_reasoning + ]) + ); + } Ok(()) } diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs index 414a32167..5bc416bbf 100644 --- a/crates/libsy-llm-client/src/lib.rs +++ b/crates/libsy-llm-client/src/lib.rs @@ -29,6 +29,7 @@ mod observation; mod observed_translation; mod openai_decisions; pub mod raw; +mod responses_reasoning; pub mod run; mod system_one; @@ -38,6 +39,7 @@ pub use error::{LlmClientError, Result}; pub use observation::{ModelCallObservation, RunObservation, RunObserver}; pub use openai_decisions::OpenAiDecisionClient; pub use raw::RawResponse; +pub use responses_reasoning::ResponsesReasoningPolicy; pub use run::{ClientRouter, decide, run}; pub use switchyard_translation::RawEventStream; pub use system_one::SystemOneClient; diff --git a/crates/libsy-llm-client/src/responses_reasoning.rs b/crates/libsy-llm-client/src/responses_reasoning.rs new file mode 100644 index 000000000..f5ba00d15 --- /dev/null +++ b/crates/libsy-llm-client/src/responses_reasoning.rs @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Target-specific replay policy for OpenAI Responses reasoning items. + +use serde::Deserialize; +use serde_json::Value; + +/// Controls which Responses reasoning items are replayed to an upstream. +#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ResponsesReasoningPolicy { + /// Preserve encrypted reasoning and stored IDs the provider can read. + /// + /// This is the default for every Responses model, regardless of its URL. + #[default] + PreserveEncrypted, + /// Drop all reasoning items while preserving messages and tool-call history. + /// + /// Use this for local Responses-compatible servers that cannot consume + /// another provider's encrypted reasoning representation. + Drop, +} + +impl ResponsesReasoningPolicy { + /// Normalizes a Responses request body for this replay policy. + pub(crate) fn normalize(self, body: &mut Value) { + let has_stored_reasoning = body.get("store").and_then(Value::as_bool) != Some(false); + let Some(Value::Array(input)) = body.get_mut("input") else { + return; + }; + input.retain_mut(|item| self.normalize_item(item, has_stored_reasoning)); + } + + // Stored reasoning can be replayed by ID without encrypted content. + fn normalize_item(self, item: &mut Value, has_stored_reasoning: bool) -> bool { + let Some(object) = item.as_object_mut() else { + return true; + }; + if object.get("type").and_then(Value::as_str) != Some("reasoning") { + return true; + } + + let encrypted_content = object.get("encrypted_content").and_then(Value::as_str); + if encrypted_content.is_some_and(switchyard_translation::is_anthropic_thinking_envelope) { + return false; + } + let has_encrypted_content = encrypted_content.is_some_and(|content| !content.is_empty()); + let has_stored_id = object + .get("id") + .and_then(Value::as_str) + .is_some_and(|id| !id.is_empty()); + if self == Self::PreserveEncrypted + && (has_encrypted_content || (has_stored_reasoning && has_stored_id)) + { + object.insert("content".to_string(), Value::Array(Vec::new())); + true + } else { + false + } + } +} diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 5d7b2c8fc..271cde0dd 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Deserializer}; use serde_json::Value; use switchyard_llm_client::{ AuxiliaryOperation, Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, - OpenAiDecisionClient, SystemOneClient, TranslatingLlmClient, + OpenAiDecisionClient, ResponsesReasoningPolicy, SystemOneClient, TranslatingLlmClient, }; use switchyard_protocol::{ModelId, RoutedDecisionClient, RoutedLlmClient, WireFormat}; @@ -349,6 +349,13 @@ impl DeploymentConfig { for (name, client_config) in &self.llm_clients { validate_value("llm client name", name)?; + if client_config.responses_reasoning.is_some() + && !matches!(client_config.format, ClientFormat::OpenAiResponses) + { + return Err(RunnerError::configuration(format!( + "llm client {name} responses_reasoning is only valid for openai_responses" + ))); + } let backend = build_backend( name, client_config, @@ -387,17 +394,20 @@ impl DeploymentConfig { ))); } } - model_configs.push(ModelConfig::new( - target.id.clone(), - build_backend( - &target.llm_client, - client_config, - &target.extra_body, - &target.omit_body_fields, - target.reasoning_effort.clone(), - )?, - None, - )); + model_configs.push( + ModelConfig::new( + target.id.clone(), + build_backend( + &target.llm_client, + client_config, + &target.extra_body, + &target.omit_body_fields, + target.reasoning_effort.clone(), + )?, + None, + ) + .with_responses_reasoning(client_config.responses_reasoning.unwrap_or_default()), + ); } let mut clients = BTreeMap::new(); @@ -760,6 +770,7 @@ struct LlmClientConfig { failure_cooldown_ms: u64, /// Deadline in milliseconds for all attempts and the complete response. Unset is unbounded. timeout_ms: Option, + responses_reasoning: Option, } #[derive(Debug, Deserialize)] @@ -2142,6 +2153,61 @@ confidence_threshold = 0.5 Ok(()) } + #[test] + fn responses_reasoning_defaults_and_accepts_drop() -> RunnerResult<()> { + let default: DeploymentConfig = toml::from_str(VALID_CONFIG).map_err(|error| { + RunnerError::configuration(format!("failed to parse config: {error}")) + })?; + let responses = default + .llm_clients + .get("responses") + .ok_or_else(|| RunnerError::configuration("responses llm client is missing"))?; + assert_eq!(responses.responses_reasoning, None); + + let configured = VALID_CONFIG.replacen( + "[llm_clients.responses]\nformat = \"openai_responses\"\nbase_url = \"https://example.test/v1\"", + "[llm_clients.responses]\nformat = \"openai_responses\"\nbase_url = \"https://example.test/v1\"\nresponses_reasoning = \"drop\"", + 1, + ); + let config: DeploymentConfig = toml::from_str(&configured).map_err(|error| { + RunnerError::configuration(format!("failed to parse config: {error}")) + })?; + let responses = config + .llm_clients + .get("responses") + .ok_or_else(|| RunnerError::configuration("responses llm client is missing"))?; + assert_eq!( + responses.responses_reasoning, + Some(ResponsesReasoningPolicy::Drop) + ); + runner_from_toml(&configured)?; + Ok(()) + } + + #[test] + fn responses_reasoning_rejects_other_formats() { + let invalid = VALID_CONFIG.replacen( + "format = \"openai_chat\"", + "format = \"openai_chat\"\nresponses_reasoning = \"drop\"", + 1, + ); + assert!( + error_message(&invalid) + .contains("responses_reasoning is only valid for openai_responses") + ); + } + + #[test] + fn responses_reasoning_rejects_unreferenced_other_format() { + let invalid = format!( + "{VALID_CONFIG}\n[llm_clients.unused]\nformat = \"openai_chat\"\nbase_url = \"https://example.test/v1\"\nresponses_reasoning = \"drop\"\n" + ); + assert!( + error_message(&invalid) + .contains("responses_reasoning is only valid for openai_responses") + ); + } + #[test] fn rejects_headers_that_switchyard_sets() { let cases = [ diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 445fde093..fb342d11f 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -5903,6 +5903,56 @@ async fn upstream_headers_forward_on_streaming_responses() -> TestResult { Ok(()) } +#[tokio::test] +async fn configured_responses_reasoning_policy_reaches_upstream() -> TestResult { + let upstream = MockUpstream::start().await?; + let base_url = upstream.base_url.replace("/v1", "/buffered"); + for (setting, expected_reasoning) in [("", 2), ("responses_reasoning = \"drop\"", 0)] { + let config = format!( + r#" +schema_version = 1 +[llm_clients.upstream] +format = "openai_responses" +base_url = "{base_url}" +{setting} +[targets] +answer = {{ id = "model/efficient", llm_client = "upstream" }} +[routes.route] +id = "route" +type = "passthrough" +target = "answer" +"# + ); + let app = build_switchyard_router(load_test_config(&config)?); + let response = send(&app, "POST", "/v1/responses", Some(json!({ + "model": "route", "input": [ + {"type": "message", "role": "user", "content": "continue"}, + {"type": "reasoning", "encrypted_content": "opaque", "summary": []}, + {"type": "reasoning", "id": "rs_stored", "summary": []}, + {"type": "reasoning", "content": [{"type": "reasoning_text", "text": "local"}], "summary": []} + ] + }))).await?; + assert_eq!(response.status, StatusCode::OK, "{}", response.json()?); + let calls = upstream.calls.lock().await; + let input = calls.last().expect("upstream request")["input"] + .as_array() + .expect("input array"); + assert_eq!( + input + .iter() + .filter(|item| item["type"] == "reasoning") + .count(), + expected_reasoning + ); + assert!( + input + .iter() + .any(|item| item["type"] == "message" && item["role"] == "user") + ); + } + Ok(()) +} + #[tokio::test] async fn rate_limit_fallback_policy() -> TestResult { for (policy, forward_auth, prompt, is_terminal) in [ diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 9497adca6..b2249d077 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -54,11 +54,22 @@ route reaches no upstream. A file without a `[targets]` table is rejected with | `api_key_env` | No | unset | Name of the environment variable holding the key. Omit to send no authentication. | | `forward_auth` | No | `false` | Forward the caller's provider credential and application headers. A route's forwarding clients must use one credential family unless they all use the same scheme, host, and port, such as one LLM gateway. | | `extra_headers` | No | `{}` | Custom HTTP headers sent to the model server. Set credentials with `api_key_env` or `forward_auth`; the server rejects headers owned by the selected auth mode. Header names are case-insensitive. | +| `responses_reasoning` | No | `"preserve_encrypted"` | Responses reasoning replay: `"preserve_encrypted"` or `"drop"`. Only valid for `openai_responses`. | | `fallback_on_rate_limit` | No | `true` | Try another completion candidate after an upstream HTTP 429 exhausts retries. Set to `false` to return the 429 instead. | | `max_retries` | No | `2` | Retry budget, `0`–`10`. | | `failure_cooldown_ms` | No | `5000` (5 seconds) | Skip a backend for this many milliseconds after an exhausted transient completion failure. Zero disables it. | | `timeout_ms` | No | unset | Deadline in milliseconds for all attempts, retry delays, and the complete response, including stream reads. Must be at least `1`. Unset leaves the wait unbounded. | +For `openai_responses`, `responses_reasoning` controls replay of reasoning +history. Every client defaults to `"preserve_encrypted"`: reasoning items with +non-empty `encrypted_content` or a stored reasoning `id` are retained with empty +`content`. ID-only reasoning is removed when the request sets `store: false`. +Switchyard's translated Anthropic reasoning is always removed. +Reasoning without either is removed. Set `responses_reasoning = "drop"` explicitly for a backend +that cannot consume provider reasoning. Both modes keep messages, tool +calls and tool results. The setting is rejected on other client formats, even +when no target uses the client. It is not inferred from the model name or URL. + The TOML never contains the secret itself. `api_key_env` names a variable that must exist and be non-empty when the server loads.