From 6e5a8293ddd77a83daa7d5dafa32905b8e15ef93 Mon Sep 17 00:00:00 2001 From: Deepak Jain Date: Sun, 27 Sep 2026 23:08:55 -0700 Subject: [PATCH 1/3] fix(client): sanitize Responses reasoning replay per client Signed-off-by: Deepak Jain --- Cargo.lock | 1 + crates/libsy-llm-client/Cargo.toml | 1 + crates/libsy-llm-client/README.md | 4 + crates/libsy-llm-client/src/client.rs | 164 ++++++++++++++++++ crates/libsy-llm-client/src/lib.rs | 2 + .../src/responses_reasoning.rs | 130 ++++++++++++++ crates/switchyard-runner/src/config.rs | 90 ++++++++-- crates/switchyard-server/tests/server.rs | 49 ++++++ docs/reference/toml_schema.md | 9 + 9 files changed, 438 insertions(+), 12 deletions(-) create mode 100644 crates/libsy-llm-client/src/responses_reasoning.rs diff --git a/Cargo.lock b/Cargo.lock index b1cbce1be9..bcd6c200a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2358,6 +2358,7 @@ dependencies = [ "opentelemetry_sdk", "parking_lot", "reqwest", + "serde", "serde_json", "switchyard-libsy", "switchyard-protocol", diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.toml index 9a9ca6e0a7..a347ed7cc5 100644 --- a/crates/libsy-llm-client/Cargo.toml +++ b/crates/libsy-llm-client/Cargo.toml @@ -29,6 +29,7 @@ parking_lot.workspace = true http.workspace = true httpdate.workspace = true serde_json.workspace = true +serde.workspace = true tokio.workspace = true tracing.workspace = true tracing-opentelemetry.workspace = true diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md index 34ec8a989e..160a428e1e 100644 --- a/crates/libsy-llm-client/README.md +++ b/crates/libsy-llm-client/README.md @@ -252,6 +252,10 @@ fn build_multi_format_client( transport failures, timeouts, HTTP 408/429, and 5xx responses. Buffered body transport failures are retried; streaming body failures are not replayed after the response has been returned. +- `ModelConfig::with_responses_reasoning` controls Responses reasoning replay. + Every Responses model defaults to `PreserveEncrypted`: plaintext is removed, + encrypted provider state is retained. Set `Drop` explicitly for a backend + that cannot consume encrypted reasoning. Messages and tool history are retained. - `HttpBackendConfig::timeout` bounds one complete response, including retries, retry delays, and every stream read. Expiry returns `LlmClientError::Timeout`, either from the call or from the returned stream, which then ends. `None` leaves diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index f2d0301dd6..4cabf6d0e0 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -63,6 +63,7 @@ pub struct ModelConfig { model_name: ModelId, default_backend: Backend, other_backends: Option>, + responses_reasoning: crate::ResponsesReasoningPolicy, } impl ModelConfig { @@ -77,8 +78,16 @@ impl ModelConfig { model_name: model_name.into(), default_backend, other_backends, + responses_reasoning: crate::ResponsesReasoningPolicy::default(), } } + + /// Sets how Responses reasoning items are replayed to this model. + #[must_use] + pub fn with_responses_reasoning(mut self, policy: crate::ResponsesReasoningPolicy) -> Self { + self.responses_reasoning = policy; + self + } } /// A model-bearing provider operation outside the normal completion endpoint. @@ -260,6 +269,13 @@ impl TranslatingLlmClient { strip_anthropic_incompatible_fields(&mut body); strip_unsigned_thinking_blocks(&mut body); } + if matches!(backend, Backend::OpenAiResponses(_)) { + self.model_to_config + .get(model) + .map(|config| config.responses_reasoning) + .unwrap_or_default() + .normalize(&mut body); + } omit_configured_body_fields(&mut body, backend.omit_body_fields()); merge_extra_body(&mut body, backend.extra_body()); // After the merge on purpose: the effort override must win over both the caller's @@ -1446,6 +1462,13 @@ mod tests { )] } + fn local_responses_map(base_url: &str) -> Vec { + vec![ + ModelConfig::new("local", Backend::OpenAiResponses(config(base_url)), None) + .with_responses_reasoning(crate::ResponsesReasoningPolicy::Drop), + ] + } + fn chat_map_with_retries(base_url: &str, max_retries: u32) -> Vec { vec![ModelConfig::new( "gpt", @@ -2192,6 +2215,147 @@ mod tests { Ok(()) } + // Strict Responses upstreams retain encrypted state without replaying plaintext. + #[tokio::test] + async fn responses_requests_drop_unsigned_reasoning_items() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(|request: &wiremock::Request| { + let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); + let Some(input) = body.get("input").and_then(Value::as_array) else { + return false; + }; + let reasoning: Vec<&Value> = input + .iter() + .filter(|item| item.get("type").and_then(Value::as_str) == Some("reasoning")) + .collect(); + reasoning.len() == 1 + && reasoning[0] + .get("encrypted_content") + .and_then(Value::as_str) + == Some("encrypted") + && reasoning[0].get("content") == Some(&json!([])) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call") + }) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call_output") + }) + }) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", + "object": "response", + "model": "gpt", + "status": "completed", + "output": [{ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }))) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&responses_map(&format!("{}/v1", server.uri())))?; + let raw = json!({ + "model": "client-facing", + "input": [ + {"type": "message", "role": "user", "content": "inspect the repo"}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "private local reasoning"}], + "encrypted_content": "" + }, + {"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "must not be replayed"}], + "encrypted_content": "encrypted" + } + ] + }); + + client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await?; + Ok(()) + } + + // Encrypted hosted reasoning is opaque to a local Responses backend. Dropping + // it avoids llama.cpp rejecting a missing or empty `content` array while + // retaining the conversation and tool-call history it can consume. + #[tokio::test] + async fn local_responses_requests_drop_encrypted_reasoning_items() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(|request: &wiremock::Request| { + let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); + let Some(input) = body.get("input").and_then(Value::as_array) else { + return false; + }; + input + .iter() + .all(|item| item.get("type").and_then(Value::as_str) != Some("reasoning")) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call") + }) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call_output") + }) + }) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_local", + "object": "response", + "model": "local", + "status": "completed", + "output": [{ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }))) + .mount(&server) + .await; + + let client = + TranslatingLlmClient::new(&local_responses_map(&format!("{}/v1", server.uri())))?; + let raw = json!({ + "model": "client-facing", + "input": [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "inspect"}]}, + { + "type": "reasoning", + "encrypted_content": "opaque-provider-reasoning" + }, + {"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "continue"}]} + ] + }); + + client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("local")), + WireFormat::OpenAiResponses, + ) + .await?; + Ok(()) + } + // A router can serve earlier turns from an OpenAI target and later turns from // an Anthropic one, so the Anthropic leg must drop OpenAI-only fields the // caller keeps sending or the upstream rejects the whole request. diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs index d5f579c7d7..714f580637 100644 --- a/crates/libsy-llm-client/src/lib.rs +++ b/crates/libsy-llm-client/src/lib.rs @@ -23,6 +23,7 @@ pub mod metrics; mod observability; mod observation; pub mod raw; +mod responses_reasoning; pub mod run; pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig}; @@ -30,6 +31,7 @@ pub use client::{AuxiliaryOperation, ModelConfig, TranslatingLlmClient}; pub use error::{LlmClientError, Result}; pub use observation::{LlmCallObservation, RunObservation, RunObserver}; pub use raw::RawResponse; +pub use responses_reasoning::ResponsesReasoningPolicy; pub use run::{ClientRouter, decide, run}; pub use switchyard_translation::RawEventStream; diff --git a/crates/libsy-llm-client/src/responses_reasoning.rs b/crates/libsy-llm-client/src/responses_reasoning.rs new file mode 100644 index 0000000000..2c0e21c306 --- /dev/null +++ b/crates/libsy-llm-client/src/responses_reasoning.rs @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Target-specific replay policy for OpenAI Responses reasoning items. + +use serde::Deserialize; +use serde_json::Value; + +/// Controls which Responses reasoning items are replayed to an upstream. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ResponsesReasoningPolicy { + /// Preserve provider-encrypted reasoning but remove plaintext reasoning. + /// + /// This is the default for every Responses model, regardless of its URL. + #[default] + PreserveEncrypted, + /// Drop all reasoning items while preserving messages and tool-call history. + /// + /// Use this for local Responses-compatible servers that cannot consume + /// another provider's encrypted reasoning representation. + Drop, +} + +impl ResponsesReasoningPolicy { + /// Normalizes a Responses request body for this replay policy. + pub(crate) fn normalize(self, body: &mut Value) { + let Some(Value::Array(input)) = body.get_mut("input") else { + return; + }; + input.retain_mut(|item| self.normalize_item(item)); + } + + // Keep non-reasoning items; PreserveEncrypted retains only non-empty encrypted_content. + fn normalize_item(self, item: &mut Value) -> bool { + let Some(object) = item.as_object_mut() else { + return true; + }; + if object.get("type").and_then(Value::as_str) != Some("reasoning") { + return true; + } + + let signed = matches!( + object.get("encrypted_content").and_then(Value::as_str), + Some(encrypted_content) if !encrypted_content.is_empty() + ); + if self == Self::PreserveEncrypted && signed { + object.insert("content".to_string(), Value::Array(Vec::new())); + true + } else { + false + } + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn mixed_history() -> Value { + json!({ + "input": [ + {"type": "message", "role": "user", "content": []}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "plaintext"}], + "encrypted_content": "" + }, + {"type": "function_call", "call_id": "call_1"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "must be removed"}], + "encrypted_content": "encrypted" + } + ] + }) + } + + #[test] + fn preserve_encrypted_drops_unsigned_and_clears_plaintext() { + let mut body = mixed_history(); + ResponsesReasoningPolicy::PreserveEncrypted.normalize(&mut body); + + let input = body["input"].as_array().expect("input array"); + let reasoning: Vec<&Value> = input + .iter() + .filter(|item| item["type"] == "reasoning") + .collect(); + assert_eq!(reasoning.len(), 1); + assert_eq!(reasoning[0]["encrypted_content"], "encrypted"); + assert_eq!(reasoning[0]["content"], json!([])); + assert!(input.iter().any(|item| item["type"] == "function_call")); + assert!( + input + .iter() + .any(|item| item["type"] == "function_call_output") + ); + } + + #[test] + fn unsigned_forms_are_removed_and_non_reasoning_is_unchanged() { + let message = json!({"type": "message", "role": "user", "content": "continue"}); + let mut body = json!({"input": [ + {"type": "reasoning"}, + {"type": "reasoning", "encrypted_content": null}, + {"type": "reasoning", "encrypted_content": ""}, + message.clone() + ]}); + ResponsesReasoningPolicy::PreserveEncrypted.normalize(&mut body); + assert_eq!(body["input"], json!([message])); + } + + #[test] + fn drop_removes_all_reasoning_and_keeps_tool_history() { + let mut body = mixed_history(); + ResponsesReasoningPolicy::Drop.normalize(&mut body); + + let input = body["input"].as_array().expect("input array"); + assert!(input.iter().all(|item| item["type"] != "reasoning")); + assert!(input.iter().any(|item| item["type"] == "function_call")); + assert!( + input + .iter() + .any(|item| item["type"] == "function_call_output") + ); + } +} diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 05f03e2ce5..b326d4cac8 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Deserializer}; use serde_json::Value; use switchyard_llm_client::{ AuxiliaryOperation, Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, - TranslatingLlmClient, + ResponsesReasoningPolicy, TranslatingLlmClient, }; use switchyard_protocol::{Category, ModelId, RoutedLlmClient, WireFormat}; @@ -281,6 +281,13 @@ impl DeploymentConfig { for (name, client_config) in &self.llm_clients { validate_value("llm client name", name)?; + if client_config.responses_reasoning.is_some() + && !matches!(client_config.format, ClientFormat::OpenAiResponses) + { + return Err(RunnerError::configuration(format!( + "llm client {name} responses_reasoning is only valid for openai_responses" + ))); + } let backend = build_backend( name, client_config, @@ -319,17 +326,20 @@ impl DeploymentConfig { ))); } } - model_configs.push(ModelConfig::new( - target.id.clone(), - build_backend( - &target.llm_client, - client_config, - &target.extra_body, - &target.omit_body_fields, - target.reasoning_effort.clone(), - )?, - None, - )); + model_configs.push( + ModelConfig::new( + target.id.clone(), + build_backend( + &target.llm_client, + client_config, + &target.extra_body, + &target.omit_body_fields, + target.reasoning_effort.clone(), + )?, + None, + ) + .with_responses_reasoning(client_config.responses_reasoning.unwrap_or_default()), + ); } let mut clients = BTreeMap::new(); @@ -578,6 +588,7 @@ struct LlmClientConfig { max_retries: u32, /// Deadline in milliseconds for all attempts and the complete response. Unset is unbounded. timeout_ms: Option, + responses_reasoning: Option, } #[derive(Debug, Deserialize)] @@ -1752,6 +1763,61 @@ confidence_threshold = 0.5 Ok(()) } + #[test] + fn responses_reasoning_defaults_and_accepts_drop() -> RunnerResult<()> { + let default: DeploymentConfig = toml::from_str(VALID_CONFIG).map_err(|error| { + RunnerError::configuration(format!("failed to parse config: {error}")) + })?; + let responses = default + .llm_clients + .get("responses") + .ok_or_else(|| RunnerError::configuration("responses llm client is missing"))?; + assert_eq!(responses.responses_reasoning, None); + + let configured = VALID_CONFIG.replacen( + "[llm_clients.responses]\nformat = \"openai_responses\"\nbase_url = \"https://example.test/v1\"", + "[llm_clients.responses]\nformat = \"openai_responses\"\nbase_url = \"https://example.test/v1\"\nresponses_reasoning = \"drop\"", + 1, + ); + let config: DeploymentConfig = toml::from_str(&configured).map_err(|error| { + RunnerError::configuration(format!("failed to parse config: {error}")) + })?; + let responses = config + .llm_clients + .get("responses") + .ok_or_else(|| RunnerError::configuration("responses llm client is missing"))?; + assert_eq!( + responses.responses_reasoning, + Some(ResponsesReasoningPolicy::Drop) + ); + runner_from_toml(&configured)?; + Ok(()) + } + + #[test] + fn responses_reasoning_rejects_other_formats() { + let invalid = VALID_CONFIG.replacen( + "format = \"openai_chat\"", + "format = \"openai_chat\"\nresponses_reasoning = \"drop\"", + 1, + ); + assert!( + error_message(&invalid) + .contains("responses_reasoning is only valid for openai_responses") + ); + } + + #[test] + fn responses_reasoning_rejects_unreferenced_other_format() { + let invalid = format!( + "{VALID_CONFIG}\n[llm_clients.unused]\nformat = \"openai_chat\"\nbase_url = \"https://example.test/v1\"\nresponses_reasoning = \"drop\"\n" + ); + assert!( + error_message(&invalid) + .contains("responses_reasoning is only valid for openai_responses") + ); + } + #[test] fn rejects_headers_that_switchyard_sets() { let cases = [ diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index cd21d3c91e..e87b37c04b 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -5245,3 +5245,52 @@ async fn upstream_headers_forward_on_streaming_responses() -> TestResult { ); Ok(()) } + +#[tokio::test] +async fn configured_responses_reasoning_policy_reaches_upstream() -> TestResult { + let upstream = MockUpstream::start().await?; + let base_url = upstream.base_url.replace("/v1", "/buffered"); + for (setting, expected_reasoning) in [("", 1), ("responses_reasoning = \"drop\"", 0)] { + let config = format!( + r#" +schema_version = 1 +[llm_clients.upstream] +format = "openai_responses" +base_url = "{base_url}" +{setting} +[targets] +answer = {{ id = "model/efficient", llm_client = "upstream" }} +[routes.route] +id = "route" +type = "passthrough" +target = "answer" +"# + ); + let app = build_switchyard_router(load_test_config(&config)?); + let response = send(&app, "POST", "/v1/responses", Some(json!({ + "model": "route", "input": [ + {"type": "message", "role": "user", "content": "continue"}, + {"type": "reasoning", "encrypted_content": "opaque", "summary": []}, + {"type": "reasoning", "content": [{"type": "reasoning_text", "text": "local"}], "summary": []} + ] + }))).await?; + assert_eq!(response.status, StatusCode::OK, "{}", response.json()?); + let calls = upstream.calls.lock().await; + let input = calls.last().expect("upstream request")["input"] + .as_array() + .expect("input array"); + assert_eq!( + input + .iter() + .filter(|item| item["type"] == "reasoning") + .count(), + expected_reasoning + ); + assert!( + input + .iter() + .any(|item| item["type"] == "message" && item["role"] == "user") + ); + } + Ok(()) +} diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 7fc96b709d..f0f4201935 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -53,9 +53,18 @@ route reaches no upstream. A file without a `[targets]` table is rejected with | `api_key_env` | No | unset | Name of the environment variable holding the key. Omit to send no authentication. | | `forward_auth` | No | `false` | Forward the caller's provider credential and application headers. All backends reachable through the route must use the same provider. | | `extra_headers` | No | `{}` | Custom HTTP headers sent to the model server. Set credentials with `api_key_env` or `forward_auth`; the server rejects headers owned by the selected auth mode. Header names are case-insensitive. | +| `responses_reasoning` | No | `"preserve_encrypted"` | Responses reasoning replay: `"preserve_encrypted"` or `"drop"`. Only valid for `openai_responses`. | | `max_retries` | No | `2` | Retry budget, `0`–`10`. | | `timeout_ms` | No | unset | Deadline in milliseconds for all attempts, retry delays, and the complete response, including stream reads. Must be at least `1`. Unset leaves the wait unbounded. | +For `openai_responses`, `responses_reasoning` controls replay of reasoning +history. Every client defaults to `"preserve_encrypted"`: reasoning items with +non-empty `encrypted_content` are retained with empty `content`, and unsigned +reasoning is removed. Set `responses_reasoning = "drop"` explicitly for a backend +that cannot consume encrypted provider state. Both modes keep messages, tool +calls and tool results. The setting is rejected on other client formats, even +when no target uses the client. It is not inferred from the model name or URL. + The TOML never contains the secret itself. `api_key_env` names a variable that must exist and be non-empty when the server loads. From 1f793065a1dd45a27f0f9ac960fbc959a69fa72b Mon Sep 17 00:00:00 2001 From: Deepak Jain Date: Sun, 27 Sep 2026 23:20:22 -0700 Subject: [PATCH 2/3] fix(client): normalize reasoning after target input defaults Signed-off-by: Deepak Jain --- crates/libsy-llm-client/src/client.rs | 60 ++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 4cabf6d0e0..8521395bfc 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -269,6 +269,8 @@ impl TranslatingLlmClient { strip_anthropic_incompatible_fields(&mut body); strip_unsigned_thinking_blocks(&mut body); } + omit_configured_body_fields(&mut body, backend.omit_body_fields()); + merge_extra_body(&mut body, backend.extra_body()); if matches!(backend, Backend::OpenAiResponses(_)) { self.model_to_config .get(model) @@ -276,8 +278,6 @@ impl TranslatingLlmClient { .unwrap_or_default() .normalize(&mut body); } - omit_configured_body_fields(&mut body, backend.omit_body_fields()); - merge_extra_body(&mut body, backend.extra_body()); // After the merge on purpose: the effort override must win over both the caller's // value and any `reasoning` default a target set through `extra_body`. apply_reasoning_effort(&mut body, backend); @@ -2215,6 +2215,62 @@ mod tests { Ok(()) } + #[tokio::test] + async fn responses_policy_normalizes_input_replaced_by_target_defaults() + -> std::result::Result<(), Box> { + for (policy, expected_reasoning) in [ + (crate::ResponsesReasoningPolicy::PreserveEncrypted, 1), + (crate::ResponsesReasoningPolicy::Drop, 0), + ] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", "object": "response", "model": "gpt", + "status": "completed", "output": [], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }))) + .expect(1) + .mount(&server) + .await; + let mut backend = config(&format!("{}/v1", server.uri())); + backend.omit_body_fields.insert("input".to_string()); + backend.extra_body.insert("input".to_string(), json!([ + {"type": "message", "role": "user", "content": "replacement"}, + {"type": "reasoning", "encrypted_content": "opaque", "content": [{"type": "reasoning_text", "text": "private"}]}, + {"type": "reasoning", "encrypted_content": null, "content": [{"type": "reasoning_text", "text": "local"}]} + ])); + let client = TranslatingLlmClient::new(&[ModelConfig::new( + "gpt", + Backend::OpenAiResponses(backend), + None, + ) + .with_responses_reasoning(policy)])?; + client + .call_rewrite_model_raw( + json!({"model": "gpt", "input": "original"}), + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await?; + let requests = server.received_requests().await.expect("recorded requests"); + let body: Value = serde_json::from_slice(&requests[0].body)?; + let input = body["input"].as_array().expect("replacement input"); + assert_eq!(input[0]["content"], "replacement"); + let reasoning: Vec<_> = input + .iter() + .filter(|item| item["type"] == "reasoning") + .collect(); + assert_eq!(reasoning.len(), expected_reasoning); + for item in reasoning { + assert_eq!(item["encrypted_content"], "opaque"); + assert_eq!(item["content"], json!([])); + } + } + Ok(()) + } + // Strict Responses upstreams retain encrypted state without replaying plaintext. #[tokio::test] async fn responses_requests_drop_unsigned_reasoning_items() From 8ab15b2d850899beaaaba096bd5e7dd18ea13b2a Mon Sep 17 00:00:00 2001 From: Deepak Jain Date: Wed, 7 Oct 2026 08:47:54 -0700 Subject: [PATCH 3/3] fix(client): preserve stored Responses reasoning references Signed-off-by: Deepak Jain --- crates/libsy-llm-client/README.md | 4 +- crates/libsy-llm-client/src/client.rs | 259 +++++------------- .../src/responses_reasoning.rs | 90 +----- crates/switchyard-server/tests/server.rs | 3 +- docs/reference/toml_schema.md | 6 +- 5 files changed, 83 insertions(+), 279 deletions(-) diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md index 6db3bc7c23..21981be9bf 100644 --- a/crates/libsy-llm-client/README.md +++ b/crates/libsy-llm-client/README.md @@ -265,8 +265,8 @@ fn build_multi_format_client( the response has been returned. - `ModelConfig::with_responses_reasoning` controls Responses reasoning replay. Every Responses model defaults to `PreserveEncrypted`: plaintext is removed, - encrypted provider state is retained. Set `Drop` explicitly for a backend - that cannot consume encrypted reasoning. Messages and tool history are retained. + encrypted provider state and stored reasoning IDs are retained. Set `Drop` for + a backend that cannot consume provider reasoning. Messages and tool history are retained. - `HttpBackendConfig::failure_cooldown` skips completion calls to a backend briefly after an exhausted transient failure. Zero disables it. State is shared per model within the client. Calls resume when the cooldown expires. Deployment TOML defaults diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 71df55ac2c..8e6d27e226 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -1504,9 +1504,9 @@ mod tests { )] } - fn local_responses_map(base_url: &str) -> Vec { + fn responses_map_with_drop_policy(base_url: &str) -> Vec { vec![ - ModelConfig::new("local", Backend::OpenAiResponses(config(base_url)), None) + ModelConfig::new("gpt", Backend::OpenAiResponses(config(base_url)), None) .with_responses_reasoning(crate::ResponsesReasoningPolicy::Drop), ] } @@ -2258,199 +2258,74 @@ mod tests { } #[tokio::test] - async fn responses_policy_normalizes_input_replaced_by_target_defaults() + async fn responses_reasoning_policy_preserves_replay_history() -> std::result::Result<(), Box> { - for (policy, expected_reasoning) in [ - (crate::ResponsesReasoningPolicy::PreserveEncrypted, 1), - (crate::ResponsesReasoningPolicy::Drop, 0), - ] { - let server = MockServer::start().await; - Mock::given(method("POST")) - .and(path("/v1/responses")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "id": "resp_1", "object": "response", "model": "gpt", - "status": "completed", "output": [], - "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} - }))) - .expect(1) - .mount(&server) - .await; - let mut backend = config(&format!("{}/v1", server.uri())); - backend.omit_body_fields.insert("input".to_string()); - backend.extra_body.insert("input".to_string(), json!([ - {"type": "message", "role": "user", "content": "replacement"}, - {"type": "reasoning", "encrypted_content": "opaque", "content": [{"type": "reasoning_text", "text": "private"}]}, - {"type": "reasoning", "encrypted_content": null, "content": [{"type": "reasoning_text", "text": "local"}]} - ])); - let client = TranslatingLlmClient::new(&[ModelConfig::new( - "gpt", - Backend::OpenAiResponses(backend), - None, - ) - .with_responses_reasoning(policy)])?; - client - .call_rewrite_model_raw( - json!({"model": "gpt", "input": "original"}), - None, - Some(&ModelId::from("gpt")), - WireFormat::OpenAiResponses, - ) - .await?; - let requests = server.received_requests().await.expect("recorded requests"); - let body: Value = serde_json::from_slice(&requests[0].body)?; - let input = body["input"].as_array().expect("replacement input"); - assert_eq!(input[0]["content"], "replacement"); - let reasoning: Vec<_> = input - .iter() - .filter(|item| item["type"] == "reasoning") - .collect(); - assert_eq!(reasoning.len(), expected_reasoning); - for item in reasoning { - assert_eq!(item["encrypted_content"], "opaque"); - assert_eq!(item["content"], json!([])); - } - } - Ok(()) - } - - // Strict Responses upstreams retain encrypted state without replaying plaintext. - #[tokio::test] - async fn responses_requests_drop_unsigned_reasoning_items() - -> std::result::Result<(), Box> { - let server = MockServer::start().await; - Mock::given(method("POST")) - .and(path("/v1/responses")) - .and(|request: &wiremock::Request| { - let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); - let Some(input) = body.get("input").and_then(Value::as_array) else { - return false; + let history = json!([ + {"type": "message", "role": "user", "content": "continue"}, + {"type": "reasoning", "encrypted_content": "opaque", "content": [{"type": "reasoning_text", "text": "private"}]}, + {"type": "reasoning", "id": "rs_stored", "summary": []}, + {"type": "reasoning", "encrypted_content": null}, + {"type": "reasoning", "encrypted_content": ""}, + {"type": "reasoning"}, + {"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"} + ]); + for is_drop in [false, true] { + for has_input_override in [false, true] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", "object": "response", "model": "gpt", + "status": "completed", "output": [], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }))) + .expect(1) + .mount(&server) + .await; + let base_url = format!("{}/v1", server.uri()); + let mut models = if is_drop { + responses_map_with_drop_policy(&base_url) + } else { + responses_map(&base_url) }; - let reasoning: Vec<&Value> = input - .iter() - .filter(|item| item.get("type").and_then(Value::as_str) == Some("reasoning")) - .collect(); - reasoning.len() == 1 - && reasoning[0] - .get("encrypted_content") - .and_then(Value::as_str) - == Some("encrypted") - && reasoning[0].get("content") == Some(&json!([])) - && input.iter().any(|item| { - item.get("type").and_then(Value::as_str) == Some("function_call") - }) - && input.iter().any(|item| { - item.get("type").and_then(Value::as_str) == Some("function_call_output") - }) - }) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "id": "resp_1", - "object": "response", - "model": "gpt", - "status": "completed", - "output": [{ - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "ok"}] - }], - "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} - }))) - .mount(&server) - .await; - - let client = TranslatingLlmClient::new(&responses_map(&format!("{}/v1", server.uri())))?; - let raw = json!({ - "model": "client-facing", - "input": [ - {"type": "message", "role": "user", "content": "inspect the repo"}, - { - "type": "reasoning", - "content": [{"type": "reasoning_text", "text": "private local reasoning"}], - "encrypted_content": "" - }, - {"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": "{}"}, - {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, - { - "type": "reasoning", - "content": [{"type": "reasoning_text", "text": "must not be replayed"}], - "encrypted_content": "encrypted" + if has_input_override { + let Backend::OpenAiResponses(backend) = &mut models[0].default_backend else { + return Err("expected Responses backend".into()); + }; + backend.omit_body_fields.insert("input".to_string()); + backend + .extra_body + .insert("input".to_string(), history.clone()); } - ] - }); - - client - .call_rewrite_model_raw( - raw, - None, - Some(&ModelId::from("gpt")), - WireFormat::OpenAiResponses, - ) - .await?; - Ok(()) - } - - // Encrypted hosted reasoning is opaque to a local Responses backend. Dropping - // it avoids llama.cpp rejecting a missing or empty `content` array while - // retaining the conversation and tool-call history it can consume. - #[tokio::test] - async fn local_responses_requests_drop_encrypted_reasoning_items() - -> std::result::Result<(), Box> { - let server = MockServer::start().await; - Mock::given(method("POST")) - .and(path("/v1/responses")) - .and(|request: &wiremock::Request| { - let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); - let Some(input) = body.get("input").and_then(Value::as_array) else { - return false; - }; - input - .iter() - .all(|item| item.get("type").and_then(Value::as_str) != Some("reasoning")) - && input.iter().any(|item| { - item.get("type").and_then(Value::as_str) == Some("function_call") - }) - && input.iter().any(|item| { - item.get("type").and_then(Value::as_str) == Some("function_call_output") - }) - }) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "id": "resp_local", - "object": "response", - "model": "local", - "status": "completed", - "output": [{ - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "ok"}] - }], - "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} - }))) - .mount(&server) - .await; - - let client = - TranslatingLlmClient::new(&local_responses_map(&format!("{}/v1", server.uri())))?; - let raw = json!({ - "model": "client-facing", - "input": [ - {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "inspect"}]}, - { - "type": "reasoning", - "encrypted_content": "opaque-provider-reasoning" - }, - {"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": "{}"}, - {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, - {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "continue"}]} - ] - }); - - client - .call_rewrite_model_raw( - raw, - None, - Some(&ModelId::from("local")), - WireFormat::OpenAiResponses, - ) - .await?; + let client = TranslatingLlmClient::new(&models)?; + client + .call_rewrite_model_raw( + json!({"model": "gpt", "input": history, "store": true}), + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await?; + let requests = server.received_requests().await.ok_or("missing requests")?; + let body: Value = serde_json::from_slice(&requests[0].body)?; + let mut expected = vec![history[0].clone()]; + if !is_drop { + let mut encrypted = history[1].clone(); + encrypted["content"] = json!([]); + expected.push(encrypted); + let mut stored = history[2].clone(); + stored["content"] = json!([]); + expected.push(stored); + } + expected.extend([history[6].clone(), history[7].clone()]); + assert_eq!( + body["input"], + json!(expected), + "drop={is_drop}, replace={has_input_override}" + ); + } + } Ok(()) } diff --git a/crates/libsy-llm-client/src/responses_reasoning.rs b/crates/libsy-llm-client/src/responses_reasoning.rs index 2c0e21c306..1c7e445604 100644 --- a/crates/libsy-llm-client/src/responses_reasoning.rs +++ b/crates/libsy-llm-client/src/responses_reasoning.rs @@ -7,10 +7,10 @@ use serde::Deserialize; use serde_json::Value; /// Controls which Responses reasoning items are replayed to an upstream. -#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq)] #[serde(rename_all = "snake_case")] pub enum ResponsesReasoningPolicy { - /// Preserve provider-encrypted reasoning but remove plaintext reasoning. + /// Preserve encrypted reasoning and stored reasoning IDs; remove plaintext. /// /// This is the default for every Responses model, regardless of its URL. #[default] @@ -31,7 +31,7 @@ impl ResponsesReasoningPolicy { input.retain_mut(|item| self.normalize_item(item)); } - // Keep non-reasoning items; PreserveEncrypted retains only non-empty encrypted_content. + // Stored reasoning can be replayed by ID without encrypted content. fn normalize_item(self, item: &mut Value) -> bool { let Some(object) = item.as_object_mut() else { return true; @@ -40,11 +40,15 @@ impl ResponsesReasoningPolicy { return true; } - let signed = matches!( + let has_encrypted_content = matches!( object.get("encrypted_content").and_then(Value::as_str), Some(encrypted_content) if !encrypted_content.is_empty() ); - if self == Self::PreserveEncrypted && signed { + let has_stored_id = object + .get("id") + .and_then(Value::as_str) + .is_some_and(|id| !id.is_empty()); + if self == Self::PreserveEncrypted && (has_encrypted_content || has_stored_id) { object.insert("content".to_string(), Value::Array(Vec::new())); true } else { @@ -52,79 +56,3 @@ impl ResponsesReasoningPolicy { } } } - -#[cfg(test)] -mod tests { - use serde_json::json; - - use super::*; - - fn mixed_history() -> Value { - json!({ - "input": [ - {"type": "message", "role": "user", "content": []}, - { - "type": "reasoning", - "content": [{"type": "reasoning_text", "text": "plaintext"}], - "encrypted_content": "" - }, - {"type": "function_call", "call_id": "call_1"}, - {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, - { - "type": "reasoning", - "content": [{"type": "reasoning_text", "text": "must be removed"}], - "encrypted_content": "encrypted" - } - ] - }) - } - - #[test] - fn preserve_encrypted_drops_unsigned_and_clears_plaintext() { - let mut body = mixed_history(); - ResponsesReasoningPolicy::PreserveEncrypted.normalize(&mut body); - - let input = body["input"].as_array().expect("input array"); - let reasoning: Vec<&Value> = input - .iter() - .filter(|item| item["type"] == "reasoning") - .collect(); - assert_eq!(reasoning.len(), 1); - assert_eq!(reasoning[0]["encrypted_content"], "encrypted"); - assert_eq!(reasoning[0]["content"], json!([])); - assert!(input.iter().any(|item| item["type"] == "function_call")); - assert!( - input - .iter() - .any(|item| item["type"] == "function_call_output") - ); - } - - #[test] - fn unsigned_forms_are_removed_and_non_reasoning_is_unchanged() { - let message = json!({"type": "message", "role": "user", "content": "continue"}); - let mut body = json!({"input": [ - {"type": "reasoning"}, - {"type": "reasoning", "encrypted_content": null}, - {"type": "reasoning", "encrypted_content": ""}, - message.clone() - ]}); - ResponsesReasoningPolicy::PreserveEncrypted.normalize(&mut body); - assert_eq!(body["input"], json!([message])); - } - - #[test] - fn drop_removes_all_reasoning_and_keeps_tool_history() { - let mut body = mixed_history(); - ResponsesReasoningPolicy::Drop.normalize(&mut body); - - let input = body["input"].as_array().expect("input array"); - assert!(input.iter().all(|item| item["type"] != "reasoning")); - assert!(input.iter().any(|item| item["type"] == "function_call")); - assert!( - input - .iter() - .any(|item| item["type"] == "function_call_output") - ); - } -} diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index f7d97a9407..1aee575372 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -5354,7 +5354,7 @@ async fn upstream_headers_forward_on_streaming_responses() -> TestResult { async fn configured_responses_reasoning_policy_reaches_upstream() -> TestResult { let upstream = MockUpstream::start().await?; let base_url = upstream.base_url.replace("/v1", "/buffered"); - for (setting, expected_reasoning) in [("", 1), ("responses_reasoning = \"drop\"", 0)] { + for (setting, expected_reasoning) in [("", 2), ("responses_reasoning = \"drop\"", 0)] { let config = format!( r#" schema_version = 1 @@ -5375,6 +5375,7 @@ target = "answer" "model": "route", "input": [ {"type": "message", "role": "user", "content": "continue"}, {"type": "reasoning", "encrypted_content": "opaque", "summary": []}, + {"type": "reasoning", "id": "rs_stored", "summary": []}, {"type": "reasoning", "content": [{"type": "reasoning_text", "text": "local"}], "summary": []} ] }))).await?; diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index fac8e87d58..db580187d2 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -60,9 +60,9 @@ route reaches no upstream. A file without a `[targets]` table is rejected with For `openai_responses`, `responses_reasoning` controls replay of reasoning history. Every client defaults to `"preserve_encrypted"`: reasoning items with -non-empty `encrypted_content` are retained with empty `content`, and unsigned -reasoning is removed. Set `responses_reasoning = "drop"` explicitly for a backend -that cannot consume encrypted provider state. Both modes keep messages, tool +non-empty `encrypted_content` or a stored reasoning `id` are retained with empty +`content`. Reasoning without either is removed. Set `responses_reasoning = "drop"` explicitly for a backend +that cannot consume provider reasoning. Both modes keep messages, tool calls and tool results. The setting is rejected on other client formats, even when no target uses the client. It is not inferred from the model name or URL.