From d6976063cb10c0d8fc7459226035e20d27a6264f Mon Sep 17 00:00:00 2001 From: ChethanUK Date: Fri, 18 Sep 2026 07:34:37 +0200 Subject: [PATCH 1/3] feat(translation): let targets choose the Chat reasoning field name Signed-off-by: ChethanUK --- crates/libsy-llm-client/README.md | 1 + crates/libsy-llm-client/src/backend.rs | 10 ++ crates/libsy-llm-client/src/client.rs | 85 ++++++++++++++- crates/libsy-llm-client/src/lib.rs | 2 +- crates/libsy-llm-client/src/run.rs | 4 + crates/switchyard-runner/src/config.rs | 101 ++++++++++++++++-- crates/switchyard-server/tests/server.rs | 1 + .../src/codecs/openai_chat/buffered.rs | 27 +++-- crates/switchyard-translation/src/helpers.rs | 12 ++- crates/switchyard-translation/src/policy.rs | 27 +++++ .../tests/request_translation.rs | 53 ++++++++- docs/reference/toml_schema.md | 1 + 12 files changed, 304 insertions(+), 20 deletions(-) diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md index 34ec8a989e..9ee4bbd5ef 100644 --- a/crates/libsy-llm-client/README.md +++ b/crates/libsy-llm-client/README.md @@ -72,6 +72,7 @@ fn build_client() -> switchyard_llm_client::Result { extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), reasoning_effort: None, + reasoning_format: Default::default(), max_retries: 2, timeout: None, }; diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs index 064b135461..312e847b64 100644 --- a/crates/libsy-llm-client/src/backend.rs +++ b/crates/libsy-llm-client/src/backend.rs @@ -13,6 +13,7 @@ use reqwest::RequestBuilder; use reqwest::header::{HeaderName, HeaderValue}; use serde_json::Value; use switchyard_protocol::{Metadata, WireFormat}; +use switchyard_translation::ReasoningFormat; use crate::error::{LlmClientError, Result, is_overflow_body}; @@ -70,6 +71,8 @@ pub struct HttpBackendConfig { /// sent. Responses carries it as `reasoning.effort`, Chat Completions as `reasoning_effort`; /// Anthropic has no equivalent and rejects the setting at configuration time. pub reasoning_effort: Option, + /// Field name used for assistant reasoning replayed to an OpenAI Chat backend. + pub reasoning_format: ReasoningFormat, /// Additional attempts after the initial upstream request. pub max_retries: u32, /// Deadline for one complete response, including retries, retry delays, and stream reads. @@ -87,6 +90,7 @@ impl fmt::Debug for HttpBackendConfig { .field("extra_body_keys", &self.extra_body.keys()) .field("omit_body_fields", &self.omit_body_fields) .field("reasoning_effort", &self.reasoning_effort) + .field("reasoning_format", &self.reasoning_format) .field("max_retries", &self.max_retries) .field("timeout", &self.timeout) .finish() @@ -314,6 +318,11 @@ impl Backend { self.config().reasoning_effort.as_deref() } + /// Field name used for assistant reasoning replayed to this backend. + pub fn reasoning_format(&self) -> ReasoningFormat { + self.config().reasoning_format + } + /// Additional attempts allowed after the initial request. pub fn max_retries(&self) -> u32 { self.config().max_retries @@ -430,6 +439,7 @@ mod tests { extra_body: BTreeMap::new(), omit_body_fields: BTreeSet::new(), reasoning_effort: None, + reasoning_format: Default::default(), max_retries: 0, timeout: None, } diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index f2d0301dd6..567c1e43ef 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -19,8 +19,10 @@ use switchyard_protocol::{ ModelId, Request, Response, RoutedLlmClient, }; use switchyard_translation::{ - TranslationError, WireFormat, decode_aggregated_response, decode_request, decode_stream, - encode_aggregated_response_with_extensions, encode_request, encode_stream_with_extensions, + TargetCapabilities, TranslationError, TranslationPolicy, WireFormat, + decode_aggregated_response, decode_request, decode_stream, + encode_aggregated_response_with_extensions, encode_request_with_policy, + encode_stream_with_extensions, }; use tracing::Instrument; @@ -245,7 +247,14 @@ impl TranslatingLlmClient { model: &ModelId, endpoint: UpstreamEndpoint, ) -> Result { - let mut body = encode_request(&llm_request, wire_format) + let policy = TranslationPolicy { + target_capabilities: TargetCapabilities { + reasoning_format: backend.reasoning_format(), + ..TargetCapabilities::default() + }, + ..TranslationPolicy::default() + }; + let mut body = encode_request_with_policy(&llm_request, wire_format, &policy) .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?; // `encode_request` round-trips a preserved same-format body verbatim, // which keeps the caller's original `model`; force the resolved model so @@ -1295,6 +1304,7 @@ mod tests { use wiremock::{Mock, MockServer, ResponseTemplate}; use super::*; + use crate::ReasoningFormat; use crate::backend::HttpBackendConfig; fn config(base_url: &str) -> HttpBackendConfig { @@ -1306,6 +1316,7 @@ mod tests { extra_body: BTreeMap::new(), omit_body_fields: BTreeSet::new(), reasoning_effort: None, + reasoning_format: Default::default(), max_retries: 0, timeout: None, } @@ -1428,6 +1439,12 @@ mod tests { vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)] } + fn chat_map_with_reasoning_format(base_url: &str, format: ReasoningFormat) -> Vec { + let mut backend = config(base_url); + backend.reasoning_format = format; + vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)] + } + fn responses_map_with_effort(base_url: &str, effort: &str) -> Vec { let mut backend = config(base_url); backend.reasoning_effort = Some(effort.to_string()); @@ -1944,6 +1961,68 @@ mod tests { /// A configured reasoning effort replaces the caller's value on both OpenAI wire formats, /// which `extra_body` (defaults only) cannot do. + /// A request rebuilt for an OpenAI Chat backend sends assistant reasoning under + /// the backend's configured field name, and never under both names. + #[tokio::test] + async fn rebuilt_chat_request_uses_the_backends_reasoning_field() + -> std::result::Result<(), Box> { + for (format, expected, absent) in [ + (ReasoningFormat::OpenAi, "reasoning", "reasoning_content"), + (ReasoningFormat::DeepSeek, "reasoning_content", "reasoning"), + ] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "1", + "model": "gpt", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop" + }], + "usage": {} + }))) + .mount(&server) + .await; + let client = TranslatingLlmClient::new(&chat_map_with_reasoning_format( + &format!("{}/v1", server.uri()), + format, + ))?; + // A Responses caller forces the Chat body to be rebuilt from the IR. + client + .call_rewrite_model_raw( + json!({ + "model": "client-facing", + "input": [ + {"type": "message", "role": "user", "content": "hi"}, + {"type": "reasoning", "content": [{"type": "reasoning_text", "text": "thinking..."}], "summary": []}, + {"type": "message", "role": "assistant", "content": "hello"}, + {"type": "message", "role": "user", "content": "next"} + ] + }), + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await?; + let received = server + .received_requests() + .await + .ok_or("request recording should be enabled")?; + let received = received.first().ok_or("expected one upstream request")?; + let body: Value = serde_json::from_slice(&received.body)?; + let assistant = &body["messages"][1]; + assert_eq!(assistant["role"], "assistant", "{body}"); + assert_eq!(assistant[expected], "thinking...", "{format:?}: {body}"); + assert!( + assistant.get(absent).is_none(), + "{format:?} also sent {absent}: {body}" + ); + } + Ok(()) + } + #[tokio::test] async fn reasoning_effort_override_replaces_the_callers_effort() -> std::result::Result<(), Box> { diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs index d5f579c7d7..0569f97058 100644 --- a/crates/libsy-llm-client/src/lib.rs +++ b/crates/libsy-llm-client/src/lib.rs @@ -31,7 +31,7 @@ pub use error::{LlmClientError, Result}; pub use observation::{LlmCallObservation, RunObservation, RunObserver}; pub use raw::RawResponse; pub use run::{ClientRouter, decide, run}; -pub use switchyard_translation::RawEventStream; +pub use switchyard_translation::{RawEventStream, ReasoningFormat}; /// Registers process-wide compatibility gauges with the global meter provider. pub fn initialize_metrics() { diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index 691e169cf6..6eec94989b 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -1212,6 +1212,7 @@ mod tests { extra_body: BTreeMap::from([("store".to_string(), json!(store))]), omit_body_fields: BTreeSet::new(), reasoning_effort: None, + reasoning_format: Default::default(), max_retries: 0, timeout: None, }; @@ -1358,6 +1359,7 @@ mod tests { extra_body: BTreeMap::new(), omit_body_fields: BTreeSet::new(), reasoning_effort: None, + reasoning_format: Default::default(), max_retries: 0, timeout: None, }), @@ -2030,6 +2032,7 @@ mod tests { extra_body: BTreeMap::new(), omit_body_fields: BTreeSet::new(), reasoning_effort: None, + reasoning_format: Default::default(), max_retries: 2, timeout: None, }) @@ -2129,6 +2132,7 @@ mod tests { extra_body: BTreeMap::new(), omit_body_fields: BTreeSet::new(), reasoning_effort: None, + reasoning_format: Default::default(), max_retries: 0, timeout: None, }) diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 05f03e2ce5..5e8f2b0a84 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Deserializer}; use serde_json::Value; use switchyard_llm_client::{ AuxiliaryOperation, Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, - TranslatingLlmClient, + ReasoningFormat, TranslatingLlmClient, }; use switchyard_protocol::{Category, ModelId, RoutedLlmClient, WireFormat}; @@ -179,7 +179,7 @@ impl DeploymentConfig { // The LLM client keeps one backend per model id, so two targets naming the same model on // the same client share it. That is harmless when their request settings agree (an alias // for a different system prompt, say) and silently wrong when they do not: the second - // target's reasoning_effort or extra_body would never reach the wire. + // target's reasoning_effort, reasoning_format or extra_body would never reach the wire. let mut seen_client_model_ids: HashMap<(&str, &str), (&String, &TargetConfig)> = HashMap::new(); for (target_name, target) in &self.targets { @@ -192,11 +192,13 @@ impl DeploymentConfig { std::collections::hash_map::Entry::Occupied(slot) => { let (first_name, first) = slot.get(); if first.reasoning_effort != target.reasoning_effort + || first.reasoning_format.unwrap_or_default() + != target.reasoning_format.unwrap_or_default() || first.extra_body != target.extra_body || first.omit_body_fields != target.omit_body_fields { return Err(RunnerError::configuration(format!( - "targets {first_name} and {target_name} both name model {} on llm client {} but with different reasoning_effort, extra_body, or omit_body_fields; one target per model id is kept, so give each its own model id or llm client", + "targets {first_name} and {target_name} both name model {} on llm client {} but with different reasoning_effort, reasoning_format, extra_body, or omit_body_fields; one target per model id is kept, so give each its own model id or llm client", target.id, target.llm_client ))); } @@ -287,6 +289,7 @@ impl DeploymentConfig { &BTreeMap::new(), &BTreeSet::new(), None, + ReasoningFormat::default(), )?; let (Backend::OpenAiChat(config) | Backend::OpenAiResponses(config) @@ -319,6 +322,13 @@ impl DeploymentConfig { ))); } } + if target.reasoning_format.is_some() + && !matches!(client_config.format, ClientFormat::OpenAiChat) + { + return Err(RunnerError::configuration(format!( + "target {target_name} reasoning_format is only supported on openai_chat clients" + ))); + } model_configs.push(ModelConfig::new( target.id.clone(), build_backend( @@ -327,6 +337,7 @@ impl DeploymentConfig { &target.extra_body, &target.omit_body_fields, target.reasoning_effort.clone(), + target.reasoning_format.unwrap_or_default(), )?, None, )); @@ -595,6 +606,9 @@ struct TargetConfig { /// Reasoning effort forced on every request to this target, replacing the caller's value. /// Only meaningful on `openai_chat` and `openai_responses` clients. reasoning_effort: Option, + /// Field name for assistant reasoning replayed to this target. Only meaningful on + /// `openai_chat` clients; unset sends `reasoning`. + reasoning_format: Option, } #[derive(Clone, Copy, Debug, Deserialize)] @@ -653,6 +667,7 @@ fn build_backend( extra_body: &BTreeMap, omit_body_fields: &BTreeSet, reasoning_effort: Option, + reasoning_format: ReasoningFormat, ) -> RunnerResult { if config.max_retries > MAX_CONFIGURED_RETRIES { return Err(RunnerError::configuration(format!( @@ -699,6 +714,7 @@ fn build_backend( extra_body: extra_body.clone(), omit_body_fields: omit_body_fields.clone(), reasoning_effort, + reasoning_format, max_retries: config.max_retries, timeout: config.timeout_ms.map(Duration::from_millis), }; @@ -1196,6 +1212,69 @@ new = ["send_message"] Ok(()) } + #[test] + fn a_target_reasoning_format_parses_and_is_rejected_where_unsupported() -> RunnerResult<()> { + let chat = "[targets.classifier]\nid = \"classifier/model\"\nllm_client = \"primary\""; + let responses = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\""; + let anthropic = "[targets.weak]\nid = \"weak/model\"\nllm_client = \"anthropic\""; + assert!( + [chat, responses, anthropic] + .iter() + .all(|t| VALID_CONFIG.contains(t)) + ); + + for format in ["openai", "deepseek"] { + let set = + VALID_CONFIG.replace(chat, &format!("{chat}\nreasoning_format = \"{format}\"")); + runner_from_toml(&set)?; + } + + let unknown = VALID_CONFIG.replace( + chat, + &format!("{chat}\nreasoning_format = \"reasoning_content\""), + ); + assert!( + error_message(&unknown).contains("unknown variant"), + "{}", + error_message(&unknown) + ); + + // Only the OpenAI Chat encoder reads the setting, so elsewhere it would do nothing. + for target in [responses, anthropic] { + let set = VALID_CONFIG.replace( + target, + &format!("{target}\nreasoning_format = \"deepseek\""), + ); + assert!( + error_message(&set) + .contains("reasoning_format is only supported on openai_chat clients"), + "{}", + error_message(&set) + ); + } + + // Two targets on one model id share a backend, so they must agree on the field name. + let alias = |format: &str| { + VALID_CONFIG.replace( + chat, + &format!( + "{chat}\n\n[targets.classifier_alias]\nid = \"classifier/model\"\nllm_client = \"primary\"{format}" + ), + ) + }; + let conflicting = alias("\nreasoning_format = \"deepseek\""); + assert!( + error_message(&conflicting).contains( + "different reasoning_effort, reasoning_format, extra_body, or omit_body_fields" + ), + "{}", + error_message(&conflicting) + ); + // Unset and "openai" send the same field, so they do not conflict. + runner_from_toml(&alias("\nreasoning_format = \"openai\""))?; + Ok(()) + } + #[test] fn duplicate_targets_with_conflicting_settings_are_rejected() -> RunnerResult<()> { let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\""; @@ -1208,8 +1287,9 @@ new = ["send_message"] ), ); assert!( - error_message(&conflicting) - .contains("different reasoning_effort, extra_body, or omit_body_fields"), + error_message(&conflicting).contains( + "different reasoning_effort, reasoning_format, extra_body, or omit_body_fields" + ), "{}", error_message(&conflicting) ); @@ -1646,6 +1726,7 @@ confidence_threshold = 0.5 &target.extra_body, &target.omit_body_fields, None, + ReasoningFormat::default(), )?; assert_eq!( @@ -1685,6 +1766,7 @@ confidence_threshold = 0.5 &target.extra_body, &target.omit_body_fields, None, + ReasoningFormat::default(), )?; assert!(backend.omit_body_fields().contains("max_output_tokens")); @@ -1702,7 +1784,14 @@ confidence_threshold = 0.5 "format = \"openai_chat\"\nbase_url = \"https://example.test/v1\"\n{setting}" ); let config: LlmClientConfig = toml::from_str(&source).expect("valid deadline config"); - let backend = build_backend("test", &config, &BTreeMap::new(), &BTreeSet::new(), None); + let backend = build_backend( + "test", + &config, + &BTreeMap::new(), + &BTreeSet::new(), + None, + ReasoningFormat::default(), + ); if expected == Some(0) { assert!(backend.is_err()); } else { diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index cd21d3c91e..aca7076bdd 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -751,6 +751,7 @@ fn random_state_with_retries( extra_body: BTreeMap::new(), omit_body_fields: BTreeSet::new(), reasoning_effort: None, + reasoning_format: Default::default(), max_retries, timeout: None, }); diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index d904235186..492433dbdf 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -922,7 +922,11 @@ fn encode_message_without_tool_results_to_openai( "role": role, "content": encode_openai_content(&content_blocks, message.role, diagnostics, policy)?, }); - encode_openai_message_reasoning(&mut message_json, &message.content); + encode_openai_message_reasoning( + &mut message_json, + &message.content, + policy.target_capabilities.reasoning_format.request_key(), + ); if !tool_calls.is_empty() { message_json["tool_calls"] = Value::Array(tool_calls); if message_json["content"] == Value::String(String::new()) { @@ -933,17 +937,25 @@ fn encode_message_without_tool_results_to_openai( } // Adds either plaintext reasoning or structured details to an OpenAI Chat message. -fn encode_openai_message_reasoning(message: &mut Value, content: &[ContentBlock]) { +fn encode_openai_message_reasoning( + message: &mut Value, + content: &[ContentBlock], + reasoning_key: &str, +) { let details = reasoning_details_from_blocks(content); if details.is_empty() { - encode_openai_message_plaintext_reasoning(message, content); + encode_openai_message_plaintext_reasoning(message, content, reasoning_key); } else { - encode_openai_message_structured_reasoning(message, content, details); + encode_openai_message_structured_reasoning(message, content, details, reasoning_key); } } // Adds reasoning blocks that have no structured provider representation. -fn encode_openai_message_plaintext_reasoning(message: &mut Value, content: &[ContentBlock]) { +fn encode_openai_message_plaintext_reasoning( + message: &mut Value, + content: &[ContentBlock], + reasoning_key: &str, +) { let reasoning = content .iter() .filter_map(|block| match block { @@ -957,7 +969,7 @@ fn encode_openai_message_plaintext_reasoning(message: &mut Value, content: &[Con .collect::>() .join("\n"); if !reasoning.is_empty() { - message["reasoning"] = Value::String(reasoning); + message[reasoning_key] = Value::String(reasoning); } } @@ -966,6 +978,7 @@ fn encode_openai_message_structured_reasoning( message: &mut Value, content: &[ContentBlock], details: Vec, + reasoning_key: &str, ) { message["reasoning_details"] = Value::Array(details); let fallback = content @@ -983,7 +996,7 @@ fn encode_openai_message_structured_reasoning( .collect::>() .join("\n"); if !fallback.is_empty() { - message["reasoning"] = Value::String(fallback); + message[reasoning_key] = Value::String(fallback); } } diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rs index 95ebb75e7d..bf19f861d4 100644 --- a/crates/switchyard-translation/src/helpers.rs +++ b/crates/switchyard-translation/src/helpers.rs @@ -37,8 +37,18 @@ pub fn decode_request(wire_format: WireFormat, body: &Value) -> Result Result { + encode_request_with_policy(request, wire_format, &DEFAULT_TRANSLATION_POLICY) +} + +/// Encodes a normalized request into `wire_format`'s JSON body under `policy`, +/// so a caller can apply a target's capabilities. +pub fn encode_request_with_policy( + request: &LlmRequest, + wire_format: WireFormat, + policy: &TranslationPolicy, +) -> Result { Ok(DEFAULT_TRANSLATION_ENGINE - .encode_request(wire_format, request, &DEFAULT_TRANSLATION_POLICY)? + .encode_request(wire_format, request, policy)? .body) } diff --git a/crates/switchyard-translation/src/policy.rs b/crates/switchyard-translation/src/policy.rs index 8da1d006ec..1611f9f7b8 100644 --- a/crates/switchyard-translation/src/policy.rs +++ b/crates/switchyard-translation/src/policy.rs @@ -57,6 +57,33 @@ pub struct TargetCapabilities { pub supports_code_execution: Option, pub supports_safety_settings: Option, pub openai_compatible: Option, + /// Field name this target expects for assistant reasoning sent back in request history. + #[serde(default)] + pub reasoning_format: ReasoningFormat, +} + +/// Field name an OpenAI Chat target uses for assistant reasoning in request history. +/// +/// Only one name is ever sent, because sending both would double the reasoning tokens. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub enum ReasoningFormat { + /// Sends `reasoning`, the name most OpenAI-compatible APIs use. + #[default] + #[serde(rename = "openai")] + OpenAi, + /// Sends `reasoning_content`, the older DeepSeek name. + #[serde(rename = "deepseek")] + DeepSeek, +} + +impl ReasoningFormat { + /// Returns the request message key this format writes reasoning under. + pub const fn request_key(self) -> &'static str { + match self { + Self::OpenAi => "reasoning", + Self::DeepSeek => "reasoning_content", + } + } } /// Provider-level profile that can be used to configure target capabilities. diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index 2af80f9d34..1d8d0316d9 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -8,8 +8,9 @@ pub mod common; use pretty_assertions::assert_eq; use serde_json::{Value, json}; use switchyard_translation::{ - ContentBlock, FormatId, LossyConversionPolicy, TranslationEngine, TranslationError, - TranslationPolicy, WireFormat, prepare_request_for_target, sanitize_anthropic_tool_use_id, + ContentBlock, FormatId, LossyConversionPolicy, ReasoningFormat, TargetCapabilities, + TranslationEngine, TranslationError, TranslationPolicy, WireFormat, prepare_request_for_target, + sanitize_anthropic_tool_use_id, }; use common::{REASONING_MODEL, normalized_policy, shell_tool_call}; @@ -2365,6 +2366,54 @@ fn openai_chat_encrypted_reasoning_details_retain_fallback() -> TestResult { Ok(()) } +// Verifies a rebuilt Chat request sends assistant reasoning under the one field +// name the target is configured for, whichever name the caller used. +#[test] +fn openai_chat_replay_uses_the_targets_configured_reasoning_field() -> TestResult { + let engine = TranslationEngine::default(); + let encrypted = json!([{"type": "reasoning.encrypted", "data": "opaque"}]); + for sent_as in ["reasoning", "reasoning_content"] { + // Plain reasoning, and reasoning whose details cannot reproduce the text. + for details in [None, Some(&encrypted)] { + let mut assistant = json!({"role": "assistant", "content": "Visible answer"}); + assistant[sent_as] = json!("Historical reasoning"); + if let Some(details) = details { + assistant["reasoning_details"] = details.clone(); + } + let body = json!({ + "model": REASONING_MODEL, + "messages": [{"role": "user", "content": "hi"}, assistant] + }); + for (format, expected, absent) in [ + (ReasoningFormat::default(), "reasoning", "reasoning_content"), + (ReasoningFormat::OpenAi, "reasoning", "reasoning_content"), + (ReasoningFormat::DeepSeek, "reasoning_content", "reasoning"), + ] { + let policy = TranslationPolicy { + target_capabilities: TargetCapabilities { + reasoning_format: format, + ..TargetCapabilities::default() + }, + ..normalized_policy() + }; + let output = engine + .translate_request( + WireFormat::OpenAiChat, + WireFormat::OpenAiChat, + &body, + &policy, + )? + .body; + let case = format!("{sent_as}, details={}, {format:?}", details.is_some()); + let message = &output["messages"][1]; + assert_eq!(message[expected], "Historical reasoning", "{case}"); + assert!(message.get(absent).is_none(), "{case} also sent {absent}"); + } + } + } + Ok(()) +} + // Verifies merged reasoning re-emerges as a Responses reasoning item ahead of // the turn's function call when encoding back to the Responses format. #[test] diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index aef6319cbe..3f58162633 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -106,6 +106,7 @@ endpoint, before it calls an upstream. | `system_prompt` | No | unset | System prompt prepended when this target serves a completion. | | `extra_body` | No | `{}` | Values merged into the upstream request when the request does not already set that key. | | `reasoning_effort` | No | unset | Reasoning effort forced on every request to this target, replacing the value the caller sent (`reasoning.effort` on `openai_responses`, `reasoning_effort` on `openai_chat`). Rejected on `anthropic_messages` clients. Use it to run one target at a different effort than the client asked for, for example a strong tier at `max` behind a client that sends `high`. Targets with different effort settings need distinct model IDs when used within one route. Separate routes may use the same model ID with separate `llm_clients` entries (same endpoint, different name). | +| `reasoning_format` | No | `openai` | Field name used to send earlier assistant reasoning back to this target. `openai` sends `reasoning`. `deepseek` sends `reasoning_content`, which DeepSeek-style models need on the next turn. Only one field is ever sent. It applies when Switchyard rebuilds the request; an unchanged same-format request is forwarded as sent. Rejected on clients other than `openai_chat`. Targets with different settings need distinct model IDs on one client. | Within one route, callable targets with the same model ID must use the same `llm_client`. This includes completion targets and targets used for judging or classification. Switchyard rejects From c881ce72cb72e0301862067a9edd72216ad005d8 Mon Sep 17 00:00:00 2001 From: ChethanUK Date: Wed, 30 Sep 2026 04:37:56 +0200 Subject: [PATCH 2/3] fix(translation): always send reasoning_content to DeepSeek-format targets When history carries reasoning_details whose text matches the reasoning, the Chat encoder dropped the plain field to avoid sending the text twice. DeepSeek does not read reasoning_details, so a deepseek-format target lost the reasoning. That format now always gets reasoning_content; the openai format keeps the existing dedup. --- .../src/codecs/openai_chat/buffered.rs | 21 ++++--- .../tests/request_translation.rs | 55 +++++++++++++++++++ 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index 492433dbdf..543c483ef4 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -23,7 +23,7 @@ use crate::llm::{ MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage, }; -use crate::policy::{DeterministicIdPolicy, TranslationPolicy}; +use crate::policy::{DeterministicIdPolicy, ReasoningFormat, TranslationPolicy}; use crate::util::{ capture_request_preservation, capture_response_preservation, embed_preservation, exact_preserved_request, exact_preserved_response, json_string, object, push_lossy, @@ -925,7 +925,7 @@ fn encode_message_without_tool_results_to_openai( encode_openai_message_reasoning( &mut message_json, &message.content, - policy.target_capabilities.reasoning_format.request_key(), + policy.target_capabilities.reasoning_format, ); if !tool_calls.is_empty() { message_json["tool_calls"] = Value::Array(tool_calls); @@ -940,13 +940,13 @@ fn encode_message_without_tool_results_to_openai( fn encode_openai_message_reasoning( message: &mut Value, content: &[ContentBlock], - reasoning_key: &str, + format: ReasoningFormat, ) { let details = reasoning_details_from_blocks(content); if details.is_empty() { - encode_openai_message_plaintext_reasoning(message, content, reasoning_key); + encode_openai_message_plaintext_reasoning(message, content, format.request_key()); } else { - encode_openai_message_structured_reasoning(message, content, details, reasoning_key); + encode_openai_message_structured_reasoning(message, content, details, format); } } @@ -974,11 +974,12 @@ fn encode_openai_message_plaintext_reasoning( } // Adds exact provider details and text that cannot be recovered from those details. +// DeepSeek does not read `reasoning_details`, so that format always gets the text. fn encode_openai_message_structured_reasoning( message: &mut Value, content: &[ContentBlock], details: Vec, - reasoning_key: &str, + format: ReasoningFormat, ) { message["reasoning_details"] = Value::Array(details); let fallback = content @@ -986,8 +987,10 @@ fn encode_openai_message_structured_reasoning( .filter_map(|block| match block { ContentBlock::Reasoning { text, details, .. } if !details.is_empty() - && reasoning_text_from_details(details).as_deref() != Some(text.as_str()) - && !text.is_empty() => + && !text.is_empty() + && (format == ReasoningFormat::DeepSeek + || reasoning_text_from_details(details).as_deref() + != Some(text.as_str())) => { Some(text.as_str()) } @@ -996,7 +999,7 @@ fn encode_openai_message_structured_reasoning( .collect::>() .join("\n"); if !fallback.is_empty() { - message[reasoning_key] = Value::String(fallback); + message[format.request_key()] = Value::String(fallback); } } diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index 1d8d0316d9..0219583d34 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -2414,6 +2414,61 @@ fn openai_chat_replay_uses_the_targets_configured_reasoning_field() -> TestResul Ok(()) } +// Verifies a DeepSeek-format target still gets `reasoning_content` when the +// history's `reasoning_details` already carry the same text, since DeepSeek does +// not read `reasoning_details`. The OpenAI format keeps the text only in details. +#[test] +fn deepseek_target_gets_reasoning_content_even_when_details_hold_the_text() -> TestResult { + let engine = TranslationEngine::default(); + let body = json!({ + "model": REASONING_MODEL, + "messages": [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": "Visible answer", + "reasoning_content": "Historical reasoning", + "reasoning_details": [{"type": "reasoning.text", "text": "Historical reasoning"}] + } + ] + }); + for (format, expected_content) in [ + (ReasoningFormat::DeepSeek, Some("Historical reasoning")), + (ReasoningFormat::OpenAi, None), + ] { + let policy = TranslationPolicy { + target_capabilities: TargetCapabilities { + reasoning_format: format, + ..TargetCapabilities::default() + }, + ..normalized_policy() + }; + let output = engine + .translate_request( + WireFormat::OpenAiChat, + WireFormat::OpenAiChat, + &body, + &policy, + )? + .body; + let message = &output["messages"][1]; + assert_eq!( + message.get("reasoning_content").and_then(Value::as_str), + expected_content, + "{format:?}" + ); + assert!( + message.get("reasoning").is_none(), + "{format:?} sent reasoning" + ); + assert_eq!( + message["reasoning_details"][0]["text"], "Historical reasoning", + "{format:?}" + ); + } + Ok(()) +} + // Verifies merged reasoning re-emerges as a Responses reasoning item ahead of // the turn's function call when encoding back to the Responses format. #[test] From eeeeb3a671d0cea547e552169e57b67c34e425f6 Mon Sep 17 00:00:00 2001 From: ChethanUK Date: Wed, 30 Sep 2026 08:01:31 +0200 Subject: [PATCH 3/3] test(llm-client): put the reasoning_effort test's doc comment back on its test The rebuilt_chat_request test was inserted between that doc comment and reasoning_effort_override_replaces_the_callers_effort, so rustdoc showed it on the wrong test and the effort test had none. Signed-off-by: ChethanUK --- crates/libsy-llm-client/src/client.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 567c1e43ef..3dee78e3da 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -1959,8 +1959,6 @@ mod tests { Ok(()) } - /// A configured reasoning effort replaces the caller's value on both OpenAI wire formats, - /// which `extra_body` (defaults only) cannot do. /// A request rebuilt for an OpenAI Chat backend sends assistant reasoning under /// the backend's configured field name, and never under both names. #[tokio::test] @@ -2023,6 +2021,8 @@ mod tests { Ok(()) } + /// A configured reasoning effort replaces the caller's value on both OpenAI wire formats, + /// which `extra_body` (defaults only) cannot do. #[tokio::test] async fn reasoning_effort_override_replaces_the_callers_effort() -> std::result::Result<(), Box> {