From 179408545315fbf2e19ea436ab42f20488444e42 Mon Sep 17 00:00:00 2001 From: deepujain Date: Mon, 21 Sep 2026 12:19:09 +0000 Subject: [PATCH 1/6] fix(translation): preserve reasoning field spelling on request rebuild Signed-off-by: deepujain --- .../libsy-llm-client/tests/observability.rs | 5 +- .../src/algorithms/advisor_gate/tests.rs | 5 +- crates/libsy/src/algorithms/llm_class.rs | 7 +- crates/libsy/src/algorithms/util/affinity.rs | 5 +- crates/libsy/src/algorithms/util/llm_judge.rs | 5 +- crates/protocol/src/llm.rs | 24 ++++++ crates/protocol/src/stream.rs | 10 ++- .../src/codecs/anthropic/buffered.rs | 7 +- .../src/codecs/openai_chat/buffered.rs | 44 +++++++++- .../src/codecs/responses/buffered.rs | 13 ++- .../tests/request_translation.rs | 80 +++++++++++++++++++ 11 files changed, 188 insertions(+), 17 deletions(-) diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index 8150564f99..87d8ebefd5 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -41,8 +41,8 @@ use switchyard_libsy::{ use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver}; use switchyard_protocol::{Category, ModelId}; use switchyard_protocol::{ - ContentBlock, LlmRequest, LlmResponse, Message, Metadata, Request, Response, Role, - RoutedLlmClient, ToolCall, ToolResult, Usage, WireFormat, + ContentBlock, LlmRequest, LlmResponse, Message, Metadata, OpenAiChatReasoningField, Request, + Response, Role, RoutedLlmClient, ToolCall, ToolResult, Usage, WireFormat, }; use switchyard_protocol::{ LlmClientError, LlmResponseChunk, LlmResponseStreamEvent, StopReason, text_request, @@ -725,6 +725,7 @@ async fn affinity_warns_once_when_request_has_no_usable_identity() -> switchyard text: "provider reasoning".to_string(), signature: None, details: Vec::new(), + openai_chat_field: OpenAiChatReasoningField::default(), }], }], ..LlmRequest::default() diff --git a/crates/libsy/src/algorithms/advisor_gate/tests.rs b/crates/libsy/src/algorithms/advisor_gate/tests.rs index 72d39615b4..e24a1cfe12 100644 --- a/crates/libsy/src/algorithms/advisor_gate/tests.rs +++ b/crates/libsy/src/algorithms/advisor_gate/tests.rs @@ -10,8 +10,8 @@ use switchyard_protocol::{ResponseOutput, ToolCall, ToolResult, completion_text} use futures::StreamExt; use switchyard_protocol::{ - AggLlmResponse, LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, ModelId, - Response, StopReason, + AggLlmResponse, ContentBlock, LlmClientError, LlmResponse, LlmResponseChunk, + LlmResponseStreamEvent, ModelId, OpenAiChatReasoningField, Response, StopReason, }; use super::transcript::{NO_TEXT_PLACEHOLDER, TRUNCATION_MARKER, middle_drop}; @@ -127,6 +127,7 @@ fn reasoning_only_turn() -> Response { text: "thinking about it".to_string(), signature: None, details: Vec::new(), + openai_chat_field: OpenAiChatReasoningField::default(), }], url_citations: Vec::new(), stop_reason: None, diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 12d9241d05..aa3bc30397 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -776,8 +776,9 @@ mod tests { use super::*; use switchyard_protocol::{ - ContentBlock, InstructionBlock, LlmClientError, LlmRequest, Metadata, ModelId, ToolCall, - ToolResult, completion_text, text_request, text_response, + ContentBlock, InstructionBlock, LlmClientError, LlmRequest, Metadata, ModelId, + OpenAiChatReasoningField, ToolCall, ToolResult, completion_text, text_request, + text_response, }; use crate::algorithms::util::llm_judge::Judge; @@ -1576,6 +1577,7 @@ mod tests { text: "private chain of thought".to_string(), signature: None, details: Vec::new(), + openai_chat_field: OpenAiChatReasoningField::default(), }, ContentBlock::Text { text: "visible answer".to_string(), @@ -1590,6 +1592,7 @@ mod tests { text: "reasoning-only turn".to_string(), signature: None, details: Vec::new(), + openai_chat_field: OpenAiChatReasoningField::default(), }], }, Message::text(Role::User, "follow-up"), diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs index 698f0a9dde..30395fdbab 100644 --- a/crates/libsy/src/algorithms/util/affinity.rs +++ b/crates/libsy/src/algorithms/util/affinity.rs @@ -288,7 +288,9 @@ mod tests { use std::sync::Arc; - use switchyard_protocol::{LlmRequest, Metadata, ToolResult, text_request}; + use switchyard_protocol::{ + LlmRequest, Metadata, OpenAiChatReasoningField, ToolResult, text_request, + }; use crate::core::algorithm::RuntimeModels; @@ -598,6 +600,7 @@ mod tests { text: "Internal provider reasoning.".to_string(), signature: Some("provider-signature".to_string()), details: Vec::new(), + openai_chat_field: OpenAiChatReasoningField::default(), }, ], }); diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs index 4798178e92..b1339aedbb 100644 --- a/crates/libsy/src/algorithms/util/llm_judge.rs +++ b/crates/libsy/src/algorithms/util/llm_judge.rs @@ -400,7 +400,9 @@ mod tests { use futures::StreamExt; use http::StatusCode; use serde::Deserialize; - use switchyard_protocol::{ContentBlock, LlmClientError, text_request, text_response}; + use switchyard_protocol::{ + ContentBlock, LlmClientError, OpenAiChatReasoningField, text_request, text_response, + }; use crate::core::algorithm::Step; use crate::core::classifier::Score; @@ -477,6 +479,7 @@ mod tests { text: r#"{"ok":false}"#.to_string(), signature: None, details: Vec::new(), + openai_chat_field: OpenAiChatReasoningField::default(), }, ); } diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rs index 7968faa85e..9950992031 100644 --- a/crates/protocol/src/llm.rs +++ b/crates/protocol/src/llm.rs @@ -72,6 +72,27 @@ impl Message { } } +/// Wire-level field used for OpenAI Chat plaintext reasoning. +/// +/// The two names are provider dialects rather than interchangeable aliases: +/// some models only accept reasoning replayed under the field they produced. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OpenAiChatReasoningField { + /// The `reasoning` field. + #[default] + Reasoning, + /// The `reasoning_content` field. + ReasoningContent, +} + +impl OpenAiChatReasoningField { + /// Returns true when the field uses the historical default spelling. + pub const fn is_default(&self) -> bool { + matches!(self, Self::Reasoning) + } +} + /// Normalized content block variants carried by messages and tool results. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] @@ -91,6 +112,9 @@ pub enum ContentBlock { /// "reasoning.encrypted", "data": "..." }` object, replayed without modification. #[serde(default, skip_serializing_if = "Vec::is_empty")] details: Vec, + /// OpenAI Chat field that carried plaintext reasoning. + #[serde(default, skip_serializing_if = "OpenAiChatReasoningField::is_default")] + openai_chat_field: OpenAiChatReasoningField, }, /// Image content. Image { diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rs index c59a66f772..28c88db336 100644 --- a/crates/protocol/src/stream.rs +++ b/crates/protocol/src/stream.rs @@ -17,7 +17,10 @@ use thiserror::Error; use crate::{ LlmClientError, format::FormatId, - llm::{AggLlmResponse, ContentBlock, ResponseOutput, Role, StopReason, ToolCall, Usage}, + llm::{ + AggLlmResponse, ContentBlock, OpenAiChatReasoningField, ResponseOutput, Role, StopReason, + ToolCall, Usage, + }, }; /// Status reported for an upstream error delivered inside a streaming body. The @@ -464,6 +467,9 @@ impl ResponseAccumulator { .into_iter() .filter(|detail| !is_reasoning_id_announcement(detail)) .collect(), + // Streamed reasoning aggregates response deltas; the Chat + // response encoder always writes `reasoning_content`. + openai_chat_field: OpenAiChatReasoningField::default(), }); } if !self.text.is_empty() { @@ -685,6 +691,7 @@ mod tests { text: "think".to_string(), signature: None, details: Vec::new(), + openai_chat_field: OpenAiChatReasoningField::default(), }, ContentBlock::Text { text: "answer".to_string(), @@ -737,6 +744,7 @@ mod tests { text: "fallback reasoning".to_string(), signature: None, details: details.clone(), + openai_chat_field: OpenAiChatReasoningField::default(), }], url_citations: Vec::new(), stop_reason: Some(StopReason::EndTurn), diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs index 67f03ebc4a..5151a054ed 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs @@ -17,8 +17,9 @@ use crate::error::{Result, TranslationError}; use crate::format::{FormatId, WireFormat}; use crate::llm::{ AggLlmResponse, ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest, - MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role, - SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage, + MediaSource, Message, OpenAiChatReasoningField, OutputParams, ProviderExtensions, + ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason, ToolCall, ToolChoice, + ToolDefinition, ToolResult, Usage, }; use crate::policy::{DeterministicIdPolicy, TranslationPolicy}; use crate::util::{ @@ -663,6 +664,8 @@ fn decode_anthropic_content_block( .filter(|signature| !signature.is_empty()) .map(ToOwned::to_owned), details: Vec::new(), + // Anthropic thinking blocks carry no OpenAI Chat wire spelling. + openai_chat_field: OpenAiChatReasoningField::default(), }], Some("tool_use") => vec![ContentBlock::ToolCall(ToolCall { id: block diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index 968b6f93b8..a40438753d 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -20,8 +20,9 @@ use crate::error::{Result, TranslationError}; use crate::format::{FormatId, WireFormat}; use crate::llm::{ AggLlmResponse, ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest, - MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role, - SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage, + MediaSource, Message, OpenAiChatReasoningField, OutputParams, ProviderExtensions, + ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason, ToolCall, ToolChoice, + ToolDefinition, ToolResult, Usage, }; use crate::policy::{DeterministicIdPolicy, TranslationPolicy}; use crate::util::{ @@ -442,6 +443,7 @@ impl FormatCodec for OpenAiChatCodec { // Pulls OpenAI-compatible reasoning fields into private reasoning IR blocks. fn prepend_openai_reasoning_blocks(content: &mut Vec, object: &Map) { + let openai_chat_field = openai_chat_reasoning_field(object); if let Some(details) = object .get("reasoning_details") .and_then(Value::as_array) @@ -466,6 +468,7 @@ fn prepend_openai_reasoning_blocks(content: &mut Vec, object: &Map text, signature, details: details.clone(), + openai_chat_field, }, ); return; @@ -478,11 +481,44 @@ fn prepend_openai_reasoning_blocks(content: &mut Vec, object: &Map text: text.to_string(), signature: None, details: Vec::new(), + openai_chat_field, }, ); } } +// Records which OpenAI Chat wire field carried plaintext reasoning so a +// rebuilt request can replay the same spelling. +fn openai_chat_reasoning_field(object: &Map) -> OpenAiChatReasoningField { + if object + .get("reasoning_content") + .and_then(Value::as_str) + .is_some_and(|text| !text.is_empty()) + { + OpenAiChatReasoningField::ReasoningContent + } else { + OpenAiChatReasoningField::Reasoning + } +} + +// Picks the OpenAI Chat wire field for encoded reasoning: the recorded source +// spelling when any reasoning block carries one, else the historical default. +fn openai_chat_reasoning_key(content: &[ContentBlock]) -> &'static str { + if content.iter().any(|block| { + matches!( + block, + ContentBlock::Reasoning { + openai_chat_field: OpenAiChatReasoningField::ReasoningContent, + .. + } + ) + }) { + "reasoning_content" + } else { + "reasoning" + } +} + // Returns structured reasoning details in their original order. fn reasoning_details_from_blocks(content: &[ContentBlock]) -> Vec { content @@ -947,7 +983,7 @@ fn encode_openai_message_plaintext_reasoning(message: &mut Value, content: &[Con .collect::>() .join("\n"); if !reasoning.is_empty() { - message["reasoning"] = Value::String(reasoning); + message[openai_chat_reasoning_key(content)] = Value::String(reasoning); } } @@ -973,7 +1009,7 @@ fn encode_openai_message_structured_reasoning( .collect::>() .join("\n"); if !fallback.is_empty() { - message["reasoning"] = Value::String(fallback); + message[openai_chat_reasoning_key(content)] = Value::String(fallback); } } diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs index 998505913e..03d08c78d3 100644 --- a/crates/switchyard-translation/src/codecs/responses/buffered.rs +++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs @@ -23,8 +23,9 @@ use crate::error::{Result, TranslationError}; use crate::format::{FormatId, WireFormat}; use crate::llm::{ AggLlmResponse, ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest, - MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role, - SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage, + MediaSource, Message, OpenAiChatReasoningField, OutputParams, ProviderExtensions, + ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason, ToolCall, ToolChoice, + ToolDefinition, ToolResult, Usage, }; use crate::policy::{DeterministicIdPolicy, TranslationPolicy}; use crate::util::{ @@ -853,6 +854,8 @@ fn decode_responses_reasoning_item(item: &Map) -> Vec) -> Vec Vec { .to_string(), signature: None, details: Vec::new(), + // Responses reasoning items carry no OpenAI Chat wire spelling. + openai_chat_field: OpenAiChatReasoningField::default(), }); } Some("input_image") => { @@ -1442,6 +1449,7 @@ fn encode_responses_special_input( text, signature: None, details, + .. } => encode_responses_reasoning_input(text, details), ContentBlock::ToolCall(call) if custom_tools.contains(&call.name) => { // A freeform tool call replays as `custom_tool_call` with its raw input. @@ -1895,6 +1903,7 @@ fn encode_responses_output(outputs: &[ResponseOutput]) -> Value { text, signature, details, + .. } = block { let encrypted = signature diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index 806e21412f..77541cec38 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -4048,3 +4048,83 @@ fn responses_stored_tool_outputs_stay_tool_results() -> TestResult { assert_eq!(output["input"], outputs); Ok(()) } + +#[test] +fn chat_request_rebuild_preserves_reasoning_field_spelling() -> TestResult { + let engine = TranslationEngine::default(); + let policy = normalized_policy(); + for field in ["reasoning_content", "reasoning"] { + let mut assistant = serde_json::Map::new(); + assistant.insert("role".to_string(), json!("assistant")); + assistant.insert("content".to_string(), json!("Visible answer")); + assistant.insert(field.to_string(), json!("Historical reasoning")); + let body = json!({ + "model": "route", + "messages": [ + {"role": "user", "content": "Hi"}, + Value::Object(assistant), + ], + }); + let output = engine + .translate_request( + WireFormat::OpenAiChat, + WireFormat::OpenAiChat, + &body, + &policy, + )? + .body; + let rebuilt = &output["messages"][1]; + assert_eq!( + rebuilt[field], + json!("Historical reasoning"), + "rebuilt request must replay reasoning under the source spelling" + ); + let other = if field == "reasoning" { + "reasoning_content" + } else { + "reasoning" + }; + assert!( + rebuilt.get(other).is_none(), + "rebuilt request must not duplicate reasoning under both spellings" + ); + } + Ok(()) +} + +#[test] +fn chat_request_rebuild_uses_default_reasoning_spelling_for_foreign_reasoning() -> TestResult { + let engine = TranslationEngine::default(); + let policy = normalized_policy(); + let body = json!({ + "model": "route", + "messages": [ + {"role": "user", "content": "Hi"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Fresh reasoning"}, + {"type": "text", "text": "Visible answer"}, + ], + }, + ], + }); + let output = engine + .translate_request( + WireFormat::AnthropicMessages, + WireFormat::OpenAiChat, + &body, + &policy, + )? + .body; + let rebuilt = &output["messages"][1]; + assert!( + rebuilt.get("reasoning").is_some(), + "reasoning from another format keeps the historical default spelling" + ); + assert!( + rebuilt.get("reasoning_content").is_none(), + "rebuilt request must not emit both spellings" + ); + Ok(()) +} From 9563d1995048abf0b28bb1d8eaf0df027b57eac7 Mon Sep 17 00:00:00 2001 From: Deepak Jain Date: Mon, 21 Sep 2026 13:01:19 +0000 Subject: [PATCH 2/6] docs(translation): document reasoning-field invariant on rebuild tests Signed-off-by: Deepak Jain --- crates/switchyard-translation/tests/request_translation.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index 77541cec38..df73ce7951 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -4050,6 +4050,7 @@ fn responses_stored_tool_outputs_stay_tool_results() -> TestResult { } #[test] +/// Rebuilt Chat requests retain the source plaintext reasoning field name. fn chat_request_rebuild_preserves_reasoning_field_spelling() -> TestResult { let engine = TranslationEngine::default(); let policy = normalized_policy(); @@ -4093,6 +4094,7 @@ fn chat_request_rebuild_preserves_reasoning_field_spelling() -> TestResult { } #[test] +/// Foreign reasoning defaults to the historical `reasoning` field in Chat requests. fn chat_request_rebuild_uses_default_reasoning_spelling_for_foreign_reasoning() -> TestResult { let engine = TranslationEngine::default(); let policy = normalized_policy(); From 6d7378928b683aa0c9315c3ac1cc159e08a11560 Mon Sep 17 00:00:00 2001 From: Deepak Jain Date: Mon, 21 Sep 2026 16:26:46 +0000 Subject: [PATCH 3/6] revert: drop source-provenance reasoning field in favor of target setting Signed-off-by: Deepak Jain --- .../libsy-llm-client/tests/observability.rs | 5 +- .../src/algorithms/advisor_gate/tests.rs | 5 +- crates/libsy/src/algorithms/llm_class.rs | 7 +- crates/libsy/src/algorithms/util/affinity.rs | 5 +- crates/libsy/src/algorithms/util/llm_judge.rs | 5 +- crates/protocol/src/llm.rs | 24 ------ crates/protocol/src/stream.rs | 10 +-- .../src/codecs/anthropic/buffered.rs | 7 +- .../src/codecs/openai_chat/buffered.rs | 44 +--------- .../src/codecs/responses/buffered.rs | 13 +-- .../tests/request_translation.rs | 82 ------------------- 11 files changed, 17 insertions(+), 190 deletions(-) diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index 87d8ebefd5..8150564f99 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -41,8 +41,8 @@ use switchyard_libsy::{ use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver}; use switchyard_protocol::{Category, ModelId}; use switchyard_protocol::{ - ContentBlock, LlmRequest, LlmResponse, Message, Metadata, OpenAiChatReasoningField, Request, - Response, Role, RoutedLlmClient, ToolCall, ToolResult, Usage, WireFormat, + ContentBlock, LlmRequest, LlmResponse, Message, Metadata, Request, Response, Role, + RoutedLlmClient, ToolCall, ToolResult, Usage, WireFormat, }; use switchyard_protocol::{ LlmClientError, LlmResponseChunk, LlmResponseStreamEvent, StopReason, text_request, @@ -725,7 +725,6 @@ async fn affinity_warns_once_when_request_has_no_usable_identity() -> switchyard text: "provider reasoning".to_string(), signature: None, details: Vec::new(), - openai_chat_field: OpenAiChatReasoningField::default(), }], }], ..LlmRequest::default() diff --git a/crates/libsy/src/algorithms/advisor_gate/tests.rs b/crates/libsy/src/algorithms/advisor_gate/tests.rs index e24a1cfe12..72d39615b4 100644 --- a/crates/libsy/src/algorithms/advisor_gate/tests.rs +++ b/crates/libsy/src/algorithms/advisor_gate/tests.rs @@ -10,8 +10,8 @@ use switchyard_protocol::{ResponseOutput, ToolCall, ToolResult, completion_text} use futures::StreamExt; use switchyard_protocol::{ - AggLlmResponse, ContentBlock, LlmClientError, LlmResponse, LlmResponseChunk, - LlmResponseStreamEvent, ModelId, OpenAiChatReasoningField, Response, StopReason, + AggLlmResponse, LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, ModelId, + Response, StopReason, }; use super::transcript::{NO_TEXT_PLACEHOLDER, TRUNCATION_MARKER, middle_drop}; @@ -127,7 +127,6 @@ fn reasoning_only_turn() -> Response { text: "thinking about it".to_string(), signature: None, details: Vec::new(), - openai_chat_field: OpenAiChatReasoningField::default(), }], url_citations: Vec::new(), stop_reason: None, diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index aa3bc30397..12d9241d05 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -776,9 +776,8 @@ mod tests { use super::*; use switchyard_protocol::{ - ContentBlock, InstructionBlock, LlmClientError, LlmRequest, Metadata, ModelId, - OpenAiChatReasoningField, ToolCall, ToolResult, completion_text, text_request, - text_response, + ContentBlock, InstructionBlock, LlmClientError, LlmRequest, Metadata, ModelId, ToolCall, + ToolResult, completion_text, text_request, text_response, }; use crate::algorithms::util::llm_judge::Judge; @@ -1577,7 +1576,6 @@ mod tests { text: "private chain of thought".to_string(), signature: None, details: Vec::new(), - openai_chat_field: OpenAiChatReasoningField::default(), }, ContentBlock::Text { text: "visible answer".to_string(), @@ -1592,7 +1590,6 @@ mod tests { text: "reasoning-only turn".to_string(), signature: None, details: Vec::new(), - openai_chat_field: OpenAiChatReasoningField::default(), }], }, Message::text(Role::User, "follow-up"), diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs index 30395fdbab..698f0a9dde 100644 --- a/crates/libsy/src/algorithms/util/affinity.rs +++ b/crates/libsy/src/algorithms/util/affinity.rs @@ -288,9 +288,7 @@ mod tests { use std::sync::Arc; - use switchyard_protocol::{ - LlmRequest, Metadata, OpenAiChatReasoningField, ToolResult, text_request, - }; + use switchyard_protocol::{LlmRequest, Metadata, ToolResult, text_request}; use crate::core::algorithm::RuntimeModels; @@ -600,7 +598,6 @@ mod tests { text: "Internal provider reasoning.".to_string(), signature: Some("provider-signature".to_string()), details: Vec::new(), - openai_chat_field: OpenAiChatReasoningField::default(), }, ], }); diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs index b1339aedbb..4798178e92 100644 --- a/crates/libsy/src/algorithms/util/llm_judge.rs +++ b/crates/libsy/src/algorithms/util/llm_judge.rs @@ -400,9 +400,7 @@ mod tests { use futures::StreamExt; use http::StatusCode; use serde::Deserialize; - use switchyard_protocol::{ - ContentBlock, LlmClientError, OpenAiChatReasoningField, text_request, text_response, - }; + use switchyard_protocol::{ContentBlock, LlmClientError, text_request, text_response}; use crate::core::algorithm::Step; use crate::core::classifier::Score; @@ -479,7 +477,6 @@ mod tests { text: r#"{"ok":false}"#.to_string(), signature: None, details: Vec::new(), - openai_chat_field: OpenAiChatReasoningField::default(), }, ); } diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rs index 9950992031..7968faa85e 100644 --- a/crates/protocol/src/llm.rs +++ b/crates/protocol/src/llm.rs @@ -72,27 +72,6 @@ impl Message { } } -/// Wire-level field used for OpenAI Chat plaintext reasoning. -/// -/// The two names are provider dialects rather than interchangeable aliases: -/// some models only accept reasoning replayed under the field they produced. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum OpenAiChatReasoningField { - /// The `reasoning` field. - #[default] - Reasoning, - /// The `reasoning_content` field. - ReasoningContent, -} - -impl OpenAiChatReasoningField { - /// Returns true when the field uses the historical default spelling. - pub const fn is_default(&self) -> bool { - matches!(self, Self::Reasoning) - } -} - /// Normalized content block variants carried by messages and tool results. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] @@ -112,9 +91,6 @@ pub enum ContentBlock { /// "reasoning.encrypted", "data": "..." }` object, replayed without modification. #[serde(default, skip_serializing_if = "Vec::is_empty")] details: Vec, - /// OpenAI Chat field that carried plaintext reasoning. - #[serde(default, skip_serializing_if = "OpenAiChatReasoningField::is_default")] - openai_chat_field: OpenAiChatReasoningField, }, /// Image content. Image { diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rs index 28c88db336..c59a66f772 100644 --- a/crates/protocol/src/stream.rs +++ b/crates/protocol/src/stream.rs @@ -17,10 +17,7 @@ use thiserror::Error; use crate::{ LlmClientError, format::FormatId, - llm::{ - AggLlmResponse, ContentBlock, OpenAiChatReasoningField, ResponseOutput, Role, StopReason, - ToolCall, Usage, - }, + llm::{AggLlmResponse, ContentBlock, ResponseOutput, Role, StopReason, ToolCall, Usage}, }; /// Status reported for an upstream error delivered inside a streaming body. The @@ -467,9 +464,6 @@ impl ResponseAccumulator { .into_iter() .filter(|detail| !is_reasoning_id_announcement(detail)) .collect(), - // Streamed reasoning aggregates response deltas; the Chat - // response encoder always writes `reasoning_content`. - openai_chat_field: OpenAiChatReasoningField::default(), }); } if !self.text.is_empty() { @@ -691,7 +685,6 @@ mod tests { text: "think".to_string(), signature: None, details: Vec::new(), - openai_chat_field: OpenAiChatReasoningField::default(), }, ContentBlock::Text { text: "answer".to_string(), @@ -744,7 +737,6 @@ mod tests { text: "fallback reasoning".to_string(), signature: None, details: details.clone(), - openai_chat_field: OpenAiChatReasoningField::default(), }], url_citations: Vec::new(), stop_reason: Some(StopReason::EndTurn), diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs index 5151a054ed..67f03ebc4a 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs @@ -17,9 +17,8 @@ use crate::error::{Result, TranslationError}; use crate::format::{FormatId, WireFormat}; use crate::llm::{ AggLlmResponse, ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest, - MediaSource, Message, OpenAiChatReasoningField, OutputParams, ProviderExtensions, - ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason, ToolCall, ToolChoice, - ToolDefinition, ToolResult, Usage, + MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role, + SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage, }; use crate::policy::{DeterministicIdPolicy, TranslationPolicy}; use crate::util::{ @@ -664,8 +663,6 @@ fn decode_anthropic_content_block( .filter(|signature| !signature.is_empty()) .map(ToOwned::to_owned), details: Vec::new(), - // Anthropic thinking blocks carry no OpenAI Chat wire spelling. - openai_chat_field: OpenAiChatReasoningField::default(), }], Some("tool_use") => vec![ContentBlock::ToolCall(ToolCall { id: block diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index a40438753d..968b6f93b8 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -20,9 +20,8 @@ use crate::error::{Result, TranslationError}; use crate::format::{FormatId, WireFormat}; use crate::llm::{ AggLlmResponse, ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest, - MediaSource, Message, OpenAiChatReasoningField, OutputParams, ProviderExtensions, - ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason, ToolCall, ToolChoice, - ToolDefinition, ToolResult, Usage, + MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role, + SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage, }; use crate::policy::{DeterministicIdPolicy, TranslationPolicy}; use crate::util::{ @@ -443,7 +442,6 @@ impl FormatCodec for OpenAiChatCodec { // Pulls OpenAI-compatible reasoning fields into private reasoning IR blocks. fn prepend_openai_reasoning_blocks(content: &mut Vec, object: &Map) { - let openai_chat_field = openai_chat_reasoning_field(object); if let Some(details) = object .get("reasoning_details") .and_then(Value::as_array) @@ -468,7 +466,6 @@ fn prepend_openai_reasoning_blocks(content: &mut Vec, object: &Map text, signature, details: details.clone(), - openai_chat_field, }, ); return; @@ -481,44 +478,11 @@ fn prepend_openai_reasoning_blocks(content: &mut Vec, object: &Map text: text.to_string(), signature: None, details: Vec::new(), - openai_chat_field, }, ); } } -// Records which OpenAI Chat wire field carried plaintext reasoning so a -// rebuilt request can replay the same spelling. -fn openai_chat_reasoning_field(object: &Map) -> OpenAiChatReasoningField { - if object - .get("reasoning_content") - .and_then(Value::as_str) - .is_some_and(|text| !text.is_empty()) - { - OpenAiChatReasoningField::ReasoningContent - } else { - OpenAiChatReasoningField::Reasoning - } -} - -// Picks the OpenAI Chat wire field for encoded reasoning: the recorded source -// spelling when any reasoning block carries one, else the historical default. -fn openai_chat_reasoning_key(content: &[ContentBlock]) -> &'static str { - if content.iter().any(|block| { - matches!( - block, - ContentBlock::Reasoning { - openai_chat_field: OpenAiChatReasoningField::ReasoningContent, - .. - } - ) - }) { - "reasoning_content" - } else { - "reasoning" - } -} - // Returns structured reasoning details in their original order. fn reasoning_details_from_blocks(content: &[ContentBlock]) -> Vec { content @@ -983,7 +947,7 @@ fn encode_openai_message_plaintext_reasoning(message: &mut Value, content: &[Con .collect::>() .join("\n"); if !reasoning.is_empty() { - message[openai_chat_reasoning_key(content)] = Value::String(reasoning); + message["reasoning"] = Value::String(reasoning); } } @@ -1009,7 +973,7 @@ fn encode_openai_message_structured_reasoning( .collect::>() .join("\n"); if !fallback.is_empty() { - message[openai_chat_reasoning_key(content)] = Value::String(fallback); + message["reasoning"] = Value::String(fallback); } } diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs index 03d08c78d3..998505913e 100644 --- a/crates/switchyard-translation/src/codecs/responses/buffered.rs +++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs @@ -23,9 +23,8 @@ use crate::error::{Result, TranslationError}; use crate::format::{FormatId, WireFormat}; use crate::llm::{ AggLlmResponse, ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest, - MediaSource, Message, OpenAiChatReasoningField, OutputParams, ProviderExtensions, - ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason, ToolCall, ToolChoice, - ToolDefinition, ToolResult, Usage, + MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role, + SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage, }; use crate::policy::{DeterministicIdPolicy, TranslationPolicy}; use crate::util::{ @@ -854,8 +853,6 @@ fn decode_responses_reasoning_item(item: &Map) -> Vec) -> Vec Vec { .to_string(), signature: None, details: Vec::new(), - // Responses reasoning items carry no OpenAI Chat wire spelling. - openai_chat_field: OpenAiChatReasoningField::default(), }); } Some("input_image") => { @@ -1449,7 +1442,6 @@ fn encode_responses_special_input( text, signature: None, details, - .. } => encode_responses_reasoning_input(text, details), ContentBlock::ToolCall(call) if custom_tools.contains(&call.name) => { // A freeform tool call replays as `custom_tool_call` with its raw input. @@ -1903,7 +1895,6 @@ fn encode_responses_output(outputs: &[ResponseOutput]) -> Value { text, signature, details, - .. } = block { let encrypted = signature diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index df73ce7951..806e21412f 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -4048,85 +4048,3 @@ fn responses_stored_tool_outputs_stay_tool_results() -> TestResult { assert_eq!(output["input"], outputs); Ok(()) } - -#[test] -/// Rebuilt Chat requests retain the source plaintext reasoning field name. -fn chat_request_rebuild_preserves_reasoning_field_spelling() -> TestResult { - let engine = TranslationEngine::default(); - let policy = normalized_policy(); - for field in ["reasoning_content", "reasoning"] { - let mut assistant = serde_json::Map::new(); - assistant.insert("role".to_string(), json!("assistant")); - assistant.insert("content".to_string(), json!("Visible answer")); - assistant.insert(field.to_string(), json!("Historical reasoning")); - let body = json!({ - "model": "route", - "messages": [ - {"role": "user", "content": "Hi"}, - Value::Object(assistant), - ], - }); - let output = engine - .translate_request( - WireFormat::OpenAiChat, - WireFormat::OpenAiChat, - &body, - &policy, - )? - .body; - let rebuilt = &output["messages"][1]; - assert_eq!( - rebuilt[field], - json!("Historical reasoning"), - "rebuilt request must replay reasoning under the source spelling" - ); - let other = if field == "reasoning" { - "reasoning_content" - } else { - "reasoning" - }; - assert!( - rebuilt.get(other).is_none(), - "rebuilt request must not duplicate reasoning under both spellings" - ); - } - Ok(()) -} - -#[test] -/// Foreign reasoning defaults to the historical `reasoning` field in Chat requests. -fn chat_request_rebuild_uses_default_reasoning_spelling_for_foreign_reasoning() -> TestResult { - let engine = TranslationEngine::default(); - let policy = normalized_policy(); - let body = json!({ - "model": "route", - "messages": [ - {"role": "user", "content": "Hi"}, - { - "role": "assistant", - "content": [ - {"type": "thinking", "thinking": "Fresh reasoning"}, - {"type": "text", "text": "Visible answer"}, - ], - }, - ], - }); - let output = engine - .translate_request( - WireFormat::AnthropicMessages, - WireFormat::OpenAiChat, - &body, - &policy, - )? - .body; - let rebuilt = &output["messages"][1]; - assert!( - rebuilt.get("reasoning").is_some(), - "reasoning from another format keeps the historical default spelling" - ); - assert!( - rebuilt.get("reasoning_content").is_none(), - "rebuilt request must not emit both spellings" - ); - Ok(()) -} From 9795f1d92348b40d09452108001cfd1ae9b3f7cc Mon Sep 17 00:00:00 2001 From: Deepak Jain Date: Mon, 21 Sep 2026 16:26:52 +0000 Subject: [PATCH 4/6] feat(runner): add reasoning_format target setting for Chat reasoning field Signed-off-by: Deepak Jain --- Cargo.lock | 1 + crates/libsy-llm-client/Cargo.toml | 1 + crates/libsy-llm-client/src/client.rs | 230 +++++++++++++++++++++++++ crates/libsy-llm-client/src/lib.rs | 2 +- crates/switchyard-runner/src/config.rs | 97 ++++++++++- docs/reference/toml_schema.md | 1 + 6 files changed, 326 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1cbce1be9..bcd6c200a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2358,6 +2358,7 @@ dependencies = [ "opentelemetry_sdk", "parking_lot", "reqwest", + "serde", "serde_json", "switchyard-libsy", "switchyard-protocol", diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.toml index 9a9ca6e0a7..ca044c4cf8 100644 --- a/crates/libsy-llm-client/Cargo.toml +++ b/crates/libsy-llm-client/Cargo.toml @@ -28,6 +28,7 @@ opentelemetry = { version = "0.32", default-features = false, features = ["metri parking_lot.workspace = true http.workspace = true httpdate.workspace = true +serde.workspace = true serde_json.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 948838f48e..0dda4d92e4 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -13,6 +13,7 @@ use futures_util::{StreamExt, stream}; use http::StatusCode; use reqwest::RequestBuilder; use reqwest::header::{HeaderMap, RETRY_AFTER}; +use serde::Deserialize; use serde_json::{Map, Value, json}; use switchyard_protocol::{ LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, Metadata, @@ -56,6 +57,20 @@ const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(250); const MAX_RETRY_BACKOFF: Duration = Duration::from_secs(2); const MAX_RETRY_AFTER: Duration = Duration::from_secs(60); +/// OpenAI Chat reasoning field dialect a target expects for assistant reasoning. +/// +/// Some OpenAI-compatible models only accept reasoning replayed under the exact +/// field they produced, so a target can pin the spelling here instead of +/// relying on whatever the inbound request carried. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum ReasoningFormat { + /// The `reasoning` field. + OpenAi, + /// The `reasoning_content` field. + Deepseek, +} + /// How one model is served: the `default_backend` used when the request does not /// pin a wire format, plus any `other_backends` reachable over additional formats. #[derive(Clone, Debug)] @@ -63,6 +78,7 @@ pub struct ModelConfig { model_name: ModelId, default_backend: Backend, other_backends: Option>, + reasoning_format: Option, } impl ModelConfig { @@ -77,8 +93,17 @@ impl ModelConfig { model_name: model_name.into(), default_backend, other_backends, + reasoning_format: None, } } + + /// Pins the OpenAI Chat reasoning field the target expects. After encoding, + /// assistant reasoning is moved to this field and the other spelling is + /// removed. Unset keeps the encoded field unchanged. + pub fn with_reasoning_format(mut self, reasoning_format: ReasoningFormat) -> Self { + self.reasoning_format = Some(reasoning_format); + self + } } /// A model-bearing provider operation outside the normal completion endpoint. @@ -183,6 +208,13 @@ impl TranslatingLlmClient { self.backend_for(model, operation.wire_format()).is_some() } + /// The OpenAI Chat reasoning field dialect `model`'s target pinned, if any. + fn reasoning_format(&self, model: &ModelId) -> Option { + self.model_to_config + .get(model) + .and_then(|config| config.reasoning_format) + } + /// Calls a model-bearing auxiliary provider operation. /// /// Returns an error when the model has no compatible backend or the upstream @@ -269,6 +301,11 @@ impl TranslatingLlmClient { } if matches!(backend, Backend::OpenAiChat(_)) { ensure_openai_stream_usage(&mut body); + // The target's field choice wins over whatever spelling the inbound + // request carried; the two fields are never emitted together. + if let Some(reasoning_format) = self.reasoning_format(model) { + apply_reasoning_format(&mut body, reasoning_format); + } } let streaming = endpoint.allows_streaming() && body.get("stream").and_then(Value::as_bool).unwrap_or(false); @@ -1135,6 +1172,31 @@ fn apply_reasoning_effort(body: &mut Value, backend: &Backend) { } } +// Moves assistant reasoning to the OpenAI Chat field the selected target +// expects, removing the other spelling. The value keeps the text it carried; +// when a message somehow carries both spellings, the target field's own value +// wins rather than concatenating the two. +fn apply_reasoning_format(body: &mut Value, reasoning_format: ReasoningFormat) { + let (target_field, other_field) = match reasoning_format { + ReasoningFormat::OpenAi => ("reasoning", "reasoning_content"), + ReasoningFormat::Deepseek => ("reasoning_content", "reasoning"), + }; + let Some(messages) = body.get_mut("messages").and_then(Value::as_array_mut) else { + return; + }; + for message in messages { + let Some(message) = message.as_object_mut() else { + continue; + }; + let Some(value) = message.remove(other_field) else { + continue; + }; + if !message.contains_key(target_field) { + message.insert(target_field.to_string(), value); + } + } +} + // Applies target defaults without overriding fields supplied by the caller. fn merge_extra_body(body: &mut Value, extra_body: &BTreeMap) { let Value::Object(object) = body else { @@ -1408,6 +1470,18 @@ mod tests { vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)] } + // A one-model config list: "gpt" served over OpenAI Chat, with the target's + // reasoning field pinned to `reasoning_format`. + fn chat_map_with_reasoning_format( + base_url: &str, + reasoning_format: ReasoningFormat, + ) -> Vec { + vec![ + ModelConfig::new("gpt", Backend::OpenAiChat(config(base_url)), None) + .with_reasoning_format(reasoning_format), + ] + } + fn responses_map_with_effort(base_url: &str, effort: &str) -> Vec { let mut backend = config(base_url); backend.reasoning_effort = Some(effort.to_string()); @@ -2854,6 +2928,162 @@ mod tests { // Raw path, buffered: decode an OpenAI Chat body -> call -> encode back to OpenAI // Chat JSON, with the served `model` restamped over the id the caller addressed. + // + // A target's `reasoning_format` moves assistant reasoning to the target's field + // even when the inbound request carried the other spelling: the target wins. + #[tokio::test] + async fn reasoning_format_openai_moves_incoming_reasoning_content() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(chat_success_response()) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&chat_map_with_reasoning_format( + &format!("{}/v1", server.uri()), + ReasoningFormat::OpenAi, + ))?; + let raw = json!({ + "model": "gpt", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "answer", "reasoning_content": "thoughts"}, + ], + }); + client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiChat, + ) + .await?; + + let received = server + .received_requests() + .await + .ok_or("request recording should be enabled")?; + let received = received.first().ok_or("expected one upstream request")?; + let body: Value = serde_json::from_slice(&received.body)?; + let assistant = &body["messages"][1]; + // The incoming `reasoning_content` spelling conflicts with the target's + // `openai` setting; the conflict resolves to the target's spelling. + assert_eq!(assistant["reasoning"], json!("thoughts")); + assert!(assistant.get("reasoning_content").is_none()); + Ok(()) + } + + #[tokio::test] + async fn reasoning_format_deepseek_moves_incoming_reasoning() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(chat_success_response()) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&chat_map_with_reasoning_format( + &format!("{}/v1", server.uri()), + ReasoningFormat::Deepseek, + ))?; + let raw = json!({ + "model": "gpt", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "answer", "reasoning": "thoughts"}, + ], + }); + client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiChat, + ) + .await?; + + let received = server + .received_requests() + .await + .ok_or("request recording should be enabled")?; + let received = received.first().ok_or("expected one upstream request")?; + let body: Value = serde_json::from_slice(&received.body)?; + let assistant = &body["messages"][1]; + assert_eq!(assistant["reasoning_content"], json!("thoughts")); + assert!(assistant.get("reasoning").is_none()); + Ok(()) + } + + // Without a target setting the encoded field is left unchanged: unset keeps + // the current behavior. + #[tokio::test] + async fn unset_reasoning_format_keeps_incoming_spelling() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(chat_success_response()) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?; + let raw = json!({ + "model": "gpt", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "answer", "reasoning_content": "thoughts"}, + ], + }); + client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiChat, + ) + .await?; + + let received = server + .received_requests() + .await + .ok_or("request recording should be enabled")?; + let received = received.first().ok_or("expected one upstream request")?; + let body: Value = serde_json::from_slice(&received.body)?; + let assistant = &body["messages"][1]; + assert_eq!(assistant["reasoning_content"], json!("thoughts")); + assert!(assistant.get("reasoning").is_none()); + Ok(()) + } + + // The two spellings are never emitted together: when a message somehow + // carries both, the target field's own value wins and the other is dropped. + #[test] + fn reasoning_format_never_emits_both_fields() { + let mut body = json!({ + "messages": [ + {"role": "assistant", "content": "answer", + "reasoning": "target spelling", "reasoning_content": "other spelling"}, + ], + }); + apply_reasoning_format(&mut body, ReasoningFormat::OpenAi); + let message = &body["messages"][0]; + assert_eq!(message["reasoning"], json!("target spelling")); + assert!(message.get("reasoning_content").is_none()); + + let mut body = json!({ + "messages": [ + {"role": "assistant", "content": "answer", + "reasoning": "other spelling", "reasoning_content": "target spelling"}, + ], + }); + apply_reasoning_format(&mut body, ReasoningFormat::Deepseek); + let message = &body["messages"][0]; + assert_eq!(message["reasoning_content"], json!("target spelling")); + assert!(message.get("reasoning").is_none()); + } #[tokio::test] async fn call_rewrite_model_raw_round_trips_buffered_json() -> std::result::Result<(), Box> { diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs index d5f579c7d7..a3aebe431e 100644 --- a/crates/libsy-llm-client/src/lib.rs +++ b/crates/libsy-llm-client/src/lib.rs @@ -26,7 +26,7 @@ pub mod raw; pub mod run; pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig}; -pub use client::{AuxiliaryOperation, ModelConfig, TranslatingLlmClient}; +pub use client::{AuxiliaryOperation, ModelConfig, ReasoningFormat, TranslatingLlmClient}; pub use error::{LlmClientError, Result}; pub use observation::{LlmCallObservation, RunObservation, RunObserver}; pub use raw::RawResponse; diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index b6f9ca58df..760bcf49c6 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Deserializer}; use serde_json::Value; use switchyard_llm_client::{ AuxiliaryOperation, Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, - TranslatingLlmClient, + ReasoningFormat, TranslatingLlmClient, }; use switchyard_protocol::{Category, ModelId, RoutedLlmClient, WireFormat}; @@ -192,10 +192,11 @@ impl DeploymentConfig { std::collections::hash_map::Entry::Occupied(slot) => { let (first_name, first) = slot.get(); if first.reasoning_effort != target.reasoning_effort + || first.reasoning_format != target.reasoning_format || first.extra_body != target.extra_body { return Err(RunnerError::configuration(format!( - "targets {first_name} and {target_name} both name model {} on llm client {} but with different reasoning_effort or extra_body; one target per model id is kept, so give each its own model id or llm client", + "targets {first_name} and {target_name} both name model {} on llm client {} but with different reasoning_effort, reasoning_format or extra_body; one target per model id is kept, so give each its own model id or llm client", target.id, target.llm_client ))); } @@ -312,7 +313,14 @@ impl DeploymentConfig { ))); } } - model_configs.push(ModelConfig::new( + if target.reasoning_format.is_some() + && !matches!(client_config.format, ClientFormat::OpenAiChat) + { + return Err(RunnerError::configuration(format!( + "target {target_name} reasoning_format is only supported on openai_chat clients" + ))); + } + let mut model_config = ModelConfig::new( target.id.clone(), build_backend( &target.llm_client, @@ -321,7 +329,11 @@ impl DeploymentConfig { target.reasoning_effort.clone(), )?, None, - )); + ); + if let Some(reasoning_format) = target.reasoning_format { + model_config = model_config.with_reasoning_format(reasoning_format); + } + model_configs.push(model_config); } let mut clients = BTreeMap::new(); @@ -583,6 +595,10 @@ struct TargetConfig { /// Reasoning effort forced on every request to this target, replacing the caller's value. /// Only meaningful on `openai_chat` and `openai_responses` clients. reasoning_effort: Option, + /// Which OpenAI Chat wire field the target expects assistant reasoning under + /// (`openai` for `reasoning`, `deepseek` for `reasoning_content`). Unset keeps + /// the encoded field unchanged. Only meaningful on `openai_chat` clients. + reasoning_format: Option, } #[derive(Clone, Copy, Debug, Deserialize)] @@ -1176,10 +1192,27 @@ new = ["send_message"] ), ); assert!( - error_message(&conflicting).contains("different reasoning_effort or extra_body"), + error_message(&conflicting) + .contains("different reasoning_effort, reasoning_format or extra_body"), "{}", error_message(&conflicting) ); + // Same model, same client, different reasoning_format: the second target's + // field choice could never take effect. + let format_conflict = VALID_CONFIG.replace( + strong, + &format!( + "{strong}\n\n[targets.strong_format_a]\nid = \"strong/model\"\nllm_client = \"responses\"\n\n[targets.strong_format_b]\nid = \"strong/model\"\nllm_client = \"responses\"\nreasoning_format = \"openai\"" + ), + ); + // The duplicate check runs before per-target validation, so one target on the + // responses client carrying reasoning_format surfaces as a settings conflict. + assert!( + error_message(&format_conflict) + .contains("different reasoning_effort, reasoning_format or extra_body"), + "{}", + error_message(&format_conflict) + ); // An alias with identical settings is still allowed (it only warns). let alias = VALID_CONFIG.replace( strong, @@ -1189,6 +1222,60 @@ new = ["send_message"] Ok(()) } + #[test] + fn reasoning_format_is_rejected_on_non_chat_clients() { + // Reasoning fields are an OpenAI Chat wire concept; the anthropic client rejects it. + let anthropic = VALID_CONFIG.replace( + "[targets.weak]\nid = \"weak/model\"\nllm_client = \"anthropic\"", + "[targets.weak]\nid = \"weak/model\"\nllm_client = \"anthropic\"\nreasoning_format = \"openai\"", + ); + assert!( + error_message(&anthropic) + .contains("reasoning_format is only supported on openai_chat clients"), + "{}", + error_message(&anthropic) + ); + // The responses client rejects it too: the setting only applies to Chat targets. + let responses = VALID_CONFIG.replace( + "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\"", + "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\"\nreasoning_format = \"deepseek\"", + ); + assert!( + error_message(&responses) + .contains("reasoning_format is only supported on openai_chat clients"), + "{}", + error_message(&responses) + ); + } + + #[test] + fn reasoning_format_accepts_openai_and_deepseek_on_chat_clients() -> RunnerResult<()> { + // The classifier target sits on the openai_chat client. + let chat_target = + "[targets.classifier]\nid = \"classifier/model\"\nllm_client = \"primary\""; + let openai = VALID_CONFIG.replace( + chat_target, + &format!("{chat_target}\nreasoning_format = \"openai\""), + ); + runner_from_toml(&openai)?; + let deepseek = VALID_CONFIG.replace( + chat_target, + &format!("{chat_target}\nreasoning_format = \"deepseek\""), + ); + runner_from_toml(&deepseek)?; + // Unknown spellings fail at TOML parse time. + let unknown = VALID_CONFIG.replace( + chat_target, + &format!("{chat_target}\nreasoning_format = \"gpt\""), + ); + assert!( + error_message(&unknown).contains("unknown variant"), + "{}", + error_message(&unknown) + ); + Ok(()) + } + #[test] fn classifier_judge_completion_caps_are_configurable() -> RunnerResult<()> { let capability = VALID_CONFIG.replace( diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 4be77ed869..1b07417cdd 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -106,6 +106,7 @@ endpoint, before it calls an upstream. | `system_prompt` | No | unset | System prompt prepended when this target serves a completion. | | `extra_body` | No | `{}` | Values merged into the upstream request when the request does not already set that key. | | `reasoning_effort` | No | unset | Reasoning effort forced on every request to this target, replacing the value the caller sent (`reasoning.effort` on `openai_responses`, `reasoning_effort` on `openai_chat`). Rejected on `anthropic_messages` clients. Use it to run one target at a different effort than the client asked for, for example a strong tier at `max` behind a client that sends `high`. Targets with different effort settings need distinct model IDs when used within one route. Separate routes may use the same model ID with separate `llm_clients` entries (same endpoint, different name). | +| `reasoning_format` | No | unset | OpenAI Chat field the target expects assistant reasoning under: `"openai"` for `reasoning`, `"deepseek"` for `reasoning_content`. After encoding, assistant reasoning is moved to this field and the other spelling is removed; the two are never sent together. Unset keeps the encoded field unchanged. Rejected on non-`openai_chat` clients. Use it when a model only accepts reasoning replayed under the exact field it produced. Targets with different format settings need distinct model IDs, same as `reasoning_effort` and `extra_body`. | Within one route, callable targets with the same model ID must use the same `llm_client`. This includes completion targets and targets used for judging or classification. Switchyard rejects From a7173e79f0812cf279b95ca5578face0fbfe256d Mon Sep 17 00:00:00 2001 From: Deepak Jain Date: Tue, 22 Sep 2026 09:11:12 -0700 Subject: [PATCH 5/6] fix(client): validate reasoning format and preserve message roles Signed-off-by: Deepak Jain --- crates/libsy-llm-client/src/client.rs | 65 +++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 0dda4d92e4..76fcc8cb17 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -158,6 +158,21 @@ impl TranslatingLlmClient { /// client and the built-in translation codecs. pub fn new(model_configs: &[ModelConfig]) -> Result { for config in model_configs { + if config.reasoning_format.is_some() + && !matches!(config.default_backend, Backend::OpenAiChat(_)) + && !config + .other_backends + .iter() + .flatten() + .any(|backend| matches!(backend, Backend::OpenAiChat(_))) + { + return Err(LlmClientError::Configuration { + message: format!( + "model {} reasoning_format requires an OpenAI Chat backend", + config.model_name + ), + }); + } config .default_backend .validate_extra_headers(&config.model_name)?; @@ -1188,6 +1203,9 @@ fn apply_reasoning_format(body: &mut Value, reasoning_format: ReasoningFormat) { let Some(message) = message.as_object_mut() else { continue; }; + if message.get("role").and_then(Value::as_str) != Some("assistant") { + continue; + } let Some(value) = message.remove(other_field) else { continue; }; @@ -2926,6 +2944,53 @@ mod tests { Ok(()) } + // A configured dialect must have a reachable Chat backend, including fallbacks. + #[test] + fn reasoning_format_requires_a_chat_backend() -> std::result::Result<(), Box> { + for backend in [ + Backend::Anthropic(config("https://example.com")), + Backend::OpenAiResponses(config("https://example.com")), + ] { + let model = ModelConfig::new("gpt", backend.clone(), None); + TranslatingLlmClient::new(std::slice::from_ref(&model))?; + let result = + TranslatingLlmClient::new( + &[model.with_reasoning_format(ReasoningFormat::Deepseek)], + ); + assert!(matches!(result, Err(LlmClientError::Configuration { .. }))); + + let mixed = ModelConfig::new( + "gpt", + backend, + Some(vec![Backend::OpenAiChat(config("https://example.com"))]), + ) + .with_reasoning_format(ReasoningFormat::Deepseek); + let client = TranslatingLlmClient::new(&[mixed])?; + assert!( + client + .backend_for(&ModelId::from("gpt"), WireFormat::OpenAiChat) + .is_some() + ); + } + Ok(()) + } + + // Provider extension fields on other message roles must survive normalization. + #[test] + fn reasoning_format_preserves_non_assistant_messages() { + let untouched = json!([ + {"role": "user", "reasoning": "user extension", "reasoning_content": "keep"}, + {"role": "system", "reasoning_content": "system extension"}, + {"role": "tool", "reasoning": "tool extension"}, + {"reasoning_content": "unknown role"}, + ]); + for format in [ReasoningFormat::OpenAi, ReasoningFormat::Deepseek] { + let mut body = json!({"messages": untouched.clone()}); + apply_reasoning_format(&mut body, format); + assert_eq!(body["messages"], untouched); + } + } + // Raw path, buffered: decode an OpenAI Chat body -> call -> encode back to OpenAI // Chat JSON, with the served `model` restamped over the id the caller addressed. // From e8a3815b841ed4f763dfc2c13e0615ed5e254bc1 Mon Sep 17 00:00:00 2001 From: Deepak Jain Date: Fri, 16 Oct 2026 21:34:58 -0700 Subject: [PATCH 6/6] fix(client): replay reasoning when fields are null Signed-off-by: Deepak Jain --- crates/libsy-llm-client/src/client.rs | 71 +++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 8030c2857d..4ad9ca8fbf 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -1329,8 +1329,11 @@ fn apply_reasoning_format(body: &mut Value, reasoning_format: ReasoningFormat) { if message.get("role").and_then(Value::as_str) != Some("assistant") { continue; } - let value = message.remove(other_field); - if message.contains_key(target_field) { + let value = message.remove(other_field).filter(|value| !value.is_null()); + if message + .get(target_field) + .is_some_and(|value| !value.is_null()) + { continue; } let value = value.or_else(|| { @@ -1354,7 +1357,7 @@ fn apply_reasoning_format(body: &mut Value, reasoning_format: ReasoningFormat) { (!text.is_empty()).then_some(Value::String(text)) }); if let Some(value) = value { - message.entry(target_field).or_insert(value); + message.insert(target_field.to_owned(), value); } } } @@ -3412,6 +3415,68 @@ mod tests { Ok(()) } + #[tokio::test] + async fn reasoning_format_replays_null_fields_with_tools() + -> std::result::Result<(), Box> { + for (format, target_field, other_field) in [ + (ReasoningFormat::Deepseek, "reasoning_content", "reasoning"), + (ReasoningFormat::OpenAi, "reasoning", "reasoning_content"), + ] { + for alternate in [None, Some(Value::Null), Some(json!("plain thoughts"))] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(chat_success_response()) + .expect(1) + .mount(&server) + .await; + let client = TranslatingLlmClient::new(&chat_map_with_reasoning_format( + &format!("{}/v1", server.uri()), + format, + ))?; + let mut request = json!({ + "model": "gpt", + "tools": [{"type": "function", "function": { + "name": "lookup", "parameters": {"type": "object"} + }}], + "messages": [ + {"role": "assistant", "content": null, (target_field): null, + "reasoning_details": [{"type": "reasoning.text", "text": "thoughts"}], + "tool_calls": [{"id": "call_1", "type": "function", "function": { + "name": "lookup", "arguments": "{}" + }}]}, + {"role": "tool", "tool_call_id": "call_1", "content": "result"} + ] + }); + let expected = alternate + .as_ref() + .and_then(Value::as_str) + .unwrap_or("thoughts"); + if let Some(value) = &alternate { + request["messages"][0][other_field] = value.clone(); + } + client + .call_rewrite_model_raw( + request, + None, + Some(&ModelId::new("gpt")), + WireFormat::OpenAiChat, + ) + .await?; + let received = server + .received_requests() + .await + .ok_or("recording disabled")?; + let body: Value = serde_json::from_slice(&received[0].body)?; + assert_eq!(body["messages"][0][target_field], expected); + assert!(body["messages"][0].get(other_field).is_none()); + assert_eq!(body["tools"][0]["function"]["name"], "lookup"); + assert_eq!(body["messages"][1]["tool_call_id"], "call_1"); + } + } + Ok(()) + } + #[tokio::test] async fn reasoning_format_openai_moves_incoming_reasoning_content() -> std::result::Result<(), Box> {