Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/libsy-llm-client/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ fn build_client() -> switchyard_llm_client::Result<TranslatingLlmClient> {
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
reasoning_effort: None,
reasoning_format: Default::default(),
max_retries: 2,
timeout: None,
};
Expand Down
10 changes: 10 additions & 0 deletions crates/libsy-llm-client/src/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use reqwest::RequestBuilder;
use reqwest::header::{HeaderName, HeaderValue};
use serde_json::Value;
use switchyard_protocol::{Metadata, WireFormat};
use switchyard_translation::ReasoningFormat;

use crate::error::{LlmClientError, Result, is_overflow_body};

Expand Down Expand Up @@ -70,6 +71,8 @@ pub struct HttpBackendConfig {
/// sent. Responses carries it as `reasoning.effort`, Chat Completions as `reasoning_effort`;
/// Anthropic has no equivalent and rejects the setting at configuration time.
pub reasoning_effort: Option<String>,
/// Field name used for assistant reasoning replayed to an OpenAI Chat backend.
pub reasoning_format: ReasoningFormat,
Comment thread
chethanuk marked this conversation as resolved.
Resolved
/// Additional attempts after the initial upstream request.
pub max_retries: u32,
/// Deadline for one complete response, including retries, retry delays, and stream reads.
Expand All @@ -87,6 +90,7 @@ impl fmt::Debug for HttpBackendConfig {
.field("extra_body_keys", &self.extra_body.keys())
.field("omit_body_fields", &self.omit_body_fields)
.field("reasoning_effort", &self.reasoning_effort)
.field("reasoning_format", &self.reasoning_format)
.field("max_retries", &self.max_retries)
.field("timeout", &self.timeout)
.finish()
Expand Down Expand Up @@ -314,6 +318,11 @@ impl Backend {
self.config().reasoning_effort.as_deref()
}

/// Field name used for assistant reasoning replayed to this backend.
pub fn reasoning_format(&self) -> ReasoningFormat {
self.config().reasoning_format
}

/// Additional attempts allowed after the initial request.
pub fn max_retries(&self) -> u32 {
self.config().max_retries
Expand Down Expand Up @@ -430,6 +439,7 @@ mod tests {
extra_body: BTreeMap::new(),
omit_body_fields: BTreeSet::new(),
reasoning_effort: None,
reasoning_format: Default::default(),
max_retries: 0,
timeout: None,
}
Expand Down
85 changes: 82 additions & 3 deletions crates/libsy-llm-client/src/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,10 @@ use switchyard_protocol::{
ModelId, Request, Response, RoutedLlmClient,
};
use switchyard_translation::{
TranslationError, WireFormat, decode_aggregated_response, decode_request, decode_stream,
encode_aggregated_response_with_extensions, encode_request, encode_stream_with_extensions,
TargetCapabilities, TranslationError, TranslationPolicy, WireFormat,
decode_aggregated_response, decode_request, decode_stream,
encode_aggregated_response_with_extensions, encode_request_with_policy,
encode_stream_with_extensions,
};
use tracing::Instrument;

Expand Down Expand Up @@ -245,7 +247,14 @@ impl TranslatingLlmClient {
model: &ModelId,
endpoint: UpstreamEndpoint,
) -> Result<EncodedResponse> {
let mut body = encode_request(&llm_request, wire_format)
let policy = TranslationPolicy {
target_capabilities: TargetCapabilities {
reasoning_format: backend.reasoning_format(),
..TargetCapabilities::default()
},
..TranslationPolicy::default()
};
let mut body = encode_request_with_policy(&llm_request, wire_format, &policy)
.map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?;
// `encode_request` round-trips a preserved same-format body verbatim,
// which keeps the caller's original `model`; force the resolved model so
Expand Down Expand Up @@ -1295,6 +1304,7 @@ mod tests {
use wiremock::{Mock, MockServer, ResponseTemplate};

use super::*;
use crate::ReasoningFormat;
use crate::backend::HttpBackendConfig;

fn config(base_url: &str) -> HttpBackendConfig {
Expand All @@ -1306,6 +1316,7 @@ mod tests {
extra_body: BTreeMap::new(),
omit_body_fields: BTreeSet::new(),
reasoning_effort: None,
reasoning_format: Default::default(),
max_retries: 0,
timeout: None,
}
Expand Down Expand Up @@ -1428,6 +1439,12 @@ mod tests {
vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]
}

fn chat_map_with_reasoning_format(base_url: &str, format: ReasoningFormat) -> Vec<ModelConfig> {
let mut backend = config(base_url);
backend.reasoning_format = format;
vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]
}

fn responses_map_with_effort(base_url: &str, effort: &str) -> Vec<ModelConfig> {
let mut backend = config(base_url);
backend.reasoning_effort = Some(effort.to_string());
Expand Down Expand Up @@ -1942,6 +1959,68 @@ mod tests {
Ok(())
}

/// A request rebuilt for an OpenAI Chat backend sends assistant reasoning under
/// the backend's configured field name, and never under both names.
#[tokio::test]
async fn rebuilt_chat_request_uses_the_backends_reasoning_field()
-> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
for (format, expected, absent) in [
(ReasoningFormat::OpenAi, "reasoning", "reasoning_content"),
(ReasoningFormat::DeepSeek, "reasoning_content", "reasoning"),
] {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"id": "1",
"model": "gpt",
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop"
}],
"usage": {}
})))
.mount(&server)
.await;
let client = TranslatingLlmClient::new(&chat_map_with_reasoning_format(
&format!("{}/v1", server.uri()),
format,
))?;
// A Responses caller forces the Chat body to be rebuilt from the IR.
client
.call_rewrite_model_raw(
json!({
"model": "client-facing",
"input": [
{"type": "message", "role": "user", "content": "hi"},
{"type": "reasoning", "content": [{"type": "reasoning_text", "text": "thinking..."}], "summary": []},
{"type": "message", "role": "assistant", "content": "hello"},
{"type": "message", "role": "user", "content": "next"}
]
}),
None,
Some(&ModelId::from("gpt")),
WireFormat::OpenAiResponses,
)
.await?;
let received = server
.received_requests()
.await
.ok_or("request recording should be enabled")?;
let received = received.first().ok_or("expected one upstream request")?;
let body: Value = serde_json::from_slice(&received.body)?;
let assistant = &body["messages"][1];
assert_eq!(assistant["role"], "assistant", "{body}");
assert_eq!(assistant[expected], "thinking...", "{format:?}: {body}");
assert!(
assistant.get(absent).is_none(),
"{format:?} also sent {absent}: {body}"
);
}
Ok(())
}

/// A configured reasoning effort replaces the caller's value on both OpenAI wire formats,
/// which `extra_body` (defaults only) cannot do.
#[tokio::test]
Expand Down
2 changes: 1 addition & 1 deletion crates/libsy-llm-client/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ pub use error::{LlmClientError, Result};
pub use observation::{LlmCallObservation, RunObservation, RunObserver};
pub use raw::RawResponse;
pub use run::{ClientRouter, decide, run};
pub use switchyard_translation::RawEventStream;
pub use switchyard_translation::{RawEventStream, ReasoningFormat};

/// Registers process-wide compatibility gauges with the global meter provider.
pub fn initialize_metrics() {
Expand Down
4 changes: 4 additions & 0 deletions crates/libsy-llm-client/src/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1212,6 +1212,7 @@ mod tests {
extra_body: BTreeMap::from([("store".to_string(), json!(store))]),
omit_body_fields: BTreeSet::new(),
reasoning_effort: None,
reasoning_format: Default::default(),
max_retries: 0,
timeout: None,
};
Expand Down Expand Up @@ -1358,6 +1359,7 @@ mod tests {
extra_body: BTreeMap::new(),
omit_body_fields: BTreeSet::new(),
reasoning_effort: None,
reasoning_format: Default::default(),
max_retries: 0,
timeout: None,
}),
Expand Down Expand Up @@ -2030,6 +2032,7 @@ mod tests {
extra_body: BTreeMap::new(),
omit_body_fields: BTreeSet::new(),
reasoning_effort: None,
reasoning_format: Default::default(),
max_retries: 2,
timeout: None,
})
Expand Down Expand Up @@ -2129,6 +2132,7 @@ mod tests {
extra_body: BTreeMap::new(),
omit_body_fields: BTreeSet::new(),
reasoning_effort: None,
reasoning_format: Default::default(),
max_retries: 0,
timeout: None,
})
Expand Down
Loading