diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index 691e169cf6..73169982c0 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -57,7 +57,7 @@ pub async fn run( request: Request, models: Arc, observer: Option, -) -> Result<(ModelId, Response)> { +) -> Result<(ModelId, Response, Option)> { let algorithm_name = algorithm.name().to_string(); let run_started = Instant::now(); let routing_clients = clients.clone(); @@ -85,6 +85,10 @@ pub async fn run( metrics::record_routing_overhead(&algorithm_name, overhead); let selected_model_id = outcome.selected_model_id()?.clone(); + // Captured before `outcome.metadata` is moved into the observer below. This is the + // algorithm's own record of WHY it chose, and the server needs it on the wire: the + // response body and the selected-model header both describe WHAT was chosen, never why. + let metadata = outcome.metadata.clone(); if let Some(observer) = &observer && let Some(metadata) = outcome.metadata { @@ -118,7 +122,7 @@ pub async fn run( if let Some(observer) = &observer { observer(RunObservation::RoutingOverhead(overhead)); } - result.map(|response| (selected_model_id, response)) + result.map(|response| (selected_model_id, response, metadata)) } /// Run an algorithm to a routing decision without serving its terminal completion. @@ -1085,6 +1089,7 @@ mod tests { None, ) .await; + let result = result.map(|(selected, response, _metadata)| (selected, response)); (client, result) } @@ -1109,7 +1114,7 @@ mod tests { let observed = Arc::clone(&observations); let observer: RunObserver = Arc::new(move |event| observed.lock().push(event)); - let (selected, response) = run( + let (selected, response, _metadata) = run( Arc::new(AnsweredAlgorithm { model: "weak".into(), }), @@ -1235,7 +1240,7 @@ mod tests { .fields .insert("conversation".to_string(), conversation.clone()); } - let (_, response) = run( + let (_, response, _metadata) = run( Arc::new(switchyard_libsy::Passthrough), clients.clone(), seed, @@ -1385,7 +1390,7 @@ mod tests { raw_request: None, metadata: None, }; - let (_, response) = run( + let (_, response, _metadata) = run( Arc::new(switchyard_libsy::Passthrough), clients.clone(), seed, @@ -1423,7 +1428,7 @@ mod tests { raw_request: None, metadata: None, }; - let (_, response) = run( + let (_, response, _metadata) = run( Arc::new(switchyard_libsy::Passthrough), clients.clone(), follow, @@ -1462,7 +1467,7 @@ mod tests { assert_eq!(history.len, 4); assert_eq!(history.segment.len(), 2); assert!(history.parent.is_some()); - let (_, response) = run( + let (_, response, _metadata) = run( Arc::new(switchyard_libsy::Passthrough), clients, third, @@ -2156,6 +2161,7 @@ mod tests { None, ) .await; + let result = result.map(|(selected, response, _metadata)| (selected, response)); (server, calls, result) } diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index 93c12adb5e..90d476efe7 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -600,6 +600,7 @@ async fn run( None, ) .await + .map(|(selected, response, _metadata)| (selected, response)) } fn classifier_router() -> switchyard_libsy::Result> { @@ -647,6 +648,7 @@ async fn run_classifier( None, ) .await + .map(|(selected, response, _metadata)| (selected, response)) } fn classifier_request() -> Request { @@ -875,7 +877,7 @@ async fn affinity_keeps_the_algorithm_selection_after_client_fallback() })?) as Arc; let request = request_with_metadata("affinity-fallback-session", "affinity-fallback-first"); - let (selected, first_response) = switchyard_llm_client::run( + let (selected, first_response, _) = switchyard_llm_client::run( Arc::clone(&router), ClientRouter::single(client.clone()), request.clone(), @@ -938,7 +940,7 @@ async fn affinity_keeps_the_algorithm_selection_after_client_fallback() } client.efficient_available.store(true, Ordering::Relaxed); - let (selected, second_response) = switchyard_llm_client::run( + let (selected, second_response, _) = switchyard_llm_client::run( router, ClientRouter::single(client.clone()), request, @@ -1289,7 +1291,7 @@ async fn stage_router_records_algorithm_owned_metrics() -> switchyard_libsy::Res usage: Usage::default(), }) as Arc; - let (selected_model, _) = switchyard_llm_client::run( + let (selected_model, _, _) = switchyard_llm_client::run( algorithm, ClientRouter::single(client), request, @@ -1343,7 +1345,7 @@ async fn observed_run_reports_one_successful_routed_call() -> switchyard_libsy:: usage: Usage::default(), }) as Arc; - let (_, response) = switchyard_llm_client::run( + let (_, response, _) = switchyard_llm_client::run( algo(ALGO, MODEL), ClientRouter::single(client), request_with_metadata("observed-session", "observed-correlation"), diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs index 9d177c7684..6ae76968f9 100644 --- a/crates/switchyard-runner/src/route.rs +++ b/crates/switchyard-runner/src/route.rs @@ -6,7 +6,7 @@ use std::error::Error; use std::sync::Arc; -use libsy::{Algorithm, LibsyError, RoutingOutcome, RuntimeModels}; +use libsy::{Algorithm, LibsyError, OutcomeMetadata, RoutingOutcome, RuntimeModels}; use serde_json::Value; use switchyard_llm_client::{AuxiliaryOperation, ClientRouter, RunObserver, TranslatingLlmClient}; use switchyard_protocol::{LlmClientError, ModelId, Request, Response, WireFormat}; @@ -124,6 +124,9 @@ pub struct Route { pub struct RunOutput { pub selected_model: ModelId, pub response: Response, + /// The algorithm's record of why it chose, carried so the server can expose the + /// decision's provenance on the wire. `None` when the algorithm produced no metadata. + pub metadata: Option, } impl Route { @@ -195,7 +198,7 @@ impl Route { request: Request, observer: Option, ) -> Result { - let (selected_model, response) = switchyard_llm_client::run( + let (selected_model, response, metadata) = switchyard_llm_client::run( Arc::clone(&self.algorithm), self.clients.clone(), request, @@ -206,6 +209,7 @@ impl Route { Ok(RunOutput { selected_model, response, + metadata, }) } diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index fe5bdef83e..cbc2b7a6ea 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -35,7 +35,7 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Extension, Json, Router}; use axum_server::tls_rustls::RustlsConfig; -use libsy::{LibsyError, RoutingOutcome}; +use libsy::{LibsyError, OutcomeMetadata, RoutingOutcome}; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -65,6 +65,15 @@ pub const DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30); pub const DEFAULT_MAX_REQUEST_BODY_BYTES: usize = 32 * 1024 * 1024; const HEADER_SELECTED_MODEL: &str = "x-model-router-selected-model"; + +/// Why the router chose this model. Paired with [`HEADER_SELECTED_MODEL`], which reports +/// WHAT was chosen; neither is meaningful without the other. +/// +/// `docs/routing_algorithms/stage_router_routing.md` describes "two routing headers" but +/// only ever listed the selected-model one. The deciding component was already recorded +/// internally as `DecisionSource` and carried as `OutcomeMetadata::evidence`, so this +/// header forwards existing state rather than introducing a new mechanism. +const HEADER_DECISION_SOURCE: &str = "x-model-router-decision-source"; const FORWARDED_UPSTREAM_HEADERS: &[&str] = &[ "baggage", "openai-processing-ms", @@ -987,6 +996,18 @@ fn resolve_route( "invalid_request_error", ) })?; + // Every wire format normalizes turns into `messages`, so an empty vec means + // the body carried no conversation at all. Reject it here, next to the + // `model` check, so a client error costs no upstream round trip and is not + // reported as an upstream failure. + if llm_request.messages.is_empty() { + return Err(error_response( + StatusCode::BAD_REQUEST, + "request body must include a non-empty `messages` array", + "invalid_request_error", + "invalid_request_error", + )); + } let route = state.route_for_model(&requested_model).ok_or_else(|| { error_response( StatusCode::NOT_FOUND, @@ -1076,6 +1097,7 @@ async fn handle_llm_request( let RunOutput { selected_model, response, + metadata, } = output; // The response carries the candidate that actually served it. Fall back to the routing // selection for algorithms that return a response without an offloaded model call. @@ -1119,7 +1141,7 @@ async fn handle_llm_request( response_headers.append(name.clone(), value.clone()); } if let Some(served_model) = served_model.as_ref() { - attach_routing_headers(&mut response, served_model.as_str()); + attach_routing_headers(&mut response, served_model.as_str(), metadata.as_ref()); } response } @@ -1238,8 +1260,25 @@ fn metadata_from_headers(headers: HeaderMap) -> Metadata { metadata } -fn attach_routing_headers(response: &mut Response, served_model: &str) { +fn attach_routing_headers( + response: &mut Response, + served_model: &str, + metadata: Option<&OutcomeMetadata>, +) { insert_routing_header(response, HEADER_SELECTED_MODEL, served_model); + if let Some(source) = metadata.and_then(decision_source_of) { + insert_routing_header(response, HEADER_DECISION_SOURCE, &source); + } +} + +/// Extracts the deciding component from an outcome's evidence. +/// +/// Built-in algorithms record a stable `source` string (`override`, `dimensions`, +/// `llm-classifier`, `fall_open`, ...) in their evidence. Anything else - a custom +/// algorithm, a route with no evidence at all - yields `None`, which simply omits the +/// header rather than inventing a reason. +fn decision_source_of(metadata: &OutcomeMetadata) -> Option { + metadata.evidence.as_ref()?.get("source")?.as_str().map(str::to_string) } fn insert_routing_header(response: &mut Response, name: &'static str, value: &str) { diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 3c26562577..04c31c91ed 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -4715,6 +4715,150 @@ async fn request_and_upstream_errors_use_the_inbound_wire_format() -> TestResult Ok(()) } +// A body with a valid model but no `messages` is a client error. It must be +// rejected with 400 before any upstream call, not forwarded and reported as an +// upstream 502. +#[tokio::test] +async fn missing_messages_is_rejected_before_upstream() -> TestResult { + let (upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?; + + let before = upstream.calls.lock().await.len(); + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({"model": ROUTE_MODEL})), + ) + .await?; + + assert_eq!(response.status, StatusCode::BAD_REQUEST); + assert_eq!(response.json()?["error"]["code"], "invalid_request_error"); + assert_eq!( + upstream.calls.lock().await.len(), + before, + "no upstream call should be made for an invalid body" + ); + Ok(()) +} + +/// The routing decision's provenance must reach the caller. +/// +/// `docs/routing_algorithms/stage_router_routing.md` states that "each response carries +/// two routing headers", but the table below it lists only +/// `x-model-router-selected-model`. The algorithm already records which component decided +/// the turn (`DecisionSource`) and libsy already carries it as +/// `OutcomeMetadata::evidence` with a stable `source` field; that evidence is simply never +/// forwarded to the wire. This test pins the missing half so the documented contract and +/// the response agree. +#[tokio::test] +async fn routing_response_exposes_decision_source() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" + +[targets.strong] +id = "model/provenance-strong" +llm_client = "upstream" + +[targets.weak] +id = "model/provenance-weak" +llm_client = "upstream" + +[routes.stage] +id = "switchyard/stage" +type = "stage_router" +capable_target = "strong" +efficient_target = "weak" +picker = "efficient_first" +confidence_threshold = 0.5 +"#, + base_url = upstream.base_url + ))?; + let app = build_switchyard_router(state); + + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/stage", + "messages": [ + {"role": "user", "content": "fix the build"}, + {"role": "assistant", "tool_calls": [{ + "id": "call_1", + "type": "function", + "function": {"name": "Bash", "arguments": "{\"command\": \"cargo test\"}"} + }]}, + {"role": "tool", "tool_call_id": "call_1", "content": "fatal runtime error: out of memory"}, + ] + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/provenance-strong"), + "the selected-model header must keep working" + ); + assert_eq!( + response + .headers + .get("x-model-router-decision-source") + .and_then(|value| value.to_str().ok()), + Some("override"), + "a critical error is an `override` decision and the caller must be able to see why" + ); + Ok(()) +} + +/// A route that records no decision evidence must omit the header entirely. +/// +/// The negative case matters as much as the positive one: a missing decision must be +/// reported as missing. Emitting an empty value, or a guessed label such as `default`, +/// would be worse than silence, because a caller reading provenance cannot distinguish +/// "this algorithm does not record one" from "this one decided, and here is why". +#[tokio::test] +async fn routing_response_omits_decision_source_without_evidence() -> TestResult { + let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?; + + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": "hello"}], + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/a"), + "the selected-model header must still be present" + ); + assert!( + !response + .headers + .contains_key("x-model-router-decision-source"), + "no decision evidence must mean no decision-source header, not an invented one" + ); + Ok(()) +} + /// A `type = "advisor"` deployment: gated executor + reviewer on one mock upstream. fn advisor_state(base_url: &str) -> TestResult { load_test_config(&format!( diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index 182013ef35..abd8c119c3 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -306,6 +306,7 @@ header: | Header | Content | |---|---| | `x-model-router-selected-model` | The model ID that served the response, including any model-call fallback. | +| `x-model-router-decision-source` | Which component decided the turn: `override`, `capable_hold`, `dimensions`, `llm-classifier`, or `fall_open`. Omitted when the algorithm records no decision evidence. | ### Decision sources @@ -320,6 +321,10 @@ paths through its cascade: | `llm-classifier` | The signals were ambiguous and the classifier returned a usable verdict. | | `fall_open` | The signal scorer abstained and the classifier was absent or could not decide, so the picker default was used. | +Those labels are also forwarded to the caller as `x-model-router-decision-source`, read +from the same evidence that feeds `routing_decisions` in the stats. The header and the +stats counter are two views of one recorded fact, not two independent signals. + ## When *not* to use stage-router - **Single-model deployments.** Use a `passthrough` route instead.