diff --git a/CHANGELOG.md b/CHANGELOG.md index 25da0ddfd5..96014c62fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,11 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). before awaiting it. See the [host integration guide](docs/getting_started.md#drive-the-algorithm). ([#948](https://github.com/NVIDIA-NeMo/Switchyard/pull/948)) +- **Fallback on capability rejections** - an upstream 400 saying the model cannot + take this input, such as image input on a text-only model, now moves the answer + call to the route's next candidate. Calls made during routing still stop on it. + **Rust source-breaking change:** callers that match exhaustively on + `RoutingFallbackReason` must handle the new `Capability` variant. ### Changed diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index d660f8c987..19a9c29c50 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -37,6 +37,7 @@ use switchyard_protocol::{ }; use switchyard_translation::prepare_request_for_target; +use crate::error::is_overflow_body; use crate::observation::{ModelCallObservation, RunObservation, RunObserver}; use crate::{metrics, observability}; @@ -518,6 +519,15 @@ async fn buffer_routing_stream( Ok((stream::iter(events.into_iter().map(Ok)).boxed(), usage)) } +// Upstream wording for "this model cannot take this input at all". Kept narrow: +// a miss keeps today's error, a false positive moves the request to another target. +const CAPABILITY_PHRASES: &[&str] = &[ + "does not support image input", // Fireworks, Moonshot + "is not a multimodal model", // vLLM; the model name prefix varies + "image input is not supported", // llama.cpp wording, when a proxy answers 400 + "image_url is only supported by certain models", // OpenAI +]; + /// Whether a failed candidate is worth routing around. fn fallback_reason( error: &LibsyError, @@ -538,6 +548,13 @@ fn fallback_reason( if policy_denial && !allow_policy_fallback { return None; } + // Reuses the overflow helper's message parsing with non-overflow phrases. + if *status == StatusCode::BAD_REQUEST + && !policy_denial + && is_overflow_body(body, |_| false, CAPABILITY_PHRASES) + { + return Some(RoutingFallbackReason::Capability); + } ((*status == StatusCode::BAD_REQUEST && policy_denial) || matches!( *status, @@ -1211,6 +1228,7 @@ mod tests { ContextWindow, ContentPolicy(StatusCode), Unauthorized, + BadRequest(&'static str), StreamSuccess, MidStreamError, } @@ -1241,6 +1259,10 @@ mod tests { status: StatusCode::UNAUTHORIZED, body: "unauthorized".to_string(), }), + FirstOutcome::BadRequest(body) => Err(LlmClientError::UpstreamHttp { + status: StatusCode::BAD_REQUEST, + body: body.to_string(), + }), FirstOutcome::StreamSuccess => Ok(stream_response(vec![ LlmResponseChunk::TextDelta { index: 0, @@ -2475,9 +2497,13 @@ mod tests { Ok(()) } + const IMAGE_REJECT: &str = r#"{"error":{"type":"invalid_request_error","message":"This model does not support image inputs"}}"#; + #[test] fn fallback_only_accepts_context_policy_and_unavailable_failures() { let error = |source| LibsyError::client_call("target", source); + let capability = Some(RoutingFallbackReason::Capability); + assert_eq!(RoutingFallbackReason::Capability.as_str(), "capability"); assert_eq!( fallback_reason( &error(LlmClientError::ContextWindowExceeded { @@ -2517,6 +2543,56 @@ mod tests { ), (r#"{"error":{"message":"content_policy_violation"}}"#, None), ("content_policy_violation", None), + (IMAGE_REJECT, capability), + ( + r#"{"error":{"message":"judge is not a multimodal model","type":"BadRequestError","code":400}}"#, + capability, + ), + ( + r#"{"error":{"message":"image input is not supported - hint: provide the mmproj"}}"#, + capability, + ), + ( + r#"{"error":{"message":"Invalid content type. image_url is only supported by certain models."}}"#, + capability, + ), + ( + r#"{"object":"error","message":"m is not a multimodal model"}"#, + capability, + ), + ("This model does not support image inputs", capability), + ( + r#"{"error":{"message":"THIS MODEL DOES NOT SUPPORT IMAGE INPUT"}}"#, + capability, + ), + ( + r#"{"error":{"code":"invalid_request_error","message":"does not support image input"}}"#, + capability, + ), + (r#"{"error":{"message":"invalid image data"}}"#, None), + ( + r#"{"error":{"message":"This model does not support images larger than 20MB"}}"#, + None, + ), + ( + r#"{"error":{"message":"does not support image generation"}}"#, + None, + ), + (r#"{"error":{"message":"image too large"}}"#, None), + ( + r#"{"error":{"message":"Multimodal processing failed"}}"#, + None, + ), + (r#"{"error":{"message":"bad request"}}"#, None), + ( + r#"{"error":{"message":"response_format json_schema is not supported"}}"#, + None, + ), + ( + r#"{"error":{"message":"rejected"},"metadata":"does not support image input"}"#, + None, + ), + (r#"{"message":"does not support image input"}"#, None), ] { assert_eq!( fallback_reason( @@ -2549,6 +2625,39 @@ mod tests { None ); } + + let mmproj = + r#"{"error":{"message":"image input is not supported - hint: provide the mmproj"}}"#; + let policy = r#"{"error":{"code":"content_policy_violation","message":"does not support image input"}}"#; + let unavailable = Some(RoutingFallbackReason::Unavailable); + for (status, body, allow_policy_fallback, expected) in [ + (401, IMAGE_REJECT, true, None), + (404, IMAGE_REJECT, true, None), + (422, IMAGE_REJECT, true, None), + (403, IMAGE_REJECT, true, unavailable), + (408, IMAGE_REJECT, true, unavailable), + (429, IMAGE_REJECT, true, unavailable), + (500, IMAGE_REJECT, true, unavailable), + (503, IMAGE_REJECT, true, unavailable), + (500, mmproj, true, unavailable), + // A policy denial keeps its own result even when the message also matches. + (400, policy, true, unavailable), + (400, policy, false, None), + // The next candidate serves the arrived request, so the flag does not apply. + (400, IMAGE_REJECT, false, capability), + ] { + assert_eq!( + fallback_reason( + &error(LlmClientError::UpstreamHttp { + status: StatusCode::from_u16(status).expect("valid status"), + body: body.to_string(), + }), + allow_policy_fallback, + ), + expected, + "{status} {body} {allow_policy_fallback}" + ); + } } #[tokio::test] @@ -2579,6 +2688,29 @@ mod tests { ); assert_eq!(response.served_model().map(ModelId::as_str), Some("strong")); + // A model that cannot take the input at all also permits another candidate. + let (client, result) = run_candidates(FirstOutcome::BadRequest(IMAGE_REJECT)).await; + let (_, response) = result?; + assert_eq!( + &*client.calls.lock(), + &[ModelId::from("weak"), "strong".into()] + ); + assert_eq!(response.served_model().map(ModelId::as_str), Some("strong")); + + // Any other 400 stays terminal, so the second candidate is untouched. + let (client, result) = run_candidates(FirstOutcome::BadRequest("failed")).await; + assert!(matches!( + result, + Err(LibsyError::ClientCall { + source: LlmClientError::UpstreamHttp { + status: StatusCode::BAD_REQUEST, + .. + }, + .. + }) + )); + assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]); + // Authentication failure is not retryable, so the second candidate is untouched. let (client, result) = run_candidates(FirstOutcome::Unauthorized).await; assert!(matches!( @@ -2667,6 +2799,32 @@ mod tests { Ok(()) } + #[tokio::test] + async fn capability_rejection_gives_the_next_candidate_the_fallback_request() -> Result<()> { + let client = Arc::new(CandidateClient { + calls: Mutex::new(Vec::new()), + requests: Mutex::new(Vec::new()), + first: FirstOutcome::BadRequest(IMAGE_REJECT), + }); + run( + Arc::new(RewritingAlgorithm {}), + ClientRouter::single(client.clone()), + request(), + to_category_map(&["weak", "strong"]), + None, + ) + .await?; + assert_eq!( + &*client.calls.lock(), + &[ModelId::from("weak"), "strong".into()] + ); + assert_eq!( + client.requests.lock()[1].llm_request.messages, + request().llm_request.messages + ); + Ok(()) + } + #[tokio::test] async fn retry_exhaustion_falls_back_and_optional_cooldown_skips_failed_backend() -> Result<()> { diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rs index f791bd8f98..848a9c59a3 100644 --- a/crates/protocol/src/client.rs +++ b/crates/protocol/src/client.rs @@ -135,6 +135,8 @@ pub enum RoutingFallbackReason { ContextWindow, /// The selected target was unavailable after its client retries finished. Unavailable, + /// The selected target rejected the request as something it cannot serve at all. + Capability, } impl RoutingFallbackReason { @@ -143,6 +145,7 @@ impl RoutingFallbackReason { match self { Self::ContextWindow => "context_window", Self::Unavailable => "unavailable", + Self::Capability => "capability", } } } diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 7261261d8d..1ca776afb5 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -213,6 +213,38 @@ async fn upstream_chat( ) .into_response(); } + if model == "model/weak" && prompt == "capability-reject" { + return ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": { + "type": "invalid_request_error", + "message": "This model does not support image inputs" + } + })), + ) + .into_response(); + } + if prompt == "all-capability-reject" { + return ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": { + "message": "judge is not a multimodal model", + "type": "BadRequestError", + "code": 400 + } + })), + ) + .into_response(); + } + if model == "model/weak" && prompt == "invalid-image-data" { + return ( + StatusCode::BAD_REQUEST, + Json(json!({"error": {"message": "invalid image data"}})), + ) + .into_response(); + } if prompt == "upstream-headers" { // Both the buffered and the streamed reply echo the same set, so the two // capture paths are compared against one expectation. @@ -4958,6 +4990,177 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted Ok(()) } +#[tokio::test] +async fn capability_rejection_tries_the_next_candidate() -> TestResult { + let upstream = MockUpstream::start().await?; + // The fixed seed selects `model/weak` first while keeping `model/strong` enabled. + let app = build_switchyard_router(weighted_random_state(&upstream.base_url, [1000, 1])?); + let cases = [ + ( + "/v1/chat/completions", + json!({ + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": "capability-reject"}] + }), + ), + ( + "/v1/messages", + json!({ + "model": ROUTE_MODEL, + "max_tokens": 16, + "messages": [{"role": "user", "content": "capability-reject"}] + }), + ), + ( + "/v1/responses", + json!({"model": ROUTE_MODEL, "input": "capability-reject"}), + ), + ]; + + for (path, body) in cases { + let previous_call_count = upstream.calls.lock().await.len(); + let response = send(&app, "POST", path, Some(body)).await?; + assert_eq!(response.status, StatusCode::OK, "{path}"); + assert_eq!( + response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/strong"), + "{path}" + ); + assert_eq!(response.json()?["model"], "model/strong", "{path}"); + assert_eq!( + upstream.models().await[previous_call_count..], + ["model/weak", "model/strong"], + "{path}" + ); + } + + let stats = send(&app, "GET", "/v1/stats", None).await?.json()?; + assert_eq!( + stats["routing_fallbacks"], + json!({"context_window": 0, "unavailable": 0}) + ); + + // The rejection is an HTTP status, so it arrives before any stream byte. + let previous_call_count = upstream.calls.lock().await.len(); + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": "capability-reject"}], + "stream": true + })), + ) + .await?; + assert_eq!(response.status, StatusCode::OK); + assert!(response.text()?.contains("data: [DONE]")); + assert_eq!( + upstream.models().await[previous_call_count..], + ["model/weak", "model/strong"] + ); + + // The caller gets the last target's error when every target rejects. + let previous_call_count = upstream.calls.lock().await.len(); + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": "all-capability-reject"}] + })), + ) + .await?; + assert_eq!(response.status, StatusCode::BAD_REQUEST); + let error = response.json()?; + assert_eq!(error["error"]["type"], "upstream_error"); + assert_eq!(error["error"]["code"], "upstream_error"); + assert!( + error["error"]["message"] + .as_str() + .is_some_and(|message| message.contains("is not a multimodal model")) + ); + assert_eq!( + upstream.models().await[previous_call_count..], + ["model/weak", "model/strong"] + ); + + // Any other 400 stops on the first target. + let previous_call_count = upstream.calls.lock().await.len(); + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": "invalid-image-data"}] + })), + ) + .await?; + assert_eq!(response.status, StatusCode::BAD_REQUEST); + assert_eq!( + upstream.models().await[previous_call_count..], + ["model/weak"] + ); + Ok(()) +} + +#[tokio::test] +async fn capability_rejection_on_a_routing_phase_call_keeps_the_400() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" + +[targets.classifier] +id = "model/strong" +llm_client = "upstream" + +[targets.strong] +id = "model/strong" +llm_client = "upstream" + +[targets.weak] +id = "model/weak" +llm_client = "upstream" + +[routes.escalation] +id = "switchyard/escalation" +type = "llm_classifier" +mode = "escalation" +classifier_target = "classifier" +strong_target = "strong" +weak_target = "weak" +escalation = {{ confirmations = 1 }} +"#, + base_url = upstream.base_url + ))?; + let app = build_switchyard_router(state); + + let response = send_with_headers( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/escalation", + "messages": [{"role": "user", "content": "capability-reject"}] + })), + &[("x-switchyard-session-id", "capability-reject")], + ) + .await?; + assert_eq!(response.status, StatusCode::BAD_REQUEST); + assert_eq!(upstream.models().await, ["model/weak"]); + Ok(()) +} + #[tokio::test] async fn streaming_response_is_framed_for_the_inbound_api() -> TestResult { let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?; diff --git a/docs/operations/context_window.md b/docs/operations/context_window.md index 6d9f5a24ee..25ec472bed 100644 --- a/docs/operations/context_window.md +++ b/docs/operations/context_window.md @@ -24,6 +24,26 @@ call fails as an overflow and falls through before anything reaches the client. An error event arriving after content has streamed is not retried — delivered output cannot be taken back — and surfaces to the client as before. +## Targets that cannot serve the request + +The same fallback applies when an upstream answers HTTP 400 and the message says +the model cannot take this input at all, such as `does not support image input` +or `is not a multimodal model`. The phrase list is short on purpose: any other 400 +fails on the spot as before. The runtime log line for the fallback gives the reason +`capability`; the routing log does not record reasons. A rejecting target is not +remembered, so it is tried again on the next request. If every target rejects, the +caller gets the last target's error. + +This covers the candidate list for the answer call, as on `random` and +`stage_router` routes. Calls made during routing still stop on the 400. That +includes the efficient-tier call on `llm_classifier` routes with +`mode = "escalation"` and on `vgr` routes, even when the capable tier could serve +the request. Hosts using the C interface run their own answer loop. Rejections of +unsupported tool schemas are not recognized. + +`vision` is a route key, so `vision = false` rejects images for the whole route. +It cannot mark one target of a route as text-only. + ## Configuration There is no eviction key. A route falls through whenever it has more than one diff --git a/docs/operations/troubleshooting.md b/docs/operations/troubleshooting.md index 4a912d4694..506234b678 100644 --- a/docs/operations/troubleshooting.md +++ b/docs/operations/troubleshooting.md @@ -16,7 +16,7 @@ log field, a log line, or a `--dry-run` message. | Requests hang or take much longer than the model needs | Server log for retries and cooldown; `/v1/stats` call counts per model | A judge or tier without `timeout_ms`; a judge call on every tool step; a dead tier retried before fallback. | Give the judge its own `[llm_clients]` entry with `timeout_ms`. On a Task or Custom `llm_classifier` route, set `classify_trigger = "user_turn"` and send a session ID. See the dead-tier row below. | | HTTP 400 from the upstream when the client sends a thinking level | The upstream error body in the response | A Claude target on an OpenAI wire format behind a gateway. | [Thinking](../integrations/pi.md#thinking) and `omit_body_fields`. | | HTTP 400 with code `context_length_exceeded` | The response body | Every target on the route overflowed. | [Context-Window Handling](context_window.md). | -| HTTP 400 on requests that contain images | The response body | The route sets `vision = false`. | Remove it, or declare `true` only when every target accepts images. See the [route keys](../reference/toml_schema.md#routesname). | +| HTTP 400 on requests that contain images | The response body | If the message says the route declares `vision = false`, the route sets it. If it is an upstream error, the upstream rejected the image and no candidate could serve it, or the rejection came from a call made during routing, which does not fall back. | Remove `vision = false`, or declare `true` only when every target accepts images. See the [route keys](../reference/toml_schema.md#routesname). For an upstream rejection, see [Context-Window Handling](context_window.md#targets-that-cannot-serve-the-request). | | The server does not start: `unknown field`, or `have no effect on the stage fallback judge` | The `--dry-run` message | An `auto` route with a tuning key; a typo; a key from another route type; `classifier.classify_trigger` other than the default, or `classifier.message_hash_fallback`, on a `stage_router` route. | [Auto](../reference/toml_schema.md#auto), the [stage_router keys](../reference/toml_schema.md#stage_router), and the [validation errors](../reference/toml_schema.md#validation-errors). | | The server does not start: two targets `both name model` | The `--dry-run` message | Two targets share a model ID on one client with different `reasoning_effort`, `extra_body`, or `omit_body_fields`. | [Reasoning effort](../routing_algorithms/model_selection.md#reasoning-effort). | | The client shows zero input tokens on `/v1/messages` | `prompt_tokens` in the routing log | The streamed `message_start` event reports zero usage when the target is on an OpenAI format. The real counts arrive in `message_delta`. | [Streamed token counts](../routing_algorithms/model_selection.md#pin-the-provider-and-endpoint). |