diff --git a/src/anthropic/translate.rs b/src/anthropic/translate.rs index a150026c..ef5f2d4b 100644 --- a/src/anthropic/translate.rs +++ b/src/anthropic/translate.rs @@ -761,6 +761,14 @@ pub fn request_to_openai(anthropic: &Value, model: &str) -> Value { } openai.insert("messages".into(), Value::Array(messages)); + // One limit, copied as is. Anthropic's `max_tokens` counts thinking (its + // thinking budget must fit inside it), and once InferenceInfra#5737 ships + // the gateway's does too: a hard total of everything the model generates, + // with no server allowance on top. `thinking.budget_tokens` has no gateway field and is dropped; only + // `thinking.type: disabled` crosses, as `reasoning_effort: none`. So a + // model that thinks by default may spend all of `max_tokens` thinking + // (deepseek-v4.1-flash was seen at 24,426 of 32,000). Mapping + // `budget_tokens` onto a `reasoning_effort` level is possible later. if let Some(max_tokens) = anthropic.get("max_tokens") { openai.insert("max_tokens".into(), max_tokens.clone()); } @@ -1746,6 +1754,32 @@ mod tests { .is_none()); } + /// Anthropic's `max_tokens` already includes thinking (the thinking budget + /// must fit inside it), which is what the gateway's `max_tokens` means + /// too, so it crosses 1:1 and is the whole budget. `budget_tokens` has no + /// gateway field and is not sent, and nothing is added to make room for + /// thinking: no `max_completion_tokens`, no second limit. + #[test] + fn max_tokens_crosses_as_the_whole_budget_and_budget_tokens_is_not_sent() { + let request = json!({ + "max_tokens": 32_000, + "thinking": {"type": "enabled", "budget_tokens": 24_000}, + "messages": [{"role": "user", "content": "hi"}], + }); + let body = request_to_openai(&request, "deepseek-v4.1-flash"); + assert_eq!(body["max_tokens"], 32_000); + assert!(body.get("max_completion_tokens").is_none(), "{body}"); + assert!(!body.to_string().contains("budget_tokens"), "{body}"); + assert!(body.get("reasoning_effort").is_none(), "{body}"); + + // No limit asked, none sent: the gateway applies the model default. + let unbounded = request_to_openai( + &json!({"messages": [{"role": "user", "content": "hi"}]}), + "deepseek-v4.1-flash", + ); + assert!(unbounded.get("max_tokens").is_none(), "{unbounded}"); + } + #[test] fn upstream_failure_no_status_is_api_error() { match upstream_failure(0, "") { diff --git a/src/harness/agents.rs b/src/harness/agents.rs index b76c974b..acf1b436 100644 --- a/src/harness/agents.rs +++ b/src/harness/agents.rs @@ -11,7 +11,7 @@ use crate::bootstrap::GlobalOptions; use crate::io::json::{dump, dump_pretty}; use crate::io::output as out; -use super::catalog_models::{catalog_models_for, CatalogModel}; +use super::catalog_models::{catalog_models_for, output_limit, CatalogModel}; use super::declared_harness::{harness_header_value, DeclaredHarness, HARNESS_HEADER}; use super::harness::{launch, release, resolve, Endpoint}; @@ -21,6 +21,13 @@ pub enum Handoff { /// extra request headers only from `config.yaml`, never the environment. /// Wally does not write those headers. It does set `model.supports_vision` /// in that file, because Hermes reads vision from there and from nowhere else. + /// + /// Nor does wally write an output limit for Hermes. Its requests follow the + /// plain API rule, which is the same rule every harness gets: whatever + /// `max_tokens` Hermes sends is the hard total, thinking included, and a + /// request naming none gets the model's default. Writing a `max_tokens` + /// into `config.yaml` is an option only once Hermes is shown to read it + /// from there. CustomEndpointEnvironment, ConfigFile, PatchOverlay, @@ -1096,13 +1103,12 @@ pub fn build_open_claw_config( }); if model.context_window > 0 { entry["contextWindow"] = json!(model.context_window); - // OpenClaw wants a maxTokens beside the window; without one it - // budgets the session against a cap it invented. - let max_tokens = if model.max_output > 0 { - model.max_output - } else { - model.context_window.min(65536) - }; + } + // OpenClaw wants a maxTokens beside the window; without one it + // budgets the session against a cap it invented. It is the whole + // budget for thinking and answer (`output_limit`), and a known cap is + // written even when the list reports no window. + if let Some(max_tokens) = output_limit(model, 65536) { entry["maxTokens"] = json!(max_tokens); } if model.input_per_mtok > 0 || model.output_per_mtok > 0 { @@ -1202,11 +1208,8 @@ pub fn build_deep_seek_llm_config( }); if model.context_window > 0 { entry["contextWindow"] = json!(model.context_window); - let max_tokens = if model.max_output > 0 { - model.max_output - } else { - model.context_window.min(32768) - }; + } + if let Some(max_tokens) = output_limit(model, 32768) { entry["maxTokens"] = json!(max_tokens); } if let Some(efforts) = deep_seek_reasoning_efforts(model) { @@ -1345,11 +1348,8 @@ pub fn build_prime_agent_models_config( }); if model.context_window > 0 { entry["contextWindow"] = json!(model.context_window); - let max_tokens = if model.max_output > 0 { - model.max_output - } else { - model.context_window.min(65536) - }; + } + if let Some(max_tokens) = output_limit(model, 65536) { entry["maxTokens"] = json!(max_tokens); } if model.input_per_mtok > 0 || model.output_per_mtok > 0 { diff --git a/src/harness/catalog_models.rs b/src/harness/catalog_models.rs index 2d610ab0..a5432039 100644 --- a/src/harness/catalog_models.rs +++ b/src/harness/catalog_models.rs @@ -35,6 +35,33 @@ pub struct CatalogModel { pub reasoning: Option, } +/// The output limit a harness config declares for `model`, which the harness +/// then sends as its `max_tokens`. The model list's `max_output` when it has +/// one, with or without a context window (the list reports the two +/// separately); `min(context_window, fallback)` when it gives a window but no +/// output cap; None when it gives neither (a private id the catalogue lists +/// without limits), so the config writes no limit and the server applies the +/// model's documented default. OpenCode is the one caller that also needs a +/// window: its `limit` pairs the output with a context size, so it writes no +/// limit for a model reported without one. +/// +/// `max_tokens` is meant as a hard total: it caps everything the model +/// generates, thinking included, which is how OpenAI (`max_completion_tokens`), +/// Anthropic (the thinking budget fits inside `max_tokens`) and OpenRouter +/// define it. Once InferenceInfra#5737 ships, the API adds no reasoning +/// allowance on top for any caller, so this limit is the whole budget for a +/// turn's thinking and its answer. Making it large enough is the client's job: +/// a reasoning model given 8,192 can spend all of it thinking and answer +/// nothing (the 2026-09-22 OpenCode incident). The writers therefore declare +/// the model's full `max_output`, which is never below min(16,384, max_output) +/// and never above max_output. +pub fn output_limit(model: &CatalogModel, fallback: i64) -> Option { + if model.max_output > 0 { + return Some(model.max_output); + } + (model.context_window > 0).then(|| model.context_window.min(fallback)) +} + /// Moves the entry whose id is `primary` to the front, or inserts a bare one /// when the catalog did not carry it — the launched model is always selectable. /// What a model on `catalog` is sent for a requested `reasoning_effort`, diff --git a/src/harness/declared_harness.rs b/src/harness/declared_harness.rs index 1c3adf87..98b0afa3 100644 --- a/src/harness/declared_harness.rs +++ b/src/harness/declared_harness.rs @@ -3,11 +3,17 @@ //! //! InferenceInfra attributes every inference request to a harness //! (api/app/services/harness.py `detect()`): a declared `X-RA-Harness` header -//! first, then a substring match over the User-Agent. The attribution shapes a -//! request -- with `RA_REASONING_ALLOWANCE_HARNESS_ONLY` on, only a recognised -//! harness gets the reasoning allowance on top of its `max_tokens` -- so a -//! harness that reaches the endpoint looking like a plain API client gets its -//! reasoning trace cut short. +//! first, then a substring match over the User-Agent. +//! +//! The attribution is bookkeeping only (usage by tool, the ledger's harness +//! column). InferenceInfra currently adds a reasoning allowance on top of a +//! named `max_tokens` (with `RA_REASONING_ALLOWANCE_HARNESS_ONLY` limiting it +//! to recognised harnesses). Once InferenceInfra#5737 ships, that allowance is +//! gone and `max_tokens` is a hard total of everything the model generates, +//! thinking included, for every caller alike. wally does not rely on the +//! allowance either way. A harness +//! gets room to think from the limit wally writes into its config +//! (`catalog_models::output_limit`), never from the header. //! //! wally launches these tools and knows which one it started, so it says so. //! The names are the contract's own `Harness` enum, never a retyped list, so a diff --git a/src/harness/opencode.rs b/src/harness/opencode.rs index 626c3166..bebc2d44 100644 --- a/src/harness/opencode.rs +++ b/src/harness/opencode.rs @@ -6,7 +6,7 @@ use crate::account::{self, ConsoleClient}; use crate::io::json::dump; use crate::io::output as out; -use super::catalog_models::{catalog_models_with, CatalogModel}; +use super::catalog_models::{catalog_models_with, output_limit, CatalogModel}; use super::declared_harness::{harness_header_value, DeclaredHarness, HARNESS_HEADER}; use super::harness::{ model_id_is_safe, refresh_and_recheck_model, report_cloud_session_invalid, @@ -286,13 +286,11 @@ pub fn build_open_code_config( // fire at the model's actual window instead of a wrong default // (which makes it nag to compact and never stop). Output is a sane // cap, never the whole context — opencode's own docs warn against - // that. - if model.context_window > 0 { - let output = if model.max_output > 0 { - model.max_output - } else { - model.context_window.min(65536) - }; + // that. It becomes opencode's `max_tokens`, the whole budget for + // thinking and answer (`output_limit`). + // opencode's `limit` needs the context size beside the output, so a + // model the list reports without a window gets no limit here. + if let Some(output) = output_limit(model, 65536).filter(|_| model.context_window > 0) { entry["limit"] = json!({ "context": model.context_window, "output": output }); } // Image input exactly where the catalog lists it. opencode 1.18 reads diff --git a/src/harness/shannon.rs b/src/harness/shannon.rs index c877514f..7dd0a96a 100644 --- a/src/harness/shannon.rs +++ b/src/harness/shannon.rs @@ -230,6 +230,9 @@ pub fn models_config(base_url: &str, models: &[CatalogModel]) -> Value { if model.context_window > 0 { entry["contextWindow"] = json!(model.context_window); } + // Only a known cap: Shannon applies its own default when none is + // written, so a window-only model is not given a guessed limit. + // For a model with a cap this matches `output_limit`. if model.max_output > 0 { entry["maxTokens"] = json!(model.max_output); } diff --git a/tests/test_wally_harness.rs b/tests/test_wally_harness.rs index 489b8453..2f2ade7c 100644 --- a/tests/test_wally_harness.rs +++ b/tests/test_wally_harness.rs @@ -1357,3 +1357,166 @@ fn deepseek_web_ui_takes_a_free_port_unless_one_was_chosen() { ); } } + +// The limit a harness config writes becomes that harness's `max_tokens`, and +// once InferenceInfra#5737 ships the API reads `max_tokens` as a hard total of +// everything the model generates, thinking included (OpenAI, Anthropic and +// OpenRouter define it the same way), with no reasoning allowance on top. So the config, not +// the server, has to make the limit big enough: a reasoning model given 8,192 +// can spend all of it thinking and answer nothing (the 2026-09-22 OpenCode +// incident). Every writer declares the model list's `max_output`: never below +// min(16,384, max_output), never above max_output. A model the list gives a +// window but no output cap gets the writer's own fallback, min(window, 65,536) +// for OpenCode, OpenClaw and Prime Agent and min(window, 32,768) for dsh; one +// with neither a window nor an output cap (a private id the catalogue lists +// without limits) gets no limit, so the harness sends its own and the server applies the model's +// default. +#[test] +fn every_harness_config_writes_an_output_limit_within_the_models_own_bounds() { + const FLOOR: i64 = 16_384; + let model = |id: &str, context_window: i64, max_output: i64| CatalogModel { + id: id.to_string(), + context_window, + max_output, + ..Default::default() + }; + // Hosted models with a max_output: the bound applies. + let capped = [ + model("deepseek-v4.1-flash", 1_048_576, 65_536), + model("glm-5.3-flash", 202_752, 131_072), + model("short-output", 32_768, 8_192), + // An output cap with no window reported: OpenClaw, Prime Agent and dsh + // apply it; OpenCode writes no limit without a window. + model("output-only", 0, 32_768), + ]; + // A window but no output cap: the writer's fallback. No window: nothing. + let uncapped = [ + model("big-window", 1_048_576, 0), + model("small-window", 20_000, 0), + model("deepseek-v4.1-flash-dedicated", 0, 0), + ]; + let catalog: Vec = capped.iter().chain(uncapped.iter()).cloned().collect(); + let base = "https://api.example.test/v1"; + let primary = catalog[0].id.as_str(); + + let by_id = |entries: &Value, field: &str| -> Vec<(String, Option)> { + entries + .as_array() + .expect("models array") + .iter() + .map(|entry| { + ( + entry["id"].as_str().expect("model id").to_string(), + entry + .get(field) + .map(|v| v.as_i64().expect("an integer limit")), + ) + }) + .collect() + }; + let open_code: Value = serde_json::from_str(&harness::build_open_code_config( + primary, base, "tok", &catalog, + )) + .unwrap(); + let open_code: Vec<(String, Option)> = open_code["provider"]["runanywhere"]["models"] + .as_object() + .expect("models object") + .iter() + .map(|(id, entry)| { + ( + id.clone(), + entry + .get("limit") + .map(|limit| limit["output"].as_i64().expect("an integer limit")), + ) + }) + .collect(); + let open_claw: Value = serde_json::from_str(&harness::build_open_claw_config( + "", primary, base, "key", &catalog, + )) + .unwrap(); + let prime: Value = serde_json::from_str( + &harness::build_prime_agent_models_config("", base, "key", &catalog).unwrap(), + ) + .unwrap(); + let dsh: Value = serde_json::from_str(&harness::build_deep_seek_llm_config( + base, + "RUNANYWHERE_API_KEY", + &catalog, + )) + .unwrap(); + let writers = [ + ("opencode", 65_536, open_code), + ( + "openclaw", + 65_536, + by_id( + &open_claw["models"]["providers"]["runanywhere"]["models"], + "maxTokens", + ), + ), + ( + "prime-agent", + 65_536, + by_id(&prime["providers"]["runanywhere"]["models"], "maxTokens"), + ), + ( + "dsh", + 32_768, + by_id(&dsh["providers"]["runanywhere"]["models"], "maxTokens"), + ), + ]; + + for (writer, fallback, written) in &writers { + assert_eq!( + written.len(), + catalog.len(), + "{writer}: every catalog model is declared" + ); + let limit_of = |id: &str| -> Option { + written + .iter() + .find(|(written_id, _)| written_id == id) + .unwrap_or_else(|| panic!("{writer} did not declare {id}")) + .1 + }; + // opencode's `limit` needs a context size, so it writes none for a + // model reported without a window; the other writers keep the cap. + let windowless = |model: &CatalogModel| *writer == "opencode" && model.context_window <= 0; + for model in capped.iter().filter(|model| !windowless(model)) { + let limit = limit_of(&model.id) + .unwrap_or_else(|| panic!("{writer} wrote no limit for {}", model.id)); + assert!( + FLOOR.min(model.max_output) <= limit && limit <= model.max_output, + "{writer}: {} limit {limit} is outside [min({FLOOR}, {max}), {max}]", + model.id, + max = model.max_output, + ); + } + assert_eq!(limit_of("big-window"), Some(*fallback), "{writer} fallback"); + assert_eq!( + limit_of("small-window"), + Some(20_000), + "{writer}: the fallback never exceeds the window" + ); + assert_eq!( + limit_of("deepseek-v4.1-flash-dedicated"), + None, + "{writer}: no window, no limit" + ); + // The one rule all four writers share. + for model in &catalog { + let expected = if windowless(model) { + None + } else { + harness::output_limit(model, *fallback) + }; + assert_eq!( + limit_of(&model.id), + expected, + "{writer}: {} must follow output_limit", + model.id + ); + } + } +}