From b02afd22a7d1e9971742ce9c8536379d109fb59e Mon Sep 17 00:00:00 2001 From: mrciffa Date: Wed, 14 Oct 2026 17:19:53 +0000 Subject: [PATCH] fix(server): a thinking request always leaves room to think The reply reserve (hard_limit_reply_budget, 4096 on the Qwen cards) came off max_tokens in full. A request that asked for thinking with max_tokens 4096 or less got none: the phase-1 cap was 0 and the budget hook closed thinking on the first token. The reserve now takes at most half of max_tokens, through one helper, reply_reserve_within(), used by the request parser, the single-request path and the scheduler. Requests with max_tokens above twice the reserve are unchanged. --- docs/specs/thinking-budget.md | 12 +++++++---- server/src/common/thinking_budget.h | 8 ++++++++ server/src/server/http_server.cpp | 25 +++++++++++++---------- server/src/server/scheduler.cpp | 7 ++++--- server/test/unit/test_server_unit.cpp | 29 +++++++++++++++++++++++++++ 5 files changed, 63 insertions(+), 18 deletions(-) diff --git a/docs/specs/thinking-budget.md b/docs/specs/thinking-budget.md index 42b6f87f0..336e03fcf 100644 --- a/docs/specs/thinking-budget.md +++ b/docs/specs/thinking-budget.md @@ -194,7 +194,7 @@ wants to use that tier's full budget needs to pass an explicit `max_tokens` ≥ `tier_value + hard_limit_reply_budget` on the request. With smaller `max_tokens`, the request parser narrows the effective phase-1 cap to `min(tier_value, request.max_tokens − -hard_limit_reply_budget)` (see §4.4). +reply_reserve)` (see §4.4). A request that asks for an effort tier exceeding the model's ceiling (e.g. `effort: "max"` on a model whose card has no @@ -286,8 +286,11 @@ error response — clamping is silent at the wire to preserve OpenAI/ Anthropic protocol compatibility. When `reasoning.effort` is set, the request's effective phase-1 -cap is `min(effort_tier_value, request.max_tokens − -hard_limit_reply_budget)`. The effort tier value is the server +cap is `min(effort_tier_value, request.max_tokens − reply_reserve)`, +where `reply_reserve = min(hard_limit_reply_budget, +request.max_tokens / 2)`. The reserve comes out of `max_tokens` but +never takes all of it: with `max_tokens` 4096 and a 4096 reserve, the +reply gets 2048 and thinking up to 2048, instead of no thinking. The effort tier value is the server configuration looked up from the resolved model card (or CLI override); the per-request narrowing accommodates clients that choose a tier (e.g. `"max"`) without also passing an explicit @@ -412,7 +415,8 @@ if thinking is still open and remaining ≤ effective_reply_budget: force-close ``` Where `effective_reply_budget` is the per-request `thinking.reply_budget` -clamped to `--hard-limit-reply-budget` (see §4.4). The initial `n_gen` is +clamped to `--hard-limit-reply-budget` (see §4.4), and at most half of +the request's `max_tokens`. The initial `n_gen` is the combined thinking ceiling and reply reserve, capped by the request's maximum output and the backend's available context. Every decode strategy receives the same controller by reference; `remaining()` is diff --git a/server/src/common/thinking_budget.h b/server/src/common/thinking_budget.h index 84eac6f99..1c16cb31b 100644 --- a/server/src/common/thinking_budget.h +++ b/server/src/common/thinking_budget.h @@ -22,6 +22,14 @@ struct BudgetHook { int hard_limit_remaining = 0; }; +// The reply reserve comes out of the request's output cap. When the cap is at +// most the configured reserve, the reply gets half of it and thinking the +// other half; the full reserve would leave no room to think at all. +inline int reply_reserve_within(int reserve, int output_cap) { + const int half = output_cap / 2; + return reserve < half ? reserve : half; +} + // One controller per generation, shared by the seed, speculative blocks and // every AR tail. The decoder supplies ACCEPTED output candidates exactly once, // before committing them to history/KV or emitting them. Rejected drafts and diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index 3ff542b0b..122f6d2e1 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -4010,21 +4010,23 @@ void apply_request_reasoning( request_budget_tokens, config.think_max_tokens); } } else if (req.thinking_opt_in && effort_set) { - // Spec §4.4: effective cap is min(tier value, max_tokens - - // hard_limit_reply_budget). Tier values can legitimately exceed - // default_max_tokens; clients that want the full tier budget must - // pass an explicit max_tokens. Otherwise we narrow silently to fit. - const int max_output_phase1_room = (std::max)( - 0, req.max_output - config.hard_limit_reply_budget); + // Spec §4.4: effective cap is min(tier value, max_tokens - reply + // reserve). Tier values can legitimately exceed default_max_tokens; + // clients that want the full tier budget must pass an explicit + // max_tokens. Otherwise we narrow silently to fit. + const int reply_reserve = luce::common::reply_reserve_within( + config.hard_limit_reply_budget, req.max_output); + const int max_output_phase1_room = + (std::max)(0, req.max_output - reply_reserve); req.per_req_phase1_cap = (std::min)(effort_phase1_cap, max_output_phase1_room); if (effort_phase1_cap > max_output_phase1_room) { std::fprintf(stderr, "[server] reasoning.effort tier=%d narrowed to %d " - "(max_tokens=%d - hard_limit_reply_budget=%d); " + "(max_tokens=%d - reply reserve=%d); " "pass a larger max_tokens to use the full tier budget\n", effort_phase1_cap, req.per_req_phase1_cap, - req.max_output, config.hard_limit_reply_budget); + req.max_output, reply_reserve); } } if (req.thinking_opt_in && request_reply_budget >= 0) { @@ -9693,9 +9695,10 @@ void HttpServer::prepare_generation_inputs( const int thinking_ceiling = req.per_req_phase1_cap >= 0 ? req.per_req_phase1_cap : config_.think_max_tokens; - const int reply_budget = req.per_req_reply_budget >= 0 - ? req.per_req_reply_budget - : config_.hard_limit_reply_budget; + const int reply_budget = luce::common::reply_reserve_within( + req.per_req_reply_budget >= 0 ? req.per_req_reply_budget + : config_.hard_limit_reply_budget, + req.max_output); inputs.generation_cap = budget_active ? (std::min)(thinking_ceiling + reply_budget, req.max_output) : req.max_output; diff --git a/server/src/server/scheduler.cpp b/server/src/server/scheduler.cpp index 8bb1e19fb..186c77ff9 100644 --- a/server/src/server/scheduler.cpp +++ b/server/src/server/scheduler.cpp @@ -463,9 +463,10 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { const int effective_think_ceiling = (req.per_req_phase1_cap >= 0) ? req.per_req_phase1_cap : config_.think_max_tokens; - const int eff_reply_for_n_gen = (req.per_req_reply_budget >= 0) - ? req.per_req_reply_budget - : config_.hard_limit_reply_budget; + const int eff_reply_for_n_gen = luce::common::reply_reserve_within( + req.per_req_reply_budget >= 0 ? req.per_req_reply_budget + : config_.hard_limit_reply_budget, + req.max_output); const int n_gen_cap = budget_active ? (std::min)(effective_think_ceiling + eff_reply_for_n_gen, req.max_output) diff --git a/server/test/unit/test_server_unit.cpp b/server/test/unit/test_server_unit.cpp index 8a5d1e6a1..1288c51e3 100644 --- a/server/test/unit/test_server_unit.cpp +++ b/server/test/unit/test_server_unit.cpp @@ -6892,6 +6892,35 @@ TEST_CASE(ServerUnitFixture, test_deepseek_reasoning_request_precedence_and_togg TEST_ASSERT(no_control.per_req_phase1_cap == -1); } +// A max_tokens at or below the reply reserve still leaves room to think: the +// reply takes half of it. The full reserve used to take all of it, so asking +// for an effort with max_tokens 4096 (reserve 4096) gave no thinking. +TEST_CASE(ServerUnitFixture, test_reply_reserve_leaves_room_to_think) { + TEST_ASSERT(luce::common::reply_reserve_within(4096, 32768) == 4096); + TEST_ASSERT(luce::common::reply_reserve_within(4096, 4096) == 2048); + TEST_ASSERT(luce::common::reply_reserve_within(4096, 1000) == 500); + + struct Case { + int max_output; + int phase1_cap; + }; + // Reserve 100, high tier 300. + const Case cases[] = { + {1000, 300}, + {150, 75}, + {100, 50}, + }; + ServerConfig config = deepseek_reasoning_test_config(); + config.arch = "qwen35"; + for (const auto & test : cases) { + ParsedRequest req; + req.max_output = test.max_output; + apply_request_reasoning({{"reasoning_effort", "high"}}, config, req); + TEST_ASSERT(req.thinking_opt_in); + TEST_ASSERT(req.per_req_phase1_cap == test.phase1_cap); + } +} + TEST_CASE(ServerUnitFixture, test_qwen_template_toggles_remain_renderer_only) { const json renderer_only_controls[] = { {{"chat_template_kwargs", {{"thinking", true}}}},