From 7b8d0f5562a4c80f61eef9b600ff989590d01f09 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:59:21 +0200 Subject: [PATCH] fix(server): a reply gets the context its prompt leaves instead of a 400 Clients send their usual output cap whatever the prompt: Claude Code asks for 32000 tokens on every turn. Once prompt + max_tokens passed the context, the request was refused with 400, so with a 131072-token context and the model cards' 32768 default a session stopped working at about 99K tokens of prompt, long before the context was full. A direct request's max_tokens is now trimmed to the context its prompt leaves, as llama.cpp does with n_predict; a prompt that fills the context is still refused. A routed request keeps the old rule, so the router can still send it whole to a model with room for it. Thinking budgets follow the trimmed cap. Co-Authored-By: Claude Opus 5.5 --- server/src/server/admission.h | 11 +++++++++++ server/src/server/http_server.cpp | 17 ++++++++++++++--- server/src/server/http_server.h | 7 +++++-- server/test/unit/test_admission.cpp | 16 +++++++++++++++- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/server/src/server/admission.h b/server/src/server/admission.h index 43a64ed32b..c2b5ac723b 100644 --- a/server/src/server/admission.h +++ b/server/src/server/admission.h @@ -2,6 +2,17 @@ // Admission gate: reject oversized requests with HTTP 400. // When compression is enabled, lets oversized requests through — post-compress check is the real gate. +#include + +// The reply a request can get: what it asked for, trimmed to the context its +// prompt leaves (0 when the prompt fills it). Clients ask for their usual cap +// whatever the prompt, Claude Code 32000 tokens, so a long session would +// otherwise be refused once prompt + cap passed the context. +inline int fit_reply_to_context(int prompt_tokens, int max_output, int max_ctx) +{ + return (std::max)(0, (std::min)(max_output, max_ctx - prompt_tokens)); +} + inline bool should_reject_oversized(int prompt_tokens, int max_output, int max_ctx, bool compression_enabled) { diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index 95562e931e..1401e7a4ff 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -2750,18 +2750,29 @@ bool HttpServer::render_and_tokenize_request( // Context-length gate. Oversized prompts pass through when compression // (pflash) will run — the post-compress check is the real gate. bool HttpServer::validate_request_context( - SocketHandle fd, const ParsedRequest & req, bool send_failure) { + SocketHandle fd, ParsedRequest & req, bool routed) { const int prompt_tokens = (int) req.prompt_tokens.size(); const bool pflash_will_run = !req.images && config_.pflash_mode != ServerConfig::PflashMode::OFF && drafter_tokenizer_ != nullptr && (config_.pflash_mode == ServerConfig::PflashMode::ALWAYS || prompt_tokens >= config_.pflash_threshold); + if (!routed && !pflash_will_run) { + const int fitted = + fit_reply_to_context(prompt_tokens, req.max_output, config_.max_ctx); + if (fitted > 0 && fitted < req.max_output) { + std::fprintf(stderr, + "[server] max_tokens=%d trimmed to %d: the prompt takes %d of " + "the %d-token context\n", + req.max_output, fitted, prompt_tokens, config_.max_ctx); + req.max_output = fitted; + } + } if (!should_reject_oversized( prompt_tokens, req.max_output, config_.max_ctx, pflash_will_run)) { return true; } - if (send_failure) send_error(fd, 400, + if (!routed) send_error(fd, 400, context_overflow_message( config_.max_ctx, prompt_tokens, req.max_output)); return false; @@ -2943,7 +2954,7 @@ bool HttpServer::handle_model_request(SocketHandle fd, ParsedRequest & req, return true; } - if (!validate_request_context(fd, req, admission == nullptr)) { + if (!validate_request_context(fd, req, admission != nullptr)) { if (admission) *admission = RoutingAdmission::unfit; return true; } diff --git a/server/src/server/http_server.h b/server/src/server/http_server.h index 264b8c043c..021ed4428e 100644 --- a/server/src/server/http_server.h +++ b/server/src/server/http_server.h @@ -667,8 +667,11 @@ class HttpServer { const std::vector & chat_messages, const ParsedRequest & req, bool add_generation_prompt, std::string & rendered, std::string & error); - bool validate_request_context(SocketHandle fd, const ParsedRequest & req, - bool send_failure = true); + // A direct request's max_tokens is trimmed to the context its prompt + // leaves; a routed one (another model may hold it whole) is refused + // without a response, so the router can try the next model. + bool validate_request_context(SocketHandle fd, ParsedRequest & req, + bool routed = false); // Constrain the reply to the request's structured output format and // tool_choice (common/output_constraint.h). False after sending a 400 // for a format it cannot serve. diff --git a/server/test/unit/test_admission.cpp b/server/test/unit/test_admission.cpp index 6ee33d7893..246bfe93fa 100644 --- a/server/test/unit/test_admission.cpp +++ b/server/test/unit/test_admission.cpp @@ -1,4 +1,4 @@ -// Unit tests for should_reject_oversized — pure, GPU-free. +// Unit tests for should_reject_oversized and fit_reply_to_context — pure, GPU-free. #include "CppUnitTestFramework.hpp" #include "server/admission.h" @@ -55,3 +55,17 @@ TEST_CASE(AdmissionFixture, test_effective_overflows_zero_length_hit_is_a_hit) { TEST_CASE(AdmissionFixture, test_effective_overflows_full_cache_hit_still_too_large) { CHECK(effective_prompt_overflows(200000, 60000, 10000, 65536)); } + +TEST_CASE(AdmissionFixture, test_reply_fits_what_the_prompt_leaves) { + // Claude Code asks for 32000 whatever the prompt. + CHECK(fit_reply_to_context(110000, 32000, 131072) == 21072); + CHECK(fit_reply_to_context(1000, 32000, 131072) == 32000); + CHECK(fit_reply_to_context(131072, 32000, 131072) == 0); + CHECK(fit_reply_to_context(140000, 32000, 131072) == 0); + // Trimmed, the request is no longer oversized; a prompt that fills the + // context still is. + CHECK(!should_reject_oversized(110000, fit_reply_to_context(110000, 32000, 131072), + 131072, false)); + CHECK(should_reject_oversized(140000, fit_reply_to_context(140000, 32000, 131072), + 131072, false)); +}