From b4c2b90c4b377a970ea47201358b70e30238171e Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Wed, 28 Oct 2026 14:39:15 -0700 Subject: [PATCH 1/5] feat(server): continue_final_message on /v1/chat/completions A trailing assistant message can now be continued instead of answered, as vLLM's and Hugging Face's continue_final_message do. The chat template is rendered without a generation prompt and cut right after the final message's text (transformers' algorithm), so logprobs give the next-token distribution after a partial answer through the model's own template. - chat_continue: the cut, shared by every model through AutoModel::_render_chat_template; a template that rewrites the message is a 400 invalid_request_error - models opt in via supports_continue_final_message(); in continue mode they keep every prompt token and feed no think preamble. gpt-oss and single-turn models are refused - server: validation (role, tool_calls, images, empty text, JSON / forced tool_choice, add_generation_prompt: true) with 400s - output parser: the continuation keeps its whitespace; reasoning state is read from the continued turn only; on a model that reasons first, a with no closes reasoning the model read the message as Fixes found on the way, all models: - max_tokens N now writes N tokens: the sample taken after prefill was emitted but not counted, so N+1 came out and completion_tokens said N - token_history now matches what the engine ran: a token cut off by max_tokens, cancel or the context cap is no longer recorded as run, and several custom loops now record the first sample they forward - SentencePiece byte tokens (<0x0D>) decode to their byte instead of their name - a single-turn model refuses history with HTTP 400 instead of 200 Co-Authored-By: Claude --- src/common/AutoModel/automodel.cpp | 74 ++++++- src/common/AutoModel/modeling_gemma3.cpp | 7 +- src/common/AutoModel/modeling_gemma3_text.cpp | 7 +- src/common/AutoModel/modeling_gemma4_12b.cpp | 38 +++- src/common/AutoModel/modeling_gemma4e.cpp | 56 ++++-- src/common/AutoModel/modeling_gpt_oss.cpp | 26 ++- src/common/AutoModel/modeling_hunyuan.cpp | 5 +- src/common/AutoModel/modeling_lfm2.cpp | 52 +++-- src/common/AutoModel/modeling_lfm3.cpp | 7 +- src/common/AutoModel/modeling_llama3.cpp | 44 +++-- src/common/AutoModel/modeling_minicpm_v.cpp | 31 ++- src/common/AutoModel/modeling_nanbeige.cpp | 39 ++-- src/common/AutoModel/modeling_phi4.cpp | 10 +- src/common/AutoModel/modeling_qwen2.cpp | 7 +- src/common/AutoModel/modeling_qwen2vl.cpp | 9 +- src/common/AutoModel/modeling_qwen3.cpp | 183 ++++++++++++------ .../AutoModel/modeling_qwen3_5_omni.cpp | 60 ++++-- src/common/AutoModel/modeling_qwen3_5vl.cpp | 42 +++- src/common/AutoModel/modeling_qwen3_6_moe.cpp | 38 +++- src/common/AutoModel/modeling_qwen3_8mtp.cpp | 62 ++++-- src/common/AutoModel/modeling_qwen3vl.cpp | 30 ++- src/common/chat/chat_continue.cpp | 60 ++++++ src/common/chat/chat_format.cpp | 4 +- src/common/chat/chat_parser.cpp | 65 +++++-- src/common/chat/chat_sources.cmake | 1 + src/common/tokenizer/tokenizer.cpp | 9 +- src/include/AutoModel/automodel.hpp | 36 +++- src/include/AutoModel/modeling_gemma3.hpp | 3 +- .../AutoModel/modeling_gemma3_text.hpp | 3 +- src/include/AutoModel/modeling_gemma4_12b.hpp | 3 +- src/include/AutoModel/modeling_gemma4e.hpp | 11 +- src/include/AutoModel/modeling_gpt_oss.hpp | 2 +- src/include/AutoModel/modeling_hunyuan.hpp | 2 +- src/include/AutoModel/modeling_lfm2.hpp | 10 +- src/include/AutoModel/modeling_lfm3.hpp | 5 +- src/include/AutoModel/modeling_llama3.hpp | 6 +- src/include/AutoModel/modeling_minicpm_v.hpp | 5 +- src/include/AutoModel/modeling_nanbeige.hpp | 3 +- src/include/AutoModel/modeling_phi4.hpp | 4 +- src/include/AutoModel/modeling_qwen2.hpp | 3 +- src/include/AutoModel/modeling_qwen2vl.hpp | 3 +- src/include/AutoModel/modeling_qwen3.hpp | 12 +- .../AutoModel/modeling_qwen3_5_omni.hpp | 3 +- src/include/AutoModel/modeling_qwen3_5vl.hpp | 3 +- .../AutoModel/modeling_qwen3_6_moe.hpp | 3 +- src/include/AutoModel/modeling_qwen3_8mtp.hpp | 3 +- src/include/AutoModel/modeling_qwen3vl.hpp | 15 +- src/include/chat/chat_continue.hpp | 25 +++ src/include/chat/chat_format.hpp | 5 +- src/include/chat/chat_parser.hpp | 13 ++ src/server/rest_handler.cpp | 89 ++++++++- src/server/streaming_ostream_openai.hpp | 3 +- 52 files changed, 965 insertions(+), 274 deletions(-) create mode 100644 src/common/chat/chat_continue.cpp create mode 100644 src/include/chat/chat_continue.hpp diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 4d6cb76e2..02818f679 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -6,6 +6,7 @@ /// \note This is a source file for the auto_model class #include "AutoModel/automodel.hpp" +#include "chat/chat_continue.hpp" #include "utils/npu_platform.hpp" #include @@ -40,12 +41,13 @@ int resolve_context_length(const nlohmann::ordered_json& model_info, ModelRequestError::ModelRequestError( - int http_code, bool session_cleared, std::string message) + int http_code, bool session_cleared, std::string message, std::string type) : std::runtime_error(std::move(message)), - http_code_(http_code), session_cleared_(session_cleared) {} + http_code_(http_code), session_cleared_(session_cleared), type_(std::move(type)) {} int ModelRequestError::http_code() const noexcept { return http_code_; } bool ModelRequestError::session_cleared() const noexcept { return session_cleared_; } +const std::string& ModelRequestError::type() const noexcept { return type_; } AutoModel::AutoModel(flm_rt::device* npu_device_inst, std::string current_model) { this->npu_device_inst = npu_device_inst; @@ -630,17 +632,61 @@ buffer AutoModel::_chunked_insert(chat_meta_info_t& meta_info, std::vector return y; } +std::string AutoModel::_render_chat_template(minja::chat_template_inputs& inputs, bool continue_final_message, + const minja::chat_template_options& options) { + inputs.add_generation_prompt = !continue_final_message; + std::string rendered = this->chat_tmpl->apply(inputs, options); + if (!continue_final_message) { + return rendered; + } + if (inputs.messages.empty() || inputs.messages.back().value("role", "") != "assistant") { + throw ModelRequestError(400, false, "continue_final_message requires the last message to have role \"assistant\"", + "invalid_request_error"); + } + std::string prompt; + try { + prompt = chat_continue_final_message(rendered, inputs.messages.back()); + } catch (const std::invalid_argument& e) { + throw ModelRequestError(400, false, e.what(), "invalid_request_error"); + } + // The continued turn starts where the same chat with a generation prompt in place of the + // final message stops matching: the output parser looks for an open reasoning block from + // there, not in what the user wrote earlier. + size_t turn = 0; + try { + minja::chat_template_inputs opened = inputs; + opened.messages.erase(opened.messages.size() - 1); + opened.add_generation_prompt = true; + const std::string head = this->chat_tmpl->apply(opened, options); + while (turn < head.size() && turn < prompt.size() && head[turn] == prompt[turn]) { + turn++; + } + } catch (const std::exception&) { + turn = 0; // a template that cannot end on a generation prompt: look at the whole prompt + } + this->continued_turn_ = prompt.substr(turn); + return prompt; +} + void AutoModel::_begin_turn(chat_meta_info_t& meta_info, const std::vector& prompt_tokens) { // keep the end of the prompt as the model sees it: generation continues from - // there, so it tells the output parser whether a reasoning block is already open + // there, so it tells the output parser whether a reasoning block is already open. + // A continued message may have opened one anywhere in its text, so then it is + // the whole continued turn. constexpr size_t PROMPT_TAIL_TOKENS = 32; meta_info.prompt_tail.clear(); - for (size_t i = prompt_tokens.size() > PROMPT_TAIL_TOKENS ? prompt_tokens.size() - PROMPT_TAIL_TOKENS : 0; - i < prompt_tokens.size(); i++) { - meta_info.prompt_tail += this->token_text(prompt_tokens[i]); + if (meta_info.continue_final_message) { + meta_info.prompt_tail = this->continued_turn_; + } + else { + for (size_t i = prompt_tokens.size() > PROMPT_TAIL_TOKENS ? prompt_tokens.size() - PROMPT_TAIL_TOKENS : 0; + i < prompt_tokens.size(); i++) { + meta_info.prompt_tail += this->token_text(prompt_tokens[i]); + } } if (meta_info.constraint) { - meta_info.constraint->start(chat_format_prefill(this->chat_format, meta_info.prompt_tail)); + meta_info.constraint->start(chat_format_prefill(this->chat_format, meta_info.prompt_tail, + meta_info.continue_final_message)); } } @@ -788,6 +834,8 @@ std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_ stop_reason_t reason = EOT_DETECTED; int last_sampled_token = this->last_token; this->token_history.push_back(this->last_token); + // the sample insert() took is the reply's first token, and counts toward the limit + meta_info.generated_tokens++; if (this->is_normal_token(last_sampled_token) && last_sampled_token != -1){ std::string token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; @@ -807,10 +855,11 @@ std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_ if (this->total_tokens >= decode_cap){ header_print("WARNING", "Max length reached, stopping generation..."); reason = MAX_LENGTH_REACHED; - meta_info.stop_reason = reason; - return result; } - while (true){ + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + } + else while (true){ // Running out of window is a truncation, not the model choosing to // stop, and the caller has to be able to tell those apart: this is // what becomes `done_reason` over the API. Testing the cap here rather @@ -864,6 +913,11 @@ std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_ } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + this->token_history.pop_back(); + } if (this->total_tokens >= decode_cap){ header_print("WARNING", "Max length reached, stopping generation..."); } diff --git a/src/common/AutoModel/modeling_gemma3.cpp b/src/common/AutoModel/modeling_gemma3.cpp index b5943c11e..a2551ee69 100644 --- a/src/common/AutoModel/modeling_gemma3.cpp +++ b/src/common/AutoModel/modeling_gemma3.cpp @@ -34,12 +34,11 @@ void Gemma3::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string Gemma3::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Gemma3::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } bool Gemma3::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -110,7 +109,7 @@ bool Gemma3::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std: filtered_messages.push_back(filtered_message); } header_print("FLM", "Total images: " << total_images); - templated_text = this->apply_chat_template(filtered_messages); + templated_text = this->apply_chat_template(filtered_messages, nlohmann::ordered_json::object(), meta_info.continue_final_message); if (total_images > 0) { pixel_values.resize(3 * 896 * 896 * sizeof(bf16) * total_images); diff --git a/src/common/AutoModel/modeling_gemma3_text.cpp b/src/common/AutoModel/modeling_gemma3_text.cpp index 7785d7dcf..21ea8478c 100644 --- a/src/common/AutoModel/modeling_gemma3_text.cpp +++ b/src/common/AutoModel/modeling_gemma3_text.cpp @@ -36,12 +36,11 @@ void Gemma3_Text_Only::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string Gemma3_Text_Only::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Gemma3_Text_Only::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } bool Gemma3_Text_Only::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -53,7 +52,7 @@ bool Gemma3_Text_Only::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& i return false; } if (!input.messages.empty()) { // already a formated messages, usually from REST API - templated_text = this->apply_chat_template(input.messages); + templated_text = this->apply_chat_template(input.messages, nlohmann::ordered_json::object(), meta_info.continue_final_message); } else if (!input.prompt.empty()) { // a pure text, usually from the cli nlohmann::ordered_json messages; diff --git a/src/common/AutoModel/modeling_gemma4_12b.cpp b/src/common/AutoModel/modeling_gemma4_12b.cpp index 2e59116da..8eedcc168 100644 --- a/src/common/AutoModel/modeling_gemma4_12b.cpp +++ b/src/common/AutoModel/modeling_gemma4_12b.cpp @@ -50,7 +50,7 @@ void Gemma4_12B::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string Gemma4_12B::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Gemma4_12B::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { // The Gemma4 template emits unquoted keys (argument_needle:<|"|>...<|"|>), which minja's // capability probe does not recognize, so it would wrongly polyfill tool calls into a // generic JSON blob. The template handles tools/tool_calls/tool_responses natively. @@ -82,13 +82,12 @@ std::string Gemma4_12B::apply_chat_template(nlohmann::ordered_json& messages, nl } minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = normalized; inputs.extra_context = this->extra_context; inputs.extra_context["enable_thinking"] = this->enable_think; if (!tools.empty()) inputs.tools = tools; - return this->chat_tmpl->apply(inputs, opt); + return this->_render_chat_template(inputs, continue_final_message, opt); } bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -275,7 +274,7 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, if (!input.messages.empty()) { // already a formated messages, usually from REST API - templated_text = this->apply_chat_template(gemma4_12b_message, input.tools); + templated_text = this->apply_chat_template(gemma4_12b_message, input.tools, meta_info.continue_final_message); } else{ @@ -472,7 +471,8 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, const bool modal_tokens = std::any_of(tokens.begin(), tokens.end(), [this](int t) { return t == image_token_id || t == audio_token_id || t == boi_token_id || t == boa_token_id; }); - const size_t slot_trim = this->enable_think ? 0 : 4; + // A continued message keeps all its tokens: they are the client's text, not the generation prompt. + const size_t slot_trim = (this->enable_think || meta_info.continue_final_message) ? 0 : 4; if (image_payload.num_images == 0 && audio_payload.num_audios == 0 && !modal_tokens && this->_prefix_slot_mode(meta_info, tokens.size(), slot_trim)) { checkpoint_his.clear(); @@ -489,7 +489,7 @@ bool Gemma4_12B::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, } size_t n = tokens.size(); - tokens.resize(n - (this->enable_think ? 0 : 4)); + tokens.resize(n - slot_trim); gemma4_12b_multi_modal_payload_t multi_modal_payload; multi_modal_payload.image_payload = image_payload; @@ -522,7 +522,8 @@ std::string Gemma4_12B::generate(chat_meta_info_t& meta_info, int length_limit, std::string token_str; int sampled_token; int last_sampled_token; - if(!enable_think) { + // A continued message has no empty thought channel to feed: it starts from insert()'s sample + if(!enable_think && !meta_info.continue_final_message) { this->token_history.push_back(boc_token_id); this->profiler_list[DECODING_TIME].start(); // fed for the KV cache only, it never reaches the output @@ -554,6 +555,8 @@ std::string Gemma4_12B::generate(chat_meta_info_t& meta_info, int length_limit, this->total_tokens++; meta_info.generated_tokens++; last_sampled_token = sampled_token; + // the reply's first token: the loop below forwards it, so the history holds it + this->token_history.push_back(last_sampled_token); token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; os << token_str << std::flush; @@ -561,6 +564,8 @@ std::string Gemma4_12B::generate(chat_meta_info_t& meta_info, int length_limit, else { last_sampled_token = this->last_token; this->token_history.push_back(this->last_token); + // the sample insert() took is the reply's first token, and counts toward the limit + meta_info.generated_tokens++; if (this->is_normal_token(last_sampled_token) && last_sampled_token != -1){ std::string token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; @@ -568,6 +573,9 @@ std::string Gemma4_12B::generate(chat_meta_info_t& meta_info, int length_limit, } if (this->is_eos(last_sampled_token)){ + // Sampled from the prefill, so no engine has run it. + this->token_history.pop_back(); + meta_info.stop_reason = reason; return result; } } @@ -575,9 +583,16 @@ std::string Gemma4_12B::generate(chat_meta_info_t& meta_info, int length_limit, if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); reason = MAX_LENGTH_REACHED; - return result; } - while (this->total_tokens < this->MAX_L){ + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + } + else while (true){ + // Running out of window is a truncation: test it here so it is recorded. + if (this->total_tokens >= this->MAX_L){ + reason = MAX_LENGTH_REACHED; + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; @@ -613,6 +628,11 @@ std::string Gemma4_12B::generate(chat_meta_info_t& meta_info, int length_limit, } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + this->token_history.pop_back(); + } if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); } diff --git a/src/common/AutoModel/modeling_gemma4e.cpp b/src/common/AutoModel/modeling_gemma4e.cpp index 10616a3a4..b71762310 100644 --- a/src/common/AutoModel/modeling_gemma4e.cpp +++ b/src/common/AutoModel/modeling_gemma4e.cpp @@ -164,6 +164,8 @@ std::string Gemma4e_Flash::generate(chat_meta_info_t& meta_info, int length_limi int last_sampled_token = this->last_token; this->token_history.push_back(last_token); + // the sample insert() took is the reply's first token, and counts toward the limit + meta_info.generated_tokens++; if (this->is_normal_token(last_sampled_token) && last_sampled_token != -1) { std::string token_str = this->tokenizer->run_time_decoder(last_sampled_token); @@ -171,6 +173,8 @@ std::string Gemma4e_Flash::generate(chat_meta_info_t& meta_info, int length_limi os << token_str << std::flush; } if (this->is_eos(last_sampled_token)) { + // Sampled from the prefill, so no engine has run it. + this->token_history.pop_back(); meta_info.stop_reason = reason; return result; } @@ -178,11 +182,17 @@ std::string Gemma4e_Flash::generate(chat_meta_info_t& meta_info, int length_limi this->profiler_list[TKOEN_DECODE_TIME].reset(); if (this->total_tokens >= this->MAX_L) { header_print("WARNING", "Max length reached, stopping generation..."); - meta_info.stop_reason = MAX_LENGTH_REACHED; - return result; + reason = MAX_LENGTH_REACHED; } - - while (this->total_tokens < this->MAX_L) { + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)) { + reason = MAX_LENGTH_REACHED; + } + else while (true) { + // Running out of window is a truncation: test it here so it is recorded. + if (this->total_tokens >= this->MAX_L) { + reason = MAX_LENGTH_REACHED; + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; @@ -212,6 +222,8 @@ std::string Gemma4e_Flash::generate(chat_meta_info_t& meta_info, int length_limi if (this->is_eos(sampled_token)) { // Single-turn: no forward_on_eos needed — kv cache is discarded // by _reset_turn() at the start of every new request. + // The engine never ran it, so the history drops it. + this->token_history.pop_back(); break; } if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)) { @@ -222,6 +234,10 @@ std::string Gemma4e_Flash::generate(chat_meta_info_t& meta_info, int length_limi meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one. + this->token_history.pop_back(); + } if (this->total_tokens >= this->MAX_L) { header_print("WARNING", "Max length reached, stopping generation..."); } @@ -271,7 +287,7 @@ void Gemma4e::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string Gemma4e::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Gemma4e::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { // Same as Gemma4_12B: the Gemma4 template emits unquoted keys (argument_needle:<|"|>...<|"|>), which // minja's capability probe does not recognize, so it would wrongly polyfill tool calls and tool // responses into a generic JSON blob. The template handles them natively. @@ -303,14 +319,13 @@ std::string Gemma4e::apply_chat_template(nlohmann::ordered_json& messages, nlohm } minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = normalized; inputs.extra_context = this->extra_context; inputs.extra_context["enable_thinking"] = this->enable_think; inputs.extra_context["conversational_mode"] = this->conversational_mode; if (!tools.empty()) inputs.tools = tools; - return this->chat_tmpl->apply(inputs, opt); + return this->_render_chat_template(inputs, continue_final_message, opt); } bool Gemma4e::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -502,7 +517,7 @@ bool Gemma4e::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std if (!input.messages.empty()) { // already a formated messages, usually from REST API // gemma4_message was already built above, in the same pass that // decoded images/audio. - templated_text = this->apply_chat_template(gemma4_message, input.tools); + templated_text = this->apply_chat_template(gemma4_message, input.tools, meta_info.continue_final_message); } else if (!input.prompt.empty()) { // a pure text, usually from the cli nlohmann::ordered_json messages; @@ -735,6 +750,8 @@ std::string Gemma4e::generate(chat_meta_info_t& meta_info, int length_limit, std int last_sampled_token = this->last_token; this->token_history.push_back(last_token); + // the sample insert() took is the reply's first token, and counts toward the limit + meta_info.generated_tokens++; if (this->is_normal_token(last_sampled_token) && last_sampled_token != -1){ std::string token_str = this->tokenizer->run_time_decoder(last_sampled_token); @@ -743,6 +760,9 @@ std::string Gemma4e::generate(chat_meta_info_t& meta_info, int length_limit, std } if (this->is_eos(last_sampled_token)){ + // Sampled from the prefill, so no engine has run it. + this->token_history.pop_back(); + meta_info.stop_reason = reason; return result; } this->profiler_list[DECODING_TIME].reset(); @@ -750,9 +770,16 @@ std::string Gemma4e::generate(chat_meta_info_t& meta_info, int length_limit, std if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); reason = MAX_LENGTH_REACHED; - return result; } - while (this->total_tokens < this->MAX_L){ + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + } + else while (true){ + // Running out of window is a truncation: test it here so it is recorded. + if (this->total_tokens >= this->MAX_L){ + reason = MAX_LENGTH_REACHED; + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; @@ -798,9 +825,14 @@ std::string Gemma4e::generate(chat_meta_info_t& meta_info, int length_limit, std int checkpoint_idx = this->lm_engine->checkpoint(); // copy the token history at the checkpoint except the last one token, which is the start token for generation and should not be included in the checkpoint history checkpoint_his = token_history; - checkpoint_his.pop_back(); + checkpoint_his.pop_back(); } - + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + this->token_history.pop_back(); + } + return result; } diff --git a/src/common/AutoModel/modeling_gpt_oss.cpp b/src/common/AutoModel/modeling_gpt_oss.cpp index 782b83521..4cd98c1e5 100644 --- a/src/common/AutoModel/modeling_gpt_oss.cpp +++ b/src/common/AutoModel/modeling_gpt_oss.cpp @@ -36,9 +36,8 @@ void GPT_OSS::setup_tokenizer(std::string model_path){ auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string GPT_OSS::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string GPT_OSS::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; inputs.extra_context["enable_thinking"] = this->enable_think; @@ -47,7 +46,7 @@ std::string GPT_OSS::apply_chat_template(nlohmann::ordered_json& messages, nlohm inputs.extra_context["role"] = this->role; //inputs.tools = tools; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } bool GPT_OSS::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -114,6 +113,8 @@ std::string GPT_OSS::generate(chat_meta_info_t& meta_info, int length_limit, std token_history.push_back(last_token); + // the sample insert() took is the reply's first token, and counts toward the limit + meta_info.generated_tokens++; if (this->is_normal_token(last_sampled_token) && last_sampled_token != -1){ std::string token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; @@ -121,6 +122,9 @@ std::string GPT_OSS::generate(chat_meta_info_t& meta_info, int length_limit, std } if (this->is_eos(last_sampled_token)){ + // Sampled from the prefill, so no engine has run it. + token_history.pop_back(); + meta_info.stop_reason = reason; return result; } this->profiler_list[DECODING_TIME].reset(); @@ -128,9 +132,16 @@ std::string GPT_OSS::generate(chat_meta_info_t& meta_info, int length_limit, std if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); reason = MAX_LENGTH_REACHED; - return result; } - while (this->total_tokens < this->MAX_L){ + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + } + else while (true){ + // Running out of window is a truncation: test it here so it is recorded. + if (this->total_tokens >= this->MAX_L){ + reason = MAX_LENGTH_REACHED; + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; @@ -166,6 +177,11 @@ std::string GPT_OSS::generate(chat_meta_info_t& meta_info, int length_limit, std } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + token_history.pop_back(); + } if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); } diff --git a/src/common/AutoModel/modeling_hunyuan.cpp b/src/common/AutoModel/modeling_hunyuan.cpp index 72a8bad12..72ef84652 100644 --- a/src/common/AutoModel/modeling_hunyuan.cpp +++ b/src/common/AutoModel/modeling_hunyuan.cpp @@ -47,12 +47,11 @@ void Hunyuan::setup_tokenizer(std::string model_path) { /// \note The template renders `system` only when it is messages[0]; every other /// role but user / assistant is dropped. It never branches on `tools`, so /// the argument is accepted for the interface and ignored. -std::string Hunyuan::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Hunyuan::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } chat_template_type_t Hunyuan::get_chat_template_type() { diff --git a/src/common/AutoModel/modeling_lfm2.cpp b/src/common/AutoModel/modeling_lfm2.cpp index 73625d0f8..1b57cdba2 100644 --- a/src/common/AutoModel/modeling_lfm2.cpp +++ b/src/common/AutoModel/modeling_lfm2.cpp @@ -35,15 +35,14 @@ void LFM2::setup_tokenizer(std::string model_path) { this->tokenizer->is_doubled_encoded = true; } -std::string LFM2::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string LFM2::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; minja::chat_template_options opt; opt.polyfill_tool_responses = false; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; // inputs.tools = tools; - return this->chat_tmpl->apply(inputs, opt); + return this->_render_chat_template(inputs, continue_final_message, opt); } bool LFM2::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -56,7 +55,7 @@ bool LFM2::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::f } if (!input.messages.empty()) { // already a formated messages, usually from REST API //templated_text = this->apply_chat_template(input.messages); - templated_text = this->apply_chat_template(input.messages, input.tools); + templated_text = this->apply_chat_template(input.messages, input.tools, meta_info.continue_final_message); } else if (!input.prompt.empty()) { // a pure text, usually from the cli nlohmann::ordered_json messages; @@ -119,15 +118,14 @@ void LFM2_5_TK::setup_tokenizer(std::string model_path) { this->tokenizer->is_doubled_encoded = true; } -std::string LFM2_5_TK::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string LFM2_5_TK::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; minja::chat_template_options opt; opt.polyfill_tool_responses = false; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; // inputs.tools = tools; - return this->chat_tmpl->apply(inputs, opt); + return this->_render_chat_template(inputs, continue_final_message, opt); } bool LFM2_5_TK::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -139,7 +137,7 @@ bool LFM2_5_TK::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s return false; } if (!input.messages.empty()) { // already a formated messages, usually from REST API - templated_text = this->apply_chat_template(input.messages); + templated_text = this->apply_chat_template(input.messages, nlohmann::ordered_json::object(), meta_info.continue_final_message); // templated_text = this->apply_chat_template(input.messages, input.tools); } else if (!input.prompt.empty()) { // a pure text, usually from the cli @@ -158,8 +156,10 @@ bool LFM2_5_TK::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s lfm2_npu *lfm2_engine = dynamic_cast(this->lm_engine); // Prefix slots. The two tokens left out ("\n") are fed by generate(); the next turn does not render them. - if (this->_prefix_slot_mode(meta_info, tokens.size(), 2)) { - return this->_prefix_text_insert(meta_info, tokens, 2, "<|im_start|>user", is_cancelled); + // A continued message keeps all its tokens: they are the client's text, not the generation prompt. + const size_t think_trim = meta_info.continue_final_message ? 0 : 2; + if (this->_prefix_slot_mode(meta_info, tokens.size(), think_trim)) { + return this->_prefix_text_insert(meta_info, tokens, think_trim, "<|im_start|>user", is_cancelled); } this->_prefix_legacy_path(); @@ -171,7 +171,7 @@ bool LFM2_5_TK::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s // Remove the starting reasoning placeholder appended by the chat template: "\n" -> token ids 151667 708. size_t n = tokens.size(); - tokens.resize(n - 2); + tokens.resize(n - think_trim); bool success = this->_shared_insert(meta_info, tokens, is_cancelled, nullptr); @@ -183,6 +183,10 @@ bool LFM2_5_TK::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s std::string LFM2_5_TK::generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { + if (meta_info.continue_final_message) { + // insert() held back no "\n": the message goes on from the sample insert() made + return this->_shared_generate(meta_info, length_limit, os, is_cancelled); + } std::vector sampled_tokens; std::string result; if (length_limit > 0){ @@ -221,18 +225,31 @@ std::string LFM2_5_TK::generate(chat_meta_info_t& meta_info, int length_limit, s os << token_str << std::flush; this->total_tokens++; + // the first sample after the preamble is the reply's first token, and counts toward the limit meta_info.generated_tokens++; int last_sampled_token = sampled_token; + token_history.push_back(last_sampled_token); token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; - os << token_str << std::flush; + os << token_str << std::flush; - if (this->total_tokens >= this->MAX_L){ + if (this->is_eos(last_sampled_token)){ + this->lm_engine->forward(last_sampled_token); + } + else if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); reason = MAX_LENGTH_REACHED; - return result; } - while (this->total_tokens < this->MAX_L){ + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + } + else while (true){ + // Running out of window is a truncation; testing here rather than in + // the loop condition records it. + if (this->total_tokens >= this->MAX_L){ + reason = MAX_LENGTH_REACHED; + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; @@ -268,6 +285,11 @@ std::string LFM2_5_TK::generate(chat_meta_info_t& meta_info, int length_limit, s } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + token_history.pop_back(); + } if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); } diff --git a/src/common/AutoModel/modeling_lfm3.cpp b/src/common/AutoModel/modeling_lfm3.cpp index af5f52572..addcb95af 100644 --- a/src/common/AutoModel/modeling_lfm3.cpp +++ b/src/common/AutoModel/modeling_lfm3.cpp @@ -35,17 +35,16 @@ void LFM3::setup_tokenizer(std::string model_path) { this->tokenizer->is_doubled_encoded = true; // byte-level BPE, like LFM2 } -std::string LFM3::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string LFM3::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; minja::chat_template_options opt; opt.polyfill_tool_responses = false; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; inputs.extra_context["enable_thinking"] = this->enable_think; if (!tools.empty() && this->tools_enabled()) inputs.tools = tools; - return this->chat_tmpl->apply(inputs, opt); + return this->_render_chat_template(inputs, continue_final_message, opt); } bool LFM3::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -56,7 +55,7 @@ bool LFM3::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::f return false; } if (!input.messages.empty()) { // already formatted messages, usually from the REST API - templated_text = this->apply_chat_template(input.messages, input.tools); + templated_text = this->apply_chat_template(input.messages, input.tools, meta_info.continue_final_message); } else { // a plain text prompt, usually from the CLI nlohmann::ordered_json messages; diff --git a/src/common/AutoModel/modeling_llama3.cpp b/src/common/AutoModel/modeling_llama3.cpp index f187fa973..11c2a44f3 100644 --- a/src/common/AutoModel/modeling_llama3.cpp +++ b/src/common/AutoModel/modeling_llama3.cpp @@ -32,12 +32,11 @@ void Llama3::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string Llama3::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Llama3::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } bool Llama3::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -49,7 +48,7 @@ bool Llama3::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std: return false; } if (!input.messages.empty()) { // already a formated messages, usually from REST API - templated_text = this->apply_chat_template(input.messages); + templated_text = this->apply_chat_template(input.messages, nlohmann::ordered_json::object(), meta_info.continue_final_message); } else if (!input.prompt.empty()) { // a pure text, usually from the cli nlohmann::ordered_json messages; @@ -113,12 +112,11 @@ void DeepSeek_r1_8b::setup_tokenizer(std::string model_path) { this->think_marker_id = this->tokenizer->encode("")[0]; } -std::string DeepSeek_r1_8b::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string DeepSeek_r1_8b::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } bool DeepSeek_r1_8b::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -130,7 +128,7 @@ bool DeepSeek_r1_8b::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& inp return false; } if (!input.messages.empty()) { // already a formated messages, usually from REST API - templated_text = this->apply_chat_template(input.messages); + templated_text = this->apply_chat_template(input.messages, nlohmann::ordered_json::object(), meta_info.continue_final_message); } else if (!input.prompt.empty()) { // a pure text, usually from the cli nlohmann::ordered_json messages; @@ -167,7 +165,10 @@ bool DeepSeek_r1_8b::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& inp std::string DeepSeek_r1_8b::generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { std::vector sampled_tokens; std::string result; - os << "\n\n"; + // A continued message already carries its own text: no think preamble of ours. + if (!meta_info.continue_final_message) { + os << "\n\n"; + } if (length_limit > 0){ sampled_tokens.reserve(length_limit); } @@ -180,22 +181,32 @@ std::string DeepSeek_r1_8b::generate(chat_meta_info_t& meta_info, int length_lim int last_sampled_token = this->last_token; token_history.push_back(this->last_token); + // the sample insert() took is the reply's first token, and counts toward the limit + meta_info.generated_tokens++; if (this->is_normal_token(last_sampled_token) && last_sampled_token != -1){ std::string token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; os << token_str << std::flush; } if (this->is_eos(last_sampled_token)){ + // Sampled from the prefill, so no engine has run it. + token_history.pop_back(); + meta_info.stop_reason = reason; return result; } this->profiler_list[DECODING_TIME].reset(); this->profiler_list[TKOEN_DECODE_TIME].reset(); if (this->total_tokens >= this->MAX_L){ - header_print("WARNING", "Max length reached, stopping generation..."); reason = MAX_LENGTH_REACHED; - return result; } - while (this->total_tokens < this->MAX_L){ + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + } + else while (true){ + if (this->total_tokens >= this->MAX_L){ + reason = MAX_LENGTH_REACHED; + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; @@ -231,12 +242,19 @@ std::string DeepSeek_r1_8b::generate(chat_meta_info_t& meta_info, int length_lim } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + token_history.pop_back(); + } if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); } std::cout << std::endl; header_print("FLM", "Model RAW Output: \n" + result); - result = "\n\n" + result; + if (!meta_info.continue_final_message) { + result = "\n\n" + result; + } return result; } diff --git a/src/common/AutoModel/modeling_minicpm_v.cpp b/src/common/AutoModel/modeling_minicpm_v.cpp index e1fb83d65..7b07b8529 100644 --- a/src/common/AutoModel/modeling_minicpm_v.cpp +++ b/src/common/AutoModel/modeling_minicpm_v.cpp @@ -45,15 +45,14 @@ void MiniCPM_V::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string MiniCPM_V::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string MiniCPM_V::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; inputs.extra_context["enable_thinking"] = this->enable_think; if (!tools.empty() && this->enable_tool) inputs.tools = tools; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } void MiniCPM_V::fail_inference() { @@ -125,7 +124,7 @@ bool MiniCPM_V::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s content.push_back({ {"type", "text"}, {"text", message["content"]} }); messages.push_back({ {"role", message["role"]}, {"content", content} }); } - templated_text = this->apply_chat_template(messages, input.tools); + templated_text = this->apply_chat_template(messages, input.tools, meta_info.continue_final_message); } else if (!input.prompt.empty()) { // a pure text, usually from the cli nlohmann::ordered_json messages; @@ -178,8 +177,9 @@ bool MiniCPM_V::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s // ---- prefix slots: resume from the longest saved prefix of this prompt ---- // The generation prompt ends in "\n" (2 tokens) or, with thinking off, "\n\n\n\n" (4): decode() feeds them, - // the next turn does not render them, so they are left out of what is saved. - const size_t think_trim = this->enable_think ? 2 : 4; + // the next turn does not render them, so they are left out of what is saved. A continued message has no generation + // prompt: all its tokens are the client's text and stay. + const size_t think_trim = meta_info.continue_final_message ? 0 : (this->enable_think ? 2 : 4); const bool slot_mode = this->_prefix_slot_mode(meta_info, tokens.size(), think_trim); size_t match_len = 0; std::vector full_tokens; // what this prompt leaves in the engine @@ -339,8 +339,11 @@ std::string MiniCPM_V::decode(chat_meta_info_t& meta_info, int length_limit, std // Some backends refuse to decode past a limit of their own, below MAX_L. const uint32_t decode_cap = this->decode_cap(); - // The tail of the generation prompt that insert() held back. - const std::vector think_prefix = this->enable_think + // The tail of the generation prompt that insert() held back. A continued + // message held nothing back: it goes on from the sample insert() made. + const std::vector think_prefix = meta_info.continue_final_message + ? std::vector{} + : this->enable_think ? std::vector{ think_start_id, 198 } // \n : std::vector{ think_start_id, 271, think_end_id, 271 }; // \n\n\n\n if (this->total_tokens + think_prefix.size() >= decode_cap) { @@ -351,7 +354,7 @@ std::string MiniCPM_V::decode(chat_meta_info_t& meta_info, int length_limit, std // The first prefix forward is left out of DECODING_TIME, as Qwen3_5VL does, // so `flm bench` measures decode speed the same way as for qwen3.5:0.8b. buffer y; - if (!this->enable_think) { + if (!this->enable_think && !think_prefix.empty()) { // fed for the KV cache only, it never reaches the output this->_skip_insert_sample(meta_info); } @@ -371,7 +374,7 @@ std::string MiniCPM_V::decode(chat_meta_info_t& meta_info, int length_limit, std os << token_str << std::flush; } } - int sampled_token = this->_sample(y, meta_info); + int sampled_token = think_prefix.empty() ? this->last_token : this->_sample(y, meta_info); while (true) { this->profiler_list[TKOEN_DECODE_TIME].start(); @@ -391,6 +394,9 @@ std::string MiniCPM_V::decode(chat_meta_info_t& meta_info, int length_limit, std this->total_tokens < decode_cap) { this->lm_engine->forward(sampled_token); this->total_tokens++; + } else { + // The engine never ran it, so the next prompt has to carry it. + this->token_history.pop_back(); } break; } @@ -419,6 +425,11 @@ std::string MiniCPM_V::decode(chat_meta_info_t& meta_info, int length_limit, std } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + this->token_history.pop_back(); + } std::cout << std::endl; if (this->log_raw_output) { diff --git a/src/common/AutoModel/modeling_nanbeige.cpp b/src/common/AutoModel/modeling_nanbeige.cpp index 950989a1e..559bacfaf 100644 --- a/src/common/AutoModel/modeling_nanbeige.cpp +++ b/src/common/AutoModel/modeling_nanbeige.cpp @@ -32,24 +32,22 @@ void Nanbeige::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string Nanbeige::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Nanbeige::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; // inputs.tools = tools; inputs.extra_context = this->extra_context; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } std::string Nanbeige::nanbeige_filter(int token) { std::string token_str = this->tokenizer->run_time_decoder(token); - // Nanbeige tokenizer encodes hex-unicode form 0x00-0xFF as <0xXX> from 3 to 258 - if (token >= 3 && token <= 258){ - char c; - sscanf(token_str.c_str(), "<0x%hhx>", &c); - token_str.resize(1); - token_str[0] = c; + // Nanbeige tokenizer encodes hex-unicode form 0x00-0xFF as <0xXX> from 3 to 258; + // run_time_decoder() already turns those into the byte, this keeps any it left alone + unsigned char c; + if (token >= 3 && token <= 258 && token_str.size() == 6 && sscanf(token_str.c_str(), "<0x%hhx>", &c) == 1){ + token_str.assign(1, static_cast(c)); } return token_str; } @@ -63,7 +61,7 @@ bool Nanbeige::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, st return false; } if (!input.messages.empty()) { // already a formated messages, usually from REST API - templated_text = this->apply_chat_template(input.messages, input.tools); + templated_text = this->apply_chat_template(input.messages, input.tools, meta_info.continue_final_message); } else if (!input.prompt.empty()) { // a pure text, usually from the cli nlohmann::ordered_json messages; @@ -116,6 +114,8 @@ std::string Nanbeige::generate(chat_meta_info_t& meta_info, int length_limit, st stop_reason_t reason = EOT_DETECTED; int last_sampled_token = this->last_token; this->token_history.push_back(this->last_token); + // the sample insert() took is the reply's first token, and counts toward the limit + meta_info.generated_tokens++; if (this->is_normal_token(last_sampled_token) && last_sampled_token != -1){ std::string token_str = this->nanbeige_filter(last_sampled_token); result += token_str; @@ -123,16 +123,24 @@ std::string Nanbeige::generate(chat_meta_info_t& meta_info, int length_limit, st } if (this->is_eos(last_sampled_token)){ + // Sampled from the prefill, so no engine has run it. + this->token_history.pop_back(); + meta_info.stop_reason = reason; return result; } this->profiler_list[DECODING_TIME].reset(); this->profiler_list[TKOEN_DECODE_TIME].reset(); if (this->total_tokens >= this->MAX_L){ - header_print("WARNING", "Max length reached, stopping generation..."); reason = MAX_LENGTH_REACHED; - return result; } - while (this->total_tokens < this->MAX_L){ + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + } + else while (true){ + if (this->total_tokens >= this->MAX_L){ + reason = MAX_LENGTH_REACHED; + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; @@ -167,6 +175,11 @@ std::string Nanbeige::generate(chat_meta_info_t& meta_info, int length_limit, st } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + this->token_history.pop_back(); + } if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); } diff --git a/src/common/AutoModel/modeling_phi4.cpp b/src/common/AutoModel/modeling_phi4.cpp index fef8866c9..bd88e5b2e 100644 --- a/src/common/AutoModel/modeling_phi4.cpp +++ b/src/common/AutoModel/modeling_phi4.cpp @@ -94,12 +94,12 @@ void Phi4::setup_tokenizer(const nlohmann::json& config) { } std::string Phi4::apply_chat_template(nlohmann::ordered_json& messages, - nlohmann::ordered_json) { + nlohmann::ordered_json, + bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = extra_context; - return chat_tmpl->apply(inputs); + return _render_chat_template(inputs, continue_final_message); } void Phi4::fail_inference(const std::string& detail) { @@ -119,7 +119,9 @@ bool Phi4::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, profiler_list[TKOEN_ENCODE_TIME].start(); std::string rendered; if (input.messages.empty() && input.prompt.empty()) return false; - if (!input.messages.empty()) rendered = apply_chat_template(input.messages); + if (!input.messages.empty()) + rendered = apply_chat_template(input.messages, nlohmann::ordered_json::object(), + meta_info.continue_final_message); else { nlohmann::ordered_json messages = nlohmann::ordered_json::array(); messages.push_back({{"role", "user"}, {"content", input.prompt}}); diff --git a/src/common/AutoModel/modeling_qwen2.cpp b/src/common/AutoModel/modeling_qwen2.cpp index 7036027a1..1c0c76071 100644 --- a/src/common/AutoModel/modeling_qwen2.cpp +++ b/src/common/AutoModel/modeling_qwen2.cpp @@ -35,12 +35,11 @@ void Qwen2::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string Qwen2::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Qwen2::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } bool Qwen2::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -52,7 +51,7 @@ bool Qwen2::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std:: return false; } if (!input.messages.empty()) { // already a formated messages, usually from REST API - templated_text = this->apply_chat_template(input.messages); + templated_text = this->apply_chat_template(input.messages, nlohmann::ordered_json::object(), meta_info.continue_final_message); } else if (!input.prompt.empty()) { // a pure text, usually from the cli nlohmann::ordered_json messages; diff --git a/src/common/AutoModel/modeling_qwen2vl.cpp b/src/common/AutoModel/modeling_qwen2vl.cpp index 118c182c2..ba80977b5 100644 --- a/src/common/AutoModel/modeling_qwen2vl.cpp +++ b/src/common/AutoModel/modeling_qwen2vl.cpp @@ -38,12 +38,11 @@ void Qwen2VL::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string Qwen2VL::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Qwen2VL::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } bool Qwen2VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -137,7 +136,7 @@ bool Qwen2VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std qwenvl_message.push_back(newItem); } - templated_text = this->apply_chat_template(qwenvl_message); + templated_text = this->apply_chat_template(qwenvl_message, nlohmann::ordered_json::object(), meta_info.continue_final_message); header_print_g("FLM", "Total images: " + std::to_string(total_images)); } else if (!input.prompt.empty()) { // a pure text, usually from the cli @@ -163,6 +162,8 @@ bool Qwen2VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std messages.push_back(content); templated_text = this->apply_chat_template(messages); } + } catch (const ModelRequestError&) { + throw; // a final message that cannot be continued is the client's error (400), not a failed prefill } catch (const std::exception& e) { header_print_r("ERROR", std::string("Exception during chat template application: ") + e.what()); return false; diff --git a/src/common/AutoModel/modeling_qwen3.cpp b/src/common/AutoModel/modeling_qwen3.cpp index 55f9759b1..09f18772d 100644 --- a/src/common/AutoModel/modeling_qwen3.cpp +++ b/src/common/AutoModel/modeling_qwen3.cpp @@ -37,15 +37,14 @@ void Qwen3::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string Qwen3::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Qwen3::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; inputs.extra_context["enable_thinking"] = this->enable_think; if (!tools.empty() && this->enable_tool) inputs.tools = tools; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } void Qwen3::fail_inference() { @@ -66,7 +65,7 @@ bool Qwen3::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std:: return false; } if (!input.messages.empty()) { // already a formated messages, usually from REST API - templated_text = this->apply_chat_template(input.messages, input.tools); + templated_text = this->apply_chat_template(input.messages, input.tools, meta_info.continue_final_message); } else if (!input.prompt.empty()) { // a pure text, usually from the cli nlohmann::ordered_json messages; @@ -81,7 +80,8 @@ bool Qwen3::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std:: // Prefix slots. With thinking off the template's empty think block is left out of the prefill (decode() feeds it), // because the next turn does not render it; with thinking on the generation prompt is how the next turn renders the reply. - const size_t trim_tokens = this->enable_think ? 0 : 4; + // A continued message keeps all its tokens: they are the client's text, not the generation prompt. + const size_t trim_tokens = (this->enable_think || meta_info.continue_final_message) ? 0 : 4; if (this->_prefix_slot_mode(meta_info, tokens.size(), trim_tokens)) { try { return this->_prefix_text_insert(meta_info, tokens, trim_tokens, "<|im_start|>user", is_cancelled, input.requested_max_new_tokens); @@ -150,7 +150,8 @@ std::string Qwen3::decode(chat_meta_info_t& meta_info, int length_limit, std::os int last_sampled_token; // Some backends refuse to decode past a limit of their own, below MAX_L. const uint32_t decode_cap = this->decode_cap(); - if(!enable_think) { + // A continued message trimmed nothing, so it starts from insert()'s sample. + if(!enable_think && !meta_info.continue_final_message) { // The empty think block insert() trimmed is decoded here, four steps // the context has to have room for. if (this->total_tokens + 4 >= decode_cap) { @@ -188,6 +189,8 @@ std::string Qwen3::decode(chat_meta_info_t& meta_info, int length_limit, std::os this->total_tokens += 4; meta_info.generated_tokens++; last_sampled_token = sampled_token; + // the loop forwards it like insert()'s sample, so history carries it too + this->token_history.push_back(last_sampled_token); token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; os << token_str << std::flush; @@ -195,6 +198,8 @@ std::string Qwen3::decode(chat_meta_info_t& meta_info, int length_limit, std::os else { last_sampled_token = this->last_token; this->token_history.push_back(this->last_token); + // the sample insert() took is the reply's first token, and counts toward the limit + meta_info.generated_tokens++; if (this->is_normal_token(last_sampled_token) && last_sampled_token != -1){ std::string token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; @@ -202,16 +207,24 @@ std::string Qwen3::decode(chat_meta_info_t& meta_info, int length_limit, std::os } if (this->is_eos(last_sampled_token)){ + // Sampled from the prefill, so no engine has run it. + this->token_history.pop_back(); + meta_info.stop_reason = reason; return result; } } if (this->total_tokens >= decode_cap){ - header_print("WARNING", "Max length reached, stopping generation..."); reason = MAX_LENGTH_REACHED; - return result; } - while (this->total_tokens < decode_cap){ + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + } + else while (true){ + if (this->total_tokens >= decode_cap){ + reason = MAX_LENGTH_REACHED; + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; @@ -239,6 +252,9 @@ std::string Qwen3::decode(chat_meta_info_t& meta_info, int length_limit, std::os if (this->forward_on_eos && (!this->backend_ || this->backend_->forwards_past_eos())) { this->lm_engine->forward(last_sampled_token); + } else { + // The engine never ran it, so the next prompt has to carry it. + this->token_history.pop_back(); } break; } @@ -250,6 +266,11 @@ std::string Qwen3::decode(chat_meta_info_t& meta_info, int length_limit, std::os } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + this->token_history.pop_back(); + } if (this->total_tokens >= decode_cap){ header_print("WARNING", "Max length reached, stopping generation..."); } @@ -290,14 +311,13 @@ void Qwen3_IT::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string Qwen3_IT::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Qwen3_IT::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; if (!tools.empty()) inputs.tools = tools; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } bool Qwen3_IT::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -309,7 +329,7 @@ bool Qwen3_IT::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, st return false; } if (!input.messages.empty()) { // already a formated messages, usually from REST API - templated_text = this->apply_chat_template(input.messages, input.tools); + templated_text = this->apply_chat_template(input.messages, input.tools, meta_info.continue_final_message); } else if (!input.prompt.empty()) { // a pure text, usually from the cli nlohmann::ordered_json messages; @@ -368,13 +388,12 @@ void Qwen3_TK::setup_tokenizer(std::string model_path) { this->think_marker_id = this->tokenizer->encode("")[0]; } -std::string Qwen3_TK::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Qwen3_TK::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; inputs.tools = tools; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } bool Qwen3_TK::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -386,7 +405,7 @@ bool Qwen3_TK::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, st return false; } if (!input.messages.empty()) { // already a formated messages, usually from REST API - templated_text = this->apply_chat_template(input.messages, input.tools); + templated_text = this->apply_chat_template(input.messages, input.tools, meta_info.continue_final_message); } else if (!input.prompt.empty()) { // a pure text, usually from the cli nlohmann::ordered_json messages; @@ -403,8 +422,10 @@ bool Qwen3_TK::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, st qwen3_npu *qwen3_engine = dynamic_cast(this->lm_engine); // Prefix slots. The two tokens left out ("\n") are fed by generate(); the next turn does not render them. - if (this->_prefix_slot_mode(meta_info, tokens.size(), 2)) { - return this->_prefix_text_insert(meta_info, tokens, 2, "<|im_start|>user", is_cancelled); + // A continued message keeps all its tokens: they are the client's text, not the generation prompt. + const size_t trim_tokens = meta_info.continue_final_message ? 0 : 2; + if (this->_prefix_slot_mode(meta_info, tokens.size(), trim_tokens)) { + return this->_prefix_text_insert(meta_info, tokens, trim_tokens, "<|im_start|>user", is_cancelled); } this->_prefix_legacy_path(); @@ -415,7 +436,7 @@ bool Qwen3_TK::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, st } size_t n = tokens.size(); - tokens.resize(n - 2); + tokens.resize(n - trim_tokens); bool success = this->_shared_insert(meta_info, tokens, is_cancelled, nullptr); @@ -441,42 +462,71 @@ std::string Qwen3_TK::generate(chat_meta_info_t& meta_info, int length_limit, st this->profiler_list[TKOEN_DECODE_TIME].reset(); std::string token_str; - int sampled_token; + int last_sampled_token; - this->token_history.push_back(think_start_id); - this->profiler_list[DECODING_TIME].start(); - this->_force_token(meta_info, think_start_id); - this->lm_engine->forward(think_start_id); - this->profiler_list[DECODING_TIME].stop(1); - token_str = this->tokenizer->run_time_decoder(think_start_id); - result += token_str; - os << token_str << std::flush; - - // \n - this->token_history.push_back(198); - this->profiler_list[DECODING_TIME].start(); - this->_force_token(meta_info, 198); - buffer y = this->lm_engine->forward(198); - this->profiler_list[DECODING_TIME].stop(1); - token_str = this->tokenizer->run_time_decoder(198); - result += token_str; - sampled_token = this->_sample(y, meta_info); - os << token_str << std::flush; - - this->total_tokens++; - meta_info.generated_tokens++; - int last_sampled_token = sampled_token; - token_str = this->tokenizer->run_time_decoder(last_sampled_token); - result += token_str; - os << token_str << std::flush; + if (!meta_info.continue_final_message) { + int sampled_token; + + this->token_history.push_back(think_start_id); + this->profiler_list[DECODING_TIME].start(); + this->_force_token(meta_info, think_start_id); + this->lm_engine->forward(think_start_id); + this->profiler_list[DECODING_TIME].stop(1); + token_str = this->tokenizer->run_time_decoder(think_start_id); + result += token_str; + os << token_str << std::flush; + + // \n + this->token_history.push_back(198); + this->profiler_list[DECODING_TIME].start(); + this->_force_token(meta_info, 198); + buffer y = this->lm_engine->forward(198); + this->profiler_list[DECODING_TIME].stop(1); + token_str = this->tokenizer->run_time_decoder(198); + result += token_str; + sampled_token = this->_sample(y, meta_info); + os << token_str << std::flush; + + this->total_tokens++; + meta_info.generated_tokens++; + last_sampled_token = sampled_token; + // the loop forwards it like insert()'s sample, so history carries it too + this->token_history.push_back(last_sampled_token); + token_str = this->tokenizer->run_time_decoder(last_sampled_token); + result += token_str; + os << token_str << std::flush; + } + else { + // A continued message trimmed nothing, so it starts from insert()'s sample. + last_sampled_token = this->last_token; + this->token_history.push_back(this->last_token); + // the sample insert() took is the reply's first token, and counts toward the limit + meta_info.generated_tokens++; + if (this->is_normal_token(last_sampled_token) && last_sampled_token != -1){ + token_str = this->tokenizer->run_time_decoder(last_sampled_token); + result += token_str; + os << token_str << std::flush; + } + if (this->is_eos(last_sampled_token)){ + // Sampled from the prefill, so no engine has run it. + this->token_history.pop_back(); + meta_info.stop_reason = reason; + return result; + } + } if (this->total_tokens >= this->MAX_L){ - header_print("WARNING", "Max length reached, stopping generation..."); reason = MAX_LENGTH_REACHED; - return result; } - while (this->total_tokens < this->MAX_L){ + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + } + else while (true){ + if (this->total_tokens >= this->MAX_L){ + reason = MAX_LENGTH_REACHED; + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; @@ -512,6 +562,11 @@ std::string Qwen3_TK::generate(chat_meta_info_t& meta_info, int length_limit, st } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + this->token_history.pop_back(); + } if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); } @@ -554,12 +609,11 @@ void DeepSeek_r1_0528_8b::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string DeepSeek_r1_0528_8b::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string DeepSeek_r1_0528_8b::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } bool DeepSeek_r1_0528_8b::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -571,7 +625,7 @@ bool DeepSeek_r1_0528_8b::insert(chat_meta_info_t& meta_info, lm_uniform_input_t return false; } if (!input.messages.empty()) { // already a formated messages, usually from REST API - templated_text = this->apply_chat_template(input.messages); + templated_text = this->apply_chat_template(input.messages, nlohmann::ordered_json::object(), meta_info.continue_final_message); } else if (!input.prompt.empty()) { // a pure text, usually from the cli nlohmann::ordered_json messages; @@ -621,6 +675,8 @@ std::string DeepSeek_r1_0528_8b::generate(chat_meta_info_t& meta_info, int lengt int last_sampled_token = this->last_token; token_history.push_back(this->last_token); + // the sample insert() took is the reply's first token, and counts toward the limit + meta_info.generated_tokens++; if (this->is_normal_token(last_sampled_token) && last_sampled_token != -1){ std::string token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; @@ -628,16 +684,24 @@ std::string DeepSeek_r1_0528_8b::generate(chat_meta_info_t& meta_info, int lengt } if (this->is_eos(last_sampled_token)){ + // Sampled from the prefill, so no engine has run it. + token_history.pop_back(); + meta_info.stop_reason = reason; return result; } this->profiler_list[DECODING_TIME].reset(); this->profiler_list[TKOEN_DECODE_TIME].reset(); if (this->total_tokens >= this->MAX_L){ - header_print("WARNING", "Max length reached, stopping generation..."); reason = MAX_LENGTH_REACHED; - return result; } - while (this->total_tokens < this->MAX_L){ + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + } + else while (true){ + if (this->total_tokens >= this->MAX_L){ + reason = MAX_LENGTH_REACHED; + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; @@ -673,6 +737,11 @@ std::string DeepSeek_r1_0528_8b::generate(chat_meta_info_t& meta_info, int lengt } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + token_history.pop_back(); + } if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); } diff --git a/src/common/AutoModel/modeling_qwen3_5_omni.cpp b/src/common/AutoModel/modeling_qwen3_5_omni.cpp index bf4e364b1..ce8546dcc 100644 --- a/src/common/AutoModel/modeling_qwen3_5_omni.cpp +++ b/src/common/AutoModel/modeling_qwen3_5_omni.cpp @@ -154,13 +154,12 @@ void Qwen3_5_Omni::load_model(std::string model_path, json model_info, int defau } -std::string Qwen3_5_Omni::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Qwen3_5_Omni::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; if (!tools.empty()) inputs.tools = tools; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } bool Qwen3_5_Omni::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -261,7 +260,7 @@ bool Qwen3_5_Omni::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input qwen_messages.push_back(new_item); } header_print("FLM", "Total images: " << total_images); - templated_text = this->apply_chat_template(qwen_messages, input.tools); + templated_text = this->apply_chat_template(qwen_messages, input.tools, meta_info.continue_final_message); } else { // CLI Processing // ---- image preprocessing ---- @@ -474,14 +473,16 @@ bool Qwen3_5_Omni::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input } size_t n = tokens.size(); - tokens.resize(n - 4); + // a continued message keeps all its tokens: they are the client's text, not the generation prompt + tokens.resize(n - (meta_info.continue_final_message ? 0 : 4)); this->_begin_turn(meta_info, tokens); // prefix check for tokens and token history to see if we can skip some tokens const size_t idx = this->token_history.size(); size_t skip_count = 0; for (size_t i = 0; i < idx; i++) { - if (tokens[i] == this->token_history[i]) { + // a continued prompt can be a strict prefix of the history (a retry with less text) + if (i < tokens.size() && tokens[i] == this->token_history[i]) { skip_count++; } else { @@ -585,8 +586,27 @@ std::string Qwen3_5_Omni::generate(chat_meta_info_t& meta_info, int length_limit int sampled_token; int last_sampled_token; + if (meta_info.continue_final_message) { + // the prompt ends inside the client's message (insert() trimmed nothing): + // no empty think block, the output starts from insert()'s sample + last_sampled_token = this->last_token; + this->token_history.push_back(last_sampled_token); + // the sample insert() took is the reply's first token, and counts toward the limit + meta_info.generated_tokens++; + if (this->is_normal_token(last_sampled_token)) { + token_str = this->tokenizer->run_time_decoder(last_sampled_token); + result += token_str; + os << token_str << std::flush; + } + if (this->is_eos(last_sampled_token)) { + // sampled from the prefill, so the engine never ran it + this->token_history.pop_back(); + meta_info.stop_reason = reason; + return result; + } + } // manually decoding \n <\think> \n - { + else { this->token_history.push_back(think_start_id); // fed for the KV cache only, it never reaches the output this->_skip_insert_sample(meta_info); @@ -614,20 +634,33 @@ std::string Qwen3_5_Omni::generate(chat_meta_info_t& meta_info, int length_limit sampled_token = this->_sample(this->last_thinker_result.logits, meta_info); this->total_tokens++; + // the first sample after the preamble is the reply's first token, and counts toward the limit meta_info.generated_tokens++; last_sampled_token = sampled_token; + this->token_history.push_back(last_sampled_token); token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; os << token_str << std::flush; } - if (this->total_tokens >= this->MAX_L){ + if (this->is_eos(last_sampled_token)) { + // only the preamble path gets here with an eos: the engine sampled it, so feed it as the loop does + this->engine->forward(last_sampled_token); + } + else if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); reason = MAX_LENGTH_REACHED; - return result; } - - while (this->total_tokens < this->MAX_L) { + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)) { + reason = MAX_LENGTH_REACHED; + } + else while (true) { + // Running out of window is a truncation; testing here rather than in + // the loop condition records it. + if (this->total_tokens >= this->MAX_L) { + reason = MAX_LENGTH_REACHED; + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; @@ -663,6 +696,11 @@ std::string Qwen3_5_Omni::generate(chat_meta_info_t& meta_info, int length_limit } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + this->token_history.pop_back(); + } if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); } diff --git a/src/common/AutoModel/modeling_qwen3_5vl.cpp b/src/common/AutoModel/modeling_qwen3_5vl.cpp index 73db6e54c..7f7eb448c 100644 --- a/src/common/AutoModel/modeling_qwen3_5vl.cpp +++ b/src/common/AutoModel/modeling_qwen3_5vl.cpp @@ -40,15 +40,14 @@ void Qwen3_5VL::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string Qwen3_5VL::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Qwen3_5VL::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; inputs.extra_context["enable_thinking"] = this->enable_think; if (!tools.empty() && this->enable_tool) inputs.tools = tools; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } @@ -148,7 +147,7 @@ bool Qwen3_5VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s qwenvl_message.push_back(newItem); } - templated_text = this->apply_chat_template(qwenvl_message, input.tools); + templated_text = this->apply_chat_template(qwenvl_message, input.tools, meta_info.continue_final_message); header_print("FLM", "Total images: " << total_images); } else if (!input.prompt.empty()) { // a pure text, usually from the cli @@ -213,7 +212,9 @@ bool Qwen3_5VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s // surviving image tokens after _shared_insert performs its own erase. // ---------------------------------------------------------------------- // ---- prefix slots: resume from the longest saved prefix of this prompt ---- - const size_t trim_tokens = this->enable_think ? 2 : 4; + // generate() feeds the generation prompt's think tokens itself, so they are left out of the prefill. A continued + // message keeps all its tokens: they are the client's text, not the generation prompt. + const size_t trim_tokens = meta_info.continue_final_message ? 0 : (this->enable_think ? 2 : 4); qwen3_5vl_npu* slot_eng = dynamic_cast(this->lm_engine); const bool slot_mode = this->prefix_cache_.enabled() && slot_eng != nullptr && meta_info.restore_allowed && tokens.size() > trim_tokens; size_t match_len = 0; @@ -336,6 +337,11 @@ bool Qwen3_5VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, s } std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { + // the think preamble below completes the generation prompt insert() left out; a continued + // message was prefilled whole, so its output starts from the sample insert() made + if (meta_info.continue_final_message) { + return this->_shared_generate(meta_info, length_limit, os, std::move(is_cancelled)); + } std::vector sampled_tokens; std::string result; @@ -402,18 +408,31 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s sampled_token = this->_sample(y, meta_info); } this->total_tokens++; + // the first sample after the preamble is the reply's first token, and counts toward the limit meta_info.generated_tokens++; int last_sampled_token = sampled_token; + this->token_history.push_back(last_sampled_token); token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; os << token_str << std::flush; - if (this->total_tokens >= this->MAX_L){ + if (this->is_eos(last_sampled_token)){ + this->lm_engine->forward(last_sampled_token); + } + else if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); reason = MAX_LENGTH_REACHED; - return result; } - while (this->total_tokens < this->MAX_L){ + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + } + else while (true){ + // Running out of window is a truncation; testing here rather than in + // the loop condition records it. + if (this->total_tokens >= this->MAX_L){ + reason = MAX_LENGTH_REACHED; + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; @@ -449,10 +468,15 @@ std::string Qwen3_5VL::generate(chat_meta_info_t& meta_info, int length_limit, s } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + this->token_history.pop_back(); + } if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); } - + std::cout << std::endl; header_print("FLM", "Model RAW Output: \n" + result); diff --git a/src/common/AutoModel/modeling_qwen3_6_moe.cpp b/src/common/AutoModel/modeling_qwen3_6_moe.cpp index c2fc877fe..2b2e14788 100644 --- a/src/common/AutoModel/modeling_qwen3_6_moe.cpp +++ b/src/common/AutoModel/modeling_qwen3_6_moe.cpp @@ -49,15 +49,14 @@ void Qwen3_6_MOE::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string Qwen3_6_MOE::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Qwen3_6_MOE::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; inputs.extra_context["enable_thinking"] = this->enable_think; if (!tools.empty() && this->enable_tool) inputs.tools = tools; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } bool Qwen3_6_MOE::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -156,7 +155,7 @@ bool Qwen3_6_MOE::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, qwenvl_message.push_back(newItem); } - templated_text = this->apply_chat_template(qwenvl_message, input.tools); + templated_text = this->apply_chat_template(qwenvl_message, input.tools, meta_info.continue_final_message); header_print("FLM", "Total images: " << total_images); } else if (!input.prompt.empty()) { // a pure text, usually from the cli @@ -221,7 +220,8 @@ bool Qwen3_6_MOE::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, // surviving image tokens after _shared_insert performs its own erase. // ---------------------------------------------------------------------- // ---- prefix slots: resume from the longest saved prefix of this prompt ---- - const size_t trim_tokens = this->enable_think ? 2 : 4; + // a continued message keeps all its tokens: they are the client's text, not the generation prompt + const size_t trim_tokens = meta_info.continue_final_message ? 0 : (this->enable_think ? 2 : 4); qwen3_6_moe_npu* slot_eng = dynamic_cast(this->lm_engine); const bool slot_mode = this->prefix_cache_.enabled() && slot_eng != nullptr && meta_info.restore_allowed && tokens.size() > trim_tokens; size_t match_len = 0; @@ -344,6 +344,10 @@ bool Qwen3_6_MOE::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, } std::string Qwen3_6_MOE::generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { + if (meta_info.continue_final_message) { + // insert() trimmed nothing: no think opening to feed, the output starts from insert()'s sample + return this->_shared_generate(meta_info, length_limit, os, is_cancelled); + } std::vector sampled_tokens; std::string result; @@ -420,13 +424,27 @@ std::string Qwen3_6_MOE::generate(chat_meta_info_t& meta_info, int length_limit, sampled_token = this->_sample(y, meta_info); } this->total_tokens += forced; + // the first sample after the preamble is the reply's first token, and counts toward the limit meta_info.generated_tokens++; int last_sampled_token = sampled_token; + this->token_history.push_back(last_sampled_token); token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; os << token_str << std::flush; - while (true){ + if (this->is_eos(last_sampled_token)){ + if (this->forward_on_eos && + (!this->backend_ || this->backend_->forwards_past_eos())) { + this->lm_engine->forward(last_sampled_token); + } else { + // The engine never ran it, so the next prompt has to carry it. + this->token_history.pop_back(); + } + } + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + } + else while (true){ // Running out of window is a truncation, not the model choosing to // stop; testing here rather than in the loop condition records it. if (this->total_tokens >= decode_cap){ @@ -460,6 +478,9 @@ std::string Qwen3_6_MOE::generate(chat_meta_info_t& meta_info, int length_limit, if (this->forward_on_eos && (!this->backend_ || this->backend_->forwards_past_eos())) { this->lm_engine->forward(last_sampled_token); + } else { + // The engine never ran it, so the next prompt has to carry it. + this->token_history.pop_back(); } break; } @@ -471,6 +492,11 @@ std::string Qwen3_6_MOE::generate(chat_meta_info_t& meta_info, int length_limit, } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + this->token_history.pop_back(); + } if (this->total_tokens >= decode_cap){ header_print("WARNING", "Max length reached, stopping generation..."); } diff --git a/src/common/AutoModel/modeling_qwen3_8mtp.cpp b/src/common/AutoModel/modeling_qwen3_8mtp.cpp index fea2e746c..c5ea0a2d6 100644 --- a/src/common/AutoModel/modeling_qwen3_8mtp.cpp +++ b/src/common/AutoModel/modeling_qwen3_8mtp.cpp @@ -81,15 +81,14 @@ void Qwen3_8MTP::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string Qwen3_8MTP::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Qwen3_8MTP::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; inputs.extra_context = this->extra_context; inputs.extra_context["enable_thinking"] = this->enable_think; if (!tools.empty() && this->enable_tool) inputs.tools = tools; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } bool Qwen3_8MTP::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -185,7 +184,7 @@ bool Qwen3_8MTP::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, qwenvl_message.push_back({ {"role", item["role"]}, {"content", newContent} }); } nlohmann::ordered_json ordered_messages = qwenvl_message; - templated_text = this->apply_chat_template(ordered_messages, input.tools); + templated_text = this->apply_chat_template(ordered_messages, input.tools, meta_info.continue_final_message); } else if (!input.prompt.empty()) { // a pure text, usually from the cli nlohmann::ordered_json messages; @@ -351,8 +350,11 @@ bool Qwen3_8MTP::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, // Thinking on still needs "\n" on screen -- record it for // generate() to echo. Thinking off streams nothing: its tokens only close // a block that was never opened. + // + // A continued message has no generation prompt: its last tokens are the + // client's own text, which the output must not repeat. this->preamble_to_stream.clear(); - if (this->enable_think && tokens.size() >= 2) + if (this->enable_think && !meta_info.continue_final_message && tokens.size() >= 2) this->preamble_to_stream.assign(tokens.end() - 2, tokens.end()); bool success = has_images @@ -443,21 +445,21 @@ std::string Qwen3_8MTP::_speculative_generate(chat_meta_info_t& meta_info, int l int last_sampled_token = this->last_token; this->token_history.push_back(this->last_token); + // the sample insert() took is the reply's first token, and counts toward the limit + meta_info.generated_tokens++; if (this->is_normal_token(last_sampled_token) && last_sampled_token != -1){ std::string token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; os << token_str << std::flush; } if (this->is_eos(last_sampled_token)){ + // Sampled from the prefill, so no engine has run it (as in _shared_generate). + this->token_history.pop_back(); + meta_info.stop_reason = reason; return result; } this->profiler_list[DECODING_TIME].reset(); this->profiler_list[TKOEN_DECODE_TIME].reset(); - if (this->total_tokens >= this->MAX_L){ - header_print("WARNING", "Max length reached, stopping generation..."); - reason = MAX_LENGTH_REACHED; - return result; - } // Commit one token: stream it, record it, test the stop rules. Shared by // both branches so a batch can't drift from the single-token path and @@ -485,6 +487,9 @@ std::string Qwen3_8MTP::_speculative_generate(chat_meta_info_t& meta_info, int l // second copy of it -- a wasted 64-layer pass and a duplicate row. if (this->forward_on_eos) { this->lm_engine->forward(token); + } else { + // The engine never ran it, so the next prompt has to carry it. + this->token_history.pop_back(); } return false; } @@ -493,19 +498,43 @@ std::string Qwen3_8MTP::_speculative_generate(chat_meta_info_t& meta_info, int l reason = MAX_LENGTH_REACHED; return false; } - return this->total_tokens < this->MAX_L; + // No MAX_L test here: a batch's tokens before its last are already in + // the caches, so stopping mid-batch would leave rows token_history + // does not describe. The loop tests MAX_L between steps instead. + return true; }; - while (this->total_tokens < this->MAX_L){ + if (this->total_tokens >= this->MAX_L){ + header_print("WARNING", "Max length reached, stopping generation..."); + reason = MAX_LENGTH_REACHED; + } + else if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + } + else while (true){ + // Tested here rather than in the loop condition so the stop is + // recorded as a truncation, as in _shared_generate. + if (this->total_tokens >= this->MAX_L){ + reason = MAX_LENGTH_REACHED; + break; + } if (is_cancelled()) { reason = CANCEL_DETECTED; break; } - if (spec_enabled) { + // A batch must not run past length_limit: speculate() returns up to + // max_draft + 1 tokens, all but the last already committed, so capping + // the batch at the tokens left keeps the limit on the last one -- the + // one token a stop can drop. With one token left, decode it plainly. + const int tokens_left = (length_limit > 0) + ? length_limit - meta_info.generated_tokens + : SPEC_MAX_DRAFT + 1; + if (spec_enabled && tokens_left > 1) { this->profiler_list[DECODING_TIME].start(); std::vector accepted = - this->lm_engine->speculate(last_sampled_token, SPEC_MAX_DRAFT); + this->lm_engine->speculate(last_sampled_token, + std::min(SPEC_MAX_DRAFT, tokens_left - 1)); // Charge the cycle to however many tokens came out of it, so // tok/s stays comparable with the non-speculative path. this->profiler_list[DECODING_TIME].stop( @@ -573,6 +602,11 @@ std::string Qwen3_8MTP::_speculative_generate(chat_meta_info_t& meta_info, int l } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + if (reason != EOT_DETECTED) { + // Cut short after sampling its last token: no engine has run that one, + // so the next prompt has to carry it. + this->token_history.pop_back(); + } if (this->total_tokens >= this->MAX_L){ header_print("WARNING", "Max length reached, stopping generation..."); } diff --git a/src/common/AutoModel/modeling_qwen3vl.cpp b/src/common/AutoModel/modeling_qwen3vl.cpp index 657a052d7..512d2ed8e 100644 --- a/src/common/AutoModel/modeling_qwen3vl.cpp +++ b/src/common/AutoModel/modeling_qwen3vl.cpp @@ -37,20 +37,20 @@ void Qwen3VL::setup_tokenizer(std::string model_path) { auto tokenizer_config = this->_shared_setup_tokenizer(model_path); } -std::string Qwen3VL::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { +std::string Qwen3VL::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools, bool continue_final_message) { minja::chat_template_inputs inputs; - inputs.add_generation_prompt = true; inputs.messages = messages; - + if (!tools.empty()) inputs.tools = tools; inputs.extra_context = this->extra_context; - return this->chat_tmpl->apply(inputs); + return this->_render_chat_template(inputs, continue_final_message); } bool Qwen3VL::_build_vl_tokens(lm_uniform_input_t& input, qwen3vl_image_payload_t& image_payload, - std::vector& tokens) { + std::vector& tokens, + bool continue_final_message) { if (input.messages.empty() && input.prompt.empty()) { header_print("WARNING", "No messages or prompt provided"); return false; @@ -129,7 +129,7 @@ bool Qwen3VL::_build_vl_tokens(lm_uniform_input_t& input, qwenvl_message.push_back(newItem); } - templated_text = this->apply_chat_template(qwenvl_message, input.tools); + templated_text = this->apply_chat_template(qwenvl_message, input.tools, continue_final_message); header_print("FLM", "Total images: " << total_images); } else if (!input.prompt.empty()) { // a pure text, usually from the cli @@ -185,7 +185,7 @@ bool Qwen3VL::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std constexpr int image_soft_token_id = IMAGE_SOFT_TOKEN_ID; qwen3vl_image_payload_t image_payload; std::vector tokens; - if (!this->_build_vl_tokens(input, image_payload, tokens)) { + if (!this->_build_vl_tokens(input, image_payload, tokens, meta_info.continue_final_message)) { return false; } @@ -507,9 +507,15 @@ std::string Qwen3VL_Flash::generate(chat_meta_info_t& meta_info, int length_limi stop_reason_t reason = EOT_DETECTED; int last_sampled_token = this->last_token; this->token_history.push_back(last_sampled_token); + // the sample insert() took is the reply's first token, and counts toward the limit + meta_info.generated_tokens++; emit(last_sampled_token); - if (!this->is_eos(last_sampled_token) && this->total_tokens < this->MAX_L) { + if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit) + && !this->is_eos(last_sampled_token)) { + reason = MAX_LENGTH_REACHED; + } + else if (!this->is_eos(last_sampled_token) && this->total_tokens < this->MAX_L) { while (true) { if (is_cancelled()) { reason = CANCEL_DETECTED; @@ -549,6 +555,9 @@ std::string Qwen3VL_Flash::generate(chat_meta_info_t& meta_info, int length_limi meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; + // Whatever the stop, eos included, its last sampled token was never + // forward()ed (this loop does not feed an eos back), so the history drops it. + this->token_history.pop_back(); if (this->log_raw_output) { std::cout << std::endl; header_print("FLM", "Model RAW Output: \n" + result); @@ -575,6 +584,11 @@ std::string Qwen3VL_Thinking::generate_with_prompt(chat_meta_info_t& meta_info, std::string Qwen3VL_Thinking::generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { + // the echo stands for the generation prompt's "\n"; a continued message + // has no generation prompt, its output picks up right after the client's text + if (meta_info.continue_final_message) { + return this->_shared_generate(meta_info, length_limit, os, is_cancelled); + } std::string result; os << "\n\n"; result = this->_shared_generate(meta_info, length_limit, os, is_cancelled); diff --git a/src/common/chat/chat_continue.cpp b/src/common/chat/chat_continue.cpp new file mode 100644 index 000000000..ee36a2823 --- /dev/null +++ b/src/common/chat/chat_continue.cpp @@ -0,0 +1,60 @@ +/// \file chat_continue.cpp +/// \brief Continue the final assistant message instead of starting a new turn + +#include "chat/chat_continue.hpp" + +#include + +namespace { + +// Python's str.strip() set, short of the non-ASCII spaces no template trims to +const char* const WHITESPACE = " \t\n\r\v\f"; + +std::string lstrip(const std::string& s) { + const size_t b = s.find_first_not_of(WHITESPACE); + return b == std::string::npos ? std::string() : s.substr(b); +} + +std::string strip(const std::string& s) { + const std::string left = lstrip(s); + const size_t e = left.find_last_not_of(WHITESPACE); + return e == std::string::npos ? std::string() : left.substr(0, e + 1); +} + +/// \brief The text the final message ends with: its content, or its last text part +std::string final_text(const nlohmann::ordered_json& message) { + const auto content = message.find("content"); + if (content != message.end() && content->is_string()) { + return content->get(); + } + if (content != message.end() && content->is_array()) { + for (auto part = content->rbegin(); part != content->rend(); ++part) { + if (part->is_object() && part->contains("text") && (*part)["text"].is_string()) { + return (*part)["text"].get(); + } + } + } + throw std::invalid_argument("continue_final_message is set but the final message has no text to continue"); +} + +} // namespace + +std::string chat_continue_final_message(const std::string& rendered, const nlohmann::ordered_json& final_message) { + const std::string text = final_text(final_message); + const std::string stripped = strip(text); + if (stripped.empty()) { + // every rendering contains "", and cutting there would leave the end of turn in place + throw std::invalid_argument("continue_final_message is set but the final message has no text to continue"); + } + const size_t at = rendered.rfind(stripped); + if (at == std::string::npos) { + throw std::invalid_argument("continue_final_message is set but the final message does not appear in the chat " + "after applying the chat template; this template rewrites the final assistant message"); + } + // a template that kept the text as it is keeps its trailing whitespace too + const size_t kept = lstrip(text).size(); + if (rendered.compare(at, kept, text) == 0) { + return rendered.substr(0, at + kept); + } + return rendered.substr(0, at + stripped.size()); +} diff --git a/src/common/chat/chat_format.cpp b/src/common/chat/chat_format.cpp index f213f48e3..bd1c59c3a 100644 --- a/src/common/chat/chat_format.cpp +++ b/src/common/chat/chat_format.cpp @@ -118,7 +118,7 @@ chat_format_spec chat_format_detect(const std::string& template_src, return spec; } -std::string chat_format_prefill(const chat_format_spec& spec, const std::string& prompt_tail) { +std::string chat_format_prefill(const chat_format_spec& spec, const std::string& prompt_tail, bool continuing) { if (spec.kind == chat_format_kind::HARMONY) { // the generation prompt ends with an open "<|start|>assistant" header, and // may have started a channel too; the header is parsed with the output @@ -133,7 +133,7 @@ std::string chat_format_prefill(const chat_format_spec& spec, const std::string& size_t open = prompt_tail.rfind(spec.reasoning_start); size_t close = prompt_tail.rfind(spec.reasoning_end); if (open == std::string::npos || (close != std::string::npos && close > open)) { - return spec.reasoning_forced ? spec.reasoning_start : ""; + return spec.reasoning_forced && !continuing ? spec.reasoning_start : ""; } // only the tag: what follows it in the prompt is whitespace, and some // decoders render that as byte tokens like <0x0A> diff --git a/src/common/chat/chat_parser.cpp b/src/common/chat/chat_parser.cpp index 6d664088f..d5f6bba84 100644 --- a/src/common/chat/chat_parser.cpp +++ b/src/common/chat/chat_parser.cpp @@ -27,6 +27,12 @@ std::string trim(const std::string& s) { return s.substr(b, e - b); } +std::string trim_end(const std::string& s) { + size_t e = s.size(); + while (e > 0 && is_ws(s[e - 1])) e--; + return s.substr(0, e); +} + size_t skip_ws(const std::string& s, size_t pos) { while (pos < s.size() && is_ws(s[pos])) pos++; return pos; @@ -76,18 +82,28 @@ size_t find_first(const std::string& s, size_t pos, std::initializer_list parts; - - void add(const std::string& raw) { - parts.push_back(raw); + struct part_t { + std::string raw; + bool exact; // the first text of a continued message + }; + std::vector parts; + + void add(const std::string& raw, bool exact = false) { + parts.push_back({raw, exact}); } - std::string join() const { + /// \param is_final no more text follows, so trailing whitespace can be released + std::string join(bool is_final = false) const { + const bool others = std::any_of(parts.begin(), parts.end(), + [](const part_t& p) { return !p.exact && !trim(p.raw).empty(); }); std::string out; - for (const auto& raw : parts) { - std::string part = trim(raw); - if (part.empty()) { + for (const auto& [raw, exact] : parts) { + std::string part = !exact ? trim(raw) : is_final ? raw : trim_end(raw); + // whitespace alone is the answer only when nothing else is + if (part.empty() || (exact && others && trim(part).empty())) { continue; } if (!out.empty()) { @@ -468,6 +484,22 @@ chat_msg parse_tagged(const chat_format_spec& spec, const std::string& full_text text_parts content; bool in_reasoning = false; size_t pos = 0; + if (options.reasoning_may_close && spec.has_reasoning()) { + const size_t close = text.find(spec.reasoning_end); + const size_t open = text.find(spec.reasoning_start); + if (close != npos && (open == npos || close < open)) { + in_reasoning = true; // what came before the close was reasoning all along + } else if (open == npos && is_partial) { + return msg; // reasoning or answer: only a marker, or the end, can tell + } + } + // the first text of a continued message keeps its whitespace + bool lead_pending = options.keep_whitespace; + auto lead = [&lead_pending](const std::string& raw) { + const bool keep = lead_pending && !raw.empty(); + lead_pending = lead_pending && raw.empty(); + return keep; + }; while (pos <= text.size()) { if (in_reasoning) { @@ -488,10 +520,11 @@ chat_msg parse_tagged(const chat_format_spec& spec, const std::string& full_text if (is_partial) { rest = cut_partial_marker(rest, {&spec.reasoning_end, &stop_marker}); } - reasoning.add(rest); + reasoning.add(rest, lead(rest)); break; } - reasoning.add(text.substr(pos, end - pos)); + const std::string part = text.substr(pos, end - pos); + reasoning.add(part, lead(part)); pos = end + spec.reasoning_end.size(); in_reasoning = false; continue; @@ -504,10 +537,11 @@ chat_msg parse_tagged(const chat_format_spec& spec, const std::string& full_text if (is_partial) { rest = cut_partial_marker(rest, {&reasoning_start, &tool_start, &stop_marker}); } - content.add(rest); + content.add(rest, lead(rest)); break; } - content.add(text.substr(pos, at - pos)); + const std::string part = text.substr(pos, at - pos); + content.add(part, lead(part)); if (which == &reasoning_start) { pos = at + reasoning_start.size(); in_reasoning = true; @@ -530,8 +564,11 @@ chat_msg parse_tagged(const chat_format_spec& spec, const std::string& full_text pos = body_end + spec.tool_end.size(); } - msg.reasoning = reasoning.join(); - msg.content = content.join(); + msg.reasoning = reasoning.join(!is_partial); + msg.content = content.join(!is_partial); + if (!msg.tool_calls.empty() && trim(msg.content).empty()) { + msg.content.clear(); // whitespace before a call is layout, kept only as a whole answer + } return msg; } diff --git a/src/common/chat/chat_sources.cmake b/src/common/chat/chat_sources.cmake index ea03a38a9..431ee7572 100644 --- a/src/common/chat/chat_sources.cmake +++ b/src/common/chat/chat_sources.cmake @@ -7,6 +7,7 @@ set(FLM_CHAT_SOURCES "${CMAKE_CURRENT_LIST_DIR}/chat_args.cpp" "${CMAKE_CURRENT_LIST_DIR}/chat_constraint.cpp" + "${CMAKE_CURRENT_LIST_DIR}/chat_continue.cpp" "${CMAKE_CURRENT_LIST_DIR}/chat_format.cpp" "${CMAKE_CURRENT_LIST_DIR}/chat_grammar.cpp" "${CMAKE_CURRENT_LIST_DIR}/chat_msg.cpp" diff --git a/src/common/tokenizer/tokenizer.cpp b/src/common/tokenizer/tokenizer.cpp index 026c43bfc..2ef114f69 100644 --- a/src/common/tokenizer/tokenizer.cpp +++ b/src/common/tokenizer/tokenizer.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -178,7 +179,13 @@ std::string Tokenizer::decode(const std::vector& tokens) { /// \param answer_token the answer token /// \return the decoded text std::string Tokenizer::run_time_decoder(int answer_token) { - return this->cpt_to_utf8(this->tokenizer->IdToToken(answer_token)); + const std::string token = this->tokenizer->IdToToken(answer_token); + // SentencePiece byte fallback: "<0x0D>" stands for the byte 0x0D, not for that text + if (!this->is_doubled_encoded && token.size() == 6 && token.compare(0, 3, "<0x") == 0 && token[5] == '>' && + std::isxdigit(static_cast(token[3])) && std::isxdigit(static_cast(token[4]))) { + return std::string(1, static_cast(std::stoi(token.substr(3, 2), nullptr, 16))); + } + return this->cpt_to_utf8(token); } /// \brief Number of UTF-8 bytes of the text one token stands for diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index f8fea5b9f..6b6bc3704 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -124,6 +124,11 @@ struct chat_meta_info_t { /// \brief The last sample added an entry to logprobs /// \note So that _skip_insert_sample() can take it back out with the sample. bool last_sample_logged = false; + /// \brief The prompt ends inside the final assistant message, and the output continues it + /// \note The request's continue_final_message. The template gets no generation + /// prompt, and the turn starts right after the message's text: insert() + /// trims no prompt tokens and generate() feeds no preamble of its own. + bool continue_final_message = false; chat_meta_info_t() : max_prefill_len(0), prompt_tokens(0), cached_prompt_tokens(0), generated_tokens(0), total_duration(0), load_duration(0), prefill_duration(0), decoding_duration(0), stop_reason(EOT_DETECTED), restore_allowed(false) {} }; @@ -161,12 +166,15 @@ using json = nlohmann::ordered_json; class ModelRequestError final : public std::runtime_error { public: - ModelRequestError(int http_code, bool session_cleared, std::string message); + /// \param type the OpenAI error type: "model_error", or "invalid_request_error" when the request itself is at fault + ModelRequestError(int http_code, bool session_cleared, std::string message, std::string type = "model_error"); int http_code() const noexcept; bool session_cleared() const noexcept; + const std::string& type() const noexcept; private: int http_code_; bool session_cleared_; + std::string type_; }; class AutoModel { @@ -353,6 +361,20 @@ class AutoModel { buffer _chunked_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0); std::string _shared_generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }); + /// \brief Render the chat template for a request + /// \param inputs the template inputs; add_generation_prompt is set here + /// \param continue_final_message render no generation prompt and stop right + /// after the final message's text (see chat_continue.hpp) + /// \param options minja's render options, for templates that need polyfills turned off + /// \throws ModelRequestError 400 when the final message cannot be continued + /// because the template rewrote it + std::string _render_chat_template(minja::chat_template_inputs& inputs, bool continue_final_message, + const minja::chat_template_options& options = minja::chat_template_options()); + /// \brief The continued assistant turn of the last prompt rendered with continue_final_message: + /// the template's opening of the turn and the message's text + /// \note _begin_turn() hands it to the output parser for such a request, which renders it just before. + std::string continued_turn_; + /// \brief Note where this turn's output starts, once the prompt is final /// \param meta_info the request; gets the prompt tail, and its constraint is started /// \param prompt_tokens the whole prompt as the model sees it @@ -563,6 +585,14 @@ class AutoModel { return true; } + /// \brief Whether a request can continue its final assistant message (continue_final_message) + /// \note Off unless a model says otherwise: its insert() must pass the flag to + /// apply_chat_template() and keep every prompt token, and its generate() + /// must start from the sample insert() made rather than feed a preamble. + virtual bool supports_continue_final_message() const { + return false; + } + /// \brief Prepare the benchmark /// \param text the text to benchmark /// \return a pair of the number of tokens and the text that produce 1000 tokens @@ -664,7 +694,9 @@ class AutoModel { return configure_parameter(parameter_name, std::any(value)); } - virtual std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) = 0; + /// \brief Render the chat + /// \param continue_final_message continue the final assistant message instead of opening a new turn + virtual std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) = 0; }; diff --git a/src/include/AutoModel/modeling_gemma3.hpp b/src/include/AutoModel/modeling_gemma3.hpp index 0b539bc02..b0fa9bd5f 100644 --- a/src/include/AutoModel/modeling_gemma3.hpp +++ b/src/include/AutoModel/modeling_gemma3.hpp @@ -34,7 +34,8 @@ class Gemma3 : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } bool check_using_checkpint() { return false; } diff --git a/src/include/AutoModel/modeling_gemma3_text.hpp b/src/include/AutoModel/modeling_gemma3_text.hpp index ba725974f..ac9cc8f05 100644 --- a/src/include/AutoModel/modeling_gemma3_text.hpp +++ b/src/include/AutoModel/modeling_gemma3_text.hpp @@ -26,7 +26,8 @@ class Gemma3_Text_Only : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } bool check_using_checkpint() { return false; } diff --git a/src/include/AutoModel/modeling_gemma4_12b.hpp b/src/include/AutoModel/modeling_gemma4_12b.hpp index a1232dbf8..f96d9bea3 100644 --- a/src/include/AutoModel/modeling_gemma4_12b.hpp +++ b/src/include/AutoModel/modeling_gemma4_12b.hpp @@ -103,7 +103,8 @@ class Gemma4_12B : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } chat_template_type_t get_chat_template_type() override { return chat_template_type_t::gemma4; } diff --git a/src/include/AutoModel/modeling_gemma4e.hpp b/src/include/AutoModel/modeling_gemma4e.hpp index 0804f9a44..1fe06124b 100644 --- a/src/include/AutoModel/modeling_gemma4e.hpp +++ b/src/include/AutoModel/modeling_gemma4e.hpp @@ -125,7 +125,8 @@ class Gemma4e : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } chat_template_type_t get_chat_template_type() { return chat_template_type_t::gemma4; } @@ -255,8 +256,12 @@ class Gemma4e_Flash : public Gemma4e { /// \brief Apply the chat template without tools — gemma4e_flash does not /// support tool calling, so the tools argument is always dropped. - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override { + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override { nlohmann::ordered_json no_tools = nlohmann::ordered_json::object(); - return Gemma4e::apply_chat_template(messages, no_tools); + return Gemma4e::apply_chat_template(messages, no_tools, continue_final_message); } + + /// \brief Single-turn (every insert() starts over), so the server rejects + /// continue_final_message for it; Gemma4e's true is not inherited. + bool supports_continue_final_message() const override { return false; } }; diff --git a/src/include/AutoModel/modeling_gpt_oss.hpp b/src/include/AutoModel/modeling_gpt_oss.hpp index 607abc56d..e273511d0 100644 --- a/src/include/AutoModel/modeling_gpt_oss.hpp +++ b/src/include/AutoModel/modeling_gpt_oss.hpp @@ -23,7 +23,7 @@ class GPT_OSS : public AutoModel { GPT_OSS(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_info, int default_context_length = -1, bool enable_preemption = false, const std::string& backend = "") override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; diff --git a/src/include/AutoModel/modeling_hunyuan.hpp b/src/include/AutoModel/modeling_hunyuan.hpp index e31b10d4d..9cbd98fe1 100644 --- a/src/include/AutoModel/modeling_hunyuan.hpp +++ b/src/include/AutoModel/modeling_hunyuan.hpp @@ -53,7 +53,7 @@ class Hunyuan : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; chat_template_type_t get_chat_template_type() override; /// \brief install a constant system prompt and pin its prefill in the KV cache diff --git a/src/include/AutoModel/modeling_lfm2.hpp b/src/include/AutoModel/modeling_lfm2.hpp index 3f105ead9..5cebcfac2 100644 --- a/src/include/AutoModel/modeling_lfm2.hpp +++ b/src/include/AutoModel/modeling_lfm2.hpp @@ -31,7 +31,10 @@ class LFM2 : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { + return true; + } bool check_using_checkpint() { return false; } @@ -62,5 +65,8 @@ class LFM2_5_TK : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { + return true; + } }; \ No newline at end of file diff --git a/src/include/AutoModel/modeling_lfm3.hpp b/src/include/AutoModel/modeling_lfm3.hpp index ff05bad5f..4f66cd060 100644 --- a/src/include/AutoModel/modeling_lfm3.hpp +++ b/src/include/AutoModel/modeling_lfm3.hpp @@ -20,10 +20,13 @@ class LFM3 : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; bool thinking_enabled() const override { return this->enable_think; } + bool supports_continue_final_message() const override { + return true; + } /// \brief enable_think / toggle_think / reasoning_effort ("none" turns thinking off) and everything the base class handles bool configure_parameter(std::string parameter_name, const std::any& value) override { diff --git a/src/include/AutoModel/modeling_llama3.hpp b/src/include/AutoModel/modeling_llama3.hpp index dfb19dbba..a6918f613 100644 --- a/src/include/AutoModel/modeling_llama3.hpp +++ b/src/include/AutoModel/modeling_llama3.hpp @@ -21,7 +21,8 @@ class Llama3 : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } bool check_using_checkpint() { return false; } @@ -46,5 +47,6 @@ class DeepSeek_r1_8b : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } }; \ No newline at end of file diff --git a/src/include/AutoModel/modeling_minicpm_v.hpp b/src/include/AutoModel/modeling_minicpm_v.hpp index 9a5b2f03f..a51ce0763 100644 --- a/src/include/AutoModel/modeling_minicpm_v.hpp +++ b/src/include/AutoModel/modeling_minicpm_v.hpp @@ -63,11 +63,14 @@ class MiniCPM_V : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; bool thinking_enabled() const override { return this->enable_think; } + bool supports_continue_final_message() const override { + return true; + } bool tools_enabled() const override { return this->enable_tool; } diff --git a/src/include/AutoModel/modeling_nanbeige.hpp b/src/include/AutoModel/modeling_nanbeige.hpp index a9fa71ff0..2df695e9a 100644 --- a/src/include/AutoModel/modeling_nanbeige.hpp +++ b/src/include/AutoModel/modeling_nanbeige.hpp @@ -23,5 +23,6 @@ class Nanbeige : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } }; diff --git a/src/include/AutoModel/modeling_phi4.hpp b/src/include/AutoModel/modeling_phi4.hpp index 032ed3d34..2373cac09 100644 --- a/src/include/AutoModel/modeling_phi4.hpp +++ b/src/include/AutoModel/modeling_phi4.hpp @@ -50,7 +50,9 @@ class Phi4 : public AutoModel { int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, - nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + nlohmann::ordered_json tools = nlohmann::ordered_json::object(), + bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } /// \note Phi-4 does not reuse a checkpointed prefix, so a matched prefix /// still costs one round. Overrides the base default of true. diff --git a/src/include/AutoModel/modeling_qwen2.hpp b/src/include/AutoModel/modeling_qwen2.hpp index b0439bc43..3364b19ba 100644 --- a/src/include/AutoModel/modeling_qwen2.hpp +++ b/src/include/AutoModel/modeling_qwen2.hpp @@ -24,7 +24,8 @@ class Qwen2 : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } bool check_using_checkpint() { return false; } diff --git a/src/include/AutoModel/modeling_qwen2vl.hpp b/src/include/AutoModel/modeling_qwen2vl.hpp index 1a2e05347..57ccacc1f 100644 --- a/src/include/AutoModel/modeling_qwen2vl.hpp +++ b/src/include/AutoModel/modeling_qwen2vl.hpp @@ -50,7 +50,8 @@ class Qwen2VL : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } bool check_using_checkpint() { return false; } diff --git a/src/include/AutoModel/modeling_qwen3.hpp b/src/include/AutoModel/modeling_qwen3.hpp index 77eec81a7..73d2d87d2 100644 --- a/src/include/AutoModel/modeling_qwen3.hpp +++ b/src/include/AutoModel/modeling_qwen3.hpp @@ -35,7 +35,8 @@ class Qwen3 : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } bool thinking_enabled() const override { return this->enable_think; } @@ -100,7 +101,8 @@ class Qwen3_IT : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } bool check_using_checkpint() { return false; } @@ -125,7 +127,8 @@ class Qwen3_TK : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } }; @@ -148,5 +151,6 @@ class DeepSeek_r1_0528_8b : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } }; \ No newline at end of file diff --git a/src/include/AutoModel/modeling_qwen3_5_omni.hpp b/src/include/AutoModel/modeling_qwen3_5_omni.hpp index abd07121b..4e6f4f095 100644 --- a/src/include/AutoModel/modeling_qwen3_5_omni.hpp +++ b/src/include/AutoModel/modeling_qwen3_5_omni.hpp @@ -35,7 +35,7 @@ class Qwen3_5_Omni : public AutoModel { void load_model(std::string model_path, json model_info, int default_context_length = -1, bool enable_preemption = false, const std::string& backend = "") override; /// \brief Apply the chat template to the messages. - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; /// \brief Encode + preprocess the input (text/image/audio) and prefill. bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; @@ -65,6 +65,7 @@ class Qwen3_5_Omni : public AutoModel { bool thinking_enabled() const override { return false; } + bool supports_continue_final_message() const override { return true; } /// \brief Configure a parameter with type-erased value /// \param parameter_name the name of the parameter diff --git a/src/include/AutoModel/modeling_qwen3_5vl.hpp b/src/include/AutoModel/modeling_qwen3_5vl.hpp index 98d2ce307..874777609 100644 --- a/src/include/AutoModel/modeling_qwen3_5vl.hpp +++ b/src/include/AutoModel/modeling_qwen3_5vl.hpp @@ -59,7 +59,8 @@ class Qwen3_5VL : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } private: diff --git a/src/include/AutoModel/modeling_qwen3_6_moe.hpp b/src/include/AutoModel/modeling_qwen3_6_moe.hpp index c809d63a1..e6646fb92 100644 --- a/src/include/AutoModel/modeling_qwen3_6_moe.hpp +++ b/src/include/AutoModel/modeling_qwen3_6_moe.hpp @@ -59,7 +59,8 @@ class Qwen3_6_MOE : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } private: diff --git a/src/include/AutoModel/modeling_qwen3_8mtp.hpp b/src/include/AutoModel/modeling_qwen3_8mtp.hpp index fab6dd505..53049db72 100644 --- a/src/include/AutoModel/modeling_qwen3_8mtp.hpp +++ b/src/include/AutoModel/modeling_qwen3_8mtp.hpp @@ -150,7 +150,8 @@ class Qwen3_8MTP : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } /// \brief the base profile plus the MTP draft/verify/replay breakdown /// \note Overridden because "Decoding time" is one number for a model with diff --git a/src/include/AutoModel/modeling_qwen3vl.hpp b/src/include/AutoModel/modeling_qwen3vl.hpp index fa9cec704..9222fc65a 100644 --- a/src/include/AutoModel/modeling_qwen3vl.hpp +++ b/src/include/AutoModel/modeling_qwen3vl.hpp @@ -52,13 +52,16 @@ class Qwen3VL : public AutoModel { /// \param image_payload receives the preprocessed pixels for every image that /// survived decoding, in prompt order /// \param tokens receives the prompt token ids + /// \param continue_final_message render `messages` so that the prompt ends + /// inside their final assistant message (the CLI prompt never does) /// \return false if there was nothing to prefill /// \note Images that fail to decode or preprocess are dropped without /// leaving a placeholder behind, so `image_payload` always stays /// aligned with the image soft tokens in `tokens`. bool _build_vl_tokens(lm_uniform_input_t& input, qwen3vl_image_payload_t& image_payload, - std::vector& tokens); + std::vector& tokens, + bool continue_final_message = false); /// \note The Qwen3-VL checkpoint is served by two engines that share this /// whole wrapper -- the tokenizer, chat template, sampler and image @@ -81,7 +84,8 @@ class Qwen3VL : public AutoModel { bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override; + bool supports_continue_final_message() const override { return true; } bool check_using_checkpint() { return false; } @@ -188,11 +192,14 @@ class Qwen3VL_Flash : public Qwen3VL { /// \brief Apply the chat template without tools — qwen3vl_flash does not /// support tool calling, so the tools argument is always dropped. - std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override { + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object(), bool continue_final_message = false) override { nlohmann::ordered_json no_tools = nlohmann::ordered_json::object(); - return Qwen3VL::apply_chat_template(messages, no_tools); + return Qwen3VL::apply_chat_template(messages, no_tools, continue_final_message); } + /// \brief Single-turn: insert() rebuilds the prompt and never continues a message + bool supports_continue_final_message() const override { return false; } + /// \brief img_pre_resize is ignored — the flash engine always forces every /// image down to a 256px longer side (see _forced_long_side above). bool configure_parameter(std::string parameter_name, const std::any& value) override { diff --git a/src/include/chat/chat_continue.hpp b/src/include/chat/chat_continue.hpp new file mode 100644 index 000000000..ce38a69db --- /dev/null +++ b/src/include/chat/chat_continue.hpp @@ -0,0 +1,25 @@ +/// \file chat_continue.hpp +/// \brief Continue the final assistant message instead of starting a new turn +/// \note The request field continue_final_message (vLLM and Hugging Face): the +/// chat is rendered without a generation prompt and cut right after the +/// final message's text, so the model writes on from there. The cut follows +/// transformers' apply_chat_template(continue_final_message=True), so the +/// same template gives the same prompt here as there. +#pragma once + +#include + +#include + +/// \brief Cut a rendered chat right after the text of its final message +/// \param rendered the chat rendered without a generation prompt +/// \param final_message the last message as the template was given it; its +/// content is a string or an array of typed parts, whose last text part counts +/// \return the rendered chat up to the end of the final message's text +/// \note Whitespace around the text is matched loosely, as transformers does: +/// templates often trim the content. Trailing whitespace is kept only when +/// the template rendered it. +/// \throws std::invalid_argument when the final message has no text, or the +/// template did not render that text as it is (some templates rewrite the +/// last assistant message, e.g. by moving a block out of it) +std::string chat_continue_final_message(const std::string& rendered, const nlohmann::ordered_json& final_message); diff --git a/src/include/chat/chat_format.hpp b/src/include/chat/chat_format.hpp index 58af89a05..74f55a12e 100644 --- a/src/include/chat/chat_format.hpp +++ b/src/include/chat/chat_format.hpp @@ -63,6 +63,9 @@ chat_format_spec chat_format_detect(const std::string& template_src, /// \brief The part of the prompt the parser has to see before the generated text /// \param spec the model's format /// \param prompt_tail the decoded end of the prompt as fed to the model +/// \param continuing the prompt ends inside the final assistant message +/// (continue_final_message), not at the start of a new turn; a model that +/// always opens its turn by reasoning has then already opened it, or not /// \return text to prepend to the output before parsing, e.g. "" when the /// template already opened the reasoning block -std::string chat_format_prefill(const chat_format_spec& spec, const std::string& prompt_tail); +std::string chat_format_prefill(const chat_format_spec& spec, const std::string& prompt_tail, bool continuing = false); diff --git a/src/include/chat/chat_parser.hpp b/src/include/chat/chat_parser.hpp index c7d587dcd..ca4d019a3 100644 --- a/src/include/chat/chat_parser.hpp +++ b/src/include/chat/chat_parser.hpp @@ -26,6 +26,19 @@ struct chat_parse_options { const nlohmann::ordered_json* tools = nullptr; /// \brief extract tool calls; when false the tool markup stays in the content bool parse_tool_calls = true; + /// \brief keep the whitespace around the output's first text, as it was written + /// \note For continue_final_message: the output continues text the client + /// already holds, so its whitespace (" yes" after "The answer is:", or a + /// lone "\n") belongs to the answer rather than to the layout. Trailing + /// whitespace is held back until the text is final, like any other. + bool keep_whitespace = false; + /// \brief the output may close a reasoning block the prompt never opened + /// \note For continue_final_message on a model that reasons first: such a + /// model reads the continued message as its reasoning, so a reasoning + /// end it writes before any reasoning start closes that, and the text + /// before it is reasoning (as vLLM's DeepSeek-R1 parser has it). Until + /// either marker shows, a partial parse yields nothing. + bool reasoning_may_close = false; }; /// \brief Parse the assistant output diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 9db4caab8..e8da0e34d 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -25,7 +25,7 @@ namespace { json ModelErrorResponse(const ModelRequestError& error) { return {{"error", {{"message", error.what()}, - {"type", "model_error"}, + {"type", error.type()}, {"code", error.http_code()}, {"session_cleared", error.session_cleared()}}}}; } @@ -311,6 +311,65 @@ static bool sampling_request_from_openai(const json& request, openai_sampling_re return true; } +///@brief Read an OpenAI chat request's continue_final_message (vLLM's extension) and check +/// that the request can continue its final message. +///@param messages the request's messages, as sent +///@param constraint the request's tool_choice and response_format +///@param continue_final set to whether the final assistant message is continued +///@param error set to the message for the client when the request is malformed +///@return false when the request is malformed +///@note add_generation_prompt is read only to refuse it alongside continue_final_message, as +/// vLLM does: the two ask for opposite things. On its own it is ignored, as before. +static bool continue_request_from_openai(const json& request, const json& messages, + const chat_constraint_request& constraint, + bool& continue_final, std::string& error) { + continue_final = false; + if (has_value(request, "continue_final_message")) { + if (!request.at("continue_final_message").is_boolean()) { + error = "continue_final_message must be a boolean"; + return false; + } + continue_final = request.at("continue_final_message").get(); + } + if (!continue_final) { + return true; + } + if (has_value(request, "add_generation_prompt") && request.at("add_generation_prompt") == true) { + error = "Cannot set both continue_final_message and add_generation_prompt to true"; + return false; + } + const json* last = messages.empty() ? nullptr : &messages.back(); + if (last == nullptr || !last->is_object() || last->value("role", "") != "assistant") { + error = "continue_final_message requires the last message to have role \"assistant\""; + return false; + } + if (has_value(*last, "tool_calls") && !(last->at("tool_calls").is_array() && last->at("tool_calls").empty())) { + error = "continue_final_message cannot continue an assistant message that has tool_calls"; + return false; + } + // the grammar of a forced call or a JSON answer starts at the beginning of a turn + if (constraint.json_object || !constraint.json_schema.is_null() || + constraint.tool_choice == chat_tool_choice::REQUIRED || constraint.tool_choice == chat_tool_choice::FUNCTION) { + error = "continue_final_message cannot be combined with response_format or a tool_choice that forces a call"; + return false; + } + return true; +} + +///@brief Check the final message once normalize_template has flattened it: text to continue, and nothing else +///@return the message for the client, empty when the message can be continued +static std::string continue_final_message_error(const json& messages) { + const json& last = messages.back(); + if (last.contains("images") || last.contains("audios")) { + return "continue_final_message requires the final assistant message to be text only"; + } + const std::string text = last.value("content", std::string()); + if (text.find_first_not_of(" \t\n\r\v\f") == std::string::npos) { + return "continue_final_message requires text in the final assistant message"; + } + return ""; +} + static void apply_sampling_request(AutoModel& engine, const openai_sampling_request& sampling) { if (sampling.penalty_window) { engine.set_penalty_window(*sampling.penalty_window); @@ -790,7 +849,8 @@ json RestHandler::build_nstream_response(const std::string& response_text, chat_ const chat_parse_options& parse_options, chat_logprobs_aligner* logprobs) { const chat_format_spec& spec = auto_chat_engine->get_chat_format(); - chat_msg parsed = chat_parse(spec, chat_format_prefill(spec, meta_info.prompt_tail) + response_text, + chat_msg parsed = chat_parse(spec, + chat_format_prefill(spec, meta_info.prompt_tail, meta_info.continue_final_message) + response_text, false, parse_options); json message; @@ -1388,6 +1448,11 @@ void RestHandler::handle_openai_chat_completion(const json& request, send_response({{"error", {{"message", request_error}, {"type", "invalid_request_error"}, {"code", 400}}}}); return; } + bool continue_final = false; + if (!continue_request_from_openai(request, current_messages, constraint_request, continue_final, request_error)) { + send_response({{"error", {{"message", request_error}, {"type", "invalid_request_error"}, {"code", 400}}}}); + return; + } auto load_start_time = time_utils::now(); if (!ensure_model_loaded(model, request_backend)) { @@ -1401,12 +1466,24 @@ void RestHandler::handle_openai_chat_completion(const json& request, {"type", "invalid_request_error"}, {"code", 400}}}}); return; } + if (continue_final && (auto_chat_engine->single_turn || !auto_chat_engine->supports_continue_final_message())) { + send_response({{"error", {{"message", "continue_final_message is not supported by " + model}, + {"type", "invalid_request_error"}, {"code", 400}}}}); + return; + } configure_chat_engine_parameters(options, request); apply_sampling_request(*auto_chat_engine, sampling); current_messages = normalize_messages(current_messages); current_messages = normalize_template(current_messages); + if (continue_final) { + request_error = continue_final_message_error(current_messages); + if (!request_error.empty()) { + send_response({{"error", {{"message", request_error}, {"type", "invalid_request_error"}, {"code", 400}}}}); + return; + } + } // see if we can use prompt cache chat_meta_info_t meta_info; @@ -1424,8 +1501,8 @@ void RestHandler::handle_openai_chat_completion(const json& request, } if (has_assistant_msg) { header_print("Warning", "Single-turn model received multi-turn request (assistant messages detected). Rejecting."); - json error_response = {{"error", "This model only supports single-turn requests. Multi-turn conversation history (assistant messages) is not allowed."}}; - send_response(error_response); + send_response({{"error", {{"message", "This model only supports single-turn requests. Multi-turn conversation history (assistant messages) is not allowed."}, + {"type", "invalid_request_error"}, {"code", 400}}}}); return; } } @@ -1483,6 +1560,7 @@ void RestHandler::handle_openai_chat_completion(const json& request, meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first; meta_info.max_prefill_len = this->prefill_chunk_len; meta_info.logprobs_top_n = logprobs_top_n; + meta_info.continue_final_message = continue_final; try { meta_info.constraint = auto_chat_engine->make_constraint(constraint_request); } catch (const std::invalid_argument& e) { @@ -1494,6 +1572,9 @@ void RestHandler::handle_openai_chat_completion(const json& request, chat_parse_options parse_options; parse_options.tools = &tools; parse_options.parse_tool_calls = !tools.empty() && constraint_request.tool_choice != chat_tool_choice::NONE; + parse_options.keep_whitespace = continue_final; + // a model that reasons first reads the continued message as its reasoning + parse_options.reasoning_may_close = continue_final && auto_chat_engine->thinking_enabled(); if (stream){ // Create a wrapper callback that passes the pre-formatted SSE string directly cancellation_token->reset(); diff --git a/src/server/streaming_ostream_openai.hpp b/src/server/streaming_ostream_openai.hpp index e7e04ee76..b8210f4d8 100644 --- a/src/server/streaming_ostream_openai.hpp +++ b/src/server/streaming_ostream_openai.hpp @@ -312,7 +312,8 @@ class streaming_buf_openai_chat : public std::streambuf { /// when it asks for logprobs it must outlive the stream void start(const chat_meta_info_t& meta_info) { const chat_format_spec& spec = auto_chat_engine->get_chat_format(); - stream = std::make_unique(spec, chat_format_prefill(spec, meta_info.prompt_tail), parse_options); + stream = std::make_unique( + spec, chat_format_prefill(spec, meta_info.prompt_tail, meta_info.continue_final_message), parse_options); if (meta_info.logprobs_top_n >= 0) { logprobs = std::make_unique(auto_chat_engine, meta_info.logprobs); } From 9c6b258b7c5c2219ea77f1dd57f409fb0a9ca6d1 Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Wed, 28 Oct 2026 14:39:16 -0700 Subject: [PATCH 2/5] test: continue_final_message, exact max_tokens, reasoning budget - src/test/chat_continue: CPU-only test of the cut on the template shapes FLM ships and of the parser for continued text; runs in the Ubuntu CI - openai_api_test/test_continue_final_message.py: continuation, logprobs, stream parity, cache resume, follow-up turn, 400s - test_logprobs.max_tokens_is_exact - --always-thinks now gives each answer a reasoning budget on top (--reasoning-budget, default 1024): those models used up the answer budgets reasoning, so content, JSON and tool calls never came Co-Authored-By: Claude --- .github/workflows/ubuntu-build.yml | 13 + src/test/chat_continue/CMakeLists.txt | 27 ++ src/test/chat_continue/test_chat_continue.cpp | 267 ++++++++++++++++++ src/test/openai_api_test/README.md | 7 +- src/test/openai_api_test/fixtures.py | 5 + src/test/openai_api_test/flm_api.py | 24 +- src/test/openai_api_test/run_all.py | 3 +- .../test_continue_final_message.py | 244 ++++++++++++++++ src/test/openai_api_test/test_logprobs.py | 23 +- 9 files changed, 605 insertions(+), 8 deletions(-) create mode 100644 src/test/chat_continue/CMakeLists.txt create mode 100644 src/test/chat_continue/test_chat_continue.cpp create mode 100644 src/test/openai_api_test/test_continue_final_message.py diff --git a/.github/workflows/ubuntu-build.yml b/.github/workflows/ubuntu-build.yml index 1dac94b0e..b2a1e87f7 100644 --- a/.github/workflows/ubuntu-build.yml +++ b/.github/workflows/ubuntu-build.yml @@ -25,6 +25,19 @@ jobs: cmake --build build-d3-contract -j"$(nproc)" ctest --test-dir build-d3-contract --output-on-failure + # continue_final_message (src/common/chat): where the prompt is cut on the template shapes the + # models ship, and how the output parser reads the continued text. CPU only, no NPU. + chat-continue: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Build and run the chat continue test + run: | + cmake -S src/test/chat_continue -B build-chat-continue -DCMAKE_BUILD_TYPE=Release + cmake --build build-chat-continue -j"$(nproc)" + ctest --test-dir build-chat-continue --output-on-failure + package-deb: runs-on: ubuntu-latest strategy: diff --git a/src/test/chat_continue/CMakeLists.txt b/src/test/chat_continue/CMakeLists.txt new file mode 100644 index 000000000..d816d90c8 --- /dev/null +++ b/src/test/chat_continue/CMakeLists.txt @@ -0,0 +1,27 @@ +# continue_final_message tests: the prompt cut on real template shapes, and the output parser for a continued message. +# No NPU, no XRT, so this builds and runs on Linux CI as well as Windows. +# cmake -S src/test/chat_continue -B build-cc && cmake --build build-cc && ctest --test-dir build-cc +cmake_minimum_required(VERSION 3.22) +project(chat_continue_tests LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(FLM_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../..") + +add_executable(test_chat_continue + test_chat_continue.cpp + "${FLM_SOURCE_DIR}/common/chat/chat_args.cpp" + "${FLM_SOURCE_DIR}/common/chat/chat_continue.cpp" + "${FLM_SOURCE_DIR}/common/chat/chat_format.cpp" + "${FLM_SOURCE_DIR}/common/chat/chat_msg.cpp" + "${FLM_SOURCE_DIR}/common/chat/chat_parser.cpp" + "${FLM_SOURCE_DIR}/common/chat/chat_stream.cpp") +target_include_directories(test_chat_continue PRIVATE "${FLM_SOURCE_DIR}/include") +if(MSVC) + # the DeepSeek markers below are UTF-8 literals + target_compile_options(test_chat_continue PRIVATE /utf-8 /bigobj) +endif() + +enable_testing() +add_test(NAME test_chat_continue COMMAND test_chat_continue) diff --git a/src/test/chat_continue/test_chat_continue.cpp b/src/test/chat_continue/test_chat_continue.cpp new file mode 100644 index 000000000..54ad7c605 --- /dev/null +++ b/src/test/chat_continue/test_chat_continue.cpp @@ -0,0 +1,267 @@ +/// \file test_chat_continue.cpp +/// \brief continue_final_message without a model: where the rendered prompt is cut, on the template shapes FLM ships, +/// and how the output parser treats the continued text. No NPU needed. +#include "chat/chat_continue.hpp" +#include "chat/chat_format.hpp" +#include "chat/chat_parser.hpp" +#include "chat/chat_stream.hpp" +#include "minja/chat-template.hpp" + +#include +#include +#include +#include + +namespace { +int failures = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::printf(" FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + failures++; \ + } \ + } while (0) + +#define CHECK_EQ(actual, expected) \ + do { \ + const std::string a_ = (actual), e_ = (expected); \ + if (a_ != e_) { \ + std::printf(" FAIL %s:%d: %s\n got: [%s]\n expected: [%s]\n", \ + __FILE__, __LINE__, #actual, a_.c_str(), e_.c_str()); \ + failures++; \ + } \ + } while (0) + +using json = nlohmann::ordered_json; + +json chat(const std::string& user, const json& assistant) { + return json::array({{{"role", "user"}, {"content", user}}, {{"role", "assistant"}, {"content", assistant}}}); +} + +/// \brief What AutoModel::_render_chat_template does when continuing +std::string continued(const std::string& source, const json& messages, const std::string& bos = "") { + minja::chat_template tmpl(source, bos, ""); + minja::chat_template_inputs inputs; + inputs.messages = messages; + inputs.add_generation_prompt = false; + return chat_continue_final_message(tmpl.apply(inputs), inputs.messages.back()); +} + +bool rejects(const std::string& rendered, const json& message) { + try { + chat_continue_final_message(rendered, message); + return false; + } catch (const std::invalid_argument&) { + return true; + } +} + +// Shapes of the templates in the model packages, cut down to the parts that decide the cut. +const std::string LLAMA3 = + "{{- bos_token }}{%- for m in messages %}" + "{{- '<|start_header_id|>' + m['role'] + '<|end_header_id|>\\n\\n' + m['content'] | trim + '<|eot_id|>' }}" + "{%- endfor %}{%- if add_generation_prompt %}{{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}{%- endif %}"; + +const std::string CHATML = + "{%- for m in messages %}{{- '<|im_start|>' + m['role'] + '\\n' + m['content'] + '<|im_end|>\\n' }}{%- endfor %}" + "{%- if add_generation_prompt %}{{- '<|im_start|>assistant\\n' }}{%- endif %}"; + +// Qwen3: the last assistant turn after the last user query always gets a think block +const std::string QWEN3 = + "{%- for m in messages %}" + "{%- if m['role'] == 'assistant' and loop.last %}" + "{{- '<|im_start|>assistant\\n\\n\\n\\n\\n' + m['content'].lstrip('\\n') + '<|im_end|>\\n' }}" + "{%- else %}{{- '<|im_start|>' + m['role'] + '\\n' + m['content'] + '<|im_end|>\\n' }}{%- endif %}" + "{%- endfor %}" + "{%- if add_generation_prompt %}{{- '<|im_start|>assistant\\n\\n\\n\\n\\n' }}{%- endif %}"; + +// Gemma: assistant is "model", content trimmed +const std::string GEMMA = + "{{- bos_token }}{%- for m in messages %}" + "{%- set role = 'model' if m['role'] == 'assistant' else m['role'] %}" + "{{- '' + role + '\\n' + m['content'] | trim + '\\n' }}{%- endfor %}" + "{%- if add_generation_prompt %}{{- 'model\\n' }}{%- endif %}"; + +// DeepSeek-R1: an assistant turn keeps only what follows +const std::string DEEPSEEK = + "{{- bos_token }}{%- for m in messages %}" + "{%- if m['role'] == 'user' %}{{- '<\xEF\xBD\x9C" "User\xEF\xBD\x9C>' + m['content'] }}" + "{%- else %}{{- '<\xEF\xBD\x9C" "Assistant\xEF\xBD\x9C>' + m['content'].split('')[-1] + '<\xEF\xBD\x9C" "end\xE2\x96\x81of\xE2\x96\x81sentence\xEF\xBD\x9C>' }}" + "{%- endif %}{%- endfor %}" + "{%- if add_generation_prompt %}{{- '<\xEF\xBD\x9C" "Assistant\xEF\xBD\x9C>' }}{%- endif %}"; + +// Qwen2-VL: typed content, images before the text +const std::string QWEN2VL = + "{%- for m in messages %}{{- '<|im_start|>' + m['role'] + '\\n' }}" + "{%- if m['content'] is string %}{{- m['content'] }}{%- else %}{%- for c in m['content'] %}" + "{%- if c['type'] == 'image' %}{{- '<|vision_start|><|image_pad|><|vision_end|>' }}{%- else %}{{- c['text'] }}{%- endif %}" + "{%- endfor %}{%- endif %}{{- '<|im_end|>\\n' }}{%- endfor %}" + "{%- if add_generation_prompt %}{{- '<|im_start|>assistant\\n' }}{%- endif %}"; + +void test_template_shapes() { + std::printf("template shapes\n"); + const std::string q = "Is Paris the capital of France? Answer yes or no."; + + CHECK_EQ(continued(LLAMA3, chat(q, "The answer is:"), "<|begin_of_text|>"), + "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" + q + + "<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nThe answer is:"); + // the template trims, so the trailing space goes, as in transformers + CHECK_EQ(continued(LLAMA3, chat(q, "The answer is: "), "<|begin_of_text|>"), + "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" + q + + "<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nThe answer is:"); + + CHECK_EQ(continued(CHATML, chat(q, "The answer is:")), + "<|im_start|>user\n" + q + "<|im_end|>\n<|im_start|>assistant\nThe answer is:"); + // the template keeps the text as it is, trailing space included + CHECK_EQ(continued(CHATML, chat(q, "The answer is: ")), + "<|im_start|>user\n" + q + "<|im_end|>\n<|im_start|>assistant\nThe answer is: "); + + CHECK_EQ(continued(QWEN3, chat(q, "The answer is:")), + "<|im_start|>user\n" + q + "<|im_end|>\n<|im_start|>assistant\n\n\n\n\nThe answer is:"); + // lstrip('\n') in the template: matched without the leading newline + CHECK_EQ(continued(QWEN3, chat(q, "\nThe answer is:")), + "<|im_start|>user\n" + q + "<|im_end|>\n<|im_start|>assistant\n\n\n\n\nThe answer is:"); + + CHECK_EQ(continued(GEMMA, chat(q, " The answer is: "), ""), + "user\n" + q + "\nmodel\nThe answer is:"); + + CHECK_EQ(continued(DEEPSEEK, chat(q, "The answer is:"), "<\xEF\xBD\x9C" "begin\xE2\x96\x81of\xE2\x96\x81sentence\xEF\xBD\x9C>"), + "<\xEF\xBD\x9C" "begin\xE2\x96\x81of\xE2\x96\x81sentence\xEF\xBD\x9C><\xEF\xBD\x9C" "User\xEF\xBD\x9C>" + q + + "<\xEF\xBD\x9C" "Assistant\xEF\xBD\x9C>The answer is:"); + // an unclosed think block is left alone, so reasoning can be continued + CHECK_EQ(continued(DEEPSEEK, chat(q, "\nParis is"), ""), + "<\xEF\xBD\x9C" "User\xEF\xBD\x9C>" + q + "<\xEF\xBD\x9C" "Assistant\xEF\xBD\x9C>\nParis is"); + // the template moves the closed block out of the message: nothing to continue from + bool threw = false; + try { + continued(DEEPSEEK, chat(q, "Paris.The answer is:")); + } catch (const std::invalid_argument&) { + threw = true; + } + CHECK(threw); + + json vl = chat(q, json::array({{{"type", "image"}}, {{"type", "text"}, {"text", "This picture shows"}}})); + CHECK_EQ(continued(QWEN2VL, vl), + "<|im_start|>user\n" + q + "<|im_end|>\n<|im_start|>assistant\n<|vision_start|><|image_pad|><|vision_end|>This picture shows"); +} + +void test_cut() { + std::printf("cut\n"); + const json msg = {{"role", "assistant"}, {"content", "The answer is:"}}; + // the last occurrence: the user may have written the same words + CHECK_EQ(chat_continue_final_message("U: The answer is: A: The answer is:", msg), "U: The answer is: A: The answer is:"); + // leading whitespace in the message, kept by the template: cut after the text + CHECK_EQ(chat_continue_final_message("A: The answer is:", {{"content", " The answer is:"}}), "A: The answer is:"); + // trailing whitespace the template kept is kept; trailing whitespace it dropped is not looked for + CHECK_EQ(chat_continue_final_message("A: x \n", {{"content", "x \n"}}), "A: x \n"); + CHECK_EQ(chat_continue_final_message("A: x", {{"content", "x \n"}}), "A: x"); + // whitespace on both sides: transformers compares against the unstripped text, so the + // trailing space goes even when the template kept it; kept that way to cut where vLLM cuts + CHECK_EQ(chat_continue_final_message("A: x ", {{"content", " x "}}), "A: x"); + + CHECK(rejects("A: ", {{"content", ""}})); + CHECK(rejects("A: \n", {{"content", " \n"}})); + CHECK(rejects("A: ", {{"content", nullptr}})); + CHECK(rejects("A: ", {{"role", "assistant"}})); + CHECK(rejects("A: ", {{"content", json::array({{{"type", "image"}}})}})); + CHECK(rejects("A: something else", msg)); +} + +void test_prefill() { + std::printf("prefill\n"); + chat_format_spec r1 = chat_format_make(chat_format_kind::CONTENT_ONLY, true); + r1.reasoning_forced = true; + const std::string tail = "<\xEF\xBD\x9C" "Assistant\xEF\xBD\x9C>The answer is:"; + // a fresh R1 turn reasons first; a continued answer is not reasoning + CHECK_EQ(chat_format_prefill(r1, tail), ""); + CHECK_EQ(chat_format_prefill(r1, tail, true), ""); + // a continued message that opened reasoning is still in it + CHECK_EQ(chat_format_prefill(r1, "<\xEF\xBD\x9C" "Assistant\xEF\xBD\x9C>\nParis is", true), ""); + + chat_format_spec qwen = chat_format_make(chat_format_kind::HERMES_JSON, true); + CHECK_EQ(chat_format_prefill(qwen, "assistant\n\n\n\n\nThe answer is:", true), ""); + CHECK_EQ(chat_format_prefill(qwen, "assistant\n\n\n\n\n\nLet me", true), ""); +} + +void test_parse() { + std::printf("parse\n"); + const chat_format_spec qwen = chat_format_make(chat_format_kind::HERMES_JSON, true); + chat_parse_options plain; + chat_parse_options keep; + keep.keep_whitespace = true; + + CHECK_EQ(chat_parse(qwen, " yes", false, plain).content, "yes"); + CHECK_EQ(chat_parse(qwen, " yes", false, keep).content, " yes"); + // the whole text, once final: prefix + content is what the model wrote + CHECK_EQ(chat_parse(qwen, " yes \n", false, keep).content, " yes \n"); + CHECK_EQ(chat_parse(qwen, " \n", false, keep).content, " \n"); + // while generating, trailing whitespace waits for what follows it + CHECK_EQ(chat_parse(qwen, " yes \n", true, keep).content, " yes"); + CHECK_EQ(chat_parse(qwen, " \n", true, keep).content, ""); + // continuing inside reasoning: the reasoning text is exact, the answer after it is laid out as usual + chat_msg in_reasoning = chat_parse(qwen, " the capital\n\nParis.\n", false, keep); + CHECK_EQ(in_reasoning.reasoning, " the capital"); + CHECK_EQ(in_reasoning.content, "Paris."); + // a newline before a tool call is layout, not an answer + CHECK_EQ(chat_parse(qwen, "\n{\"name\": \"f\", \"arguments\": {}}", false, keep).content, ""); + + // a model that reasons first reads the continued message as reasoning, and closes it + chat_parse_options closes = keep; + closes.reasoning_may_close = true; + chat_msg closed = chat_parse(qwen, " Paris.\n\n\nThe capital of France is Paris.", false, closes); + CHECK_EQ(closed.reasoning, " Paris.\n"); + CHECK_EQ(closed.content, "The capital of France is Paris."); + // no marker: the continuation is the answer, as written + CHECK_EQ(chat_parse(qwen, " Yes", false, closes).content, " Yes"); + CHECK_EQ(chat_parse(qwen, " Yes", false, closes).reasoning, ""); + // while neither marker has shown, nothing is released + CHECK_EQ(chat_parse(qwen, " Paris.\nhmParis.", false, closes); + CHECK_EQ(opened.reasoning, "hm"); + CHECK_EQ(opened.content, "Paris."); + { + chat_stream stream(qwen, "", closes); + std::string r, c; + for (const char* piece : {" Paris", ".\n\n\nThe", " capital."}) { + for (const chat_msg_delta& d : stream.update(piece, false)) { r += d.reasoning; c += d.content; } + } + for (const chat_msg_delta& d : stream.update("", true)) { r += d.reasoning; c += d.content; } + CHECK_EQ(r, " Paris.\n"); + CHECK_EQ(c, "The capital."); + } + + // the stream sends the same text, and never takes back what it sent + for (const std::vector& pieces : std::vector>{ + {" ", "yes", ",", " it", " is.", "\n"}, {"\n", "\n"}}) { + chat_stream stream(qwen, "", keep); + std::string streamed, whole; + for (const std::string& piece : pieces) { + whole += piece; + for (const chat_msg_delta& d : stream.update(piece, false)) { + streamed += d.content; + } + } + for (const chat_msg_delta& d : stream.update("", true)) { + streamed += d.content; + } + CHECK_EQ(streamed, whole); + CHECK_EQ(streamed, chat_parse(qwen, whole, false, keep).content); + } +} + +} // namespace + +int main() { + test_template_shapes(); + test_cut(); + test_prefill(); + test_parse(); + if (failures) { + std::printf("%d check(s) failed\n", failures); + return 1; + } + std::printf("all checks passed\n"); + return 0; +} diff --git a/src/test/openai_api_test/README.md b/src/test/openai_api_test/README.md index 8414f5f12..d4066706b 100644 --- a/src/test/openai_api_test/README.md +++ b/src/test/openai_api_test/README.md @@ -21,7 +21,8 @@ python3 run_all.py qwen3:4b -k schema # only cases whose name contains "schema" |---|---| | `--endpoint URL` | Server base URL, default `http://127.0.0.1:52625` | | `--timeout T` | Per-request timeout in seconds, default 300 | -| `--always-thinks` | The model reasons even with `think: false` (`qwen3-tk`, `lfm2.5-tk`, `deepseek-r1`, `gpt-oss`). Don't require reasoning to disappear. | +| `--always-thinks` | The model reasons even with `think: false` (`qwen3-tk`, `lfm2.5-tk`, `deepseek-r1`, `deepseek-r1-0528`, `nanbeige4.1`, `gpt-oss`). Don't require reasoning to disappear, and give every answer room for the reasoning before it. | +| `--reasoning-budget N` | With `--always-thinks`, tokens added to each request that expects an answer (`max_tokens` ≥ 32), default 1024 | | `-k NAME` | Run only matching cases | | `-v` | Print tracebacks for failures | @@ -35,8 +36,9 @@ Each case prints `PASS`, `FAIL` or `SKIP`. The exit code is non-zero if anything | `test_reasoning.py` | `think: true` → `reasoning_content`, `think: false` / `reasoning_effort: "none"` → none, reasoning streamed before content, switch is per request, reasoning in history is accepted | | `test_tool_calls.py` | `auto` / `required` / forced function / `none`, `parallel_tool_calls: false`, ids, arguments are JSON matching the tool schema (typed: integer, number, boolean, enum, arrays), `strict: false`, streamed call deltas, a tool round trip | | `test_response_format.py` | `json_object`, `json_schema` (flat, enum, Pydantic `$defs`/`$ref`, `const`/`anyOf`/bounds/`pattern`), streamed, with and without thinking, schema vs. `auto` tools, `text` | -| `test_logprobs.py` | `logprobs` / `top_logprobs` shape in both modes, greedy token heads its `top_logprobs`, entries spell the content and nothing else (no reasoning, no tool call markup), stream and non-stream report the same tokens, malformed values get 400 | +| `test_logprobs.py` | `logprobs` / `top_logprobs` shape in both modes, greedy token heads its `top_logprobs`, entries spell the content and nothing else (no reasoning, no tool call markup), stream and non-stream report the same tokens, `max_tokens` N writes at most N tokens and `completion_tokens` counts each, malformed values get 400 | | `test_sampling.py` | The same `seed` samples the same text (with an unseeded request in between, streamed or not), `repeat_last_n: 0` turns penalties off, `-1` covers the whole reply, malformed values get 400 | +| `test_continue_final_message.py` | `continue_final_message` continues a trailing assistant message instead of answering it: content is only the new text, whitespace as written (a model that reasons first may read the prefix as reasoning, and close it), `logprobs` cover exactly the continued tokens (the first entry scores the next token), stream matches non-stream, a cache-resumed continuation matches a cold one, a normal turn can follow, malformed combinations get 400 | | `test_errors.py` | Every malformed `tool_choice` / `response_format` gets HTTP 400 with an OpenAI error body, streaming too, and the server keeps serving | Some cases skip by design: @@ -44,6 +46,7 @@ Some cases skip by design: - **Tool cases** skip on models whose chat format has no tool call syntax (the server answers `has no tool call syntax`), and on models too small to be shown tools (`does not support tool calling`). - **Thinking-on cases** skip on models that never reason. - **Multi-turn cases** skip on single-turn (flash) models. +- **`continue_final_message` cases** skip on models that cannot continue a message (single-turn models, `gpt-oss`); the server answers `continue_final_message is not supported`. - **Stream/non-stream parity** compares greedy outputs. Greedy decoding on the NPU isn't always bit-reproducible between identical requests (mostly over long outputs), so the check retries and only fails if the non-streaming output reproduced itself while the stream never matched it. Otherwise it skips. - **Seed and penalty-window comparisons** in `test_sampling.py` retry the same way, and skip when two identical greedy requests report different logprobs, since then no output of the model's is repeatable. diff --git a/src/test/openai_api_test/fixtures.py b/src/test/openai_api_test/fixtures.py index a24bb713f..f74ea1ae6 100644 --- a/src/test/openai_api_test/fixtures.py +++ b/src/test/openai_api_test/fixtures.py @@ -10,6 +10,11 @@ def user(text: str) -> list[dict[str, Any]]: return [{"role": "user", "content": text}] +def continued(text: str, assistant: Any) -> list[dict[str, Any]]: + """A question and the start of its answer, for continue_final_message.""" + return user(text) + [{"role": "assistant", "content": assistant}] + + def function_choice(name: str) -> dict[str, Any]: """tool_choice forcing one function.""" return {"type": "function", "function": {"name": name}} diff --git a/src/test/openai_api_test/flm_api.py b/src/test/openai_api_test/flm_api.py index bb7906ed1..70d59037b 100644 --- a/src/test/openai_api_test/flm_api.py +++ b/src/test/openai_api_test/flm_api.py @@ -38,6 +38,10 @@ DEFAULT_ENDPOINT = "http://127.0.0.1:52625" DEFAULT_TIMEOUT = 300.0 +# room a model that always reasons gets on top of a request's answer budget (--reasoning-budget) +DEFAULT_REASONING_BUDGET = 1024 +# smaller budgets are about where generation stops, not about getting an answer +ANSWER_BUDGET = 32 # Raw markup any supported format may write; none of it may reach the client. MARKERS = ( @@ -85,16 +89,24 @@ class Stream: class Api: """Talks to one FastFlowLM server about one model.""" - def __init__(self, endpoint: str, model: str, timeout: float, always_thinks: bool = False) -> None: + def __init__(self, endpoint: str, model: str, timeout: float, always_thinks: bool = False, + reasoning_budget: int = DEFAULT_REASONING_BUDGET) -> None: self.url = endpoint.rstrip("/") + "/v1/chat/completions" self.model = model self.timeout = timeout # the model reasons whatever think says (qwen3-tk, deepseek-r1, gpt-oss, ...) self.always_thinks = always_thinks + self.reasoning_budget = reasoning_budget def body(self, messages: list[dict[str, Any]], *, think: bool = False, max_tokens: int = 256, **fields: Any) -> dict[str, Any]: - """Build a request body with sampling pinned (see the module note).""" + """Build a request body with sampling pinned (see the module note). + + A model that always reasons spends tokens on that before answering, so a + budget meant for an answer gets the reasoning budget on top. + """ + if self.always_thinks and max_tokens >= ANSWER_BUDGET: + max_tokens += self.reasoning_budget body: dict[str, Any] = { "model": self.model, "messages": messages, @@ -185,6 +197,8 @@ def raise_for_capability(message: str) -> None: raise Skip("the model only supports single-turn requests") if "logprobs is not supported" in message: raise Skip("the model cannot report logprobs") + if "continue_final_message is not supported" in message: + raise Skip("the model cannot continue a message") # ---------------------------------------------------------------- stream folding @@ -546,13 +560,17 @@ def parse_args(description: str) -> argparse.Namespace: parser.add_argument("--always-thinks", action="store_true", help="the model cannot turn thinking off (e.g. qwen3-tk, deepseek-r1, gpt-oss): " "do not require think=false to drop reasoning_content") + parser.add_argument("--reasoning-budget", type=int, default=DEFAULT_REASONING_BUDGET, + help="with --always-thinks, tokens added to every answer budget for the reasoning " + "that comes first (default: %(default)s)") parser.add_argument("-k", dest="only", help="run only cases whose name contains this text") parser.add_argument("-v", "--verbose", action="store_true", help="print tracebacks of failures") return parser.parse_args() def make_api(args: argparse.Namespace) -> Api: - return Api(args.endpoint, args.model, args.timeout, always_thinks=args.always_thinks) + return Api(args.endpoint, args.model, args.timeout, always_thinks=args.always_thinks, + reasoning_budget=args.reasoning_budget) def summarize(tally: Tally) -> int: diff --git a/src/test/openai_api_test/run_all.py b/src/test/openai_api_test/run_all.py index 3f31d245d..58df824f4 100644 --- a/src/test/openai_api_test/run_all.py +++ b/src/test/openai_api_test/run_all.py @@ -13,6 +13,7 @@ import sys import flm_api +import test_continue_final_message import test_errors import test_logprobs import test_reasoning @@ -22,7 +23,7 @@ import test_tool_calls MODULES = [test_response_shape, test_reasoning, test_tool_calls, test_response_format, test_logprobs, test_sampling, - test_errors] + test_continue_final_message, test_errors] def main() -> None: diff --git a/src/test/openai_api_test/test_continue_final_message.py b/src/test/openai_api_test/test_continue_final_message.py new file mode 100644 index 000000000..6270ec947 --- /dev/null +++ b/src/test/openai_api_test/test_continue_final_message.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""continue_final_message on /v1/chat/completions, streamed and not. + +Checks what a client scoring or completing a partial answer relies on (the +field is vLLM's; Hugging Face's apply_chat_template has the same flag): + +* a trailing assistant message is continued, not answered: the content is + only the new text, and the prefix is not repeated; +* the content keeps the whitespace the continuation starts with, so + ``prefix + content`` reads as the model wrote it; +* ``logprobs`` covers exactly the continued tokens; the first entry is the + next-token distribution right after the prefix; +* streamed and non-streamed content are the same; +* a prompt resumed from the cache continues the same way as a cold one, and + a normal turn can follow a continued one; +* without the flag (false or null) a trailing assistant message is answered + as before; +* malformed combinations get HTTP 400. + +Models that cannot continue a message (single-turn models, gpt-oss) answer +400 "continue_final_message is not supported", which turns the cases into +skips. + +Usage:: + + python3 test_continue_final_message.py [--endpoint URL] [-k NAME] [-v] +""" + +from __future__ import annotations + +from typing import Any + +import flm_api +from flm_api import Api, Skip, case, expect +from fixtures import WEATHER_TOOLS, continued, user +from test_logprobs import (check_entry, check_greedy, expect_400, expect_covers, joined, logprobs_body, + nonstream_entries, stream_entries) + +QUESTION = "What is the capital of France? Answer in one sentence." +PREFIX = "The capital of France is" +YES_NO = "Is Paris the capital of France? Answer yes or no." + + +def continue_body(api: Api, messages: list[dict[str, Any]], **fields: Any) -> dict[str, Any]: + fields.setdefault("max_tokens", 16) + return api.body(messages, continue_final_message=True, **fields) + + +def expect_continuation(api: Api, msg: dict[str, Any], prefix: str, where: str) -> str: + """The reply goes on from the prefix rather than starting over; returns the text that does. + + That is the content, or on a model that reasons first (--always-thinks) the + reasoning it may read the prefix as: it then closes that reasoning and answers. + """ + content = msg.get("content") or "" + reasoning = msg.get("reasoning_content") or "" + expect(not reasoning or api.always_thinks, f"{where}: a continued answer was reported as reasoning: {msg!r}") + flm_api.assert_no_markers(content, f"{where} content") + flm_api.assert_no_markers(reasoning, f"{where} reasoning") + text = reasoning or content + expect(text.strip() != "", f"{where}: no continuation: {msg!r}") + expect(not text.lstrip().startswith(prefix.strip()), f"{where}: the reply repeats the prefix: {text!r}") + return text + + +def folded_message(folded: flm_api.Folded) -> dict[str, Any]: + return {"content": folded.content, "reasoning_content": folded.reasoning} + + +@case +def continues_not_repeats(api: Api) -> None: + """The partial answer is continued: the content is the rest of it.""" + choice = api.ok(continue_body(api, continued(QUESTION, PREFIX))) + msg = flm_api.check_message_contract(choice) + text = expect_continuation(api, msg, PREFIX, "non-stream") + expect("Paris" in text, f"greedy continuation of {PREFIX!r} lacks Paris: {text!r}") + + +@case +def stream_continues(api: Api) -> None: + """The stream sends the same continuation, starting with its own whitespace.""" + folded = flm_api.assert_parity(api, continue_body(api, continued(QUESTION, PREFIX)), + lambda m: (m["content"], m["reasoning_content"])) + expect_continuation(api, folded_message(folded), PREFIX, "stream") + + +@case +def logprobs_cover_continuation(api: Api) -> None: + """logprobs has the continued tokens only, and they spell the content exactly, leading space included.""" + body = logprobs_body(api, continued(QUESTION, PREFIX), continue_final_message=True, max_tokens=12) + msg, entries = nonstream_entries(api, body) + expect_continuation(api, msg, PREFIX, "non-stream") + if msg.get("reasoning_content"): + # the model read the prefix as reasoning: only its answer has logprobs + expect_covers(entries, msg.get("content"), "non-stream after reasoning") + return + content = msg["content"] + expect(len(entries) > 0, "no logprobs entries for the continuation") + for i, entry in enumerate(entries): + check_entry(entry, 3, f"content[{i}]") + check_greedy(entry, f"content[{i}]") + # rstrip only: whitespace the continuation starts with is part of the answer + expect(joined(entries).rstrip() == content.rstrip(), + f"the logprobs tokens spell {joined(entries)!r}, the content is {content!r}") + folded, streamed = stream_entries(api, body) + expect(joined(streamed).rstrip() == folded.content.rstrip(), + f"stream: the logprobs tokens spell {joined(streamed)!r}, the content is {folded.content!r}") + + +@case +def score_next_token(api: Api) -> None: + """max_tokens 1 after "The answer is:" gives the next-token distribution, with yes or no in it.""" + body = logprobs_body(api, continued(YES_NO, "The answer is:"), continue_final_message=True, + max_tokens=1, top_logprobs=5) + choice = api.ok(body) + msg = flm_api.check_message_contract(choice) + entries = choice["logprobs"]["content"] + if not entries and msg.get("reasoning_content"): + raise Skip("the model reasons before answering, so its first token is not the answer") + expect(len(entries) == 1, f"max_tokens 1 should give one entry, got {len(entries)}: {entries!r}") + check_entry(entries[0], 5, "content[0]") + check_greedy(entries[0], "content[0]") + tops = [alt["token"].strip().lower() for alt in entries[0]["top_logprobs"]] + expect("yes" in tops or "no" in tops, f"neither yes nor no among the likeliest next tokens: {tops!r}") + expect(choice.get("finish_reason") in ("length", "stop"), f"finish_reason is {choice.get('finish_reason')!r}") + expect(msg.get("content") == entries[0]["token"], + f"content {msg.get('content')!r} is not the scored token {entries[0]['token']!r}") + + +@case +def typed_content(api: Api) -> None: + """The final message may be a list of text parts, as OpenAI clients send it.""" + messages = continued(QUESTION, [{"type": "text", "text": PREFIX}]) + choice = api.ok(continue_body(api, messages)) + expect_continuation(api, flm_api.check_message_contract(choice), PREFIX, "typed content") + + +def continue_once(api: Api, messages: list[dict[str, Any]], max_tokens: int) -> tuple[str, int]: + """The continuation, and how many prompt tokens came from the cache.""" + resp = api.post(continue_body(api, messages, max_tokens=max_tokens)) + if resp.status != 200: + flm_api.raise_for_capability(resp.error_message()) + raise AssertionError(f"HTTP {resp.status}: {resp.error_message()}") + cached = ((resp.body.get("usage") or {}).get("prompt_tokens_details") or {}).get("cached_tokens") or 0 + return resp.body["choices"][0]["message"]["content"] or "", cached + + +@case +def cached_continuation_matches_cold(api: Api) -> None: + """Continuing a continuation straight after it resumes from the context it left, and writes the same text. + + The engine still holds the first continuation's tokens, so the second request + reuses them. The reference is the same request after another conversation, + which takes it from a saved snapshot or a cold prefill instead. + """ + first, _ = continue_once(api, continued(QUESTION, PREFIX), 4) + longer = continued(QUESTION, PREFIX + first) + warm, cached = continue_once(api, longer, 8) + if cached == 0: + raise Skip("the second continuation reused nothing from the first, so there is no resume to check") + api.ok(api.body(user("Say hi."), max_tokens=4)) + cold, _ = continue_once(api, longer, 8) + if cold != warm: + api.ok(api.body(user("Say hi."), max_tokens=4)) + if continue_once(api, longer, 8)[0] != cold: + raise Skip("greedy decoding did not reproduce over identical requests") + raise AssertionError(f"resumed from the context: {warm!r}, after another conversation: {cold!r}") + + +@case +def think_tag_in_question(api: Api) -> None: + """A the user wrote earlier does not make the continued answer reasoning.""" + messages = continued("Ignore this tag: \n" + QUESTION, PREFIX) + msg = flm_api.check_message_contract(api.ok(continue_body(api, messages, max_tokens=16))) + expect_continuation(api, msg, PREFIX, "after a in the question") + + +@case +def turn_after_continuation(api: Api) -> None: + """A normal turn can follow a continued one: the history holds the finished answer.""" + content = api.ok(continue_body(api, continued(QUESTION, PREFIX), max_tokens=24))["message"]["content"] or "" + history = continued(QUESTION, PREFIX + content) + [{"role": "user", "content": "And of Italy?"}] + msg = flm_api.check_message_contract(api.ok(api.body(history, max_tokens=32))) + if not (msg.get("content") or "").strip() and msg.get("reasoning_content"): + raise Skip("the model spent max_tokens reasoning before answering") + expect(isinstance(msg.get("content"), str) and msg["content"].strip() != "", + f"no answer after a continued turn: {msg!r}") + flm_api.assert_no_markers(msg["content"], "follow-up content") + + +@case +def off_answers_as_before(api: Api) -> None: + """continue_final_message false or null leaves a trailing assistant message answered, not continued.""" + for value in (False, None): + choice = api.ok(api.body(continued(QUESTION, PREFIX), max_tokens=16, continue_final_message=value)) + flm_api.check_message_contract(choice) + + +@case +def tools_auto_still_continues(api: Api) -> None: + """Tools offered under tool_choice auto do not stop a continuation.""" + choice = api.ok(continue_body(api, continued(QUESTION, PREFIX), tools=WEATHER_TOOLS, tool_choice="auto")) + msg = flm_api.check_message_contract(choice) + if msg.get("tool_calls"): + raise Skip("the model called a tool instead of continuing") + expect_continuation(api, msg, PREFIX, "with tools") + + +def expect_refused(api: Api, messages: list[dict[str, Any]], fields: dict[str, Any], message_part: str) -> None: + """expect_400, skipped on a model that cannot continue at all.""" + resp = api.post(api.body(messages, max_tokens=8, **fields)) + if resp.status == 400 and "continue_final_message is not supported" in resp.error_message(): + raise Skip("the model cannot continue a message") + expect_400(api, fields, message_part, messages=messages) + + +@case +def bad_requests(api: Api) -> None: + """What cannot be continued is refused with 400, streaming too.""" + on = {"continue_final_message": True} + expect_400(api, {"continue_final_message": "yes"}, "must be a boolean", messages=continued(QUESTION, PREFIX)) + expect_400(api, on, "role \"assistant\"", messages=user(QUESTION)) + expect_400(api, {**on, "add_generation_prompt": True}, "Cannot set both", messages=continued(QUESTION, PREFIX)) + call = {"id": "call_0", "type": "function", "function": {"name": "get_weather", "arguments": "{\"city\": \"Paris\"}"}} + expect_400(api, on, "tool_calls", + messages=user(QUESTION) + [{"role": "assistant", "content": "", "tool_calls": [call]}]) + expect_400(api, {**on, "response_format": {"type": "json_object"}}, "response_format", + messages=continued(QUESTION, PREFIX)) + expect_400(api, {**on, "tools": WEATHER_TOOLS, "tool_choice": "required"}, "tool_choice", + messages=continued(QUESTION, PREFIX)) + expect_refused(api, continued(QUESTION, " "), on, "requires text") + + +@case +def vllm_style_request(api: Api) -> None: + """vLLM clients send add_generation_prompt false alongside the flag.""" + choice = api.ok(continue_body(api, continued(QUESTION, PREFIX), add_generation_prompt=False)) + expect_continuation(api, flm_api.check_message_contract(choice), PREFIX, "vLLM style") + + +CASES = flm_api.collect(globals()) + +if __name__ == "__main__": + flm_api.main(CASES, __doc__.splitlines()[0]) diff --git a/src/test/openai_api_test/test_logprobs.py b/src/test/openai_api_test/test_logprobs.py index 34701a830..573621593 100644 --- a/src/test/openai_api_test/test_logprobs.py +++ b/src/test/openai_api_test/test_logprobs.py @@ -163,6 +163,24 @@ def stream_matches_nonstream(api: Api) -> None: f" non-stream: {plain[0]!r}") +@case +def max_tokens_is_exact(api: Api) -> None: + """max_tokens N writes N tokens at most, and usage.completion_tokens counts every one.""" + for n in (1, 2, 5): + resp = api.post(logprobs_body(api, user("Count from 1 to 20, separated by spaces."), max_tokens=n)) + if resp.status != 200: + flm_api.raise_for_capability(resp.error_message()) + raise AssertionError(f"HTTP {resp.status}: {resp.error_message()}") + choice = resp.body["choices"][0] + entries = choice["logprobs"]["content"] + completion = resp.body["usage"]["completion_tokens"] + # trailing whitespace waits for text that never comes, so a token can be missing, never extra + expect(len(entries) <= n, f"max_tokens {n} gave {len(entries)} tokens: {joined(entries)!r}") + expect(completion <= n, f"max_tokens {n} reported {completion} completion tokens") + if choice.get("finish_reason") == "length" and not choice["message"].get("reasoning_content"): + expect(completion == n, f"max_tokens {n} stopped for length after {completion} completion tokens") + + @case def top_logprobs_default(api: Api) -> None: """logprobs without top_logprobs gives every entry an empty top_logprobs.""" @@ -213,9 +231,10 @@ def tool_call_has_no_logprobs(api: Api) -> None: expect_covers(streamed, folded.content, "stream tool call") -def expect_400(api: Api, fields: dict[str, Any], message_part: str) -> None: +def expect_400(api: Api, fields: dict[str, Any], message_part: str, + messages: list[dict[str, Any]] | None = None) -> None: """Non-streaming and streaming both refuse, naming the problem.""" - body = api.body(user("hi"), max_tokens=8, **fields) + body = api.body(messages if messages is not None else user("hi"), max_tokens=8, **fields) resp = api.post(body) expect(resp.status == 400, f"expected HTTP 400, got {resp.status}: {resp.body}") err = resp.body.get("error") From 8f1175a30714c348517371ab51dce5a2305f7212 Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Wed, 28 Oct 2026 14:39:16 -0700 Subject: [PATCH 3/5] docs: continue_final_message and scoring the next token Co-Authored-By: Claude --- docs/docs/instructions/server/openapi.md | 37 +++++++++++++ .../instructions/server/request_parameters.md | 52 ++++++++++++++++++- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/docs/docs/instructions/server/openapi.md b/docs/docs/instructions/server/openapi.md index 89c5cb06f..8b9ae12e8 100644 --- a/docs/docs/instructions/server/openapi.md +++ b/docs/docs/instructions/server/openapi.md @@ -17,6 +17,8 @@ parent: Local Server (Server Mode) - [Example: Use a File as the Prompt](#-example-use-a-file-as-the-prompt) - [Example: Batch Requests (Multiple Prompts)](#-example-batch-requests-multiple-prompts) - [Example: Use Temperature, Top-p, and Presence Penalty](#-example-use-temperature-top-p-and-presence-penalty) + - [Example: Token Log Probabilities](#-example-token-log-probabilities) + - [Example: Score the Next Token After a Partial Answer](#-example-score-the-next-token-after-a-partial-answer) - [Example: Multi-Image Input](#-example-multi-image-input) - [Example: Multi-Modal Input](#️-example-multi-modal-input) - [Example: Flash Models (Single-Turn, Pinned System Prompt)](#-example-flash-models-single-turn-pinned-system-prompt) @@ -358,6 +360,41 @@ Streaming works the same way: each content chunk carries `choices[0].logprobs.co --- +## 🎯 Example: Score the Next Token After a Partial Answer + +> Available from FLM v1.0.11. + +To see how likely each answer is, end `messages` with the start of the assistant's reply and set `continue_final_message` (an `extra_body` field). The model continues that text instead of starting a new reply, and the first log probability entry is its distribution for the very next token: + +```python +from openai import OpenAI + +client = OpenAI(base_url="http://127.0.0.1:52625/v1", api_key="flm") + +response = client.chat.completions.create( + model="llama3.2:1b", + messages=[ + {"role": "user", "content": "Is Paris the capital of France? Answer yes or no."}, + {"role": "assistant", "content": "The answer is:"}, + ], + max_tokens=1, + temperature=0, + logprobs=True, + top_logprobs=5, + extra_body={"continue_final_message": True}, +) + +for alt in response.choices[0].logprobs.content[0].top_logprobs: + print(f"{alt.token!r:10} {alt.logprob:8.4f}") +``` + +- The prompt goes through the model's own chat template, so the score is for the context the model is used to. +- `content` is only the continuation, without your text, and keeps its whitespace as the model wrote it (`' Yes'`, not `'Yes'`). +- Use a model with thinking off (`think: false`, or a non-reasoning model). A model that always reasons first reads your text as its reasoning, so its first token is not the answer. +- Continuing works on every chat model except single-turn (flash) models and `gpt-oss`. What else is refused, and how the prompt is cut: [Continuing an Assistant Message](https://fastflowlm.com/docs/instructions/server/request_parameters/#-continuing-an-assistant-message-continue_final_message). + +--- + ## 📷 Example: Multi-Image Input Send multiple images together with a text prompt. diff --git a/docs/docs/instructions/server/request_parameters.md b/docs/docs/instructions/server/request_parameters.md index 28bbe8a55..fd5f66174 100644 --- a/docs/docs/instructions/server/request_parameters.md +++ b/docs/docs/instructions/server/request_parameters.md @@ -20,6 +20,7 @@ This page says what FLM does with the fields of a `/v1/chat/completions` or `/v1 - [The Penalty Window](#-the-penalty-window) - [`seed` and Repeatable Output](#-seed-and-repeatable-output) - [Reasoning in `reasoning_content`](#-reasoning-in-reasoning_content) +- [Continuing an Assistant Message (`continue_final_message`)](#-continuing-an-assistant-message-continue_final_message) - [Fields FLM Doesn't Know](#-fields-flm-doesnt-know) - [Long Prompts: Prefill Chunking](#-long-prompts-prefill-chunking) @@ -179,6 +180,54 @@ Turning thinking on and off, streaming examples and the full message format: [Re --- +## ⏩ Continuing an Assistant Message (`continue_final_message`) + +> Available from FLM v1.0.11, on `/v1/chat/completions`. + +By default, a request whose last message is from the assistant gets a **new** reply: the chat template closes that message and opens a fresh assistant turn. Set `continue_final_message: true` and the model **continues** the last message instead, from exactly where its text ends. The field is vLLM's, and Hugging Face's `apply_chat_template` has the same flag. + +```python +response = client.chat.completions.create( + model="llama3.2:1b", + messages=[ + {"role": "user", "content": "Is Paris the capital of France? Answer yes or no."}, + {"role": "assistant", "content": "The answer is:"}, + ], + max_tokens=1, + logprobs=True, + top_logprobs=5, + extra_body={"continue_final_message": True}, +) +first = response.choices[0].logprobs.content[0] +print(repr(first.token), first.logprob) # ' Yes' -0.098 on llama3.2:1b +``` + +Use it to score what the model would say at an exact point, through the model's own chat template: classification, multiple-choice evaluation, or checking how confident the model is in an answer. With `logprobs`, the first entry is the model's distribution for the token right after your text (see [the example](https://fastflowlm.com/docs/instructions/server/openapi/#-example-score-the-next-token-after-a-partial-answer)). + +What you get back: + +- **`content` is only the continuation.** Your text isn't repeated, so `prefix + content` is the whole message. Its whitespace is kept as the model wrote it (`' Yes'`, not `'Yes'`), even when the continuation is only whitespace. In a stream, trailing whitespace arrives once the text after it, or the end of the reply, does. +- **`logprobs` cover only the continued tokens.** Your text is part of the prompt, so it has no entries. +- **`usage.prompt_tokens` includes your text**, and `completion_tokens` counts the continuation. +- **The prompt is cut the way Hugging Face cuts it.** FLM renders the chat with no generation prompt, finds the last occurrence of the final message's text, and stops right after it. Templates that trim message content also drop trailing whitespace from your text, so `"The answer is: "` is read as `"The answer is:"`. A template that keeps the space keeps it, and the model then sees a separate space token, which changes what it predicts next. +- **Models that reason first may read your text as reasoning.** `deepseek-r1`, `deepseek-r1-0528`, `qwen3-tk`, `lfm2.5-tk`, `nanbeige4.1`, and any model with `think` on, start every reply by reasoning. They can treat the message you continue as the start of that reasoning: they go on, close it with ``, then answer. FLM then returns the continuation in `reasoning_content` and the answer after it in `content`, as vLLM's reasoning parsers do. A stream from such a model sends nothing until it either writes that tag or ends. To score the next token, use a model with `think` off: a reasoning model's first token is reasoning, not the answer. + +Rejected with HTTP 400 (`invalid_request_error`), for streaming requests too: + +| Request | Why | +|---|---| +| `continue_final_message` that isn't a boolean | | +| The last message isn't from the assistant | There is nothing to continue. | +| `add_generation_prompt: true` together with it | They ask for opposite things; vLLM refuses this too. Sending `add_generation_prompt: false` is fine. | +| The last message has `tool_calls`, images or audio, or no text | Only text can be continued. | +| `response_format` (JSON), or `tool_choice` `"required"` / a named function | The grammar for these starts at the beginning of a reply. `tool_choice` `"auto"` and `"none"` work. | +| The chat template rewrites the last message, for example by moving a closed `…` block out of it | Its text no longer appears as you sent it, so there is no place to continue from. An unclosed `` is kept, and the model goes on reasoning. | +| The model can't continue a message: single-turn (flash) models and `gpt-oss` | The message is `continue_final_message is not supported by `. | + +`continue_final_message: false` or `null` keeps the default behaviour. FLM does not read `add_generation_prompt` otherwise. + +--- + ## 📨 Fields FLM Doesn't Know **FLM ignores fields it doesn't read. It doesn't reject them.** OpenAI SDKs, LangChain and agent frameworks send fields that FLM has no use for, and they still work against FLM. The downside is that a misspelt field (`temprature`) is dropped without any error or log line, so check your spelling if a setting seems to have no effect. @@ -191,7 +240,7 @@ Turning thinking on and off, streaming examples and the full message format: [Re | An `options` object (Ollama style) on a `/v1` endpoint | Ignored. Put sampling fields at the top level of the request. | | A sampling field outside its range | Ignored with a warning in the server log. The previous value stays in effect ([see above](#️-sampling-fields)). | | A sampling field of the wrong JSON type, such as `"temperature": "0.7"` | The request fails with an error. Send numbers as JSON numbers. | -| A malformed `messages`, `tool_choice`, `parallel_tool_calls`, `response_format`, `logprobs`, `top_logprobs`, `seed` or `repeat_last_n` | Rejected with HTTP 400 and `"type": "invalid_request_error"` before anything is generated, for streaming requests too. | +| A malformed `messages`, `tool_choice`, `parallel_tool_calls`, `response_format`, `logprobs`, `top_logprobs`, `seed`, `repeat_last_n` or `continue_final_message` | Rejected with HTTP 400 and `"type": "invalid_request_error"` before anything is generated, for streaming requests too. | FLM's own extensions to the OpenAI request, all top-level fields (`extra_body` in the OpenAI SDK): @@ -201,6 +250,7 @@ FLM's own extensions to the OpenAI request, all top-level fields (`extra_body` i | `top_k`, `repetition_penalty` | [Sampling Fields](#️-sampling-fields) | | `repeat_last_n` | [The Penalty Window](#-the-penalty-window) | | `backend` | [Choose an Execution Backend](https://fastflowlm.com/docs/instructions/cli/#-choose-an-execution-backend) | +| `continue_final_message` (vLLM's name) | [Continuing an Assistant Message](#-continuing-an-assistant-message-continue_final_message) | --- From 139252ffeeea8c0093e8c58b0d2ff31d1f62652c Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Wed, 28 Oct 2026 14:59:13 -0700 Subject: [PATCH 4/5] fix: continue the reply's opening, cached-prompt crash, loop accounting - continue_final_message keeps transformers' cut wherever the template opens a reply the way it renders a finished message. Where it does not (Qwen3.5 drops the empty think block, Gemma4 12B with thinking off its empty thought channel, LFM2.5-Thinking its opening ), the cut read as a finished turn and Qwen3.5 ended at once with empty content; the text now follows the reply's opening. chat_continue_prompt holds the rule and the turn the output parser reads, and is unit tested - _shared_insert (and Qwen3.5-Omni's own insert) read the prompt in again when the engine already holds all of it: the same request twice left nothing to prefill, and the empty prefill crashed hy-mt2-flash once the unrun token no longer sat in the history - parser: whitespace before later text is settled in partial parses too, so a stream with a tool call or think block after the continued text no longer drops a delta; a closed whitespace-only think block is layout - Gemma4e checkpoint history keeps an eos the engine ran; the forced think paths of Qwen3, Qwen3-TK and Gemma4 12B filter and stop on an eos first sample, and count every forced token in the context Co-Authored-By: Claude --- .../instructions/server/request_parameters.md | 2 +- src/common/AutoModel/automodel.cpp | 41 +++++------ src/common/AutoModel/modeling_gemma4_12b.cpp | 17 ++++- src/common/AutoModel/modeling_gemma4e.cpp | 11 ++- src/common/AutoModel/modeling_qwen3.cpp | 31 ++++++-- .../AutoModel/modeling_qwen3_5_omni.cpp | 3 +- src/common/chat/chat_continue.cpp | 42 ++++++++++- src/common/chat/chat_parser.cpp | 12 ++- src/include/chat/chat_continue.hpp | 30 +++++++- src/test/chat_continue/test_chat_continue.cpp | 73 +++++++++++++++++++ .../test_continue_final_message.py | 2 +- 11 files changed, 216 insertions(+), 48 deletions(-) diff --git a/docs/docs/instructions/server/request_parameters.md b/docs/docs/instructions/server/request_parameters.md index fd5f66174..82c3c698b 100644 --- a/docs/docs/instructions/server/request_parameters.md +++ b/docs/docs/instructions/server/request_parameters.md @@ -209,7 +209,7 @@ What you get back: - **`content` is only the continuation.** Your text isn't repeated, so `prefix + content` is the whole message. Its whitespace is kept as the model wrote it (`' Yes'`, not `'Yes'`), even when the continuation is only whitespace. In a stream, trailing whitespace arrives once the text after it, or the end of the reply, does. - **`logprobs` cover only the continued tokens.** Your text is part of the prompt, so it has no entries. - **`usage.prompt_tokens` includes your text**, and `completion_tokens` counts the continuation. -- **The prompt is cut the way Hugging Face cuts it.** FLM renders the chat with no generation prompt, finds the last occurrence of the final message's text, and stops right after it. Templates that trim message content also drop trailing whitespace from your text, so `"The answer is: "` is read as `"The answer is:"`. A template that keeps the space keeps it, and the model then sees a separate space token, which changes what it predicts next. +- **The prompt is cut the way Hugging Face cuts it.** FLM renders the chat with no generation prompt, finds the last occurrence of the final message's text, and stops right after it. One exception: a few templates write a finished message differently from how they open a reply (Qwen3.5 leaves out the empty `` block, Gemma4 12B with thinking off its empty thought channel, LFM2.5-Thinking its opening ``). Cut that way, the prompt looks like a finished turn and the model ends it, so for those templates FLM puts your text after the reply's opening instead. Templates that trim message content also drop trailing whitespace from your text, so `"The answer is: "` is read as `"The answer is:"`. A template that keeps the space keeps it, unless your text also starts with whitespace (Hugging Face drops the trailing space then too), and the model then sees a separate space token, which changes what it predicts next. - **Models that reason first may read your text as reasoning.** `deepseek-r1`, `deepseek-r1-0528`, `qwen3-tk`, `lfm2.5-tk`, `nanbeige4.1`, and any model with `think` on, start every reply by reasoning. They can treat the message you continue as the start of that reasoning: they go on, close it with ``, then answer. FLM then returns the continuation in `reasoning_content` and the answer after it in `content`, as vLLM's reasoning parsers do. A stream from such a model sends nothing until it either writes that tag or ends. To score the next token, use a model with `think` off: a reasoning model's first token is reasoning, not the answer. Rejected with HTTP 400 (`invalid_request_error`), for streaming requests too: diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 02818f679..ebb3a9b93 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -539,6 +539,13 @@ bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& to clear_context(); skip_count = 0; } + else if (skip_count >= tokens.size() && !tokens.empty()) { + // The engine holds the whole prompt already (the same request again), but + // sampling needs the logits of its last token, and the cache cannot step + // back one: read it in again from the start. + clear_context(); + skip_count = 0; + } tokens.erase(tokens.begin(), tokens.begin() + skip_count); if (this->total_tokens + tokens.size() >= this->MAX_L){ @@ -634,38 +641,28 @@ buffer AutoModel::_chunked_insert(chat_meta_info_t& meta_info, std::vector std::string AutoModel::_render_chat_template(minja::chat_template_inputs& inputs, bool continue_final_message, const minja::chat_template_options& options) { - inputs.add_generation_prompt = !continue_final_message; - std::string rendered = this->chat_tmpl->apply(inputs, options); if (!continue_final_message) { - return rendered; + inputs.add_generation_prompt = true; + return this->chat_tmpl->apply(inputs, options); } if (inputs.messages.empty() || inputs.messages.back().value("role", "") != "assistant") { throw ModelRequestError(400, false, "continue_final_message requires the last message to have role \"assistant\"", "invalid_request_error"); } - std::string prompt; + const auto render = [&](const nlohmann::ordered_json& messages, bool add_generation_prompt) { + minja::chat_template_inputs chat = inputs; + chat.messages = messages; + chat.add_generation_prompt = add_generation_prompt; + return this->chat_tmpl->apply(chat, options); + }; try { - prompt = chat_continue_final_message(rendered, inputs.messages.back()); + chat_continued continued = chat_continue_prompt(render, inputs.messages); + // the output parser looks for an open reasoning block in this turn only, not in what the user wrote earlier + this->continued_turn_ = std::move(continued.turn); + return continued.prompt; } catch (const std::invalid_argument& e) { throw ModelRequestError(400, false, e.what(), "invalid_request_error"); } - // The continued turn starts where the same chat with a generation prompt in place of the - // final message stops matching: the output parser looks for an open reasoning block from - // there, not in what the user wrote earlier. - size_t turn = 0; - try { - minja::chat_template_inputs opened = inputs; - opened.messages.erase(opened.messages.size() - 1); - opened.add_generation_prompt = true; - const std::string head = this->chat_tmpl->apply(opened, options); - while (turn < head.size() && turn < prompt.size() && head[turn] == prompt[turn]) { - turn++; - } - } catch (const std::exception&) { - turn = 0; // a template that cannot end on a generation prompt: look at the whole prompt - } - this->continued_turn_ = prompt.substr(turn); - return prompt; } void AutoModel::_begin_turn(chat_meta_info_t& meta_info, const std::vector& prompt_tokens) { diff --git a/src/common/AutoModel/modeling_gemma4_12b.cpp b/src/common/AutoModel/modeling_gemma4_12b.cpp index 8eedcc168..821839278 100644 --- a/src/common/AutoModel/modeling_gemma4_12b.cpp +++ b/src/common/AutoModel/modeling_gemma4_12b.cpp @@ -552,14 +552,23 @@ std::string Gemma4_12B::generate(chat_meta_info_t& meta_info, int length_limit, token_str = this->tokenizer->run_time_decoder(eoc_token_id); sampled_token = this->_sample(y, meta_info); - this->total_tokens++; + // all four went through the engine, so all four occupy the context + this->total_tokens += 4; meta_info.generated_tokens++; last_sampled_token = sampled_token; // the reply's first token: the loop below forwards it, so the history holds it this->token_history.push_back(last_sampled_token); - token_str = this->tokenizer->run_time_decoder(last_sampled_token); - result += token_str; - os << token_str << std::flush; + if (this->is_normal_token(last_sampled_token)) { + token_str = this->tokenizer->run_time_decoder(last_sampled_token); + result += token_str; + os << token_str << std::flush; + } + if (this->is_eos(last_sampled_token)) { + // the reply ended before it started; no engine has run the eos + this->token_history.pop_back(); + meta_info.stop_reason = reason; + return result; + } } else { last_sampled_token = this->last_token; diff --git a/src/common/AutoModel/modeling_gemma4e.cpp b/src/common/AutoModel/modeling_gemma4e.cpp index b71762310..6af9600e5 100644 --- a/src/common/AutoModel/modeling_gemma4e.cpp +++ b/src/common/AutoModel/modeling_gemma4e.cpp @@ -821,17 +821,16 @@ std::string Gemma4e::generate(chat_meta_info_t& meta_info, int length_limit, std std::cout << std::endl; header_print("FLM", "Model RAW Output: \n" + result); - if (!this->enable_think) { - int checkpoint_idx = this->lm_engine->checkpoint(); - // copy the token history at the checkpoint except the last one token, which is the start token for generation and should not be included in the checkpoint history - checkpoint_his = token_history; - checkpoint_his.pop_back(); - } if (reason != EOT_DETECTED) { // Cut short after sampling its last token: no engine has run that one, // so the next prompt has to carry it. this->token_history.pop_back(); } + if (!this->enable_think) { + // the history the checkpoint holds: an eos ending the turn was run, a cut-off token was not + int checkpoint_idx = this->lm_engine->checkpoint(); + checkpoint_his = token_history; + } return result; } diff --git a/src/common/AutoModel/modeling_qwen3.cpp b/src/common/AutoModel/modeling_qwen3.cpp index 09f18772d..c4c58b12c 100644 --- a/src/common/AutoModel/modeling_qwen3.cpp +++ b/src/common/AutoModel/modeling_qwen3.cpp @@ -191,9 +191,17 @@ std::string Qwen3::decode(chat_meta_info_t& meta_info, int length_limit, std::os last_sampled_token = sampled_token; // the loop forwards it like insert()'s sample, so history carries it too this->token_history.push_back(last_sampled_token); - token_str = this->tokenizer->run_time_decoder(last_sampled_token); - result += token_str; - os << token_str << std::flush; + if (this->is_normal_token(last_sampled_token)) { + token_str = this->tokenizer->run_time_decoder(last_sampled_token); + result += token_str; + os << token_str << std::flush; + } + if (this->is_eos(last_sampled_token)) { + // the reply ended before it started; no engine has run the eos + this->token_history.pop_back(); + meta_info.stop_reason = reason; + return result; + } } else { last_sampled_token = this->last_token; @@ -487,14 +495,23 @@ std::string Qwen3_TK::generate(chat_meta_info_t& meta_info, int length_limit, st sampled_token = this->_sample(y, meta_info); os << token_str << std::flush; - this->total_tokens++; + // both went through the engine, so both occupy the context + this->total_tokens += 2; meta_info.generated_tokens++; last_sampled_token = sampled_token; // the loop forwards it like insert()'s sample, so history carries it too this->token_history.push_back(last_sampled_token); - token_str = this->tokenizer->run_time_decoder(last_sampled_token); - result += token_str; - os << token_str << std::flush; + if (this->is_normal_token(last_sampled_token)) { + token_str = this->tokenizer->run_time_decoder(last_sampled_token); + result += token_str; + os << token_str << std::flush; + } + if (this->is_eos(last_sampled_token)) { + // the reply ended before it started; no engine has run the eos + this->token_history.pop_back(); + meta_info.stop_reason = reason; + return result; + } } else { // A continued message trimmed nothing, so it starts from insert()'s sample. diff --git a/src/common/AutoModel/modeling_qwen3_5_omni.cpp b/src/common/AutoModel/modeling_qwen3_5_omni.cpp index ce8546dcc..a2533e41c 100644 --- a/src/common/AutoModel/modeling_qwen3_5_omni.cpp +++ b/src/common/AutoModel/modeling_qwen3_5_omni.cpp @@ -489,7 +489,8 @@ bool Qwen3_5_Omni::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input break; } } - if (skip_count != idx) { + // the whole prompt cached leaves no last token to sample from: read it in again (see _shared_insert) + if (skip_count != idx || (skip_count >= tokens.size() && !tokens.empty())) { clear_context(); skip_count = 0; } diff --git a/src/common/chat/chat_continue.cpp b/src/common/chat/chat_continue.cpp index ee36a2823..4a382f2b8 100644 --- a/src/common/chat/chat_continue.cpp +++ b/src/common/chat/chat_continue.cpp @@ -37,9 +37,18 @@ std::string final_text(const nlohmann::ordered_json& message) { throw std::invalid_argument("continue_final_message is set but the final message has no text to continue"); } +size_t common_prefix(const std::string& a, const std::string& b) { + size_t n = 0; + while (n < a.size() && n < b.size() && a[n] == b[n]) { + n++; + } + return n; +} + } // namespace -std::string chat_continue_final_message(const std::string& rendered, const nlohmann::ordered_json& final_message) { +std::string chat_continue_final_message(const std::string& rendered, const nlohmann::ordered_json& final_message, + size_t* text_at) { const std::string text = final_text(final_message); const std::string stripped = strip(text); if (stripped.empty()) { @@ -51,6 +60,9 @@ std::string chat_continue_final_message(const std::string& rendered, const nlohm throw std::invalid_argument("continue_final_message is set but the final message does not appear in the chat " "after applying the chat template; this template rewrites the final assistant message"); } + if (text_at != nullptr) { + *text_at = at; + } // a template that kept the text as it is keeps its trailing whitespace too const size_t kept = lstrip(text).size(); if (rendered.compare(at, kept, text) == 0) { @@ -58,3 +70,31 @@ std::string chat_continue_final_message(const std::string& rendered, const nlohm } return rendered.substr(0, at + stripped.size()); } + +chat_continued chat_continue_prompt(const chat_render_fn& render, const nlohmann::ordered_json& messages) { + chat_continued out; + size_t text_at = 0; + out.prompt = chat_continue_final_message(render(messages, false), messages.back(), &text_at); + + nlohmann::ordered_json before = messages; + before.erase(before.size() - 1); + try { + // how the template opens a reply after the same conversation + const std::string opening = render(before, true); + if (opening.size() > text_at || out.prompt.compare(0, opening.size(), opening) != 0) { + out.prompt = opening + out.prompt.substr(text_at); + } + } catch (const std::exception&) { + // a template that cannot open a reply here: the cut stands + } + + size_t turn = 0; + try { + // the continued turn starts where the conversation before it ends + turn = common_prefix(render(before, false), out.prompt); + } catch (const std::exception&) { + turn = 0; + } + out.turn = out.prompt.substr(turn); + return out; +} diff --git a/src/common/chat/chat_parser.cpp b/src/common/chat/chat_parser.cpp index d5f6bba84..ebf42a549 100644 --- a/src/common/chat/chat_parser.cpp +++ b/src/common/chat/chat_parser.cpp @@ -100,8 +100,12 @@ struct text_parts { const bool others = std::any_of(parts.begin(), parts.end(), [](const part_t& p) { return !p.exact && !trim(p.raw).empty(); }); std::string out; - for (const auto& [raw, exact] : parts) { - std::string part = !exact ? trim(raw) : is_final ? raw : trim_end(raw); + for (size_t i = 0; i < parts.size(); i++) { + const auto& [raw, exact] = parts[i]; + // text after it settles its trailing whitespace as surely as the end does + const bool followed = std::any_of(parts.begin() + i + 1, parts.end(), + [](const part_t& p) { return !trim(p.raw).empty(); }); + std::string part = !exact ? trim(raw) : (is_final || followed) ? raw : trim_end(raw); // whitespace alone is the answer only when nothing else is if (part.empty() || (exact && others && trim(part).empty())) { continue; @@ -524,7 +528,9 @@ chat_msg parse_tagged(const chat_format_spec& spec, const std::string& full_text break; } const std::string part = text.substr(pos, end - pos); - reasoning.add(part, lead(part)); + // closed with nothing but whitespace in it, the block was layout, not an answer + const bool exact = lead(part); + reasoning.add(part, exact && !trim(part).empty()); pos = end + spec.reasoning_end.size(); in_reasoning = false; continue; diff --git a/src/include/chat/chat_continue.hpp b/src/include/chat/chat_continue.hpp index ce38a69db..7bcd65fa6 100644 --- a/src/include/chat/chat_continue.hpp +++ b/src/include/chat/chat_continue.hpp @@ -4,9 +4,13 @@ /// chat is rendered without a generation prompt and cut right after the /// final message's text, so the model writes on from there. The cut follows /// transformers' apply_chat_template(continue_final_message=True), so the -/// same template gives the same prompt here as there. +/// same template gives the same prompt here as there, except where the +/// template writes a finished message differently from how it opens a reply +/// (see chat_continue_prompt). #pragma once +#include +#include #include #include @@ -15,6 +19,7 @@ /// \param rendered the chat rendered without a generation prompt /// \param final_message the last message as the template was given it; its /// content is a string or an array of typed parts, whose last text part counts +/// \param text_at set to where the message's text starts in the result, when not null /// \return the rendered chat up to the end of the final message's text /// \note Whitespace around the text is matched loosely, as transformers does: /// templates often trim the content. Trailing whitespace is kept only when @@ -22,4 +27,25 @@ /// \throws std::invalid_argument when the final message has no text, or the /// template did not render that text as it is (some templates rewrite the /// last assistant message, e.g. by moving a block out of it) -std::string chat_continue_final_message(const std::string& rendered, const nlohmann::ordered_json& final_message); +std::string chat_continue_final_message(const std::string& rendered, const nlohmann::ordered_json& final_message, + size_t* text_at = nullptr); + +/// \brief Renders a chat: the messages, and whether to end on a generation prompt +using chat_render_fn = std::function; + +/// \brief A prompt that continues the final message +struct chat_continued { + std::string prompt; ///< what the model is fed + std::string turn; ///< the continued assistant turn: the template's opening of it and the message's text +}; + +/// \brief Build the prompt that continues the final assistant message of a chat +/// \param render renders the chat with the model's template +/// \param messages the chat, ending with the assistant message to continue +/// \note The prompt is transformers' cut wherever the template opens a reply the +/// way it renders a finished message. Where it does not, the cut reads as a +/// finished turn and the model ends it (Qwen3.5 leaves the empty think block +/// out of a finished message, Gemma4 with thinking off its empty thought +/// channel), so the prompt is then the reply's opening followed by the text. +/// \throws std::invalid_argument as chat_continue_final_message +chat_continued chat_continue_prompt(const chat_render_fn& render, const nlohmann::ordered_json& messages); diff --git a/src/test/chat_continue/test_chat_continue.cpp b/src/test/chat_continue/test_chat_continue.cpp index 54ad7c605..fcb177be1 100644 --- a/src/test/chat_continue/test_chat_continue.cpp +++ b/src/test/chat_continue/test_chat_continue.cpp @@ -98,6 +98,56 @@ const std::string QWEN2VL = "{%- endfor %}{%- endif %}{{- '<|im_end|>\\n' }}{%- endfor %}" "{%- if add_generation_prompt %}{{- '<|im_start|>assistant\\n' }}{%- endif %}"; +// Qwen3.5: a reply opens with an empty think block, a finished message has none +const std::string QWEN35 = + "{%- for m in messages %}{{- '<|im_start|>' + m['role'] + '\\n' + m['content'] | trim + '<|im_end|>\\n' }}{%- endfor %}" + "{%- if add_generation_prompt %}{{- '<|im_start|>assistant\\n\\n\\n\\n\\n' }}{%- endif %}"; + +// LFM2.5-Thinking: a reply opens inside a think block, a finished message is the answer alone +const std::string THINKING = + "{%- for m in messages %}{{- '<|im_start|>' + m['role'] + '\\n' + m['content'] + '<|im_end|>\\n' }}{%- endfor %}" + "{%- if add_generation_prompt %}{{- '<|im_start|>assistant\\n\\n' }}{%- endif %}"; + +/// \brief What AutoModel::_render_chat_template does when continuing +chat_continued continued_prompt(const std::string& source, const json& messages) { + minja::chat_template tmpl(source, "", ""); + return chat_continue_prompt( + [&tmpl](const json& chat, bool add_generation_prompt) { + minja::chat_template_inputs inputs; + inputs.messages = chat; + inputs.add_generation_prompt = add_generation_prompt; + return tmpl.apply(inputs); + }, + messages); +} + +void test_prompt() { + std::printf("prompt\n"); + const std::string q = "Mind the tag. Is Paris the capital of France?"; + const std::string user_turn = "<|im_start|>user\n" + q + "<|im_end|>\n"; + + // where the template opens a reply the way it renders a finished message, the cut is transformers' + chat_continued chatml = continued_prompt(CHATML, chat(q, "The answer is:")); + CHECK_EQ(chatml.prompt, continued(CHATML, chat(q, "The answer is:"))); + CHECK_EQ(chatml.turn, "<|im_start|>assistant\nThe answer is:"); + chat_continued qwen3 = continued_prompt(QWEN3, chat(q, "The answer is:")); + CHECK_EQ(qwen3.prompt, continued(QWEN3, chat(q, "The answer is:"))); + CHECK_EQ(qwen3.turn, "<|im_start|>assistant\n\n\n\n\nThe answer is:"); + + // where it does not, the message's text follows the reply's opening, as the model writes a reply + chat_continued qwen35 = continued_prompt(QWEN35, chat(q, "The answer is: ")); + CHECK_EQ(qwen35.prompt, user_turn + "<|im_start|>assistant\n\n\n\n\nThe answer is:"); + CHECK_EQ(qwen35.turn, "<|im_start|>assistant\n\n\n\n\nThe answer is:"); + chat_continued thinking = continued_prompt(THINKING, chat(q, "The answer is:")); + CHECK_EQ(thinking.prompt, user_turn + "<|im_start|>assistant\n\nThe answer is:"); + + // the turn is what the output parser reads: the user's is not in it, the template's are + const chat_format_spec spec = chat_format_make(chat_format_kind::HERMES_JSON, true); + CHECK_EQ(chat_format_prefill(spec, chatml.turn, true), ""); + CHECK_EQ(chat_format_prefill(spec, qwen35.turn, true), ""); + CHECK_EQ(chat_format_prefill(spec, thinking.turn, true), ""); +} + void test_template_shapes() { std::printf("template shapes\n"); const std::string q = "Is Paris the capital of France? Answer yes or no."; @@ -249,12 +299,35 @@ void test_parse() { CHECK_EQ(streamed, whole); CHECK_EQ(streamed, chat_parse(qwen, whole, false, keep).content); } + // text after a tool call or a think block settles the whitespace before it, so a partial + // parse is still a prefix of the final one and no delta gets dropped + for (const std::vector& pieces : std::vector>{ + {" Paris. ", "{\"name\": \"f\", \"arguments\": {}}", "", "\nDo", "ne"}, + {"foo ", "x", " b", "ar"}}) { + chat_stream stream(qwen, "", keep); + std::string streamed, whole; + for (const std::string& piece : pieces) { + whole += piece; + for (const chat_msg_delta& d : stream.update(piece, false)) { + streamed += d.content; + } + } + for (const chat_msg_delta& d : stream.update("", true)) { + streamed += d.content; + } + CHECK_EQ(streamed, chat_parse(qwen, whole, false, keep).content); + } + // an empty block the model closes is layout: the answer after it is the content + chat_msg empty_block = chat_parse(qwen, "\n\n\nParis.", false, closes); + CHECK_EQ(empty_block.reasoning, ""); + CHECK_EQ(empty_block.content, "Paris."); } } // namespace int main() { test_template_shapes(); + test_prompt(); test_cut(); test_prefill(); test_parse(); diff --git a/src/test/openai_api_test/test_continue_final_message.py b/src/test/openai_api_test/test_continue_final_message.py index 6270ec947..01e33f477 100644 --- a/src/test/openai_api_test/test_continue_final_message.py +++ b/src/test/openai_api_test/test_continue_final_message.py @@ -115,7 +115,7 @@ def score_next_token(api: Api) -> None: choice = api.ok(body) msg = flm_api.check_message_contract(choice) entries = choice["logprobs"]["content"] - if not entries and msg.get("reasoning_content"): + if api.always_thinks and (not entries or not entries[0]["token"].strip()): raise Skip("the model reasons before answering, so its first token is not the answer") expect(len(entries) == 1, f"max_tokens 1 should give one entry, got {len(entries)}: {entries!r}") check_entry(entries[0], 5, "content[0]") From e62c97e10cfed56129db1bba04a19731f00ade0d Mon Sep 17 00:00:00 2001 From: zaneni6 Date: Wed, 28 Oct 2026 16:52:53 -0700 Subject: [PATCH 5/5] test(logprobs): give top_logprobs_default an answer budget At 16 tokens a model that reasons first spent the whole budget reasoning and returned no content to take logprobs from. Co-Authored-By: Claude --- src/test/openai_api_test/test_logprobs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/openai_api_test/test_logprobs.py b/src/test/openai_api_test/test_logprobs.py index 573621593..7dfc78b60 100644 --- a/src/test/openai_api_test/test_logprobs.py +++ b/src/test/openai_api_test/test_logprobs.py @@ -184,7 +184,8 @@ def max_tokens_is_exact(api: Api) -> None: @case def top_logprobs_default(api: Api) -> None: """logprobs without top_logprobs gives every entry an empty top_logprobs.""" - body = logprobs_body(api, user(PROMPT), max_tokens=16) + # an answer budget: a model that reasons first gets room for that too + body = logprobs_body(api, user(PROMPT), max_tokens=32) del body["top_logprobs"] _, entries = nonstream_entries(api, body) expect(len(entries) > 0, "no logprobs entries")