diff --git a/docs/cli.md b/docs/cli.md index b680d288cb..ba821f3585 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -339,7 +339,8 @@ and rendered prompt mode: | Qwen3.8-Flash-Next | non-thinking | `0.7` | `0.80` | `20` | `0` | `1.5` | Frequency penalty is `0` in every registered preset. Task-specific profiles such as Qwen's -precise-coding profile use explicit sampling overrides. +precise-coding profile use explicit sampling overrides. `ninfer-serve` drops the presence penalty +for a request that offers tools ([HTTP serving](serving.md#server-options)). Repeat `--stop-token-id`, `--stop`, or `--reasoning-stop` to add stop conditions. Use `--raw-output` to expose the frontend's raw output stream and `--print-token-ids` to include diff --git a/docs/serving.md b/docs/serving.md index 5bbb0857a4..4e099323e9 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -1534,8 +1534,10 @@ separate `namespace` and `name` fields in aggregate output, SSE events, and repl function name may therefore appear in different namespaces. Namespace members remain ordinary client-executed functions; this does not add a remote MCP executor. -NInfer renders these definitions in the Qwen prompt and parses model output into separate -`function_call` output Items. Each output has a protocol Item `id` (`fc_...`) and a distinct +NInfer renders a function definition in the Qwen prompt, whichever protocol declared it, as +`{"type":"function","function":{"name":…,"description":…,"parameters":…}}` in declaration order +and without `strict`, which only selects the server's output contract, and parses model output +into separate `function_call` output Items. Each output has a protocol Item `id` (`fc_...`) and a distinct `call_id` (`call_...`). The client executes the function and sends a `function_call_output` Item in a later request. Only functions in the current effective tool set can become structured calls; undeclared model output remains ordinary text. `allowed_tools` filters that set without changing @@ -2082,6 +2084,10 @@ Qwen3.6-27B, Qwen3.8-27B and the artifacts built from it, and Qwen3.8-Flash-Next penalty, which is `1.5`. Frequency penalty is `0` for all registered presets. Process flags override registered values, request fields override process flags, and `--greedy` finally forces temperature `0`. +A request that offers tools (`tools` with a `tool_choice` other than `none`, on every protocol) +samples without the presence penalty unless the request or `--presence-penalty` sets one: a second +tool call repeats the first call's markup and function name, and under `1.5` the model drops calls +it would otherwise make. The request log records the resolved value. ### Post-thinking sampling diff --git a/include/ninfer/types.h b/include/ninfer/types.h index 469b8f597d..8cf70efe3e 100644 --- a/include/ninfer/types.h +++ b/include/ninfer/types.h @@ -1075,6 +1075,8 @@ struct PromptOptions { // JSON object of template parameters. Unset typed fields leave template defaults intact. std::string chat_template_kwargs_json; bool add_vision_id = false; + // OpenAI-shaped function definitions in declaration order. `function.strict` configures the + // output contract and is not rendered into the prompt. std::vector tool_jsons; // Function the caller selected. Its call opener is appended to the generation prompt, so the // answer can only continue inside that call. Requires a new assistant turn with thinking off. diff --git a/src/models/qwen3_5/frontend/chat_template.cpp b/src/models/qwen3_5/frontend/chat_template.cpp index 4a5fd3f271..8a580766d5 100644 --- a/src/models/qwen3_5/frontend/chat_template.cpp +++ b/src/models/qwen3_5/frontend/chat_template.cpp @@ -344,7 +344,14 @@ RenderedChat CompiledChatTemplate::render_as(const std::vector& mes if (!options.tool_jsons.empty()) { context["tools"] = Json::array(); for (std::size_t i = 0; i < options.tool_jsons.size(); ++i) { - context["tools"].push_back(Json::parse(options.tool_jsons[i])); + Json tool = Json::parse(options.tool_jsons[i]); + // `strict` selects the output contract's grammar; the model reads a definition as its + // template was trained on it, without that server flag. + if (tool.is_object()) { + const auto function = tool.find("function"); + if (function != tool.end() && function->is_object()) { function->erase("strict"); } + } + context["tools"].push_back(std::move(tool)); tool_tags.push_back(tag("/tools/" + std::to_string(i))); } } diff --git a/src/ops/sparse_moe/streamed_cpu_avx2.cpp b/src/ops/sparse_moe/streamed_cpu_avx2.cpp index 6162b74e33..f479d2d06c 100644 --- a/src/ops/sparse_moe/streamed_cpu_avx2.cpp +++ b/src/ops/sparse_moe/streamed_cpu_avx2.cpp @@ -20,7 +20,10 @@ std::uint16_t load16(const std::uint8_t* p) noexcept { return value; } -float half(std::uint16_t bits) noexcept { return _cvtsh_ss(bits); } +// The vector F16C form: MSVC does not declare the scalar _cvtsh_ss. +float half(std::uint16_t bits) noexcept { + return _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(bits))); +} // Signed integer codes to weights: (code ^ bias) - bias, times the scale. __m256 weight(__m256i code, int bias, __m256 scale) noexcept { diff --git a/src/serve/translate.cpp b/src/serve/translate.cpp index b16ea96e61..239c51e346 100644 --- a/src/serve/translate.cpp +++ b/src/serve/translate.cpp @@ -125,14 +125,19 @@ std::vector effective_tools(const GenerationRequest& requ return tools; } +// The members follow the order the Qwen templates were trained on, name, description and +// parameters; `strict` is the server's enforcement flag for the output contract, and the frontend +// leaves it out of the rendered prompt. std::string render_tool_definition(const ToolDefinition& tool) { using Json = RequestJson; Json schema = Json::parse(tool.input_schema_json); - Json function{{"name", tool.name}, {"parameters", std::move(schema)}, {"strict", tool.strict}}; + Json function{{"name", tool.name}}; if (!tool.description.empty()) { function["description"] = tool.description; } + function["parameters"] = std::move(schema); if (tool.input_examples_json) { function["input_examples"] = Json::parse(*tool.input_examples_json); } + function["strict"] = tool.strict; return Json{{"type", "function"}, {"function", std::move(function)}}.dump(); } @@ -454,6 +459,18 @@ ninfer::RequestOptions to_request_options(const GenerationRequest& request, options.execution.sampling = resolve_sampling_overrides(request.sampling, server); options.execution.post_thinking_sampling = resolve_post_thinking_overrides(request.post_thinking, server); + // A second tool call repeats the first one's markup and function name, which a presence penalty + // punishes: under the Qwen presets' 1.5 the model drops calls it would otherwise make. A request + // that offers tools samples without one unless the request or the server named a value. + if (request.uses_tools()) { + if (!options.execution.sampling.presence_penalty) { + options.execution.sampling.presence_penalty = 0.0F; + } + if (options.execution.post_thinking_sampling && + !options.execution.post_thinking_sampling->presence_penalty) { + options.execution.post_thinking_sampling->presence_penalty = 0.0F; + } + } options.execution.logprobs = request.logprobs; options.output.raw = false; options.output.preserve_special_tokens = diff --git a/tests/models/qwen3_5/test_frontend.cpp b/tests/models/qwen3_5/test_frontend.cpp index 6a13ffc442..c4252e3f16 100644 --- a/tests/models/qwen3_5/test_frontend.cpp +++ b/tests/models/qwen3_5/test_frontend.cpp @@ -7,6 +7,7 @@ #include "models/qwen3_5/frontend/processor.h" #include "models/qwen3_5/frontend/test_access.h" #include "models/qwen3_5/frontend/tokenizer.h" +#include "models/qwen3_5/frontend/tool_call_parser.h" #include "text/unicode.h" #include "text/structured_output.h" @@ -1031,6 +1032,33 @@ int test_selected_template_recovery_boundary() { return failures; } +// The model reads a tool as the maintained templates were trained on it: name, description and +// parameters in that order, without the server's `strict` flag, which still selects the output +// contract's grammar. +int test_tool_definitions_in_prompt() { + fi::ChatRenderOptions options; + options.enable_thinking = false; + options.tool_jsons = { + R"({"type":"function","function":{"name":"weather","description":"Get weather","parameters":{"type":"object","properties":{"city":{"type":"string"}}},"strict":true}})"}; + const std::string expected = + R"({"type": "function", "function": {"name": "weather", "description": "Get weather", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}})"; + int failures = 0; + for (const auto& source : + {thinking_toggle_template_source(), reasoning_effort_template_source()}) { + const std::string text = + fi::CompiledChatTemplate::resolve(source) + .render({chat_message(ninfer::ChatRole::User, "weather?")}, options) + .text; + failures += check(text.find(expected) != std::string::npos && + text.find("strict") == std::string::npos, + "a tool reached the prompt out of order or with the strict flag"); + } + const auto contract = fi::build_tool_call_output_contract(options.tool_jsons, true); + failures += check(contract && contract->tools.size() == 1 && contract->tools.front().strict, + "the strict flag no longer reaches the output contract"); + return failures; +} + int test_adjacent_tool_message_boundary() { fi::ChatMessage assistant = chat_message(ninfer::ChatRole::Assistant, ""); assistant.tool_calls.push_back( @@ -4300,6 +4328,7 @@ int main() { failures += test_open_reasoning_tool_recovery(); failures += test_forced_tool_call(frontend); failures += test_tool_marker_after_quoted_marker(); + failures += test_tool_definitions_in_prompt(); failures += test_reasoning_split(frontend); failures += test_reasoning_close_requires_boundary(frontend); failures += test_reasoning_close_resolves_at_terminal(frontend); diff --git a/tests/ops/test_gguf_expert_cpu.cpp b/tests/ops/test_gguf_expert_cpu.cpp index 4894b9d206..274d093b52 100644 --- a/tests/ops/test_gguf_expert_cpu.cpp +++ b/tests/ops/test_gguf_expert_cpu.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -496,7 +497,9 @@ double ulps(float got, double exact) { double silu64(double x) { return x / (1.0 + std::exp(-x)); } // 0.5 x (1 + tanh(u)) = x / (1 + e^-2u), the second form keeping its precision for negative u. -double gelu_u(double x) { return std::sqrt(2.0 / M_PI) * (x + 0.044715 * x * x * x); } +double gelu_u(double x) { + return std::sqrt(2.0 / std::numbers::pi) * (x + 0.044715 * x * x * x); +} double gelu64(double x) { return x / (1.0 + std::exp(-2.0 * gelu_u(x))); } void check_silu(std::mt19937& rng) { diff --git a/tests/test_openai_schema.cpp b/tests/test_openai_schema.cpp index 6f35b5c338..8820c2b9df 100644 --- a/tests/test_openai_schema.cpp +++ b/tests/test_openai_schema.cpp @@ -563,6 +563,39 @@ int test_tools() { failures += check(automatic.generation.uses_tools(), "function tools default to auto"); failures += check(prompt(automatic.generation).options.tool_jsons.size() == 1, "auto tools reach PromptInput"); + // The definition keeps the member order the templates were trained on; `strict` rides along + // for the output contract and the frontend leaves it out of the prompt. + failures += check( + prompt(automatic.generation).options.tool_jsons[0] == + R"({"type":"function","function":{"name":"weather","description":"Get weather","parameters":{"type":"object"},"strict":false}})", + "a tool definition did not reach the prompt as name, description, parameters"); + // A second call repeats the first call's markup and name, so a request that offers tools + // samples without the preset's presence penalty unless the request or the server names one. + failures += check(options(automatic.generation).execution.sampling.presence_penalty == 0.0F, + "a request with tools kept the preset's presence penalty"); + failures += check(!options(parse(base_request()).generation).execution.sampling.presence_penalty, + "a request without tools lost the preset's presence penalty"); + Json penalized = body; + penalized["presence_penalty"] = 1.0; + failures += + check(options(parse(penalized).generation).execution.sampling.presence_penalty == 1.0F, + "a request with tools lost its own presence penalty"); + ServeOptions penalizing; + penalizing.sampling_overrides.presence_penalty = 0.5F; + failures += check(to_request_options(automatic.generation, penalizing, + semantics(automatic.generation), true) + .execution.sampling.presence_penalty == 0.5F, + "a request with tools lost the server's presence penalty"); + Json post = body; + post["post_thinking"] = Json{{"temperature", 0.1}}; + const ninfer::RequestOptions post_options = options(parse(post).generation); + failures += check(post_options.execution.post_thinking_sampling && + post_options.execution.post_thinking_sampling->presence_penalty == 0.0F, + "a request with tools kept the post-thinking preset's presence penalty"); + post["post_thinking"] = Json{{"presence_penalty", 0.7}}; + failures += check(options(parse(post).generation) + .execution.post_thinking_sampling->presence_penalty == 0.7F, + "a request with tools lost its own post-thinking presence penalty"); body["tools"][0]["future_item_field"] = "ignored"; body["tools"][0]["function"]["future_function_field"] = "ignored"; @@ -577,6 +610,8 @@ int test_tools() { failures += check(!none.generation.uses_tools() && prompt(none.generation).options.tool_jsons.empty(), "tool_choice none makes parallel_tool_calls neutral and removes executable tools"); + failures += check(!options(none.generation).execution.sampling.presence_penalty, + "tool_choice none did not restore the preset's presence penalty"); body["parallel_tool_calls"] = true; body["enable_thinking"] = false;