From e0ee297691f5a397ba0d4b6ccde5c61ad8974c5c Mon Sep 17 00:00:00 2001 From: dhandhalyabhavik Date: Sat, 9 Aug 2025 03:39:06 +0530 Subject: [PATCH 1/7] common : add GLM-4.5 tool calling support - Add COMMON_CHAT_FORMAT_GLM_4_5 format enum - Implement GLM-4.5 tool call parser for format - Add template detection based on and tags - Fix null content handling in message parsing and serialization - Ensure GLM-4.5 detection runs before Hermes to avoid misidentification This enables tool calling functionality for GLM-4.5 models when using --jinja flag. The parser handles GLM-4.5's XML-like tool call format with key-value argument pairs. --- common/chat.cpp | 73 +++++++++++++++++++++++++++++++++++++++++++++++-- common/chat.h | 1 + 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 60805ab3b53f..467d0028e4ef 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -203,7 +203,10 @@ std::vector common_chat_msgs_parse_oaicompat(const json & messa msg_part.text = part.at("text"); msg.content_parts.push_back(msg_part); } - } else if (!content.is_null()) { + } else if (content.is_null()) { + // Handle null content by setting it to empty string + msg.content = ""; + } else { throw std::runtime_error("Invalid 'content' type: expected string or array, got " + content.dump() + " (ref: https://github.com/ggml-org/llama.cpp/issues/8367)"); } } @@ -292,7 +295,7 @@ json common_chat_msgs_to_json_oaicompat(const std::vector & msg } } } else { - jmsg["content"] = json(); // null + jmsg["content"] = ""; // empty string instead of null } if (!msg.reasoning_content.empty()) { jmsg["reasoning_content"] = msg.reasoning_content; @@ -607,6 +610,7 @@ const char * common_chat_format_name(common_chat_format format) { case COMMON_CHAT_FORMAT_HERMES_2_PRO: return "Hermes 2 Pro"; case COMMON_CHAT_FORMAT_COMMAND_R7B: return "Command R7B"; case COMMON_CHAT_FORMAT_GPT_OSS: return "GPT-OSS"; + case COMMON_CHAT_FORMAT_GLM_4_5: return "GLM 4.5"; default: throw std::runtime_error("Unknown chat format"); } @@ -1325,6 +1329,63 @@ static void common_chat_parse_gpt_oss(common_chat_msg_parser & builder) { } } +static common_chat_params common_chat_params_init_glm_4_5(const common_chat_template & tmpl, const struct templates_params & inputs) { + common_chat_params data; + data.prompt = apply(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_GLM_4_5; + return data; +} + +static void common_chat_parse_glm_4_5(common_chat_msg_parser & builder) { + builder.try_parse_reasoning("", ""); + if (!builder.syntax().parse_tool_calls) { + builder.add_content(builder.consume_rest()); + return; + } + + // GLM 4.5 uses format: function_name\nkey\nvalue\n + static const common_regex tool_call_start("([^\n<]+)"); + static const common_regex arg_key_regex("([^<]+)"); + static const common_regex arg_value_regex("([^<]*)"); + static const common_regex tool_call_end(""); + + while (auto res = builder.try_find_regex(tool_call_start)) { + // Move to the start of the tool call and consume it + builder.move_to(res->groups[0].begin); + builder.consume_regex(tool_call_start); + + std::string function_name = builder.str(res->groups[1]); + json arguments = json::object(); + + builder.consume_spaces(); + + // Parse all arg_key/arg_value pairs + while (auto key_res = builder.try_consume_regex(arg_key_regex)) { + std::string key = builder.str(key_res->groups[1]); + builder.consume_spaces(); + + if (auto value_res = builder.try_consume_regex(arg_value_regex)) { + std::string value = builder.str(value_res->groups[1]); + arguments[key] = value; + builder.consume_spaces(); + } else { + throw common_chat_msg_partial_exception("Expected after "); + } + } + + // Consume closing tag + builder.consume_regex(tool_call_end); + builder.consume_spaces(); + + // Add the parsed tool call + if (!builder.add_tool_call(function_name, "", arguments.dump())) { + throw common_chat_msg_partial_exception("Failed to add GLM tool call"); + } + } + + builder.add_content(builder.consume_rest()); +} + static common_chat_params common_chat_params_init_firefunction_v2(const common_chat_template & tmpl, const struct templates_params & inputs) { LOG_DBG("%s\n", __func__); common_chat_params data; @@ -1805,6 +1866,11 @@ static common_chat_params common_chat_templates_apply_jinja( return common_chat_params_init_command_r7b(tmpl, params); } + // GLM 4.5: detect by and tags (check before Hermes since both use ) + if (src.find("") != std::string::npos && src.find("") != std::string::npos && params.json_schema.is_null()) { + return common_chat_params_init_glm_4_5(tmpl, params); + } + // Hermes 2/3 Pro, Qwen 2.5 Instruct (w/ tools) if (src.find("") != std::string::npos && params.json_schema.is_null()) { return common_chat_params_init_hermes_2_pro(tmpl, params); @@ -1969,6 +2035,9 @@ static void common_chat_parse(common_chat_msg_parser & builder) { case COMMON_CHAT_FORMAT_GPT_OSS: common_chat_parse_gpt_oss(builder); break; + case COMMON_CHAT_FORMAT_GLM_4_5: + common_chat_parse_glm_4_5(builder); + break; default: throw std::runtime_error(std::string("Unsupported format: ") + common_chat_format_name(builder.syntax().format)); } diff --git a/common/chat.h b/common/chat.h index b014f9f0aaeb..f31bd085729a 100644 --- a/common/chat.h +++ b/common/chat.h @@ -110,6 +110,7 @@ enum common_chat_format { COMMON_CHAT_FORMAT_HERMES_2_PRO, COMMON_CHAT_FORMAT_COMMAND_R7B, COMMON_CHAT_FORMAT_GPT_OSS, + COMMON_CHAT_FORMAT_GLM_4_5, COMMON_CHAT_FORMAT_COUNT, // Not a format, just the # formats }; From 5432887c291fb63f7aa81264e20fef7d3c80166b Mon Sep 17 00:00:00 2001 From: Bhavik Dhandhalya <86345824+dhandhalyabhavik@users.noreply.github.com> Date: Sat, 23 Aug 2025 04:08:15 +0530 Subject: [PATCH 2/7] fixed reasoning_content This fixes the problem of thinking message not coming in reasoning_content variable. --- common/chat.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/chat.cpp b/common/chat.cpp index 467d0028e4ef..ebebabf6bb8c 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1337,6 +1337,7 @@ static common_chat_params common_chat_params_init_glm_4_5(const common_chat_temp } static void common_chat_parse_glm_4_5(common_chat_msg_parser & builder) { + builder.consume_spaces(); builder.try_parse_reasoning("", ""); if (!builder.syntax().parse_tool_calls) { builder.add_content(builder.consume_rest()); @@ -1867,7 +1868,7 @@ static common_chat_params common_chat_templates_apply_jinja( } // GLM 4.5: detect by and tags (check before Hermes since both use ) - if (src.find("") != std::string::npos && src.find("") != std::string::npos && params.json_schema.is_null()) { + if (src.find("[gMASK]") != std::string::npos && src.find("") != std::string::npos && src.find("") != std::string::npos && params.json_schema.is_null()) { return common_chat_params_init_glm_4_5(tmpl, params); } From fb14b0c3aa93da5b2c2c605d81df1133974612ad Mon Sep 17 00:00:00 2001 From: Bhavik Dhandhalya <86345824+dhandhalyabhavik@users.noreply.github.com> Date: Sat, 23 Aug 2025 04:16:02 +0530 Subject: [PATCH 3/7] regex arg_value fix --- common/chat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chat.cpp b/common/chat.cpp index ebebabf6bb8c..79074b032983 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1347,7 +1347,7 @@ static void common_chat_parse_glm_4_5(common_chat_msg_parser & builder) { // GLM 4.5 uses format: function_name\nkey\nvalue\n static const common_regex tool_call_start("([^\n<]+)"); static const common_regex arg_key_regex("([^<]+)"); - static const common_regex arg_value_regex("([^<]*)"); + static const common_regex arg_value_regex("(.*)"); static const common_regex tool_call_end(""); while (auto res = builder.try_find_regex(tool_call_start)) { From 6f74eebac902d4ba815fdeca1352f831b025a98f Mon Sep 17 00:00:00 2001 From: Bhavik Dhandhalya <86345824+dhandhalyabhavik@users.noreply.github.com> Date: Sat, 23 Aug 2025 04:17:45 +0530 Subject: [PATCH 4/7] regex fix --- common/chat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chat.cpp b/common/chat.cpp index 79074b032983..7cf8a021a414 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1347,7 +1347,7 @@ static void common_chat_parse_glm_4_5(common_chat_msg_parser & builder) { // GLM 4.5 uses format: function_name\nkey\nvalue\n static const common_regex tool_call_start("([^\n<]+)"); static const common_regex arg_key_regex("([^<]+)"); - static const common_regex arg_value_regex("(.*)"); + static const common_regex arg_value_regex("(.*?)"); static const common_regex tool_call_end(""); while (auto res = builder.try_find_regex(tool_call_start)) { From 45fac2a30441307772e069041b50323215ba536d Mon Sep 17 00:00:00 2001 From: Bhavik Dhandhalya <86345824+dhandhalyabhavik@users.noreply.github.com> Date: Sat, 23 Aug 2025 12:41:44 +0530 Subject: [PATCH 5/7] added fix for arg_value (it can read multiple lines with ASCIIs) --- common/chat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chat.cpp b/common/chat.cpp index 7cf8a021a414..fa59698c34d2 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1347,7 +1347,7 @@ static void common_chat_parse_glm_4_5(common_chat_msg_parser & builder) { // GLM 4.5 uses format: function_name\nkey\nvalue\n static const common_regex tool_call_start("([^\n<]+)"); static const common_regex arg_key_regex("([^<]+)"); - static const common_regex arg_value_regex("(.*?)"); + static const common_regex arg_value_regex("([\\s\\S]*?)"); static const common_regex tool_call_end(""); while (auto res = builder.try_find_regex(tool_call_start)) { From 623f3dd04b05999ffcb598a88d4f79fa9c3af849 Mon Sep 17 00:00:00 2001 From: dhandhalyabhavik Date: Sat, 30 Aug 2025 19:25:34 +0530 Subject: [PATCH 6/7] GLM-4.5 tool calling: Complete implementation with schema-aware parsing - Added GLM-4.5 chat format with native XML template support - Implemented schema-aware type conversion for tool arguments - Added JSON unpacking logic for single-argument cases - Enhanced server integration with tools_schema support - Added comprehensive test coverage and documentation - Fixed template polyfill conflicts and type conversion issues --- .github/chatmodes/custom_models.chatmode.md | 0 common/chat.cpp | 208 ++++++++++++++++---- common/chat.h | 3 + models/templates/glm_4_5.jinja | 119 +++++++++++ tests/test-chat.cpp | 1 + tools/server/server.cpp | 1 + tools/server/utils.hpp | 3 + 7 files changed, 298 insertions(+), 37 deletions(-) create mode 100644 .github/chatmodes/custom_models.chatmode.md create mode 100644 models/templates/glm_4_5.jinja diff --git a/.github/chatmodes/custom_models.chatmode.md b/.github/chatmodes/custom_models.chatmode.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/common/chat.cpp b/common/chat.cpp index fa59698c34d2..e9ce8028726c 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1330,63 +1330,197 @@ static void common_chat_parse_gpt_oss(common_chat_msg_parser & builder) { } static common_chat_params common_chat_params_init_glm_4_5(const common_chat_template & tmpl, const struct templates_params & inputs) { + LOG_INF("%s: initializing GLM-4.5 chat params\n", __func__); common_chat_params data; - data.prompt = apply(tmpl, inputs); + + // Configure template inputs + minja::chat_template_inputs tmpl_inputs; + tmpl_inputs.messages = inputs.messages; + tmpl_inputs.tools = inputs.tools.empty() ? json() : inputs.tools; + tmpl_inputs.add_generation_prompt = inputs.add_generation_prompt; + tmpl_inputs.extra_context = inputs.extra_context; + tmpl_inputs.now = inputs.now; // Use the consistent timestamp from params + + // Configure template options to disable polyfills and enforce native XML format + minja::chat_template_options opts; + opts.apply_polyfills = false; // Hard disable all polyfills + + // The prompt is generated here + data.prompt = tmpl.apply(tmpl_inputs, opts); data.format = COMMON_CHAT_FORMAT_GLM_4_5; + + data.preserved_tokens = { + "<|system|>", "<|assistant|>", "<|observation|>", + "", "", "", "", + "", "", "", "", + "", "", + }; + + // Store tools schema for type-aware parsing later + data.tools_schema = inputs.tools; + + LOG_INF("%s: GLM-4.5 native XML format enforced\n", __func__); return data; } static void common_chat_parse_glm_4_5(common_chat_msg_parser & builder) { - builder.consume_spaces(); - builder.try_parse_reasoning("", ""); - if (!builder.syntax().parse_tool_calls) { - builder.add_content(builder.consume_rest()); - return; - } - - // GLM 4.5 uses format: function_name\nkey\nvalue\n - static const common_regex tool_call_start("([^\n<]+)"); - static const common_regex arg_key_regex("([^<]+)"); - static const common_regex arg_value_regex("([\\s\\S]*?)"); - static const common_regex tool_call_end(""); + + auto get_expected_type = [&](const std::string& tool_name, const std::string& param_name) -> std::string { + // Access tools schema from builder syntax + const auto& tools_schema = builder.syntax().tools_schema; + if (tools_schema.is_array()) { + for (const auto& tool : tools_schema) { + if (tool.contains("function") && tool["function"]["name"] == tool_name) { + auto params = tool["function"]["parameters"]; + if (params.contains("properties") && params["properties"].contains(param_name)) { + return params["properties"][param_name].value("type", "string"); + } + } + } + } + return "string"; // Default fallback + }; - while (auto res = builder.try_find_regex(tool_call_start)) { - // Move to the start of the tool call and consume it - builder.move_to(res->groups[0].begin); - builder.consume_regex(tool_call_start); + auto handle_tool_call_end = [&] (common_chat_msg_parser & builder, auto end_pos) { + builder.move_to(end_pos); + builder.consume_literal(""); - std::string function_name = builder.str(res->groups[1]); - json arguments = json::object(); + size_t obs_pos = builder.input().find("<|observation|>", builder.pos()); + if (obs_pos != std::string::npos) { + if (obs_pos > builder.pos()) { + std::string content = builder.input().substr(builder.pos(), obs_pos - builder.pos()); + builder.add_content(content); + } + + builder.move_to(obs_pos); + builder.consume_literal("<|observation|>"); + } else { + std::string remaining = builder.consume_rest(); + if (!remaining.empty()) builder.add_content(remaining); + } + }; + + builder.consume_spaces(); + + builder.try_parse_reasoning("", ""); + + size_t curr_pos = builder.pos(); + while (builder.input().find("", builder.pos()) != std::string::npos) { + size_t tool_call_start = builder.input().find("", builder.pos()); + if (tool_call_start > builder.pos()) { + std::string content = builder.input().substr(builder.pos(), tool_call_start - builder.pos()); + builder.add_content(content); + } + size_t tool_call_end = builder.input().find("", tool_call_start); + if (tool_call_end == std::string::npos) return; + + builder.move_to(tool_call_start); + builder.consume_literal(""); builder.consume_spaces(); - - // Parse all arg_key/arg_value pairs - while (auto key_res = builder.try_consume_regex(arg_key_regex)) { - std::string key = builder.str(key_res->groups[1]); - builder.consume_spaces(); + + size_t arg_key_start = builder.input().find("", tool_call_start); + if (arg_key_start == std::string::npos || arg_key_start > tool_call_end) { + std::string function_content = builder.input().substr(builder.pos(), tool_call_end - builder.pos()); + std::string function_name = string_strip(function_content); + + if (!builder.add_tool_call(function_name, "", "{}")) { + LOG_INF("%s: failed to add tool call\n", __func__); + } + + handle_tool_call_end(builder, tool_call_end); + + } else { + std::string function_content = builder.input().substr(builder.pos(), arg_key_start - builder.pos()); + std::string function_name = string_strip(function_content); + + json args_json = json::object(); + builder.move_to(arg_key_start); - if (auto value_res = builder.try_consume_regex(arg_value_regex)) { - std::string value = builder.str(value_res->groups[1]); - arguments[key] = value; + while (builder.pos() < tool_call_end && builder.input().substr(builder.pos()).find("") == 0) { + if (!builder.try_consume_literal("")) break; + + auto key_close = builder.try_find_literal(""); + if (!key_close || key_close->groups[0].end > tool_call_end) { + throw common_chat_msg_partial_exception("incomplete tool call"); + return; + } + + std::string key = string_strip(key_close->prelude); + + builder.consume_spaces(); + + if (!builder.try_consume_literal("")) { + throw common_chat_msg_partial_exception("incomplete tool call"); + return; + } + + auto value_close = builder.try_find_literal(""); + if (!value_close || value_close->groups[0].end > tool_call_end) { + throw common_chat_msg_partial_exception("incomplete tool call"); + return; + } + + std::string value = string_strip(value_close->prelude); + + // Schema-aware type conversion + std::string expected_type = get_expected_type(function_name, key); + json parsed_value; + + if (expected_type == "array" || expected_type == "object") { + try { + parsed_value = json::parse(value); + } catch (...) { + parsed_value = value; + } + } else { + // For all other types, store as string and let the unpacking logic handle it + parsed_value = value; + } + + args_json[key] = parsed_value; builder.consume_spaces(); + } + + if (args_json.size() == 1) { + const auto key = args_json.begin().key(); + auto& value = args_json.begin().value(); + + if (value.is_string()) { + try { + json unpacked_json = json::parse(value.get()); + if (unpacked_json.is_object()) { + args_json = unpacked_json; + } + } catch (const std::exception&) { + // Not a valid JSON string, proceed as normal + } + } + } + + if (!builder.add_tool_call(function_name, "", args_json.dump())) { + LOG_INF("%s: failed to add tool call with arguments\n", __func__); } else { - throw common_chat_msg_partial_exception("Expected after "); + LOG_INF("%s: successfully added tool call with arguments\n", __func__); } + + handle_tool_call_end(builder, tool_call_end); } - - // Consume closing tag - builder.consume_regex(tool_call_end); - builder.consume_spaces(); - - // Add the parsed tool call - if (!builder.add_tool_call(function_name, "", arguments.dump())) { - throw common_chat_msg_partial_exception("Failed to add GLM tool call"); + + if (curr_pos == builder.pos()) { + // No progress made, avoid infinite loop + LOG_INF("%s: no progress in parsing, stopping to avoid infinite loop\n", __func__); + break; } + curr_pos = builder.pos(); } - builder.add_content(builder.consume_rest()); + if (builder.pos() < builder.input().size()) { + builder.add_content(builder.consume_rest()); + } } + static common_chat_params common_chat_params_init_firefunction_v2(const common_chat_template & tmpl, const struct templates_params & inputs) { LOG_DBG("%s\n", __func__); common_chat_params data; diff --git a/common/chat.h b/common/chat.h index f31bd085729a..625b50d05362 100644 --- a/common/chat.h +++ b/common/chat.h @@ -3,6 +3,7 @@ #pragma once #include "common.h" +#include #include #include #include @@ -142,6 +143,7 @@ struct common_chat_params { std::vector grammar_triggers; std::vector preserved_tokens; std::vector additional_stops; + nlohmann::ordered_json tools_schema = nlohmann::ordered_json(); // Schema for tools to pass to parser }; struct common_chat_syntax { @@ -151,6 +153,7 @@ struct common_chat_syntax { bool reasoning_in_content = false; bool thinking_forced_open = false; bool parse_tool_calls = true; + nlohmann::ordered_json tools_schema = nlohmann::ordered_json(); // Schema for tools to enable type-aware parsing }; // Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid diff --git a/models/templates/glm_4_5.jinja b/models/templates/glm_4_5.jinja new file mode 100644 index 000000000000..08fb5d1f97f0 --- /dev/null +++ b/models/templates/glm_4_5.jinja @@ -0,0 +1,119 @@ +[gMASK] +{%- if tools -%} +<|system|> +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{% for tool in tools %} +{{ tool | tojson }} +{% endfor %} + + +For each function call, output the function name and arguments within the following XML format: +{function-name} +{arg-key-1} +{arg-value-1} +{arg-key-2} +{arg-value-2} +... +{%- endif -%} +{%- macro visible_text(content) -%} + {%- if content is string -%} + {{- content }} + {%- elif content is iterable and content is not mapping -%} + {%- for item in content -%} + {%- if item is mapping and item.type == 'text' -%} + {{- item.text }} + {%- elif item is string -%} + {{- item }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- content }} + {%- endif -%} +{%- endmacro -%} +{%- set ns = namespace(last_user_index=-1) %} +{%- for m in messages %} + {%- if m.role == 'user' %} + {% set ns.last_user_index = loop.index0 -%} + {%- endif %} +{%- endfor %} +{% for m in messages %} +{%- if m.role == 'user' -%}<|user|> +{%- set user_content = visible_text(m.content) -%} +{{ user_content }} +{{- '/nothink' if (enable_thinking is defined and not enable_thinking and not user_content.endswith("/nothink")) else '' -}} +{%- elif m.role == 'assistant' -%} +<|assistant|> +{%- set reasoning_content = '' %} +{%- set content = visible_text(m.content) %} +{%- if m.reasoning_content is string %} + {%- set reasoning_content = m.reasoning_content %} +{%- else %} + {%- if '' in content %} + {%- set parts = content.split('') -%} + {%- set before_first_close = parts | first -%} + {%- set inner_parts = before_first_close.rstrip('\n').split('') -%} + {%- set extracted_reasoning = inner_parts | last -%} + {%- set reasoning_content = extracted_reasoning.lstrip('\n') -%} + {%- set after_last_close = parts | last -%} + {%- set content = after_last_close.lstrip('\n') -%} + {%- endif %} +{%- endif %} +{%- if loop.index0 > ns.last_user_index and reasoning_content -%} +{{ '\n' + reasoning_content.strip() + ''}} +{%- else -%} +{{ '\n' }} +{%- endif -%} +{%- if content.strip() -%} +{{ '\n' + content.strip() }} +{%- endif -%} +{% if m.tool_calls %} +{% for tc in m.tool_calls %} +{%- if tc.function %} + {%- set tc_obj = tc.function %} +{%- else %} + {%- set tc_obj = tc %} +{%- endif %} +{{ '\n' + tc_obj.name }} +{%- if tc_obj.arguments is mapping -%} + {%- for k, v in tc_obj.arguments.items() -%} + +{{ k }} +{{ v | tojson if v is not string else v }} + {%- endfor -%} +{%- else -%} + {#- Arguments came as string - this shouldn't happen with polyfills disabled -#} + {#- Output as single argument for debugging -#} + +raw_arguments +{{ tc_obj.arguments }} +{%- endif -%} +{% endfor %} +{% endif %} +{%- elif m.role == 'tool' -%} +{%- if m.content is string -%} +{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|observation|>' }} +{%- endif %} +{{- '\n\n' }} +{{- m.content }} +{{- '\n' }} +{%- else -%} +<|observation|>{% for tr in m.content %} + + +{{ tr.output if tr.output is defined else tr }} +{% endfor -%} +{% endif -%} +{%- elif m.role == 'system' -%} +<|system|> +{{ visible_text(m.content) }} +{%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + <|assistant|>{{- '\n' if (enable_thinking is defined and not enable_thinking) else '' -}} +{%- endif -%} \ No newline at end of file diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 73c98bfa207f..5918ff1f4594 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -294,6 +294,7 @@ static void test_templates(const struct common_chat_templates * tmpls, const std common_chat_syntax syntax; syntax.format = data.params.format; syntax.reasoning_format = reasoning_format; + syntax.tools_schema = data.params.tools_schema; const auto msg = common_chat_parse(data.delta, /* is_partial= */ false, syntax); assert_msg_equals(test_message, msg); } diff --git a/tools/server/server.cpp b/tools/server/server.cpp index a255d481a4d1..a70508b3568e 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -387,6 +387,7 @@ struct server_task { params.oaicompat_chat_syntax.reasoning_in_content = params.stream && (params_base.reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY); params.oaicompat_chat_syntax.thinking_forced_open = json_value(data, "thinking_forced_open", false); params.oaicompat_chat_syntax.parse_tool_calls = json_value(data, "parse_tool_calls", false); + params.oaicompat_chat_syntax.tools_schema = json_value(data, "tools_schema", nlohmann::ordered_json()); } { diff --git a/tools/server/utils.hpp b/tools/server/utils.hpp index f3dfc8225da4..84fb72a6fb5c 100644 --- a/tools/server/utils.hpp +++ b/tools/server/utils.hpp @@ -848,6 +848,9 @@ static json oaicompat_chat_params_parse( } } + // Store tools schema for parser + llama_params["tools_schema"] = chat_params.tools_schema; + return llama_params; } From ab55d7611b50f913e980b3b631af468ce56aa6dd Mon Sep 17 00:00:00 2001 From: Bhavik Dhandhalya <86345824+dhandhalyabhavik@users.noreply.github.com> Date: Tue, 2 Sep 2025 14:18:10 +0530 Subject: [PATCH 7/7] fixed type conversion rare case, it was failing with tool argument type --- common/chat.cpp | 80 ++++++++++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 34 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index e9ce8028726c..3c4fd78fa6a0 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1384,14 +1384,14 @@ static void common_chat_parse_glm_4_5(common_chat_msg_parser & builder) { auto handle_tool_call_end = [&] (common_chat_msg_parser & builder, auto end_pos) { builder.move_to(end_pos); builder.consume_literal(""); - + size_t obs_pos = builder.input().find("<|observation|>", builder.pos()); if (obs_pos != std::string::npos) { if (obs_pos > builder.pos()) { std::string content = builder.input().substr(builder.pos(), obs_pos - builder.pos()); builder.add_content(content); } - + builder.move_to(obs_pos); builder.consume_literal("<|observation|>"); } else { @@ -1401,9 +1401,8 @@ static void common_chat_parse_glm_4_5(common_chat_msg_parser & builder) { }; builder.consume_spaces(); - builder.try_parse_reasoning("", ""); - + size_t curr_pos = builder.pos(); while (builder.input().find("", builder.pos()) != std::string::npos) { size_t tool_call_start = builder.input().find("", builder.pos()); @@ -1411,7 +1410,7 @@ static void common_chat_parse_glm_4_5(common_chat_msg_parser & builder) { std::string content = builder.input().substr(builder.pos(), tool_call_start - builder.pos()); builder.add_content(content); } - + size_t tool_call_end = builder.input().find("", tool_call_start); if (tool_call_end == std::string::npos) return; @@ -1419,73 +1418,88 @@ static void common_chat_parse_glm_4_5(common_chat_msg_parser & builder) { builder.consume_literal(""); builder.consume_spaces(); - size_t arg_key_start = builder.input().find("", tool_call_start); + size_t arg_key_start = builder.input().find("", builder.pos()); if (arg_key_start == std::string::npos || arg_key_start > tool_call_end) { std::string function_content = builder.input().substr(builder.pos(), tool_call_end - builder.pos()); std::string function_name = string_strip(function_content); - + if (!builder.add_tool_call(function_name, "", "{}")) { LOG_INF("%s: failed to add tool call\n", __func__); } - handle_tool_call_end(builder, tool_call_end); - } else { std::string function_content = builder.input().substr(builder.pos(), arg_key_start - builder.pos()); std::string function_name = string_strip(function_content); - + json args_json = json::object(); builder.move_to(arg_key_start); - - while (builder.pos() < tool_call_end && builder.input().substr(builder.pos()).find("") == 0) { + + while (builder.pos() < tool_call_end && builder.input().substr(builder.pos()).rfind("", 0) == 0) { if (!builder.try_consume_literal("")) break; - + auto key_close = builder.try_find_literal(""); if (!key_close || key_close->groups[0].end > tool_call_end) { - throw common_chat_msg_partial_exception("incomplete tool call"); - return; + throw common_chat_msg_partial_exception("incomplete tool call (arg_key)"); } - std::string key = string_strip(key_close->prelude); - + builder.consume_spaces(); - if (!builder.try_consume_literal("")) { - throw common_chat_msg_partial_exception("incomplete tool call"); - return; + throw common_chat_msg_partial_exception("incomplete tool call (arg_value)"); } - + auto value_close = builder.try_find_literal(""); if (!value_close || value_close->groups[0].end > tool_call_end) { - throw common_chat_msg_partial_exception("incomplete tool call"); - return; + throw common_chat_msg_partial_exception("incomplete tool call (arg_value content)"); } - std::string value = string_strip(value_close->prelude); - // Schema-aware type conversion std::string expected_type = get_expected_type(function_name, key); json parsed_value; - if (expected_type == "array" || expected_type == "object") { + if (expected_type == "integer" || expected_type == "number") { + try { + if (value.find('.') != std::string::npos) { + parsed_value = std::stod(value); + } else { + parsed_value = std::stoll(value); + } + } catch (const std::exception&) { + LOG_WRN("%s: Failed to parse '%s' as a number for key '%s', falling back to string.\n", __func__, value.c_str(), key.c_str()); + parsed_value = value; + } + } else if (expected_type == "boolean") { + std::string lower_val = value; + std::transform(lower_val.begin(), lower_val.end(), lower_val.begin(), + [](unsigned char c){ return std::tolower(c); }); + if (lower_val == "true" || lower_val == "1") { + parsed_value = true; + } else if (lower_val == "false" || lower_val == "0") { + parsed_value = false; + } else { + LOG_WRN("%s: Ambiguous boolean value '%s' for key '%s', falling back to string.\n", __func__, value.c_str(), key.c_str()); + parsed_value = value; + } + } else if (expected_type == "array" || expected_type == "object") { try { parsed_value = json::parse(value); - } catch (...) { + } catch (const json::parse_error&) { + LOG_WRN("%s: Failed to parse '%s' as JSON for key '%s', falling back to raw string.\n", __func__, value.c_str(), key.c_str()); parsed_value = value; } } else { - // For all other types, store as string and let the unpacking logic handle it + // Default case is "string". parsed_value = value; } - + args_json[key] = parsed_value; builder.consume_spaces(); } - + + // This is a special case to handle when the model outputs a single JSON object as a string if (args_json.size() == 1) { const auto key = args_json.begin().key(); auto& value = args_json.begin().value(); - if (value.is_string()) { try { json unpacked_json = json::parse(value.get()); @@ -1503,12 +1517,10 @@ static void common_chat_parse_glm_4_5(common_chat_msg_parser & builder) { } else { LOG_INF("%s: successfully added tool call with arguments\n", __func__); } - handle_tool_call_end(builder, tool_call_end); } - + if (curr_pos == builder.pos()) { - // No progress made, avoid infinite loop LOG_INF("%s: no progress in parsing, stopping to avoid infinite loop\n", __func__); break; }