From 8416c69e0e1374951b2fd52a84b5a9d59aac962e Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:21:33 +0200 Subject: [PATCH 1/2] feat(server): structured output with grammar-constrained decoding response_format was ignored: a request for JSON got the model's habits, JSON in a code fence or after a sentence, and tool_choice "required" was only a hint. The reply now follows the schema exactly: - common/gbnf: llama.cpp's GBNF parser and pushdown matcher (b6500, MIT), matching tokens by their decoded text; - server/json_schema_grammar: llama.cpp's JSON Schema to GBNF converter, fed the schema as ordered JSON so properties keep their order; - common/output_constraint: one per request, shared through SamplerCfg::constraint. The sampler draws as usual and, when the grammar rejects the draw, draws again from the tokens it allows (exactly the masked distribution). A redraw at the start of the reply skips blank tokens and tool-call openers, since the rejected draw meant to write the reply. It waits for when the request starts thinking, lets go once the reply opens a tool call (openers from the tool parser), never allows a token whose text holds a NUL, and stops constraining rather than dead-end if a token is injected past the sampler; - the commit points advance it: the scheduler's advance_slot for paged engines, the token callback for the single-request path. A mask holds for one position only, so a constrained request never speculates; - response_format (chat), text.format (responses) and output_format (messages); json_object, json_schema, an unsupported type is a 400; tool_choice required/any/named forces a tool call. Requests without a format or a forced tool call run the code they ran before. Co-Authored-By: Claude Opus 5.5 --- server/CMakeLists.txt | 4 + server/docs/API.md | 11 +- server/src/common/gbnf.cpp | 818 +++++++++++++++ server/src/common/gbnf.h | 120 +++ server/src/common/output_constraint.cpp | 214 ++++ server/src/common/output_constraint.h | 107 ++ server/src/common/sampler.cpp | 39 +- server/src/common/sampler.h | 20 +- server/src/server/http_server.cpp | 173 ++- server/src/server/http_server.h | 29 + server/src/server/json_schema_grammar.cpp | 1045 +++++++++++++++++++ server/src/server/json_schema_grammar.h | 33 + server/src/server/scheduler.cpp | 9 +- server/src/server/tool_parser.cpp | 40 +- server/src/server/tool_parser.h | 4 + server/test/unit/test_output_constraint.cpp | 327 ++++++ 16 files changed, 2958 insertions(+), 35 deletions(-) create mode 100644 server/src/common/gbnf.cpp create mode 100644 server/src/common/gbnf.h create mode 100644 server/src/common/output_constraint.cpp create mode 100644 server/src/common/output_constraint.h create mode 100644 server/src/server/json_schema_grammar.cpp create mode 100644 server/src/server/json_schema_grammar.h create mode 100644 server/test/unit/test_output_constraint.cpp diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index d79af1092c..d7ab0fd957 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -703,6 +703,8 @@ add_library(luce_common STATIC src/qwen35/layer_split_daemon_loop.cpp src/qwen35/qwen35_daemon.cpp src/common/sampler.cpp + src/common/gbnf.cpp + src/common/output_constraint.cpp src/common/backend_precision.cpp src/common/daemon_loop.cpp src/common/gguf_inspect.cpp @@ -731,6 +733,7 @@ add_library(luce_common STATIC src/server/tokenizer.cpp src/server/chat_template.cpp src/server/tool_parser.cpp + src/server/json_schema_grammar.cpp src/server/tool_hint.cpp src/server/tool_memory.cpp src/server/response_error.cpp @@ -2427,6 +2430,7 @@ if(LUCE_TESTS) test/unit/test_mmq_tile_selection.cpp test/unit/test_mixed_mmq_policy.cpp test/unit/test_spec_acceptance.cpp + test/unit/test_output_constraint.cpp test/unit/test_ds4_spec_sampling.cpp test/unit/test_ds4_env.cpp test/unit/test_kvflash_ubatch.cpp diff --git a/server/docs/API.md b/server/docs/API.md index 8fb6ff3d58..3bdac26126 100644 --- a/server/docs/API.md +++ b/server/docs/API.md @@ -80,8 +80,8 @@ Successful requests expose the measured result under `usage.timings`: | `n` | int | — | Number of completions | ❌ TODO | | `logprobs` | bool | — | Return log probabilities | ❌ TODO | | `top_logprobs` | int | — | Number of top logprobs per token | ❌ TODO | -| `response_format` | object | — | JSON mode / structured output | ❌ TODO | -| `tool_choice` | string/object | — | Tool choice / force tool usage | ✅ | +| `response_format` | object | — | Structured output: `json_object`, or `json_schema` with a schema the reply follows exactly (grammar-constrained decoding) | ✅ | +| `tool_choice` | string/object | — | Tool choice; `required` or a named function forces a tool call | ✅ | | `logit_bias` | object | — | Per-token logit adjustments | ❌ TODO | | `user` | string | — | End-user identifier (tracking) | ❌ TODO | | `stream_options` | object | — | Streaming options (e.g. include_usage) | ❌ TODO 🔴 | @@ -124,7 +124,8 @@ Successful requests expose the measured result under `usage.timings`: | `presence_penalty` | float | 0.0 | Penalize present tokens | ✅ | | `thinking` | object | — | Thinking mode (`{"type":"enabled"}`) | ✅ | | `tools` | array | — | Tool definitions | ✅ | -| `tool_choice` | string/object | — | Tool choice / force tool usage | ✅ | +| `tool_choice` | string/object | — | Tool choice; `any` or a named tool forces a tool call | ✅ | +| `output_format` | object | — | Structured output (`{"type":"json_schema","schema":{...}}`) | ✅ | | `stop_sequences` | array | — | Stop sequences | ✅ | | `metadata` | object | — | Request metadata (tracing) | ❌ TODO | @@ -159,7 +160,7 @@ Follows Anthropic Messages API structure with `content` blocks: | `parallel_tool_calls` | bool | — | Allow parallel tool calls | ❌ TODO 🔴 | | `store` | bool | — | Persist response | ❌ TODO 🔴 | | `include` | array | — | Include extra response fields | ❌ TODO 🔴 | -| `text` | object | — | Structured output / JSON schema | ❌ TODO 🔴 | +| `text` | object | — | Structured output (`{"format":{"type":"json_schema","schema":{...}}}`) | ✅ | | `service_tier` | string | — | Routing hint | ❌ TODO 🔴 | | `previous_response_id` | string | — | Multi-turn chaining | ❌ TODO | @@ -247,14 +248,12 @@ support causes errors or silent feature degradation. | **`parallel_tool_calls`** | Codex (Responses API) | Codex always sends `true`. Can accept and ignore (we serialize calls). | | **`store`** | Codex (Responses API) | Controls response persistence. Accept field; can be no-op locally. | | **`include`** | Codex (Responses API) | Controls what's included in response events. Accept field. | -| **`text` (structured output)** | Codex (Responses API) | JSON schema output formatting (`{"format":{"type":"json_schema","schema":{...}}}`). Needed for structured tool outputs. | | **`service_tier`** | Codex (Responses API) | Routing hint (e.g., `"default"`). Accept and ignore. | ### 🟡 Medium Priority | Feature | Used By | Notes | |---------|---------|-------| -| **`response_format`** | Chat Completions API | JSON mode / structured output (OpenAI Chat format). | | **`metadata`** | Claude Code (Anthropic) | Request metadata for tracing. Accept and ignore. | | **`stream_options`** | Some OpenAI clients | `{"include_usage": true}` — usage in final streaming chunk. | | **Input validation** | — | Clamp penalty ranges [-2,2], reject invalid params with 400. | diff --git a/server/src/common/gbnf.cpp b/server/src/common/gbnf.cpp new file mode 100644 index 0000000000..60b4f72aa1 --- /dev/null +++ b/server/src/common/gbnf.cpp @@ -0,0 +1,818 @@ +// GBNF parser and pushdown matcher. See gbnf.h for provenance: the parsing +// and matching below follow llama.cpp src/llama-grammar.cpp (b6500) function +// by function, so upstream fixes port over directly. + +#include "gbnf.h" + +#include +#include + +namespace luce::common::gbnf { + +namespace { + +// NOTE: assumes valid UTF-8 (but checks for overrun). +std::pair decode_utf8_char(const char * src) { + static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; + const uint8_t first_byte = static_cast(*src); + const uint8_t highbits = first_byte >> 4; + const int len = lookup[highbits]; + const uint8_t mask = (1 << (8 - len)) - 1; + uint32_t value = first_byte & mask; + const char * end = src + len; // may overrun! + const char * pos = src + 1; + for (; pos < end && *pos; pos++) { + value = (value << 6) + (static_cast(*pos) & 0x3F); + } + return std::make_pair(value, pos); +} + +bool is_digit_char(char c) { + return '0' <= c && c <= '9'; +} + +bool is_word_char(char c) { + return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '-' || is_digit_char(c); +} + +std::pair parse_hex(const char * src, int size) { + const char * pos = src; + const char * end = src + size; + uint32_t value = 0; + for (; pos < end && *pos; pos++) { + value <<= 4; + const char c = *pos; + if ('a' <= c && c <= 'f') { + value += c - 'a' + 10; + } else if ('A' <= c && c <= 'F') { + value += c - 'A' + 10; + } else if ('0' <= c && c <= '9') { + value += c - '0'; + } else { + break; + } + } + if (pos != end) { + throw std::runtime_error("expecting " + std::to_string(size) + " hex chars at " + src); + } + return std::make_pair(value, pos); +} + +const char * parse_space(const char * src, bool newline_ok) { + const char * pos = src; + while (*pos == ' ' || *pos == '\t' || *pos == '#' || + (newline_ok && (*pos == '\r' || *pos == '\n'))) { + if (*pos == '#') { + while (*pos && *pos != '\r' && *pos != '\n') pos++; + } else { + pos++; + } + } + return pos; +} + +const char * parse_name(const char * src) { + const char * pos = src; + while (is_word_char(*pos)) pos++; + if (pos == src) { + throw std::runtime_error(std::string("expecting name at ") + src); + } + return pos; +} + +const char * parse_int(const char * src) { + const char * pos = src; + while (is_digit_char(*pos)) pos++; + if (pos == src) { + throw std::runtime_error(std::string("expecting integer at ") + src); + } + return pos; +} + +std::pair parse_char(const char * src) { + if (*src == '\\') { + switch (src[1]) { + case 'x': return parse_hex(src + 2, 2); + case 'u': return parse_hex(src + 2, 4); + case 'U': return parse_hex(src + 2, 8); + case 't': return std::make_pair('\t', src + 2); + case 'r': return std::make_pair('\r', src + 2); + case 'n': return std::make_pair('\n', src + 2); + case '\\': + case '"': + case '[': + case ']': + return std::make_pair(src[1], src + 2); + default: + throw std::runtime_error(std::string("unknown escape at ") + src); + } + } else if (*src) { + return decode_utf8_char(src); + } + throw std::runtime_error("unexpected end of input"); +} + +void print_grammar_char(FILE * file, uint32_t c) { + if (0x20 <= c && c <= 0x7f) { + std::fprintf(file, "%c", static_cast(c)); + } else { + std::fprintf(file, "", c); + } +} + +bool is_char_element(Element elem) { + switch (elem.type) { + case ElementType::CHAR: + case ElementType::CHAR_NOT: + case ElementType::CHAR_ALT: + case ElementType::CHAR_RNG_UPPER: + case ElementType::CHAR_ANY: + return true; + default: + return false; + } +} + +void print_rule(FILE * file, uint32_t rule_id, const Rule & rule, + const std::map & symbol_id_names) { + if (rule.empty() || rule.back().type != ElementType::END) { + throw std::runtime_error( + "malformed rule, does not end with END: " + std::to_string(rule_id)); + } + std::fprintf(file, "%s ::= ", symbol_id_names.at(rule_id).c_str()); + for (size_t i = 0, end = rule.size() - 1; i < end; i++) { + const Element elem = rule[i]; + switch (elem.type) { + case ElementType::END: + throw std::runtime_error( + "unexpected end of rule: " + std::to_string(rule_id) + "," + std::to_string(i)); + case ElementType::ALT: + std::fprintf(file, "| "); + break; + case ElementType::RULE_REF: + std::fprintf(file, "%s ", symbol_id_names.at(elem.value).c_str()); + break; + case ElementType::CHAR: + std::fprintf(file, "["); + print_grammar_char(file, elem.value); + break; + case ElementType::CHAR_NOT: + std::fprintf(file, "[^"); + print_grammar_char(file, elem.value); + break; + case ElementType::CHAR_RNG_UPPER: + if (i == 0 || !is_char_element(rule[i - 1])) { + throw std::runtime_error( + "CHAR_RNG_UPPER without preceding char: " + + std::to_string(rule_id) + "," + std::to_string(i)); + } + std::fprintf(file, "-"); + print_grammar_char(file, elem.value); + break; + case ElementType::CHAR_ALT: + if (i == 0 || !is_char_element(rule[i - 1])) { + throw std::runtime_error( + "CHAR_ALT without preceding char: " + + std::to_string(rule_id) + "," + std::to_string(i)); + } + print_grammar_char(file, elem.value); + break; + case ElementType::CHAR_ANY: + std::fprintf(file, "."); + break; + } + if (is_char_element(elem)) { + switch (rule[i + 1].type) { + case ElementType::CHAR_ALT: + case ElementType::CHAR_RNG_UPPER: + case ElementType::CHAR_ANY: + break; + default: + std::fprintf(file, "] "); + } + } + } + std::fprintf(file, "\n"); +} + +// True when pos ends one of the definitions of a rule. +bool is_end_of_sequence(const Element * pos) { + return pos->type == ElementType::END || pos->type == ElementType::ALT; +} + +// Whether chr satisfies the char range at pos (regular or inverse range), +// and the element after the range. pos must point at a char range element. +std::pair match_char(const Element * pos, uint32_t chr) { + bool found = false; + const bool is_positive_char = + pos->type == ElementType::CHAR || pos->type == ElementType::CHAR_ANY; + if (!is_positive_char && pos->type != ElementType::CHAR_NOT) { + throw std::logic_error("gbnf: match_char on a non-char element"); + } + do { + if (pos[1].type == ElementType::CHAR_RNG_UPPER) { + found = found || (pos->value <= chr && chr <= pos[1].value); + pos += 2; + } else if (pos->type == ElementType::CHAR_ANY) { + found = true; + pos += 1; + } else { + found = found || pos->value == chr; + pos += 1; + } + } while (pos->type == ElementType::CHAR_ALT); + return std::make_pair(found == is_positive_char, pos); +} + +// Whether some continuation of the partial UTF-8 sequence could satisfy the +// char range at pos (regular or inverse range). +bool match_partial_char(const Element * pos, PartialUtf8 partial_utf8) { + const bool is_positive_char = + pos->type == ElementType::CHAR || pos->type == ElementType::CHAR_ANY; + const uint32_t partial_value = partial_utf8.value; + const int n_remain = partial_utf8.n_remain; + + // invalid sequence or 7-bit char split across 2 bytes (overlong) + if (n_remain < 0 || (n_remain == 1 && partial_value < 2)) { + return false; + } + + // range of possible code points this partial UTF-8 sequence could complete to + uint32_t low = partial_value << (n_remain * 6); + uint32_t high = low | ((1 << (n_remain * 6)) - 1); + if (low == 0) { + if (n_remain == 2) { + low = 1 << 11; + } else if (n_remain == 3) { + low = 1 << 16; + } + } + + do { + if (pos[1].type == ElementType::CHAR_RNG_UPPER) { + if (pos->value <= high && low <= pos[1].value) return is_positive_char; + pos += 2; + } else if (pos->type == ElementType::CHAR_ANY) { + return true; + } else { + if (low <= pos->value && pos->value <= high) return is_positive_char; + pos += 1; + } + } while (pos->type == ElementType::CHAR_ALT); + + return !is_positive_char; +} + +// Expand a pushdown stack into the stacks that end at a character range +// (terminal element), or the empty stack when the sentence can end there. +void advance_stack(const Rules & rules, const Stack & stack, Stacks & new_stacks) { + if (stack.empty()) { + if (std::find(new_stacks.begin(), new_stacks.end(), stack) == new_stacks.end()) { + new_stacks.emplace_back(stack); + } + return; + } + + const Element * pos = stack.back(); + switch (pos->type) { + case ElementType::RULE_REF: { + const size_t rule_id = static_cast(pos->value); + const Element * subpos = rules[rule_id].data(); + do { + // the stack without the top (pos) + Stack new_stack(stack.begin(), stack.end() - 1); + if (!is_end_of_sequence(pos + 1)) { + // the rule ref is followed by another element + new_stack.push_back(pos + 1); + } + if (!is_end_of_sequence(subpos)) { + // the alternate is nonempty + new_stack.push_back(subpos); + } + advance_stack(rules, new_stack, new_stacks); + while (!is_end_of_sequence(subpos)) subpos++; + if (subpos->type == ElementType::ALT) { + subpos++; // another alternate of this rule + } else { + break; + } + } while (true); + break; + } + case ElementType::CHAR: + case ElementType::CHAR_NOT: + case ElementType::CHAR_ANY: + if (std::find(new_stacks.begin(), new_stacks.end(), stack) == new_stacks.end()) { + new_stacks.emplace_back(stack); + } + break; + default: + // END/ALT or the middle of a char range: a stack never rests there. + throw std::logic_error("gbnf: stack rests on a non-terminal element"); + } +} + +std::vector reject_for_stack(const Rules & rules, const Stack & stack, + const std::vector & candidates); + +std::vector reject_for_stacks(const Rules & rules, const Stacks & stacks, + const std::vector & candidates) { + if (candidates.empty()) return {}; + if (stacks.empty()) return candidates; + auto rejects = reject_for_stack(rules, stacks.front(), candidates); + for (size_t i = 1, size = stacks.size(); i < size; ++i) { + rejects = reject_for_stack(rules, stacks[i], rejects); + } + return rejects; +} + +std::vector reject_for_stack(const Rules & rules, const Stack & stack, + const std::vector & candidates) { + std::vector rejects; + rejects.reserve(candidates.size()); + + if (stack.empty()) { + for (const auto & tok : candidates) { + if (*tok.code_points != 0 || tok.partial.n_remain != 0) { + rejects.push_back(tok); + } + } + return rejects; + } + + const Element * stack_pos = stack.back(); + + std::vector next_candidates; + next_candidates.reserve(candidates.size()); + for (const auto & tok : candidates) { + if (*tok.code_points == 0) { + // End of the token's full code points: reject iff it ended in a + // partial sequence that cannot satisfy this position. + if (tok.partial.n_remain != 0 && !match_partial_char(stack_pos, tok.partial)) { + rejects.push_back(tok); + } + } else if (match_char(stack_pos, *tok.code_points).first) { + next_candidates.push_back({ tok.index, tok.code_points + 1, tok.partial }); + } else { + rejects.push_back(tok); + } + } + + const Element * stack_pos_after = match_char(stack_pos, 0).second; + + // the top of the stack moves to the next element, if any + Stack stack_after(stack.begin(), stack.end() - 1); + if (!is_end_of_sequence(stack_pos_after)) { + stack_after.push_back(stack_pos_after); + } + Stacks next_stacks; + advance_stack(rules, stack_after, next_stacks); + + const auto next_rejects = reject_for_stacks(rules, next_stacks, next_candidates); + for (const auto & tok : next_rejects) { + rejects.push_back({ tok.index, tok.code_points - 1, tok.partial }); + } + return rejects; +} + +bool detect_left_recursion(const Rules & rules, size_t rule_index, + std::vector & rules_visited, + std::vector & rules_in_progress, + std::vector & rules_may_be_empty) { + if (rules_in_progress[rule_index]) return true; + rules_in_progress[rule_index] = true; + + const Rule & rule = rules[rule_index]; + + // Can the rule produce the empty string? + bool at_rule_start = true; + for (size_t i = 0; i < rule.size(); i++) { + if (is_end_of_sequence(&rule[i])) { + if (at_rule_start) { + rules_may_be_empty[rule_index] = true; + break; + } + at_rule_start = true; + } else { + at_rule_start = false; + } + } + + // Recurse into leftmost nonterminals (or next-leftmost while the previous + // nonterminal may be empty). + bool recurse_into_nonterminal = true; + for (size_t i = 0; i < rule.size(); i++) { + if (rule[i].type == ElementType::RULE_REF && recurse_into_nonterminal) { + if (detect_left_recursion(rules, (size_t) rule[i].value, rules_visited, + rules_in_progress, rules_may_be_empty)) { + return true; + } + if (!rules_may_be_empty[(size_t) rule[i].value]) { + recurse_into_nonterminal = false; + } + } else if (is_end_of_sequence(&rule[i])) { + recurse_into_nonterminal = true; + } else { + recurse_into_nonterminal = false; + } + } + + rules_in_progress[rule_index] = false; + rules_visited[rule_index] = true; + return false; +} + +} // namespace + +std::pair, PartialUtf8> decode_utf8( + const std::string & bytes, PartialUtf8 partial_start) { + static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 3, 4 }; + const char * pos = bytes.c_str(); + std::vector code_points; + // common english strings have the same number of codepoints and bytes; + // + 1 for the terminating 0 + code_points.reserve(bytes.size() + 1); + uint32_t value = partial_start.value; + int n_remain = partial_start.n_remain; + + // continue the previous decode, if any + while (*pos != 0 && n_remain > 0) { + const uint8_t next_byte = static_cast(*pos); + if ((next_byte >> 6) != 2) { + // invalid sequence, abort + code_points.push_back(0); + return std::make_pair(std::move(code_points), PartialUtf8{ 0, -1 }); + } + value = (value << 6) + (next_byte & 0x3F); + ++pos; + --n_remain; + } + + if (partial_start.n_remain > 0 && n_remain == 0) { + code_points.push_back(value); + } + + // decode the following sequences, which may end in an incomplete one + while (*pos != 0) { + const uint8_t first_byte = static_cast(*pos); + const uint8_t highbits = first_byte >> 4; + n_remain = lookup[highbits] - 1; + + if (n_remain < 0) { + // invalid sequence, abort + code_points.clear(); + code_points.push_back(0); + return std::make_pair(std::move(code_points), PartialUtf8{ 0, n_remain }); + } + + const uint8_t mask = (1 << (7 - n_remain)) - 1; + value = first_byte & mask; + + ++pos; + while (*pos != 0 && n_remain > 0) { + value = (value << 6) + (static_cast(*pos) & 0x3F); + ++pos; + --n_remain; + } + if (n_remain == 0) { + code_points.push_back(value); + } + } + code_points.push_back(0); + + return std::make_pair(std::move(code_points), PartialUtf8{ value, n_remain }); +} + +uint32_t Parser::get_symbol_id(const char * src, size_t len) { + const uint32_t next_id = static_cast(symbol_ids.size()); + const auto result = symbol_ids.emplace(std::string(src, len), next_id); + return result.first->second; +} + +uint32_t Parser::generate_symbol_id(const std::string & base_name) { + const uint32_t next_id = static_cast(symbol_ids.size()); + symbol_ids[base_name + '_' + std::to_string(next_id)] = next_id; + return next_id; +} + +void Parser::add_rule(uint32_t rule_id, const Rule & rule) { + if (rules.size() <= rule_id) rules.resize(rule_id + 1); + rules[rule_id] = rule; +} + +const char * Parser::parse_alternates(const char * src, const std::string & rule_name, + uint32_t rule_id, bool is_nested) { + Rule rule; + const char * pos = parse_sequence(src, rule_name, rule, is_nested); + while (*pos == '|') { + rule.push_back({ ElementType::ALT, 0 }); + pos = parse_space(pos + 1, true); + pos = parse_sequence(pos, rule_name, rule, is_nested); + } + rule.push_back({ ElementType::END, 0 }); + add_rule(rule_id, rule); + return pos; +} + +const char * Parser::parse_sequence(const char * src, const std::string & rule_name, + Rule & rule, bool is_nested) { + size_t last_sym_start = rule.size(); + const char * pos = src; + + // S{m,n} --> S S S (m times) S'(n-m), S'(x) ::= S S'(x-1) | ... S'(1) ::= S | + // S{m,} --> S S S (m times) S', S' ::= S S' | + // S* = S{0,}, S+ = S{1,}, S? = S{0,1} + auto handle_repetitions = [&](int min_times, int max_times) { + if (last_sym_start == rule.size()) { + throw std::runtime_error(std::string("expecting preceding item to */+/?/{ at ") + pos); + } + Rule prev_rule(rule.begin() + last_sym_start, rule.end()); + if (min_times == 0) { + rule.resize(last_sym_start); + } else { + // repeat the previous elements (min_times - 1) times + for (int i = 1; i < min_times; i++) { + rule.insert(rule.end(), prev_rule.begin(), prev_rule.end()); + } + } + + uint32_t last_rec_rule_id = 0; + const int n_opt = max_times < 0 ? 1 : max_times - min_times; + + Rule rec_rule(prev_rule); + for (int i = 0; i < n_opt; i++) { + rec_rule.resize(prev_rule.size()); + const uint32_t rec_rule_id = generate_symbol_id(rule_name); + if (i > 0 || max_times < 0) { + rec_rule.push_back({ ElementType::RULE_REF, + max_times < 0 ? rec_rule_id : last_rec_rule_id }); + } + rec_rule.push_back({ ElementType::ALT, 0 }); + rec_rule.push_back({ ElementType::END, 0 }); + add_rule(rec_rule_id, rec_rule); + last_rec_rule_id = rec_rule_id; + } + if (n_opt > 0) { + rule.push_back({ ElementType::RULE_REF, last_rec_rule_id }); + } + }; + + while (*pos) { + if (*pos == '"') { // literal string + pos++; + last_sym_start = rule.size(); + while (*pos != '"') { + if (!*pos) throw std::runtime_error("unexpected end of input"); + const auto char_pair = parse_char(pos); + pos = char_pair.second; + rule.push_back({ ElementType::CHAR, char_pair.first }); + } + pos = parse_space(pos + 1, is_nested); + } else if (*pos == '[') { // char range(s) + pos++; + ElementType start_type = ElementType::CHAR; + if (*pos == '^') { + pos++; + start_type = ElementType::CHAR_NOT; + } + last_sym_start = rule.size(); + while (*pos != ']') { + if (!*pos) throw std::runtime_error("unexpected end of input"); + const auto char_pair = parse_char(pos); + pos = char_pair.second; + const ElementType type = last_sym_start < rule.size() + ? ElementType::CHAR_ALT + : start_type; + rule.push_back({ type, char_pair.first }); + if (pos[0] == '-' && pos[1] != ']') { + if (!pos[1]) throw std::runtime_error("unexpected end of input"); + const auto endchar_pair = parse_char(pos + 1); + pos = endchar_pair.second; + rule.push_back({ ElementType::CHAR_RNG_UPPER, endchar_pair.first }); + } + } + pos = parse_space(pos + 1, is_nested); + } else if (is_word_char(*pos)) { // rule reference + const char * name_end = parse_name(pos); + const uint32_t ref_rule_id = get_symbol_id(pos, name_end - pos); + pos = parse_space(name_end, is_nested); + last_sym_start = rule.size(); + rule.push_back({ ElementType::RULE_REF, ref_rule_id }); + } else if (*pos == '(') { // grouping + // nested alternates become a synthesized rule + pos = parse_space(pos + 1, true); + const uint32_t sub_rule_id = generate_symbol_id(rule_name); + pos = parse_alternates(pos, rule_name, sub_rule_id, true); + last_sym_start = rule.size(); + rule.push_back({ ElementType::RULE_REF, sub_rule_id }); + if (*pos != ')') throw std::runtime_error(std::string("expecting ')' at ") + pos); + pos = parse_space(pos + 1, is_nested); + } else if (*pos == '.') { // any char + last_sym_start = rule.size(); + rule.push_back({ ElementType::CHAR_ANY, 0 }); + pos = parse_space(pos + 1, is_nested); + } else if (*pos == '*') { + pos = parse_space(pos + 1, is_nested); + handle_repetitions(0, -1); + } else if (*pos == '+') { + pos = parse_space(pos + 1, is_nested); + handle_repetitions(1, -1); + } else if (*pos == '?') { + pos = parse_space(pos + 1, is_nested); + handle_repetitions(0, 1); + } else if (*pos == '{') { + pos = parse_space(pos + 1, is_nested); + if (!is_digit_char(*pos)) throw std::runtime_error(std::string("expecting an int at ") + pos); + const char * int_end = parse_int(pos); + const int min_times = std::stoul(std::string(pos, int_end - pos)); + pos = parse_space(int_end, is_nested); + + int max_times = -1; + if (*pos == '}') { + max_times = min_times; + pos = parse_space(pos + 1, is_nested); + } else if (*pos == ',') { + pos = parse_space(pos + 1, is_nested); + if (is_digit_char(*pos)) { + const char * max_end = parse_int(pos); + max_times = std::stoul(std::string(pos, max_end - pos)); + pos = parse_space(max_end, is_nested); + } + if (*pos != '}') throw std::runtime_error(std::string("expecting '}' at ") + pos); + pos = parse_space(pos + 1, is_nested); + } else { + throw std::runtime_error(std::string("expecting ',' at ") + pos); + } + handle_repetitions(min_times, max_times); + } else { + break; + } + } + return pos; +} + +const char * Parser::parse_rule(const char * src) { + const char * name_end = parse_name(src); + const char * pos = parse_space(name_end, false); + const size_t name_len = name_end - src; + const uint32_t rule_id = get_symbol_id(src, name_len); + const std::string name(src, name_len); + + if (!(pos[0] == ':' && pos[1] == ':' && pos[2] == '=')) { + throw std::runtime_error(std::string("expecting ::= at ") + pos); + } + pos = parse_space(pos + 3, true); + pos = parse_alternates(pos, name, rule_id, false); + + if (*pos == '\r') { + pos += pos[1] == '\n' ? 2 : 1; + } else if (*pos == '\n') { + pos++; + } else if (*pos) { + throw std::runtime_error(std::string("expecting newline or end at ") + pos); + } + return parse_space(pos, true); +} + +bool Parser::parse(const char * src) { + try { + const char * pos = parse_space(src, true); + while (*pos) pos = parse_rule(pos); + // every referenced rule must be defined + for (const auto & rule : rules) { + if (rule.empty()) throw std::runtime_error("Undefined rule"); + for (const auto & elem : rule) { + if (elem.type != ElementType::RULE_REF) continue; + if (elem.value >= rules.size() || rules[elem.value].empty()) { + for (const auto & kv : symbol_ids) { + if (kv.second == elem.value) { + throw std::runtime_error("Undefined rule identifier '" + kv.first + "'"); + } + } + throw std::runtime_error("Undefined rule"); + } + } + } + } catch (const std::exception & err) { + error = err.what(); + rules.clear(); + return false; + } + return true; +} + +void Parser::print(FILE * file) const { + try { + std::map symbol_id_names; + for (const auto & kv : symbol_ids) symbol_id_names[kv.second] = kv.first; + for (size_t i = 0, end = rules.size(); i < end; i++) { + print_rule(file, uint32_t(i), rules[i], symbol_id_names); + } + } catch (const std::exception & err) { + std::fprintf(stderr, "\ngbnf: error printing grammar: %s\n", err.what()); + } +} + +std::unique_ptr Matcher::compile(const std::string & text, + const std::string & root, + std::string & error) { + Parser parser; + if (!parser.parse(text.c_str()) || parser.rules.empty()) { + error = parser.error.empty() ? "empty grammar" : parser.error; + return nullptr; + } + const auto root_it = parser.symbol_ids.find(root); + if (root_it == parser.symbol_ids.end()) { + error = "grammar does not contain a '" + root + "' rule"; + return nullptr; + } + + const size_t n_rules = parser.rules.size(); + std::vector rules_visited(n_rules); + std::vector rules_in_progress(n_rules); + std::vector rules_may_be_empty(n_rules); + for (size_t i = 0; i < n_rules; i++) { + if (rules_visited[i]) continue; + if (detect_left_recursion(parser.rules, i, rules_visited, rules_in_progress, + rules_may_be_empty)) { + error = "unsupported grammar, left recursion at rule " + std::to_string(i); + return nullptr; + } + } + + std::unique_ptr matcher(new Matcher()); + auto rules = std::make_shared(std::move(parser.rules)); + matcher->symbol_ids_ = std::move(parser.symbol_ids); + + // the initial stacks: one per alternate of the root rule + const Element * pos = (*rules)[root_it->second].data(); + do { + Stack stack; + if (!is_end_of_sequence(pos)) stack.push_back(pos); + advance_stack(*rules, stack, matcher->stacks_); + while (!is_end_of_sequence(pos)) pos++; + if (pos->type == ElementType::ALT) { + pos++; + } else { + break; + } + } while (true); + + // The stacks point into *rules: shared and never mutated, so copies of + // the matcher stay valid. + matcher->rules_ = std::move(rules); + return matcher; +} + +bool Matcher::can_end() const { + if (partial_.n_remain != 0) return false; + for (const auto & stack : stacks_) { + if (stack.empty()) return true; + } + return false; +} + +int Matcher::rule_id(const std::string & name) const { + const auto it = symbol_ids_.find(name); + return it == symbol_ids_.end() ? -1 : (int) it->second; +} + +void Matcher::accept_bytes(const std::string & bytes) { + const auto decoded = decode_utf8(bytes, partial_); + if (decoded.second.n_remain < 0) { + stacks_.clear(); + return; + } + const auto & code_points = decoded.first; + for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { + Stacks stacks_new; + stacks_new.reserve(stacks_.size()); + for (const auto & stack : stacks_) { + if (stack.empty()) continue; + const auto match = match_char(stack.back(), *it); + if (!match.first) continue; + // the top of the stack moves to the next element, if any + Stack new_stack(stack.begin(), stack.end() - 1); + if (!is_end_of_sequence(match.second)) new_stack.push_back(match.second); + advance_stack(*rules_, new_stack, stacks_new); + } + stacks_ = std::move(stacks_new); + if (stacks_.empty()) return; + } + partial_ = decoded.second; + if (partial_.n_remain > 0) { + // The piece ends inside a character: some stack must accept a + // continuation of it, or the text is already off the grammar. + const bool reachable = std::any_of(stacks_.begin(), stacks_.end(), + [&](const Stack & stack) { + return !stack.empty() && match_partial_char(stack.back(), partial_); + }); + if (!reachable) stacks_.clear(); + } +} + +std::vector Matcher::reject(const std::vector & candidates) const { + return reject_for_stacks(*rules_, stacks_, candidates); +} + +} // namespace luce::common::gbnf diff --git a/server/src/common/gbnf.h b/server/src/common/gbnf.h new file mode 100644 index 0000000000..cb2acd5f53 --- /dev/null +++ b/server/src/common/gbnf.h @@ -0,0 +1,120 @@ +// GBNF grammars: the parser and the pushdown matcher that constrained +// decoding runs on (output_constraint.h). +// +// Adapted from llama.cpp src/llama-grammar.{h,cpp} (release b6500), +// MIT License, Copyright (c) 2023-2024 The ggml authors. The grammar syntax +// is llama.cpp's own (grammars/README.md upstream), so a grammar written for +// llama-server works here unchanged. Left out: lazy trigger words and the +// llama_vocab coupling; tokens are matched by their decoded bytes. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace luce::common::gbnf { + +enum class ElementType : uint8_t { + END, // end of rule definition + ALT, // start of an alternate definition of the rule + RULE_REF, // non-terminal: reference to a rule + CHAR, // terminal: code point + CHAR_NOT, // inverse char(s): [^a], [^a-b], [^abc] + CHAR_RNG_UPPER, // makes the preceding CHAR/CHAR_ALT an inclusive range: [a-z] + CHAR_ALT, // adds an alternate char to the preceding CHAR/range: [ab], [a-zA] + CHAR_ANY, // any character: . +}; + +struct Element { + ElementType type; + uint32_t value; // code point or rule id +}; + +struct PartialUtf8 { + uint32_t value = 0; // bit value so far (unshifted) + int n_remain = 0; // bytes still missing; -1 marks an invalid sequence +}; + +using Rule = std::vector; +using Rules = std::vector; +using Stack = std::vector; +using Stacks = std::vector; + +// Decode `bytes` as UTF-8, continuing a sequence an earlier piece left open. +// The code points end with a 0 terminator; the second value is the sequence +// this piece leaves open (n_remain 0 when it ends on a character boundary). +std::pair, PartialUtf8> decode_utf8( + const std::string & bytes, PartialUtf8 start); + +struct Parser { + std::map symbol_ids; + Rules rules; + std::string error; + + // False (with `error` set and `rules` cleared) when the text is not a + // well-formed grammar or references an undefined rule. + bool parse(const char * src); + void print(FILE * file) const; + +private: + uint32_t get_symbol_id(const char * src, size_t len); + uint32_t generate_symbol_id(const std::string & base_name); + void add_rule(uint32_t rule_id, const Rule & rule); + const char * parse_alternates(const char * src, const std::string & rule_name, + uint32_t rule_id, bool is_nested); + const char * parse_sequence(const char * src, const std::string & rule_name, + Rule & rule, bool is_nested); + const char * parse_rule(const char * src); +}; + +// A token under test: its decoded code points (0-terminated) from the +// matcher's current partial UTF-8 state. `index` is the caller's own id. +struct Candidate { + size_t index; + const uint32_t * code_points; + PartialUtf8 partial; +}; + +// One position in a grammar: the rules (shared, immutable) and the pushdown +// stacks every way of reaching this position leaves. Copies share the rules, +// so a copy is a cheap snapshot of the position. +class Matcher { +public: + // Parse `text` and start at rule `root`. Null with `error` set on a parse + // failure, a missing root, or left recursion. + static std::unique_ptr compile(const std::string & text, + const std::string & root, + std::string & error); + + // True when the text so far is a complete sentence of the grammar. + bool can_end() const; + // True when no continuation is possible (the last piece did not match). + bool dead() const { return stacks_.empty(); } + PartialUtf8 partial() const { return partial_; } + const Stacks & stacks() const { return stacks_; } + const Rules & rules() const { return *rules_; } + // Id of a named rule, -1 when absent. + int rule_id(const std::string & name) const; + + // Advance over `bytes`. On a mismatch the matcher becomes dead(). + void accept_bytes(const std::string & bytes); + + // Every candidate that cannot continue the text from this position. + std::vector reject(const std::vector & candidates) const; + +private: + Matcher() = default; + + std::shared_ptr rules_; + std::map symbol_ids_; + Stacks stacks_; + PartialUtf8 partial_; +}; + +} // namespace luce::common::gbnf diff --git a/server/src/common/output_constraint.cpp b/server/src/common/output_constraint.cpp new file mode 100644 index 0000000000..c37e2bd536 --- /dev/null +++ b/server/src/common/output_constraint.cpp @@ -0,0 +1,214 @@ +#include "output_constraint.h" + +#include +#include +#include +#include +#include + +namespace luce::common { + +std::shared_ptr ConstraintVocab::build( + std::vector pieces, const std::vector & end_tokens) { + auto vocab = std::make_shared(); + const size_t n = pieces.size(); + vocab->code_points.resize(n); + vocab->partials.resize(n); + vocab->is_end.assign(n, 0); + for (size_t i = 0; i < n; ++i) { + // The matcher reads text up to a NUL: a piece holding one (the byte + // token for 0x00) would match as if it were shorter, so it gets no + // text at all and the grammar never allows it. + if (pieces[i].find('\0') != std::string::npos) pieces[i].clear(); + auto decoded = gbnf::decode_utf8(pieces[i], {}); + vocab->code_points[i] = std::move(decoded.first); + vocab->partials[i] = decoded.second; + } + for (int32_t token : end_tokens) { + if (token >= 0 && (size_t) token < n) vocab->is_end[(size_t) token] = 1; + } + vocab->pieces = std::move(pieces); + return vocab; +} + +OutputConstraint::OutputConstraint(std::shared_ptr vocab, + std::unique_ptr grammar, + Options options) + : vocab_(std::move(vocab)), + initial_(std::move(grammar)), + options_(std::move(options)) { + reset(); +} + +OutputConstraint::State OutputConstraint::initial_state() const { + return options_.wait_for.empty() ? State::engaged : State::waiting; +} + +void OutputConstraint::reset() { + grammar_ = std::make_unique(*initial_); + state_ = initial_state(); + tail_.clear(); + head_.clear(); + started_ = false; + release_window_ = 0; + for (const auto & prefix : options_.release_prefixes) { + release_window_ = std::max(release_window_, prefix.size()); + } +} + +bool OutputConstraint::allows(int32_t token) const { + if (!engaged()) return true; + if (token < 0 || token >= vocab_->size()) return false; + if (vocab_->is_end[(size_t) token]) return grammar_->can_end(); + const std::string & piece = vocab_->pieces[(size_t) token]; + if (piece.empty()) return false; + const gbnf::PartialUtf8 partial = grammar_->partial(); + if (partial.n_remain == 0) { + return grammar_->reject({{ 0, vocab_->code_points[(size_t) token].data(), + vocab_->partials[(size_t) token] }}).empty(); + } + const auto decoded = gbnf::decode_utf8(piece, partial); + return grammar_->reject({{ 0, decoded.first.data(), decoded.second }}).empty(); +} + +int OutputConstraint::mask(float * logits, int n_vocab, bool content_only) const { + if (!engaged()) return n_vocab; + const bool can_end = grammar_->can_end(); + const gbnf::PartialUtf8 partial = grammar_->partial(); + const bool from_boundary = partial.n_remain == 0; + const int n_known = std::min(n_vocab, vocab_->size()); + + // Tokens past the tokenizer's vocabulary (padding rows) never match. + for (int t = n_known; t < n_vocab; ++t) logits[t] = -INFINITY; + + thread_local std::vector candidates; + thread_local std::vector, gbnf::PartialUtf8>> decoded; + candidates.clear(); + decoded.clear(); + if (!from_boundary) decoded.reserve((size_t) n_known); + for (int t = 0; t < n_known; ++t) { + if (logits[t] == -INFINITY) continue; + if (vocab_->is_end[(size_t) t]) { + if (!can_end) logits[t] = -INFINITY; + continue; + } + const std::string & piece = vocab_->pieces[(size_t) t]; + if (piece.empty()) { + logits[t] = -INFINITY; + continue; + } + if (from_boundary) { + candidates.push_back({ (size_t) t, vocab_->code_points[(size_t) t].data(), + vocab_->partials[(size_t) t] }); + } else { + decoded.push_back(gbnf::decode_utf8(piece, partial)); + candidates.push_back({ (size_t) t, decoded.back().first.data(), + decoded.back().second }); + } + } + for (const auto & reject : grammar_->reject(candidates)) { + logits[reject.index] = -INFINITY; + } + if (content_only && !started_) { + const bool keep_calls = options_.calls_only; + for (const auto & candidate : candidates) { + const size_t t = candidate.index; + if (logits[t] == -INFINITY) continue; + const std::string & piece = vocab_->pieces[t]; + if (piece.find_first_not_of(" \t\r\n") == std::string::npos || + (!keep_calls && starts_release(piece))) { + logits[t] = -INFINITY; + } + } + } + int allowed = 0; + for (int t = 0; t < n_known; ++t) { + if (logits[t] != -INFINITY) ++allowed; + } + return allowed; +} + +void OutputConstraint::accept(int32_t token) { + if (state_ == State::released || state_ == State::broken) return; + const bool known = token >= 0 && token < vocab_->size(); + // An end token ends the output; nothing follows it to constrain. + if (known && vocab_->is_end[(size_t) token]) return; + static const std::string no_text; + const std::string & piece = known ? vocab_->pieces[(size_t) token] : no_text; + if (state_ == State::waiting) { + if (!known) return; + tail_ += piece; + const size_t at = tail_.find(options_.wait_for); + if (at == std::string::npos) { + // Keep enough to see a marker split across tokens. + if (tail_.size() > options_.wait_for.size()) { + tail_.erase(0, tail_.size() - options_.wait_for.size()); + } + return; + } + const std::string rest = tail_.substr(at + options_.wait_for.size()); + tail_.clear(); + state_ = State::engaged; + if (!rest.empty()) feed(rest, token); + return; + } + if (!known || piece.empty()) { + std::fprintf(stderr, + "[constraint] token %d has no text; output is no longer constrained\n", + token); + state_ = State::broken; + return; + } + feed(piece, token); +} + +void OutputConstraint::feed(const std::string & text, int32_t token) { + grammar_->accept_bytes(text); + if (grammar_->dead()) { + // Only a token injected past the sampler (a forced close, a hint) + // gets here: let the request finish unconstrained rather than leave + // it without a legal token. + std::fprintf(stderr, + "[constraint] token %d does not follow the grammar; output is no " + "longer constrained\n", token); + state_ = State::broken; + return; + } + if (!started_ && text.find_first_not_of(" \t\r\n") != std::string::npos) started_ = true; + check_release(text); +} + +bool OutputConstraint::starts_release(const std::string & piece) const { + const std::string text = head_ + piece; + const size_t start = text.find_first_not_of(" \t\r\n"); + if (start == std::string::npos) return false; + const std::string_view output = std::string_view(text).substr(start); + for (const auto & prefix : options_.release_prefixes) { + const size_t n = std::min(output.size(), prefix.size()); + if (output.compare(0, n, prefix, 0, n) == 0) return true; + } + return false; +} + +void OutputConstraint::check_release(const std::string & text) { + if (release_window_ == 0) return; + head_ += text; + const size_t start = head_.find_first_not_of(" \t\r\n"); + if (start == std::string::npos) return; + const std::string_view output = std::string_view(head_).substr(start); + for (const auto & prefix : options_.release_prefixes) { + if (output.size() >= prefix.size() && output.compare(0, prefix.size(), prefix) == 0) { + state_ = State::released; + head_.clear(); + return; + } + } + if (output.size() >= release_window_) { + // Past every opener without matching one: the output is the + // constrained text, not a tool call. + release_window_ = 0; + head_.clear(); + } +} + +} // namespace luce::common diff --git a/server/src/common/output_constraint.h b/server/src/common/output_constraint.h new file mode 100644 index 0000000000..6e433142d7 --- /dev/null +++ b/server/src/common/output_constraint.h @@ -0,0 +1,107 @@ +// Constrained decoding: a request's output follows a grammar (a JSON schema +// from response_format, a forced tool call). Model-neutral. +// +// One OutputConstraint per request, shared through SamplerCfg::constraint: +// - the sampler (sample_logits) draws the request's token as usual and, if +// the grammar rejects it, masks every token the grammar rejects and draws +// again from what is left. Drawing first and masking only on a miss gives +// exactly the masked distribution, and costs one token check per step +// while the model already writes what the grammar wants; +// - the commit point advances it with every token the request keeps: the +// scheduler (advance_slot) for paged engines, the token callback for the +// single-request path. +// Both run on the request's worker thread. Masks are only valid for the next +// position, so a constrained request never speculates: SamplerCfg marks it as +// needing logit processing, the scheduler disables speculation for it and the +// single-request path decodes it autoregressively. + +#pragma once + +#include "gbnf.h" + +#include +#include +#include +#include +#include + +namespace luce::common { + +// Every token's text, decoded once per model and shared by all requests. +struct ConstraintVocab { + std::vector pieces; // decoded bytes per token id + // pieces[i] decoded from a character boundary (0-terminated), and the + // UTF-8 sequence it leaves open + std::vector> code_points; + std::vector partials; + std::vector is_end; // end-of-generation tokens + + static std::shared_ptr build( + std::vector pieces, const std::vector & end_tokens); + + int size() const { return (int) pieces.size(); } +}; + +class OutputConstraint { +public: + struct Options { + // The request starts inside a thinking block: the constraint waits + // until the output has closed it with this text (``); what + // follows it in the same token is constrained. + std::string wait_for; + // Openers of the tool-call syntax. Once the output (after leading + // whitespace) starts with one, the request is calling a tool and the + // constraint lets go; the grammar admits them for that hand-over. + std::vector release_prefixes; + // The grammar admits only a tool call (tool_choice "required"), so a + // redraw never keeps to a reply there is none of. + bool calls_only = false; + }; + + OutputConstraint(std::shared_ptr vocab, + std::unique_ptr grammar, + Options options); + + // Constraining the next token: not waiting, released, finished or broken. + bool engaged() const { return state_ == State::engaged; } + bool released() const { return state_ == State::released; } + bool broken() const { return state_ == State::broken; } + // The output so far is a complete sentence of the grammar. + bool complete() const { return grammar_ && grammar_->can_end(); } + + bool allows(int32_t token) const; + // Set every token the grammar rejects to -inf; returns how many remain. + // `content_only` is for a redraw after the request drew a token the + // grammar rejects (a code fence, a sentence): it meant to start writing. + // Before the output has started it also drops blank tokens, which would + // only postpone the choice, and the tokens that would start a tool call + // when the grammar admits a reply. + int mask(float * logits, int n_vocab, bool content_only = false) const; + // Advance over a token the request kept. A token the grammar rejects + // (one injected past the sampler) breaks the constraint: it stops + // constraining instead of leaving the request with no legal token. + void accept(int32_t token); + // Back to the start, for a request that decodes again from its prompt. + void reset(); + +private: + enum class State { waiting, engaged, released, broken }; + + State initial_state() const; + void feed(const std::string & text, int32_t token); + void check_release(const std::string & text); + // The output with `piece` appended would begin a tool-call opener. + bool starts_release(const std::string & piece) const; + + std::shared_ptr vocab_; + std::unique_ptr initial_; + std::unique_ptr grammar_; + Options options_; + State state_ = State::engaged; + std::string tail_; // output tail while waiting + std::string head_; // output start, for release detection + size_t release_window_ = 0; // longest release prefix + bool started_ = false; // the output has more than whitespace +}; + +} // namespace luce::common diff --git a/server/src/common/sampler.cpp b/server/src/common/sampler.cpp index 8ca688a731..80665fd5da 100644 --- a/server/src/common/sampler.cpp +++ b/server/src/common/sampler.cpp @@ -2,6 +2,8 @@ #include "sampler.h" +#include "output_constraint.h" + #include #include #include @@ -135,13 +137,11 @@ void apply_sampler_penalties(const SamplerCfg & cfg, const std::vector } } -} // namespace - -int sample_logits(const float * logits_in, - int vocab, - const SamplerCfg & cfg, - const std::vector & history, - std::mt19937_64 & rng) { +int sample_logits_chain(const float * logits_in, + int vocab, + const SamplerCfg & cfg, + const std::vector & history, + std::mt19937_64 & rng) { // Draw the single uniform up front, exactly once, and only when we will // actually sample (temp>0; greedy returns before any draw). It is then // threaded through every path below — the full-GPU draw, the GPU-assisted @@ -261,6 +261,30 @@ int sample_logits(const float * logits_in, return draw_from_weights(cand, r_uniform); } +} // namespace + +int sample_logits(const float * logits_in, + int vocab, + const SamplerCfg & cfg, + const std::vector & history, + std::mt19937_64 & rng) { + const int token = sample_logits_chain(logits_in, vocab, cfg, history, rng); + OutputConstraint * constraint = cfg.constraint.get(); + if (!constraint || !constraint->engaged() || constraint->allows(token)) return token; + // The grammar rejects the draw: draw again from the tokens it allows. + // Drawing first and masking only on a miss leaves each allowed token + // with exactly its masked probability. A rejected draw is the request + // writing its reply, so the redraw keeps to the reply and leaves a tool + // call out, unless a tool call is all the grammar allows. + thread_local std::vector masked; + masked.assign(logits_in, logits_in + vocab); + if (constraint->mask(masked.data(), vocab, /*content_only=*/true) == 0) { + masked.assign(logits_in, logits_in + vocab); + if (constraint->mask(masked.data(), vocab) == 0) return token; + } + return sample_logits_chain(masked.data(), vocab, cfg, history, rng); +} + void sampler_distribution(const float * logits_in, int vocab, const SamplerCfg & cfg, @@ -275,6 +299,7 @@ void sampler_distribution(const float * logits_in, thread_local std::vector z; thread_local std::vector zd; z.assign(logits_in, logits_in + vocab); + if (cfg.constraint && cfg.constraint->engaged()) cfg.constraint->mask(z.data(), vocab); apply_sampler_penalties(cfg, history, vocab, [&](int t) -> float & { return z[(size_t) t]; }); if (cfg.temp <= 0.0f) { const int best = (int) (std::max_element(z.begin(), z.end()) - z.begin()); diff --git a/server/src/common/sampler.h b/server/src/common/sampler.h index 05c963f1dd..e795585b72 100644 --- a/server/src/common/sampler.h +++ b/server/src/common/sampler.h @@ -12,6 +12,7 @@ #pragma once #include +#include #include #include #include @@ -19,6 +20,8 @@ namespace luce::common { +class OutputConstraint; + struct SamplerCfg { float temp = 0.0f; float top_p = 1.0f; @@ -34,15 +37,23 @@ struct SamplerCfg { float freq_pen = 0.0f; float pres_pen = 0.0f; - // True when any logit modifier is active (penalties or stochastic sampling). - // Backends should use CPU sample_logits() path when this returns true. + // Structured output (common/output_constraint.h): the request's tokens + // follow a grammar. Shared by every copy of the config, so the sampler + // and the commit point see one position. + std::shared_ptr constraint; + + // True when any logit modifier is active (penalties, stochastic sampling + // or a grammar). Backends should use CPU sample_logits() path when this + // returns true. bool needs_logit_processing() const { - return temp > 0.0f || rep_pen > 1.0f || freq_pen != 0.0f || pres_pen != 0.0f; + return temp > 0.0f || rep_pen > 1.0f || freq_pen != 0.0f || pres_pen != 0.0f || + constraint != nullptr; } }; // Returns the chosen token id. cfg.temp == 0 -> caller should use argmax; // the chain assumes a positive temperature and falls back to a small floor. +// With cfg.constraint the token is one the grammar allows at its position. int sample_logits(const float * logits_in, int vocab, const SamplerCfg & cfg, @@ -52,7 +63,8 @@ int sample_logits(const float * logits_in, // The distribution sample_logits draws from, after the whole chain (penalties, // top_k, temperature, top_p) as (probability, token) pairs summing to 1. With // temp <= 0 it is the single post-penalty argmax. Speculative sampling uses it -// to accept a drafted token with the target's own probability. +// to accept a drafted token with the target's own probability. A constraint +// masks the distribution at its current position only. void sampler_distribution(const float * logits_in, int vocab, const SamplerCfg & cfg, diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index e71d34f108..976097d2c7 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -26,6 +26,8 @@ #include "sse_emitter.h" #include "prompt_normalize.h" #include "tool_hint.h" +#include "tool_parser.h" +#include "json_schema_grammar.h" #include "pin_friendly_prompt.h" #include "common/kv_rotation.h" #include "common/sha1.h" @@ -2200,6 +2202,165 @@ json HttpServer::model_routing_status() { {"queue_limit", config_.routing_queue_limit}, {"models", models}}; } +// ─── Structured output ────────────────────────────────────────────────── + +namespace http_detail { + +nlohmann::ordered_json request_output_format(const std::string & body) { + if (body.find("\"response_format\"") == std::string::npos && + body.find("\"output_format\"") == std::string::npos && + body.find("\"format\"") == std::string::npos) { + return nullptr; + } + const auto ordered = nlohmann::ordered_json::parse(body, nullptr, false); + if (!ordered.is_object()) return nullptr; + if (ordered.contains("response_format")) return ordered["response_format"]; + if (ordered.contains("output_format")) return ordered["output_format"]; + const auto text = ordered.find("text"); + if (text != ordered.end() && text->is_object() && text->contains("format")) { + return (*text)["format"]; + } + return nullptr; +} + +bool output_format_schema(const nlohmann::ordered_json & format, + nlohmann::ordered_json & schema, std::string & error) { + schema = nullptr; + if (format.is_null()) return true; + if (!format.is_object()) { + error = "response_format must be an object"; + return false; + } + const std::string type = format.value("type", std::string()); + if (type.empty() || type == "text") return true; + if (type == "json_object") { + // A schema beside json_object narrows it, as in llama-server. + schema = format.contains("schema") ? format["schema"] + : nlohmann::ordered_json::object(); + } else if (type == "json_schema") { + // Chat completions nest the schema under json_schema; responses and + // messages put it on the format itself. + const auto & holder = format.contains("json_schema") ? format["json_schema"] : format; + schema = holder.is_object() && holder.contains("schema") + ? holder["schema"] : nlohmann::ordered_json::object(); + } else { + error = "response_format type must be text, json_object or json_schema, not '" + + type + "'"; + return false; + } + if (!schema.is_object()) { + error = "the response_format schema must be a JSON object"; + return false; + } + if (schema.empty()) schema["type"] = "object"; // an empty schema means any object + return true; +} + +bool tool_choice_requires_call(const json & tool_choice) { + if (tool_choice.is_string()) { + return tool_choice == "required" || tool_choice == "any"; + } + if (!tool_choice.is_object()) return false; + const std::string type = tool_choice.value("type", std::string()); + return type == "any" || type == "tool" || type == "function" || + tool_choice.contains("function"); +} + +static std::string gbnf_literal(const std::string & text) { + std::string out = "\""; + for (const char c : text) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: out += c; + } + } + return out + "\""; +} + +std::string output_constraint_grammar(const nlohmann::ordered_json & schema, + const std::vector & tool_openers) { + return build_grammar([&](const GrammarBuilder & builder) { + std::string replies; + if (!schema.is_null()) { + auto resolved = schema; + builder.resolve_refs(resolved); + replies = builder.add_schema("reply", resolved); + } + if (!tool_openers.empty()) { + std::string openers; + for (const auto & opener : tool_openers) { + if (!openers.empty()) openers += " | "; + openers += gbnf_literal(opener); + } + // The grammar only admits the opener; the tool parser reads the + // call, and the constraint lets go of the output once it starts. + if (!replies.empty()) replies += " | "; + replies += builder.add_rule("tool-call", "(" + openers + ") .*"); + } + builder.add_rule("root", "[ \\t\\n]{0,4} (" + replies + ")"); + }); +} + +} // namespace http_detail + +std::shared_ptr HttpServer::output_constraint_vocab() { + std::call_once(constraint_vocab_once_, [this] { + std::vector pieces((size_t) tokenizer_.vocab_size()); + for (int32_t id = 0; id < tokenizer_.vocab_size(); ++id) { + pieces[(size_t) id] = tokenizer_.token_text(id); + } + constraint_vocab_ = ConstraintVocab::build( + std::move(pieces), {tokenizer_.eos_id(), tokenizer_.eos_chat_id()}); + }); + return constraint_vocab_; +} + +bool HttpServer::attach_output_constraint(SocketHandle fd, ParsedRequest & req) { + nlohmann::ordered_json schema; + std::string error; + if (!http_detail::output_format_schema(req.output_format, schema, error)) { + send_error(fd, 400, error); + return false; + } + const bool tools_off = req.tool_choice == "none" || + (req.tool_choice.is_object() && req.tool_choice.value("type", "") == "none"); + const bool has_tools = req.tools.is_array() && !req.tools.empty() && !tools_off; + const bool must_call = has_tools && http_detail::tool_choice_requires_call(req.tool_choice); + if (schema.is_null() && !must_call) return true; + + // With tools the reply may call one first; a forced call must. + std::vector openers = has_tools ? tool_call_openers(req.tools) + : std::vector{}; + std::string grammar; + try { + grammar = http_detail::output_constraint_grammar( + must_call ? nlohmann::ordered_json() : schema, openers); + } catch (const std::exception & e) { + send_error(fd, 400, std::string("response_format: ") + e.what()); + return false; + } + auto matcher = gbnf::Matcher::compile(grammar, "root", error); + if (!matcher) { + send_error(fd, 400, "response_format: " + error); + return false; + } + OutputConstraint::Options options; + if (req.started_in_thinking) { + options.wait_for = config_.think_marker_token_ids.empty() + ? std::string("") + : tokenizer_.decode(config_.think_marker_token_ids); + } + options.release_prefixes = std::move(openers); + options.calls_only = must_call; + req.sampler.constraint = std::make_shared( + output_constraint_vocab(), std::move(matcher), std::move(options)); + return true; +} + // ─── Request parsing ──────────────────────────────────────────────────── // Fields shared by every endpoint: stream/model/max-tokens, sampler, @@ -2596,7 +2757,7 @@ void HttpServer::log_parsed_request(const ParsedRequest & req) const { std::fprintf(stderr, "[server] chat %s format=%s stream=%s msgs=%zu tools=%zu prompt_tokens=%zu " "max_tokens=%d max_ctx=%d thinking=%s reasoning_effort=%s " - "started_in_thinking=%s stops=%zu model=%s\n", + "started_in_thinking=%s stops=%zu grammar=%s model=%s\n", req.response_id.c_str(), api_format_name(req.format), req.stream ? "true" : "false", json_array_size(req.messages), json_array_size(req.tools), @@ -2606,7 +2767,8 @@ void HttpServer::log_parsed_request(const ParsedRequest & req) const { (req.reasoning_effort.empty() ? "default" : req.reasoning_effort.c_str()), req.started_in_thinking ? "true" : "false", - req.stop_sequences.size(), req.model.c_str()); + req.stop_sequences.size(), req.sampler.constraint ? "on" : "off", + req.model.c_str()); } RoutingAdmission HttpServer::enqueue_request_and_wait(SocketHandle fd, ParsedRequest req, @@ -2654,6 +2816,7 @@ bool HttpServer::route_request(SocketHandle fd, const HttpRequest & hr) { bool count_tokens_only = false; try { req.raw_body = json::parse(hr.body); + req.output_format = http_detail::request_output_format(hr.body); if (!parse_endpoint_request( hr.path, req.raw_body, req, count_tokens_only)) return false; return models_.empty() @@ -2757,6 +2920,7 @@ bool HttpServer::handle_model_request(SocketHandle fd, ParsedRequest & req, send_response(fd, 200, "application/json", response.dump() + "\n"); return true; } + if (!attach_output_constraint(fd, req)) return true; } catch (const json::parse_error & e) { send_error(fd, 400, std::string("JSON parse error: ") + e.what()); return true; @@ -4593,6 +4757,8 @@ void HttpServer::prepare_generation_inputs( inputs.request.n_gen = inputs.generation_cap; inputs.request.sampler = req.sampler; inputs.request.do_sample = req.sampler.needs_logit_processing(); + // A grammar masks one position at a time: no speculative blocks. + inputs.request.force_ar_decode = req.sampler.constraint != nullptr; // Tokens are delivered through DaemonIO so all API formats share the // same disconnect and streaming state machine. inputs.request.stream = false; @@ -4662,6 +4828,9 @@ void HttpServer::configure_generation_io( io.on_token = [this, job, &req, &emitter, &output]( int32_t token) -> bool { + // The grammar moves with every token the request keeps, before the + // backend samples the next one. + if (req.sampler.constraint) req.sampler.constraint->accept(token); if (output.client_disconnected || job->client_disconnected.load(std::memory_order_acquire)) { output.client_disconnected = true; diff --git a/server/src/server/http_server.h b/server/src/server/http_server.h index 0e9d9a029b..264b8c043c 100644 --- a/server/src/server/http_server.h +++ b/server/src/server/http_server.h @@ -17,6 +17,7 @@ #include "client_send_buffer.h" #include "common/model_backend.h" #include "common/concurrency/paged_kv_offload.h" +#include "common/output_constraint.h" #include "tokenizer.h" #include "chat_template.h" #include "tool_memory.h" @@ -292,6 +293,23 @@ bool should_clamp_flowkv_disk_cache( // Agent clients append system notes (a context budget, reminders) after // the tool results; those notes are append-only history too. bool ends_with_tool_result(const std::vector & messages); +// Structured output. The format object a request asks its output to follow: +// response_format (chat completions), text.format (responses) or +// output_format (messages); null when it asks for none. Read from the body +// text as ordered JSON, so the grammar keeps the schema's property order. +nlohmann::ordered_json request_output_format(const std::string & body); +// The JSON schema a format object asks for. False with `error` set for an +// unsupported format type; true with a null schema for plain text. +bool output_format_schema(const nlohmann::ordered_json & format, + nlohmann::ordered_json & schema, std::string & error); +// Whether tool_choice obliges the reply to call a tool ("required", or a +// named function in any of the three APIs). +bool tool_choice_requires_call(const json & tool_choice); +// The grammar for a constrained reply: optional leading whitespace, then the +// schema's JSON (when a schema is given) or a tool call that starts with one +// of `tool_openers` (when given). Throws on a schema it cannot convert. +std::string output_constraint_grammar(const nlohmann::ordered_json & schema, + const std::vector & tool_openers); bool canonical_turn_matches_checkpoint( const std::vector & prompt, const std::vector & completed_turn, @@ -344,6 +362,9 @@ struct ParsedRequest { json messages; // Original request body (for upstream proxy forwarding) json raw_body; + // Structured output format (http_detail::request_output_format), null + // when the request asks for none. + nlohmann::ordered_json output_format; // Response ID std::string response_id; // Thinking/reasoning state @@ -648,6 +669,12 @@ class HttpServer { std::string & rendered, std::string & error); bool validate_request_context(SocketHandle fd, const ParsedRequest & req, bool send_failure = true); + // Constrain the reply to the request's structured output format and + // tool_choice (common/output_constraint.h). False after sending a 400 + // for a format it cannot serve. + bool attach_output_constraint(SocketHandle fd, ParsedRequest & req); + // Every token's text, built on first use and shared by all requests. + std::shared_ptr output_constraint_vocab(); void log_parsed_request(const ParsedRequest & req) const; RoutingAdmission enqueue_request_and_wait(SocketHandle fd, ParsedRequest req, bool report_admission = false); @@ -678,6 +705,8 @@ class HttpServer { Tokenizer * drafter_tokenizer_ = nullptr; // pflash drafter (optional) ServerConfig config_; ChatFormat chat_format_; + std::once_flag constraint_vocab_once_; + std::shared_ptr constraint_vocab_; PFlashDrafterIpcClient pflash_remote_; ToolMemory tool_memory_; PrefixCache prefix_cache_; diff --git a/server/src/server/json_schema_grammar.cpp b/server/src/server/json_schema_grammar.cpp new file mode 100644 index 0000000000..9534a3ab6a --- /dev/null +++ b/server/src/server/json_schema_grammar.cpp @@ -0,0 +1,1045 @@ +// JSON Schema -> GBNF grammar, for structured output (response_format and +// its equivalents) and anything else that constrains output to a schema. +// +// Adapted from llama.cpp common/json-schema-to-grammar.cpp (release b6500), +// MIT License, Copyright (c) 2023-2024 The ggml authors. The conversion is +// upstream's own, so a schema behaves here as it does in llama-server. +// Changes: the luce::common namespace, the llguidance path left out, and +// the string helpers from llama.cpp's common.h inlined below. + +#include "json_schema_grammar.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace luce::common { + +namespace { + +using json = nlohmann::ordered_json; + +std::string string_repeat(const std::string & str, size_t n) { + std::string result; + result.reserve(str.length() * n); + for (size_t i = 0; i < n; ++i) result += str; + return result; +} + +std::string string_join(const std::vector & values, const std::string & separator) { + std::ostringstream result; + for (size_t i = 0; i < values.size(); ++i) { + if (i > 0) result << separator; + result << values[i]; + } + return result.str(); +} + +std::vector string_split(const std::string & str, const std::string & delimiter) { + std::vector parts; + size_t start = 0; + size_t end = str.find(delimiter); + while (end != std::string::npos) { + parts.push_back(str.substr(start, end - start)); + start = end + delimiter.length(); + end = str.find(delimiter, start); + } + parts.push_back(str.substr(start)); + return parts; +} + +static std::string build_repetition(const std::string & item_rule, int min_items, int max_items, const std::string & separator_rule = "") { + auto has_max = max_items != std::numeric_limits::max(); + + if (max_items == 0) { + return ""; + } + if (min_items == 0 && max_items == 1) { + return item_rule + "?"; + } + + if (separator_rule.empty()) { + if (min_items == 1 && !has_max) { + return item_rule + "+"; + } else if (min_items == 0 && !has_max) { + return item_rule + "*"; + } else { + return item_rule + "{" + std::to_string(min_items) + "," + (has_max ? std::to_string(max_items) : "") + "}"; + } + } + + auto result = item_rule + " " + build_repetition("(" + separator_rule + " " + item_rule + ")", min_items == 0 ? 0 : min_items - 1, has_max ? max_items - 1 : max_items); + if (min_items == 0) { + result = "(" + result + ")?"; + } + return result; +} + +static void _build_min_max_int(int min_value, int max_value, std::stringstream & out, int decimals_left = 16, bool top_level = true) { + auto has_min = min_value != std::numeric_limits::min(); + auto has_max = max_value != std::numeric_limits::max(); + + auto digit_range = [&](char from, char to) { + out << "["; + if (from == to) { + out << from; + } else { + out << from << "-" << to; + } + out << "]"; + }; + auto more_digits = [&](int min_digits, int max_digits) { + out << "[0-9]"; + if (min_digits == max_digits && min_digits == 1) { + return; + } + out << "{"; + out << min_digits; + if (max_digits != min_digits) { + out << ","; + if (max_digits != std::numeric_limits::max()) { + out << max_digits; + } + } + out << "}"; + }; + std::function uniform_range = + [&](const std::string_view & from, const std::string_view & to) { + size_t i = 0; + while (i < from.length() && i < to.length() && from[i] == to[i]) { + i++; + } + if (i > 0) { + out << "\"" << from.substr(0, i) << "\""; + } + if (i < from.length() && i < to.length()) { + if (i > 0) { + out << " "; + } + auto sub_len = from.length() - i - 1; + if (sub_len > 0) { + auto from_sub = from.substr(i + 1); + auto to_sub = to.substr(i + 1); + auto sub_zeros = string_repeat("0", sub_len); + auto sub_nines = string_repeat("9", sub_len); + + auto to_reached = false; + out << "("; + if (from_sub == sub_zeros) { + digit_range(from[i], to[i] - 1); + out << " "; + more_digits(sub_len, sub_len); + } else { + out << "[" << from[i] << "] "; + out << "("; + uniform_range(from_sub, sub_nines); + out << ")"; + if (from[i] < to[i] - 1) { + out << " | "; + if (to_sub == sub_nines) { + digit_range(from[i] + 1, to[i]); + to_reached = true; + } else { + digit_range(from[i] + 1, to[i] - 1); + } + out << " "; + more_digits(sub_len, sub_len); + } + } + if (!to_reached) { + out << " | "; + digit_range(to[i], to[i]); + out << " "; + uniform_range(sub_zeros, to_sub); + } + out << ")"; + } else { + out << "[" << from[i] << "-" << to[i] << "]"; + } + } + }; + + if (has_min && has_max) { + if (min_value < 0 && max_value < 0) { + out << "\"-\" ("; + _build_min_max_int(-max_value, -min_value, out, decimals_left, /* top_level= */ true); + out << ")"; + return; + } + + if (min_value < 0) { + out << "\"-\" ("; + _build_min_max_int(0, -min_value, out, decimals_left, /* top_level= */ true); + out << ") | "; + min_value = 0; + } + + auto min_s = std::to_string(min_value); + auto max_s = std::to_string(max_value); + auto min_digits = min_s.length(); + auto max_digits = max_s.length(); + + for (auto digits = min_digits; digits < max_digits; digits++) { + uniform_range(min_s, string_repeat("9", digits)); + min_s = "1" + string_repeat("0", digits); + out << " | "; + } + uniform_range(min_s, max_s); + return; + } + + auto less_decimals = std::max(decimals_left - 1, 1); + + if (has_min) { + if (min_value < 0) { + out << "\"-\" ("; + _build_min_max_int(std::numeric_limits::min(), -min_value, out, decimals_left, /* top_level= */ false); + out << ") | [0] | [1-9] "; + more_digits(0, decimals_left - 1); + } else if (min_value == 0) { + if (top_level) { + out << "[0] | [1-9] "; + more_digits(0, less_decimals); + } else { + more_digits(1, decimals_left); + } + } else if (min_value <= 9) { + char c = '0' + min_value; + auto range_start = top_level ? '1' : '0'; + if (c > range_start) { + digit_range(range_start, c - 1); + out << " "; + more_digits(1, less_decimals); + out << " | "; + } + digit_range(c, '9'); + out << " "; + more_digits(0, less_decimals); + } else { + auto min_s = std::to_string(min_value); + auto len = min_s.length(); + auto c = min_s[0]; + + if (c > '1') { + digit_range(top_level ? '1' : '0', c - 1); + out << " "; + more_digits(len, less_decimals); + out << " | "; + } + digit_range(c, c); + out << " ("; + _build_min_max_int(std::stoi(min_s.substr(1)), std::numeric_limits::max(), out, less_decimals, /* top_level= */ false); + out << ")"; + if (c < '9') { + out << " | "; + digit_range(c + 1, '9'); + out << " "; + more_digits(len - 1, less_decimals); + } + } + return; + } + + if (has_max) { + if (max_value >= 0) { + if (top_level) { + out << "\"-\" [1-9] "; + more_digits(0, less_decimals); + out << " | "; + } + _build_min_max_int(0, max_value, out, decimals_left, /* top_level= */ true); + } else { + out << "\"-\" ("; + _build_min_max_int(-max_value, std::numeric_limits::max(), out, decimals_left, /* top_level= */ false); + out << ")"; + } + return; + } + + throw std::runtime_error("At least one of min_value or max_value must be set"); +} + +const std::string SPACE_RULE = "| \" \" | \"\\n\"{1,2} [ \\t]{0,20}"; + +struct BuiltinRule { + std::string content; + std::vector deps; +}; + +std::unordered_map PRIMITIVE_RULES = { + {"boolean", {"(\"true\" | \"false\") space", {}}}, + {"decimal-part", {"[0-9]{1,16}", {}}}, + {"integral-part", {"[0] | [1-9] [0-9]{0,15}", {}}}, + {"number", {"(\"-\"? integral-part) (\".\" decimal-part)? ([eE] [-+]? integral-part)? space", {"integral-part", "decimal-part"}}}, + {"integer", {"(\"-\"? integral-part) space", {"integral-part"}}}, + {"value", {"object | array | string | number | boolean | null", {"object", "array", "string", "number", "boolean", "null"}}}, + {"object", {"\"{\" space ( string \":\" space value (\",\" space string \":\" space value)* )? \"}\" space", {"string", "value"}}}, + {"array", {"\"[\" space ( value (\",\" space value)* )? \"]\" space", {"value"}}}, + {"uuid", {"\"\\\"\" [0-9a-fA-F]{8} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{12} \"\\\"\" space", {}}}, + {"char", {"[^\"\\\\\\x7F\\x00-\\x1F] | [\\\\] ([\"\\\\bfnrt] | \"u\" [0-9a-fA-F]{4})", {}}}, + {"string", {"\"\\\"\" char* \"\\\"\" space", {"char"}}}, + {"null", {"\"null\" space", {}}}, +}; + +std::unordered_map STRING_FORMAT_RULES = { + {"date", {"[0-9]{4} \"-\" ( \"0\" [1-9] | \"1\" [0-2] ) \"-\" ( \"0\" [1-9] | [1-2] [0-9] | \"3\" [0-1] )", {}}}, + {"time", {"([01] [0-9] | \"2\" [0-3]) \":\" [0-5] [0-9] \":\" [0-5] [0-9] ( \".\" [0-9]{3} )? ( \"Z\" | ( \"+\" | \"-\" ) ( [01] [0-9] | \"2\" [0-3] ) \":\" [0-5] [0-9] )", {}}}, + {"date-time", {"date \"T\" time", {"date", "time"}}}, + {"date-string", {"\"\\\"\" date \"\\\"\" space", {"date"}}}, + {"time-string", {"\"\\\"\" time \"\\\"\" space", {"time"}}}, + {"date-time-string", {"\"\\\"\" date-time \"\\\"\" space", {"date-time"}}} +}; + +static bool is_reserved_name(const std::string & name) { + static const std::unordered_set RESERVED_NAMES = [] { + std::unordered_set s; + s.insert("root"); + for (const auto & p : PRIMITIVE_RULES) s.insert(p.first); + for (const auto & p : STRING_FORMAT_RULES) s.insert(p.first); + return s; + }(); + return RESERVED_NAMES.find(name) != RESERVED_NAMES.end(); +} + +std::regex INVALID_RULE_CHARS_RE("[^a-zA-Z0-9-]+"); +std::regex GRAMMAR_LITERAL_ESCAPE_RE("[\r\n\"]"); +std::regex GRAMMAR_RANGE_LITERAL_ESCAPE_RE("[\r\n\"\\]\\-\\\\]"); +std::unordered_map GRAMMAR_LITERAL_ESCAPES = { + {'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"} +}; + +std::unordered_set NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?'}; +std::unordered_set ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = {'^', '$', '.', '[', ']', '(', ')', '|', '{', '}', '*', '+', '?'}; + +static std::string replacePattern(const std::string & input, const std::regex & regex, const std::function & replacement) { + std::smatch match; + std::string result; + + std::string::const_iterator searchStart(input.cbegin()); + std::string::const_iterator searchEnd(input.cend()); + + while (std::regex_search(searchStart, searchEnd, match, regex)) { + result.append(searchStart, searchStart + match.position()); + result.append(replacement(match)); + searchStart = match.suffix().first; + } + + result.append(searchStart, searchEnd); + + return result; +} + +static std::string format_literal(const std::string & literal) { + std::string escaped = replacePattern(literal, GRAMMAR_LITERAL_ESCAPE_RE, [&](const std::smatch & match) { + char c = match.str()[0]; + return GRAMMAR_LITERAL_ESCAPES.at(c); + }); + return "\"" + escaped + "\""; +} + +class SchemaConverter { +public: + std::function _fetch_json; + bool _dotall; + std::map _rules; + std::unordered_map _refs; + std::unordered_set _refs_being_resolved; + std::vector _errors; + std::vector _warnings; + + std::string _add_rule(const std::string & name, const std::string & rule) { + std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-"); + if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) { + _rules[esc_name] = rule; + return esc_name; + } else { + int i = 0; + while (_rules.find(esc_name + std::to_string(i)) != _rules.end() && _rules[esc_name + std::to_string(i)] != rule) { + i++; + } + std::string key = esc_name + std::to_string(i); + _rules[key] = rule; + return key; + } + } + + std::string _generate_union_rule(const std::string & name, const std::vector & alt_schemas) { + std::vector rules; + for (size_t i = 0; i < alt_schemas.size(); i++) { + rules.push_back(visit(alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i))); + } + return string_join(rules, " | "); + } + + std::string _visit_pattern(const std::string & pattern, const std::string & name) { + if (!(pattern.front() == '^' && pattern.back() == '$')) { + _errors.push_back("Pattern must start with '^' and end with '$'"); + return ""; + } + std::string sub_pattern = pattern.substr(1, pattern.length() - 2); + std::unordered_map sub_rule_ids; + + size_t i = 0; + size_t length = sub_pattern.length(); + + using literal_or_rule = std::pair; + auto to_rule = [&](const literal_or_rule & ls) { + auto is_literal = ls.second; + auto s = ls.first; + return is_literal ? "\"" + s + "\"" : s; + }; + std::function transform = [&]() -> literal_or_rule { + size_t start = i; + std::vector seq; + + auto get_dot = [&]() { + std::string rule; + if (_dotall) { + rule = "[\\U00000000-\\U0010FFFF]"; + } else { + rule = "[^\\x0A\\x0D]"; + } + return _add_rule("dot", rule); + }; + + // Joins the sequence, merging consecutive literals together. + auto join_seq = [&]() { + std::vector ret; + + std::string literal; + auto flush_literal = [&]() { + if (literal.empty()) { + return false; + } + ret.emplace_back(literal, true); + literal.clear(); + return true; + }; + + for (const auto & item : seq) { + auto is_literal = item.second; + if (is_literal) { + literal += item.first; + } else { + flush_literal(); + ret.push_back(item); + } + } + flush_literal(); + + std::vector results; + for (const auto & item : ret) { + results.push_back(to_rule(item)); + } + return std::make_pair(string_join(results, " "), false); + }; + + while (i < length) { + char c = sub_pattern[i]; + if (c == '.') { + seq.emplace_back(get_dot(), false); + i++; + } else if (c == '(') { + i++; + if (i < length) { + if (sub_pattern[i] == '?') { + _warnings.push_back("Unsupported pattern syntax"); + } + } + seq.emplace_back("(" + to_rule(transform()) + ")", false); + } else if (c == ')') { + i++; + if (start > 0 && sub_pattern[start - 1] != '(') { + _errors.push_back("Unbalanced parentheses"); + } + return join_seq(); + } else if (c == '[') { + std::string square_brackets = std::string(1, c); + i++; + while (i < length && sub_pattern[i] != ']') { + if (sub_pattern[i] == '\\') { + square_brackets += sub_pattern.substr(i, 2); + i += 2; + } else { + square_brackets += sub_pattern[i]; + i++; + } + } + if (i >= length) { + _errors.push_back("Unbalanced square brackets"); + } + square_brackets += ']'; + i++; + seq.emplace_back(square_brackets, false); + } else if (c == '|') { + seq.emplace_back("|", false); + i++; + } else if (c == '*' || c == '+' || c == '?') { + seq.back() = std::make_pair(to_rule(seq.back()) + c, false); + i++; + } else if (c == '{') { + std::string curly_brackets = std::string(1, c); + i++; + while (i < length && sub_pattern[i] != '}') { + curly_brackets += sub_pattern[i]; + i++; + } + if (i >= length) { + _errors.push_back("Unbalanced curly brackets"); + } + curly_brackets += '}'; + i++; + auto nums = string_split(curly_brackets.substr(1, curly_brackets.length() - 2), ","); + int min_times = 0; + int max_times = std::numeric_limits::max(); + try { + if (nums.size() == 1) { + min_times = max_times = std::stoi(nums[0]); + } else if (nums.size() != 2) { + _errors.push_back("Wrong number of values in curly brackets"); + } else { + if (!nums[0].empty()) { + min_times = std::stoi(nums[0]); + } + if (!nums[1].empty()) { + max_times = std::stoi(nums[1]); + } + } + } catch (const std::invalid_argument & e) { + _errors.push_back("Invalid number in curly brackets"); + return std::make_pair("", false); + } + auto &last = seq.back(); + auto &sub = last.first; + auto sub_is_literal = last.second; + + if (!sub_is_literal) { + std::string & sub_id = sub_rule_ids[sub]; + if (sub_id.empty()) { + sub_id = _add_rule(name + "-" + std::to_string(sub_rule_ids.size()), sub); + } + sub = sub_id; + } + seq.back().first = build_repetition( + sub_is_literal ? "\"" + sub + "\"" : sub, + min_times, + max_times, + "" + ); + seq.back().second = false; + } else { + std::string literal; + auto is_non_literal = [&](char c) { + return NON_LITERAL_SET.find(c) != NON_LITERAL_SET.end(); + }; + while (i < length) { + if (sub_pattern[i] == '\\' && i < length - 1) { + char next = sub_pattern[i + 1]; + if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.find(next) != ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.end()) { + i++; + literal += sub_pattern[i]; + i++; + } else { + literal += sub_pattern.substr(i, 2); + i += 2; + } + } else if (sub_pattern[i] == '"') { + literal += "\\\""; + i++; + } else if (!is_non_literal(sub_pattern[i]) && + (i == length - 1 || literal.empty() || sub_pattern[i + 1] == '.' || !is_non_literal(sub_pattern[i + 1]))) { + literal += sub_pattern[i]; + i++; + } else { + break; + } + } + if (!literal.empty()) { + seq.emplace_back(literal, true); + } + } + } + return join_seq(); + }; + return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\" space"); + } + + /* + Returns a rule that matches a JSON string that is none of the provided strings + + not_strings({"a"}) + -> ["] ( [a] char+ | [^"a] char* )? ["] space + not_strings({"and", "also"}) + -> ["] ( [a] ([l] ([s] ([o] char+ | [^"o] char*) | [^"s] char*) | [n] ([d] char+ | [^"d] char*) | [^"ln] char*) | [^"a] char* )? ["] space + */ + std::string _not_strings(const std::vector & strings) { + + struct TrieNode { + std::map children; + bool is_end_of_string; + + TrieNode() : is_end_of_string(false) {} + + void insert(const std::string & string) { + auto node = this; + for (char c : string) { + node = &node->children[c]; + } + node->is_end_of_string = true; + } + }; + + TrieNode trie; + for (const auto & s : strings) { + trie.insert(s); + } + + std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char")); + std::ostringstream out; + out << "[\"] ( "; + std::function visit = [&](const TrieNode & node) { + std::ostringstream rejects; + auto first = true; + for (const auto & kv : node.children) { + rejects << kv.first; + if (first) { + first = false; + } else { + out << " | "; + } + out << "[" << kv.first << "]"; + if (!kv.second.children.empty()) { + out << " ("; + visit(kv.second); + out << ")"; + } else if (kv.second.is_end_of_string) { + out << " " << char_rule << "+"; + } + } + if (!node.children.empty()) { + if (!first) { + out << " | "; + } + out << "[^\"" << rejects.str() << "] " << char_rule << "*"; + } + }; + visit(trie); + + out << " )"; + if (!trie.is_end_of_string) { + out << "?"; + } + out << " [\"] space"; + return out.str(); + } + + std::string _resolve_ref(const std::string & ref) { + std::string ref_name = ref.substr(ref.find_last_of('/') + 1); + if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref) == _refs_being_resolved.end()) { + _refs_being_resolved.insert(ref); + json resolved = _refs[ref]; + ref_name = visit(resolved, ref_name); + _refs_being_resolved.erase(ref); + } + return ref_name; + } + + std::string _build_object_rule( + const std::vector> & properties, + const std::unordered_set & required, + const std::string & name, + const json & additional_properties) + { + std::vector required_props; + std::vector optional_props; + std::unordered_map prop_kv_rule_names; + std::vector prop_names; + for (const auto & kv : properties) { + const auto &prop_name = kv.first; + const auto &prop_schema = kv.second; + + std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name); + prop_kv_rule_names[prop_name] = _add_rule( + name + (name.empty() ? "" : "-") + prop_name + "-kv", + format_literal(json(prop_name).dump()) + " space \":\" space " + prop_rule_name + ); + if (required.find(prop_name) != required.end()) { + required_props.push_back(prop_name); + } else { + optional_props.push_back(prop_name); + } + prop_names.push_back(prop_name); + } + if ((additional_properties.is_boolean() && additional_properties.get()) || additional_properties.is_object()) { + std::string sub_name = name + (name.empty() ? "" : "-") + "additional"; + std::string value_rule = + additional_properties.is_object() ? visit(additional_properties, sub_name + "-value") + : _add_primitive("value", PRIMITIVE_RULES.at("value")); + + auto key_rule = + prop_names.empty() ? _add_primitive("string", PRIMITIVE_RULES.at("string")) + : _add_rule(sub_name + "-k", _not_strings(prop_names)); + std::string kv_rule = _add_rule(sub_name + "-kv", key_rule + " \":\" space " + value_rule); + prop_kv_rule_names["*"] = kv_rule; + optional_props.push_back("*"); + } + + std::string rule = "\"{\" space "; + for (size_t i = 0; i < required_props.size(); i++) { + if (i > 0) { + rule += " \",\" space "; + } + rule += prop_kv_rule_names[required_props[i]]; + } + + if (!optional_props.empty()) { + rule += " ("; + if (!required_props.empty()) { + rule += " \",\" space ( "; + } + + std::function &, bool)> get_recursive_refs = [&](const std::vector & ks, bool first_is_optional) { + std::string res; + if (ks.empty()) { + return res; + } + std::string k = ks[0]; + std::string kv_rule_name = prop_kv_rule_names[k]; + std::string comma_ref = "( \",\" space " + kv_rule_name + " )"; + if (first_is_optional) { + res = comma_ref + (k == "*" ? "*" : "?"); + } else { + res = kv_rule_name + (k == "*" ? " " + comma_ref + "*" : ""); + } + if (ks.size() > 1) { + res += " " + _add_rule( + name + (name.empty() ? "" : "-") + k + "-rest", + get_recursive_refs(std::vector(ks.begin() + 1, ks.end()), true) + ); + } + return res; + }; + + for (size_t i = 0; i < optional_props.size(); i++) { + if (i > 0) { + rule += " | "; + } + rule += get_recursive_refs(std::vector(optional_props.begin() + i, optional_props.end()), false); + } + if (!required_props.empty()) { + rule += " )"; + } + rule += " )?"; + } + + rule += " \"}\" space"; + + return rule; + } + + std::string _add_primitive(const std::string & name, const BuiltinRule & rule) { + auto n = _add_rule(name, rule.content); + for (const auto & dep : rule.deps) { + BuiltinRule dep_rule; + auto it = PRIMITIVE_RULES.find(dep); + if (it == PRIMITIVE_RULES.end()) { + it = STRING_FORMAT_RULES.find(dep); + if (it == STRING_FORMAT_RULES.end()) { + _errors.push_back("Rule " + dep + " not known"); + continue; + } + } + if (_rules.find(dep) == _rules.end()) { + _add_primitive(dep, it->second); + } + } + return n; + } + +public: + SchemaConverter( + const std::function & fetch_json, + bool dotall) + : _fetch_json(fetch_json), _dotall(dotall) + { + _rules["space"] = SPACE_RULE; + } + + void resolve_refs(json & schema, const std::string & url) { + /* + * Resolves all $ref fields in the given schema, fetching any remote schemas, + * replacing each $ref with absolute reference URL and populates _refs with the + * respective referenced (sub)schema dictionaries. + */ + std::function visit_refs = [&](json & n) { + if (n.is_array()) { + for (auto & x : n) { + visit_refs(x); + } + } else if (n.is_object()) { + if (n.contains("$ref")) { + std::string ref = n["$ref"]; + if (_refs.find(ref) == _refs.end()) { + json target; + if (ref.find("https://") == 0) { + std::string base_url = ref.substr(0, ref.find('#')); + auto it = _refs.find(base_url); + if (it != _refs.end()) { + target = it->second; + } else { + // Fetch the referenced schema and resolve its refs + auto referenced = _fetch_json(ref); + resolve_refs(referenced, base_url); + _refs[base_url] = referenced; + } + if (ref.find('#') == std::string::npos || ref.substr(ref.find('#') + 1).empty()) { + return; + } + } else if (ref.find("#/") == 0) { + target = schema; + n["$ref"] = url + ref; + ref = url + ref; + } else { + _errors.push_back("Unsupported ref: " + ref); + return; + } + std::string pointer = ref.substr(ref.find('#') + 1); + std::vector tokens = string_split(pointer, "/"); + for (size_t i = 1; i < tokens.size(); ++i) { + std::string sel = tokens[i]; + if (target.is_null() || !target.contains(sel)) { + _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump()); + return; + } + target = target[sel]; + } + _refs[ref] = target; + } + } else { + for (auto & kv : n.items()) { + visit_refs(kv.value()); + } + } + } + }; + + visit_refs(schema); + } + + std::string _generate_constant_rule(const json & value) { + return format_literal(value.dump()); + } + + std::string visit(const json & schema, const std::string & name) { + json schema_type = schema.contains("type") ? schema["type"] : json(); + std::string schema_format = schema.contains("format") ? schema["format"].get() : ""; + std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name; + + if (schema.contains("$ref")) { + return _add_rule(rule_name, _resolve_ref(schema["$ref"])); + } else if (schema.contains("oneOf") || schema.contains("anyOf")) { + std::vector alt_schemas = schema.contains("oneOf") ? schema["oneOf"].get>() : schema["anyOf"].get>(); + return _add_rule(rule_name, _generate_union_rule(name, alt_schemas)); + } else if (schema_type.is_array()) { + std::vector schema_types; + for (const auto & t : schema_type) { + json schema_copy(schema); + schema_copy["type"] = t; + schema_types.push_back(schema_copy); + } + return _add_rule(rule_name, _generate_union_rule(name, schema_types)); + } else if (schema.contains("const")) { + return _add_rule(rule_name, _generate_constant_rule(schema["const"]) + " space"); + } else if (schema.contains("enum")) { + std::vector enum_values; + for (const auto & v : schema["enum"]) { + enum_values.push_back(_generate_constant_rule(v)); + } + return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ") space"); + } else if ((schema_type.is_null() || schema_type == "object") + && (schema.contains("properties") || + (schema.contains("additionalProperties") && schema["additionalProperties"] != true))) { + std::unordered_set required; + if (schema.contains("required") && schema["required"].is_array()) { + for (const auto & item : schema["required"]) { + if (item.is_string()) { + required.insert(item.get()); + } + } + } + std::vector> properties; + if (schema.contains("properties")) { + for (const auto & prop : schema["properties"].items()) { + properties.emplace_back(prop.key(), prop.value()); + } + } + return _add_rule(rule_name, + _build_object_rule( + properties, required, name, + schema.contains("additionalProperties") ? schema["additionalProperties"] : json())); + } else if ((schema_type.is_null() || schema_type == "object" || schema_type == "string") && schema.contains("allOf")) { + std::unordered_set required; + std::vector> properties; + std::map enum_values; + std::string hybrid_name = name; + std::function add_component = [&](const json & comp_schema, bool is_required) { + if (comp_schema.contains("$ref")) { + add_component(_refs[comp_schema["$ref"]], is_required); + } else if (comp_schema.contains("properties")) { + for (const auto & prop : comp_schema["properties"].items()) { + properties.emplace_back(prop.key(), prop.value()); + if (is_required) { + required.insert(prop.key()); + } + } + } else if (comp_schema.contains("enum")) { + for (const auto & v : comp_schema["enum"]) { + const auto rule = _generate_constant_rule(v); + if (enum_values.find(rule) == enum_values.end()) { + enum_values[rule] = 0; + } + enum_values[rule] += 1; + } + } else { + // todo warning + } + }; + for (auto & t : schema["allOf"]) { + if (t.contains("anyOf")) { + for (auto & tt : t["anyOf"]) { + add_component(tt, false); + } + } else { + add_component(t, true); + } + } + if (!enum_values.empty()) { + std::vector enum_intersection; + for (const auto & p : enum_values) { + if (p.second == schema["allOf"].size()) { + enum_intersection.push_back(p.first); + } + } + if (!enum_intersection.empty()) { + return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ") space"); + } + } + return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json())); + } else if ((schema_type.is_null() || schema_type == "array") && (schema.contains("items") || schema.contains("prefixItems"))) { + json items = schema.contains("items") ? schema["items"] : schema["prefixItems"]; + if (items.is_array()) { + std::string rule = "\"[\" space "; + for (size_t i = 0; i < items.size(); i++) { + if (i > 0) { + rule += " \",\" space "; + } + rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i)); + } + rule += " \"]\" space"; + return _add_rule(rule_name, rule); + } else { + std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item"); + int min_items = schema.contains("minItems") ? schema["minItems"].get() : 0; + json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json(); + int max_items = max_items_json.is_number_integer() ? max_items_json.get() : std::numeric_limits::max(); + + return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " \"]\" space"); + } + } else if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) { + return _visit_pattern(schema["pattern"], rule_name); + } else if ((schema_type.is_null() || schema_type == "string") && std::regex_match(schema_format, std::regex("^uuid[1-5]?$"))) { + return _add_primitive(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid")); + } else if ((schema_type.is_null() || schema_type == "string") && STRING_FORMAT_RULES.find(schema_format + "-string") != STRING_FORMAT_RULES.end()) { + auto prim_name = schema_format + "-string"; + return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name))); + } else if (schema_type == "string" && (schema.contains("minLength") || schema.contains("maxLength"))) { + std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char")); + int min_len = schema.contains("minLength") ? schema["minLength"].get() : 0; + int max_len = schema.contains("maxLength") ? schema["maxLength"].get() : std::numeric_limits::max(); + return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\" space"); + } else if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) { + int min_value = std::numeric_limits::min(); + int max_value = std::numeric_limits::max(); + if (schema.contains("minimum")) { + min_value = schema["minimum"].get(); + } else if (schema.contains("exclusiveMinimum")) { + min_value = schema["exclusiveMinimum"].get() + 1; + } + if (schema.contains("maximum")) { + max_value = schema["maximum"].get(); + } else if (schema.contains("exclusiveMaximum")) { + max_value = schema["exclusiveMaximum"].get() - 1; + } + std::stringstream out; + out << "("; + _build_min_max_int(min_value, max_value, out); + out << ") space"; + return _add_rule(rule_name, out.str()); + } else if (schema.empty() || schema_type == "object") { + return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object"))); + } else { + if (!schema_type.is_string() || PRIMITIVE_RULES.find(schema_type.get()) == PRIMITIVE_RULES.end()) { + _errors.push_back("Unrecognized schema: " + schema.dump()); + return ""; + } + // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero + return _add_primitive(rule_name == "root" ? "root" : schema_type.get(), PRIMITIVE_RULES.at(schema_type.get())); + } + } + + void check_errors() { + if (!_errors.empty()) { + throw std::runtime_error("JSON schema conversion failed:\n" + string_join(_errors, "\n")); + } + if (!_warnings.empty()) { + fprintf(stderr, "WARNING: JSON schema conversion was incomplete: %s\n", string_join(_warnings, "; ").c_str()); + } + } + + std::string format_grammar() { + std::stringstream ss; + for (const auto & kv : _rules) { + ss << kv.first << " ::= " << kv.second << std::endl; + } + return ss.str(); + } +}; + +} // namespace + +std::string json_schema_to_grammar(const nlohmann::ordered_json & schema) { + return build_grammar([&](const GrammarBuilder & callbacks) { + auto copy = schema; + callbacks.resolve_refs(copy); + callbacks.add_schema("", copy); + }); +} + +std::string build_grammar(const std::function & cb, + const GrammarOptions & options) { + SchemaConverter converter([&](const std::string &) { return json(); }, options.dotall); + GrammarBuilder builder { + /* .add_rule = */ [&](const std::string & name, const std::string & rule) { + return converter._add_rule(name, rule); + }, + /* .add_schema = */ [&](const std::string & name, const nlohmann::ordered_json & schema) { + return converter.visit(schema, name == "root" ? "" : name); + }, + /* .resolve_refs = */ [&](nlohmann::ordered_json & schema) { + converter.resolve_refs(schema, ""); + }, + }; + cb(builder); + converter.check_errors(); + return converter.format_grammar(); +} + +} // namespace luce::common diff --git a/server/src/server/json_schema_grammar.h b/server/src/server/json_schema_grammar.h new file mode 100644 index 0000000000..456f2ff6de --- /dev/null +++ b/server/src/server/json_schema_grammar.h @@ -0,0 +1,33 @@ +// JSON Schema -> GBNF grammar (common/gbnf.h). Adapted from llama.cpp's +// json-schema-to-grammar; see the .cpp for provenance. + +#pragma once + +#include + +#include +#include + +namespace luce::common { + +// The GBNF grammar whose `root` rule matches exactly the JSON documents the +// schema describes (with the whitespace llama.cpp allows between tokens). +// Throws std::runtime_error when the schema cannot be converted. +std::string json_schema_to_grammar(const nlohmann::ordered_json & schema); + +// Compose a grammar from schemas and hand-written rules: add_schema returns +// the rule name for a schema, add_rule adds (or reuses) a named rule. +struct GrammarBuilder { + std::function add_rule; + std::function add_schema; + std::function resolve_refs; +}; + +struct GrammarOptions { + bool dotall = false; +}; + +std::string build_grammar(const std::function & cb, + const GrammarOptions & options = {}); + +} // namespace luce::common diff --git a/server/src/server/scheduler.cpp b/server/src/server/scheduler.cpp index 0d228e306a..c3797b121d 100644 --- a/server/src/server/scheduler.cpp +++ b/server/src/server/scheduler.cpp @@ -10,6 +10,7 @@ #include "http_server.h" #include "common/concurrency/seq_engine.h" +#include "common/output_constraint.h" #include "parallel_prefix_txn.h" #include "response_error.h" @@ -215,6 +216,10 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { // slot — on EOS, gen cap, stop-sequence hit, or degenerate repetition. auto advance_slot = [&](SchedSlot & s, int32_t tok) { tok = s.budget.apply(tok).token; + // The grammar moves with the token fed back, the budget's included. + if (OutputConstraint * constraint = s.job->req.sampler.constraint.get()) { + constraint->accept(tok); + } s.gen_tokens.push_back(tok); const bool cont = deliver_generation_token( s.job, s.job->req, *s.emitter, tok, s.completion_tokens, @@ -884,8 +889,10 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { const auto & s = slots[(size_t)i]; if (!s.job || parked(i)) continue; if (!s.prefilling) { + // A grammar masks one position at a time: no bursts. step_plan.decode.push_back( - {i, s.pending_tok, s.budget.settled()}); + {i, s.pending_tok, + s.budget.settled() && !s.job->req.sampler.constraint}); } else { prefill_candidates.push_back({i, s.admission_order}); } diff --git a/server/src/server/tool_parser.cpp b/server/src/server/tool_parser.cpp index 390a96808a..f6f134de88 100644 --- a/server/src/server/tool_parser.cpp +++ b/server/src/server/tool_parser.cpp @@ -104,6 +104,14 @@ static std::string declared_tool_name(const json & tool) { return valid_tool_name(name) ? name : std::string(); } +// Literal openers of the tool-call forms above. ARG_KEY_OPEN is not one: it +// sits after the tool name it belongs to. +static const char * const TOOL_OPENERS[] = { + TOOL_OPEN, TOOL_CALLS_OPEN, DSML_PREFIX, INVOKE_OPEN, FUNCTION_CALL_OPEN, + FUNCTION_CALLS_OPEN, FUNCTION_OPEN, BARE_FUNCTION_OPEN, FUNCTION_SPACE_OPEN, + FUNCNAME_OPEN, TOOL_CODE_OPEN, ATTRIBUTE_PARAMETER_OPEN, +}; + static bool declared_tool_open_at(const std::string & text, size_t pos, const json & tools) { if (!tools.is_array()) return false; @@ -116,25 +124,27 @@ static bool declared_tool_open_at(const std::string & text, size_t pos, return false; } +std::vector tool_call_openers(const json & tools) { + std::vector openers(std::begin(TOOL_OPENERS), std::end(TOOL_OPENERS)); + if (tools.is_array()) { + for (const auto & tool : tools) { + const std::string name = declared_tool_name(tool); + if (!name.empty()) openers.push_back("<" + name + ">"); + } + } + return openers; +} + bool find_tool_syntax_start(const std::string & text, const json & tools, size_t & pos) { size_t idx = text.find('<'); while (idx != std::string::npos) { - if (text.compare(idx, sizeof(TOOL_OPEN) - 1, TOOL_OPEN) == 0 || - text.compare(idx, sizeof(TOOL_CALLS_OPEN) - 1, TOOL_CALLS_OPEN) == 0 || - text.compare(idx, sizeof(DSML_PREFIX) - 1, DSML_PREFIX) == 0 || - text.compare(idx, sizeof(INVOKE_OPEN) - 1, INVOKE_OPEN) == 0 || - text.compare(idx, sizeof(FUNCTION_CALL_OPEN) - 1, FUNCTION_CALL_OPEN) == 0 || - text.compare(idx, sizeof(FUNCTION_CALLS_OPEN) - 1, FUNCTION_CALLS_OPEN) == 0 || - text.compare(idx, sizeof(FUNCTION_OPEN) - 1, FUNCTION_OPEN) == 0 || - text.compare(idx, sizeof(BARE_FUNCTION_OPEN) - 1, BARE_FUNCTION_OPEN) == 0 || - text.compare(idx, sizeof(FUNCTION_SPACE_OPEN) - 1, - FUNCTION_SPACE_OPEN) == 0 || - text.compare(idx, sizeof(FUNCNAME_OPEN) - 1, FUNCNAME_OPEN) == 0 || - text.compare(idx, sizeof(TOOL_CODE_OPEN) - 1, TOOL_CODE_OPEN) == 0 || - text.compare(idx, sizeof(ATTRIBUTE_PARAMETER_OPEN) - 1, - ATTRIBUTE_PARAMETER_OPEN) == 0 || - declared_tool_open_at(text, idx, tools)) { + const bool opener = std::any_of( + std::begin(TOOL_OPENERS), std::end(TOOL_OPENERS), + [&](const char * open) { + return text.compare(idx, std::strlen(open), open) == 0; + }); + if (opener || declared_tool_open_at(text, idx, tools)) { pos = idx; return true; } diff --git a/server/src/server/tool_parser.h b/server/src/server/tool_parser.h index 35e3299bbe..a9e32c5f8b 100644 --- a/server/src/server/tool_parser.h +++ b/server/src/server/tool_parser.h @@ -42,6 +42,10 @@ struct ToolParseResult { bool find_tool_syntax_start(const std::string & text, const json & tools, size_t & pos); +// The literal openers find_tool_syntax_start recognizes (declared tools' +// `` included): text starting with one of them is a tool call. +std::vector tool_call_openers(const json & tools); + // Number of trailing bytes the streaming emitter must retain to recognize an // opener split across token boundaries. size_t tool_syntax_holdback(const json & tools); diff --git a/server/test/unit/test_output_constraint.cpp b/server/test/unit/test_output_constraint.cpp new file mode 100644 index 0000000000..55f2e36464 --- /dev/null +++ b/server/test/unit/test_output_constraint.cpp @@ -0,0 +1,327 @@ +// Structured output: GBNF matching, JSON schema grammars, the per-request +// constraint and the sampler that honours it. Pure logic: no model or GPU. +#include "CppUnitTestFramework.hpp" + +#include "common/gbnf.h" +#include "common/output_constraint.h" +#include "common/sampler.h" +#include "server/http_server.h" +#include "server/json_schema_grammar.h" +#include "server/tool_parser.h" + +#include +#include +#include +#include + +using namespace luce::common; + +namespace { + +struct OutputConstraintFixture {}; + +std::unique_ptr compile(const std::string & grammar) { + std::string error; + auto matcher = gbnf::Matcher::compile(grammar, "root", error); + if (!matcher) std::fprintf(stderr, "grammar error: %s\n", error.c_str()); + return matcher; +} + +bool matches(const std::string & grammar, const std::string & text) { + auto matcher = compile(grammar); + if (!matcher) return false; + matcher->accept_bytes(text); + return !matcher->dead() && matcher->can_end(); +} + +// A toy vocabulary: a fenced-JSON habit the grammar must break, the JSON +// pieces it should pick instead, a tool-call opener and an end token. +enum Tok : int32_t { + FENCE, OBJ_OPEN, OBJ_CLOSE, KEY_A, COLON, ONE, NEWLINE, TOOL_CALL, THINK_END, + PROSE, NUL_BYTE, END, N_TOK, +}; + +std::shared_ptr toy_vocab() { + std::vector pieces(N_TOK); + pieces[FENCE] = "```json"; + pieces[OBJ_OPEN] = "{"; + pieces[OBJ_CLOSE] = "}"; + pieces[KEY_A] = "\"a\""; + pieces[COLON] = ":"; + pieces[ONE] = "1"; + pieces[NEWLINE] = "\n"; + pieces[TOOL_CALL] = ""; + pieces[THINK_END] = ""; + pieces[PROSE] = "Sure"; + pieces[NUL_BYTE] = std::string(1, '\0'); + pieces[END] = "<|im_end|>"; + return ConstraintVocab::build(std::move(pieces), {END}); +} + +const char * kSchemaA = R"({"type":"object","properties":{"a":{"type":"integer"}},"required":["a"],"additionalProperties":false})"; + +std::shared_ptr schema_constraint(OutputConstraint::Options options = {}, + std::vector openers = {}) { + const auto schema = nlohmann::ordered_json::parse(kSchemaA); + auto matcher = compile(http_detail::output_constraint_grammar(schema, openers)); + options.release_prefixes = std::move(openers); + return std::make_shared(toy_vocab(), std::move(matcher), std::move(options)); +} + +// Logits that prefer `best`, then everything else equally. +std::vector prefer(int32_t best) { + std::vector logits(N_TOK, 0.0f); + logits[(size_t) best] = 10.0f; + return logits; +} + +} // namespace + +TEST_CASE(OutputConstraintFixture, gbnf_matches_literals_ranges_and_repetition) { + const std::string grammar = "root ::= \"x\" [0-9]{1,3} (\",\" [a-z]+)*\n"; + CHECK(matches(grammar, "x1")); + CHECK(matches(grammar, "x123,ab,c")); + CHECK(!matches(grammar, "x1234")); + CHECK(!matches(grammar, "y1")); + CHECK(!matches(grammar, "x")); // a prefix, not a sentence +} + +TEST_CASE(OutputConstraintFixture, gbnf_reports_parse_errors) { + std::string error; + CHECK(gbnf::Matcher::compile("root ::= missing\n", "root", error) == nullptr); + CHECK(!error.empty()); + CHECK(gbnf::Matcher::compile("start ::= \"a\"\n", "root", error) == nullptr); + CHECK(gbnf::Matcher::compile("root ::= root \"a\"\n", "root", error) == nullptr); +} + +TEST_CASE(OutputConstraintFixture, gbnf_follows_utf8_split_across_pieces) { + auto matcher = compile("root ::= \"é\" \"!\"\n"); + CHECK(matcher != nullptr); + matcher->accept_bytes("\xC3"); // first byte of é + CHECK(!matcher->dead()); + matcher->accept_bytes("\xA9!"); // its continuation, then ! + CHECK(!matcher->dead()); + CHECK(matcher->can_end()); +} + +TEST_CASE(OutputConstraintFixture, schema_grammar_accepts_documents_and_rejects_others) { + const auto schema = nlohmann::ordered_json::parse(kSchemaA); + const std::string grammar = json_schema_to_grammar(schema); + CHECK(matches(grammar, "{\"a\": 1}")); + CHECK(matches(grammar, "{\n \"a\": 42\n}")); + CHECK(!matches(grammar, "{\"a\": \"one\"}")); // wrong type + CHECK(!matches(grammar, "{\"a\": 1, \"b\": 2}")); // additionalProperties + CHECK(!matches(grammar, "```json\n{\"a\": 1}\n```")); // fenced +} + +TEST_CASE(OutputConstraintFixture, schema_grammar_keeps_property_order_and_enums) { + const auto schema = nlohmann::ordered_json::parse(R"({ + "type": "object", + "properties": { + "title": {"type": "string"}, + "genre": {"type": "string", "enum": ["drama", "sci-fi"]} + }, + "required": ["title", "genre"] + })"); + const std::string grammar = json_schema_to_grammar(schema); + CHECK(matches(grammar, R"({"title": "Alien", "genre": "sci-fi"})")); + CHECK(!matches(grammar, R"({"title": "Alien", "genre": "horror"})")); + CHECK(!matches(grammar, R"({"genre": "sci-fi", "title": "Alien"})")); +} + +TEST_CASE(OutputConstraintFixture, sampler_redraws_a_rejected_token_from_the_allowed_ones) { + auto constraint = schema_constraint(); + SamplerCfg cfg; + cfg.constraint = constraint; + CHECK(cfg.needs_logit_processing()); + std::mt19937_64 rng(1); + const std::vector history; + + // Greedy: the model wants a code fence, the grammar wants an object. + auto logits = prefer(FENCE); + CHECK(sample_logits(logits.data(), N_TOK, cfg, history, rng) == OBJ_OPEN); + + // A token the grammar already allows is kept as drawn. + logits = prefer(OBJ_OPEN); + const int32_t first = sample_logits(logits.data(), N_TOK, cfg, history, rng); + CHECK(first == OBJ_OPEN); + constraint->accept(first); + + // Sampled at a temperature, the draw still lands on an allowed token. + cfg.temp = 1.0f; + for (int i = 0; i < 50; ++i) { + logits = prefer(PROSE); + const int32_t token = sample_logits(logits.data(), N_TOK, cfg, history, rng); + CHECK(token == KEY_A || token == NEWLINE); + } +} + +TEST_CASE(OutputConstraintFixture, nul_byte_token_is_never_allowed) { + // Text is read up to a NUL, so this token would match "nothing" and slip + // into a string (seen live: "answer_cent\0s"). + auto constraint = schema_constraint(); + CHECK(!constraint->allows(NUL_BYTE)); + constraint->accept(OBJ_OPEN); + constraint->accept(KEY_A); + CHECK(!constraint->allows(NUL_BYTE)); + auto logits = prefer(NUL_BYTE); + constraint->mask(logits.data(), N_TOK); + CHECK(logits[NUL_BYTE] == -INFINITY); +} + +TEST_CASE(OutputConstraintFixture, redraw_after_a_fence_writes_the_reply_not_a_tool_call) { + // The model wants a code fence and would call a tool before opening the + // object; the fence says it is writing the reply. + auto constraint = schema_constraint({}, {""}); + SamplerCfg cfg; + cfg.constraint = constraint; + std::mt19937_64 rng(1); + auto logits = prefer(FENCE); + logits[TOOL_CALL] = 9.0f; + logits[OBJ_OPEN] = 8.0f; + CHECK(sample_logits(logits.data(), N_TOK, cfg, {}, rng) == OBJ_OPEN); + // A tool call the model draws itself is kept. + logits = prefer(TOOL_CALL); + CHECK(sample_logits(logits.data(), N_TOK, cfg, {}, rng) == TOOL_CALL); +} + +TEST_CASE(OutputConstraintFixture, redraw_at_the_start_skips_blank_tokens) { + // A newline would only postpone the choice; after it the model may well + // drift into the tool call the fence said it did not want. + auto constraint = schema_constraint({}, {""}); + SamplerCfg cfg; + cfg.constraint = constraint; + std::mt19937_64 rng(1); + auto logits = prefer(FENCE); + logits[NEWLINE] = 9.0f; + logits[OBJ_OPEN] = 8.0f; + CHECK(sample_logits(logits.data(), N_TOK, cfg, {}, rng) == OBJ_OPEN); + // Drawn by the model itself, a leading newline is kept. + logits = prefer(NEWLINE); + CHECK(sample_logits(logits.data(), N_TOK, cfg, {}, rng) == NEWLINE); +} + +TEST_CASE(OutputConstraintFixture, forced_tool_call_redraws_onto_the_opener) { + auto matcher = compile(http_detail::output_constraint_grammar(nullptr, {""})); + OutputConstraint::Options options; + options.release_prefixes = {""}; + options.calls_only = true; + SamplerCfg cfg; + cfg.constraint = std::make_shared(toy_vocab(), std::move(matcher), options); + std::mt19937_64 rng(1); + auto logits = prefer(PROSE); + CHECK(sample_logits(logits.data(), N_TOK, cfg, {}, rng) == TOOL_CALL); +} + +TEST_CASE(OutputConstraintFixture, end_token_only_once_the_document_is_complete) { + auto constraint = schema_constraint(); + for (int32_t token : {OBJ_OPEN, KEY_A, COLON, ONE}) { + CHECK(!constraint->allows(END)); + CHECK(constraint->allows(token)); + constraint->accept(token); + } + CHECK(!constraint->allows(END)); + constraint->accept(OBJ_CLOSE); + CHECK(constraint->complete()); + CHECK(constraint->allows(END)); + CHECK(!constraint->allows(OBJ_OPEN)); + + auto logits = prefer(PROSE); + CHECK(constraint->mask(logits.data(), N_TOK) >= 1); + CHECK(logits[END] != -INFINITY); + CHECK(logits[PROSE] == -INFINITY); +} + +TEST_CASE(OutputConstraintFixture, tool_call_opener_releases_the_output) { + auto constraint = schema_constraint({}, {""}); + CHECK(constraint->allows(TOOL_CALL)); + CHECK(constraint->allows(OBJ_OPEN)); + CHECK(!constraint->allows(PROSE)); + constraint->accept(TOOL_CALL); + CHECK(constraint->released()); + CHECK(!constraint->engaged()); + CHECK(constraint->allows(PROSE)); // the tool parser reads the call +} + +TEST_CASE(OutputConstraintFixture, waits_for_the_end_of_thinking) { + OutputConstraint::Options options; + options.wait_for = ""; + auto constraint = schema_constraint(options); + CHECK(!constraint->engaged()); + CHECK(constraint->allows(PROSE)); // free while thinking + constraint->accept(PROSE); + CHECK(!constraint->engaged()); + constraint->accept(THINK_END); + CHECK(constraint->engaged()); + constraint->accept(NEWLINE); // leading whitespace is allowed + CHECK(!constraint->allows(PROSE)); + CHECK(constraint->allows(OBJ_OPEN)); +} + +TEST_CASE(OutputConstraintFixture, off_grammar_token_breaks_instead_of_dead_ending) { + auto constraint = schema_constraint(); + constraint->accept(PROSE); // injected past the sampler + CHECK(constraint->broken()); + CHECK(constraint->allows(FENCE)); + constraint->reset(); + CHECK(constraint->engaged()); + CHECK(!constraint->allows(FENCE)); +} + +TEST_CASE(OutputConstraintFixture, request_output_format_reads_all_three_apis) { + using http_detail::request_output_format; + CHECK(request_output_format(R"({"messages":[]})").is_null()); + const auto chat = request_output_format( + R"({"response_format":{"type":"json_schema","json_schema":{"name":"x","schema":{"type":"object","properties":{"b":{},"a":{}}}}}})"); + CHECK(chat.value("type", "") == "json_schema"); + // Property order survives (nlohmann::json would sort a before b). + const auto props = chat["json_schema"]["schema"]["properties"]; + CHECK(props.begin().key() == "b"); + CHECK(request_output_format(R"({"text":{"format":{"type":"json_object"}}})") + .value("type", "") == "json_object"); + CHECK(request_output_format(R"({"output_format":{"type":"json_schema","schema":{}}})") + .value("type", "") == "json_schema"); +} + +TEST_CASE(OutputConstraintFixture, output_format_schema_normalizes_formats) { + using http_detail::output_format_schema; + nlohmann::ordered_json schema; + std::string error; + CHECK(output_format_schema(nullptr, schema, error)); + CHECK(schema.is_null()); + CHECK(output_format_schema({{"type", "text"}}, schema, error)); + CHECK(schema.is_null()); + CHECK(output_format_schema({{"type", "json_object"}}, schema, error)); + CHECK(schema.value("type", "") == "object"); + CHECK(output_format_schema(nlohmann::ordered_json::parse( + R"({"type":"json_schema","json_schema":{"schema":{"type":"array"}}})"), schema, error)); + CHECK(schema.value("type", "") == "array"); + CHECK(output_format_schema(nlohmann::ordered_json::parse( + R"({"type":"json_schema","schema":{"type":"string"}})"), schema, error)); + CHECK(schema.value("type", "") == "string"); + CHECK(!output_format_schema({{"type", "xml"}}, schema, error)); + CHECK(!error.empty()); +} + +TEST_CASE(OutputConstraintFixture, tool_choice_requires_call_covers_each_api) { + using http_detail::tool_choice_requires_call; + CHECK(tool_choice_requires_call("required")); + CHECK(!tool_choice_requires_call("auto")); + CHECK(!tool_choice_requires_call(nullptr)); + CHECK(tool_choice_requires_call(json::parse(R"({"type":"function","function":{"name":"f"}})"))); + CHECK(tool_choice_requires_call(json::parse(R"({"type":"function","name":"f"})"))); + CHECK(tool_choice_requires_call(json::parse(R"({"type":"any"})"))); + CHECK(tool_choice_requires_call(json::parse(R"({"type":"tool","name":"f"})"))); + CHECK(!tool_choice_requires_call(json::parse(R"({"type":"auto"})"))); +} + +TEST_CASE(OutputConstraintFixture, forced_tool_call_grammar_admits_only_openers) { + const json tools = json::parse(R"([{"type":"function","function":{"name":"get_weather"}}])"); + const auto openers = tool_call_openers(tools); + CHECK(std::find(openers.begin(), openers.end(), "") != openers.end()); + CHECK(std::find(openers.begin(), openers.end(), "") != openers.end()); + const std::string grammar = http_detail::output_constraint_grammar(nullptr, openers); + CHECK(matches(grammar, "\n\n\n")); + CHECK(matches(grammar, "\n<｜DSML｜ calls>")); + CHECK(!matches(grammar, "I will call the tool.")); +} From b68c7ee8e9dc5b1b7c7d8cac10c45d6af3a0d5d0 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:41:57 +0200 Subject: [PATCH 2/2] fix(server): a non-string format or tool_choice type is a clean 400 response_format {"type": 1} or a tool_choice object with a non-string type made json::value throw; read those members only when they are strings, and answer a non-string response_format type with its own 400. The sampler's comment now says what the redraw gives: exactly the masked distribution for temperature sampling, llama.cpp's rejection sampling with top_k/top_p. Co-Authored-By: Claude Opus 5.5 --- server/src/common/output_constraint.h | 7 ++++--- server/src/common/sampler.cpp | 13 ++++++++----- server/src/server/http_server.cpp | 20 +++++++++++++++++--- server/test/unit/test_output_constraint.cpp | 6 ++++++ 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/server/src/common/output_constraint.h b/server/src/common/output_constraint.h index 6e433142d7..5944784d45 100644 --- a/server/src/common/output_constraint.h +++ b/server/src/common/output_constraint.h @@ -4,9 +4,10 @@ // One OutputConstraint per request, shared through SamplerCfg::constraint: // - the sampler (sample_logits) draws the request's token as usual and, if // the grammar rejects it, masks every token the grammar rejects and draws -// again from what is left. Drawing first and masking only on a miss gives -// exactly the masked distribution, and costs one token check per step -// while the model already writes what the grammar wants; +// again from what is left (llama.cpp's grammar rejection sampling). It +// costs one token check per step while the model already writes what the +// grammar wants, and gives the masked distribution exactly for +// temperature sampling (see sample_logits for top_k/top_p); // - the commit point advances it with every token the request keeps: the // scheduler (advance_slot) for paged engines, the token callback for the // single-request path. diff --git a/server/src/common/sampler.cpp b/server/src/common/sampler.cpp index 80665fd5da..665103081f 100644 --- a/server/src/common/sampler.cpp +++ b/server/src/common/sampler.cpp @@ -271,11 +271,14 @@ int sample_logits(const float * logits_in, const int token = sample_logits_chain(logits_in, vocab, cfg, history, rng); OutputConstraint * constraint = cfg.constraint.get(); if (!constraint || !constraint->engaged() || constraint->allows(token)) return token; - // The grammar rejects the draw: draw again from the tokens it allows. - // Drawing first and masking only on a miss leaves each allowed token - // with exactly its masked probability. A rejected draw is the request - // writing its reply, so the redraw keeps to the reply and leaves a tool - // call out, unless a tool call is all the grammar allows. + // The grammar rejects the draw: draw again from the tokens it allows + // (llama.cpp's grammar rejection sampling). With temperature and + // penalties alone this is exactly the masked distribution; top_k and + // top_p rank before the mask, so tokens the unmasked draw could reach + // weigh a little more than the masked chain would give them, in exchange + // for not masking the whole vocabulary every step. A rejected draw is + // the request writing its reply, so the redraw keeps to the reply and + // leaves a tool call out, unless a tool call is all the grammar allows. thread_local std::vector masked; masked.assign(logits_in, logits_in + vocab); if (constraint->mask(masked.data(), vocab, /*content_only=*/true) == 0) { diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index 976097d2c7..95562e931e 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -2223,6 +2223,16 @@ nlohmann::ordered_json request_output_format(const std::string & body) { return nullptr; } +// A string member of a client object; "" when it is absent or not a string. +// (json::value would throw on a client's {"type": 1}.) +template +std::string string_member(const Json & object, const char * key) { + if (!object.is_object()) return {}; + const auto it = object.find(key); + return it != object.end() && it->is_string() ? it->template get() + : std::string(); +} + bool output_format_schema(const nlohmann::ordered_json & format, nlohmann::ordered_json & schema, std::string & error) { schema = nullptr; @@ -2231,7 +2241,11 @@ bool output_format_schema(const nlohmann::ordered_json & format, error = "response_format must be an object"; return false; } - const std::string type = format.value("type", std::string()); + if (format.contains("type") && !format["type"].is_string()) { + error = "response_format type must be a string"; + return false; + } + const std::string type = string_member(format, "type"); if (type.empty() || type == "text") return true; if (type == "json_object") { // A schema beside json_object narrows it, as in llama-server. @@ -2261,7 +2275,7 @@ bool tool_choice_requires_call(const json & tool_choice) { return tool_choice == "required" || tool_choice == "any"; } if (!tool_choice.is_object()) return false; - const std::string type = tool_choice.value("type", std::string()); + const std::string type = string_member(tool_choice, "type"); return type == "any" || type == "tool" || type == "function" || tool_choice.contains("function"); } @@ -2327,7 +2341,7 @@ bool HttpServer::attach_output_constraint(SocketHandle fd, ParsedRequest & req) return false; } const bool tools_off = req.tool_choice == "none" || - (req.tool_choice.is_object() && req.tool_choice.value("type", "") == "none"); + http_detail::string_member(req.tool_choice, "type") == "none"; const bool has_tools = req.tools.is_array() && !req.tools.empty() && !tools_off; const bool must_call = has_tools && http_detail::tool_choice_requires_call(req.tool_choice); if (schema.is_null() && !must_call) return true; diff --git a/server/test/unit/test_output_constraint.cpp b/server/test/unit/test_output_constraint.cpp index 55f2e36464..c73897ea63 100644 --- a/server/test/unit/test_output_constraint.cpp +++ b/server/test/unit/test_output_constraint.cpp @@ -301,6 +301,11 @@ TEST_CASE(OutputConstraintFixture, output_format_schema_normalizes_formats) { CHECK(schema.value("type", "") == "string"); CHECK(!output_format_schema({{"type", "xml"}}, schema, error)); CHECK(!error.empty()); + // A non-string type is the client's mistake: an error, not an exception. + error.clear(); + CHECK(!output_format_schema({{"type", 1}}, schema, error)); + CHECK(!error.empty()); + CHECK(!output_format_schema(nlohmann::ordered_json::parse(R"({"type":null})"), schema, error)); } TEST_CASE(OutputConstraintFixture, tool_choice_requires_call_covers_each_api) { @@ -313,6 +318,7 @@ TEST_CASE(OutputConstraintFixture, tool_choice_requires_call_covers_each_api) { CHECK(tool_choice_requires_call(json::parse(R"({"type":"any"})"))); CHECK(tool_choice_requires_call(json::parse(R"({"type":"tool","name":"f"})"))); CHECK(!tool_choice_requires_call(json::parse(R"({"type":"auto"})"))); + CHECK(!tool_choice_requires_call(json::parse(R"({"type":1})"))); } TEST_CASE(OutputConstraintFixture, forced_tool_call_grammar_admits_only_openers) {