From 014211a857bf5912604ed69f4dc35e8544a78382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20Pa=C3=9Fgang?= Date: Sun, 19 Apr 2026 14:12:54 +0200 Subject: [PATCH 1/6] This commit enables the router to forward form-data to model server. Fixes #22044 (enabling to use the /v1/audio/transcriptions in router mode) --- tools/server/server-cors-proxy.h | 1 + tools/server/server-models.cpp | 91 ++++++++++++++++++++++++++++++-- tools/server/server-models.h | 1 + 3 files changed, 89 insertions(+), 4 deletions(-) diff --git a/tools/server/server-cors-proxy.h b/tools/server/server-cors-proxy.h index 887013152bf..2af0c7e1c21 100644 --- a/tools/server/server-cors-proxy.h +++ b/tools/server/server-cors-proxy.h @@ -49,6 +49,7 @@ static server_http_res_ptr proxy_request(const server_http_req & req, std::strin parsed_url.path, headers, req.body, + req.files, req.should_stop, 600, // timeout_read (default to 10 minutes) 600 // timeout_write (default to 10 minutes) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index a1eeec30e99..c5fa0a5f876 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #ifdef _WIN32 @@ -818,6 +819,7 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co proxy_path, req.headers, req.body, + req.files, req.should_stop, base_params.timeout_read, base_params.timeout_write @@ -1121,6 +1123,53 @@ static bool should_strip_proxy_header(const std::string & header_name) { return false; } +static std::string generate_multipart_boundary() { + static const char chars[] = "0123456789abcdefghijklmnopqrstuvwxyz"; + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis(0, sizeof(chars) - 2); + std::string boundary = "----llama-cpp-proxy-"; + for (int i = 0; i < 16; i++) { + boundary += chars[dis(gen)]; + } + return boundary; +} + +static std::string build_multipart_body( + const json & form_fields, + const std::map & files, + const std::string & boundary) { + std::string body; + + for (const auto & [key, value] : form_fields.items()) { + if (value.is_array()) { + for (const auto & item : value) { + body += "--" + boundary + "\r\n"; + body += "Content-Disposition: form-data; name=\"" + key + "\"\r\n"; + body += "\r\n"; + body += item.get() + "\r\n"; + } + } else { + body += "--" + boundary + "\r\n"; + body += "Content-Disposition: form-data; name=\"" + key + "\"\r\n"; + body += "\r\n"; + body += value.get() + "\r\n"; + } + } + + for (const auto & [key, data] : files) { + body += "--" + boundary + "\r\n"; + body += "Content-Disposition: form-data; name=\"" + key + "\"; filename=\"" + key + "\"\r\n"; + body += "Content-Type: application/octet-stream\r\n"; + body += "\r\n"; + body.append(data.begin(), data.end()); + body += "\r\n"; + } + + body += "--" + boundary + "--\r\n"; + return body; +} + server_http_proxy::server_http_proxy( const std::string & method, const std::string & scheme, @@ -1129,6 +1178,7 @@ server_http_proxy::server_http_proxy( const std::string & path, const std::map & headers, const std::string & body, + const std::map & files, const std::function should_stop, int32_t timeout_read, int32_t timeout_write @@ -1190,28 +1240,61 @@ server_http_proxy::server_http_proxy( return pipe->write({{}, 0, std::string(data, data_length), ""}); }; + // when files are present, the body was converted from multipart form data to JSON + // we need to reconstruct the multipart body for the downstream server + std::string effective_body = body; + std::string override_content_type; + bool has_files = !files.empty(); + + if (has_files) { + auto boundary = generate_multipart_boundary(); + json form_fields = json::parse(body); + effective_body = build_multipart_body(form_fields, files, boundary); + override_content_type = "multipart/form-data; boundary=" + boundary; + } + // prepare the request to destination server httplib::Request req; { req.method = method; req.path = path; for (const auto & [key, value] : headers) { - if (key == "Accept-Encoding") { + const auto lowered = to_lower_copy(key); + if (lowered == "accept-encoding") { // disable Accept-Encoding to avoid compressed responses continue; } - if (key == "Transfer-Encoding") { + if (lowered == "transfer-encoding") { // the body is already decoded continue; } - if (key == "Host" || key == "host") { + if (lowered == "content-length") { + // let httplib calculate Content-Length from the actual body + continue; + } + if (lowered == "content-type") { + if (has_files) { + // we set our own Content-Type with the new boundary + continue; + } + // when no files but the original request was multipart, + // the body is now JSON, so correct the Content-Type + if (value.find("multipart/form-data") != std::string::npos) { + override_content_type = "application/json; charset=utf-8"; + continue; + } + } + if (lowered == "host") { bool is_default_port = (scheme == "https" && port == 443) || (scheme == "http" && port == 80); req.set_header(key, is_default_port ? host : host + ":" + std::to_string(port)); } else { req.set_header(key, value); } } - req.body = body; + req.body = effective_body; + if (!override_content_type.empty()) { + req.set_header("Content-Type", override_content_type); + } req.response_handler = response_handler; req.content_receiver = content_receiver; } diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 1db34b6c4df..6b31d78c7a3 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -202,6 +202,7 @@ struct server_http_proxy : server_http_res { const std::string & path, const std::map & headers, const std::string & body, + const std::map & files, const std::function should_stop, int32_t timeout_read, int32_t timeout_write From 81e53053caa57a718d1406e7d64f02247674ce9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20Pa=C3=9Fgang?= Date: Mon, 20 Apr 2026 15:33:56 +0200 Subject: [PATCH 2/6] * Applied the suggestion from Copilots first comment: using the non-throwing json::parse overload. * Addressed Copilots third comment by extending the files representation to also include filename and content-type * Addressed Copilots fourth comment by making the RNG thread_local --- tools/server/server-common.cpp | 4 ++-- tools/server/server-common.h | 3 ++- tools/server/server-http.cpp | 8 ++++++-- tools/server/server-http.h | 8 +++++++- tools/server/server-models.cpp | 35 ++++++++++++++++++++++------------ tools/server/server-models.h | 2 +- 6 files changed, 41 insertions(+), 19 deletions(-) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index cae64884b36..abacfc3a234 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1457,14 +1457,14 @@ json convert_responses_to_chatcmpl(const json & response_body) { json convert_transcriptions_to_chatcmpl( const json & inp_body, - const std::map & in_files, + const std::map & in_files, std::vector & out_files) { // TODO @ngxson : this function may need to be improved in the future // handle input files out_files.clear(); auto it = in_files.find("file"); if (it != in_files.end()) { - out_files.push_back(it->second); + out_files.push_back(it->second.data); } else { throw std::invalid_argument("No input file found for transcription"); } diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 093a43453c2..02b9475c438 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -5,6 +5,7 @@ #include "llama.h" #include "chat.h" #include "mtmd.h" +#include "server-http.h" #define JSON_ASSERT GGML_ASSERT #include @@ -313,7 +314,7 @@ json convert_responses_to_chatcmpl(const json & body); // convert OpenAI transcriptions API format to OpenAI Chat Completions API format json convert_transcriptions_to_chatcmpl( const json & body, - const std::map & in_files, + const std::map & in_files, std::vector & out_files); // convert Anthropic Messages API format to OpenAI Chat Completions API format diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index 83f656f5c9d..8554e00e76f 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -439,7 +439,7 @@ void server_http_context::get(const std::string & path, const server_http_contex void server_http_context::post(const std::string & path, const server_http_context::handler_t & handler) const { pimpl->srv->Post(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) { std::string body = req.body; - std::map files; + std::map files; if (req.is_multipart_form_data()) { // translate text fields to a JSON object and use it as the body @@ -460,7 +460,11 @@ void server_http_context::post(const std::string & path, const server_http_conte // populate files from multipart form for (const auto & [key, file] : req.form.files) { - files[key] = raw_buffer(file.content.begin(), file.content.end()); + files[key] = uploaded_file{ + raw_buffer(file.content.begin(), file.content.end()), + file.filename, + file.content_type, + }; } } diff --git a/tools/server/server-http.h b/tools/server/server-http.h index 68ae2170cf6..d4d3b6e5368 100644 --- a/tools/server/server-http.h +++ b/tools/server/server-http.h @@ -36,13 +36,19 @@ struct server_http_res { using server_http_res_ptr = std::unique_ptr; using raw_buffer = std::vector; +struct uploaded_file { + raw_buffer data; + std::string filename; + std::string content_type; +}; + struct server_http_req { std::map params; // path_params + query_params std::map headers; // used by MCP proxy std::string path; std::string query_string; // query parameters string (e.g. "action=save") std::string body; - std::map files; // used for file uploads (form data) + std::map files; // used for file uploads (form data) const std::function & should_stop; std::string get_param(const std::string & key, const std::string & def = "") const { diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index c5fa0a5f876..3861ee87820 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1124,9 +1124,8 @@ static bool should_strip_proxy_header(const std::string & header_name) { } static std::string generate_multipart_boundary() { + thread_local std::mt19937 gen(std::random_device{}()); static const char chars[] = "0123456789abcdefghijklmnopqrstuvwxyz"; - std::random_device rd; - std::mt19937 gen(rd()); std::uniform_int_distribution<> dis(0, sizeof(chars) - 2); std::string boundary = "----llama-cpp-proxy-"; for (int i = 0; i < 16; i++) { @@ -1137,7 +1136,7 @@ static std::string generate_multipart_boundary() { static std::string build_multipart_body( const json & form_fields, - const std::map & files, + const std::map & files, const std::string & boundary) { std::string body; @@ -1157,12 +1156,20 @@ static std::string build_multipart_body( } } - for (const auto & [key, data] : files) { + for (const auto & [key, file] : files) { body += "--" + boundary + "\r\n"; - body += "Content-Disposition: form-data; name=\"" + key + "\"; filename=\"" + key + "\"\r\n"; - body += "Content-Type: application/octet-stream\r\n"; + body += "Content-Disposition: form-data; name=\"" + key + "\""; + if (!file.filename.empty()) { + body += "; filename=\"" + file.filename + "\""; + } + body += "\r\n"; + if (!file.content_type.empty()) { + body += "Content-Type: " + file.content_type + "\r\n"; + } else { + body += "Content-Type: application/octet-stream\r\n"; + } body += "\r\n"; - body.append(data.begin(), data.end()); + body.append(file.data.begin(), file.data.end()); body += "\r\n"; } @@ -1178,7 +1185,7 @@ server_http_proxy::server_http_proxy( const std::string & path, const std::map & headers, const std::string & body, - const std::map & files, + const std::map & files, const std::function should_stop, int32_t timeout_read, int32_t timeout_write @@ -1247,10 +1254,14 @@ server_http_proxy::server_http_proxy( bool has_files = !files.empty(); if (has_files) { - auto boundary = generate_multipart_boundary(); - json form_fields = json::parse(body); - effective_body = build_multipart_body(form_fields, files, boundary); - override_content_type = "multipart/form-data; boundary=" + boundary; + json form_fields = json::parse(body, nullptr, false); + if (!form_fields.is_discarded()) { + auto boundary = generate_multipart_boundary(); + effective_body = build_multipart_body(form_fields, files, boundary); + override_content_type = "multipart/form-data; boundary=" + boundary; + } else { + SRV_ERR("%s", "failed to parse multipart form fields JSON\n"); + } } // prepare the request to destination server diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 6b31d78c7a3..b3428ef5447 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -202,7 +202,7 @@ struct server_http_proxy : server_http_res { const std::string & path, const std::map & headers, const std::string & body, - const std::map & files, + const std::map & files, const std::function should_stop, int32_t timeout_read, int32_t timeout_write From 2d226f96c026bc81511dd8597395635698a4e243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20Pa=C3=9Fgang?= Date: Fri, 24 Apr 2026 01:28:27 +0200 Subject: [PATCH 3/6] Changed variable body from std::string to std::ostringstream in build_multipart_body as suggested by ngxson in https://github.com/ggml-org/llama.cpp/pull/22118#discussion_r3127099053 --- tools/server/server-models.cpp | 41 +++++++++++++++++----------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index b78e17a96cb..a2433741ad3 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #ifdef _WIN32 @@ -1143,43 +1144,43 @@ static std::string build_multipart_body( const json & form_fields, const std::map & files, const std::string & boundary) { - std::string body; + std::ostringstream body; for (const auto & [key, value] : form_fields.items()) { if (value.is_array()) { for (const auto & item : value) { - body += "--" + boundary + "\r\n"; - body += "Content-Disposition: form-data; name=\"" + key + "\"\r\n"; - body += "\r\n"; - body += item.get() + "\r\n"; + body << "--" << boundary << "\r\n"; + body << "Content-Disposition: form-data; name=\"" << key << "\"\r\n"; + body << "\r\n"; + body << item.get() << "\r\n"; } } else { - body += "--" + boundary + "\r\n"; - body += "Content-Disposition: form-data; name=\"" + key + "\"\r\n"; - body += "\r\n"; - body += value.get() + "\r\n"; + body << "--" << boundary << "\r\n"; + body << "Content-Disposition: form-data; name=\"" << key << "\"\r\n"; + body << "\r\n"; + body << value.get() << "\r\n"; } } for (const auto & [key, file] : files) { - body += "--" + boundary + "\r\n"; - body += "Content-Disposition: form-data; name=\"" + key + "\""; + body << "--" << boundary << "\r\n"; + body << "Content-Disposition: form-data; name=\"" << key << "\""; if (!file.filename.empty()) { - body += "; filename=\"" + file.filename + "\""; + body << "; filename=\"" << file.filename << "\""; } - body += "\r\n"; + body << "\r\n"; if (!file.content_type.empty()) { - body += "Content-Type: " + file.content_type + "\r\n"; + body << "Content-Type: " << file.content_type << "\r\n"; } else { - body += "Content-Type: application/octet-stream\r\n"; + body << "Content-Type: application/octet-stream\r\n"; } - body += "\r\n"; - body.append(file.data.begin(), file.data.end()); - body += "\r\n"; + body << "\r\n"; + body.write(reinterpret_cast(file.data.data()), file.data.size()); + body << "\r\n"; } - body += "--" + boundary + "--\r\n"; - return body; + body << "--" << boundary << "--\r\n"; + return body.str(); } server_http_proxy::server_http_proxy( From 191730a1b313144c8902ec89005d4e62ca3db01c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20Pa=C3=9Fgang?= Date: Fri, 24 Apr 2026 02:36:22 +0200 Subject: [PATCH 4/6] Added sanitize_field lambda in build_multipart_body for key, filename and content_type as suggested by ngxson in https://github.com/ggml-org/llama.cpp/pull/22118#discussion_r3127104647 --- tools/server/server-models.cpp | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index a2433741ad3..83cfbe04c1c 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1144,19 +1144,30 @@ static std::string build_multipart_body( const json & form_fields, const std::map & files, const std::string & boundary) { + static auto sanitize_field = [](const std::string & text) { + std::string result; + result.reserve(text.size()); + for (char c : text) { + if (c != '\n' && c != '\r') { + result += c; + } + } + return result; + }; + std::ostringstream body; for (const auto & [key, value] : form_fields.items()) { if (value.is_array()) { for (const auto & item : value) { body << "--" << boundary << "\r\n"; - body << "Content-Disposition: form-data; name=\"" << key << "\"\r\n"; + body << "Content-Disposition: form-data; name=\"" << sanitize_field(key) << "\"\r\n"; body << "\r\n"; body << item.get() << "\r\n"; } } else { body << "--" << boundary << "\r\n"; - body << "Content-Disposition: form-data; name=\"" << key << "\"\r\n"; + body << "Content-Disposition: form-data; name=\"" << sanitize_field(key) << "\"\r\n"; body << "\r\n"; body << value.get() << "\r\n"; } @@ -1164,13 +1175,13 @@ static std::string build_multipart_body( for (const auto & [key, file] : files) { body << "--" << boundary << "\r\n"; - body << "Content-Disposition: form-data; name=\"" << key << "\""; + body << "Content-Disposition: form-data; name=\"" << sanitize_field(key) << "\""; if (!file.filename.empty()) { - body << "; filename=\"" << file.filename << "\""; + body << "; filename=\"" << sanitize_field(file.filename) << "\""; } body << "\r\n"; if (!file.content_type.empty()) { - body << "Content-Type: " << file.content_type << "\r\n"; + body << "Content-Type: " << sanitize_field(file.content_type) << "\r\n"; } else { body << "Content-Type: application/octet-stream\r\n"; } From e7df9c9faac9959a01d122ecd526935e356d427c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20Pa=C3=9Fgang?= Date: Fri, 24 Apr 2026 17:39:18 +0200 Subject: [PATCH 5/6] explicitly checking if value/item is string before calling value/item.get() as requested by ngxson in https://github.com/ggml-org/llama.cpp/pull/22118#discussion_r3127111279 --- tools/server/server-models.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 83cfbe04c1c..c806359e683 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1163,12 +1163,18 @@ static std::string build_multipart_body( body << "--" << boundary << "\r\n"; body << "Content-Disposition: form-data; name=\"" << sanitize_field(key) << "\"\r\n"; body << "\r\n"; + if (!item.is_string()) { + throw std::invalid_argument("expected string"); + } body << item.get() << "\r\n"; } } else { body << "--" << boundary << "\r\n"; body << "Content-Disposition: form-data; name=\"" << sanitize_field(key) << "\"\r\n"; body << "\r\n"; + if (!value.is_string()) { + throw std::invalid_argument("expected string"); + } body << value.get() << "\r\n"; } } From 94c2f44edbb99626216771a8076162818c5cfdd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20Pa=C3=9Fgang?= Date: Mon, 27 Apr 2026 18:39:11 +0200 Subject: [PATCH 6/6] Added double quote to the sanitize lambda and throw on json parse failure --- tools/server/server-models.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index c806359e683..db6cbce8f9d 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1148,7 +1148,7 @@ static std::string build_multipart_body( std::string result; result.reserve(text.size()); for (char c : text) { - if (c != '\n' && c != '\r') { + if (c != '\n' && c != '\r' && c != '"') { result += c; } } @@ -1283,7 +1283,7 @@ server_http_proxy::server_http_proxy( effective_body = build_multipart_body(form_fields, files, boundary); override_content_type = "multipart/form-data; boundary=" + boundary; } else { - SRV_ERR("%s", "failed to parse multipart form fields JSON\n"); + throw std::runtime_error("failed to parse multipart form fields JSON"); } }