From 9b2aa8c935df333942d024b8b23dacc06b727e33 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Fri, 24 Apr 2026 18:30:42 -0500 Subject: [PATCH 01/14] ggml: implement `gguf_init_from_buffer` --- ggml/include/gguf.h | 2 +- ggml/src/gguf.cpp | 88 ++++++++++++++++++++++++++++++++++++++------- 2 files changed, 77 insertions(+), 13 deletions(-) diff --git a/ggml/include/gguf.h b/ggml/include/gguf.h index 02d5f221c03..ce43c5f4379 100644 --- a/ggml/include/gguf.h +++ b/ggml/include/gguf.h @@ -79,7 +79,7 @@ extern "C" { GGML_API struct gguf_context * gguf_init_empty(void); GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); - //GGML_API struct gguf_context * gguf_init_from_buffer(..); + GGML_API struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, struct gguf_init_params params); GGML_API void gguf_free(struct gguf_context * ctx); diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index ab3cc974867..7b56606b480 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -230,7 +230,15 @@ struct gguf_context { struct gguf_reader { gguf_reader(FILE * file) : file(file) { // read the remaining bytes once and update on each read + const int64_t cur = gguf_ftell(file); nbytes_remain = file_remain(file); + total_size = cur < 0 + ? nbytes_remain + : static_cast(cur) + nbytes_remain; + } + + gguf_reader(const void * data, size_t size) + : data(static_cast(data)), nbytes_remain(size), total_size(size) { } // helper for remaining bytes in a file @@ -260,7 +268,7 @@ struct gguf_reader { if (nbytes_remain < size) { return false; } - const size_t nread = fread(&dst, 1, size, file); + const size_t nread = read_raw(&dst, size); nbytes_remain -= nread; return nread == size; } @@ -344,7 +352,7 @@ struct gguf_reader { return false; } dst.resize(static_cast(size)); - const size_t nread = fread(dst.data(), 1, size, file); + const size_t nread = read_raw(dst.data(), static_cast(size)); nbytes_remain -= nread; return nread == size; } @@ -353,15 +361,58 @@ struct gguf_reader { if (size > nbytes_remain) { return false; } - const size_t nread = fread(dst, 1, size, file); + const size_t nread = read_raw(dst, size); nbytes_remain -= nread; return nread == size; } + uint64_t tell() const { + if (file != nullptr) { + const int64_t cur = gguf_ftell(file); + return cur < 0 + ? 0 + : static_cast(cur); + } + + return offset; + } + + bool seek(uint64_t absolute_offset) const { + if (absolute_offset > total_size) { + return false; + } + + if (file != nullptr) { + if (gguf_fseek(file, absolute_offset, SEEK_SET) != 0) { + return false; + } + } else { + offset = static_cast(absolute_offset); + } + + nbytes_remain = total_size - absolute_offset; + return true; + } + private: - FILE * file; + size_t read_raw(void * dst, size_t size) const { + if (file != nullptr) { + return fread(dst, 1, size, file); + } else if (data == nullptr || size > nbytes_remain || offset + size < offset) { + return 0; + } + memcpy(dst, data + offset, size); + offset += size; + return size; + } + + FILE * file = nullptr; + const uint8_t * data = nullptr; + + mutable size_t offset = 0; mutable uint64_t nbytes_remain; + uint64_t total_size = 0; }; struct gguf_context * gguf_init_empty(void) { @@ -394,12 +445,7 @@ bool gguf_read_emplace_helper(const struct gguf_reader & gr, std::vectorinfo.size()) == n_tensors); // we require the data section to be aligned, so take into account any padding - if (gguf_fseek(file, GGML_PAD(gguf_ftell(file), ctx->alignment), SEEK_SET) != 0) { + if (!gr.seek(GGML_PAD(gr.tell(), ctx->alignment))) { GGML_LOG_ERROR("%s: failed to seek to beginning of data section\n", __func__); gguf_free(ctx); return nullptr; } // store the current file offset - this is where the data section starts - ctx->offset = gguf_ftell(file); + ctx->offset = gr.tell(); // compute the total size of the data section, taking into account the alignment { @@ -844,6 +890,24 @@ struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_para return ctx; } +struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params) { + if (!file) { + return nullptr; + } + + const struct gguf_reader gr(file); + return gguf_init_from_reader(gr, params); +} + +struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, struct gguf_init_params params) { + if (data == nullptr || size == 0) { + return nullptr; + } + + const struct gguf_reader gr(data, size); + return gguf_init_from_reader(gr, params); +} + struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params) { FILE * file = ggml_fopen(fname, "rb"); From de9ebec64ca03c2c05276f04c712753e6983bef6 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Fri, 24 Apr 2026 18:50:22 -0500 Subject: [PATCH 02/14] test: `gguf_init_from_buffer` --- tests/test-gguf.cpp | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/tests/test-gguf.cpp b/tests/test-gguf.cpp index ed3070dc4de..86dd888304c 100644 --- a/tests/test-gguf.cpp +++ b/tests/test-gguf.cpp @@ -162,6 +162,22 @@ static void helper_write(FILE * file, const void * data, const size_t nbytes) { GGML_ASSERT(fwrite(data, 1, nbytes, file) == nbytes); } +static std::vector read_file_to_buffer(FILE * file) { + GGML_ASSERT(file != nullptr); + GGML_ASSERT(fseek(file, 0, SEEK_END) == 0); + + const long size = ftell(file); + GGML_ASSERT(size >= 0); + + rewind(file); + + std::vector data(static_cast(size)); + GGML_ASSERT(fread(data.data(), 1, data.size(), file) == data.size()); + + rewind(file); + return data; +} + static FILE * get_handcrafted_file(const unsigned int seed, const enum handcrafted_file_type hft, const int extra_bytes = 0) { FILE * file = tmpfile(); @@ -1095,10 +1111,11 @@ static bool same_tensor_data(const struct ggml_context * orig, const struct ggml return ok; } -static std::pair test_roundtrip(ggml_backend_dev_t dev, const unsigned int seed, const bool only_meta) { +static std::pair test_roundtrip(ggml_backend_dev_t dev, const unsigned int seed, const bool only_meta, const bool from_buffer = false) { ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); - printf("%s: device=%s, backend=%s, only_meta=%s\n", - __func__, ggml_backend_dev_description(dev), ggml_backend_name(backend), only_meta ? "yes" : "no"); + printf("%s: device=%s, backend=%s, only_meta=%s, from_buffer=%s\n", + __func__, ggml_backend_dev_description(dev), ggml_backend_name(backend), + only_meta ? "yes" : "no", from_buffer ? "yes" : "no"); int npass = 0; int ntest = 0; @@ -1133,7 +1150,14 @@ static std::pair test_roundtrip(ggml_backend_dev_t dev, const unsigned /*no_alloc =*/ false, /*ctx =*/ only_meta ? nullptr : &ctx_1, }; - struct gguf_context * gguf_ctx_1 = gguf_init_from_file_ptr(file, gguf_params); + struct gguf_context * gguf_ctx_1 = nullptr; + + if (from_buffer) { + const std::vector data = read_file_to_buffer(file); + gguf_ctx_1 = gguf_init_from_buffer(data.data(), data.size(), gguf_params); + } else { + gguf_ctx_1 = gguf_init_from_file_ptr(file, gguf_params); + } printf("%s: same_version: ", __func__); if (gguf_get_version(gguf_ctx_0) == gguf_get_version(gguf_ctx_1)) { @@ -1347,6 +1371,11 @@ int main(int argc, char ** argv) { npass += result.first; ntest += result.second; } + { + std::pair result = test_roundtrip(dev, seed, /*only_meta=*/false, /*from_buffer=*/true); + npass += result.first; + ntest += result.second; + } { std::pair result = test_gguf_set_kv(dev, seed); From 59883b3cafae9481475fcb2d0981311051a6df96 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Fri, 24 Apr 2026 20:07:59 -0500 Subject: [PATCH 03/14] fix: memory breakdown for a model loaded with `no_alloc` from a file is consistent with being loaded from a buffer --- src/llama-model-loader.cpp | 20 +++++- tests/CMakeLists.txt | 4 ++ tests/test-model-load-buffer.cpp | 107 +++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 tests/test-model-load-buffer.cpp diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 4e65a45a50d..2de7e7bcd1f 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -697,7 +697,9 @@ llama_model_loader::llama_model_loader( } n_kv = gguf_get_n_kv(metadata); - n_tensors = weights_map.size(); + n_tensors = files.empty() + ? gguf_get_n_tensors(metadata) + : weights_map.size(); fver = (enum llama_fver) gguf_get_version(metadata); @@ -1218,6 +1220,12 @@ struct ggml_tensor * llama_model_loader::create_tensor( const int64_t tid = gguf_find_tensor(metadata, tn.str().c_str()); if (tid != -1) { type = gguf_get_tensor_type(metadata, tid); + } else if (no_alloc) { + if (flags & TENSOR_NOT_REQUIRED) { + return nullptr; + } + + throw std::runtime_error(format("missing tensor '%s'", tn.str().c_str())); } // for tensors that are not required some of the dimensions can be invalid: @@ -1243,6 +1251,16 @@ struct ggml_tensor * llama_model_loader::create_tensor( ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta); GGML_ASSERT(buft != nullptr); ggml_context * ctx = ctx_for_buft(buft); + + if (flags & TENSOR_DUPLICATED) { + ggml_tensor * t = ggml_get_tensor(ctx, tn.str().c_str()); + if (t) { + return t; + } + } else { + n_created++; + } + ggml_tensor * ret = ggml_dup_tensor(ctx, &t_meta); ggml_set_name(ret, tn.str().c_str()); return ret; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index edb585b9f65..2b886c74b8d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -243,6 +243,10 @@ llama_build_and_test(test-gguf.cpp) llama_build_and_test(test-backend-ops.cpp) llama_build_and_test(test-model-load-cancel.cpp LABEL "model") + +llama_build_and_test(test-model-load-buffer.cpp LABEL "model" ARGS "${MODEL_DEST}") +set_tests_properties(test-model-load-buffer PROPERTIES FIXTURES_REQUIRED test-download-model) + llama_build_and_test(test-autorelease.cpp LABEL "model") llama_build_and_test(test-backend-sampler.cpp LABEL "model") diff --git a/tests/test-model-load-buffer.cpp b/tests/test-model-load-buffer.cpp new file mode 100644 index 00000000000..c36ac19666d --- /dev/null +++ b/tests/test-model-load-buffer.cpp @@ -0,0 +1,107 @@ +#include "ggml-backend.h" +#include "get-model.h" +#include "llama.h" +#include "gguf.h" + +#include "../src/llama-model.h" + +#include +#include +#include +#include + +static std::vector read_file_to_buffer(FILE * file) { + if (file == nullptr || fseek(file, 0, SEEK_END) != 0) { + return {}; + } + + const long size = ftell(file); + if (size < 0) { + return {}; + } + + rewind(file); + + std::vector data(static_cast(size)); + if (fread(data.data(), 1, data.size(), file) != data.size()) { + return {}; + } + + return data; +} + +static void set_tensor_data_noop(struct ggml_tensor * tensor, void * userdata) { + (void) tensor; + (void) userdata; +} + +int main(int argc, char * argv[]) { + char * model_path = get_model_or_exit(argc, argv); + FILE * file = fopen(model_path, "rb"); + if (file == nullptr) { + fprintf(stderr, "failed to open model at '%s'\n", model_path); + return EXIT_FAILURE; + } + + const std::vector data = read_file_to_buffer(file); + fclose(file); + if (data.empty()) { + fprintf(stderr, "failed to read model at '%s'\n", model_path); + return EXIT_FAILURE; + } + + llama_backend_init(); + + ggml_backend_dev_t cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); + if (cpu_dev == nullptr) { + llama_backend_free(); + return EXIT_FAILURE; + } + + ggml_backend_dev_t devices[] = { cpu_dev, nullptr }; + + llama_model_params model_params = llama_model_default_params(); + model_params.devices = devices; + model_params.no_alloc = true; + model_params.use_mmap = false; + model_params.progress_callback = [](float /*progress*/, void * /*user_data*/) { + return true; + }; + + gguf_init_params gguf_params = { + /*.no_alloc = */ true, + /*.ctx = */ nullptr, + }; + gguf_context * gguf_ctx = gguf_init_from_buffer(data.data(), data.size(), gguf_params); + if (gguf_ctx == nullptr || gguf_get_n_tensors(gguf_ctx) <= 0) { + gguf_free(gguf_ctx); + llama_backend_free(); + return EXIT_FAILURE; + } + + llama_model * model_from_file = llama_model_load_from_file(model_path, model_params); + if (model_from_file == nullptr) { + gguf_free(gguf_ctx); + llama_backend_free(); + return EXIT_FAILURE; + } + + llama_model * model_from_buffer = llama_model_init_from_user(gguf_ctx, set_tensor_data_noop, nullptr, model_params); + if (model_from_buffer == nullptr) { + llama_model_free(model_from_file); + gguf_free(gguf_ctx); + llama_backend_free(); + return EXIT_FAILURE; + } + + const auto mb_from_file = model_from_file->memory_breakdown(); + const auto mb_from_buffer = model_from_buffer->memory_breakdown(); + const bool ok = !mb_from_file.empty() && mb_from_file == mb_from_buffer; + + llama_model_free(model_from_buffer); + llama_model_free(model_from_file); + gguf_free(gguf_ctx); + llama_backend_free(); + + return ok ? EXIT_SUCCESS : EXIT_FAILURE; +} From 5950daa76eba822dc4f75b95de0fa0c1e6a920bb Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Fri, 24 Apr 2026 20:10:37 -0500 Subject: [PATCH 04/14] fix: use `GGML_UNUSED` Co-authored-by: Copilot --- tests/test-model-load-buffer.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test-model-load-buffer.cpp b/tests/test-model-load-buffer.cpp index c36ac19666d..0dcc63cdb38 100644 --- a/tests/test-model-load-buffer.cpp +++ b/tests/test-model-load-buffer.cpp @@ -31,8 +31,8 @@ static std::vector read_file_to_buffer(FILE * file) { } static void set_tensor_data_noop(struct ggml_tensor * tensor, void * userdata) { - (void) tensor; - (void) userdata; + GGML_UNUSED(tensor); + GGML_UNUSED(userdata); } int main(int argc, char * argv[]) { @@ -64,7 +64,9 @@ int main(int argc, char * argv[]) { model_params.devices = devices; model_params.no_alloc = true; model_params.use_mmap = false; - model_params.progress_callback = [](float /*progress*/, void * /*user_data*/) { + model_params.progress_callback = [](float progress, void * user_data) { + GGML_UNUSED(progress); + GGML_UNUSED(user_data); return true; }; From b120c9bd110e887645a6f733f826ca3036675e19 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Mon, 27 Apr 2026 00:10:15 -0500 Subject: [PATCH 05/14] fix: remove `total_size` from `gguf_reader` --- ggml/src/gguf.cpp | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index 7b56606b480..b0fdfad4b8b 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -230,15 +230,11 @@ struct gguf_context { struct gguf_reader { gguf_reader(FILE * file) : file(file) { // read the remaining bytes once and update on each read - const int64_t cur = gguf_ftell(file); nbytes_remain = file_remain(file); - total_size = cur < 0 - ? nbytes_remain - : static_cast(cur) + nbytes_remain; } gguf_reader(const void * data, size_t size) - : data(static_cast(data)), nbytes_remain(size), total_size(size) { + : data(static_cast(data)), nbytes_remain(size) { } // helper for remaining bytes in a file @@ -378,19 +374,21 @@ struct gguf_reader { } bool seek(uint64_t absolute_offset) const { - if (absolute_offset > total_size) { - return false; - } - if (file != nullptr) { if (gguf_fseek(file, absolute_offset, SEEK_SET) != 0) { return false; } + + nbytes_remain = end_offset - absolute_offset; } else { + const uint64_t end_offset = offset + nbytes_remain; + if (absolute_offset > end_offset) { + return false; + } + offset = static_cast(absolute_offset); + nbytes_remain = end_offset - absolute_offset; } - - nbytes_remain = total_size - absolute_offset; return true; } @@ -412,7 +410,6 @@ struct gguf_reader { mutable size_t offset = 0; mutable uint64_t nbytes_remain; - uint64_t total_size = 0; }; struct gguf_context * gguf_init_empty(void) { From 65f7136014e55e95536f2a2b308497b7f670ab7f Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Mon, 27 Apr 2026 00:27:32 -0500 Subject: [PATCH 06/14] fix: file offset calculation, rename `offset` to `data_offset` Co-authored-by: Copilot --- ggml/src/gguf.cpp | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index b0fdfad4b8b..f3775b235b2 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -370,25 +370,31 @@ struct gguf_reader { : static_cast(cur); } - return offset; + return data_offset; } bool seek(uint64_t absolute_offset) const { if (file != nullptr) { - if (gguf_fseek(file, absolute_offset, SEEK_SET) != 0) { + const int64_t cur = gguf_ftell(file); + const uint64_t end_offset = cur < 0 + ? nbytes_remain + : static_cast(cur) + nbytes_remain; + + if (absolute_offset > end_offset || gguf_fseek(file, absolute_offset, SEEK_SET) != 0) { return false; } nbytes_remain = end_offset - absolute_offset; } else { - const uint64_t end_offset = offset + nbytes_remain; + const uint64_t end_offset = data_offset + nbytes_remain; if (absolute_offset > end_offset) { return false; } - - offset = static_cast(absolute_offset); + + data_offset = static_cast(absolute_offset); nbytes_remain = end_offset - absolute_offset; } + return true; } @@ -396,19 +402,19 @@ struct gguf_reader { size_t read_raw(void * dst, size_t size) const { if (file != nullptr) { return fread(dst, 1, size, file); - } else if (data == nullptr || size > nbytes_remain || offset + size < offset) { + } else if (data == nullptr || size > nbytes_remain || data_offset + size < data_offset) { return 0; } - memcpy(dst, data + offset, size); - offset += size; + memcpy(dst, data + data_offset, size); + data_offset += size; return size; } FILE * file = nullptr; const uint8_t * data = nullptr; - mutable size_t offset = 0; + mutable size_t data_offset = 0; mutable uint64_t nbytes_remain; }; From 0a59d4e23d7e144dda96e51c773163ed62dbea2f Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Mon, 27 Apr 2026 14:03:58 -0500 Subject: [PATCH 07/14] refactor: extract model loader bug fixes to another PR --- src/llama-model-loader.cpp | 20 +----- tests/CMakeLists.txt | 4 -- tests/test-model-load-buffer.cpp | 109 ------------------------------- 3 files changed, 1 insertion(+), 132 deletions(-) delete mode 100644 tests/test-model-load-buffer.cpp diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 2de7e7bcd1f..4e65a45a50d 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -697,9 +697,7 @@ llama_model_loader::llama_model_loader( } n_kv = gguf_get_n_kv(metadata); - n_tensors = files.empty() - ? gguf_get_n_tensors(metadata) - : weights_map.size(); + n_tensors = weights_map.size(); fver = (enum llama_fver) gguf_get_version(metadata); @@ -1220,12 +1218,6 @@ struct ggml_tensor * llama_model_loader::create_tensor( const int64_t tid = gguf_find_tensor(metadata, tn.str().c_str()); if (tid != -1) { type = gguf_get_tensor_type(metadata, tid); - } else if (no_alloc) { - if (flags & TENSOR_NOT_REQUIRED) { - return nullptr; - } - - throw std::runtime_error(format("missing tensor '%s'", tn.str().c_str())); } // for tensors that are not required some of the dimensions can be invalid: @@ -1251,16 +1243,6 @@ struct ggml_tensor * llama_model_loader::create_tensor( ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta); GGML_ASSERT(buft != nullptr); ggml_context * ctx = ctx_for_buft(buft); - - if (flags & TENSOR_DUPLICATED) { - ggml_tensor * t = ggml_get_tensor(ctx, tn.str().c_str()); - if (t) { - return t; - } - } else { - n_created++; - } - ggml_tensor * ret = ggml_dup_tensor(ctx, &t_meta); ggml_set_name(ret, tn.str().c_str()); return ret; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2b886c74b8d..edb585b9f65 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -243,10 +243,6 @@ llama_build_and_test(test-gguf.cpp) llama_build_and_test(test-backend-ops.cpp) llama_build_and_test(test-model-load-cancel.cpp LABEL "model") - -llama_build_and_test(test-model-load-buffer.cpp LABEL "model" ARGS "${MODEL_DEST}") -set_tests_properties(test-model-load-buffer PROPERTIES FIXTURES_REQUIRED test-download-model) - llama_build_and_test(test-autorelease.cpp LABEL "model") llama_build_and_test(test-backend-sampler.cpp LABEL "model") diff --git a/tests/test-model-load-buffer.cpp b/tests/test-model-load-buffer.cpp deleted file mode 100644 index 0dcc63cdb38..00000000000 --- a/tests/test-model-load-buffer.cpp +++ /dev/null @@ -1,109 +0,0 @@ -#include "ggml-backend.h" -#include "get-model.h" -#include "llama.h" -#include "gguf.h" - -#include "../src/llama-model.h" - -#include -#include -#include -#include - -static std::vector read_file_to_buffer(FILE * file) { - if (file == nullptr || fseek(file, 0, SEEK_END) != 0) { - return {}; - } - - const long size = ftell(file); - if (size < 0) { - return {}; - } - - rewind(file); - - std::vector data(static_cast(size)); - if (fread(data.data(), 1, data.size(), file) != data.size()) { - return {}; - } - - return data; -} - -static void set_tensor_data_noop(struct ggml_tensor * tensor, void * userdata) { - GGML_UNUSED(tensor); - GGML_UNUSED(userdata); -} - -int main(int argc, char * argv[]) { - char * model_path = get_model_or_exit(argc, argv); - FILE * file = fopen(model_path, "rb"); - if (file == nullptr) { - fprintf(stderr, "failed to open model at '%s'\n", model_path); - return EXIT_FAILURE; - } - - const std::vector data = read_file_to_buffer(file); - fclose(file); - if (data.empty()) { - fprintf(stderr, "failed to read model at '%s'\n", model_path); - return EXIT_FAILURE; - } - - llama_backend_init(); - - ggml_backend_dev_t cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); - if (cpu_dev == nullptr) { - llama_backend_free(); - return EXIT_FAILURE; - } - - ggml_backend_dev_t devices[] = { cpu_dev, nullptr }; - - llama_model_params model_params = llama_model_default_params(); - model_params.devices = devices; - model_params.no_alloc = true; - model_params.use_mmap = false; - model_params.progress_callback = [](float progress, void * user_data) { - GGML_UNUSED(progress); - GGML_UNUSED(user_data); - return true; - }; - - gguf_init_params gguf_params = { - /*.no_alloc = */ true, - /*.ctx = */ nullptr, - }; - gguf_context * gguf_ctx = gguf_init_from_buffer(data.data(), data.size(), gguf_params); - if (gguf_ctx == nullptr || gguf_get_n_tensors(gguf_ctx) <= 0) { - gguf_free(gguf_ctx); - llama_backend_free(); - return EXIT_FAILURE; - } - - llama_model * model_from_file = llama_model_load_from_file(model_path, model_params); - if (model_from_file == nullptr) { - gguf_free(gguf_ctx); - llama_backend_free(); - return EXIT_FAILURE; - } - - llama_model * model_from_buffer = llama_model_init_from_user(gguf_ctx, set_tensor_data_noop, nullptr, model_params); - if (model_from_buffer == nullptr) { - llama_model_free(model_from_file); - gguf_free(gguf_ctx); - llama_backend_free(); - return EXIT_FAILURE; - } - - const auto mb_from_file = model_from_file->memory_breakdown(); - const auto mb_from_buffer = model_from_buffer->memory_breakdown(); - const bool ok = !mb_from_file.empty() && mb_from_file == mb_from_buffer; - - llama_model_free(model_from_buffer); - llama_model_free(model_from_file); - gguf_free(gguf_ctx); - llama_backend_free(); - - return ok ? EXIT_SUCCESS : EXIT_FAILURE; -} From 62f4e9531c2dc0cc3ebe5b2a25661c95f5f9bfe3 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Mon, 27 Apr 2026 19:52:13 -0500 Subject: [PATCH 08/14] feat: add `gguf_init_from_callback` --- ggml/include/gguf.h | 8 ++ ggml/src/gguf.cpp | 192 +++++++++++++++++++++++++++++++------------- tests/test-gguf.cpp | 64 +++++++++++++-- 3 files changed, 203 insertions(+), 61 deletions(-) diff --git a/ggml/include/gguf.h b/ggml/include/gguf.h index ce43c5f4379..dfab09a68a3 100644 --- a/ggml/include/gguf.h +++ b/ggml/include/gguf.h @@ -76,11 +76,19 @@ extern "C" { struct ggml_context ** ctx; }; + // reads up to `len` bytes at `offset` into `output` and returns the number of bytes read. + // may be called with `len == 0` to seek/synchronize to `offset` without reading. + // when `len == 0` returns 0 on success and non-zero on failure + typedef size_t (*gguf_reader_callback_t)(void * userdata, uint8_t * output, size_t offset, size_t len); + GGML_API struct gguf_context * gguf_init_empty(void); GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); GGML_API struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, struct gguf_init_params params); + // when `total_size == 0` then end of file offset will be determined when the returned read size is smaller than the requested `len` + GGML_API struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, size_t total_size, struct gguf_init_params params); + GGML_API void gguf_free(struct gguf_context * ctx); GGML_API const char * gguf_type_name(enum gguf_type type); diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index f3775b235b2..6759f66e20e 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -228,13 +228,19 @@ struct gguf_context { }; struct gguf_reader { - gguf_reader(FILE * file) : file(file) { - // read the remaining bytes once and update on each read - nbytes_remain = file_remain(file); - } - - gguf_reader(const void * data, size_t size) - : data(static_cast(data)), nbytes_remain(size) { + gguf_reader( + gguf_reader_callback_t callback, + void * userdata, + size_t max_chunk_read, + size_t data_offset = 0, + uint64_t nbytes_remain = 0, + bool has_nbytes_remain = false) + : callback(callback), + userdata(userdata), + max_chunk_read(max_chunk_read), + data_offset(data_offset), + nbytes_remain(nbytes_remain), + has_nbytes_remain(has_nbytes_remain) { } // helper for remaining bytes in a file @@ -261,12 +267,10 @@ struct gguf_reader { template bool read(T & dst) const { const size_t size = sizeof(dst); - if (nbytes_remain < size) { + if (has_nbytes_remain && nbytes_remain < size) { return false; } - const size_t nread = read_raw(&dst, size); - nbytes_remain -= nread; - return nread == size; + return read_raw(&dst, size) == size; } template @@ -279,14 +283,14 @@ struct gguf_reader { if (n > SIZE_MAX / sizeof(uint64_t)) { return false; } - if (nbytes_remain < n * sizeof(uint64_t)) { + if (has_nbytes_remain && nbytes_remain < n * sizeof(uint64_t)) { return false; } } else { if (n > SIZE_MAX / sizeof(T)) { return false; } - if (nbytes_remain < n * sizeof(T)) { + if (has_nbytes_remain && nbytes_remain < n * sizeof(T)) { return false; } } @@ -343,55 +347,42 @@ struct gguf_reader { GGML_LOG_ERROR("%s: string length %" PRIu64 " exceeds maximum %" PRIu64 "\n", __func__, size, (uint64_t) GGUF_MAX_STRING_LENGTH); return false; } - if (size > nbytes_remain) { + if (has_nbytes_remain && size > nbytes_remain) { GGML_LOG_ERROR("%s: string length %" PRIu64 " exceeds remaining file size %" PRIu64 " bytes\n", __func__, size, nbytes_remain); return false; } dst.resize(static_cast(size)); - const size_t nread = read_raw(dst.data(), static_cast(size)); - nbytes_remain -= nread; - return nread == size; + return read_raw(dst.data(), static_cast(size)) == size; } bool read(void * dst, const size_t size) const { - if (size > nbytes_remain) { + if (has_nbytes_remain && size > nbytes_remain) { return false; } - const size_t nread = read_raw(dst, size); - nbytes_remain -= nread; - return nread == size; + return read_raw(dst, size) == size; } uint64_t tell() const { - if (file != nullptr) { - const int64_t cur = gguf_ftell(file); - return cur < 0 - ? 0 - : static_cast(cur); - } - return data_offset; } bool seek(uint64_t absolute_offset) const { - if (file != nullptr) { - const int64_t cur = gguf_ftell(file); - const uint64_t end_offset = cur < 0 - ? nbytes_remain - : static_cast(cur) + nbytes_remain; + if (absolute_offset > SIZE_MAX) { + return false; + } - if (absolute_offset > end_offset || gguf_fseek(file, absolute_offset, SEEK_SET) != 0) { - return false; - } + const uint64_t end_offset = uint64_t(data_offset) + nbytes_remain; + if (has_nbytes_remain && absolute_offset > end_offset) { + return false; + } - nbytes_remain = end_offset - absolute_offset; - } else { - const uint64_t end_offset = data_offset + nbytes_remain; - if (absolute_offset > end_offset) { - return false; - } + const size_t offset = static_cast(absolute_offset); + if (offset != data_offset && callback(userdata, nullptr, offset, 0) != 0) { + return false; + } - data_offset = static_cast(absolute_offset); + data_offset = offset; + if (has_nbytes_remain) { nbytes_remain = end_offset - absolute_offset; } @@ -400,22 +391,45 @@ struct gguf_reader { private: size_t read_raw(void * dst, size_t size) const { - if (file != nullptr) { - return fread(dst, 1, size, file); - } else if (data == nullptr || size > nbytes_remain || data_offset + size < data_offset) { + if (callback == nullptr) { return 0; } - memcpy(dst, data + data_offset, size); - data_offset += size; - return size; - } + uint8_t * data = static_cast(dst); + size_t total_nread = 0; + bool reached_eof = false; + + while (total_nread < size) { + const size_t chunk = std::min(max_chunk_read, size - total_nread); + if (data_offset + total_nread < data_offset) { + break; + } + const size_t nread = callback(userdata, data + total_nread, data_offset + total_nread, chunk); + total_nread += nread; + if (nread != chunk) { + reached_eof = true; + break; + } + } - FILE * file = nullptr; - const uint8_t * data = nullptr; + data_offset += total_nread; + if (has_nbytes_remain) { + GGML_ASSERT(total_nread <= nbytes_remain); + nbytes_remain -= total_nread; + } else if (reached_eof) { + nbytes_remain = 0; + has_nbytes_remain = true; + } + return total_nread; + } + + gguf_reader_callback_t callback = nullptr; + void * userdata = nullptr; + size_t max_chunk_read = 0; mutable size_t data_offset = 0; - mutable uint64_t nbytes_remain; + mutable uint64_t nbytes_remain = 0; + mutable bool has_nbytes_remain = false; }; struct gguf_context * gguf_init_empty(void) { @@ -893,21 +907,89 @@ static struct gguf_context * gguf_init_from_reader(const struct gguf_reader & gr return ctx; } +struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, size_t total_size, struct gguf_init_params params) { + if (callback == nullptr || max_chunk_read == 0) { + return nullptr; + } + + const struct gguf_reader gr(callback, userdata, max_chunk_read, 0, total_size, total_size != 0); + return gguf_init_from_reader(gr, params); +} + +struct gguf_file_reader { + FILE * file; + size_t offset; +}; + +static size_t gguf_file_reader_callback(void * userdata, uint8_t * output, size_t offset, size_t len) { + gguf_file_reader & reader = *static_cast(userdata); + + if (reader.offset != offset) { + if (gguf_fseek(reader.file, offset, SEEK_SET) != 0) { + return len == 0 ? 1 : 0; + } + + reader.offset = offset; + } + + if (len == 0) { + return 0; + } + + const size_t nread = fread(output, 1, len, reader.file); + reader.offset += nread; + return nread; +} + struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params) { if (!file) { return nullptr; } - const struct gguf_reader gr(file); + const int64_t cur = gguf_ftell(file); + if (cur < 0) { + return nullptr; + } + + gguf_file_reader reader = { + /*.file = */ file, + /*.offset = */ static_cast(cur), + }; + const struct gguf_reader gr(gguf_file_reader_callback, &reader, SIZE_MAX, reader.offset, gguf_reader::file_remain(file), true); return gguf_init_from_reader(gr, params); } +struct gguf_buffer_reader { + const uint8_t * data; + size_t size; +}; + +static size_t gguf_buffer_reader_callback(void * userdata, uint8_t * output, size_t offset, size_t len) { + const gguf_buffer_reader & reader = *static_cast(userdata); + + if (offset > reader.size) { + return len == 0 ? 1 : 0; + } + + if (len == 0) { + return 0; + } + + const size_t nread = std::min(len, reader.size - offset); + memcpy(output, reader.data + offset, nread); + return nread; +} + struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, struct gguf_init_params params) { if (data == nullptr || size == 0) { return nullptr; } - const struct gguf_reader gr(data, size); + gguf_buffer_reader reader = { + /*.data = */ static_cast(data), + /*.size = */ size, + }; + const struct gguf_reader gr(gguf_buffer_reader_callback, &reader, SIZE_MAX, 0, size, true); return gguf_init_from_reader(gr, params); } diff --git a/tests/test-gguf.cpp b/tests/test-gguf.cpp index 86dd888304c..013f62e69ee 100644 --- a/tests/test-gguf.cpp +++ b/tests/test-gguf.cpp @@ -178,6 +178,27 @@ static std::vector read_file_to_buffer(FILE * file) { return data; } +struct callback_reader_data { + const uint8_t * data; + size_t size; +}; + +static size_t read_buffer_callback(void * userdata, uint8_t * output, size_t offset, size_t len) { + const callback_reader_data & reader = *static_cast(userdata); + + if (offset > reader.size) { + return len == 0 ? 1 : 0; + } + + if (len == 0) { + return 0; + } + + const size_t nread = std::min(len, reader.size - offset); + memcpy(output, reader.data + offset, nread); + return nread; +} + static FILE * get_handcrafted_file(const unsigned int seed, const enum handcrafted_file_type hft, const int extra_bytes = 0) { FILE * file = tmpfile(); @@ -1111,11 +1132,29 @@ static bool same_tensor_data(const struct ggml_context * orig, const struct ggml return ok; } -static std::pair test_roundtrip(ggml_backend_dev_t dev, const unsigned int seed, const bool only_meta, const bool from_buffer = false) { +enum roundtrip_read_mode { + ROUNDTRIP_READ_FILE, + ROUNDTRIP_READ_BUFFER, + ROUNDTRIP_READ_CALLBACK, +}; + +static const char * roundtrip_read_mode_name(const roundtrip_read_mode mode) { + switch (mode) { + case ROUNDTRIP_READ_FILE: return "file"; + case ROUNDTRIP_READ_BUFFER: return "buffer"; + case ROUNDTRIP_READ_CALLBACK: return "callback"; + } + + GGML_ABORT("fatal error"); +} + +static std::pair test_roundtrip( + ggml_backend_dev_t dev, const unsigned int seed, const bool only_meta, + const roundtrip_read_mode read_mode = ROUNDTRIP_READ_FILE) { ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); - printf("%s: device=%s, backend=%s, only_meta=%s, from_buffer=%s\n", + printf("%s: device=%s, backend=%s, only_meta=%s, read_mode=%s\n", __func__, ggml_backend_dev_description(dev), ggml_backend_name(backend), - only_meta ? "yes" : "no", from_buffer ? "yes" : "no"); + only_meta ? "yes" : "no", roundtrip_read_mode_name(read_mode)); int npass = 0; int ntest = 0; @@ -1151,10 +1190,18 @@ static std::pair test_roundtrip(ggml_backend_dev_t dev, const unsigned /*ctx =*/ only_meta ? nullptr : &ctx_1, }; struct gguf_context * gguf_ctx_1 = nullptr; + const std::vector data = read_mode == ROUNDTRIP_READ_FILE + ? std::vector() + : read_file_to_buffer(file); - if (from_buffer) { - const std::vector data = read_file_to_buffer(file); + if (read_mode == ROUNDTRIP_READ_BUFFER) { gguf_ctx_1 = gguf_init_from_buffer(data.data(), data.size(), gguf_params); + } else if (read_mode == ROUNDTRIP_READ_CALLBACK) { + callback_reader_data reader = { + /*.data = */ data.data(), + /*.size = */ data.size(), + }; + gguf_ctx_1 = gguf_init_from_callback(read_buffer_callback, &reader, 4096, 0, gguf_params); } else { gguf_ctx_1 = gguf_init_from_file_ptr(file, gguf_params); } @@ -1372,7 +1419,12 @@ int main(int argc, char ** argv) { ntest += result.second; } { - std::pair result = test_roundtrip(dev, seed, /*only_meta=*/false, /*from_buffer=*/true); + std::pair result = test_roundtrip(dev, seed, /*only_meta=*/false, ROUNDTRIP_READ_BUFFER); + npass += result.first; + ntest += result.second; + } + { + std::pair result = test_roundtrip(dev, seed, /*only_meta=*/false, ROUNDTRIP_READ_CALLBACK); npass += result.first; ntest += result.second; } From 78bff38f5bd2ec2acfc65a80845dcc48ec5fd9d3 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Tue, 28 Apr 2026 15:35:27 -0500 Subject: [PATCH 09/14] fix: always require a max expected size --- ggml/include/gguf.h | 4 +--- ggml/src/gguf.cpp | 42 ++++++++++++++++++------------------------ 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/ggml/include/gguf.h b/ggml/include/gguf.h index dfab09a68a3..2640f6e5153 100644 --- a/ggml/include/gguf.h +++ b/ggml/include/gguf.h @@ -85,9 +85,7 @@ extern "C" { GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); GGML_API struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, struct gguf_init_params params); - - // when `total_size == 0` then end of file offset will be determined when the returned read size is smaller than the requested `len` - GGML_API struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, size_t total_size, struct gguf_init_params params); + GGML_API struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, size_t max_expected_size, struct gguf_init_params params); GGML_API void gguf_free(struct gguf_context * ctx); diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index 6759f66e20e..5f667e89075 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -233,14 +233,12 @@ struct gguf_reader { void * userdata, size_t max_chunk_read, size_t data_offset = 0, - uint64_t nbytes_remain = 0, - bool has_nbytes_remain = false) + uint64_t nbytes_remain = 0) : callback(callback), userdata(userdata), max_chunk_read(max_chunk_read), data_offset(data_offset), - nbytes_remain(nbytes_remain), - has_nbytes_remain(has_nbytes_remain) { + nbytes_remain(nbytes_remain) { } // helper for remaining bytes in a file @@ -267,7 +265,7 @@ struct gguf_reader { template bool read(T & dst) const { const size_t size = sizeof(dst); - if (has_nbytes_remain && nbytes_remain < size) { + if (size > nbytes_remain) { return false; } return read_raw(&dst, size) == size; @@ -283,14 +281,14 @@ struct gguf_reader { if (n > SIZE_MAX / sizeof(uint64_t)) { return false; } - if (has_nbytes_remain && nbytes_remain < n * sizeof(uint64_t)) { + if (nbytes_remain < n * sizeof(uint64_t)) { return false; } } else { if (n > SIZE_MAX / sizeof(T)) { return false; } - if (has_nbytes_remain && nbytes_remain < n * sizeof(T)) { + if (nbytes_remain < n * sizeof(T)) { return false; } } @@ -347,7 +345,7 @@ struct gguf_reader { GGML_LOG_ERROR("%s: string length %" PRIu64 " exceeds maximum %" PRIu64 "\n", __func__, size, (uint64_t) GGUF_MAX_STRING_LENGTH); return false; } - if (has_nbytes_remain && size > nbytes_remain) { + if (size > nbytes_remain) { GGML_LOG_ERROR("%s: string length %" PRIu64 " exceeds remaining file size %" PRIu64 " bytes\n", __func__, size, nbytes_remain); return false; } @@ -356,7 +354,7 @@ struct gguf_reader { } bool read(void * dst, const size_t size) const { - if (has_nbytes_remain && size > nbytes_remain) { + if (size > nbytes_remain) { return false; } return read_raw(dst, size) == size; @@ -372,7 +370,7 @@ struct gguf_reader { } const uint64_t end_offset = uint64_t(data_offset) + nbytes_remain; - if (has_nbytes_remain && absolute_offset > end_offset) { + if (absolute_offset > end_offset) { return false; } @@ -382,16 +380,14 @@ struct gguf_reader { } data_offset = offset; - if (has_nbytes_remain) { - nbytes_remain = end_offset - absolute_offset; - } + nbytes_remain = end_offset - absolute_offset; return true; } private: size_t read_raw(void * dst, size_t size) const { - if (callback == nullptr) { + if (callback == nullptr || size == 0) { return 0; } @@ -413,12 +409,11 @@ struct gguf_reader { } data_offset += total_nread; - if (has_nbytes_remain) { - GGML_ASSERT(total_nread <= nbytes_remain); - nbytes_remain -= total_nread; - } else if (reached_eof) { + GGML_ASSERT(total_nread <= nbytes_remain); + nbytes_remain -= total_nread; + + if (reached_eof) { nbytes_remain = 0; - has_nbytes_remain = true; } return total_nread; @@ -429,7 +424,6 @@ struct gguf_reader { size_t max_chunk_read = 0; mutable size_t data_offset = 0; mutable uint64_t nbytes_remain = 0; - mutable bool has_nbytes_remain = false; }; struct gguf_context * gguf_init_empty(void) { @@ -907,12 +901,12 @@ static struct gguf_context * gguf_init_from_reader(const struct gguf_reader & gr return ctx; } -struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, size_t total_size, struct gguf_init_params params) { +struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, size_t max_expected_size, struct gguf_init_params params) { if (callback == nullptr || max_chunk_read == 0) { return nullptr; } - const struct gguf_reader gr(callback, userdata, max_chunk_read, 0, total_size, total_size != 0); + const struct gguf_reader gr(callback, userdata, max_chunk_read, 0, max_expected_size); return gguf_init_from_reader(gr, params); } @@ -955,7 +949,7 @@ struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_para /*.file = */ file, /*.offset = */ static_cast(cur), }; - const struct gguf_reader gr(gguf_file_reader_callback, &reader, SIZE_MAX, reader.offset, gguf_reader::file_remain(file), true); + const struct gguf_reader gr(gguf_file_reader_callback, &reader, SIZE_MAX, reader.offset, gguf_reader::file_remain(file)); return gguf_init_from_reader(gr, params); } @@ -989,7 +983,7 @@ struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, stru /*.data = */ static_cast(data), /*.size = */ size, }; - const struct gguf_reader gr(gguf_buffer_reader_callback, &reader, SIZE_MAX, 0, size, true); + const struct gguf_reader gr(gguf_buffer_reader_callback, &reader, SIZE_MAX, 0, size); return gguf_init_from_reader(gr, params); } From dcca71d3d9c91c8584d5d6202284365f1187f33f Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Tue, 28 Apr 2026 16:11:57 -0500 Subject: [PATCH 10/14] fix: change `gguf_reader_callback_t`'s `output` type to `void *`, change `max_expected_size` and offsets to `uint64_t` --- ggml/include/gguf.h | 10 +++++----- ggml/src/gguf.cpp | 34 +++++++++++++++------------------- tests/test-gguf.cpp | 9 +++++---- 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/ggml/include/gguf.h b/ggml/include/gguf.h index 2640f6e5153..387f2a07d9c 100644 --- a/ggml/include/gguf.h +++ b/ggml/include/gguf.h @@ -76,16 +76,16 @@ extern "C" { struct ggml_context ** ctx; }; - // reads up to `len` bytes at `offset` into `output` and returns the number of bytes read. - // may be called with `len == 0` to seek/synchronize to `offset` without reading. - // when `len == 0` returns 0 on success and non-zero on failure - typedef size_t (*gguf_reader_callback_t)(void * userdata, uint8_t * output, size_t offset, size_t len); + // callback to simulate or wrap a FILE pointer: + // - by default, read up to `len` bytes at `offset` into `output` and return the number of bytes read + // - if called with `len == 0`, seek/synchronize to `offset` without reading, return 0 on success, non-zero for failure + typedef size_t (*gguf_reader_callback_t)(void * userdata, void * output, uint64_t offset, size_t len); GGML_API struct gguf_context * gguf_init_empty(void); GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); GGML_API struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, struct gguf_init_params params); - GGML_API struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, size_t max_expected_size, struct gguf_init_params params); + GGML_API struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, uint64_t max_expected_size, struct gguf_init_params params); GGML_API void gguf_free(struct gguf_context * ctx); diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index 5f667e89075..f0664481823 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -232,7 +232,7 @@ struct gguf_reader { gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, - size_t data_offset = 0, + uint64_t data_offset = 0, uint64_t nbytes_remain = 0) : callback(callback), userdata(userdata), @@ -365,21 +365,16 @@ struct gguf_reader { } bool seek(uint64_t absolute_offset) const { - if (absolute_offset > SIZE_MAX) { - return false; - } - const uint64_t end_offset = uint64_t(data_offset) + nbytes_remain; if (absolute_offset > end_offset) { return false; } - const size_t offset = static_cast(absolute_offset); - if (offset != data_offset && callback(userdata, nullptr, offset, 0) != 0) { + if (absolute_offset != data_offset && callback(userdata, nullptr, absolute_offset, 0) != 0) { return false; } - data_offset = offset; + data_offset = absolute_offset; nbytes_remain = end_offset - absolute_offset; return true; @@ -400,7 +395,7 @@ struct gguf_reader { if (data_offset + total_nread < data_offset) { break; } - const size_t nread = callback(userdata, data + total_nread, data_offset + total_nread, chunk); + const size_t nread = callback(userdata, static_cast(data + total_nread), data_offset + total_nread, chunk); total_nread += nread; if (nread != chunk) { reached_eof = true; @@ -422,7 +417,7 @@ struct gguf_reader { gguf_reader_callback_t callback = nullptr; void * userdata = nullptr; size_t max_chunk_read = 0; - mutable size_t data_offset = 0; + mutable uint64_t data_offset = 0; mutable uint64_t nbytes_remain = 0; }; @@ -901,7 +896,7 @@ static struct gguf_context * gguf_init_from_reader(const struct gguf_reader & gr return ctx; } -struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, size_t max_expected_size, struct gguf_init_params params) { +struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, uint64_t max_expected_size, struct gguf_init_params params) { if (callback == nullptr || max_chunk_read == 0) { return nullptr; } @@ -912,14 +907,14 @@ struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, v struct gguf_file_reader { FILE * file; - size_t offset; + uint64_t offset; }; -static size_t gguf_file_reader_callback(void * userdata, uint8_t * output, size_t offset, size_t len) { +static size_t gguf_file_reader_callback(void * userdata, void * output, uint64_t offset, size_t len) { gguf_file_reader & reader = *static_cast(userdata); if (reader.offset != offset) { - if (gguf_fseek(reader.file, offset, SEEK_SET) != 0) { + if (offset > INT64_MAX || gguf_fseek(reader.file, static_cast(offset), SEEK_SET) != 0) { return len == 0 ? 1 : 0; } @@ -930,7 +925,7 @@ static size_t gguf_file_reader_callback(void * userdata, uint8_t * output, size_ return 0; } - const size_t nread = fread(output, 1, len, reader.file); + const size_t nread = fread(static_cast(output), 1, len, reader.file); reader.offset += nread; return nread; } @@ -947,7 +942,7 @@ struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_para gguf_file_reader reader = { /*.file = */ file, - /*.offset = */ static_cast(cur), + /*.offset = */ static_cast(cur), }; const struct gguf_reader gr(gguf_file_reader_callback, &reader, SIZE_MAX, reader.offset, gguf_reader::file_remain(file)); return gguf_init_from_reader(gr, params); @@ -958,7 +953,7 @@ struct gguf_buffer_reader { size_t size; }; -static size_t gguf_buffer_reader_callback(void * userdata, uint8_t * output, size_t offset, size_t len) { +static size_t gguf_buffer_reader_callback(void * userdata, void * output, uint64_t offset, size_t len) { const gguf_buffer_reader & reader = *static_cast(userdata); if (offset > reader.size) { @@ -969,8 +964,9 @@ static size_t gguf_buffer_reader_callback(void * userdata, uint8_t * output, siz return 0; } - const size_t nread = std::min(len, reader.size - offset); - memcpy(output, reader.data + offset, nread); + const size_t data_offset = static_cast(offset); + const size_t nread = std::min(len, reader.size - data_offset); + memcpy(static_cast(output), reader.data + data_offset, nread); return nread; } diff --git a/tests/test-gguf.cpp b/tests/test-gguf.cpp index 013f62e69ee..a6b69818b25 100644 --- a/tests/test-gguf.cpp +++ b/tests/test-gguf.cpp @@ -183,7 +183,7 @@ struct callback_reader_data { size_t size; }; -static size_t read_buffer_callback(void * userdata, uint8_t * output, size_t offset, size_t len) { +static size_t read_buffer_callback(void * userdata, void * output, uint64_t offset, size_t len) { const callback_reader_data & reader = *static_cast(userdata); if (offset > reader.size) { @@ -194,8 +194,9 @@ static size_t read_buffer_callback(void * userdata, uint8_t * output, size_t off return 0; } - const size_t nread = std::min(len, reader.size - offset); - memcpy(output, reader.data + offset, nread); + const size_t data_offset = static_cast(offset); + const size_t nread = std::min(len, reader.size - data_offset); + memcpy(static_cast(output), reader.data + data_offset, nread); return nread; } @@ -1201,7 +1202,7 @@ static std::pair test_roundtrip( /*.data = */ data.data(), /*.size = */ data.size(), }; - gguf_ctx_1 = gguf_init_from_callback(read_buffer_callback, &reader, 4096, 0, gguf_params); + gguf_ctx_1 = gguf_init_from_callback(read_buffer_callback, &reader, 4096, 4ull << 30 /* 4GB */, gguf_params); } else { gguf_ctx_1 = gguf_init_from_file_ptr(file, gguf_params); } From 6248d3cd5f9e8d1f011d3354eaa3d9a64ec34773 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Tue, 28 Apr 2026 16:24:39 -0500 Subject: [PATCH 11/14] fix: harden against offset overflow in buffer read --- ggml/src/gguf.cpp | 2 +- tests/test-gguf.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index f0664481823..be35bde9a4c 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -960,7 +960,7 @@ static size_t gguf_buffer_reader_callback(void * userdata, void * output, uint64 return len == 0 ? 1 : 0; } - if (len == 0) { + if (len == 0 || offset + len > reader.size) { return 0; } diff --git a/tests/test-gguf.cpp b/tests/test-gguf.cpp index a6b69818b25..72d1e9eac83 100644 --- a/tests/test-gguf.cpp +++ b/tests/test-gguf.cpp @@ -190,7 +190,7 @@ static size_t read_buffer_callback(void * userdata, void * output, uint64_t offs return len == 0 ? 1 : 0; } - if (len == 0) { + if (len == 0 || offset + len > reader.size) { return 0; } From 913a28f5e1c844d655463015b5ed69779ac01d60 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Thu, 21 May 2026 20:36:11 +0200 Subject: [PATCH 12/14] fix: remove seek behavior from the callback --- ggml/include/gguf.h | 4 +--- ggml/src/gguf.cpp | 27 ++++++++++----------------- tests/test-gguf.cpp | 34 ++++++++++++++++------------------ 3 files changed, 27 insertions(+), 38 deletions(-) diff --git a/ggml/include/gguf.h b/ggml/include/gguf.h index 387f2a07d9c..0ce079cac33 100644 --- a/ggml/include/gguf.h +++ b/ggml/include/gguf.h @@ -76,9 +76,7 @@ extern "C" { struct ggml_context ** ctx; }; - // callback to simulate or wrap a FILE pointer: - // - by default, read up to `len` bytes at `offset` into `output` and return the number of bytes read - // - if called with `len == 0`, seek/synchronize to `offset` without reading, return 0 on success, non-zero for failure + // callback to simulate or wrap a FILE pointer - read up to `len` bytes at `offset` into `output` and return the number of bytes read typedef size_t (*gguf_reader_callback_t)(void * userdata, void * output, uint64_t offset, size_t len); GGML_API struct gguf_context * gguf_init_empty(void); diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index be35bde9a4c..3a395dcaf30 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -239,6 +239,7 @@ struct gguf_reader { max_chunk_read(max_chunk_read), data_offset(data_offset), nbytes_remain(nbytes_remain) { + GGML_ASSERT(max_chunk_read > 0); } // helper for remaining bytes in a file @@ -370,10 +371,6 @@ struct gguf_reader { return false; } - if (absolute_offset != data_offset && callback(userdata, nullptr, absolute_offset, 0) != 0) { - return false; - } - data_offset = absolute_offset; nbytes_remain = end_offset - absolute_offset; @@ -391,13 +388,13 @@ struct gguf_reader { bool reached_eof = false; while (total_nread < size) { - const size_t chunk = std::min(max_chunk_read, size - total_nread); + const size_t chunk_size = std::min(max_chunk_read, size - total_nread); if (data_offset + total_nread < data_offset) { break; } - const size_t nread = callback(userdata, static_cast(data + total_nread), data_offset + total_nread, chunk); + const size_t nread = callback(userdata, static_cast(data + total_nread), data_offset + total_nread, chunk_size); total_nread += nread; - if (nread != chunk) { + if (nread != chunk_size) { reached_eof = true; break; } @@ -911,20 +908,18 @@ struct gguf_file_reader { }; static size_t gguf_file_reader_callback(void * userdata, void * output, uint64_t offset, size_t len) { + GGML_ASSERT(len > 0); + gguf_file_reader & reader = *static_cast(userdata); if (reader.offset != offset) { if (offset > INT64_MAX || gguf_fseek(reader.file, static_cast(offset), SEEK_SET) != 0) { - return len == 0 ? 1 : 0; + return 0; } reader.offset = offset; } - if (len == 0) { - return 0; - } - const size_t nread = fread(static_cast(output), 1, len, reader.file); reader.offset += nread; return nread; @@ -954,13 +949,11 @@ struct gguf_buffer_reader { }; static size_t gguf_buffer_reader_callback(void * userdata, void * output, uint64_t offset, size_t len) { - const gguf_buffer_reader & reader = *static_cast(userdata); + GGML_ASSERT(len > 0); - if (offset > reader.size) { - return len == 0 ? 1 : 0; - } + const gguf_buffer_reader & reader = *static_cast(userdata); - if (len == 0 || offset + len > reader.size) { + if (offset > reader.size || len > reader.size - offset) { return 0; } diff --git a/tests/test-gguf.cpp b/tests/test-gguf.cpp index 72d1e9eac83..1ae468fbd65 100644 --- a/tests/test-gguf.cpp +++ b/tests/test-gguf.cpp @@ -184,13 +184,11 @@ struct callback_reader_data { }; static size_t read_buffer_callback(void * userdata, void * output, uint64_t offset, size_t len) { - const callback_reader_data & reader = *static_cast(userdata); + GGML_ASSERT(len > 0); - if (offset > reader.size) { - return len == 0 ? 1 : 0; - } + const callback_reader_data & reader = *static_cast(userdata); - if (len == 0 || offset + len > reader.size) { + if (offset > reader.size || len > reader.size - offset) { return 0; } @@ -1134,16 +1132,16 @@ static bool same_tensor_data(const struct ggml_context * orig, const struct ggml } enum roundtrip_read_mode { - ROUNDTRIP_READ_FILE, - ROUNDTRIP_READ_BUFFER, - ROUNDTRIP_READ_CALLBACK, + ROUNDTRIP_READ_MODE_FILE, + ROUNDTRIP_READ_MODE_BUFFER, + ROUNDTRIP_READ_MODE_CALLBACK, }; static const char * roundtrip_read_mode_name(const roundtrip_read_mode mode) { switch (mode) { - case ROUNDTRIP_READ_FILE: return "file"; - case ROUNDTRIP_READ_BUFFER: return "buffer"; - case ROUNDTRIP_READ_CALLBACK: return "callback"; + case ROUNDTRIP_READ_MODE_FILE: return "file"; + case ROUNDTRIP_READ_MODE_BUFFER: return "buffer"; + case ROUNDTRIP_READ_MODE_CALLBACK: return "callback"; } GGML_ABORT("fatal error"); @@ -1151,7 +1149,7 @@ static const char * roundtrip_read_mode_name(const roundtrip_read_mode mode) { static std::pair test_roundtrip( ggml_backend_dev_t dev, const unsigned int seed, const bool only_meta, - const roundtrip_read_mode read_mode = ROUNDTRIP_READ_FILE) { + const roundtrip_read_mode read_mode) { ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); printf("%s: device=%s, backend=%s, only_meta=%s, read_mode=%s\n", __func__, ggml_backend_dev_description(dev), ggml_backend_name(backend), @@ -1191,13 +1189,13 @@ static std::pair test_roundtrip( /*ctx =*/ only_meta ? nullptr : &ctx_1, }; struct gguf_context * gguf_ctx_1 = nullptr; - const std::vector data = read_mode == ROUNDTRIP_READ_FILE + const std::vector data = read_mode == ROUNDTRIP_READ_MODE_FILE ? std::vector() : read_file_to_buffer(file); - if (read_mode == ROUNDTRIP_READ_BUFFER) { + if (read_mode == ROUNDTRIP_READ_MODE_BUFFER) { gguf_ctx_1 = gguf_init_from_buffer(data.data(), data.size(), gguf_params); - } else if (read_mode == ROUNDTRIP_READ_CALLBACK) { + } else if (read_mode == ROUNDTRIP_READ_MODE_CALLBACK) { callback_reader_data reader = { /*.data = */ data.data(), /*.size = */ data.size(), @@ -1415,17 +1413,17 @@ int main(int argc, char ** argv) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); for (bool only_meta : {true, false}) { - std::pair result = test_roundtrip(dev, seed, only_meta); + std::pair result = test_roundtrip(dev, seed, only_meta, ROUNDTRIP_READ_MODE_FILE); npass += result.first; ntest += result.second; } { - std::pair result = test_roundtrip(dev, seed, /*only_meta=*/false, ROUNDTRIP_READ_BUFFER); + std::pair result = test_roundtrip(dev, seed, /*only_meta=*/false, ROUNDTRIP_READ_MODE_BUFFER); npass += result.first; ntest += result.second; } { - std::pair result = test_roundtrip(dev, seed, /*only_meta=*/false, ROUNDTRIP_READ_CALLBACK); + std::pair result = test_roundtrip(dev, seed, /*only_meta=*/false, ROUNDTRIP_READ_MODE_CALLBACK); npass += result.first; ntest += result.second; } From 7157cc0a56c25f6253583487554e5a6bdaca405f Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Sat, 23 May 2026 03:17:56 +0200 Subject: [PATCH 13/14] feat: `max_chunk_read == 0` means `SIZE_MAX` --- ggml/include/gguf.h | 2 ++ ggml/src/gguf.cpp | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ggml/include/gguf.h b/ggml/include/gguf.h index 0ce079cac33..a2e9b8c505f 100644 --- a/ggml/include/gguf.h +++ b/ggml/include/gguf.h @@ -83,6 +83,8 @@ extern "C" { GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); GGML_API struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, struct gguf_init_params params); + + // max_chunk_read is the maximum number of bytes that the GGUF code will read at once from the callback, a value of 0 means no limit GGML_API struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, uint64_t max_expected_size, struct gguf_init_params params); GGML_API void gguf_free(struct gguf_context * ctx); diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index 3a395dcaf30..b980ee2c70d 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -894,11 +894,11 @@ static struct gguf_context * gguf_init_from_reader(const struct gguf_reader & gr } struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, uint64_t max_expected_size, struct gguf_init_params params) { - if (callback == nullptr || max_chunk_read == 0) { + if (callback == nullptr) { return nullptr; } - const struct gguf_reader gr(callback, userdata, max_chunk_read, 0, max_expected_size); + const struct gguf_reader gr(callback, userdata, max_chunk_read == 0 ? SIZE_MAX : max_chunk_read, 0, max_expected_size); return gguf_init_from_reader(gr, params); } From 5908c5893a64fc666657eb9fe0130f047892e7d5 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Sun, 24 May 2026 22:07:00 +0200 Subject: [PATCH 14/14] fix: seeking in a gguf file with no tensors --- ggml/include/gguf.h | 2 +- ggml/src/gguf.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/include/gguf.h b/ggml/include/gguf.h index a2e9b8c505f..67851ba6f16 100644 --- a/ggml/include/gguf.h +++ b/ggml/include/gguf.h @@ -93,7 +93,7 @@ extern "C" { GGML_API uint32_t gguf_get_version (const struct gguf_context * ctx); GGML_API size_t gguf_get_alignment (const struct gguf_context * ctx); - GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx); + GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx); // padded to gguf_get_alignment if and only if the gguf_context contains at least one tensor GGML_API int64_t gguf_get_n_kv(const struct gguf_context * ctx); GGML_API int64_t gguf_find_key(const struct gguf_context * ctx, const char * key); // returns -1 if key is not found diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index b980ee2c70d..5e198618251 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -749,7 +749,7 @@ static struct gguf_context * gguf_init_from_reader(const struct gguf_reader & gr GGML_ASSERT(int64_t(ctx->info.size()) == n_tensors); // we require the data section to be aligned, so take into account any padding - if (!gr.seek(GGML_PAD(gr.tell(), ctx->alignment))) { + if (n_tensors > 0 && !gr.seek(GGML_PAD(gr.tell(), ctx->alignment))) { GGML_LOG_ERROR("%s: failed to seek to beginning of data section\n", __func__); gguf_free(ctx); return nullptr;