From 271f94738e3fde9eedbe3e4b56586522dff3a7e5 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 30 Aug 2026 08:02:22 +0000 Subject: [PATCH 1/6] ggml-cuda: key the CUDA graph cache by shape The graph cache is keyed on cgraph->nodes[0] alone, so two evaluations that share a first node but differ in shape collide on one entry. Warmup needs two consecutive calls with unchanged node properties, so a workload whose batch shape varies resets warmup on nearly every call and falls back to eager launch. Speculative decoding is exactly that workload. The qwen4exp verify batch is distributed 2:13 percent, 3:11 percent, 4:75 percent as the accepted count varies, where qwen35 sits at 4:98 percent and is effectively constant. Host launch time for the qwen4exp target decode was 1.52 ms with the draft head disabled and 12.35 ms with it enabled, while GPU time was unchanged, so the regression was entirely host side. The key now mixes the first node, the last node and the node count. This is O(1) rather than a walk over every node: the existing uid early return fires on 127 of 128 decodes, so the hot path must not touch node data. An earlier all-nodes hash reintroduced exactly the per-node walk a CUDA graph exists to avoid. Measured overhead against the previous key is 0.2 to 0.6 percent, with both variants built into one binary to avoid comparing across runs. Capture churn on Qwen3.8-27B UD-Q2_K_XL drops from 52 captures and 50 destroys to 4 and 0, with identical output md5 and an unchanged speculative ratio. Across 14 distinct prefill shapes the cache instantiates 16 entries against 14 before, with no destroys and no growth, and is capped at 64 by LRU on top of the existing sweep. test-backend-ops passes 13646 of 13646 on CUDA0, and Llama-3.2-1B-Instruct Q8_0 is byte identical with no throughput change. (cherry picked from commit 5a08a717da20caa6c5c4dfaa85024adf6fc4e7fa) --- ggml/src/ggml-cuda/common.cuh | 27 +++++++++++++++++------ ggml/src/ggml-cuda/ggml-cuda.cu | 38 ++++++++++++++++++++++++++------- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 50713c012cce..fdb75a916fd1 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1456,13 +1456,18 @@ struct ggml_backend_cuda_context { int curr_stream_no = 0; #ifdef USE_CUDA_GRAPH - // Map from first_node_ptr to cuda_graph - allows multiple graphs per context - // when the computation is split across CPU/GPU (e.g., with --n-cpu-moe) - std::unordered_map> cuda_graphs; + // Map from graph key to cuda_graph - allows multiple graphs per context when the + // computation is split across CPU/GPU (e.g., with --n-cpu-moe), and when the same + // split is called with different tensor shapes (e.g. a speculative verify batch) + std::unordered_map> cuda_graphs; + + // a cuda graph instance is only valid for the shapes it captured, so a caller that + // alternates shapes needs one instance per shape to stay on the graph path + static const size_t max_cuda_graphs = 64; int64_t last_graph_eviction_sweep = 0; - ggml_cuda_graph * cuda_graph(const void * first_node_ptr) { + ggml_cuda_graph * cuda_graph(uint64_t graph_key) { const int64_t time_now = ggml_time_us(); // sweep every 5s, evicting cuda graphs unused for >=10s @@ -1477,9 +1482,19 @@ struct ggml_backend_cuda_context { } } - auto it = cuda_graphs.find(first_node_ptr); + auto it = cuda_graphs.find(graph_key); if (it == cuda_graphs.end()) { - it = cuda_graphs.emplace(first_node_ptr, std::make_unique()).first; + // a workload with many distinct shapes must not grow this without bound + while (cuda_graphs.size() >= max_cuda_graphs) { + auto lru = cuda_graphs.begin(); + for (auto c = cuda_graphs.begin(); c != cuda_graphs.end(); ++c) { + if (c->second->last_used_time < lru->second->last_used_time) { + lru = c; + } + } + cuda_graphs.erase(lru); + } + it = cuda_graphs.emplace(graph_key, std::make_unique()).first; } it->second->last_used_time = time_now; return it->second.get(); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 043511b72327..7160bedb55f8 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2594,14 +2594,36 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { return use_cuda_graph; } -static const void * ggml_cuda_graph_get_key(ggml_cgraph * cgraph) { - return cgraph->nodes[0]; +// the key identifies both the split (its first node) and the shapes it was called with. +// a captured cuda graph hard-codes the shapes, so a caller that alternates shapes - a +// speculative verify batch, for example - needs a separate instance per shape. with a +// single key per split, every shape change resets the warmup and no graph is ever used. +// +// this stays O(1) on purpose: walking every node undoes the point of a cuda graph, which is +// to not touch per-node data on the hot path. the first and last node carry the batch +// dimension, which is what changes when a verify batch changes size. a shape this does not +// separate just shares an entry and re-captures, exactly as before, so it can only help. +static uint64_t ggml_cuda_graph_get_key(ggml_cgraph * cgraph) { + uint64_t key = (uint64_t) (uintptr_t) cgraph->nodes[0]; + + auto mix = [&key](uint64_t v) { + key = (key ^ v) * 0x100000001b3ull; + }; + + mix(cgraph->n_nodes); + + for (int d = 0; d < GGML_MAX_DIMS; d++) { + mix(cgraph->nodes[0]->ne[d]); + mix(cgraph->nodes[cgraph->n_nodes - 1]->ne[d]); + } + + return key; } static bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph) { bool res = false; - const void * graph_key = ggml_cuda_graph_get_key(cgraph); + const uint64_t graph_key = ggml_cuda_graph_get_key(cgraph); ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); if (cgraph->uid != 0 && @@ -2640,7 +2662,7 @@ static bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx return res; } -static void ggml_cuda_graph_update_executable(ggml_backend_cuda_context * cuda_ctx, const void * graph_key) { +static void ggml_cuda_graph_update_executable(ggml_backend_cuda_context * cuda_ctx, uint64_t graph_key) { ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); #if CUDART_VERSION >= 12000 @@ -4209,7 +4231,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph return 0; } -static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, const bool use_cuda_graph, const bool cuda_graph_update_required, const void * graph_key) { +static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, const bool use_cuda_graph, const bool cuda_graph_update_required, uint64_t graph_key) { bool graph_evaluated_or_captured = false; // flag used to determine whether it is an integrated_gpu @@ -4428,7 +4450,7 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud } #ifdef USE_CUDA_GRAPH -static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, const void * graph_key) { +static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, uint64_t graph_key) { ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); if (graph->graph == nullptr) { @@ -4451,7 +4473,7 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, bool use_cuda_graph = false; bool cuda_graph_update_required = false; - const void * graph_key = nullptr; + uint64_t graph_key = 0; #ifdef USE_CUDA_GRAPH graph_key = ggml_cuda_graph_get_key(cgraph); @@ -4621,7 +4643,7 @@ static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph } #ifdef USE_CUDA_GRAPH - const void * graph_key = ggml_cuda_graph_get_key(cgraph); + const uint64_t graph_key = ggml_cuda_graph_get_key(cgraph); const bool use_cuda_graph = ggml_cuda_graph_set_enabled(cuda_ctx, graph_key); #else const bool use_cuda_graph = false; From 52d536067a83512f215a55d42aae2e48eeac9f62 Mon Sep 17 00:00:00 2001 From: Daniel Han <23090290+danielhanchen@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:24:53 -0700 Subject: [PATCH 2/6] llama: prefetch lazily read gather rows on macOS too llama_prefetch_rows only issued madvise(MADV_WILLNEED) on Linux, so the macOS prebuilt read per-layer embedding rows (Gemma 4 PLE, the qwen4exp n-gram table) one page fault at a time. madvise and MADV_WILLNEED exist there with the same meaning. Carries the macOS half of unslothai#137. --- src/llama-mmap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp index 045489df7a35..5b2756360bba 100644 --- a/src/llama-mmap.cpp +++ b/src/llama-mmap.cpp @@ -835,7 +835,7 @@ const bool llama_mlock::SUPPORTED = false; #endif void llama_prefetch(llama_memory_ranges mr) { -#if defined(__linux__) || (defined(_WIN32) && _WIN32_WINNT >= 0x602) +#if defined(__linux__) || defined(__APPLE__) || (defined(_WIN32) && _WIN32_WINNT >= 0x602) if (mr.empty()) { return; } From 5ddb9233c3c34b7e05085f3ec89ca092b6817439 Mon Sep 17 00:00:00 2001 From: Daniel Han <23090290+danielhanchen@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:25:17 -0700 Subject: [PATCH 3/6] llama: keep the glm5-next indexer cache F16 under a quantized -ctk The glm5-next indexer cache rows hold the compressor gates next to the keys, and the gates feed a softmax. Passing -ctk q8_0 through to that cache quantized them too. Keep it F16 then, as unslothai's earlier GLM-5-Next support did; qwen4exp rows carry no gate and are unchanged. --- src/llama-memory-hybrid-idx.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/llama-memory-hybrid-idx.cpp b/src/llama-memory-hybrid-idx.cpp index 0bfba3ce0b51..03ce00140d8c 100644 --- a/src/llama-memory-hybrid-idx.cpp +++ b/src/llama-memory-hybrid-idx.cpp @@ -65,10 +65,13 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx( hparams_idx.n_embd_head_k_mla_impl = model.hparams.indexer_head_size; hparams_idx.n_embd_head_v_mla_impl = model.hparams.indexer_head_size; - LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size); + // glm5-next caches the compressor gates beside the keys and they feed a softmax, so a quantized -ctk stays F16 here + const ggml_type type_idx = model.arch == LLM_ARCH_GLM5_NEXT && ggml_is_quantized(type_k) ? GGML_TYPE_F16 : type_k; + + LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells, type = %s\n", __func__, kv_size, ggml_type_name(type_idx)); return new llama_kv_cache( - model, hparams_idx, type_k, type_v, v_trans, offload, unified, + model, hparams_idx, type_idx, type_v, v_trans, offload, unified, kv_size, n_seq_max, n_pad, n_swa, swa_type, nullptr, filter_idx, nullptr, nullptr, "idx_"); }()) {} From b96a713a48313a32c92fd6f77b367ea9d803417b Mon Sep 17 00:00:00 2001 From: Daniel Han <23090290+danielhanchen@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:04:43 -0700 Subject: [PATCH 4/6] glm5-next : NextN (MTP) draft graph The NextN block was loaded but build_arch_graph threw for LLM_GRAPH_TYPE_DECODER_MTP. Build it: eh_proj([enorm(e) ; hnorm(h)]), then one DSA layer with a plain residual (the block has no mHC tensors), the MoE + shared expert FFN, and the head norm and head (nextn shared ones when present). It reuses the trunk's k-pool input and DSA layer; the MTP context memory already holds only the NextN block. h_nextn is the post-norm hidden state, the same kind the trunk exports. --- src/models/glm5-next.cpp | 88 +++++++++++++++++++++++++++++++++++++++- src/models/models.h | 9 ++++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/src/models/glm5-next.cpp b/src/models/glm5-next.cpp index 09382fdac98f..f33f334633d2 100644 --- a/src/models/glm5-next.cpp +++ b/src/models/glm5-next.cpp @@ -187,7 +187,7 @@ void llama_model_glm5_next::load_arch_tensors(llama_model_loader & ml) { std::unique_ptr llama_model_glm5_next::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { - throw std::runtime_error("GLM5-Next NextN graph not implemented yet"); + return std::make_unique(*this, params); } return std::make_unique(*this, params); } @@ -681,6 +681,92 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_p ggml_build_forward_expand(gf, cur); } +// NextN draft head: one DSA layer with a plain residual (no mHC), fed by eh_proj([enorm(e) ; hnorm(h)]) +llama_model_glm5_next::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : + graph(model, params, no_build{}) { + GGML_ASSERT(hparams.n_layer_nextn == 1 && "GLM5-Next MTP has a single NextN block"); + GGML_ASSERT(ubatch.token && "GLM5-Next MTP requires token input"); + + const int il = hparams.n_layer(); + const auto & layer = model.layers[il]; + + GGML_ASSERT(layer.nextn.eh_proj && layer.nextn.enorm && layer.nextn.hnorm && + "NextN block missing, load the model with MTP enabled"); + + auto inp = std::make_unique(n_embd); + + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->tokens); + + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens); + ggml_set_input(inp->embd); + + inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens); + ggml_set_input(inp->h); + ggml_set_name(inp->h, "mtp_h_input"); + + ggml_tensor * tok_embd = ggml_get_rows(ctx0, layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd, inp->tokens); + cb(tok_embd, "mtp_tok_embd", il); + + ggml_tensor * h = inp->h; + + res->add_input(std::move(inp)); + + const auto * mctx_hyb = static_cast(mctx); + + auto * inp_hyb = build_inp_mem_hybrid_k(); + auto * inp_kpool = build_inp_kpool(mctx_hyb); + + // the draft memory has no recurrent layer, but its input still has to be allocated + ggml_build_forward_expand(gf, inp_hyb->get_recr()->s_copy); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il); + ggml_tensor * h_norm = build_norm(h, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il); + + ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, ggml_concat(ctx0, e_norm, h_norm, 0)); + cb(cur, "mtp_eh_proj", il); + + ggml_tensor * residual = cur; + ggml_tensor * prev_sel = nullptr; + + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cur = build_dsa_layer(cur, layer, mctx_hyb, inp_hyb->get_attn(), inp_kpool, &prev_sel, il); + cur = ggml_add(ctx0, cur, residual); + residual = cur; + + cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, layer.ffn_up_exps, layer.ffn_gate_exps, layer.ffn_down_exps, + layer.ffn_exp_probs_b, n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, il); + ggml_tensor * ffn_shexp = build_ffn(cur, + layer.ffn_up_shexp, nullptr, nullptr, + layer.ffn_gate_shexp, nullptr, nullptr, + layer.ffn_down_shexp, nullptr, nullptr, + nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il); + cur = ggml_add(ctx0, ggml_add(ctx0, moe_out, ffn_shexp), residual); + cb(cur, "mtp_ffn_out", il); + + // post-norm like the trunk's h_nextn, so the next draft step reads the same kind of h + cur = build_norm(cur, layer.nextn.shared_head_norm ? layer.nextn.shared_head_norm : model.output_norm, nullptr, LLM_NORM_RMS, -1); + + ggml_tensor * cur_out = inp_out_ids ? ggml_get_rows(ctx0, cur, inp_out_ids) : cur; + res->t_h_nextn = cparams.embeddings_nextn_masked ? cur_out : cur; + cb(res->t_h_nextn, "h_nextn", -1); + ggml_build_forward_expand(gf, res->t_h_nextn); + + res->t_embd = cur_out; + + cur = ggml_mul_mat(ctx0, layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output, cur_out); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} + // KDA layer, g_a/g_b output gate ggml_tensor * llama_model_glm5_next::graph::build_kda_layer( diff --git a/src/models/models.h b/src/models/models.h index 8f04933a67ad..590f40852077 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2631,6 +2631,11 @@ struct llama_model_glm5_next : public llama_model_base { struct graph : public llm_build_delta_net_base { graph(const llama_model & model, const llm_graph_params & params); + // the helpers alone, graph_mtp builds its own body + struct no_build {}; + graph(const llama_model & model, const llm_graph_params & params, no_build) : + llm_build_delta_net_base(params), model(model) {} + // collapse the hc streams with per-stream weights ggml_tensor * build_hc_pre( ggml_tensor * x, @@ -2679,6 +2684,10 @@ struct llama_model_glm5_next : public llama_model_base { }; + struct graph_mtp : public graph { + graph_mtp(const llama_model & model, const llm_graph_params & params); + }; + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; From 0e8ffdc72fce157ef5dbd72134b6171367bee6ce Mon Sep 17 00:00:00 2001 From: Daniel Han <23090290+danielhanchen@users.noreply.github.com> Date: Thu, 8 Oct 2026 03:49:33 -0700 Subject: [PATCH 5/6] ggml-cuda: drop the CUDA graph cache count cap, as upstream does The shape keyed graph cache added a 64 entry LRU cap on top of the existing time based sweep, to bound a workload with many distinct shapes. Tensor split needs about 2 * n_layers + 1 graphs per device for a single shape, one per segment between the cross-device reductions, so every model past ~31 layers exceeds 64. The LRU then evicts graphs that are still in use and every token re-captures instead of replaying, which is as slow as running with GGML_CUDA_DISABLE_GRAPHS=1. Upstream tried the same cap (64, then 128) in ggml-org/llama.cpp#21611 and dropped it in review for this reason, keeping only the sweep that evicts graphs unused for 10 s. Do the same here. The shape key stays, so an alternating speculative verify batch keeps one graph per shape. Refs unslothai/unsloth#12468 --- ggml/src/ggml-cuda/common.cuh | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index fdb75a916fd1..72b6b16a319f 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1459,11 +1459,12 @@ struct ggml_backend_cuda_context { // Map from graph key to cuda_graph - allows multiple graphs per context when the // computation is split across CPU/GPU (e.g., with --n-cpu-moe), and when the same // split is called with different tensor shapes (e.g. a speculative verify batch) - std::unordered_map> cuda_graphs; - // a cuda graph instance is only valid for the shapes it captured, so a caller that - // alternates shapes needs one instance per shape to stay on the graph path - static const size_t max_cuda_graphs = 64; + // alternates shapes needs one instance per shape to stay on the graph path. the map is + // bounded only by the time based sweep below, as upstream does: tensor split mode alone + // needs about 2 * n_layers + 1 graphs per device and shape, so any count cap small enough + // to matter evicts graphs that are still in use and re-captures them on every token + std::unordered_map> cuda_graphs; int64_t last_graph_eviction_sweep = 0; @@ -1484,16 +1485,6 @@ struct ggml_backend_cuda_context { auto it = cuda_graphs.find(graph_key); if (it == cuda_graphs.end()) { - // a workload with many distinct shapes must not grow this without bound - while (cuda_graphs.size() >= max_cuda_graphs) { - auto lru = cuda_graphs.begin(); - for (auto c = cuda_graphs.begin(); c != cuda_graphs.end(); ++c) { - if (c->second->last_used_time < lru->second->last_used_time) { - lru = c; - } - } - cuda_graphs.erase(lru); - } it = cuda_graphs.emplace(graph_key, std::make_unique()).first; } it->second->last_used_time = time_now; From 7fa42ff5efb5388ed8f47cec7bce4f8fa7dfe546 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 14 Oct 2026 13:59:00 -0700 Subject: [PATCH 6/6] ggml-cuda : graph key: handle a graph with no nodes --- ggml/src/ggml-cuda/ggml-cuda.cu | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 395cef18f174..331e2bc95496 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2684,6 +2684,10 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { // dimension, which is what changes when a verify batch changes size. a shape this does not // separate just shares an entry and re-captures, exactly as before, so it can only help. static uint64_t ggml_cuda_graph_get_key(ggml_cgraph * cgraph) { + if (cgraph->n_nodes == 0) { + return 0; + } + uint64_t key = (uint64_t) (uintptr_t) cgraph->nodes[0]; auto mix = [&key](uint64_t v) {