From 350dfccb9f647873099ef94c865b28e57a175808 Mon Sep 17 00:00:00 2001 From: bri-prism <288398250+bri-prism@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:48:41 -0700 Subject: [PATCH 1/2] speculative: take the draft block layout from the draft model, not the CLI type A DSpark-lineage drafter reads its drafted block anchor-first; a DFlash drafter puts the anchor token in row 0 and predicts from row 1. That choice was driven solely by --spec-type, so running a DSpark drafter as draft-dflash took the DFlash branch and read every drafted token one row late. Nothing errored: the drafter loaded, drafted at full speed, and produced tokens the target rejected almost every time. Output stayed correct, so only throughput showed it. Measured on one RTX 4090, same binary, same weights, same prompts: dflash layout (as requested): 0.272% acceptance, mean len 1.01 dspark layout (as required): 51.385% acceptance, mean len 3.04 These files declare general.architecture = dflash whichever lineage they come from, so the requested type cannot be trusted here. The Markov head is the on-disk marker of the DSpark lineage and is already loaded, so derive the layout from the model and warn when the requested type disagrees. The resolved layout now appears in the init log next to sample_from_anchor. Adds an acceptance-floor test. The existing coverage asserts that drafting happens, not that anything is accepted, which is why a 185x collapse passed every test in the file. The fixture drafter is deliberately mismatched so its acceptance is only 15-17%; the floor sits at 5%, well under that and well over a collapse. --- common/speculative.cpp | 27 ++++++++++++--- src/llama-ext.h | 4 +++ src/llama-model.cpp | 4 +++ tools/server/tests/unit/test_speculative.py | 37 +++++++++++++++++++++ 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/common/speculative.cpp b/common/speculative.cpp index f2ef82e5f424..fc476159905f 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -923,8 +923,11 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { int32_t block_size = 0; llama_token mask_token_id = 0; - // draft-dspark: the draft carries a Markov head and uses an anchor-first block layout - const bool is_dspark; + // draft-dspark: the draft carries a Markov head and uses an anchor-first block layout. + // Derived from the draft model itself, not from the requested type: reading a DSpark + // draft with the DFlash layout is off by one row and silently collapses acceptance + // (measured 0.27% against 51% on the same binary and weights). + bool is_dspark = false; // dspark speculators bool sample_from_anchor = true; @@ -939,7 +942,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH) : common_speculative_impl(type, n_seq) , params(params.draft) - , is_dspark(type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK) { auto * ctx_tgt = this->params.ctx_tgt; auto * ctx_dft = this->params.ctx_dft; @@ -952,6 +954,20 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { target_layer_ids_n = llama_model_target_layer_ids_n(model_dft); GGML_ASSERT(target_layer_ids_n > 0 && "DFlash model has no target_layer_ids"); + // These files declare general.architecture = dflash whichever lineage they come from, + // so the requested type cannot be trusted to pick the block layout. The Markov head is + // the on-disk marker of the DSpark lineage and is already loaded by then. + is_dspark = llama_model_has_dspark_markov_head(model_dft); + const bool type_says_dspark = (type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK); + if (type_says_dspark != is_dspark) { + LOG_WRN("%s: draft model carries %s, but --spec-type requested %s. Using the %s block " + "layout that the model requires; the other layout reads drafts one row off and " + "acceptance collapses to near zero.\n", __func__, + is_dspark ? "a DSpark Markov head" : "no DSpark Markov head", + common_speculative_type_to_str(type).c_str(), + is_dspark ? "DSpark" : "DFlash"); + } + n_embd_tgt = llama_model_n_embd(model_tgt); n_embd_dec = llama_model_n_embd(model_dft); n_embd_enc = (int32_t) target_layer_ids_n * n_embd_tgt; @@ -971,8 +987,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str()); LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min); - LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u, sample_from_anchor=%s\n", __func__, - block_size, mask_token_id, target_layer_ids_n, sample_from_anchor ? "true" : "false"); + LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u, sample_from_anchor=%s, block_layout=%s\n", __func__, + block_size, mask_token_id, target_layer_ids_n, sample_from_anchor ? "true" : "false", + is_dspark ? "dspark (anchor-first)" : "dflash (anchor at row 0)"); // DFlash input is [id_last, * (block_size-1)]: in-place denoising yields at most // block_size-1 draft tokens, anchor-first DSpark yields a full block_size draft tokens diff --git a/src/llama-ext.h b/src/llama-ext.h index 35d6e58adfa8..3cc2bb2c42d3 100644 --- a/src/llama-ext.h +++ b/src/llama-ext.h @@ -124,6 +124,10 @@ LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx); LLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_model * model); // returns the number of extracted layers from target model LLAMA_API uint32_t llama_model_target_layer_ids_n(const struct llama_model * model); +// returns true if the draft model carries a DSpark Markov head, which implies the +// anchor-first block layout -- the file says general.architecture = dflash either way, +// so this is the only reliable way to tell the two lineages apart +LLAMA_API bool llama_model_has_dspark_markov_head(const struct llama_model * model); // retrieves the whole token embedding matrix in F32 format (n_embd * n_vocab) // returns total number of elements or 0 on error diff --git a/src/llama-model.cpp b/src/llama-model.cpp index c34700ff563b..2882241abe69 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -3123,6 +3123,10 @@ uint32_t llama_model_target_layer_ids_n(const struct llama_model * model) { return (uint32_t) model->target_layer_ids.size(); } +bool llama_model_has_dspark_markov_head(const struct llama_model * model) { + return model->dspark_markov_w1 != nullptr; +} + uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out) { if (model->vocab.n_tokens() == 0 || model->tok_embd == nullptr) { return 0; diff --git a/tools/server/tests/unit/test_speculative.py b/tools/server/tests/unit/test_speculative.py index 5837195006bc..33bcffb065a9 100644 --- a/tools/server/tests/unit/test_speculative.py +++ b/tools/server/tests/unit/test_speculative.py @@ -53,6 +53,43 @@ def test_with_and_without_draft(): assert tokens_no_draft == tokens_draft +def test_draft_acceptance_floor(): + # test_with_and_without_draft asserts that drafting happens (draft_n > 0), not that + # any draft is accepted. Those are different failures: a drafter can run at full + # speed, produce well-formed blocks, and have essentially every token rejected. That + # is what a wrong block layout or a mis-tapped feature looks like, and it is a pure + # slowdown with identical output, so every existing assertion here still passes. + # + # This drafter/target pair is deliberately mismatched (q4_0 stories15M drafting for + # the F16 MoE), so acceptance is low by nature: 15.0% and 17.4% measured on the two + # prompts below, deterministic at temperature 0. The floor sits well under that and + # well over a collapse -- the regression this guards against measured 0.27%. + # + # Scope: this covers the draft-simple path only. The block-layout-sensitive draft + # types need their own drafters and are not exercised by any tiny fixture model. + global server + server.start() + for prompt in [ + "I believe the meaning of life is", + "Once upon a time there was a little girl who", + ]: + res = server.make_request("POST", "/completion", data={ + "prompt": prompt, + "temperature": 0.0, + "top_k": 1, + "n_predict": 64, + }) + assert res.status_code == 200 + draft_n = res.body["timings"]["draft_n"] + draft_n_accepted = res.body["timings"]["draft_n_accepted"] + assert draft_n > 0, f"nothing was drafted for {prompt!r}" + accept_rate = draft_n_accepted / draft_n + assert accept_rate > 0.05, ( + f"draft acceptance collapsed to {100 * accept_rate:.2f}% " + f"({draft_n_accepted}/{draft_n}) for {prompt!r}" + ) + + def test_different_draft_min_draft_max(): global server test_values = [ From c4c46677713e2b04b04c2d0783322a6ee213b422 Mon Sep 17 00:00:00 2001 From: bri-prism <288398250+bri-prism@users.noreply.github.com> Date: Wed, 26 Aug 2026 06:57:13 -0700 Subject: [PATCH 2/2] speculative: narrow the layout claims to what the Markov head establishes Review caught that the Markov head does not imply an anchor-first block layout. A DSpark export can set dflash.sample_from_anchor=false (bonus anchor, slot 0 is a bonus token), and that path starts drafts at row 1, same as DFlash. For those models the DSpark branch differs by confidence truncation, not a row offset. The detection itself is unchanged and still correct. What was wrong was the surrounding documentation and diagnostics: - the public accessor comment claimed the head implies the layout - the is_dspark comment claimed every DSpark model is anchor-first - the init log always printed "anchor-first", even when the code started at row 1 - the mismatch warning promised acceptance collapse for a case where both paths read the same rows The log field is now lineage=dspark|dflash next to the existing sample_from_anchor, so the two facts are reported separately instead of one being inferred from the other. The warning names the real consequences. Also shortens the acceptance-floor test comment to two lines per AGENTS.md. --- common/speculative.cpp | 22 ++++++++++----------- src/llama-ext.h | 5 ++--- tools/server/tests/unit/test_speculative.py | 15 ++------------ 3 files changed, 14 insertions(+), 28 deletions(-) diff --git a/common/speculative.cpp b/common/speculative.cpp index fc476159905f..9e4f7e22cbdf 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -923,10 +923,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { int32_t block_size = 0; llama_token mask_token_id = 0; - // draft-dspark: the draft carries a Markov head and uses an anchor-first block layout. - // Derived from the draft model itself, not from the requested type: reading a DSpark - // draft with the DFlash layout is off by one row and silently collapses acceptance - // (measured 0.27% against 51% on the same binary and weights). + // draft-dspark: the draft carries a Markov head. Comes from the model, not the + // requested type. The DSpark path also truncates on confidence, and for + // sample_from_anchor models it starts one row earlier. bool is_dspark = false; // dspark speculators @@ -954,15 +953,14 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { target_layer_ids_n = llama_model_target_layer_ids_n(model_dft); GGML_ASSERT(target_layer_ids_n > 0 && "DFlash model has no target_layer_ids"); - // These files declare general.architecture = dflash whichever lineage they come from, - // so the requested type cannot be trusted to pick the block layout. The Markov head is - // the on-disk marker of the DSpark lineage and is already loaded by then. + // Both lineages declare general.architecture = dflash, so the requested type cannot + // pick the draft path. The Markov head is the on-disk marker and is already loaded. is_dspark = llama_model_has_dspark_markov_head(model_dft); const bool type_says_dspark = (type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK); if (type_says_dspark != is_dspark) { - LOG_WRN("%s: draft model carries %s, but --spec-type requested %s. Using the %s block " - "layout that the model requires; the other layout reads drafts one row off and " - "acceptance collapses to near zero.\n", __func__, + LOG_WRN("%s: draft model carries %s, but --spec-type requested %s. Using %s, which is " + "what the model needs. The wrong path drops confidence truncation, and for " + "sample_from_anchor models it also reads the drafts one row late.\n", __func__, is_dspark ? "a DSpark Markov head" : "no DSpark Markov head", common_speculative_type_to_str(type).c_str(), is_dspark ? "DSpark" : "DFlash"); @@ -987,9 +985,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str()); LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min); - LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u, sample_from_anchor=%s, block_layout=%s\n", __func__, + LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u, sample_from_anchor=%s, lineage=%s\n", __func__, block_size, mask_token_id, target_layer_ids_n, sample_from_anchor ? "true" : "false", - is_dspark ? "dspark (anchor-first)" : "dflash (anchor at row 0)"); + is_dspark ? "dspark" : "dflash"); // DFlash input is [id_last, * (block_size-1)]: in-place denoising yields at most // block_size-1 draft tokens, anchor-first DSpark yields a full block_size draft tokens diff --git a/src/llama-ext.h b/src/llama-ext.h index 3cc2bb2c42d3..f1d9800a7b9c 100644 --- a/src/llama-ext.h +++ b/src/llama-ext.h @@ -124,9 +124,8 @@ LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx); LLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_model * model); // returns the number of extracted layers from target model LLAMA_API uint32_t llama_model_target_layer_ids_n(const struct llama_model * model); -// returns true if the draft model carries a DSpark Markov head, which implies the -// anchor-first block layout -- the file says general.architecture = dflash either way, -// so this is the only reliable way to tell the two lineages apart +// returns true if the draft model carries a DSpark Markov head. Both lineages declare +// general.architecture = dflash, so this is how to tell them apart. LLAMA_API bool llama_model_has_dspark_markov_head(const struct llama_model * model); // retrieves the whole token embedding matrix in F32 format (n_embd * n_vocab) diff --git a/tools/server/tests/unit/test_speculative.py b/tools/server/tests/unit/test_speculative.py index 33bcffb065a9..871e5abaf93c 100644 --- a/tools/server/tests/unit/test_speculative.py +++ b/tools/server/tests/unit/test_speculative.py @@ -54,19 +54,8 @@ def test_with_and_without_draft(): def test_draft_acceptance_floor(): - # test_with_and_without_draft asserts that drafting happens (draft_n > 0), not that - # any draft is accepted. Those are different failures: a drafter can run at full - # speed, produce well-formed blocks, and have essentially every token rejected. That - # is what a wrong block layout or a mis-tapped feature looks like, and it is a pure - # slowdown with identical output, so every existing assertion here still passes. - # - # This drafter/target pair is deliberately mismatched (q4_0 stories15M drafting for - # the F16 MoE), so acceptance is low by nature: 15.0% and 17.4% measured on the two - # prompts below, deterministic at temperature 0. The floor sits well under that and - # well over a collapse -- the regression this guards against measured 0.27%. - # - # Scope: this covers the draft-simple path only. The block-layout-sensitive draft - # types need their own drafters and are not exercised by any tiny fixture model. + # The other tests check that drafting happens, not that drafts are accepted. + # This drafter/target pair is mismatched on purpose, so accept is only 15-17%. global server server.start() for prompt in [