From 817eb251f9f603aa2f5226a7c80d59b237527f62 Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 22 Oct 2026 02:42:34 +0900 Subject: [PATCH 01/10] =?UTF-8?q?accepts:=20Body35's=20placed=20card=20leg?= =?UTF-8?q?=20takes=20the=20common=20leg's=20routed=20rows=20(Q6=5FK,=20IQ?= =?UTF-8?q?3=5FXXS,=20IQ4=5FXS=20and=20Q8=5F0=20gate=C2=B7up;=20IQ3=5FXXS,?= =?UTF-8?q?=20IQ4=5FXS,=20Q5=5F1,=20Q8=5F0=20and=20IQ4=5FNL=20down);=20the?= =?UTF-8?q?=20coverage=20pins=20and=20the=20host-only=20causes=20follow=20?= =?UTF-8?q?the=20table=20(#75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/gguf/src/card.rs | 4 + crates/gguf/src/qtype.rs | 2 +- crates/gpu-gates/src/record.rs | 15 ++- crates/model/src/arch/coverage.rs | 207 +++++++++++++++++------------- crates/placement/src/accepts.rs | 180 +++++++++++++++++++++++--- crates/placement/src/kernels.rs | 51 +++++++- crates/placement/src/placement.rs | 12 +- 7 files changed, 354 insertions(+), 117 deletions(-) diff --git a/crates/gguf/src/card.rs b/crates/gguf/src/card.rs index 606b6d5c8..81560f119 100644 --- a/crates/gguf/src/card.rs +++ b/crates/gguf/src/card.rs @@ -87,6 +87,10 @@ pub enum GateUpEntry { /// `KquantKernels::enqueue_gate_up_q8_0` (`kq_gate_up_act_q8_0`, the /// file's `block_q8_0` stream) in `crates/gpu/src/kquant/sel.rs`. Q8_0, + /// `Q6kSelKernels::enqueue_gate_up` (`q6k_gate_up_sel`, the file's 210-byte + /// `block_q6_K` stream over the `Q8Act`'s q6 plane) in + /// `crates/gpu/src/q6k_sel.rs`. + Q6k, /// `QuantSelKernels::enqueue_gate_up` (`iq2_xs_gate_up_sel`, the file's /// 74-byte `block_iq2_xs` stream) in `crates/gpu/src/iq_sel.rs`. Iq2xs, diff --git a/crates/gguf/src/qtype.rs b/crates/gguf/src/qtype.rs index 70e9d837b..359c3e585 100644 --- a/crates/gguf/src/qtype.rs +++ b/crates/gguf/src/qtype.rs @@ -96,7 +96,7 @@ pub const QROWS: &[QRow] = &[ QRow { ty: T::Q5_K, name: "q5_K", blck: 256, bytes: 176, dequant: true, host_act: H::Q8_2X4, card: card!(KQuant - Q5k (Q5k, Q8Act) Q5k Q5k - Q5k) }, QRow { ty: T::Q6_K, name: "q6_K", blck: 256, bytes: 210, dequant: true, host_act: H::Q8_2X4, - card: card!(KQuant KQuant - (Q6k, Q8Act) Q6k Q6k Q6k Q6k) }, + card: card!(KQuant KQuant Q6k (Q6k, Q8Act) Q6k Q6k Q6k Q6k) }, // q8_K: ggml's activation form, mainline's block (llama.cpp 36a73916 ggml-common.h:371-376, ggml.c:880): float d, 256 int8 // codes, 16 int16 bsums = 292 B. ik's block adds a float sum, 296 B (c10fbbcc ggml-common.h:405-411): a divergence. QRow { ty: T::Q8_K, name: "q8_K", blck: 256, bytes: 292, dequant: false, host_act: H::None, card: Card::NONE }, diff --git a/crates/gpu-gates/src/record.rs b/crates/gpu-gates/src/record.rs index d5efaf410..085ce4f4c 100644 --- a/crates/gpu-gates/src/record.rs +++ b/crates/gpu-gates/src/record.rs @@ -4013,7 +4013,7 @@ mod tests { #[test] fn host_only_records_name_causes_and_skip_an_all_host_choice() { - use gguf::GgmlType::{Q4_K, Q5_K, Q6_K, Q8_0}; + use gguf::GgmlType::{IQ2_XS, MXFP4, Q4_K, Q5_K, Q6_K}; use model::placement::accepts::{Program, routed_rule}; use model::placement::{ModelTensor, ModelTensors, Role}; let mut model = ModelTensors { @@ -4023,10 +4023,15 @@ mod tests { experts_used: 2, bf16_words: |_| false, }; + // PIN(2026-10-22): the Q8_0 down and the Q6_K gate and up the layers named + // here once were refused by the qwen35moe card leg; the common leg runs the + // Q8_0 down now, so the layers take types the leg still has no GEMM for + // (an MXFP4 down, an IQ2_XS gate and up); the causes and their order are + // the ones the test held. for (layer, types) in [ [Q4_K, Q4_K, Q6_K], - [Q4_K, Q4_K, Q8_0], - [Q6_K, Q6_K, Q6_K], + [Q4_K, Q4_K, MXFP4], + [IQ2_XS, IQ2_XS, Q6_K], [Q4_K, Q5_K, Q4_K], ] .into_iter() @@ -4066,7 +4071,7 @@ mod tests { ] ); // The rendered reason carries the offending type and keeps distinct - // causes apart: the Q8_0 down is an unsupported part, the Q4_K/Q5_K + // causes apart: the MXFP4 down is an unsupported part, the Q4_K/Q5_K // gate/up a mismatch. The wording itself is not pinned. let why = |layer: u64, tensor: &str| { causes @@ -4078,7 +4083,7 @@ mod tests { }; let unsupported = why(1, "ffn_down_exps.weight"); let mismatch = why(3, "ffn_up_exps.weight"); - assert!(unsupported.starts_with(&Q8_0.to_string()), "{unsupported}"); + assert!(unsupported.starts_with(&MXFP4.to_string()), "{unsupported}"); assert!(mismatch.starts_with(&Q5_K.to_string()), "{mismatch}"); assert_ne!(unsupported, mismatch, "distinct causes must not collapse"); // The renderer hands the policy's own reason through, so losing or diff --git a/crates/model/src/arch/coverage.rs b/crates/model/src/arch/coverage.rs index 98f573d0b..d0e8d2c94 100644 --- a/crates/model/src/arch/coverage.rs +++ b/crates/model/src/arch/coverage.rs @@ -20,6 +20,7 @@ use bloomery_placement::accepts::{ BOLT_LEG, embed_of_widened, head_of_widened, qwen3moe_dense, resident_down_runs, }; pub use bloomery_placement::accepts::{Program, program_of, qwen35_down, qwen35_gate_up}; +use bloomery_placement::accepts::{qwen35_common_down, qwen35_common_gate_up}; pub use bloomery_placement::kernels::{DownEntry, GateUpEntry, SiteEntry}; use gguf::GgmlType; use models::shape::{RouterBody, gqa_row, gqa_row_v, select_router}; @@ -1378,9 +1379,10 @@ struct RowPin { /// body's attention sites ([`qwen3moe_dense`]: the table's dense row kept to /// the forms that body's arena carries, the fused Q4_K ones among them), head /// ([`head_of_widened`]: `head.rs`) and token embedding ([`embed_of_widened`]: -/// `elem.rs`), each the table's row but BF16, and routed stacks ([`qwen35_gate_up`], [`qwen35_down`]: the FFN -/// dispatch both bodies share), and `Body35`'s FFN ([`qwen35_gate_up`], -/// [`qwen35_down`], [`qwen35_shared`]), +/// `elem.rs`), each the table's row but BF16, and routed stacks (the qwen3moe +/// body's [`qwen35_gate_up`], [`qwen35_down`]: the FFN dispatch; `Body35`'s +/// [`qwen35_common_gate_up`], [`qwen35_common_down`]: the dispatch and the +/// common card leg), and `Body35`'s shared expert ([`qwen35_shared`]), /// and the glm5next and mimo2 bodies' token embedding (the host's rows, /// `GgmlType::has_dequant`), and the gemma-embedding2 encoder's token table /// and per-layer input projection (the host's rows), each `what` naming the @@ -1479,16 +1481,16 @@ const ROW_PINS: &[RowPin] = &[ role: Role::RoutedExperts, matrices: true, names: Some(routed_gate_up), - what: "routed experts gate and up (the body reads q4_K and q5_K)", - row: |ty| qwen35_gate_up(ty).is_some(), + what: "routed experts gate and up (the body reads q4_K, q5_K, q6_K, iq3_xxs, iq4_xs and q8_0)", + row: qwen35_common_gate_up, }, RowPin { program: Program::Qwen35Body, role: Role::RoutedExperts, matrices: true, names: Some(routed_down), - what: "routed experts down (the body reads q4_K, q5_K and q6_K)", - row: |ty| qwen35_down(ty).is_some(), + what: "routed experts down (the body reads q4_K, q5_K, q6_K, iq3_xxs, iq4_xs, q5_1, q8_0 and iq4_nl)", + row: qwen35_common_down, }, RowPin { program: Program::Raon1Body, @@ -2125,43 +2127,54 @@ mod tests { } } - /// The whole-card bodies' routed pins, by part: both bodies run the FFN - /// dispatch's rows of the card kernel table, q4_K or q5_K gate and up + /// The whole-card bodies' routed pins, by part: the qwen3moe body runs the + /// FFN dispatch's rows of the card kernel table, q4_K or q5_K gate and up /// and q4_K, q5_K or q6_K down — the parts the default expert rule's card /// formats pass (q3_K, q6_K, q8_0 gate·up; q3_K, q8_0 down) reach the - /// loads' refusal otherwise. + /// loads' refusal otherwise; `Body35` runs those and the common card leg's + /// rows beside them (`qwen35_common_gate_up`, `qwen35_common_down`). // PIN(2026-10-10): `Body35`'s routed pins moved from the joined stacks' // q4_K gate·up and q4_K or q6_K down to the table's rows its dispatch // runs (`qwen35_gate_up`, `qwen35_down`): q5_K gate·up and down left its // refused types; red on the tree before the move by those two. The // qwen3moe body's moved the same way (q4_K gate·up, q4_K or q6_K down // before): the dispatch is one code path, so one row decides both. + // PIN(2026-10-22): `Body35`'s pins moved on to the rows of its placed card + // leg (`qwen35_common_gate_up`, `qwen35_common_down`: the dispatch's types + // and the common leg's): a q6_K, iq3_xxs, iq4_xs or q8_0 gate·up and an + // iq3_xxs, iq4_xs, q5_1, q8_0 or iq4_nl down left its refused types, each a type the + // common leg launches with a prompt GEMM; red on the tree before the move + // by those. The qwen3moe body keeps the dispatch's rows (its placed side + // has no common leg). #[test] fn the_whole_card_bodies_read_routed_stacks_by_part() { + use GgmlType::{ + IQ2_XS, IQ3_S, IQ3_XXS, IQ4_NL, IQ4_XS, MXFP4, Q3_K, Q4_K, Q5_1, Q5_K, Q6_K, Q8_0, + }; type Parts = (&'static str, &'static [GgmlType], &'static [GgmlType]); - let gate_up: Parts = ( - "q4_K and q5_K", - &[GgmlType::Q4_K, GgmlType::Q5_K], - &[ - GgmlType::Q3_K, - GgmlType::Q6_K, - GgmlType::Q8_0, - GgmlType::IQ4_XS, - ], - ); - let down: Parts = ( - "q4_K, q5_K and q6_K", - &[GgmlType::Q4_K, GgmlType::Q5_K, GgmlType::Q6_K], - &[ - GgmlType::Q3_K, - GgmlType::Q8_0, - GgmlType::Q5_1, - GgmlType::IQ4_NL, - ], - ); - let bodies = [ - (Program::Qwen3moeBody, gate_up, down), - (Program::Qwen35Body, gate_up, down), + let bodies: [(Program, Parts, Parts); 2] = [ + ( + Program::Qwen3moeBody, + ("q4_K and q5_K", &[Q4_K, Q5_K], &[Q3_K, Q6_K, Q8_0, IQ4_XS]), + ( + "q4_K, q5_K and q6_K", + &[Q4_K, Q5_K, Q6_K], + &[Q3_K, Q8_0, Q5_1, IQ4_NL], + ), + ), + ( + Program::Qwen35Body, + ( + "q4_K, q5_K, q6_K, iq3_xxs, iq4_xs and q8_0", + &[Q4_K, Q5_K, Q6_K, IQ3_XXS, IQ4_XS, Q8_0], + &[Q3_K, IQ2_XS, IQ3_S, MXFP4], + ), + ( + "q4_K, q5_K, q6_K, iq3_xxs, iq4_xs, q5_1, q8_0 and iq4_nl", + &[Q4_K, Q5_K, Q6_K, IQ3_XXS, IQ4_XS, Q5_1, Q8_0, IQ4_NL], + &[Q3_K, IQ2_XS, IQ3_S, MXFP4], + ), + ), ]; for (program, gate_up, down) in bodies { for (names, part, (reads, runs, refused)) in [ @@ -2250,24 +2263,48 @@ mod tests { } /// The FFN pins are the card kernel table's: over every type id, a - /// routed part's matrix is an item (of either body) exactly when - /// [`qwen35_gate_up`] or [`qwen35_down`] gives its type no entry, `Body35`'s + /// routed part's matrix is an item exactly when its body's rows give its + /// type no entry (the qwen3moe body's [`qwen35_gate_up`], [`qwen35_down`]; + /// `Body35`'s [`qwen35_common_gate_up`], [`qwen35_common_down`]), `Body35`'s /// shared expert's exactly when [`qwen35_shared`] does not, and each pin's /// text names the types its row takes (a pin reads no F32 matrix). #[test] fn the_ffn_pins_are_the_card_kernel_table() { - use super::{qwen35_down, qwen35_gate_up, qwen35_shared}; + use super::{ + qwen35_common_down, qwen35_common_gate_up, qwen35_down, qwen35_gate_up, qwen35_shared, + }; type Row = fn(GgmlType) -> bool; - let parts: [(&str, Role, Row); 3] = [ - ("ffn_gate_exps.weight", Role::RoutedExperts, |t| { - qwen35_gate_up(t).is_some() - }), - ("ffn_down_exps.weight", Role::RoutedExperts, |t| { - qwen35_down(t).is_some() - }), - ("ffn_up_shexp.weight", Role::SharedExpert, |t| { - qwen35_shared(t).is_some() - }), + let parts: [(&str, Role, Program, Row); 5] = [ + ( + "ffn_gate_exps.weight", + Role::RoutedExperts, + Program::Qwen3moeBody, + |t| qwen35_gate_up(t).is_some(), + ), + ( + "ffn_down_exps.weight", + Role::RoutedExperts, + Program::Qwen3moeBody, + |t| qwen35_down(t).is_some(), + ), + ( + "ffn_gate_exps.weight", + Role::RoutedExperts, + Program::Qwen35Body, + qwen35_common_gate_up, + ), + ( + "ffn_down_exps.weight", + Role::RoutedExperts, + Program::Qwen35Body, + qwen35_common_down, + ), + ( + "ffn_up_shexp.weight", + Role::SharedExpert, + Program::Qwen35Body, + |t| qwen35_shared(t).is_some(), + ), ]; let rows = |row: Row| -> Vec { (0..64u32) @@ -2275,50 +2312,39 @@ mod tests { .filter(|&t| t != GgmlType::F32 && row(t)) .collect() }; - let programs = |role: Role| -> &'static [Program] { - if role == Role::SharedExpert { - &[Program::Qwen35Body] - } else { - &[Program::Qwen3moeBody, Program::Qwen35Body] - } - }; - for (name, role, row) in parts { - for program in programs(role) { - for id in 0..64u32 { - let ty = GgmlType::from_u32(id); - if ty == GgmlType::F32 { - continue; - } - assert_eq!( - items(*program, &matrix(name, role, ty)).is_empty(), - row(ty), - "{program:?} {name} {ty:?}" - ); + for (name, role, program, row) in parts { + for id in 0..64u32 { + let ty = GgmlType::from_u32(id); + if ty == GgmlType::F32 { + continue; } + assert_eq!( + items(program, &matrix(name, role, ty)).is_empty(), + row(ty), + "{program:?} {name} {ty:?}" + ); } } // Each pin's text names, in any order, exactly the types its row // takes: the list inside its parentheses, "a, b and c". - for (name, role, row) in parts { - for program in programs(role) { - let text = items(*program, &matrix(name, role, GgmlType::BF16)); - let [text] = text.as_slice() else { - panic!("{program:?} {name}: one item for bf16, got {text:?}"); - }; - let list = text - .split_once("(the body reads ") - .and_then(|(_, l)| l.strip_suffix(')')) - .unwrap_or_else(|| panic!("{name}: no type list in {text:?}")); - let mut named: Vec = list - .replace(" and ", ", ") - .split(", ") - .map(str::to_string) - .collect(); - let mut want: Vec = rows(row).iter().map(ToString::to_string).collect(); - named.sort(); - want.sort(); - assert_eq!(named, want, "{program:?} {name}: {text}"); - } + for (name, role, program, row) in parts { + let text = items(program, &matrix(name, role, GgmlType::BF16)); + let [text] = text.as_slice() else { + panic!("{program:?} {name}: one item for bf16, got {text:?}"); + }; + let list = text + .split_once("(the body reads ") + .and_then(|(_, l)| l.strip_suffix(')')) + .unwrap_or_else(|| panic!("{name}: no type list in {text:?}")); + let mut named: Vec = list + .replace(" and ", ", ") + .split(", ") + .map(str::to_string) + .collect(); + let mut want: Vec = rows(row).iter().map(ToString::to_string).collect(); + named.sort(); + want.sort(); + assert_eq!(named, want, "{program:?} {name}: {text}"); } } @@ -2331,6 +2357,9 @@ mod tests { /// table names no `Body35` launch for stays an item by name. // PIN(2026-10-10): red on the tree before the table's rows (the q5_K // down, the q8_0 and q6_K shared experts beside stacks of other types). + // PIN(2026-10-22): the q8_0 routed gate·up and down it held as items run on + // the common card leg now; the routed items it keeps are types that leg has + // no prompt GEMM for (an iq2_xs gate·up, an mxfp4 down). #[test] fn the_hf_qwen36_layer_shapes_are_no_items() { use GgmlType::{Q4_K, Q5_K, Q6_K, Q8_0}; @@ -2374,14 +2403,14 @@ mod tests { ( "ffn_gate_exps.weight", Role::RoutedExperts, - Q8_0, - "q8_0 routed experts gate and up (the body reads q4_K and q5_K)", + GgmlType::IQ2_XS, + "iq2_xs routed experts gate and up (the body reads q4_K, q5_K, q6_K, iq3_xxs, iq4_xs and q8_0)", ), ( "ffn_down_exps.weight", Role::RoutedExperts, - Q8_0, - "q8_0 routed experts down (the body reads q4_K, q5_K and q6_K)", + GgmlType::MXFP4, + "mxfp4 routed experts down (the body reads q4_K, q5_K, q6_K, iq3_xxs, iq4_xs, q5_1, q8_0 and iq4_nl)", ), ( "ffn_gate_shexp.weight", diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index 89112223e..a09c53a55 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -106,15 +106,17 @@ impl Program { } } -/// The card gate·up entry `Body35` runs a routed stack of `ty` with: the -/// card kernel table's row ([`kernels::gate_up`], the decode `_sel`) when -/// the table names the type's prompt GEMM too ([`kernels::gemm`]), among -/// the entries the family's FFN dispatch launches -/// (`gpu/src/arch/qwen3moe/dispatch.rs`): the K-quant family's Q4_K and -/// Q5_K. The table's Q8_0 row reads a stack in the file's block stream, its -/// i-quant rows are `iq_sel.rs`'s, its MXFP4 row is `mxfp4_sel.rs`'s and its -/// NVFP4 row `mxfp4_sel.rs`'s; the dispatch launches none of them, so -/// such a stack is refused by name. +/// The card gate·up entry the qwen3moe family's FFN dispatch +/// (`gpu/src/arch/qwen3moe/dispatch.rs`, and the placed side's dispatch arms +/// for the types it keeps) runs a routed stack of `ty` with: the card kernel +/// table's row ([`kernels::gate_up`], the decode `_sel`) when the table names +/// the type's prompt GEMM too ([`kernels::gemm`]), among the entries that +/// dispatch launches: the K-quant family's Q4_K and Q5_K. The table's Q8_0 +/// row reads a stack in the file's block stream, its i-quant rows are +/// `iq_sel.rs`'s, its MXFP4 row is `mxfp4_sel.rs`'s and its NVFP4 row +/// `mxfp4_sel.rs`'s; the dispatch launches none of them, so a whole-card load +/// refuses such a stack by name. `Body35`'s placed card leg runs those types +/// through the common leg ([`qwen35_leg`]). #[must_use] pub const fn qwen35_gate_up(ty: GgmlType) -> Option { let (Some(entry), Some(_)) = (kernels::gate_up(ty), kernels::gemm(ty)) else { @@ -123,6 +125,7 @@ pub const fn qwen35_gate_up(ty: GgmlType) -> Option { match entry { GateUpEntry::Q4k | GateUpEntry::Q5k => Some(entry), GateUpEntry::Q8_0 + | GateUpEntry::Q6k | GateUpEntry::Iq2xs | GateUpEntry::Iq3xxs | GateUpEntry::Iq3s @@ -132,12 +135,14 @@ pub const fn qwen35_gate_up(ty: GgmlType) -> Option { } } -/// The card down entry `Body35` runs a routed stack of `ty` with: the -/// table's row ([`kernels::down`]) when it names the type's prompt GEMM too -/// ([`kernels::gemm`]), among the entries the FFN dispatch launches: the -/// downs over `Q8Act`, Q4_K, Q5_K and Q6_K. The downs over `Q8Blocks32` -/// (Q5_0, Q5_1, Q8_0, IQ4_NL, NVFP4) read an activation form the dispatch does not -/// quantize, so such a stack is refused by name. +/// The card down entry the qwen3moe family's FFN dispatch runs a routed stack +/// of `ty` with: the table's row ([`kernels::down`]) when it names the type's +/// prompt GEMM too ([`kernels::gemm`]), among the entries the FFN dispatch +/// launches: the downs over `Q8Act`, Q4_K, Q5_K and Q6_K. The downs over +/// `Q8Blocks32` (Q5_0, Q5_1, Q8_0, IQ4_NL, NVFP4) read an activation form the +/// dispatch does not quantize, so a whole-card load refuses such a stack by +/// name; `Body35`'s placed card leg runs the ones the common leg does +/// ([`qwen35_leg`]). #[must_use] pub const fn qwen35_down(ty: GgmlType) -> Option { let (Some((entry, _)), Some(_)) = (kernels::down(ty), kernels::gemm(ty)) else { @@ -158,6 +163,64 @@ pub const fn qwen35_down(ty: GgmlType) -> Option { } } +/// Whether `Body35`'s placed card leg runs a routed gate or up stack of `ty` +/// through the common leg (`crate::card_leg`'s `CardLeg`): the common leg's +/// gate·up arms ([`kernels::card_gate_up_runs`], and the Q8_0 and Q6_K arms the +/// leg launches for the stacks no other family's plan reads) of the types whose +/// prompt GEMM the table names ([`kernels::gemm`]). The wide arm reads that +/// GEMM, so IQ2_XS, IQ3_S and MXFP4, which have none, stay on the host by +/// name; NVFP4 is not admitted (no qwen35 file or fixture proves it). +#[must_use] +pub const fn qwen35_common_gate_up(ty: GgmlType) -> bool { + (kernels::card_gate_up_runs(ty) + || matches!( + kernels::gate_up(ty), + Some(GateUpEntry::Q8_0 | GateUpEntry::Q6k) + )) + && kernels::gemm(ty).is_some() +} + +/// Whether `Body35`'s placed card leg runs a routed down stack of `ty` +/// through the common leg: the arms a tensor-parallel rank runs +/// ([`kernels::rank_down_runs`]: [`kernels::card_down_runs`] and the Q6_K +/// one) of the types whose prompt GEMM the table names. The 32-value downs +/// (Q5_1, Q8_0, IQ4_NL) read the `Q8Blocks32` form the leg quantizes for +/// them. +#[must_use] +pub const fn qwen35_common_down(ty: GgmlType) -> bool { + kernels::rank_down_runs(ty) && kernels::gemm(ty).is_some() +} + +/// How `Body35`'s placed card leg runs a layer's routed stacks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Leg35 { + /// The family's own dispatch launches ([`qwen35_gate_up`], [`qwen35_down`]): + /// a Q4_K or Q5_K gate and up with a Q4_K, Q5_K or Q6_K down, the arms + /// whose launches and graph are the load's reference. + Dispatch, + /// The common leg's launches ([`qwen35_common_gate_up`], + /// [`qwen35_common_down`]): any other layer its arms run. + Common, +} + +/// The leg that runs a layer of routed stacks of types `[gate, up, down]` on +/// `Body35`'s placed card, or `None` when none does (the layer is the host +/// tier's). The one classifier the plan's rule, the load's check, the +/// residency stack table and the placed side read. +#[must_use] +pub fn qwen35_leg([gate, up, down]: [GgmlType; 3]) -> Option { + if gate != up || !qwen35_common_gate_up(gate) || !qwen35_common_down(down) { + return None; + } + Some( + if qwen35_gate_up(gate).is_some() && qwen35_down(down).is_some() { + Leg35::Dispatch + } else { + Leg35::Common + }, + ) +} + /// The dense site the qwen3moe body runs a projection of `ty` with: the /// table's row ([`kernels::dense`]) when that site reads an activation the /// body's arena carries ([`SiteAct::Q8x128`], [`SiteAct::Q8x32`], @@ -288,10 +351,14 @@ pub struct LegArms { #[must_use] pub fn leg_arms(p: Program) -> Option { match p { - Program::Qwen3moeBody | Program::Qwen35Body => Some(LegArms { + Program::Qwen3moeBody => Some(LegArms { gate_up: |ty| qwen35_gate_up(ty).is_some(), down: |ty| qwen35_down(ty).is_some(), }), + Program::Qwen35Body => Some(LegArms { + gate_up: qwen35_common_gate_up, + down: qwen35_common_down, + }), Program::Qwen38Body => Some(LegArms { gate_up: |ty| qwen38_gate_up(ty).is_some(), down: |ty| qwen38_down(ty).is_some(), @@ -338,10 +405,14 @@ pub fn leg_runs(p: Program, [gate, up, down]: [GgmlType; 3]) -> Option { #[must_use] pub fn routed_rule(p: Program) -> RoutedRule { match p { - Program::Qwen3moeBody | Program::Qwen35Body => RoutedRule::Leg { + Program::Qwen3moeBody => RoutedRule::Leg { gate_up: |ty| qwen35_gate_up(ty).map(|_| leg_card_format(ty)), down: |ty| qwen35_down(ty).map(|_| leg_card_format(ty)), }, + Program::Qwen35Body => RoutedRule::Leg { + gate_up: |ty| qwen35_common_gate_up(ty).then(|| leg_card_format(ty)), + down: |ty| qwen35_common_down(ty).then(|| leg_card_format(ty)), + }, Program::Qwen38Body => RoutedRule::Leg { gate_up: |ty| qwen38_gate_up(ty).map(|_| leg_card_format(ty)), down: |ty| qwen38_down(ty).map(|_| leg_card_format(ty)), @@ -487,6 +558,81 @@ mod tests { ); } + /// `Body35`'s placed card leg: a Q4_K or Q5_K gate and up with a Q4_K, Q5_K + /// or Q6_K down are the dispatch's own launches ([`Leg35::Dispatch`], the + /// layers whose graph the load's reference holds); a Q4_K, Q5_K, Q6_K, IQ3_XXS, + /// IQ4_XS or Q8_0 gate and up of one type with a Q4_K, Q5_K, Q6_K, + /// IQ3_XXS, IQ4_XS, Q5_1, Q8_0 or IQ4_NL down are the rest of the common + /// leg's ([`Leg35::Common`]); every other triple, a gate and up of two + /// types among them, is the host tier's. The sets are spelled out, so a + /// type entering or leaving an arm goes red here. Mutant: the dispatch + /// row widened to the common leg's gate·up arms — the IQ4_XS layer reads + /// `Dispatch`. + #[test] + fn qwen35_leg_is_the_dispatch_for_its_types_and_the_common_leg_for_the_rest() { + use GgmlType::{IQ3_XXS, IQ4_NL, IQ4_XS, Q4_K, Q5_1, Q5_K, Q6_K, Q8_0}; + let gate_up = [Q4_K, Q5_K, Q6_K, IQ3_XXS, IQ4_XS, Q8_0]; + let down = [Q4_K, Q5_K, Q6_K, IQ3_XXS, IQ4_XS, Q5_1, Q8_0, IQ4_NL]; + let dispatch_gate_up = [Q4_K, Q5_K]; + let dispatch_down = [Q4_K, Q5_K, Q6_K]; + for g in types() { + assert_eq!( + qwen35_common_gate_up(g), + gate_up.contains(&g), + "{g} gate·up" + ); + assert_eq!(qwen35_common_down(g), down.contains(&g), "{g} down"); + for u in types() { + for d in types() { + let want = if g == u && gate_up.contains(&g) && down.contains(&d) { + Some( + if dispatch_gate_up.contains(&g) && dispatch_down.contains(&d) { + Leg35::Dispatch + } else { + Leg35::Common + }, + ) + } else { + None + }; + assert_eq!(qwen35_leg([g, u, d]), want, "{g} {u} {d}"); + } + } + } + } + + /// `Body35`'s plan rule and load check are the classifier's: a layer the + /// rule cards is one [`qwen35_leg`] runs, and the arms are the common + /// leg's. The qwen3moe body (Qwen3-30B-A3B) keeps the dispatch's arms: its + /// placed side has no common layer. + #[test] + fn qwen35_arms_are_the_classifiers_and_the_qwen3moe_body_keeps_the_dispatchs() { + let arms = leg_arms(Program::Qwen35Body).expect("a card leg"); + let old = leg_arms(Program::Qwen3moeBody).expect("a card leg"); + for ty in types() { + assert_eq!( + (arms.gate_up)(ty), + qwen35_common_gate_up(ty), + "{ty} gate·up" + ); + assert_eq!((arms.down)(ty), qwen35_common_down(ty), "{ty} down"); + assert_eq!( + (old.gate_up)(ty), + qwen35_gate_up(ty).is_some(), + "{ty} old gate·up" + ); + assert_eq!((old.down)(ty), qwen35_down(ty).is_some(), "{ty} old down"); + } + assert_eq!( + leg_runs(Program::Qwen35Body, [GgmlType::IQ4_XS; 3]), + Some(true) + ); + assert_eq!( + leg_runs(Program::Qwen3moeBody, [GgmlType::IQ4_XS; 3]), + Some(false) + ); + } + /// Qwen3.8's arms are the card leg's launches: a Q4_K, Q5_K, IQ3_XXS, /// IQ4_XS or NVFP4 gate·up and a Q5_1, Q8_0, IQ4_NL or NVFP4 down, no /// other type. diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 657b72e9d..2c3aafade 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -47,6 +47,25 @@ pub const fn gemm(ty: GgmlType) -> Option { ty.card().gemm } +/// Whether the prompt GEMM `entry` reads its activation in 32-value blocks +/// (`Gemm32Weight`'s entries: Q8_0, Q5_1, IQ4_NL and NVFP4, over a +/// `GemmAct32`), else in the grouped GEMM's q8_1 blocks of 128 (`GemmAct`): +/// the form a prompt walk quantizes a unit's normed rows and SwiGLU rows into +/// for a stack of that type. +#[must_use] +pub const fn gemm_reads_blocks32(entry: GemmEntry) -> bool { + match entry { + GemmEntry::Q8_0 | GemmEntry::Q5_1 | GemmEntry::Iq4nl | GemmEntry::Nvfp4 => true, + GemmEntry::Q3k + | GemmEntry::Q4k + | GemmEntry::Q5k + | GemmEntry::Q6k + | GemmEntry::Iq2xxs + | GemmEntry::Iq3xxs + | GemmEntry::Iq4xs => false, + } +} + /// The dense gemv site launch family for a file tensor of type `ty`. #[must_use] pub const fn dense(ty: GgmlType) -> Option { @@ -86,8 +105,13 @@ pub const fn q8_act_bytes(cols: u64, k: u64) -> u64 { /// gate·up entry of a routed gate or up stack of type `ty` in the file's /// blocks ([`crate::placement::CardFormat::KQuant`]): the table's [`gate_up`] /// row, for the entries whose launchers read a `Q8Act` of more columns than a -/// run uses. The Q8_0 row has a launcher of that shape and no arm in the leg -/// yet. A stack in its planes is [`card_planes_runs`]'. +/// run uses. The Q8_0 row has a launcher of that shape and an arm in the leg +/// (`CardLeg::launch`), which this set leaves out: the families that plan by +/// it (GLM-5.3, MiMo, Solar-Open2, MiniMax-M3, Raon) hold `Q8Act` down buffers +/// only, so a Q8_0 layer they planned on the card would fail at its first +/// launch where it runs on the host today; a program whose buffers follow the +/// down's form reads the arm through its own row ([`crate::accepts::qwen35_common_gate_up`]). +/// A stack in its planes is [`card_planes_runs`]'. #[must_use] pub const fn card_gate_up_runs(ty: GgmlType) -> bool { matches!( @@ -347,6 +371,7 @@ mod tests { (GgmlType::Q8_0, GateUpEntry::Q8_0), (GgmlType::Q4_K, GateUpEntry::Q4k), (GgmlType::Q5_K, GateUpEntry::Q5k), + (GgmlType::Q6_K, GateUpEntry::Q6k), (GgmlType::MXFP4, GateUpEntry::Mxfp4), (GgmlType::NVFP4, GateUpEntry::Nvfp4), (GgmlType::IQ2_XS, GateUpEntry::Iq2xs), @@ -453,6 +478,28 @@ mod tests { assert!(!card_runs(GgmlType::NVFP4) && !card_sel32_runs(GgmlType::NVFP4)); } + /// The GEMM entries that read 32-value blocks are the `Gemm32Weight` + /// ones (the file's Q8_0, Q5_1 and IQ4_NL blocks, NVFP4's planes); the + /// grouped GEMM's read q8_1 blocks of 128. A type's GEMM form is one fact: + /// the table's `gemm` row read through this. + #[test] + fn gemm_form_is_the_32_value_entries() { + let blocks32: Vec<_> = ALL + .iter() + .filter(|&&ty| gemm(ty).is_some_and(gemm_reads_blocks32)) + .copied() + .collect(); + assert_eq!( + blocks32, + [ + GgmlType::Q5_1, + GgmlType::Q8_0, + GgmlType::NVFP4, + GgmlType::IQ4_NL + ] + ); + } + // PIN(2026-10-20): the GEMM pairs gain NVFP4 (`gemm_nvfp4p` over its // planes); every other pair is unchanged. // PIN(2026-10-21): and IQ2_XXS (`gemm_iq2xxs`, which `GemmWeight` already diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 24524177d..a4db1420a 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -5498,14 +5498,19 @@ mod tests { assert_eq!(plan.n_l, vec![8]); } + // PIN(2026-10-22): the Qwen3.6 leg runs a Q8_0 down and (once the leg has the + // arm) a Q6_K gate and up through the common leg, so the layers that held those + // types hold types it still has no GEMM for (an MXFP4 down, an IQ2_XS gate and + // up); its rows are the ones the test held, and Qwen3.8's gain the MXFP4 down + // its Q8_0 down did not have. #[test] fn host_only_reasons_follow_the_program_and_part_in_file_order() { use crate::accepts::{Program, routed_rule}; - use GgmlType::{Q4_K, Q5_K, Q6_K, Q8_0}; + use GgmlType::{IQ2_XS, MXFP4, Q4_K, Q5_K, Q6_K}; let model = legged(&[ [Q4_K, Q4_K, Q6_K], - [Q4_K, Q4_K, Q8_0], - [Q6_K, Q6_K, Q6_K], + [Q4_K, Q4_K, MXFP4], + [IQ2_XS, IQ2_XS, Q6_K], [Q4_K, Q5_K, Q4_K], ]); let rows = |program| { @@ -5529,6 +5534,7 @@ mod tests { rows(Program::Qwen38Body), [ (0, "ffn_down_exps.weight"), + (1, "ffn_down_exps.weight"), (2, "ffn_gate_exps.weight"), (2, "ffn_up_exps.weight"), (2, "ffn_down_exps.weight"), From ee7ff791c6e1d5369352e871c52baa86efdba5ce Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 22 Oct 2026 02:42:34 +0900 Subject: [PATCH 02/10] =?UTF-8?q?card=20leg:=20a=20Q6=5FK=20gate=C2=B7up?= =?UTF-8?q?=20kernel=20(q6k=5Fgate=5Fup=5Fsel)=20and=20the=20Q8=5F0=20and?= =?UTF-8?q?=20Q6=5FK=20gate=C2=B7up=20arms;=20Body35's=20placed=20layers?= =?UTF-8?q?=20launch=20them=20(the=20common=20leg's=20plan=20rule=20still?= =?UTF-8?q?=20admits=20neither)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/gpu/src/card_leg.rs | 505 ++++++++++++++++++++++++++++++++++--- crates/gpu/src/q6k_sel.rs | 184 +++++++++++++- tools/ref/ptx-shapes.tsv | 1 + 3 files changed, 659 insertions(+), 31 deletions(-) diff --git a/crates/gpu/src/card_leg.rs b/crates/gpu/src/card_leg.rs index ecbffc8c3..8d57b50f1 100644 --- a/crates/gpu/src/card_leg.rs +++ b/crates/gpu/src/card_leg.rs @@ -45,9 +45,12 @@ //! (the expert tier's staged rows) runs [`CardLeg::enqueue_rows`]. //! //! The run launches the entries [`kernels::card_gate_up_runs`] and -//! [`kernels::card_down_runs`] admit, each stack by its role: those whose -//! launchers read an activation of more columns than a run uses (a -//! [`LegBufs`] sized for `cap` tokens serves a unit of fewer in one launch). +//! [`kernels::card_down_runs`] admit, each stack by its role, and a Q8_0 gate +//! and up ([`GateUpEntry::Q8_0`]) for a program whose arms name it +//! ([`LegStacks::of_arms`]; the other families' constructors read the first two +//! tables and keep a Q8_0 gate·up on the host). The launchers read an +//! activation of more columns than a run uses (a [`LegBufs`] sized for `cap` +//! tokens serves a unit of fewer in one launch). //! The buffers hold the down's activation in one form ([`LegBufs::for_down`]); //! a stack whose down entry reads the other is refused by name before the //! first launch. @@ -59,12 +62,17 @@ //! //! A prompt walk that runs a unit's card slots through a route table reads //! each resident stack by the GEMM entry [`kernels::gemm`] names for its type -//! ([`StackW::route`]); the table, the launches and the SwiGLU between them -//! stay with the walk. +//! ([`StackW::route`], [`LegStacks::routes`]); the table and the SwiGLU between +//! the launches stay with the walk, which runs the block as [`RouteBlock`]: +//! the gate and up GEMMs, the SwiGLU into the form the down GEMM reads, and the +//! down GEMM. use crate::card_slots::{CardAcc, CardSlotKernels, CardSlots}; use crate::fault::FaultSink; -use crate::gemm::{Gemm32Weight, GemmWeight}; +use crate::gemm::{ + Gemm32Args, Gemm32Kernels, Gemm32Weight, GemmAct, GemmAct32, GemmArgs, GemmInput, GemmKernels, + GemmRoute, GemmWeight, +}; use crate::iq_sel::{IqDown, IqGateUp}; use crate::kquant::{Act, GateUpAct, SelDown}; use crate::mxfp4_sel::{Mxfp4SelKernels, Nvfp4Down, Nvfp4GateUp, Nvfp4SelKernels, Nvfp4Stack}; @@ -78,7 +86,7 @@ use crate::weights::{DevWeight, Weights}; use crate::{DeviceTensor, Gpu, GpuError, Q8Act}; use cuda_core::{CudaStream, DeviceBuffer}; use gguf::GgmlType; -use model::placement::accepts; +use model::placement::accepts::{self, LegArms}; use model::placement::kernels::{self, DownAct, DownEntry, GateUpEntry, GemmEntry, q8_act_bytes}; /// What the leg's errors name. @@ -272,6 +280,159 @@ pub(crate) enum RouteW<'a> { Blocks32(Gemm32Weight<'a>), } +/// A routed block over a route table, the wide arm of a prompt walk that runs +/// a layer's card experts: the gate and up GEMMs, the SwiGLU into the form the +/// down GEMM reads, and the down GEMM, each as the kernel table's GEMM entry +/// of its stack's type reads it ([`StackW::route`]). The gate and up are one +/// type, so one input form; the down's form is its own. +pub(crate) struct RouteBlock<'a> { + pub(crate) gate: RouteW<'a>, + pub(crate) up: RouteW<'a>, + pub(crate) down: RouteW<'a>, + /// Rows an expert of the gate and up: the SwiGLU's values a column. + pub(crate) ff: usize, + /// Rows an expert of the down. + pub(crate) hidden: usize, + pub(crate) rule: Act, +} + +/// The rows a [`RouteBlock`] reads and writes: the unit's normed rows in the +/// form the gate and up read (the other absent), the gate and up outputs, the +/// SwiGLU rows in the form the down reads (the other absent), and the down's +/// output. +pub(crate) struct RouteIo<'a> { + pub(crate) x128: Option<&'a GemmAct>, + pub(crate) x32: Option<&'a GemmAct32>, + pub(crate) g: &'a mut DeviceBuffer, + pub(crate) u: &'a mut DeviceBuffer, + pub(crate) h128: Option<&'a mut GemmAct>, + pub(crate) h32: Option<&'a mut GemmAct32>, + pub(crate) y: &'a mut DeviceBuffer, +} + +impl RouteBlock<'_> { + /// Enqueue the block over `n_slots` slot columns of `route`'s table, the + /// gate and up reading each slot's token column (`top_k` slots a token) and + /// the down its own slot's: six launches. A form a stack reads that `io` + /// lacks, a gate and up the table reads by two GEMMs, and a block whose + /// gate and up are of two types are refused by name before the first + /// launch. Asynchronous, allocation-free, capturable. + #[allow( + clippy::too_many_arguments, + reason = "the card, the two GEMM kernel sets, the table and its column rule, the slots, the rows and the fault sink (rust-quality R8)" + )] + pub(crate) fn enqueue( + &self, + gpu: &Gpu, + (gemm, g32): (&GemmKernels, &Gemm32Kernels), + (route, top_k): (&GemmRoute, usize), + n_slots: usize, + io: RouteIo<'_>, + fault: FaultSink, + ) -> Result<(), GpuError> { + const WHAT_B: &str = "card_leg::RouteBlock"; + let stream = gpu.stream(); + let RouteIo { + x128, + x32, + g, + u, + h128, + h32, + y, + } = io; + let lacks = |form: &'static str| { + GpuError::shape( + WHAT_B, + format!("a block that reads {form}, which the rows lack"), + ) + }; + let input = GemmInput::Shared { top_k }; + match (self.gate, self.up) { + (RouteW::Grouped(ty, wg), RouteW::Grouped(tu, wu)) if ty == tu => { + let act = x128.ok_or_else(|| lacks("q8_1 blocks of 128 (the gate and up)"))?; + for (w, y) in [(wg, &mut *g), (wu, &mut *u)] { + gemm.enqueue_gemm( + stream, + GemmArgs { + ty, + w, + rows_per_expert: self.ff, + act, + route, + input, + y, + }, + )?; + } + } + (RouteW::Blocks32(wg), RouteW::Blocks32(wu)) => { + let act = x32.ok_or_else(|| lacks("q8 blocks of 32 (the gate and up)"))?; + for (w, y) in [(wg, &mut *g), (wu, &mut *u)] { + g32.enqueue_gemm32( + stream, + Gemm32Args { + w, + rows_per_expert: self.ff, + act, + route, + input, + y, + }, + )?; + } + } + _ => { + return Err(GpuError::shape( + WHAT_B, + "a gate and an up the table does not read by one GEMM of one type", + )); + } + } + let per_slot = GemmInput::PerSlot; + match self.down { + RouteW::Grouped(ty, w) => { + let act = h128.ok_or_else(|| lacks("q8_1 blocks of 128 (the down)"))?; + gemm.enqueue_swiglu_act_quant(stream, g, u, self.rule, n_slots, act, fault)?; + gemm.enqueue_gemm( + stream, + GemmArgs { + ty, + w, + rows_per_expert: self.hidden, + act, + route, + input: per_slot, + y, + }, + ) + } + RouteW::Blocks32(w) => { + let act = h32.ok_or_else(|| lacks("q8 blocks of 32 (the down)"))?; + match self.rule { + Act::SiluMul => { + g32.enqueue_swiglu_quant32(stream, g, u, n_slots, act, fault)? + } + rule => { + g32.enqueue_swiglu_act_quant32(stream, g, u, rule, n_slots, act, fault)?; + } + } + g32.enqueue_gemm32( + stream, + Gemm32Args { + w, + rows_per_expert: self.hidden, + act, + route, + input: per_slot, + y, + }, + ) + } + } + } +} + /// One resident routed stack: its name, file type, device form and row width. struct LegStack<'a> { name: &'a str, @@ -358,6 +519,10 @@ impl<'a> LegStack<'a> { } } +/// What a refusal of a gate or up stack the common card leg does not run says. +const GATE_UP: &str = + "a resident routed gate or up stack of a type the card leg runs (`kernels::card_gate_up_runs`)"; + /// A layer's three resident routed stacks, the gate and up of one type. pub struct LegStacks<'a> { gate: LegStack<'a>, @@ -375,7 +540,15 @@ impl<'a> LegStacks<'a> { pub fn of(w: &'a Weights, names: [&'a str; 3]) -> Result, GpuError> { const DOWN: &str = "a resident routed down stack of a type the card leg runs (`kernels::card_down_runs`)"; - LegStacks::with_downs(w, names, (kernels::card_down_runs, DOWN)) + LegStacks::with_arms( + w, + names, + LegArms { + gate_up: kernels::card_gate_up_runs, + down: kernels::card_down_runs, + }, + (GATE_UP, DOWN), + ) } /// [`LegStacks::of`] for a tensor-parallel rank, whose downs may also be @@ -383,7 +556,15 @@ impl<'a> LegStacks<'a> { pub fn of_rank(w: &'a Weights, names: [&'a str; 3]) -> Result, GpuError> { const DOWN: &str = "a resident routed down stack of a type a rank's card leg runs \ (`kernels::rank_down_runs`)"; - LegStacks::with_downs(w, names, (kernels::rank_down_runs, DOWN)) + LegStacks::with_arms( + w, + names, + LegArms { + gate_up: kernels::card_gate_up_runs, + down: kernels::rank_down_runs, + }, + (GATE_UP, DOWN), + ) } /// [`LegStacks::of`] for a body that loads every tensor in the format @@ -393,33 +574,45 @@ impl<'a> LegStacks<'a> { pub fn of_resident(w: &'a Weights, names: [&'a str; 3]) -> Result, GpuError> { const DOWN: &str = "a resident routed down stack of a type a whole-file body's card leg \ runs (`kernels::card_down_runs` or Q5_0)"; - LegStacks::with_downs(w, names, (accepts::resident_down_runs, DOWN)) + LegStacks::with_arms( + w, + names, + LegArms { + gate_up: kernels::card_gate_up_runs, + down: accepts::resident_down_runs, + }, + (GATE_UP, DOWN), + ) } - /// The stacks with the down admitted by `downs.0`, refused as `downs.1`. - fn with_downs( + /// [`LegStacks::of`] for a program whose card leg runs the types `arms` + /// name ([`accepts::leg_arms`]): the one constructor behind the others, + /// each a program's arms over the launches [`CardLeg`] has arms for. A + /// stack of a type no arm of `arms` admits is refused by name (`roles`: + /// the gate or up refusal's words, then the down's). Allocation-free on + /// success. + pub fn of_arms( w: &'a Weights, names: [&'a str; 3], - downs: (fn(GgmlType) -> bool, &'static str), + arms: LegArms, + roles: (&'static str, &'static str), + ) -> Result, GpuError> { + LegStacks::with_arms(w, names, arms, roles) + } + + /// The stacks with the gate and up admitted by `arms.gate_up` and the + /// down by `arms.down`, refused as `roles.0` and `roles.1`. + fn with_arms( + w: &'a Weights, + names: [&'a str; 3], + arms: LegArms, + roles: (&'static str, &'static str), ) -> Result, GpuError> { - const GATE_UP: &str = "a resident routed gate or up stack of a type the card leg runs (`kernels::card_gate_up_runs`)"; let [gate, up, down] = names; let (gate, up, down) = ( - LegStack::of( - w, - gate, - kernels::card_gate_up_runs, - kernels::card_planes_runs, - GATE_UP, - )?, - LegStack::of( - w, - up, - kernels::card_gate_up_runs, - kernels::card_planes_runs, - GATE_UP, - )?, - LegStack::of(w, down, downs.0, kernels::card_planes_runs, downs.1)?, + LegStack::of(w, gate, arms.gate_up, kernels::card_planes_runs, roles.0)?, + LegStack::of(w, up, arms.gate_up, kernels::card_planes_runs, roles.0)?, + LegStack::of(w, down, arms.down, kernels::card_planes_runs, roles.1)?, ); if up.ty != gate.ty { return Err(GpuError::Tensor { @@ -431,6 +624,21 @@ impl<'a> LegStacks<'a> { Ok(LegStacks { gate, up, down }) } + /// The three stacks as a prompt walk's route GEMMs read them + /// ([`StackW::route`]), `[gate, up, down]`: each stack's own type through + /// the kernel table's GEMM entry. A stack no route GEMM reads is refused + /// by name. + pub(crate) fn routes(&self) -> Result<[RouteW<'a>; 3], GpuError> { + let one = |st: &LegStack<'a>| { + st.w.route(st.ty).ok_or_else(|| GpuError::Tensor { + what: WHAT, + name: st.name.to_owned(), + need: "a type some route GEMM reads (`kernels::gemm`)", + }) + }; + Ok([one(&self.gate)?, one(&self.up)?, one(&self.down)?]) + } + /// The stacks hold `n_card` experts of `shape` each: the gate's and up's /// `ff` rows of `n` values, the down's `n` rows of `ff`. pub fn check(&self, n_card: usize, shape: LegShape) -> Result<(), GpuError> { @@ -652,6 +860,11 @@ impl CardLeg { self.quant .enqueue_gate_up_q5k(stream, &words()?, fault, h)? } + Some(GateUpEntry::Q8_0) => { + self.quant + .enqueue_gate_up_q8_0(stream, &words()?, fault, h)? + } + Some(GateUpEntry::Q6k) => self.q6k.enqueue_gate_up(stream, &words()?, fault, h)?, Some(GateUpEntry::Mxfp4) => self.mxfp4.enqueue_gate_up(stream, &words()?, fault, h)?, Some(GateUpEntry::Nvfp4) => { let a = Nvfp4GateUp { @@ -1448,6 +1661,240 @@ pub(crate) mod tests { } } + /// A gate and up of `ty` beside a Q4_K down, a combination the qwen35moe + /// program's arms admit ([`accepts::leg_arms`]): the leg's rows are the + /// hand launches' bit for bit (the table's gate·up entry, the quantizer + /// and the Q4_K down), for a run of every token its buffers hold and of + /// fewer, and the constructors of the families whose plan reads + /// [`kernels::card_gate_up_runs`] refuse the stacks by name. + fn leg_runs_gate_up_as_its_launch(ty: GgmlType) { + let gpu = Gpu::with_device(0).expect("a Gpu on device 0"); + let stream = gpu.stream(); + let leg = CardLeg::load(&gpu).expect("the leg"); + let kq = QuantSelKernels::load(gpu.context(), gpu.fault_word()).expect("the kquant module"); + let q6 = Q6kSelKernels::load(gpu.context(), gpu.fault_word()).expect("the q6k module"); + let tys = [ty, GgmlType::Q4_K]; + let w = weights_for(&gpu, SHAPE, &tys, |role, t| match role { + "down" => t == GgmlType::Q4_K, + _ => t == ty, + }); + let names = [ + stack_name("gate", ty), + stack_name("up", ty), + stack_name("down", GgmlType::Q4_K), + ]; + let names = [names[0].as_str(), names[1].as_str(), names[2].as_str()]; + assert!( + matches!( + LegStacks::of(&w, names), + Err(GpuError::Tensor { what: WHAT, .. }) + ), + "the common constructor keeps a {ty} gate and up off the card" + ); + let arms = accepts::leg_arms(accepts::Program::Qwen35Body).expect("the body's arms"); + let stacks = LegStacks::of_arms(&w, names, arms, (GATE_UP, "a down the body's arms run")) + .expect("the stacks"); + stacks.check(N_CARD, SHAPE).expect("the stacks' shape"); + let host_normed: Vec = (0..CAP * N) + .map(|i| ((i * 2654435761usize) % 2001) as f32 / 1000.0 - 1.0) + .collect(); + let normed = DeviceBuffer::from_host(stream, &host_normed).expect("the normed rows"); + let sel = DeviceBuffer::from_host(stream, &SEL).expect("the places"); + let mut bufs = LegBufs::new(stream, SHAPE).expect("the leg's buffers"); + let rule = Act::SiluMul; + for m in [CAP, CAP - 1] { + let case = format!("{ty} gate·up, {m} tokens"); + let slots = m * SPC; + let got = leg_rows(&gpu, &leg, &mut bufs, &stacks, &normed, &sel, m); + + let mut act_x = Q8Act::with_tier_cols(stream, m, N).expect("act_x"); + let mut h = DeviceBuffer::::zeroed(stream, slots * FF).expect("h"); + let mut act_h = Q8Act::with_slots(stream, slots, FF).expect("act_h"); + let mut want = + DeviceBuffer::from_host(stream, &vec![SENT; slots * N]).expect("the rows"); + let sink = gpu.layer_sink(0).expect("a sink"); + gpu.enqueue_quantize_q8_1_cols(&normed, &mut act_x, m, 0) + .expect("q8_1"); + let a = GateUpAct { + wg: words(&w, names[0]), + wu: words(&w, names[1]), + act: &act_x, + sel: &sel, + n_slots: slots, + rows_per_expert: FF, + slots_per_col: SPC, + rule, + }; + match ty { + GgmlType::Q8_0 => kq.enqueue_gate_up_q8_0(stream, &a, sink, &mut h), + _ => q6.enqueue_gate_up(stream, &a, sink, &mut h), + } + .unwrap_or_else(|e| panic!("{case}: the gate·up: {e}")); + let q = QuantSel { + x: &h, + cols: 0..slots, + sel: &sel, + n_card: N_CARD, + }; + gpu.q4k_sel() + .enqueue_quantize_sel(stream, &q, sink, &mut act_h) + .unwrap_or_else(|e| panic!("{case}: the quantizer: {e}")); + gpu.q4k_sel() + .enqueue_gemv_q4k_sel( + stream, + words(&w, names[2]), + &act_h, + &sel, + slots, + N, + &mut want, + ) + .unwrap_or_else(|e| panic!("{case}: the down: {e}")); + let want = want.to_host_vec(stream).expect("the hand rows"); + assert!( + gpu.take_fault().expect("the fault word").is_none(), + "{case}" + ); + assert_hand_rows(&case, &got, &want); + } + } + + /// A Q8_0 gate and up through the leg. Mutant: the arm launching the Q5_K + /// gate·up for a Q8_0 stack — the rows part from the hand launch. + #[test] + #[ignore = "needs a CUDA device; `just gate-gpu-lib` runs it on the box"] + fn hw_the_leg_runs_a_q8_0_gate_up_as_its_sel_launch() { + leg_runs_gate_up_as_its_launch(GgmlType::Q8_0); + } + + /// A Q6_K gate and up through the leg. Mutant: the arm launching the Q8_0 + /// gate·up for a Q6_K stack — the stacks are refused by name or the rows + /// part from the hand launch. + #[test] + #[ignore = "needs a CUDA device; `just gate-gpu-lib` runs it on the box"] + fn hw_the_leg_runs_a_q6k_gate_up_as_its_sel_launch() { + leg_runs_gate_up_as_its_launch(GgmlType::Q6_K); + } + + /// `q6k_gate_up_sel` against its parts: the gate and up rows of a slot are + /// the two dots `q6k_gemv_sel` writes for the slot's column (the same row + /// walk, bit for bit, one activation column a slot), and `h` is the rule + /// over them — within the device's fused multiply-adds of the host's + /// transcription ([`crate::kquant::act::apply`]), for a run of every token the + /// activation holds and of fewer; a slot the card does not serve keeps + /// its sentinel, and an id past the stack's experts that is not the host's + /// raises [`FaultSite::ExpertId`] and writes NaN into its rows. Mutant: the + /// up sum read from the gate stack — `h` is `silu(g) · g`. + #[test] + #[ignore = "needs a CUDA device; `just gate-gpu-lib` runs it on the box"] + fn hw_the_q6k_gate_up_is_its_two_dots_and_the_rule() { + use crate::fault::{FaultSite, LAYER_NONE}; + let gpu = Gpu::with_device(0).expect("a Gpu on device 0"); + let stream = gpu.stream(); + let q6 = Q6kSelKernels::load(gpu.context(), gpu.fault_word()).expect("the q6k module"); + let w = weights_for(&gpu, SHAPE, &[GgmlType::Q6_K], |role, _| role != "down"); + let (wg, wu) = ( + words(&w, &stack_name("gate", GgmlType::Q6_K)), + words(&w, &stack_name("up", GgmlType::Q6_K)), + ); + let rows_of = |t: usize| row_of(t, N); + let (code, limit) = Act::SiluMul.code("test").expect("the rule's code"); + for m in [CAP, CAP - 1] { + let slots = m * SPC; + let case = format!("q6_K gate·up, {m} tokens"); + let normed_host: Vec = (0..m).flat_map(rows_of).collect(); + let normed = DeviceBuffer::from_host(stream, &normed_host).expect("the normed rows"); + // One column a slot for the hand dots: the slot's token's row. + let by_slot: Vec = (0..slots).flat_map(|s| rows_of(s / SPC)).collect(); + let by_slot = DeviceBuffer::from_host(stream, &by_slot).expect("the rows a slot"); + let mut act_x = Q8Act::with_tier_cols(stream, m, N).expect("act_x"); + let mut act_s = Q8Act::with_slots(stream, slots, N).expect("act_s"); + gpu.enqueue_quantize_q8_1_cols(&normed, &mut act_x, m, 0) + .expect("q8_1"); + gpu.enqueue_quantize_q8_1_cols(&by_slot, &mut act_s, slots, 0) + .expect("q8_1 a slot"); + let sel = DeviceBuffer::from_host(stream, &SEL[..slots]).expect("the places"); + let sink = gpu.layer_sink(0).expect("a sink"); + let mut h = DeviceBuffer::from_host(stream, &vec![SENT; slots * FF]).expect("h"); + let a = GateUpAct { + wg, + wu, + act: &act_x, + sel: &sel, + n_slots: slots, + rows_per_expert: FF, + slots_per_col: SPC, + rule: Act::SiluMul, + }; + q6.enqueue_gate_up(stream, &a, sink, &mut h) + .unwrap_or_else(|e| panic!("{case}: the gate·up: {e}")); + let mut g = DeviceBuffer::from_host(stream, &vec![SENT; slots * FF]).expect("g"); + let mut u = DeviceBuffer::from_host(stream, &vec![SENT; slots * FF]).expect("u"); + q6.enqueue_gemv_q6k_sel(stream, wg, &act_s, &sel, slots, FF, &mut g) + .unwrap_or_else(|e| panic!("{case}: the gate dots: {e}")); + q6.enqueue_gemv_q6k_sel(stream, wu, &act_s, &sel, slots, FF, &mut u) + .unwrap_or_else(|e| panic!("{case}: the up dots: {e}")); + let (h, g, u) = ( + h.to_host_vec(stream).expect("h"), + g.to_host_vec(stream).expect("g"), + u.to_host_vec(stream).expect("u"), + ); + assert!( + gpu.take_fault().expect("the fault word").is_none(), + "{case}" + ); + for (s, &place) in SEL[..slots].iter().enumerate() { + for r in s * FF..(s + 1) * FF { + if place == HOST { + assert_eq!(h[r].to_bits(), SENT.to_bits(), "{case}: host slot {s}"); + continue; + } + let want = crate::kquant::act::apply(code, limit, g[r], u[r]); + assert!( + want.is_finite() && (h[r] - want).abs() <= 2e-6 * (1.0 + want.abs()), + "{case}: slot {s} row {}: kernel {} against the rule over the two dots {want}", + r - s * FF, + h[r] + ); + } + } + } + // An id past the stack: fault and NaN, the other slot untouched. + let bad = DeviceBuffer::from_host(stream, &[N_CARD as u32 + 5, 1]).expect("the places"); + let normed_host: Vec = rows_of(0); + let normed = DeviceBuffer::from_host(stream, &normed_host).expect("the normed row"); + let mut act_x = Q8Act::with_tier_cols(stream, 1, N).expect("act_x"); + gpu.enqueue_quantize_q8_1_cols(&normed, &mut act_x, 1, 0) + .expect("q8_1"); + let mut h = DeviceBuffer::from_host(stream, &vec![SENT; 2 * FF]).expect("h"); + let a = GateUpAct { + wg, + wu, + act: &act_x, + sel: &bad, + n_slots: 2, + rows_per_expert: FF, + slots_per_col: 2, + rule: Act::SiluMul, + }; + q6.enqueue_gate_up(stream, &a, gpu.unlabelled_sink(), &mut h) + .expect("the launch"); + let h = h.to_host_vec(stream).expect("h"); + let fault = gpu.take_fault().expect("the fault word"); + assert!( + fault.is_some_and(|f| f.code == FaultSite::ExpertId as u32 && f.layer == LAYER_NONE), + "an id past the stack raises ExpertId, got {fault:?}" + ); + assert!( + h[..FF].iter().all(|v| v.is_nan()), + "the bad slot's rows are NaN" + ); + assert!( + h[FF..].iter().all(|v| v.is_finite() && *v != SENT), + "the good slot is written" + ); + } + #[test] #[ignore = "needs a CUDA card"] fn hw_gate_up_sums_fault_before_activation() { diff --git a/crates/gpu/src/q6k_sel.rs b/crates/gpu/src/q6k_sel.rs index 6838856c5..2572a225f 100644 --- a/crates/gpu/src/q6k_sel.rs +++ b/crates/gpu/src/q6k_sel.rs @@ -2,7 +2,9 @@ //! projection of a Q6_K expert stack in the decode shape, all `n_slots` //! routed experts in one launch — the per-slot-column shape of //! `q4k_gemv_sel`: slot `s` dots the rows of expert `sel[s]` with -//! activation column `s`. +//! activation column `s` — and the gate·up of Q6_K stacks (`q6k_gate_up_sel`), +//! the same row walk twice and the activation rule, slot `s` against column +//! `s / slots_per_col`. //! //! The per-row body is `q6k_gemv`'s column-0 path verbatim — the same //! byte-window loads with the 16-bit funnel for a super-block at 2 mod 4, @@ -18,7 +20,11 @@ use crate::cores::{funnel16, half_to_f32, l2_prefetch, q6k_chain, q6k_dequant, q6k_sub_scale}; use crate::fault::{FaultSink, FaultSite, LAYER_NONE}; use crate::hybrid::HOST; -use crate::kquant::sel::{ARENA_PARTS, ARENA_SKIP, PART_DOWN, SelDownAt, arena_part, down_at_dims}; +use crate::kquant::act; +use crate::kquant::sel::{ + ARENA_PARTS, ARENA_SKIP, GateUpAct, PART_DOWN, SelDownAt, arena_part, check_gate_up_slots, + down_at_dims, +}; use crate::tensor::{DeviceTensor, Q8Act}; use crate::{GpuError, launch_u32}; use cuda_core::{CudaContext, CudaStream, DeviceBuffer, LaunchConfig1D}; @@ -288,6 +294,96 @@ mod q6k_sel_kernels { } } + /// Q6_K gate·up over expert slots selected on the device: one warp per + /// output row, eight rows per 256-thread block; thread row `n = slot · + /// rows_per_expert + r` stores `h[n]`, the activation rule `act` over the + /// dots of weight row `sel[slot] · rows_per_expert + r` of the gate and of + /// the up stack (`210 · n_sb` bytes at byte `row · 210 · n_sb` of each) + /// against q8_1 column `slot / slots_per_col` in the q6 permutation. Each + /// dot is [`q6k_sel_row_dot`], `q6k_gemv_sel`'s row walk, so the sums are + /// the down launch's of the same column bit for bit. An id `>= n_experts` + /// returns the slot's warps before their first weight load (warp-uniform); + /// one that is not [`HOST`] (a slot the host tier serves, skipped by + /// contract) raises [`FaultSite::ExpertId`] on `fault` first and writes NaN + /// into its rows, as the common leg's other gate·up entries. `m_cols` only + /// bounds the activation in the launch contract. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + 4 * wg.len() >= n_experts * rows_per_expert * 210 * n_sb, + 4 * wu.len() >= n_experts * rows_per_expert * 210 * n_sb, + slots_per_col >= 1, + n_slots <= m_cols * slots_per_col, + q.len() >= m_cols * 128 * iters, + d8.len() >= m_cols * 2 * n_sb, + sel.len() >= n_slots, + h.len() >= n_slots * rows_per_expert + ) + )] + pub fn q6k_gate_up_sel( + wg: &[u32], + wu: &[u32], + q: &[u32], + d8: &[f32], + sel: &[u32], + n_experts: u32, + rows_per_expert: u32, + n_slots: u32, + m_cols: u32, + slots_per_col: u32, + n_sb: u32, + iters: u32, + act: u32, + limit: f32, + fault: FaultSink, + mut h: DisjointSlice, + ) { + let _ = m_cols; + let t = thread::index_1d().get() % 256; + let row = (thread::index_1d().get() / 256) * 8 + t / 32; + if row >= n_slots as usize * rows_per_expert as usize { + return; + } + let slot = row / rows_per_expert as usize; + // SAFETY: slot < n_slots <= sel.len() by the launch contract; the + // load is warp-uniform (all 32 lanes share `row`, hence `slot`). + let id = unsafe { *sel.get_unchecked(slot) }; + let lane = warp::lane_id() as usize; + if id >= n_experts { + if id != HOST && lane == 0 { + fault.raise(FaultSite::ExpertId); + // SAFETY: row < n_slots·rows_per_expert <= h.len() by the + // launch contract; only lane 0 of the warp writes h[row]. + unsafe { *h.get_unchecked_mut(row) = f32::NAN }; + } + return; + } + let row_abs = id as usize * rows_per_expert as usize + row % rows_per_expert as usize; + let col = slot / slots_per_col as usize; + let n_sb = n_sb as usize; + let row_at = row_abs * 210 * n_sb; + // SAFETY: row_abs bounds both stacks' row by the launch contract and + // col < m_cols bounds q and d8 (slot < n_slots <= m_cols · + // slots_per_col); the host passes iters = ceil(n_sb/2); the returns + // above are warp-uniform, so all 32 lanes are here. + let g = q6k_sel_row_dot!(wg, row_at, q, d8, n_sb, iters, col, lane); + let u = q6k_sel_row_dot!(wu, row_at, q, d8, n_sb, iters, col, lane); + if lane == 0 { + // SAFETY: row < n_slots·rows_per_expert <= h.len() by the launch + // contract; only lane 0 of the warp writes h[row]. + unsafe { + *h.get_unchecked_mut(row) = act::apply_checked(act, limit, g, u, fault); + } + } + } + /// The Q6_K down arena entry: [`q6k_gemv_sel`] with slot `slot`'s /// expert read from its down part in `arena` (word /// `offs[slot · ARENA_PARTS + PART_DOWN]`, the expert's rows as its file @@ -525,6 +621,90 @@ impl Q6kSelKernels { Ok(()) } + /// Enqueue the device-indirect MoE gate·up for Q6_K expert stacks + /// (`q6k_gate_up_sel`): slot `s` writes `h[s·rows_per_expert ..][.. + /// rows_per_expert]`, `a.rule` over expert `sel[s]`'s rows of the gate and + /// up stacks against `a.act` column `s / slots_per_col`. The stacks are + /// the byte streams [`Self::enqueue_gemv_q6k_sel`] reads, both of one + /// shape; `a.act` holds at least `ceil(n_slots / slots_per_col)` columns. + /// An id `>= n_experts` writes nothing, and one that is not [`HOST`] raises + /// [`FaultSite::ExpertId`] on `fault` and writes NaN into its rows. + /// Asynchronous, allocation-free, capturable. + pub fn enqueue_gate_up( + &self, + stream: &CudaStream, + a: &GateUpAct<'_>, + fault: FaultSink, + h: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + let what = "enqueue_gate_up_q6k"; + let n_sb = a.act.n_sb(); + let (rpe, rows) = (a.rows_per_expert, a.wg.rows()); + if a.wu.rows() != rows || a.wu.cols() != a.wg.cols() { + return Err(GpuError::shape( + what, + format!( + "the up stack {} x {} must be the gate stack's {rows} x {}", + a.wu.rows(), + a.wu.cols(), + a.wg.cols() + ), + )); + } + if rpe == 0 || !rows.is_multiple_of(rpe) { + return Err(GpuError::shape( + what, + format!("w.rows() {rows} must be a positive multiple of rows_per_expert {rpe}"), + )); + } + let need = (rows * 210 * n_sb).div_ceil(4); + if a.wg.buf().len() < need || a.wu.buf().len() < need { + return Err(GpuError::shape( + what, + format!( + "{rows} rows of 210*{n_sb} bytes need {need} words, got {} and {}", + a.wg.buf().len(), + a.wu.buf().len() + ), + )); + } + let slot_rows = check_gate_up_slots( + what, + a.n_slots, + rpe, + a.act.m(), + a.slots_per_col, + a.sel.len(), + h.len(), + )?; + let (code, limit) = a.rule.code(what)?; + let grid = launch_u32(what, "grid", slot_rows.div_ceil(8))?; + let prep = self + .module + .prepare_q6k_gate_up_sel(LaunchConfig1D::new(grid, 256, 0))?; + self.module.q6k_gate_up_sel( + stream, + &prep, + a.wg.buf(), + a.wu.buf(), + &a.act.q6, + &a.act.d8, + a.sel, + launch_u32(what, "n_experts", rows / rpe)?, + launch_u32(what, "rows_per_expert", rpe)?, + launch_u32(what, "n_slots", a.n_slots)?, + launch_u32(what, "m_cols", a.act.m())?, + launch_u32(what, "slots_per_col", a.slots_per_col)?, + launch_u32(what, "n_sb", n_sb)?, + launch_u32(what, "iters", n_sb.div_ceil(2))?, + code, + limit, + fault, + h, + )?; + Ok(()) + } + /// Enqueue the Q6_K down arena entry (`q6k_gemv_sel_at`): as /// [`Self::enqueue_gemv_q6k_sel`], slot `s`'s expert the down part at its /// offset in `a.arena` (`kquant::sel`'s table; a slot marked diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index 03eff9c46..15bca5a8f 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -316,6 +316,7 @@ generate_ds41 q5_quantize_q8_sel_relu_sqr 0 0 generate_ds41 q5k_gemv_sel 0 0 generate_ds41 q5k_gemv_sel_at 0 0 generate_ds41 q5k_gemv_tiles 0 0 +generate_ds41 q6k_gate_up_sel 0 0 generate_ds41 q6k_gemv 0 0 generate_ds41 q6k_gemv_ids 0 0 generate_ds41 q6k_gemv_ids_mcol 0 0 From 205bb2c0fd71955c2e47eb97b5350fbbba2b6d1a Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 22 Oct 2026 02:42:34 +0900 Subject: [PATCH 03/10] qwen3moe: Body35's placed load runs a layer the dispatch refuses on the common card leg; the dispatch's layers stay bit-identical Co-Authored-By: Claude Opus 5.5 --- crates/gpu/src/arch/qwen3moe/body35.rs | 82 ++- crates/gpu/src/arch/qwen3moe/dispatch.rs | 2 + crates/gpu/src/arch/qwen3moe/placed.rs | 633 ++++++++++++++++++++--- crates/gpu/src/arch/qwen3moe/swap.rs | 53 +- crates/gpu/src/arch/qwen3moe/swap35.rs | 7 +- 5 files changed, 662 insertions(+), 115 deletions(-) diff --git a/crates/gpu/src/arch/qwen3moe/body35.rs b/crates/gpu/src/arch/qwen3moe/body35.rs index 587495fd1..11adb4c1b 100644 --- a/crates/gpu/src/arch/qwen3moe/body35.rs +++ b/crates/gpu/src/arch/qwen3moe/body35.rs @@ -83,8 +83,8 @@ use super::image::{ }; use super::mtp35::Mtp35; use super::placed::{ - BatchWalk, PLACED_ROWS, Placed, PlacedOpen, StepWalk, WalkParts, apart_plans, counted_arena, - placed_bytes, placed_scratch, prompt_rows, tally_shape, + BatchWalk, CommonForms, PLACED_ROWS, Placed, PlacedOpen, StepWalk, WalkParts, apart_plans, + counted_arena, placed_bytes_with, placed_scratch_with, prompt_rows, tally_shape, }; use super::plan::{ DeltaPlan, FfnPlan, FfnRoute, Flash, GqaKind, GqaPlan, Kind35, LayerPlan, MixerPlan, @@ -130,6 +130,7 @@ use gguf::quant::GgmlType; use model::arch::Arch; use model::arch::models::shape::MoeShape; use model::arch::models::{Ffn, LayerSpec, Mixer, ModelSpec, MtpDraft}; +use model::placement::accepts::Leg35; use model::placement::{Plan, WholeLoad, accepts}; use runtime::seqstate::{Kept, Price}; use std::num::NonZeroUsize; @@ -490,7 +491,9 @@ fn joint(l: usize, part: &str) -> String { /// Q4_K or Q8_0 (`Head`). A routed layer's stacks and its shared expert are /// read at any site type ([`FFN_READ`]) and admitted by the card kernel /// table's entries ([`plan_ffn`]); under a placed read the stacks of a layer -/// the card leg does not run are the host tier's instead ([`card_runs`]). +/// the dispatch's launches do not run are typed as the host tier's instead +/// ([`card_leg`]): a layer the common leg runs holds its stacks to the placed +/// side, which reads their types from the resident stacks. const PROJ: &[SiteTy] = &[ SiteTy::Q3K, SiteTy::Q4K, @@ -576,13 +579,18 @@ fn stack_ty(file: &Split, name: &str, (rows, k): (usize, usize)) -> Result Result { +/// The leg that runs layer `l`'s routed stacks (`n` experts of `ff` rows of +/// `h` values) on the placed card, with the stacks' file types, read from +/// `file`'s header before any upload: the planner's own verdict +/// ([`accepts::qwen35_leg`]), `None` for a layer the host tier runs. A stack of +/// a type that neither the card leg's arm for its part ([`accepts::leg_arms`]) +/// nor the host tier's fused rows ([`model::ops::host_stack_runs`]) runs is +/// refused by name, with the layer: no plan can place it. +fn card_leg( + file: &Split, + l: usize, + (ff, h, n): (usize, usize, usize), +) -> Result, GpuError> { let program = accepts::Program::Qwen35Body; let arms = accepts::leg_arms(program) .ok_or_else(|| GpuError::shape(WHAT, "the qwen35moe body runs no card leg"))?; @@ -611,7 +619,30 @@ fn card_runs(file: &Split, l: usize, (ff, h, n): (usize, usize, usize)) -> Resul stack("up", (ff, h), gate_up)?, stack("down", (h, ff), ("down", arms.down))?, ]; - Ok(accepts::leg_runs(program, tys) == Some(true)) + Ok(accepts::qwen35_leg(tys).map(|leg| (leg, tys))) +} + +/// The forms the layers the common card leg runs ([`Leg35::Common`]) hold +/// buffers of, from `file`'s header: [`CommonForms::NONE`] unless the read is a +/// placed one, which is the only one a card leg runs a layer of by it. +fn common_forms( + file: &Split, + d: &Dims, + layers: &[LayerSpec], + read: Read35, +) -> Result { + if read != Read35::Placed { + return Ok(CommonForms::NONE); + } + let mut tys = Vec::new(); + for (l, layer) in layers.iter().enumerate() { + let Ffn::Moe(m) = &layer.ffn else { continue }; + let n = m.experts as usize; + if let Some((Leg35::Common, [gate_up, _, down])) = card_leg(file, l, (d.ff, d.hidden, n))? { + tys.push([gate_up, down]); + } + } + Ok(CommonForms::of(tys)) } /// The types of routed part `part` of layer `l` from `file`'s header: its @@ -644,11 +675,13 @@ fn part_tys( /// into the stacks as expert `n` when its gate, up and down are each its /// stack's type ([`SharedPlan::Joined`]), else kept apart at its own types /// ([`SharedPlan::Apart`]). Under a placed read ([`Read35::Placed`]) a layer -/// whose stacks the card leg does not run ([`card_runs`]) is the host -/// tier's: its stacks are read as [`HOSTED`] and no card entry is asked of -/// them, its shared expert kept apart at its own types, which the card runs -/// whatever the stacks are. A dense one: its three matrices a stack of one -/// expert on the arena's fixed route. +/// whose stacks the dispatch's launches do not run ([`card_leg`]) is typed as +/// the host tier's: its stacks are read as [`HOSTED`] and no card entry is +/// asked of them, its shared expert kept apart at its own types, which the +/// card runs whatever the stacks are; the common leg's layers +/// ([`Leg35::Common`]) are among them, their types the placed side's to +/// read. A dense one: its three matrices a stack of one expert on the +/// arena's fixed route. fn plan_ffn( file: &Split, d: &Dims, @@ -661,7 +694,8 @@ fn plan_ffn( match &spec.ffn { Ffn::Moe(m) => { let n = m.experts as usize; - let hosted = read == Read35::Placed && !card_runs(file, l, (ff, h, n))?; + let hosted = read == Read35::Placed + && !matches!(card_leg(file, l, (ff, h, n))?, Some((Leg35::Dispatch, _))); let (gate_ty, sh_gate) = part_tys(file, l, "gate", (ff, h, n), hosted)?; let (up_ty, sh_up) = part_tys(file, l, "up", (ff, h, n), hosted)?; let (down_ty, sh_down) = part_tys(file, l, "down", (h, ff, n), hosted)?; @@ -993,6 +1027,9 @@ struct Pre35 { /// The file's IMROPE sections and `rope.section_order`, read when the /// load takes media. sections: Option<([u32; 4], vision::mrope::SectionOrder)>, + /// The forms the layers the common card leg runs hold buffers of + /// ([`CommonForms::NONE`] under a whole read). + common: CommonForms, } impl Pre35 { @@ -1072,6 +1109,7 @@ impl Pre35 { d.hidden, HEAD_TY, )?; + let common = common_forms(file, &d, &spec.layers, mode)?; let plans = kinds .iter() .zip(&spec.layers) @@ -1097,6 +1135,7 @@ impl Pre35 { ubatch, base, sections, + common, }) } } @@ -1359,7 +1398,11 @@ impl GpuModel { /// refused by name here, before any plan. pub fn placed_arena_bytes(file: &Split, o: Open35) -> Result { let pre = Pre35::read_placed(file, placed_open(o))?; - placed_scratch(&pre.d, pre.plans.len(), Forms::of(&pre.plans, &pre.d)) + placed_scratch_with( + &pre.d, + pre.plans.len(), + (Forms::of(&pre.plans, &pre.d), pre.common), + ) } /// What a whole load of `file` opened with `o` ([`GpuModel::open`]) @@ -1563,6 +1606,7 @@ impl Body35 { ubatch, base, sections, + common, } = pre; let ctx = o.ctx; let stream = gpu.stream(); @@ -1616,7 +1660,7 @@ impl Body35 { } Some((open, counted)) => { refuse_stage_rows_past(WHAT, open.plan, 0, rope.table.num_bytes() as u64)?; - let side = placed_bytes(&d, plans.len(), open.rows)?; + let side = placed_bytes_with(&d, plans.len(), open.rows, common)?; let u = counted_arena(stream, (d, forms), ubatch.get(), counted, side)?; (u, Some(Placed::new(gpu, w, &plans, &d, open)?)) } diff --git a/crates/gpu/src/arch/qwen3moe/dispatch.rs b/crates/gpu/src/arch/qwen3moe/dispatch.rs index d5c86b7b2..0b0df56b7 100644 --- a/crates/gpu/src/arch/qwen3moe/dispatch.rs +++ b/crates/gpu/src/arch/qwen3moe/dispatch.rs @@ -1151,6 +1151,7 @@ fn slots_down( &mut s.h, )?, GateUpEntry::Q8_0 + | GateUpEntry::Q6k | GateUpEntry::Iq2xs | GateUpEntry::Iq3xxs | GateUpEntry::Iq3s @@ -1376,6 +1377,7 @@ fn apart_down( GateUpEntry::Q4k => kk.kq.enqueue_gate_up_q4k(stream, &gate_up, c.sink, h)?, GateUpEntry::Q5k => kk.kq.enqueue_gate_up_q5k(stream, &gate_up, c.sink, h)?, GateUpEntry::Q8_0 + | GateUpEntry::Q6k | GateUpEntry::Iq2xs | GateUpEntry::Iq3xxs | GateUpEntry::Iq3s diff --git a/crates/gpu/src/arch/qwen3moe/placed.rs b/crates/gpu/src/arch/qwen3moe/placed.rs index 43f1643a0..4e75c671d 100644 --- a/crates/gpu/src/arch/qwen3moe/placed.rs +++ b/crates/gpu/src/arch/qwen3moe/placed.rs @@ -18,14 +18,17 @@ //! and the go; a unit's download of its rows and routing with its places //! (`hostleg`); //! - the shadow: the card's routed experts of the layer when it has any — -//! the K-quant family's gate·up over the places ([`GateUpEntry`]: Q4_K or -//! Q5_K, the table's entry for the stack's type), the q8_1 of the card -//! slots' rows, the down `_sel` the table names for the stack's type -//! ([`DownArm`]: Q4_K, Q5_K or Q6_K) — and on Qwen3.6 the shared -//! expert by its types ([`SharedFfn`]): on its own stacks, a stack of one -//! expert on a fixed route of expert 0, when the same table runs all three -//! parts, else as its own dense FFN at its site types; a slot the host -//! serves is [`HOST`] there and no launch reads or writes it; +//! by the layer's leg ([`FfnLeg`], `accepts::qwen35_leg`): the dispatch's +//! K-quant launches over the places ([`GateUpEntry`]: Q4_K or Q5_K, the +//! table's entry for the stack's type; the down `_sel` the table names for +//! the stack's type, [`DownArm`]: Q4_K, Q5_K or Q6_K), or the common card +//! leg's ([`CardLeg`]: every type its arms run, the unit's wide arm its +//! route GEMMs, [`RouteBlock`]) — the q8_1 of the card slots' rows, and on +//! Qwen3.6 the shared expert by its types ([`SharedFfn`]): on its own +//! stacks, a stack of one expert on a fixed route of expert 0, when the +//! same table runs all three parts, else as its own dense FFN at its site +//! types; a slot the host serves is [`HOST`] there and no launch reads or +//! writes it; //! - the back: the step's wait (a unit's serve and upload are the walk's), //! then the combine of the card's slots with the host's routed sum, the //! residual and the shared expert (`hostleg`), into the next layer's @@ -67,6 +70,9 @@ use super::plan::{ use super::program::{Stores, Tail, enqueue_tail}; use super::scratch::{Arena, Dims, Forms, Io}; use super::wide::{self, CardRoute, GEMV_COLS, HostSide, arena_bytes}; +use crate::card_leg::{ + CardLeg, LegBufs, LegRun, LegShape, LegStacks, RouteBlock, RouteIo, RouteW, leg_bytes_for, +}; use crate::gemm::{Gemm32Kernels, GemmRoute}; use crate::host::handoff::{Handoff, HandoffKernels}; use crate::host::prompt_stream::PromptStream; @@ -79,13 +85,13 @@ use crate::host::xstream::{self, Costs, LAND_BATCHES, LandBatch, RingLayer, XCfg use crate::host::{BatchLeg, PassKind, StepLeg}; use crate::hybrid::{Boundary, BoundaryShape, Chain, HOST, HostResidency, Hybrid, SlotMap}; use crate::kquant::{Act, GateUpAct, SelDown}; -use crate::model::fit::{act_col_bytes, route_bytes}; +use crate::model::fit::{act_col_bytes, act32_col_bytes, route_bytes}; use crate::model::lookup::{f32_gain, f32_tensor, kq_weight}; use crate::q4k_sel::QuantSel; use crate::quant_sel::QuantSelKernels; -use crate::site; +use crate::site::{self, WideActs}; use crate::tensor::{DeviceTensor, Q8Act, Window, WindowMut}; -use crate::weights::Weights; +use crate::weights::{DevWeight, Weights}; use crate::{FaultSink, Gpu, GpuError}; use bloomery_levers::HostCfg; use cuda_core::{CudaStream, DeviceBuffer}; @@ -93,6 +99,7 @@ use gguf::Split; use gguf::quant::GgmlType; use model::moe::UNION_MAX_COLS; use model::placement::Plan; +use model::placement::accepts::{self, Leg35}; use model::placement::kernels::{self, DownAct, DownEntry}; use runtime::sched::{self, At, LayerProgram, Overlap, PortKind}; use std::sync::Arc; @@ -133,11 +140,28 @@ pub(super) struct SharedStack { down_ty: SiteTy, } +/// How a placed layer's card experts run: the dispatch's own launches for the +/// types the family's FFN dispatch holds, or the common card leg +/// (`crate::card_leg`) for the rest of the types its arms run +/// ([`accepts::qwen35_leg`]). +#[derive(Clone, Copy)] +enum FfnLeg { + /// The K-quant family's gate·up at the table's entry and its down by the + /// stack's site type ([`gate_up_sel`], [`down_sel`]). + Dispatch { + gate_up: GateUpEntry, + down_ty: SiteTy, + }, + /// The common leg's launches over the stacks' file types: the gate and up + /// of `gate_up`, the down of `down`. + Common { gate_up: GgmlType, down: GgmlType }, +} + /// A placed layer's routed experts on the card: its slots there (the slot /// map's capacity, fixed for the load's life: the residency machine moves /// which experts fill them, never how many there are), the file's three -/// stacks (the card holds their rows of those slots), the gate·up entry and -/// the down's type; and the shared expert, on a chain that has one. +/// stacks (the card holds their rows of those slots), the leg that runs them +/// ([`FfnLeg`]); and the shared expert, on a chain that has one. pub(super) struct PlacedFfn { /// The slots of the layer's card stacks: the bound a place must be below /// to name one, and the width of the layer's routed table. @@ -146,11 +170,110 @@ pub(super) struct PlacedFfn { up: String, down: String, /// `None` on a layer with no card expert, which launches none. - gate_up: Option, - down_ty: SiteTy, + leg: Option, shared: Option, } +impl PlacedFfn { + /// The types of a layer the common leg runs, when it is one with card + /// experts. + fn common(&self) -> Option<[GgmlType; 2]> { + match self.leg { + Some(FfnLeg::Common { gate_up, down }) => Some([gate_up, down]), + Some(FfnLeg::Dispatch { .. }) | None => None, + } + } +} + +/// The forms a placed load's common-leg layers hold buffers for: which of the +/// leg's two down forms ([`DownAct`]) some layer's down reads, and which of the +/// wide arm's two GEMM forms its gate·up reads (the unit's normed rows) and its +/// down reads (the SwiGLU rows). A load with no common layer holds +/// [`CommonForms::NONE`] and allocates none of it. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct CommonForms { + act8: bool, + blocks32: bool, + x128: bool, + x32: bool, + h128: bool, + h32: bool, +} + +impl CommonForms { + /// No common layer. + pub(super) const NONE: CommonForms = CommonForms { + act8: false, + blocks32: false, + x128: false, + x32: false, + h128: false, + h32: false, + }; + + /// The forms of the layers whose `[gate·up, down]` file types are `tys`: + /// the down's launch form ([`kernels::down`]) and each GEMM's form + /// ([`kernels::gemm`], [`kernels::gemm_reads_blocks32`]). + pub(super) fn of(tys: impl IntoIterator) -> CommonForms { + let mut f = CommonForms::NONE; + for [gate_up, down] in tys { + match kernels::down(down) { + Some((_, DownAct::Q8Act)) => f.act8 = true, + Some((_, DownAct::Q8Blocks32)) => f.blocks32 = true, + None => {} + } + match kernels::gemm(gate_up).map(kernels::gemm_reads_blocks32) { + Some(true) => f.x32 = true, + Some(false) => f.x128 = true, + None => {} + } + match kernels::gemm(down).map(kernels::gemm_reads_blocks32) { + Some(true) => f.h32 = true, + Some(false) => f.h128 = true, + None => {} + } + } + f + } + + /// Whether any layer is a common one. + fn any(self) -> bool { + self != CommonForms::NONE + } + + /// The leg's device bytes for an arena of `rows` rows of `shape` + /// ([`LegBufs`] of each down form held), and past [`GEMV_COLS`] rows the + /// wide arm's: the gate and up rows, and the unit's normed rows and the + /// SwiGLU rows in the forms the GEMMs read. + fn bytes(self, shape: RouteShape, rows: usize) -> usize { + let leg = LegShape { + cap: rows.min(GEMV_COLS), + slots_per_col: shape.pitch, + n: shape.hidden, + ff: shape.ff, + }; + let forms = [ + (self.act8, DownAct::Q8Act), + (self.blocks32, DownAct::Q8Blocks32), + ]; + let bufs: usize = forms + .into_iter() + .filter(|&(held, _)| held) + .map(|(_, act)| leg_bytes_for(leg, act)) + .sum(); + if rows <= GEMV_COLS || !self.any() { + return bufs; + } + let slots = rows * shape.pitch; + let acts = |cols: usize, k: usize, (q128, q32): (bool, bool)| { + cols * (usize::from(q128) * act_col_bytes(k) + usize::from(q32) * act32_col_bytes(k)) + }; + bufs + 2 * slots * shape.ff * 4 + + acts(rows, shape.hidden, (self.x128, self.x32)) + + acts(slots, shape.ff, (self.h128, self.h32)) + } +} + /// The residency machine's tally shape for a placed load over arena dims /// `d`: the ids a row routes a layer, and the rows a pass runs. The ids are /// the step boundary's `n_used` (the router's routed slots, not the shared @@ -208,6 +331,95 @@ pub(super) struct PlacedRows { shared: Option, routes: Vec>, ring: Option, + /// The buffers of the load's common-leg layers; `None` on a load with none. + common: Option, +} + +/// An arena's buffers for the common-leg layers ([`FfnLeg::Common`]): the +/// common card leg's own (one [`LegBufs`] per down form the load's layers +/// read, of the gemv arm's [`GEMV_COLS`] rows at most), and, past that many +/// rows, the wide arm's: the unit's normed rows +/// and the SwiGLU rows in the forms the route GEMMs read, and the gate and up +/// rows. +struct CommonRows { + act8: Option, + blocks32: Option, + wide: Option, +} + +/// The wide arm's buffers of the common-leg layers ([`CommonRows`]). +struct CommonWide { + x: WideActs, + h: WideActs, + g: DeviceBuffer, + u: DeviceBuffer, +} + +impl CommonRows { + /// The buffers of an arena of `rows` rows for `shape` over the forms + /// `forms` name; `None` for a load with no common layer. Load-time only. + fn new( + stream: &CudaStream, + shape: RouteShape, + rows: usize, + forms: CommonForms, + ) -> Result, GpuError> { + if !forms.any() { + return Ok(None); + } + let leg = LegShape { + cap: rows.min(GEMV_COLS), + slots_per_col: shape.pitch, + n: shape.hidden, + ff: shape.ff, + }; + let slots = rows * shape.pitch; + let wide = (rows > GEMV_COLS) + .then(|| { + Ok::<_, GpuError>(CommonWide { + x: WideActs::new(stream, (forms.x128, forms.x32), rows, shape.hidden)?, + h: WideActs::new(stream, (forms.h128, forms.h32), slots, shape.ff)?, + g: DeviceBuffer::zeroed(stream, slots * shape.ff)?, + u: DeviceBuffer::zeroed(stream, slots * shape.ff)?, + }) + }) + .transpose()?; + Ok(Some(CommonRows { + act8: forms.act8.then(|| LegBufs::new(stream, leg)).transpose()?, + blocks32: forms + .blocks32 + .then(|| LegBufs::for_down(stream, leg, DownAct::Q8Blocks32)) + .transpose()?, + wide, + })) + } + + /// The leg's buffers a down of file type `down` reads its columns in + /// ([`kernels::down`]'s form), refused by name when the load holds none. + fn bufs(&mut self, down: GgmlType) -> Result<&mut LegBufs, GpuError> { + let held = match kernels::down(down) { + Some((_, DownAct::Q8Act)) => self.act8.as_mut(), + Some((_, DownAct::Q8Blocks32)) => self.blocks32.as_mut(), + None => None, + }; + held.ok_or_else(|| { + GpuError::shape( + WHAT, + format!( + "a common-leg layer of a {down} down, whose form the load holds no buffers of" + ), + ) + }) + } + + /// Device bytes. + fn bytes(&self) -> usize { + self.act8.as_ref().map_or(0, LegBufs::bytes) + + self.blocks32.as_ref().map_or(0, LegBufs::bytes) + + self.wide.as_ref().map_or(0, |w| { + w.x.bytes() + w.h.bytes() + w.g.num_bytes() + w.u.num_bytes() + }) + } } /// The ring pass's buffers of a wide arena ([`ring_pass`]): the routed table @@ -254,7 +466,7 @@ impl PlacedRows { stream: &CudaStream, shape: RouteShape, rows: usize, - cards: &[usize], + (cards, common): (&[usize], CommonForms), ) -> Result { if rows == 0 { return Err(GpuError::shape( @@ -294,6 +506,7 @@ impl PlacedRows { shared, routes, ring: None, + common: CommonRows::new(stream, shape, rows, common)?, }) } @@ -347,6 +560,7 @@ impl PlacedRows { + r.union.num_bytes() + r.maps.iter().map(DeviceBuffer::num_bytes).sum::() }) + + self.common.as_ref().map_or(0, CommonRows::bytes) } } @@ -360,6 +574,8 @@ pub(super) struct PlacedKernels { kquant: QuantSelKernels, leg: HostLegKernels, g32: Option, + /// The common card leg's modules, on a load with a common-leg layer. + card: Option, } /// What every placed walk reads besides the arena: each layer's card @@ -522,6 +738,7 @@ impl Placed { let stage = map.stage_view(); let slots = Arc::new(DeviceTensor::upload(stream, &stage, n, shape.n_expert)?); let cards: Vec = ffn.iter().map(|f| f.n_card).collect(); + let common = CommonForms::of(ffn.iter().filter_map(PlacedFfn::common)); let live = o.glue.source().is_some(); let maps = if o.rows > GEMV_COLS { layer_maps(stream, &stage, n, shape.n_expert)? @@ -574,6 +791,7 @@ impl Placed { g32: (o.rows > GEMV_COLS) .then(|| Gemm32Kernels::load(ctx)) .transpose()?, + card: common.any().then(|| CardLeg::load(gpu)).transpose()?, }; let placed = Placed { hybrid, @@ -586,8 +804,8 @@ impl Placed { k, cap: o.rows, }, - step: PlacedRows::new(stream, shape, STEP_ROWS, &[])?, - pass: PlacedRows::new(stream, shape, o.rows, &cards)?, + step: PlacedRows::new(stream, shape, STEP_ROWS, (&[], common))?, + pass: PlacedRows::new(stream, shape, o.rows, (&cards, common))?, pass_hsum: DeviceBuffer::zeroed(stream, o.rows * shape.hidden)?, glue: o.glue, }; @@ -597,7 +815,12 @@ impl Placed { .filter(|&&c| c > 0) .map(|&c| route_bytes(o.rows * shape.pitch, c)) .sum(); - let counted = placed_total(d, n, o.rows, if o.rows > GEMV_COLS { tables } else { 0 })?; + let counted = placed_total( + d, + n, + o.rows, + (if o.rows > GEMV_COLS { tables } else { 0 }, common), + )?; if made as u64 != counted { return Err(GpuError::shape( WHAT, @@ -619,6 +842,15 @@ impl Placed { /// [`HostTier::xstream_start`]: crate::host::HostTier::xstream_start pub(super) fn start_xstream(&mut self, gpu: &Gpu, costs: Costs) -> Result<(), GpuError> { let shape = self.side.shape; + if let Some(l) = self.side.ffn.iter().position(|f| f.common().is_some()) { + return Err(GpuError::shape( + "BLOOMERY_XSTREAM", + format!( + "split on a placed load whose layer {l} runs its card experts through the \ + common leg: the ring pass reads the dispatch's K-quant words only" + ), + )); + } if self.side.cap <= GEMV_COLS { return Err(GpuError::shape( "BLOOMERY_XSTREAM", @@ -764,6 +996,19 @@ impl Placed { /// know; [`Placed::new`] checks the load against the count at its split. A /// chain of dense FFNs is refused by name. pub(super) fn placed_bytes(d: &Dims, layers: usize, rows: usize) -> Result { + placed_bytes_with(d, layers, rows, CommonForms::NONE) +} + +/// [`placed_bytes`] for a load with the common-leg layers `common` names: +/// their buffers counted beside the rest, at the forms the plan's types hold +/// (a load whose plan gives none of a form's layers card experts holds fewer, +/// which [`Placed::new`] checks at its split). +pub(super) fn placed_bytes_with( + d: &Dims, + layers: usize, + rows: usize, + common: CommonForms, +) -> Result { let s = RouteShape::of(d)?; let slots = rows * s.pitch; let table = route_bytes(slots, s.n_expert).max(route_bytes(slots, 1)); @@ -771,13 +1016,19 @@ pub(super) fn placed_bytes(d: &Dims, layers: usize, rows: usize) -> Result GEMV_COLS { layers * table } else { 0 }, + (if rows > GEMV_COLS { layers * table } else { 0 }, common), ) } /// [`placed_bytes`] with the layers' tables counted as `tables` bytes in all -/// (past [`GEMV_COLS`] rows). -fn placed_total(d: &Dims, layers: usize, rows: usize, tables: usize) -> Result { +/// (past [`GEMV_COLS`] rows), and the common-leg layers' buffers of `common`'s +/// forms in the step's and the pass's arena. +fn placed_total( + d: &Dims, + layers: usize, + rows: usize, + (tables, common): (usize, CommonForms), +) -> Result { let s = RouteShape::of(d)?; let shared = |r: usize| { if s.shared() { @@ -799,7 +1050,9 @@ fn placed_total(d: &Dims, layers: usize, rows: usize, tables: usize) -> Result usize { /// (`model::placement::workstation::placed_with_arena`), and what [`counted_arena`] /// holds the load to. pub(super) fn placed_scratch(d: &Dims, layers: usize, forms: Forms) -> Result { + placed_scratch_with(d, layers, (forms, CommonForms::NONE)) +} + +/// [`placed_scratch`] for a load with the common-leg layers `common` names +/// ([`placed_bytes_with`]). +pub(super) fn placed_scratch_with( + d: &Dims, + layers: usize, + (forms, common): (Forms, CommonForms), +) -> Result { let arena = arena_bytes(d, prompt_rows(PLACED_ROWS, d.ctx), forms) as u64; - Ok(arena + placed_bytes(d, layers, PLACED_ROWS)?) + Ok(arena + placed_bytes_with(d, layers, PLACED_ROWS, common)?) } /// The ubatch arena of `d` for ubatches of `u` tokens on a placed load, @@ -964,6 +1227,44 @@ pub(super) fn apart_plans(plans: Vec) -> Vec { .collect() } +/// Layer `l`'s routed stacks `[gate, up, down]` of `n` experts of `ff` rows of +/// `h` values, each a resident word plane of its shape ([`DevWeight::KQuant`]: +/// the file's blocks), as their file types: the types the placed card leg's +/// classifier reads ([`accepts::qwen35_leg`]). A stack that is not one, or of +/// another shape, is refused by name with the layer. +fn stack_files( + w: &Weights, + names: [&str; 3], + (n, ff, h): (usize, usize, usize), + l: usize, +) -> Result<[GgmlType; 3], GpuError> { + let stack = |name: &str, rows: usize, k: usize| -> Result { + let Some(DevWeight::KQuant { ty, w: t, k: wk }) = w.get(name) else { + return Err(GpuError::tensor( + WHAT, + name, + "a resident K-quant word plane (the file's blocks)", + )); + }; + if t.rows() != rows || *wk != k { + return Err(GpuError::shape( + WHAT, + format!( + "layer {l}: {name} is {ty} {} rows x {wk}; the chain takes {rows} rows x {k}", + t.rows() + ), + )); + } + Ok(*ty) + }; + let [gate, up, down] = names; + Ok([ + stack(gate, n * ff, h)?, + stack(up, n * ff, h)?, + stack(down, n * h, ff)?, + ]) +} + /// Layer `l`'s card stacks under `map` (module doc, [`Placed::new`]): their /// slots are the map's capacity, which a residency machine's evictions and /// admissions leave as it is (`SlotMap::evict`), where the live count drops @@ -982,11 +1283,32 @@ fn placed_ffn( model::arch::qwen3moe::names::ffn_down_exps(l), ); let (h, ff) = (s.hidden, s.ff); - let (gate_up, down_ty) = if n_card > 0 { - let [g, u, d] = stack_tys(w, [&gate, &up, &down], (n_card, ff, h), l)?; - (Some(gate_up_entry(g, u, WHAT)?), d) + let leg = if n_card > 0 { + let tys = stack_files(w, [&gate, &up, &down], (n_card, ff, h), l)?; + Some(match accepts::qwen35_leg(tys) { + Some(Leg35::Dispatch) => { + let [g, u, d] = stack_tys(w, [&gate, &up, &down], (n_card, ff, h), l)?; + FfnLeg::Dispatch { + gate_up: gate_up_entry(g, u, WHAT)?, + down_ty: d, + } + } + Some(Leg35::Common) => FfnLeg::Common { + gate_up: tys[0], + down: tys[2], + }, + None => { + return Err(GpuError::shape( + WHAT, + format!( + "layer {l}'s card stacks are {tys:?} (gate, up, down): no card leg runs \ + them (`accepts::qwen35_leg`)" + ), + )); + } + }) } else { - (None, p.ffn.down_ty) + None }; let shared = match &p.ffn.route { FfnRoute::Router { @@ -1028,8 +1350,7 @@ fn placed_ffn( gate, up, down, - gate_up, - down_ty, + leg, shared, }) } @@ -1188,47 +1509,53 @@ fn shadow( } = side.shape; let i = s.col(m)?; if f.n_card > 0 { - let n_slots = m * pitch; - let entry = f.gate_up.ok_or(GpuError::state( - WHAT, - "a gate·up entry for a layer with card experts", - ))?; - gate_up_sel( - (stream, &side.k.kquant), - entry, - &GateUpAct { - wg: kq_weight(w, &f.gate)?, - wu: kq_weight(w, &f.up)?, - act: &s.act_ffn[i], - sel: &rows.sel, - n_slots, - rows_per_expert: ff, - slots_per_col: pitch, - rule: Act::SiluMul, - }, - c.sink, - &mut s.h, - )?; - gpu.q4k_sel().enqueue_quantize_sel( - stream, - &QuantSel { - x: &s.h, - cols: 0..n_slots, - sel: &rows.sel, - n_card: f.n_card, - }, - c.sink, - &mut s.act_h[i], - )?; - down_sel( - (gpu, k, &side.k.kquant), - (f.down_ty, kq_weight(w, &f.down)?), - (&s.act_h[i], &rows.sel), - n_slots, - hidden, - c.sink, - &mut s.down, - )?; + match f.leg { + Some(FfnLeg::Dispatch { gate_up, down_ty }) => { + let n_slots = m * pitch; + gate_up_sel( + (stream, &side.k.kquant), + gate_up, + &GateUpAct { + wg: kq_weight(w, &f.gate)?, + wu: kq_weight(w, &f.up)?, + act: &s.act_ffn[i], + sel: &rows.sel, + n_slots, + rows_per_expert: ff, + slots_per_col: pitch, + rule: Act::SiluMul, + }, + c.sink, + &mut s.h, + )?; + gpu.q4k_sel().enqueue_quantize_sel( + stream, + &QuantSel { + x: &s.h, + cols: 0..n_slots, + sel: &rows.sel, + n_card: f.n_card, + }, + c.sink, + &mut s.act_h[i], + )?; + down_sel( + (gpu, k, &side.k.kquant), + (down_ty, kq_weight(w, &f.down)?), + (&s.act_h[i], &rows.sel), + n_slots, + hidden, + c.sink, + &mut s.down, + )?; + } + Some(FfnLeg::Common { down, .. }) => { + common_shadow(c, side, rows, s, normed, (f, down, m))?; + } + None => { + return Err(GpuError::state(WHAT, "a leg for a layer with card experts")); + } + } } match (&f.shared, rows.shared.as_mut()) { (Some(SharedFfn::Stack(sh)), Some(r)) => shared_stack(c, side, (sh, r), s, m)?, @@ -1241,6 +1568,146 @@ fn shadow( Ok(()) } +/// The plan's card leg arms the common-leg layers' stacks are admitted by +/// ([`accepts::leg_arms`] of the qwen35moe body), and what a refusal of a +/// stack of another type names. +fn common_stacks<'a>(w: &'a Weights, f: &'a PlacedFfn) -> Result, GpuError> { + let arms = accepts::leg_arms(accepts::Program::Qwen35Body) + .ok_or(GpuError::state(WHAT, "the qwen35moe body's card leg arms"))?; + LegStacks::of_arms( + w, + [&f.gate, &f.up, &f.down], + arms, + ( + "a resident routed gate or up stack of a type the qwen35moe card leg runs \ + (`accepts::qwen35_common_gate_up`)", + "a resident routed down stack of a type the qwen35moe card leg runs \ + (`accepts::qwen35_common_down`)", + ), + ) +} + +/// Layer `c.layer`'s card experts through the common leg over `m` rows of the +/// gemv arm's width (at most [`GEMV_COLS`]): the four launches of +/// [`CardLeg::enqueue_rows`] over the f32 normed rows (`normed`, the step +/// boundary's, or the arena's own when `None`) and the places in `rows`, +/// writing the slots' down rows into the arena's `down` as the dispatch's +/// launches do. `f` the layer's placed FFN and `down` the file type of its +/// down. +fn common_shadow( + c: &Ctx<'_>, + side: &PlacedSide, + rows: &mut PlacedRows, + s: &mut Arena, + normed: Option<&DeviceBuffer>, + (f, down, m): (&PlacedFfn, GgmlType, usize), +) -> Result<(), GpuError> { + let leg = side.k.card.as_ref().ok_or(GpuError::state( + WHAT, + "the common card leg's modules on a load with a common-leg layer", + ))?; + let PlacedRows { sel, common, .. } = rows; + let bufs = common + .as_mut() + .ok_or(GpuError::state( + WHAT, + "the common-leg buffers of a load with a common-leg layer", + ))? + .bufs(down)?; + let stacks = common_stacks(c.w, f)?; + stacks.check(f.n_card, bufs.shape())?; + let Arena { + normed: own, + down: out, + .. + } = s; + let run = LegRun { + layer: c.layer, + stacks: &stacks, + normed: normed.unwrap_or(&*own), + m, + sel, + n_card: f.n_card, + rule: Act::SiluMul, + }; + leg.enqueue_rows(c.gpu, bufs, &run, out) +} + +/// Layer `c.layer`'s card experts through the common leg over `m` rows past +/// the gemv arm's width: the layer's routed table through its map, the unit's +/// normed rows in the form the gate and up route GEMMs read, and the routed +/// block ([`RouteBlock`]: gate, up, the SwiGLU into the form the down reads, +/// down) over the card's stacks, writing the slots' down rows into the arena's +/// `down` as the dispatch's wide route does. The shared expert is the caller's. +fn common_wide( + c: &Ctx<'_>, + side: &PlacedSide, + (table, common): (&mut GemmRoute, &mut CommonRows), + s: &mut Arena, + (f, map, m): (&PlacedFfn, &DeviceBuffer, usize), +) -> Result<(), GpuError> { + let g32 = side.k.g32.as_ref().ok_or(GpuError::state( + WHAT, + "the 32-value kernels of a side whose units run the wide arm", + ))?; + let stack = common_stacks(c.w, f)?; + let [gate, up, down] = stack.routes()?; + let (gpu, stream) = (c.gpu, c.gpu.stream()); + let CommonWide { x, h, g, u } = common.wide.as_mut().ok_or(GpuError::state( + WHAT, + "the wide arm's buffers of a load with a common-leg layer", + ))?; + let d = s.dims; + let slots = d.slots(); + let Arena { + normed, + route, + down: out, + .. + } = s; + g32.enqueue_route_remap(stream, route.ids(), map, m * slots, table, c.sink)?; + match gate { + RouteW::Grouped(..) => { + let a = x.q128.as_mut().ok_or(GpuError::state( + WHAT, + "the unit's q8_1 rows of 128 a grouped gate and up read", + ))?; + gpu.enqueue_quantize_gemm(normed, m, a, c.sink)?; + } + RouteW::Blocks32(_) => { + let a = x.q32.as_mut().ok_or(GpuError::state( + WHAT, + "the unit's q8 rows of 32 a 32-value gate and up read", + ))?; + g32.enqueue_quantize_gemm32(stream, normed, m, a, c.sink)?; + } + } + let blk = RouteBlock { + gate, + up, + down, + ff: d.ff, + hidden: d.hidden, + rule: Act::SiluMul, + }; + blk.enqueue( + gpu, + (&c.k.gemm, g32), + (&*table, slots), + m * slots, + RouteIo { + x128: x.q128.as_ref(), + x32: x.q32.as_ref(), + g, + u, + h128: h.q128.as_mut(), + h32: h.q32.as_mut(), + y: out, + }, + c.sink, + ) +} + /// Layer `c.layer`'s shared expert as a stack of one expert on a fixed route /// of expert 0 ([`SharedFfn::Stack`]) over `m` rows: the gate·up `_sel` /// over the q8_1 of the normed rows into the SwiGLU rows, their q8_1, and @@ -1356,6 +1823,7 @@ fn gate_up_sel( GateUpEntry::Q4k => kq.enqueue_gate_up_q4k(stream, a, sink, h), GateUpEntry::Q5k => kq.enqueue_gate_up_q5k(stream, a, sink, h), GateUpEntry::Q8_0 + | GateUpEntry::Q6k | GateUpEntry::Iq2xs | GateUpEntry::Iq3xxs | GateUpEntry::Iq3s @@ -1530,8 +1998,8 @@ fn card_wide( WHAT, "a map of the card's experts for every layer of a wide unit", ))?; - let table = rows - .routes + let PlacedRows { routes, common, .. } = rows; + let table = routes .get_mut(c.layer) .ok_or(GpuError::state( WHAT, @@ -1542,6 +2010,25 @@ fn card_wide( WHAT, "the remap kernels of a side whose units run the wide arm", ))?; + let f = side + .ffn + .get(c.layer) + .ok_or(GpuError::state(WHAT, "a placed FFN for every layer"))?; + // A common-leg layer's routed experts run through the common leg; the + // wide arm's own routed GEMMs (the table) are the dispatch layers'. + let table = match (f.common(), table, common.as_mut()) { + (Some(_), Some(table), Some(common)) => { + common_wide(c, side, (table, common), s, (f, map, m))?; + None + } + (Some(_), _, _) => { + return Err(GpuError::state( + WHAT, + "a table and the common-leg buffers for a common-leg layer with card experts", + )); + } + (None, table, _) => table, + }; wide::ffn_card(c, &c.p.ffn, s, m, Some(CardRoute { map, table, remap })) } diff --git a/crates/gpu/src/arch/qwen3moe/swap.rs b/crates/gpu/src/arch/qwen3moe/swap.rs index db83caa79..4db152a62 100644 --- a/crates/gpu/src/arch/qwen3moe/swap.rs +++ b/crates/gpu/src/arch/qwen3moe/swap.rs @@ -136,36 +136,49 @@ mod tests { } } - /// Qwen3.6's leg: a Q4_K or Q5_K gate and up of one type with a Q4_K, - /// Q5_K or Q6_K down is a card layer; a Q6_K gate and up (a type its - /// down runs, not its gate·up), a gate and up of two types, a Q6_K up - /// beside a Q4_K gate and a carded layer with a stack of a type the leg - /// does not run (a Q5_1 down, an IQ4_XS gate) are refused by name. - /// Mutant: Qwen3.8's down arm for Qwen3.6's — the Q4_K and Q6_K downs are - /// refused and the Q5_1 one passes. + /// Qwen3.6's leg: the dispatch's types (a Q4_K or Q5_K gate and up of one + /// type with a Q4_K, Q5_K or Q6_K down) and the common leg's (a Q6_K, Q8_0, + /// IQ3_XXS or IQ4_XS gate and up of one type with a down any of the + /// common down arm's types) are card layers; a gate and up of two types, + /// and a carded layer with a stack of a type no card launch runs (an + /// IQ2_XS gate: no prompt GEMM, an MXFP4 down) are refused by name, the + /// refusal listing both arms' types in file order. PIN(2026-10-22): the + /// IQ4_XS and Q6_K gate and up and the Q5_1 down moved from refused to card + /// layers when `placement::accepts` gave the body the common leg's arms. + /// Mutant: the dispatch's arms (`accepts::qwen35_gate_up`/`qwen35_down`) + /// for the common ones in `leg_arms` — the IQ4_XS, Q6_K, Q8_0 and IQ3_XXS + /// layers are refused. #[test] fn qwen36s_leg_checks_a_card_layer_against_its_launches() { - use GgmlType::{IQ4_XS, Q4_K, Q5_1, Q5_K, Q6_K}; + use GgmlType::{IQ2_XS, IQ3_XXS, IQ4_NL, IQ4_XS, MXFP4, Q4_K, Q5_1, Q5_K, Q6_K, Q8_0}; for ok in [ [Q4_K, Q4_K, Q4_K], [Q4_K, Q4_K, Q6_K], [Q5_K, Q5_K, Q5_K], [Q5_K, Q5_K, Q6_K], + [Q4_K, Q4_K, Q5_1], + [IQ4_XS, IQ4_XS, Q4_K], + [Q8_0, Q8_0, Q8_0], + [IQ3_XXS, IQ3_XXS, IQ4_NL], + [Q6_K, Q6_K, Q4_K], + [Q6_K, Q6_K, Q8_0], ] { assert_eq!(layer::(ok), Ok(()), "{ok:?}"); } for bad in [ - [Q4_K, Q4_K, Q5_1], - [IQ4_XS, IQ4_XS, Q4_K], - [Q6_K, Q6_K, Q4_K], + [IQ2_XS, IQ2_XS, Q4_K], + [Q4_K, Q4_K, MXFP4], [Q4_K, Q5_K, Q4_K], [Q4_K, Q6_K, Q4_K], - [Q6_K, Q6_K, Q6_K], + [Q6_K, Q8_0, Q6_K], ] { let text = layer::(bad).expect_err(&format!("{bad:?} is refused")); assert!( text.contains("layer 0's gate") - && text.ends_with("a gate and an up of one type, [Q4_K, Q5_K], and a down of [Q4_K, Q5_K, Q6_K]"), + && text.ends_with( + "a gate and an up of one type, [Q8_0, Q4_K, Q5_K, Q6_K, IQ3_XXS, IQ4_XS], \ + and a down of [Q5_1, Q8_0, Q4_K, Q5_K, Q6_K, IQ3_XXS, IQ4_NL, IQ4_XS]" + ), "{text}" ); } @@ -214,17 +227,17 @@ mod tests { /// layer 2 arm catches it. #[test] fn the_table_checks_every_layer() { - use GgmlType::{Q4_K, Q6_K}; + use GgmlType::{IQ2_XS, Q4_K, Q6_K}; let three = |l: usize| { if l == 2 { - [Q6_K, Q6_K, Q4_K] + [IQ2_XS, IQ2_XS, Q4_K] } else { [Q4_K, Q4_K, Q6_K] } }; match stacks::(&model(4, three), |_| Ok(true)) { Err(GpuError::Shape { detail, .. }) => { - assert!(detail.starts_with("layer 2's gate q6_K"), "{detail}"); + assert!(detail.starts_with("layer 2's gate iq2_xs"), "{detail}"); } _ => panic!("layer 2 is refused by name"), } @@ -240,14 +253,14 @@ mod tests { } /// A layer the plan gives no card expert loads whatever its types: the - /// leg never runs its stacks, so a gate and an up of two card types (or a - /// Q6_K gate and up on Qwen3.6's leg) is refused only on a carded layer. + /// leg never runs its stacks, so a gate and an up of two card types (or an + /// IQ2_XS gate and up on Qwen3.6's leg) is refused only on a carded layer. /// Mutant: the check run on every layer whatever `carded` says — the /// uncarded arms are refused. #[test] fn a_layer_with_no_card_expert_is_not_checked() { - use GgmlType::{Q4_K, Q5_K, Q6_K}; - for bad in [[Q4_K, Q5_K, Q4_K], [Q6_K, Q6_K, Q4_K]] { + use GgmlType::{IQ2_XS, Q4_K, Q5_K}; + for bad in [[Q4_K, Q5_K, Q4_K], [IQ2_XS, IQ2_XS, Q4_K]] { let two = model(2, |_| bad); assert!( stacks::(&two, |_| Ok(false)).is_ok(), diff --git a/crates/gpu/src/arch/qwen3moe/swap35.rs b/crates/gpu/src/arch/qwen3moe/swap35.rs index f6c34fcf1..0584669fa 100644 --- a/crates/gpu/src/arch/qwen3moe/swap35.rs +++ b/crates/gpu/src/arch/qwen3moe/swap35.rs @@ -18,8 +18,8 @@ use model::placement::ModelTensors; use model::placement::accepts::Program; /// Qwen3.6's card leg ([`CardLeg`]): the qwen35moe body's arms -/// (`placement::accepts::leg_arms`: a Q4_K or Q5_K gate and up, a Q4_K, Q5_K -/// or Q6_K down), which its place rule reads too. +/// (`placement::accepts::leg_arms`: the dispatch's types and the common leg's, +/// `qwen35_leg`), which its place rule reads too. pub(super) struct Leg35; impl CardLeg for Leg35 { @@ -30,7 +30,8 @@ impl CardLeg for Leg35 { /// The stacks of `model`'s layers, each layer's types read from its tensors, /// each layer `carded` names checked against [`Leg35`] ([`swap::stacks`]). /// Refused by name: a layer without all three stacks, and a carded layer the -/// leg refuses (a Q6_K gate or up among them). Load-time only. +/// leg refuses (a gate or up of a type no card gate·up launch reads among +/// them). Load-time only. pub(super) fn stacks( model: &ModelTensors, carded: impl Fn(usize) -> Result, From 4bfe02fd9de4c4c720e9bdf4326ce5e768b671a6 Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 22 Oct 2026 02:42:34 +0900 Subject: [PATCH 04/10] gate-gpu-qwen35moe-leg: the common-placed layers against the host on four fixture variants (q35iq4xs, q35q80, q35iq4q8, q35q6k) Co-Authored-By: Claude Opus 5.5 --- .../gpu-gates/src/bin/gate_qwen35moe_e2e.rs | 319 +++++++++++++++++- justfile | 10 + tools/fixture-variants.tsv | 4 + tools/gate-class.tsv | 1 + 4 files changed, 333 insertions(+), 1 deletion(-) diff --git a/crates/gpu-gates/src/bin/gate_qwen35moe_e2e.rs b/crates/gpu-gates/src/bin/gate_qwen35moe_e2e.rs index a1210bd60..e09a5c164 100644 --- a/crates/gpu-gates/src/bin/gate_qwen35moe_e2e.rs +++ b/crates/gpu-gates/src/bin/gate_qwen35moe_e2e.rs @@ -425,12 +425,13 @@ mod gate { use bloomery_levers::{CARD_BUDGET, HostCfg}; use cuda_core::sys; use gguf::Split; - use gguf::quant::half_to_f32; + use gguf::quant::{GgmlType, half_to_f32}; use model::arch::models::Mixer; use model::arch::models::shape::{AttnShape, MoeShape}; use model::arch::qwen35moe::hparams::Hparams; use model::arch::qwen35moe::names; use model::placement::PlanLevers; + use model::placement::accepts::{self, Leg35}; use refset::arch::qwen35moe::fixture as fx; use refset::arch::qwen35moe::{BATCH, D1K, IK, MODEL, STEP4}; use runtime::{Out, Target, Want}; @@ -1463,6 +1464,14 @@ mod gate { /// their arithmetic is the host's own, 5.1e-3 to 5.5e-3 in today's six /// runs; 2e-2 is four times the larger. const FREE_PROBE_HOST: f64 = 2e-2; + /// PIN(2026-10-22): (o9)'s wide unit's last-row logits against the host + /// tier's wide unit over the same ids (the card runs the experts through + /// the route GEMMs on q8_1 rows, the host through ik's 8-bit rule): + /// 1.90e-2 to 2.26e-2 in today's five runs (the base fixture's dispatch + /// layers and the variants `q35iq4xs`, `q35q80`, `q35iq4q8` and `q35q6k`); + /// 1e-1 is four times the larger. A route block whose gate and up weights + /// are swapped reads 4.8e-1 on `q35q6k`. + const COMMON_WIDE_REL: f64 = 1e-1; /// The free-running clause: every layer's output of the one-token /// chain against ik's `l_out-L` within [`FREE_BAND`], and the last @@ -3794,6 +3803,309 @@ mod gate { Ok(ok) } + // ------------------------------- (o9) the common card leg on the placed load + + /// The layers of `file` whose three routed stacks the common card leg runs + /// ([`Leg35::Common`]: a type the dispatch's launches do not), with the + /// stacks' file types, read from the header. + fn common_layers(file: &Split) -> Result, GateError> { + let mut out = Vec::new(); + for l in 0..n_layer() { + let mut tys = [GgmlType::F32; 3]; + for (ty, name) in tys.iter_mut().zip(model::arch::names::routed_stacks(l)) { + let (_, t) = file + .find(&name) + .ok_or_else(|| format!("no tensor {name}"))?; + *ty = t.ty; + } + if accepts::qwen35_leg(tys) == Some(Leg35::Common) { + out.push((l, tys)); + } + } + Ok(out) + } + + /// What one placed load of (o9) saw: its plan, its layers' card experts, the + /// arena the planner counts against the one the load counts, five graph + /// steps with each layer's output, the prompt call against the fifth step, + /// a wide unit. + struct CommonRun { + /// The plan's card experts and host experts. + experts: (u64, u64), + /// The card experts each layer holds. + counts: Vec, + /// The plan's counted arena and placed side, and the load's. + scratch: (u64, u64), + /// Five graph steps over the batch tokens. + steps: PathRun, + /// The layers' output rows of each of the five steps, `[step][layer]`. + taps: Vec>>, + /// The prompt call (a unit within the gemv arm's cut) left the fifth + /// step's bits. + call_same: bool, + /// The wide unit's greedy id and logits. + wide: (u32, Vec), + /// The host tier's slots on the step and the batch ports. + host_slots: (u64, u64), + } + + /// One placed load of (o9) at the card budget `pick` names for the file's + /// planner, run over `toks` (the gemv arm) and a wide unit of an lcg's ids. + fn common_chain( + label: &str, + pick: impl FnOnce(&PlaceQ3) -> Result, + toks: &[u32], + host: HostCfg, + ) -> Result { + let file = open_model()?; + let o = Open35 { + ctx: CTX, + mma: true, + ubatch: U_GATE, + kv: KvQ8::F16, + media: false, + }; + let q = PlaceQ3::qwen35(&file, Place::parse("cuda0")?, o)?; + let budget = pick(&q)?; + let levers = PlanLevers { + card_budget_bytes: Some(budget), + }; + let plan = q.plan(CTX, &levers)?; + for line in q.records(&plan, None)? { + line.print(); + } + let experts = ( + plan.cards.first().ok_or("a plan of no card")?.experts, + plan.host.experts, + ); + let card = plan.machine.cards.first().ok_or("a plan of no card")?; + let scratch = ( + model::placement::workstation::counted_arena_bytes(card), + GpuModel::::placed_arena_bytes(&file, o)?, + ); + let t = Instant::now(); + let mut m = GpuModel::::open_placed(file, &plan, o, host)?; + let counts = m + .body("gate_qwen35moe_e2e")? + .placed() + .ok_or("an open_placed load with no placed side")? + .card_counts() + .to_vec(); + println!( + "placed common {label}: card budget {budget} B card_experts={} host_experts={} card \ + experts per layer {counts:?} resident_bytes={} in {:.1} s (runtime value)", + experts.0, + experts.1, + m.resident_bytes(), + t.elapsed().as_secs_f64() + ); + // The gemv arm: five graph steps, then the same five eager with each + // layer's output kept. + m.set_mode(StepMode::Graph); + println!( + "placed common {label} capture graph_nodes={}", + m.capture_step()? + ); + let mut steps = decode_run(&mut m, toks, false)?; + m.set_mode(StepMode::Eager); + m.set_layer_taps(true)?; + fresh(&mut m)?; + let mut taps = Vec::new(); + for &tok in toks { + m.step(&[tok])?; + taps.push(m.layer_taps()?); + } + m.set_layer_taps(false)?; + // The prompt call against the fifth step. + m.set_mode(StepMode::Graph); + fresh(&mut m)?; + let plan_p = m.prefill_plan(toks.len(), PrefillPath::Auto)?; + let tok = m.prefill_with(toks, PrefillPath::Auto)?; + let call = PathRun { + logits: vec![m.logits()?], + tokens: vec![tok], + stores: stores(&mut m)?, + }; + let fifth = PathRun { + logits: steps.logits.last().cloned().into_iter().collect(), + tokens: steps.tokens.last().copied().into_iter().collect(), + stores: std::mem::take(&mut steps.stores), + }; + let call_same = same_run( + &format!("placed common {label}: the prompt call ({plan_p}) vs the fifth graph step"), + &call, + &fifth, + ); + // The wide arm: one unit past the gemv arm's cut, then its ids as steps. + let vocab = m.body("gate_qwen35moe_e2e")?.vocab(); + let ids = lcg_ids(PLACED_UBATCH, vocab); + fresh(&mut m)?; + let plan_w = m.prefill_plan(ids.len(), PrefillPath::Gemm)?; + let next = m.prefill_with(&ids, PrefillPath::Gemm)?; + let wide = (next, m.logits()?); + println!( + "placed common {label}: wide unit of {} ids (plan={plan_w}) token {next}", + ids.len() + ); + let stats = m + .body("gate_qwen35moe_e2e")? + .placed() + .ok_or("an open_placed load with no placed side")? + .hybrid() + .stats(); + Ok(CommonRun { + experts, + counts, + scratch, + steps, + taps, + call_same, + wide, + host_slots: (stats.host_slots, stats.batch_host_slots), + }) + } + + /// (o9) (module doc): the file's placed load with no card expert (the host + /// tier's chain, ik's 8-bit rule) first, then its placed load at (o)'s + /// budget, whose card runs every layer's experts by the leg its types name + /// ([`accepts::qwen35_leg`]); each load opened and dropped alone. + fn placed_common(host: HostCfg) -> Result { + tier::sc( + "(o9) the common card leg on the placed load: the host tier's chain, the card's, the \ + wide arm", + )?; + let file = open_model()?; + let common = common_layers(&file)?; + drop(file); + let variant = refset::fixture::selected()?.is_some(); + let premise = !variant || !common.is_empty(); + println!( + "placed common file: {} of {} layers hold stacks the common card leg runs {:?} {}", + common.len(), + n_layer(), + common.iter().map(|(l, t)| (*l, *t)).collect::>(), + verdict(premise) + ); + if !premise { + return Ok(false); + } + let toks = TEXT_TOKENS; + let want = common_chain( + "host-only plan", + |q| budget_for_card_experts(q, 0), + &toks, + host, + )?; + let got = common_chain("plan", |_| Ok(placed_budget()), &toks, host)?; + let mut ok = true; + // The plans: the reference leaves (about) nothing on the card, the run + // splits the experts, and both count the arena the load made. + let split = got.experts.0 > 0 && got.experts.1 > 0; + let lean = want.experts.0 * 8 <= got.experts.0; + let counted = want.scratch.0 == want.scratch.1 && got.scratch.0 == got.scratch.1; + println!( + "placed common plans: reference card_experts={} host_experts={}, run card_experts={} \ + host_experts={}; the reference holds at most an eighth of the run's card experts {} \ + and the run splits {}; the planner's counted scratch is the load's: reference {} B \ + = {} B, run {} B = {} B {}", + want.experts.0, + want.experts.1, + got.experts.0, + got.experts.1, + verdict(lean), + verdict(split), + want.scratch.0, + want.scratch.1, + got.scratch.0, + got.scratch.1, + verdict(counted) + ); + ok &= lean && split && counted; + // The legs ran: every common layer with card experts is one the run + // carded, and a variant has one. + let carded: Vec = common + .iter() + .map(|&(l, _)| l) + .filter(|&l| got.counts.get(l).is_some_and(|&n| n > 0)) + .collect(); + let ran = !variant || !carded.is_empty(); + println!( + "placed common legs: {} of the file's {} common layers hold card experts in the run \ + (layers {carded:?}) {}; host slots on the run's step and batch ports {:?} (both \ + above 0) {}", + carded.len(), + common.len(), + verdict(ran), + got.host_slots, + verdict(got.host_slots.0 > 0 && got.host_slots.1 > 0) + ); + ok &= ran && got.host_slots.0 > 0 && got.host_slots.1 > 0; + // The gemv arm: the prompt call is the fifth step's bits on each load; + // the steps and the layers' outputs against the host tier's. + ok &= want.call_same && got.call_same; + ok &= forced_calls( + "placed common gemv arm, the batch tokens' steps against the host tier's chain", + (&got.steps.tokens, &got.steps.logits), + (&want.steps.tokens, &want.steps.logits), + )?; + // Each layer's output of each step against the host tier's: layer l + // carries l + 1 layers of the host tier's arithmetic, so its bound is + // (l + 1) times one layer's. A near-tie router flip moves one token's + // expert set from its layer on, and a wiring fault moves every token's + // rows, so the rows over their bound may belong to one token alone. + let mut over = std::collections::BTreeSet::new(); + let mut first_clean = true; + for l in 0..want.taps[0].len() { + let per_token: Vec = got + .taps + .iter() + .zip(&want.taps) + .map(|(g, w)| rel_to(&g[l], &w[l])) + .collect(); + let bound = (l + 1) as f64 * FREE_PROBE_HOST; + let exceed: Vec = (0..per_token.len()) + .filter(|&t| per_token[t] > bound || per_token[t].is_nan()) + .collect(); + over.extend(exceed.iter().copied()); + first_clean &= l > 0 || exceed.is_empty(); + println!( + "placed common layer={l} l_out_rel={:.3e} (bound {bound:.1e}, {} layers of the \ + host tier's arithmetic) per step {:?}; steps over the bound {exceed:?}", + per_token.iter().copied().fold(0.0f64, worse), + l + 1, + per_token + .iter() + .map(|e| format!("{e:.2e}")) + .collect::>(), + ); + } + let layers_ok = over.len() <= 1 && first_clean; + println!( + "placed common layers: the steps over their bound are {over:?} (at most one step, and \ + none at layer 0, whose router sees the same input in both loads: a router flip moves \ + one token's experts, a fault every token's) {}", + verdict(layers_ok) + ); + ok &= layers_ok; + // The wide arm: a unit past the gemv arm's cut against the host tier's + // wide unit over the same ids. + ok &= forced_calls( + "placed common wide arm against the host tier's", + (&[got.wide.0], std::slice::from_ref(&got.wide.1)), + (&[want.wide.0], std::slice::from_ref(&want.wide.1)), + )?; + let (rel, _) = e_distance(&got.wide.1, &want.wide.1); + let tight = rel <= COMMON_WIDE_REL; + println!( + "placed common wide arm: logits rel {rel:.3e} against the host tier's wide unit \ + (bound {COMMON_WIDE_REL:.0e}) {}", + verdict(tight) + ); + ok &= tight; + println!("placed common: {}", verdict(ok)); + Ok(ok) + } + const NAME: &str = "gate_qwen35moe_e2e"; // ------------------------------------- (e) residency on the placed load @@ -4792,6 +5104,11 @@ mod gate { ok &= placed_unjoined(&toks, host)?; return close("gate_qwen35moe_e2e --unjoined-placed-only", ok, 0); } + if std::env::args().any(|a| a == "--common-placed-only") { + let mut ok = proof; + ok &= placed_common(host)?; + return close("gate_qwen35moe_e2e --common-placed-only", ok, 0); + } if std::env::args().any(|a| a == "--residency-only") { let mut ok = proof; ok &= residency(host)?; diff --git a/justfile b/justfile index 459cd5f95..e71c7d0c2 100644 --- a/justfile +++ b/justfile @@ -1277,6 +1277,16 @@ gate-gpu-qwen35moe-moe *ARGS: gate-gpu-qwen35moe-e2e: BLOOMERY_MODEL=qwen35moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_qwen35moe_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen35moe_e2e' +# Qwen3.6's placed load on the common card leg's types (`gate_qwen35moe_e2e --common-placed-only`, clause (o9)): the file planned +# with no card expert (the host tier's chain) and at (o)'s budget, the card running every layer's experts by the leg its types name +# (`accepts::qwen35_leg`). Asserted: the planner's counted arena is the load's, the common layers hold card experts, the prompt +# call is the fifth step's bits, each layer's output after five steps stays within the host tier's arithmetic band, and a wide +# unit's logits stand within the band of the host tier's and of the same load's one-token steps. Runs on the fixture variants whose +# routed stacks are a type the dispatch's launches do not run (tools/fixture-variants.tsv); the base fixture runs the dispatch's. +[group('pool')] +gate-gpu-qwen35moe-leg: + BLOOMERY_MODEL=qwen35moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_qwen35moe_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen35moe_e2e --common-placed-only' + # The Qwen3.6 verify gate (one card): a load opened to verify drafted rows (`open_verifying`: 8 stamped lanes a delta layer) # commits every `accepted` in 1..=m of a verify of m = 4 and 8 rows with no re-feed and no copy, and the 16 steps after it # give the plain load's ids and last logits bit for bit (graph and eager, a chain of commits too); a lookup draft over diff --git a/tools/fixture-variants.tsv b/tools/fixture-variants.tsv index 204d53a7f..7ec83fd24 100644 --- a/tools/fixture-variants.tsv +++ b/tools/fixture-variants.tsv @@ -18,3 +18,7 @@ glmffnkq glm5next all ffn_gate=q6_K ffn_up=q6_K ffn_down=q5_K ffn_gate_shexp=q6_ q6kdown glm5next trunk ffn_down_exps=q6_K gate-gpu-glm5next-nvtier the GLM gate file's Q6_K routed downs (gate_glm5next_residency: a layer whose down is Q6_K keeps its experts on the host, as `card_routed` reads no Q6_K): every trunk routed layer stays on the host (the fixture's layer 5 down is Q6_K already, so the map retypes layers 2 to 4), so the NVMe tier's paged clauses have layers to page on a small file unjoined qwen35moe trunk attn_qkv=q8_0 attn_gate=q8_0 ssm_alpha=f32 ssm_beta=f32 ssm_out=q8_0 attn_q=q8_0 attn_k=q8_0 attn_v=q8_0 attn_output=q8_0 ffn_gate_shexp=q8_0 ffn_up_shexp=q8_0 ffn_down_shexp=q8_0 ffn_down_exps=q5_K gate-gpu-qwen35moe-e2e unsloth Qwen3.6-35B-A3B UD-Q4_K_M's layout on its 37 layers with a Q5_K routed down: every mixer site Q8_0 (attn_qkv, attn_gate, ssm_out; attn_q, attn_k, attn_v, attn_output) but β and α, F32, and a Q8_0 shared expert (gate, up and down) apart from the Q4_K routed gate and up, as its header states them; the small proof of clauses (j) and (o8) of gate_qwen35moe_e2e and of the file's node counts (NODES_DECODE_UD, NODES_PASS_UD), the unjoined shared expert on the whole-card and the placed load. A map is per role, so the layers 34, 38 and 39 with a Q6_K down are the row unjoinedq6 unjoinedq6 qwen35moe trunk attn_qkv=q8_0 attn_gate=q8_0 ssm_alpha=f32 ssm_beta=f32 ssm_out=q8_0 attn_q=q8_0 attn_k=q8_0 attn_v=q8_0 attn_output=q8_0 ffn_gate_shexp=q8_0 ffn_up_shexp=q8_0 ffn_down_shexp=q8_0 ffn_down_exps=q6_K gate-gpu-qwen35moe-e2e unsloth Qwen3.6-35B-A3B UD-Q4_K_M's layout on its layers 34, 38 and 39: the row unjoined's map with a Q6_K routed down beside the Q8_0 shared expert; the same clauses on that down type +q35iq4xs qwen35moe trunk attn_qkv=q8_0 attn_gate=q8_0 ssm_alpha=f32 ssm_beta=f32 ssm_out=q8_0 attn_q=q8_0 attn_k=q8_0 attn_v=q8_0 attn_output=q8_0 ffn_gate_exps=iq4_xs ffn_up_exps=iq4_xs ffn_down_exps=iq4_xs gate-gpu-qwen35moe-leg unsloth Qwen3.6-35B-A3B UD-IQ4_XS's routed stacks on UD-Q4_K_M's mixer layout: gate, up and down IQ4_XS on every trunk layer, every mixer site Q8_0 but β and α F32 (the row unjoined's), the shared expert and every other tensor as the fixture has it, so every layer is apart (the gate's launch rules know the joined Q4_K and the unjoined Q8_0 layouts only); the small proof of the common card leg's IQ4_XS gate·up and 128-value down on Body35's placed load (accepts::qwen35_leg is Common for it) and of the route GEMMs' grouped form on the wide arm +q35q80 qwen35moe trunk attn_qkv=q8_0 attn_gate=q8_0 ssm_alpha=f32 ssm_beta=f32 ssm_out=q8_0 attn_q=q8_0 attn_k=q8_0 attn_v=q8_0 attn_output=q8_0 ffn_gate_exps=q8_0 ffn_up_exps=q8_0 ffn_down_exps=q8_0 gate-gpu-qwen35moe-leg unsloth Qwen3.6-35B-A3B Q8_0's routed stacks on UD-Q4_K_M's mixer layout: gate, up and down Q8_0 on every trunk layer, every mixer site Q8_0 but β and α F32 (the row unjoined's), the shared expert and every other tensor as the fixture has it, so every layer is apart; the small proof of the common card leg's Q8_0 gate·up arm and 32-value down on Body35's placed load and of the route GEMMs' 32-value form on the wide arm +q35iq4q8 qwen35moe trunk attn_qkv=q8_0 attn_gate=q8_0 ssm_alpha=f32 ssm_beta=f32 ssm_out=q8_0 attn_q=q8_0 attn_k=q8_0 attn_v=q8_0 attn_output=q8_0 ffn_gate_exps=iq4_xs ffn_up_exps=iq4_xs ffn_down_exps=q8_0 gate-gpu-qwen35moe-leg an IQ4_XS gate and up beside a Q8_0 down on every trunk layer on UD-Q4_K_M's mixer layout (the row q35iq4xs's mixers, the shared expert as the fixture has it): the two activation forms in one layer (the gate·up reads the 128-value rows, the down the 32-value ones), which the buffers and the arena count of a placed load with common layers hold side by side +q35q6k qwen35moe trunk attn_qkv=q8_0 attn_gate=q8_0 ssm_alpha=f32 ssm_beta=f32 ssm_out=q8_0 attn_q=q8_0 attn_k=q8_0 attn_v=q8_0 attn_output=q8_0 ffn_gate_exps=q6_K ffn_up_exps=q6_K ffn_down_exps=q6_K gate-gpu-qwen35moe-leg unsloth Qwen3.6-35B-A3B UD-Q6_K's routed stacks on UD-Q4_K_M's mixer layout: gate, up and down Q6_K on every trunk layer (the file holds a Q8_0 down on three of its layers), every mixer site Q8_0 but β and α F32 (the row unjoined's), the shared expert and every other tensor as the fixture has it, so every layer is apart; the small proof of the common card leg's Q6_K gate·up (q6k_gate_up_sel) and Q6_K down on Body35's placed load and of the grouped route GEMM of a Q6_K gate and up on the wide arm diff --git a/tools/gate-class.tsv b/tools/gate-class.tsv index cc6373c6f..0d063ff06 100644 --- a/tools/gate-class.tsv +++ b/tools/gate-class.tsv @@ -143,6 +143,7 @@ gate-gpu-qwen35-dspark family qwen35moe draft drafts_dspark gate-gpu-qwen35-media family qwen35 media serve_media gate-gpu-qwen35moe-attn common kernel attn_gqa gate-gpu-qwen35moe-e2e family qwen35moe e2e graph_fault+attn_linear_gdn_kda+attn_gqa+moe_route +gate-gpu-qwen35moe-leg family qwen35moe tier tier_legs+placement_sizing gate-gpu-qwen35moe-media family qwen35moe media serve_media gate-gpu-qwen35moe-moe family qwen35moe kernel moe_route gate-gpu-qwen35moe-mtp family qwen35moe draft drafts_mtp From c861b0b11efc7d15d5483edec6b160ae95690b04 Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 22 Oct 2026 02:52:58 +0900 Subject: [PATCH 05/10] accepts: the common leg's prompt rows by their own names (prompt_leg_gate_up, prompt_leg_down, LegSplit); only qwen35_leg names the family Co-Authored-By: Claude Opus 5.5 --- .../gpu-gates/src/bin/gate_qwen35moe_e2e.rs | 6 +-- crates/gpu/src/arch/qwen3moe/body35.rs | 17 ++++--- crates/gpu/src/arch/qwen3moe/placed.rs | 10 ++-- crates/gpu/src/arch/qwen3moe/swap.rs | 14 +++--- crates/gpu/src/arch/qwen3moe/swap35.rs | 8 +-- crates/model/src/arch/coverage.rs | 20 ++++---- crates/placement/src/accepts.rs | 50 ++++++++----------- crates/placement/src/kernels.rs | 2 +- 8 files changed, 62 insertions(+), 65 deletions(-) diff --git a/crates/gpu-gates/src/bin/gate_qwen35moe_e2e.rs b/crates/gpu-gates/src/bin/gate_qwen35moe_e2e.rs index e09a5c164..ae0fedf6a 100644 --- a/crates/gpu-gates/src/bin/gate_qwen35moe_e2e.rs +++ b/crates/gpu-gates/src/bin/gate_qwen35moe_e2e.rs @@ -431,7 +431,7 @@ mod gate { use model::arch::qwen35moe::hparams::Hparams; use model::arch::qwen35moe::names; use model::placement::PlanLevers; - use model::placement::accepts::{self, Leg35}; + use model::placement::accepts::{self, LegSplit}; use refset::arch::qwen35moe::fixture as fx; use refset::arch::qwen35moe::{BATCH, D1K, IK, MODEL, STEP4}; use runtime::{Out, Target, Want}; @@ -3806,7 +3806,7 @@ mod gate { // ------------------------------- (o9) the common card leg on the placed load /// The layers of `file` whose three routed stacks the common card leg runs - /// ([`Leg35::Common`]: a type the dispatch's launches do not), with the + /// ([`LegSplit::Common`]: a type the dispatch's launches do not), with the /// stacks' file types, read from the header. fn common_layers(file: &Split) -> Result, GateError> { let mut out = Vec::new(); @@ -3818,7 +3818,7 @@ mod gate { .ok_or_else(|| format!("no tensor {name}"))?; *ty = t.ty; } - if accepts::qwen35_leg(tys) == Some(Leg35::Common) { + if accepts::qwen35_leg(tys) == Some(LegSplit::Common) { out.push((l, tys)); } } diff --git a/crates/gpu/src/arch/qwen3moe/body35.rs b/crates/gpu/src/arch/qwen3moe/body35.rs index 11adb4c1b..3d7344a3a 100644 --- a/crates/gpu/src/arch/qwen3moe/body35.rs +++ b/crates/gpu/src/arch/qwen3moe/body35.rs @@ -130,7 +130,7 @@ use gguf::quant::GgmlType; use model::arch::Arch; use model::arch::models::shape::MoeShape; use model::arch::models::{Ffn, LayerSpec, Mixer, ModelSpec, MtpDraft}; -use model::placement::accepts::Leg35; +use model::placement::accepts::LegSplit; use model::placement::{Plan, WholeLoad, accepts}; use runtime::seqstate::{Kept, Price}; use std::num::NonZeroUsize; @@ -590,7 +590,7 @@ fn card_leg( file: &Split, l: usize, (ff, h, n): (usize, usize, usize), -) -> Result, GpuError> { +) -> Result, GpuError> { let program = accepts::Program::Qwen35Body; let arms = accepts::leg_arms(program) .ok_or_else(|| GpuError::shape(WHAT, "the qwen35moe body runs no card leg"))?; @@ -622,7 +622,7 @@ fn card_leg( Ok(accepts::qwen35_leg(tys).map(|leg| (leg, tys))) } -/// The forms the layers the common card leg runs ([`Leg35::Common`]) hold +/// The forms the layers the common card leg runs ([`LegSplit::Common`]) hold /// buffers of, from `file`'s header: [`CommonForms::NONE`] unless the read is a /// placed one, which is the only one a card leg runs a layer of by it. fn common_forms( @@ -638,7 +638,9 @@ fn common_forms( for (l, layer) in layers.iter().enumerate() { let Ffn::Moe(m) = &layer.ffn else { continue }; let n = m.experts as usize; - if let Some((Leg35::Common, [gate_up, _, down])) = card_leg(file, l, (d.ff, d.hidden, n))? { + if let Some((LegSplit::Common, [gate_up, _, down])) = + card_leg(file, l, (d.ff, d.hidden, n))? + { tys.push([gate_up, down]); } } @@ -679,7 +681,7 @@ fn part_tys( /// the host tier's: its stacks are read as [`HOSTED`] and no card entry is /// asked of them, its shared expert kept apart at its own types, which the /// card runs whatever the stacks are; the common leg's layers -/// ([`Leg35::Common`]) are among them, their types the placed side's to +/// ([`LegSplit::Common`]) are among them, their types the placed side's to /// read. A dense one: its three matrices a stack of one expert on the /// arena's fixed route. fn plan_ffn( @@ -695,7 +697,10 @@ fn plan_ffn( Ffn::Moe(m) => { let n = m.experts as usize; let hosted = read == Read35::Placed - && !matches!(card_leg(file, l, (ff, h, n))?, Some((Leg35::Dispatch, _))); + && !matches!( + card_leg(file, l, (ff, h, n))?, + Some((LegSplit::Dispatch, _)) + ); let (gate_ty, sh_gate) = part_tys(file, l, "gate", (ff, h, n), hosted)?; let (up_ty, sh_up) = part_tys(file, l, "up", (ff, h, n), hosted)?; let (down_ty, sh_down) = part_tys(file, l, "down", (h, ff, n), hosted)?; diff --git a/crates/gpu/src/arch/qwen3moe/placed.rs b/crates/gpu/src/arch/qwen3moe/placed.rs index 4e75c671d..3e6b82f37 100644 --- a/crates/gpu/src/arch/qwen3moe/placed.rs +++ b/crates/gpu/src/arch/qwen3moe/placed.rs @@ -99,7 +99,7 @@ use gguf::Split; use gguf::quant::GgmlType; use model::moe::UNION_MAX_COLS; use model::placement::Plan; -use model::placement::accepts::{self, Leg35}; +use model::placement::accepts::{self, LegSplit}; use model::placement::kernels::{self, DownAct, DownEntry}; use runtime::sched::{self, At, LayerProgram, Overlap, PortKind}; use std::sync::Arc; @@ -1286,14 +1286,14 @@ fn placed_ffn( let leg = if n_card > 0 { let tys = stack_files(w, [&gate, &up, &down], (n_card, ff, h), l)?; Some(match accepts::qwen35_leg(tys) { - Some(Leg35::Dispatch) => { + Some(LegSplit::Dispatch) => { let [g, u, d] = stack_tys(w, [&gate, &up, &down], (n_card, ff, h), l)?; FfnLeg::Dispatch { gate_up: gate_up_entry(g, u, WHAT)?, down_ty: d, } } - Some(Leg35::Common) => FfnLeg::Common { + Some(LegSplit::Common) => FfnLeg::Common { gate_up: tys[0], down: tys[2], }, @@ -1580,9 +1580,9 @@ fn common_stacks<'a>(w: &'a Weights, f: &'a PlacedFfn) -> Result, arms, ( "a resident routed gate or up stack of a type the qwen35moe card leg runs \ - (`accepts::qwen35_common_gate_up`)", + (`accepts::prompt_leg_gate_up`)", "a resident routed down stack of a type the qwen35moe card leg runs \ - (`accepts::qwen35_common_down`)", + (`accepts::prompt_leg_down`)", ), ) } diff --git a/crates/gpu/src/arch/qwen3moe/swap.rs b/crates/gpu/src/arch/qwen3moe/swap.rs index 4db152a62..829d16664 100644 --- a/crates/gpu/src/arch/qwen3moe/swap.rs +++ b/crates/gpu/src/arch/qwen3moe/swap.rs @@ -92,7 +92,7 @@ pub(super) fn stacks( #[cfg(test)] mod tests { - use super::super::swap35::Leg35; + use super::super::swap35::LegSplit; use super::super::swap38::Leg38; use super::*; use model::placement::{ModelTensor, Role}; @@ -163,7 +163,7 @@ mod tests { [Q6_K, Q6_K, Q4_K], [Q6_K, Q6_K, Q8_0], ] { - assert_eq!(layer::(ok), Ok(()), "{ok:?}"); + assert_eq!(layer::(ok), Ok(()), "{ok:?}"); } for bad in [ [IQ2_XS, IQ2_XS, Q4_K], @@ -172,7 +172,7 @@ mod tests { [Q4_K, Q6_K, Q4_K], [Q6_K, Q8_0, Q6_K], ] { - let text = layer::(bad).expect_err(&format!("{bad:?} is refused")); + let text = layer::(bad).expect_err(&format!("{bad:?} is refused")); assert!( text.contains("layer 0's gate") && text.ends_with( @@ -235,7 +235,7 @@ mod tests { [Q4_K, Q4_K, Q6_K] } }; - match stacks::(&model(4, three), |_| Ok(true)) { + match stacks::(&model(4, three), |_| Ok(true)) { Err(GpuError::Shape { detail, .. }) => { assert!(detail.starts_with("layer 2's gate iq2_xs"), "{detail}"); } @@ -245,7 +245,7 @@ mod tests { short.tensors.pop(); assert!( matches!( - stacks::(&short, |_| Ok(true)), + stacks::(&short, |_| Ok(true)), Err(GpuError::Tensor { .. }) ), "a layer without its down is a missing tensor" @@ -263,10 +263,10 @@ mod tests { for bad in [[Q4_K, Q5_K, Q4_K], [IQ2_XS, IQ2_XS, Q4_K]] { let two = model(2, |_| bad); assert!( - stacks::(&two, |_| Ok(false)).is_ok(), + stacks::(&two, |_| Ok(false)).is_ok(), "{bad:?} on layers with no card expert loads" ); - match stacks::(&two, |l| Ok(l == 1)) { + match stacks::(&two, |l| Ok(l == 1)) { Err(GpuError::Shape { detail, .. }) => { assert!(detail.starts_with("layer 1's gate"), "{detail}"); } diff --git a/crates/gpu/src/arch/qwen3moe/swap35.rs b/crates/gpu/src/arch/qwen3moe/swap35.rs index 0584669fa..e586148e0 100644 --- a/crates/gpu/src/arch/qwen3moe/swap35.rs +++ b/crates/gpu/src/arch/qwen3moe/swap35.rs @@ -20,15 +20,15 @@ use model::placement::accepts::Program; /// Qwen3.6's card leg ([`CardLeg`]): the qwen35moe body's arms /// (`placement::accepts::leg_arms`: the dispatch's types and the common leg's, /// `qwen35_leg`), which its place rule reads too. -pub(super) struct Leg35; +pub(super) struct LegSplit; -impl CardLeg for Leg35 { +impl CardLeg for LegSplit { const WHAT: &'static str = "swap35::stacks"; const PROGRAM: Program = Program::Qwen35Body; } /// The stacks of `model`'s layers, each layer's types read from its tensors, -/// each layer `carded` names checked against [`Leg35`] ([`swap::stacks`]). +/// each layer `carded` names checked against [`LegSplit`] ([`swap::stacks`]). /// Refused by name: a layer without all three stacks, and a carded layer the /// leg refuses (a gate or up of a type no card gate·up launch reads among /// them). Load-time only. @@ -36,5 +36,5 @@ pub(super) fn stacks( model: &ModelTensors, carded: impl Fn(usize) -> Result, ) -> Result { - swap::stacks::(model, carded) + swap::stacks::(model, carded) } diff --git a/crates/model/src/arch/coverage.rs b/crates/model/src/arch/coverage.rs index d0e8d2c94..33e92cacc 100644 --- a/crates/model/src/arch/coverage.rs +++ b/crates/model/src/arch/coverage.rs @@ -20,7 +20,7 @@ use bloomery_placement::accepts::{ BOLT_LEG, embed_of_widened, head_of_widened, qwen3moe_dense, resident_down_runs, }; pub use bloomery_placement::accepts::{Program, program_of, qwen35_down, qwen35_gate_up}; -use bloomery_placement::accepts::{qwen35_common_down, qwen35_common_gate_up}; +use bloomery_placement::accepts::{prompt_leg_down, prompt_leg_gate_up}; pub use bloomery_placement::kernels::{DownEntry, GateUpEntry, SiteEntry}; use gguf::GgmlType; use models::shape::{RouterBody, gqa_row, gqa_row_v, select_router}; @@ -1381,7 +1381,7 @@ struct RowPin { /// ([`head_of_widened`]: `head.rs`) and token embedding ([`embed_of_widened`]: /// `elem.rs`), each the table's row but BF16, and routed stacks (the qwen3moe /// body's [`qwen35_gate_up`], [`qwen35_down`]: the FFN dispatch; `Body35`'s -/// [`qwen35_common_gate_up`], [`qwen35_common_down`]: the dispatch and the +/// [`prompt_leg_gate_up`], [`prompt_leg_down`]: the dispatch and the /// common card leg), and `Body35`'s shared expert ([`qwen35_shared`]), /// and the glm5next and mimo2 bodies' token embedding (the host's rows, /// `GgmlType::has_dequant`), and the gemma-embedding2 encoder's token table @@ -1482,7 +1482,7 @@ const ROW_PINS: &[RowPin] = &[ matrices: true, names: Some(routed_gate_up), what: "routed experts gate and up (the body reads q4_K, q5_K, q6_K, iq3_xxs, iq4_xs and q8_0)", - row: qwen35_common_gate_up, + row: prompt_leg_gate_up, }, RowPin { program: Program::Qwen35Body, @@ -1490,7 +1490,7 @@ const ROW_PINS: &[RowPin] = &[ matrices: true, names: Some(routed_down), what: "routed experts down (the body reads q4_K, q5_K, q6_K, iq3_xxs, iq4_xs, q5_1, q8_0 and iq4_nl)", - row: qwen35_common_down, + row: prompt_leg_down, }, RowPin { program: Program::Raon1Body, @@ -2132,7 +2132,7 @@ mod tests { /// and q4_K, q5_K or q6_K down — the parts the default expert rule's card /// formats pass (q3_K, q6_K, q8_0 gate·up; q3_K, q8_0 down) reach the /// loads' refusal otherwise; `Body35` runs those and the common card leg's - /// rows beside them (`qwen35_common_gate_up`, `qwen35_common_down`). + /// rows beside them (`prompt_leg_gate_up`, `prompt_leg_down`). // PIN(2026-10-10): `Body35`'s routed pins moved from the joined stacks' // q4_K gate·up and q4_K or q6_K down to the table's rows its dispatch // runs (`qwen35_gate_up`, `qwen35_down`): q5_K gate·up and down left its @@ -2140,7 +2140,7 @@ mod tests { // qwen3moe body's moved the same way (q4_K gate·up, q4_K or q6_K down // before): the dispatch is one code path, so one row decides both. // PIN(2026-10-22): `Body35`'s pins moved on to the rows of its placed card - // leg (`qwen35_common_gate_up`, `qwen35_common_down`: the dispatch's types + // leg (`prompt_leg_gate_up`, `prompt_leg_down`: the dispatch's types // and the common leg's): a q6_K, iq3_xxs, iq4_xs or q8_0 gate·up and an // iq3_xxs, iq4_xs, q5_1, q8_0 or iq4_nl down left its refused types, each a type the // common leg launches with a prompt GEMM; red on the tree before the move @@ -2265,13 +2265,13 @@ mod tests { /// The FFN pins are the card kernel table's: over every type id, a /// routed part's matrix is an item exactly when its body's rows give its /// type no entry (the qwen3moe body's [`qwen35_gate_up`], [`qwen35_down`]; - /// `Body35`'s [`qwen35_common_gate_up`], [`qwen35_common_down`]), `Body35`'s + /// `Body35`'s [`prompt_leg_gate_up`], [`prompt_leg_down`]), `Body35`'s /// shared expert's exactly when [`qwen35_shared`] does not, and each pin's /// text names the types its row takes (a pin reads no F32 matrix). #[test] fn the_ffn_pins_are_the_card_kernel_table() { use super::{ - qwen35_common_down, qwen35_common_gate_up, qwen35_down, qwen35_gate_up, qwen35_shared, + prompt_leg_down, prompt_leg_gate_up, qwen35_down, qwen35_gate_up, qwen35_shared, }; type Row = fn(GgmlType) -> bool; let parts: [(&str, Role, Program, Row); 5] = [ @@ -2291,13 +2291,13 @@ mod tests { "ffn_gate_exps.weight", Role::RoutedExperts, Program::Qwen35Body, - qwen35_common_gate_up, + prompt_leg_gate_up, ), ( "ffn_down_exps.weight", Role::RoutedExperts, Program::Qwen35Body, - qwen35_common_down, + prompt_leg_down, ), ( "ffn_up_shexp.weight", diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index a09c53a55..5f6340791 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -171,7 +171,7 @@ pub const fn qwen35_down(ty: GgmlType) -> Option { /// GEMM, so IQ2_XS, IQ3_S and MXFP4, which have none, stay on the host by /// name; NVFP4 is not admitted (no qwen35 file or fixture proves it). #[must_use] -pub const fn qwen35_common_gate_up(ty: GgmlType) -> bool { +pub const fn prompt_leg_gate_up(ty: GgmlType) -> bool { (kernels::card_gate_up_runs(ty) || matches!( kernels::gate_up(ty), @@ -187,19 +187,19 @@ pub const fn qwen35_common_gate_up(ty: GgmlType) -> bool { /// (Q5_1, Q8_0, IQ4_NL) read the `Q8Blocks32` form the leg quantizes for /// them. #[must_use] -pub const fn qwen35_common_down(ty: GgmlType) -> bool { +pub const fn prompt_leg_down(ty: GgmlType) -> bool { kernels::rank_down_runs(ty) && kernels::gemm(ty).is_some() } /// How `Body35`'s placed card leg runs a layer's routed stacks. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Leg35 { +pub enum LegSplit { /// The family's own dispatch launches ([`qwen35_gate_up`], [`qwen35_down`]): /// a Q4_K or Q5_K gate and up with a Q4_K, Q5_K or Q6_K down, the arms /// whose launches and graph are the load's reference. Dispatch, - /// The common leg's launches ([`qwen35_common_gate_up`], - /// [`qwen35_common_down`]): any other layer its arms run. + /// The common leg's launches ([`prompt_leg_gate_up`], + /// [`prompt_leg_down`]): any other layer its arms run. Common, } @@ -208,15 +208,15 @@ pub enum Leg35 { /// tier's). The one classifier the plan's rule, the load's check, the /// residency stack table and the placed side read. #[must_use] -pub fn qwen35_leg([gate, up, down]: [GgmlType; 3]) -> Option { - if gate != up || !qwen35_common_gate_up(gate) || !qwen35_common_down(down) { +pub fn qwen35_leg([gate, up, down]: [GgmlType; 3]) -> Option { + if gate != up || !prompt_leg_gate_up(gate) || !prompt_leg_down(down) { return None; } Some( if qwen35_gate_up(gate).is_some() && qwen35_down(down).is_some() { - Leg35::Dispatch + LegSplit::Dispatch } else { - Leg35::Common + LegSplit::Common }, ) } @@ -356,8 +356,8 @@ pub fn leg_arms(p: Program) -> Option { down: |ty| qwen35_down(ty).is_some(), }), Program::Qwen35Body => Some(LegArms { - gate_up: qwen35_common_gate_up, - down: qwen35_common_down, + gate_up: prompt_leg_gate_up, + down: prompt_leg_down, }), Program::Qwen38Body => Some(LegArms { gate_up: |ty| qwen38_gate_up(ty).is_some(), @@ -410,8 +410,8 @@ pub fn routed_rule(p: Program) -> RoutedRule { down: |ty| qwen35_down(ty).map(|_| leg_card_format(ty)), }, Program::Qwen35Body => RoutedRule::Leg { - gate_up: |ty| qwen35_common_gate_up(ty).then(|| leg_card_format(ty)), - down: |ty| qwen35_common_down(ty).then(|| leg_card_format(ty)), + gate_up: |ty| prompt_leg_gate_up(ty).then(|| leg_card_format(ty)), + down: |ty| prompt_leg_down(ty).then(|| leg_card_format(ty)), }, Program::Qwen38Body => RoutedRule::Leg { gate_up: |ty| qwen38_gate_up(ty).map(|_| leg_card_format(ty)), @@ -559,11 +559,11 @@ mod tests { } /// `Body35`'s placed card leg: a Q4_K or Q5_K gate and up with a Q4_K, Q5_K - /// or Q6_K down are the dispatch's own launches ([`Leg35::Dispatch`], the + /// or Q6_K down are the dispatch's own launches ([`LegSplit::Dispatch`], the /// layers whose graph the load's reference holds); a Q4_K, Q5_K, Q6_K, IQ3_XXS, /// IQ4_XS or Q8_0 gate and up of one type with a Q4_K, Q5_K, Q6_K, /// IQ3_XXS, IQ4_XS, Q5_1, Q8_0 or IQ4_NL down are the rest of the common - /// leg's ([`Leg35::Common`]); every other triple, a gate and up of two + /// leg's ([`LegSplit::Common`]); every other triple, a gate and up of two /// types among them, is the host tier's. The sets are spelled out, so a /// type entering or leaving an arm goes red here. Mutant: the dispatch /// row widened to the common leg's gate·up arms — the IQ4_XS layer reads @@ -576,20 +576,16 @@ mod tests { let dispatch_gate_up = [Q4_K, Q5_K]; let dispatch_down = [Q4_K, Q5_K, Q6_K]; for g in types() { - assert_eq!( - qwen35_common_gate_up(g), - gate_up.contains(&g), - "{g} gate·up" - ); - assert_eq!(qwen35_common_down(g), down.contains(&g), "{g} down"); + assert_eq!(prompt_leg_gate_up(g), gate_up.contains(&g), "{g} gate·up"); + assert_eq!(prompt_leg_down(g), down.contains(&g), "{g} down"); for u in types() { for d in types() { let want = if g == u && gate_up.contains(&g) && down.contains(&d) { Some( if dispatch_gate_up.contains(&g) && dispatch_down.contains(&d) { - Leg35::Dispatch + LegSplit::Dispatch } else { - Leg35::Common + LegSplit::Common }, ) } else { @@ -610,12 +606,8 @@ mod tests { let arms = leg_arms(Program::Qwen35Body).expect("a card leg"); let old = leg_arms(Program::Qwen3moeBody).expect("a card leg"); for ty in types() { - assert_eq!( - (arms.gate_up)(ty), - qwen35_common_gate_up(ty), - "{ty} gate·up" - ); - assert_eq!((arms.down)(ty), qwen35_common_down(ty), "{ty} down"); + assert_eq!((arms.gate_up)(ty), prompt_leg_gate_up(ty), "{ty} gate·up"); + assert_eq!((arms.down)(ty), prompt_leg_down(ty), "{ty} down"); assert_eq!( (old.gate_up)(ty), qwen35_gate_up(ty).is_some(), diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 2c3aafade..14c3e6432 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -110,7 +110,7 @@ pub const fn q8_act_bytes(cols: u64, k: u64) -> u64 { /// it (GLM-5.3, MiMo, Solar-Open2, MiniMax-M3, Raon) hold `Q8Act` down buffers /// only, so a Q8_0 layer they planned on the card would fail at its first /// launch where it runs on the host today; a program whose buffers follow the -/// down's form reads the arm through its own row ([`crate::accepts::qwen35_common_gate_up`]). +/// down's form reads the arm through its own row ([`crate::accepts::prompt_leg_gate_up`]). /// A stack in its planes is [`card_planes_runs`]'. #[must_use] pub const fn card_gate_up_runs(ty: GgmlType) -> bool { From a8ca6516b971b4ae088dcdf2f3b6b79bd42e2dce Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 22 Oct 2026 02:52:58 +0900 Subject: [PATCH 06/10] ratchets: q6k_gate_up_sel's unsafe blocks; accepts.rs's qwen row holds Qwen35Body's own program rows, coverage.rs's comes down Co-Authored-By: Claude Opus 5.5 --- tools/model-tax.tsv | 4 ++-- tools/unsafe-ratchet.txt | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/model-tax.tsv b/tools/model-tax.tsv index 22dd3eb99..14fe0dd0d 100644 --- a/tools/model-tax.tsv +++ b/tools/model-tax.tsv @@ -140,7 +140,7 @@ crates/model/src/arch/coverage.rs gemma 18 baseline P 18: MK2 program by body crates/model/src/arch/coverage.rs glm 21 baseline P 21: MK2 program by body crates/model/src/arch/coverage.rs mimo 10 baseline P 10: MK2 program by body crates/model/src/arch/coverage.rs nemotron 36 baseline P 36: MK2 program by body -crates/model/src/arch/coverage.rs qwen 63 baseline P 63: MK2 program by body +crates/model/src/arch/coverage.rs qwen 61 baseline P 63: MK2 program by body crates/model/src/arch/draft.rs mimo 3 baseline D 4: MK1a draft usability as an ArchRow row crates/model/src/arch/draft.rs qwen 13 baseline D 15: MK1a draft usability as an ArchRow row crates/model/src/arch/dspark.rs deepseek 1 baseline O 1: own path: an alias row in tools/model-tokens.tsv or a move into the family directory @@ -166,7 +166,7 @@ crates/placement/src/accepts.rs gemma 5 PIN(2026-10-21) gemma: MK2 program by bo crates/placement/src/accepts.rs glm 5 PIN(2026-10-21) glm: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet crates/placement/src/accepts.rs mimo 5 PIN(2026-10-21) mimo: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet crates/placement/src/accepts.rs nemotron 17 PIN(2026-10-21) nemotron: MK2 program by body; +2 is the family's word in the common Program::name owner, which the model's own code does not carry yet -crates/placement/src/accepts.rs qwen 26 PIN(2026-10-21) qwen: MK2 program by body; +3 is the family's word in the common Program::name owner, which the model's own code does not carry yet +crates/placement/src/accepts.rs qwen 30 PIN(2026-10-21) qwen: MK2 program by body; +3 is the family's word in the common Program::name owner, which the model's own code does not carry yet. PIN(2026-10-22): +4 is Qwen35Body's own program rows, split from Qwen3moeBody's because its placed card leg also runs the common leg (qwen35_leg, the one classifier both rules read) crates/placement/src/placement/nodes_scale.rs deepseek 10 baseline D 10: shapes read from the fixture ModelSpec rows (deferred) crates/placement/src/placement/nodes_scale.rs glm 2 baseline D 2: shapes read from the fixture ModelSpec rows (deferred) crates/placement/src/placement/nodes_scale.rs qwen 32 baseline D 32: shapes read from the fixture ModelSpec rows (deferred) diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index 5aabb6afe..1e29b4bef 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -413,7 +413,9 @@ gguf 7 2 0 # PIN(2026-10-21): gpu 2101 -> 2112 blocks, 243 -> 247 fns: both head-256 # appends carry their distinct cache-writer contract through the KV and selecting-attention wrappers. # Eleven audited caller blocks discharge that obligation; no device code or launch changes. -gpu 2112 247 10 +# PIN(2026-10-22): gpu 2112 -> 2115 blocks, a new kernel entry, not old code relaxed: q6k_gate_up_sel (q6k_sel.rs), the +# Q6_K gate·up the common card leg launches for Body35's placed layers. Each block has its SAFETY line. +gpu 2115 247 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line. From 0146392cabb055511a81b852abaa95c20b9b55ce Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 22 Oct 2026 05:23:25 +0900 Subject: [PATCH 07/10] card leg: LegStacks::of_arms is the one constructor over a program's arms (with_arms folded in) Co-Authored-By: Claude Opus 5.5 --- crates/gpu/src/card_leg.rs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/crates/gpu/src/card_leg.rs b/crates/gpu/src/card_leg.rs index 8d57b50f1..cdcc151d7 100644 --- a/crates/gpu/src/card_leg.rs +++ b/crates/gpu/src/card_leg.rs @@ -540,7 +540,7 @@ impl<'a> LegStacks<'a> { pub fn of(w: &'a Weights, names: [&'a str; 3]) -> Result, GpuError> { const DOWN: &str = "a resident routed down stack of a type the card leg runs (`kernels::card_down_runs`)"; - LegStacks::with_arms( + LegStacks::of_arms( w, names, LegArms { @@ -556,7 +556,7 @@ impl<'a> LegStacks<'a> { pub fn of_rank(w: &'a Weights, names: [&'a str; 3]) -> Result, GpuError> { const DOWN: &str = "a resident routed down stack of a type a rank's card leg runs \ (`kernels::rank_down_runs`)"; - LegStacks::with_arms( + LegStacks::of_arms( w, names, LegArms { @@ -574,7 +574,7 @@ impl<'a> LegStacks<'a> { pub fn of_resident(w: &'a Weights, names: [&'a str; 3]) -> Result, GpuError> { const DOWN: &str = "a resident routed down stack of a type a whole-file body's card leg \ runs (`kernels::card_down_runs` or Q5_0)"; - LegStacks::with_arms( + LegStacks::of_arms( w, names, LegArms { @@ -596,17 +596,6 @@ impl<'a> LegStacks<'a> { names: [&'a str; 3], arms: LegArms, roles: (&'static str, &'static str), - ) -> Result, GpuError> { - LegStacks::with_arms(w, names, arms, roles) - } - - /// The stacks with the gate and up admitted by `arms.gate_up` and the - /// down by `arms.down`, refused as `roles.0` and `roles.1`. - fn with_arms( - w: &'a Weights, - names: [&'a str; 3], - arms: LegArms, - roles: (&'static str, &'static str), ) -> Result, GpuError> { let [gate, up, down] = names; let (gate, up, down) = ( From 237641622bd025d662a68fafa95aa4661b9eeec5 Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 22 Oct 2026 05:25:33 +0900 Subject: [PATCH 08/10] cards: q75-decode times Qwen3.6's Q8_0 and UD-Q6_K decode with the placed card leg (RTX 3090, ctx 262144) Co-Authored-By: Claude Opus 5.5 --- docs/cards/q75-decode.card | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 docs/cards/q75-decode.card diff --git a/docs/cards/q75-decode.card b/docs/cards/q75-decode.card new file mode 100644 index 000000000..1998ddb08 --- /dev/null +++ b/docs/cards/q75-decode.card @@ -0,0 +1,25 @@ +# Qwen3.6-35B-A3B's placed card leg runs Q8_0 and Q6_K experts (branch q75, PR #629): decode at the serving +# defaults, --place unset, on one RTX 3090. +# Derivation [derived], from the plan measured on the 3090 (q75-design.md section 5): at ctx 262144 the card holds +# 4505 of 10240 experts of the Q8_0 file (f 0.440) and 5844 of the UD-Q6_K file (f 0.571), where main holds 0. +# Per layer: front 0.045 ms + the host leg (0.209 ms Q8_0, 0.161 ms UD-Q6_K, at 128 GB/s) x (1 - f) + back +# 0.06 ms; the card's shadow (0.05 ms) sits under the host leg. Q8_0 0.314 -> 0.222 ms (x1.4); UD-Q6_K 0.266 -> +# 0.174 ms (x1.5). Uniform routing assumed; skewed routing raises the card's share. +# Timeline per token: host DRAM carries the wall (the host leg is the max term); card SMs and PCIe sit under it. +kind: ab +question: does moving 44-57 % of the experts of a Q8_0 or UD-Q6_K file onto the card read the derived decode gain + on one RTX 3090 at the serving defaults? +term: tg tok/s @ D = 6 (prose corpus ids), Qwen3.6-35B-A3B Q8_0, bloomery-serve at the serving defaults, ctx + 262144, --place unset, RTX 3090; the UD-Q6_K file in the same lease as the second table +unit: % +predict: +25..+60 +rounds: 4 +sd: 0.6 % AGENTS.md Know the ruler, same-binary scatter; the ratio at 4 rounds reads +-1.0 % +condition: one lease with the box quiet (no gate on either card); depth-serve.sh, one fresh server an arm and a + round, the base arm main 1e8e923e9's bloomery-serve and the changed arm q75's, D 6 at C 262144 with N 128; + rows tagged [cpu-busy], [other-busy] or [cold] are rerun +decide-in: land #629; the README's Qwen3.6 rows name the Q8_0 and UD-Q6_K decode rates +decide-below: under +25 %: the host leg is not the max term, or the card's shadow is above it; read one nsys of a + decode step on the Q8_0 file before landing +decide-above: beyond +60 %: routing is skewed toward the experts the plan placed; land, and read the residency + hit rate from the step stats From a656c439994c5c379e86f742b477e6819d4d0523 Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 22 Oct 2026 07:06:23 +0900 Subject: [PATCH 09/10] qwen3moe, card_leg, placement: review fixes on the placed leg body::stack_tys reads its stacks through placed::stack_files, qwen35_leg reads leg_runs(Program::Qwen35Body, ...), and RouteBlock drops its rule field: every caller ran SiluMul. Co-Authored-By: Claude Opus 5.5 --- crates/gpu/src/arch/qwen3moe/body.rs | 49 ++++++++------------------ crates/gpu/src/arch/qwen3moe/placed.rs | 15 ++++---- crates/gpu/src/card_leg.rs | 12 ++----- crates/placement/src/accepts.rs | 2 +- 4 files changed, 25 insertions(+), 53 deletions(-) diff --git a/crates/gpu/src/arch/qwen3moe/body.rs b/crates/gpu/src/arch/qwen3moe/body.rs index 9be85ed0b..61dd0b708 100644 --- a/crates/gpu/src/arch/qwen3moe/body.rs +++ b/crates/gpu/src/arch/qwen3moe/body.rs @@ -275,48 +275,27 @@ fn pins(hp: &Hparams) -> Result { /// Layer `l`'s routed stacks `[gate, up, down]` of `n` experts (or a stack of /// one expert: a placed layer's shared expert) of `ff` rows of `h` values: -/// each a resident K-quant word plane of its shape, its type the file's, then -/// the types the card kernel table's entries admit for the family's FFN -/// dispatch ([`gate_up_entry`]: one launch reads the gate and up of one -/// type; [`down_entry`]), a type with no entry refused by name with the -/// layer. Returns the gate, up and down types. +/// each a resident K-quant word plane of its shape, its type the file's (the +/// layout checked by [`super::placed::stack_files`]), then the types the card +/// kernel table's entries admit for the family's FFN dispatch +/// ([`gate_up_entry`]: one launch reads the gate and up of one type; +/// [`down_entry`]), a type with no entry refused by name with the layer. +/// Returns the gate, up and down types. pub(super) fn stack_tys( w: &Weights, - [gate, up, down]: [&str; 3], + names: [&str; 3], (n, ff, h): (usize, usize, usize), l: usize, ) -> Result<[SiteTy; 3], GpuError> { let what = "qwen3moe::Body::load"; - let stack = |name: &str, rows: usize, k: usize| -> Result { - let Some(DevWeight::KQuant { ty, w: t, k: wk }) = w.get(name) else { - return Err(GpuError::tensor( - what, - name, - "a resident K-quant word plane", - )); - }; - let Some(site) = SiteTy::of_ggml(*ty) else { - return Err(GpuError::shape( - what, - format!("{name} is {ty}; no launch family reads it"), - )); - }; - if t.rows() != rows || *wk != k { - return Err(GpuError::shape( - what, - format!( - "{name} is {ty} {} rows x {wk}; the chain takes {rows} rows x {k}", - t.rows() - ), - )); - } - Ok(site) + let raw = super::placed::stack_files(w, names, (n, ff, h), l, what)?; + let site_ty = |i: usize| { + let (name, ty) = (names[i], raw[i]); + SiteTy::of_ggml(ty).ok_or_else(|| { + GpuError::shape(what, format!("{name} is {ty}; no launch family reads it")) + }) }; - let tys = [ - stack(gate, n * ff, h)?, - stack(up, n * ff, h)?, - stack(down, n * h, ff)?, - ]; + let tys = [site_ty(0)?, site_ty(1)?, site_ty(2)?]; stack_entries(tys, l)?; Ok(tys) } diff --git a/crates/gpu/src/arch/qwen3moe/placed.rs b/crates/gpu/src/arch/qwen3moe/placed.rs index 3e6b82f37..a3f909edc 100644 --- a/crates/gpu/src/arch/qwen3moe/placed.rs +++ b/crates/gpu/src/arch/qwen3moe/placed.rs @@ -1231,24 +1231,26 @@ pub(super) fn apart_plans(plans: Vec) -> Vec { /// `h` values, each a resident word plane of its shape ([`DevWeight::KQuant`]: /// the file's blocks), as their file types: the types the placed card leg's /// classifier reads ([`accepts::qwen35_leg`]). A stack that is not one, or of -/// another shape, is refused by name with the layer. -fn stack_files( +/// another shape, is refused by name with the layer, `what` naming the entry +/// point that read the stacks. +pub(super) fn stack_files( w: &Weights, names: [&str; 3], (n, ff, h): (usize, usize, usize), l: usize, + what: &'static str, ) -> Result<[GgmlType; 3], GpuError> { let stack = |name: &str, rows: usize, k: usize| -> Result { let Some(DevWeight::KQuant { ty, w: t, k: wk }) = w.get(name) else { return Err(GpuError::tensor( - WHAT, + what, name, - "a resident K-quant word plane (the file's blocks)", + "a resident K-quant word plane", )); }; if t.rows() != rows || *wk != k { return Err(GpuError::shape( - WHAT, + what, format!( "layer {l}: {name} is {ty} {} rows x {wk}; the chain takes {rows} rows x {k}", t.rows() @@ -1284,7 +1286,7 @@ fn placed_ffn( ); let (h, ff) = (s.hidden, s.ff); let leg = if n_card > 0 { - let tys = stack_files(w, [&gate, &up, &down], (n_card, ff, h), l)?; + let tys = stack_files(w, [&gate, &up, &down], (n_card, ff, h), l, WHAT)?; Some(match accepts::qwen35_leg(tys) { Some(LegSplit::Dispatch) => { let [g, u, d] = stack_tys(w, [&gate, &up, &down], (n_card, ff, h), l)?; @@ -1688,7 +1690,6 @@ fn common_wide( down, ff: d.ff, hidden: d.hidden, - rule: Act::SiluMul, }; blk.enqueue( gpu, diff --git a/crates/gpu/src/card_leg.rs b/crates/gpu/src/card_leg.rs index cdcc151d7..149524fbc 100644 --- a/crates/gpu/src/card_leg.rs +++ b/crates/gpu/src/card_leg.rs @@ -293,7 +293,6 @@ pub(crate) struct RouteBlock<'a> { pub(crate) ff: usize, /// Rows an expert of the down. pub(crate) hidden: usize, - pub(crate) rule: Act, } /// The rows a [`RouteBlock`] reads and writes: the unit's normed rows in the @@ -393,7 +392,7 @@ impl RouteBlock<'_> { match self.down { RouteW::Grouped(ty, w) => { let act = h128.ok_or_else(|| lacks("q8_1 blocks of 128 (the down)"))?; - gemm.enqueue_swiglu_act_quant(stream, g, u, self.rule, n_slots, act, fault)?; + gemm.enqueue_swiglu_quant(stream, g, u, n_slots, act, fault)?; gemm.enqueue_gemm( stream, GemmArgs { @@ -409,14 +408,7 @@ impl RouteBlock<'_> { } RouteW::Blocks32(w) => { let act = h32.ok_or_else(|| lacks("q8 blocks of 32 (the down)"))?; - match self.rule { - Act::SiluMul => { - g32.enqueue_swiglu_quant32(stream, g, u, n_slots, act, fault)? - } - rule => { - g32.enqueue_swiglu_act_quant32(stream, g, u, rule, n_slots, act, fault)?; - } - } + g32.enqueue_swiglu_quant32(stream, g, u, n_slots, act, fault)?; g32.enqueue_gemm32( stream, Gemm32Args { diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index 5f6340791..fcb4df469 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -209,7 +209,7 @@ pub enum LegSplit { /// residency stack table and the placed side read. #[must_use] pub fn qwen35_leg([gate, up, down]: [GgmlType; 3]) -> Option { - if gate != up || !prompt_leg_gate_up(gate) || !prompt_leg_down(down) { + if leg_runs(Program::Qwen35Body, [gate, up, down]) != Some(true) { return None; } Some( From a1782c1f10480566a11c2844b53537e59fe85881 Mon Sep 17 00:00:00 2001 From: midagedev Date: Thu, 22 Oct 2026 08:59:43 +0900 Subject: [PATCH 10/10] model-tax: accepts.rs's qwen row takes qwen35_leg's leg_runs line Co-Authored-By: Claude Opus 5.5 --- tools/model-tax.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/model-tax.tsv b/tools/model-tax.tsv index 14fe0dd0d..05e54b21d 100644 --- a/tools/model-tax.tsv +++ b/tools/model-tax.tsv @@ -166,7 +166,7 @@ crates/placement/src/accepts.rs gemma 5 PIN(2026-10-21) gemma: MK2 program by bo crates/placement/src/accepts.rs glm 5 PIN(2026-10-21) glm: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet crates/placement/src/accepts.rs mimo 5 PIN(2026-10-21) mimo: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet crates/placement/src/accepts.rs nemotron 17 PIN(2026-10-21) nemotron: MK2 program by body; +2 is the family's word in the common Program::name owner, which the model's own code does not carry yet -crates/placement/src/accepts.rs qwen 30 PIN(2026-10-21) qwen: MK2 program by body; +3 is the family's word in the common Program::name owner, which the model's own code does not carry yet. PIN(2026-10-22): +4 is Qwen35Body's own program rows, split from Qwen3moeBody's because its placed card leg also runs the common leg (qwen35_leg, the one classifier both rules read) +crates/placement/src/accepts.rs qwen 31 PIN(2026-10-21) qwen: MK2 program by body; +3 is the family's word in the common Program::name owner, which the model's own code does not carry yet. PIN(2026-10-22): +4 is Qwen35Body's own program rows, split from Qwen3moeBody's because its placed card leg also runs the common leg (qwen35_leg, the one classifier both rules read); +1 is qwen35_leg reading leg_runs(Program::Qwen35Body, …) so its rule and the plan read one table crates/placement/src/placement/nodes_scale.rs deepseek 10 baseline D 10: shapes read from the fixture ModelSpec rows (deferred) crates/placement/src/placement/nodes_scale.rs glm 2 baseline D 2: shapes read from the fixture ModelSpec rows (deferred) crates/placement/src/placement/nodes_scale.rs qwen 32 baseline D 32: shapes read from the fixture ModelSpec rows (deferred)