diff --git a/crates/chain/src/site.rs b/crates/chain/src/site.rs index da5ccefcf..d44ac713e 100644 --- a/crates/chain/src/site.rs +++ b/crates/chain/src/site.rs @@ -38,6 +38,8 @@ pub enum SiteTy { /// entry is none; a backend whose launch reads it names it in the list /// [`file_site`] takes. MlxA4, + /// IQ4_XS in its original blocks, over q8_1 columns, narrow and wide. + Iq4Xs, } impl SiteTy { @@ -56,6 +58,7 @@ impl SiteTy { SiteTy::Mxfp8, SiteTy::Fp8B128, SiteTy::Bf16, + SiteTy::Iq4Xs, ]; /// The family of a file tensor of type `ty`, the one that launches the @@ -99,6 +102,7 @@ impl SiteTy { SiteTy::Mxfp8 => Some(SiteEntry::Mxfp8), SiteTy::Fp8B128 => Some(SiteEntry::Fp8B128), SiteTy::Bf16 => Some(SiteEntry::Bf16), + SiteTy::Iq4Xs => Some(SiteEntry::Iq4xs), SiteTy::MlxA4 => None, } } diff --git a/crates/gguf/src/card.rs b/crates/gguf/src/card.rs index fec369482..d6ed30de3 100644 --- a/crates/gguf/src/card.rs +++ b/crates/gguf/src/card.rs @@ -290,6 +290,9 @@ pub enum SiteEntry { /// `Q8F32Kernels::enqueue_bf16_gemv` over BF16 words and f32 rows; /// wide, `Q8F32Kernels::enqueue_bf16_tile` over the same words and rows. Bf16, + /// `QuantSelKernels::enqueue_gemv_sel` over one matrix in the file's + /// IQ4_XS blocks and `Q8Act` columns; wide, the grouped IQ4_XS GEMM. + Iq4xs, } /// The activation a dense site's launch reads; `crates/gpu/src/site.rs` @@ -316,7 +319,11 @@ impl SiteEntry { #[must_use] pub const fn reads(self) -> SiteAct { match self { - SiteEntry::Q3k | SiteEntry::Q4k | SiteEntry::Q5k | SiteEntry::Q6k => SiteAct::Q8x128, + SiteEntry::Q3k + | SiteEntry::Q4k + | SiteEntry::Q5k + | SiteEntry::Q6k + | SiteEntry::Iq4xs => SiteAct::Q8x128, SiteEntry::Q8_0 => SiteAct::Q8x32, SiteEntry::F32 => SiteAct::F32, SiteEntry::Q5_0 => SiteAct::Q8Blocks32, @@ -335,7 +342,7 @@ impl SiteEntry { } /// One past the last [`SiteEntry`]: the span [`DENSE_TYPE`] covers. -const DENSE_SPAN: usize = SiteEntry::Bf16 as usize + 1; +const DENSE_SPAN: usize = SiteEntry::Iq4xs as usize + 1; /// Dense entry to the file type of the row that names it. The build refuses /// an entry two rows name, an entry no row names, and an entry past diff --git a/crates/gguf/src/qtype.rs b/crates/gguf/src/qtype.rs index 2d0bacf62..5051e1f5a 100644 --- a/crates/gguf/src/qtype.rs +++ b/crates/gguf/src/qtype.rs @@ -117,7 +117,7 @@ pub const QROWS: &[QRow] = &[ QRow { ty: T::IQ2_S, name: "iq2_s", blck: 256, bytes: 82, dequant: true, host_act: H::Q8K, card: card!(KQuant KQuant Iq2s (Iq2s, Q8Act) Iq2s - - -) }, QRow { ty: T::IQ4_XS, name: "iq4_xs", blck: 256, bytes: 136, dequant: true, host_act: H::Q8K, - card: card!(- - Iq4xs (Iq4xs, Q8Act) Iq4xs - - -) }, + card: card!(KQuant - Iq4xs (Iq4xs, Q8Act) Iq4xs Iq4xs - -) }, // The C scalars: ggml.c:621-649. QRow { ty: T::I8, name: "i8", blck: 1, bytes: 1, dequant: false, host_act: H::None, card: Card::NONE }, QRow { ty: T::I16, name: "i16", blck: 1, bytes: 2, dequant: false, host_act: H::None, card: Card::NONE }, diff --git a/crates/gpu-deepseek41/src/block.rs b/crates/gpu-deepseek41/src/block.rs index 807482311..02923e18a 100644 --- a/crates/gpu-deepseek41/src/block.rs +++ b/crates/gpu-deepseek41/src/block.rs @@ -41,12 +41,13 @@ const BLOCK: &str = "gated block"; /// The types a block's matrices read, one launch family each ([`site::file_site`]'s `allowed`); a model's /// coverage pin for the block lists the same set. -pub const BLOCK_TYPES: [SiteTy; 5] = [ +pub const BLOCK_TYPES: [SiteTy; 6] = [ SiteTy::Q8_0, SiteTy::Q4K, SiteTy::Q5K, SiteTy::Q6K, SiteTy::Mxfp8, + SiteTy::Iq4Xs, ]; /// The most columns [`enqueue_block_cols`] takes: the q8_0 block's multi-column launches' cap, kept for every plan. @@ -57,6 +58,11 @@ fn refuse(what: &'static str, detail: String) -> GpuError { GpuError::Shape { what, detail } } +/// A site whose narrow and wide launches both read the 128-value q8_1 form. +fn q128(ty: SiteTy) -> bool { + ty.reads() == site::Form::Q8x128 && site::gemm_weight(ty).is_some() +} + /// A block's gate·up half, picked once at load from the file's types. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum GateUp { @@ -109,28 +115,28 @@ impl BlockPlan { self == BlockPlan::Q8 } - /// Whether a row's run leaves its SwiGLU row in `h` as f32 (the q8_0 pair, a fused K-quant gate·up, an - /// f32-form pair, a split gate·up over a down that is not a K-quant); a split gate·up over a K-quant down - /// writes it only into the down's q8_1 blocks, from the f32 gate and up rows - /// ([`BlockScratch::gate_up_rows`]). + /// Whether a row's run leaves its SwiGLU row in `h` as f32. A split gate/up + /// feeding a down over 128-value q8_1 blocks writes only those blocks; + /// other plans leave the f32 row ([`BlockScratch::gate_up_rows`]). #[must_use] pub fn writes_h(self) -> bool { - !matches!(self.gate_up, GateUp::Split { .. }) || !self.down.kquant() + !matches!(self.gate_up, GateUp::Split { .. }) || !q128(self.down) } - /// The plan of a block whose gate, up and down are `tys`: the q8_0 pair, two K-quants (the same Q4_K or Q5_K - /// fused, else split) or two types that read the f32 row for gate·up, and any of [`BLOCK_TYPES`] for the - /// down. Another mix is refused by name as `what`'s. + /// The plan of a block whose gate, up and down are `tys`: the q8_0 pair, + /// two sites over 128-value q8_1 blocks (the same Q4_K or Q5_K fused, + /// otherwise split), or two types that read the f32 row for gate/up. + /// The down is any of [`BLOCK_TYPES`]; another gate/up mix is refused by name. /// /// # Errors - /// A gate and up that are not two q8_0, two K-quants or two f32-form types. + /// A gate and up whose activation forms disagree. pub fn of_types( what: &'static str, [gate, up, down]: [SiteTy; 3], ) -> Result { let gate_up = match (gate, up) { (SiteTy::Q8_0, SiteTy::Q8_0) => GateUp::Ds41, - (g, u) if g.kquant() && u.kquant() => match (g == u, g) { + (g, u) if q128(g) && q128(u) => match (g == u, g) { (true, SiteTy::Q4K | SiteTy::Q5K) => GateUp::Fused(g), _ => GateUp::Split { gate: g, up: u }, }, @@ -141,8 +147,8 @@ impl BlockPlan { return Err(refuse( what, format!( - "gate is {gate} and up is {up}: a block takes two q8_0, two K-quants or two \ - types that read the f32 row" + "gate is {gate} and up is {up}: a block takes two q8_0, two sites over \ + 128-value q8_1 or two types that read the f32 row" ), )); } @@ -177,7 +183,7 @@ impl BlockPlan { /// already wrote it). #[must_use] pub fn extra_launches(self) -> usize { - let down_quantize = usize::from(self.down.kquant()); + let down_quantize = usize::from(q128(self.down)); match self.gate_up { GateUp::Ds41 => down_quantize, GateUp::Fused(_) => 1 + down_quantize, @@ -291,7 +297,7 @@ impl BlockScratch { let (mut any, mut split_ff) = (false, 0); for (plan, ff) in blocks.into_iter().filter(|(p, _)| !p.is_q8()) { any = true; - if plan.down.kquant() && !ha.iter().any(|(f, _)| *f == ff) { + if q128(plan.down) && !ha.iter().any(|(f, _)| *f == ff) { ha.push((ff, Q8Act::with_k(stream, 1, ff)?)); } if matches!(plan.gate_up, GateUp::Split { .. } | GateUp::Site { .. }) { @@ -450,7 +456,7 @@ pub fn enqueue_block_act( gpu.layer_sink(blk.layer)?, h, )?; - if !blk.plan.down.kquant() { + if !q128(blk.plan.down) { return site::resident_gemv(gpu, w, down, h, y); } } @@ -485,8 +491,8 @@ pub fn enqueue_block_act( missing: "the split block's f32 gate and up", }); }; - // A K-quant down reads the SwiGLU as q8_1 blocks; any other down reads the f32 `h`. - let act = if blk.plan.down.kquant() { + // A 128-value q8_1 down reads the quantizer's blocks; another down reads f32 `h`. + let act = if q128(blk.plan.down) { Some(h_blocks(ha, blk.ff)?) } else { None @@ -517,7 +523,7 @@ pub fn enqueue_block_act( .enqueue_swiglu_rule(stream, rule, (&*g, &*u), blk.ff, (fault, &mut *h))?; } } - if !blk.plan.down.kquant() { + if !q128(blk.plan.down) { return site::resident_gemv(gpu, w, down, h, y); } let (kk, sc) = kit(k, s)?; @@ -667,16 +673,6 @@ mod tests { BlockPlan::of_types(WHAT, tys) } - fn refused(r: Result, words: &[&str]) { - let msg = match r { - Err(GpuError::Shape { what, detail }) => format!("{what}: {detail}"), - other => panic!("expected a named shape refusal, got {other:?}"), - }; - for w in words { - assert!(msg.contains(w), "{msg:?} lacks {w:?}"); - } - } - /// The plan is the types' dispatch: the q8_0 triple is `BlockPlan::Q8`, one Q4_K or Q5_K for gate and up /// is fused, any other K-quant pair splits whatever its down (a K-quant down reads the SwiGLU as q8_1 /// blocks, a q8_0 or MXFP8 down as the f32 `h`), an MXFP8 pair runs the f32-form arm, and every plan reads @@ -686,7 +682,7 @@ mod tests { /// it in the last layer's shared expert (gate and up Q6_K, down Q8_0), and its launches take it. #[test] fn the_plan_is_the_types_dispatch() { - use SiteTy::{Mxfp8, Q3K, Q4K, Q5K, Q6K, Q8_0}; + use SiteTy::{Iq4Xs, Mxfp8, Q3K, Q4K, Q5K, Q6K, Q8_0}; assert_eq!(plan([Q8_0, Q8_0, Q8_0]).unwrap(), BlockPlan::Q8); assert_eq!(plan([Q8_0, Q8_0, Q6K]).unwrap().gate_up, GateUp::Ds41); assert_eq!(plan([Q4K, Q4K, Q6K]).unwrap().gate_up, GateUp::Fused(Q4K)); @@ -697,6 +693,10 @@ mod tests { assert_eq!(plan([Q3K, Q3K, Q6K]).unwrap().gate_up, split(Q3K, Q3K)); assert_eq!(plan([Q6K, Q6K, Q8_0]).unwrap().gate_up, split(Q6K, Q6K)); assert_eq!(plan([Q4K, Q5K, Mxfp8]).unwrap().gate_up, split(Q4K, Q5K)); + assert_eq!( + plan([Iq4Xs, Iq4Xs, Iq4Xs]).unwrap().gate_up, + split(Iq4Xs, Iq4Xs) + ); assert_eq!( plan([Mxfp8, Mxfp8, Mxfp8]).unwrap().gate_up, GateUp::Site { @@ -718,11 +718,10 @@ mod tests { } } - /// A row's SwiGLU is f32 in `h` unless a split gate·up feeds a K-quant down, which takes the q8_1 blocks the - /// SwiGLU's quantizer writes. Mutant: `writes_h` false for every split plan. + /// The observable SwiGLU scratch follows the down's actual activation form. #[test] - fn a_row_leaves_f32_h_unless_a_split_pair_feeds_a_kquant_down() { - use SiteTy::{Mxfp8, Q4K, Q5K, Q6K, Q8_0}; + fn a_row_leaves_f32_h_unless_a_split_pair_feeds_q8x128_down() { + use SiteTy::{Iq4Xs, Mxfp8, Q4K, Q5K, Q6K, Q8_0}; let writes = |tys| plan(tys).unwrap().writes_h(); assert!(writes([Q8_0, Q8_0, Q6K])); assert!(writes([Q4K, Q4K, Q8_0])); @@ -730,19 +729,29 @@ mod tests { assert!(!writes([Q6K, Q6K, Q5K])); assert!(writes([Q6K, Q6K, Q8_0])); assert!(writes([Q4K, Q5K, Mxfp8])); + assert!(writes([Iq4Xs, Iq4Xs, Q8_0])); + assert!(!writes([Iq4Xs, Iq4Xs, Iq4Xs])); } - /// A mix no arm covers is refused by name, with the types: gate and up of two kinds. #[test] - fn a_mix_no_arm_covers_is_refused() { + fn incompatible_activation_forms_refuse_before_launch() { use SiteTy::{Mxfp8, Q4K, Q6K, Q8_0}; - refused( - plan([Q8_0, Q4K, Q8_0]), - &["test", "gate is Q8_0", "up is Q4_K"], - ); - refused(plan([Q6K, Q8_0, Q6K]), &["gate is Q6_K", "up is Q8_0"]); - refused(plan([Mxfp8, Q8_0, Mxfp8]), &["gate is MXFP8", "up is Q8_0"]); - refused(plan([Q4K, Mxfp8, Q4K]), &["gate is Q4_K", "up is MXFP8"]); + for types in [ + [Q8_0, Q4K, Q8_0], + [Q6K, Q8_0, Q6K], + [Mxfp8, Q8_0, Mxfp8], + [Q4K, Mxfp8, Q4K], + ] { + let Err(GpuError::Shape { what, detail }) = plan(types) else { + panic!( + "{types:?}: incompatible activation forms were not refused as a shape error" + ); + }; + assert_eq!(what, WHAT); + for ty in types { + assert!(detail.contains(&ty.to_string()), "{types:?}: {detail}"); + } + } } /// The launches beyond the q8_0 block's two, by plan: 0 for the q8_0 block, and the other arms' derived diff --git a/crates/gpu-deepseek41/src/shared_routed.rs b/crates/gpu-deepseek41/src/shared_routed.rs index e71e92788..29bfb4742 100644 --- a/crates/gpu-deepseek41/src/shared_routed.rs +++ b/crates/gpu-deepseek41/src/shared_routed.rs @@ -1184,3 +1184,205 @@ fn no_batch() -> GpuError { missing: "the batch's buffers", } } + +#[cfg(test)] +mod tests { + use super::SharedWide; + use crate::block::{ + Block, BlockIo, BlockKernels, BlockPlan, BlockScratch, KBlockKernels, + enqueue_block_cols_act, + }; + use crate::experts::ExpertKernels; + use bloomery_gpu::Gpu; + use bloomery_gpu::gemm::Gemm32Kernels; + use bloomery_gpu::kquant::Act; + use bloomery_gpu::weights::Weights; + use cuda_core::DeviceBuffer; + use gguf::Split; + use gguf::quant::GgmlType; + use gguf::write::{Layout, TensorDecl, Writer}; + + const N: usize = 256; + const FF: usize = 512; + const NAMES: [&str; 3] = [ + "blk.42.ffn_gate_shexp.weight", + "blk.42.ffn_up_shexp.weight", + "blk.42.ffn_down_shexp.weight", + ]; + + fn iq4_shared_file() -> (std::path::PathBuf, Split) { + let path = + std::env::temp_dir().join(format!("bloomery-iq4-shared-{}.gguf", std::process::id())); + let specs = [(N, FF, 0x1c00u16), (N, FF, 0x1800), (FF, N, 0x1800)]; + let payloads: Vec> = specs + .iter() + .enumerate() + .map(|(i, &(k, rows, scale))| { + let mut bytes = Vec::with_capacity(rows * (k / 256) * 136); + for row in 0..rows { + let d = if i == 2 && row % 2 == 1 { + scale - 0x400 + } else { + scale + }; + let mut block = [0x88u8; 136]; + block[..2].copy_from_slice(&d.to_le_bytes()); + block[2..4].copy_from_slice(&0xaaaau16.to_le_bytes()); + block[4..8].fill(0x11); + for _ in 0..k / 256 { + bytes.extend_from_slice(&block); + } + } + bytes + }) + .collect(); + let decls = NAMES + .iter() + .zip(specs) + .zip(&payloads) + .map(|((&name, (k, rows, _)), bytes)| TensorDecl { + name: name.to_owned(), + dims: vec![u64::try_from(k).unwrap(), u64::try_from(rows).unwrap()], + type_id: GgmlType::IQ4_XS.as_u32(), + nbytes: u64::try_from(bytes.len()).unwrap(), + }) + .collect(); + let layout = Layout::new(&[], decls).unwrap(); + let file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .unwrap(); + let mut writer = Writer::new(file, layout).unwrap(); + for (name, bytes) in NAMES.iter().zip(payloads) { + writer.tensor(name, &bytes).unwrap(); + } + writer.finish().unwrap(); + let split = Split::open(&path).unwrap(); + (path, split) + } + + fn check_shared_rows(input: &[f32], rows: &[f32]) { + // Uniform positive rows quantize to code127 without integer rounding. + // The gate is at least128, so SiLU rounds to the gate itself. The + // scalar positive-dot upper bound has four input-width operations, + // two output-width operations and the scale/product roundings. + let operations = 4.0 * 256.0 + 2.0 * 512.0 + 16.0; + let eps = f64::from(f32::EPSILON); + let gamma = operations * eps / (1.0 - operations * eps); + for (c, row) in rows.as_chunks::().0.iter().enumerate() { + let x = f64::from(input[c * N]); + for (r, &got) in row.iter().enumerate() { + let expected = x * x * if r % 2 == 0 { 0.5 } else { 0.25 }; + assert!(got.is_finite()); + assert!( + (f64::from(got) - expected).abs() <= gamma * expected, + "column {c} row {r}: {got} vs {expected}, relative bound {gamma}" + ); + } + } + } + + #[test] + #[ignore = "needs a CUDA card"] + fn hw_iq4_shared_loads_and_runs_narrow_columns_and_wide_replay() { + let (path, split) = iq4_shared_file(); + let gpu = Gpu::new().expect("card"); + let stream = gpu.stream(); + let weights = Weights::load_where(stream, &split, |name| NAMES.contains(&name)).unwrap(); + assert_eq!( + model::placement::card_format_of_role(GgmlType::IQ4_XS, models::Role::SharedExpert), + Some(model::placement::CardFormat::KQuant) + ); + for name in NAMES { + let Some(bloomery_gpu::weights::DevWeight::KQuant { ty, w, .. }) = weights.get(name) + else { + panic!("{name} must retain its IQ4_XS words"); + }; + assert_eq!(*ty, GgmlType::IQ4_XS); + let words = w.buf().to_host_vec(stream).unwrap(); + let stored: Vec = words.iter().flat_map(|word| word.to_le_bytes()).collect(); + assert_eq!(stored, split.data(name).unwrap(), "{name}"); + } + let plan = BlockPlan::of_file(&split, "IQ4_XS shared", NAMES, (FF, N)).unwrap(); + plan.check_resident(&weights, "IQ4_XS shared", NAMES, (FF, N)) + .unwrap(); + let block = Block { + plan, + names: NAMES, + ff: FF, + limit: 0.0, + layer: 42, + }; + let experts = ExpertKernels::load(gpu.context()).unwrap(); + let quant = KBlockKernels::load(&gpu).unwrap(); + let kernels = BlockKernels { + experts: &experts, + kquant: Some(&quant), + }; + let mut scratch = BlockScratch::new(stream, N, [(plan, FF)]).unwrap(); + let mut h = DeviceBuffer::zeroed(stream, FF).unwrap(); + let g32 = Gemm32Kernels::load(gpu.context()).unwrap(); + for cols in [1, 8, super::GATED_FROM] { + let input: Vec = (0..cols) + .flat_map(|c| std::iter::repeat_n([128.0, 256.0, 384.0][c % 3], N)) + .collect(); + let x = DeviceBuffer::from_host(stream, &input).unwrap(); + let mut y = DeviceBuffer::zeroed(stream, cols * N).unwrap(); + if cols < super::GATED_FROM { + enqueue_block_cols_act( + &gpu, + &weights, + &kernels, + &mut scratch, + (&block, Act::SiluMul), + cols, + BlockIo { + x: &x, + h: &mut h, + y: &mut y, + }, + ) + .unwrap(); + } else { + let types = plan.types(); + let mut wide = + SharedWide::new(&gpu, (&types[..2], &types[2..]), (cols, N, FF)).unwrap(); + assert!(wide.x.q128.is_some() && wide.h.q128.is_some()); + assert!(wide.x.q32.is_none() && wide.h.q32.is_none()); + assert!(wide.x.f32.is_none() && wide.h.f32.is_none()); + wide.run( + (&gpu, &weights, &g32), + (&block, Act::SiluMul), + (cols, N), + (&x, &mut y), + ) + .unwrap(); + let eager = y.to_host_vec(stream).unwrap(); + check_shared_rows(&input, &eager); + let graph = gpu + .capture(|_| { + wide.run( + (&gpu, &weights, &g32), + (&block, Act::SiluMul), + (cols, N), + (&x, &mut y), + ) + }) + .unwrap(); + graph.launch(stream).unwrap(); + let replay = y.to_host_vec(stream).unwrap(); + assert!( + eager + .iter() + .zip(&replay) + .all(|(a, b)| a.to_bits() == b.to_bits()) + ); + } + check_shared_rows(&input, &y.to_host_vec(stream).unwrap()); + assert_eq!(gpu.take_fault().unwrap(), None); + } + stream.synchronize().unwrap(); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/crates/gpu-gates/src/act_rule.rs b/crates/gpu-gates/src/act_rule.rs index b9c0d9c2b..969255e06 100644 --- a/crates/gpu-gates/src/act_rule.rs +++ b/crates/gpu-gates/src/act_rule.rs @@ -237,8 +237,11 @@ impl SiteRel { #[must_use] pub fn site_rel(ty: GgmlType, arm: Arm) -> Option { let ours = match (ty, arm) { - (GgmlType::Q3_K | GgmlType::Q4_K | GgmlType::Q5_K | GgmlType::Q6_K, _) - | (GgmlType::IQ3_XXS | GgmlType::IQ4_XS, Arm::Expert) => Some(Q8_1_BLOCK), + ( + GgmlType::Q3_K | GgmlType::Q4_K | GgmlType::Q5_K | GgmlType::Q6_K | GgmlType::IQ4_XS, + _, + ) + | (GgmlType::IQ3_XXS, Arm::Expert) => Some(Q8_1_BLOCK), // Bolt's IQ2_XXS: the routed `iq2_xxs_gemv_sel` and the dense `iq2_xxs_rows` read q8_1. (GgmlType::IQ2_XXS, Arm::Expert | Arm::Gemv) => Some(Q8_1_BLOCK), // Bolt's dense NVFP4 (`nvfp4_gemv`, the ssm_in/ssm_out) reads the engine's q8_1. @@ -993,6 +996,11 @@ mod site_tests { let (q32, q128, q256) = (Some(32), Some(Q8_1_BLOCK), Some(QK_K)); for arm in [Arm::Gemv, Arm::Gemm, Arm::Expert] { assert_eq!(site_rel(GgmlType::Q3_K, arm), q(q256, q128), "Q3_K {arm:?}"); + assert_eq!( + site_rel(GgmlType::IQ4_XS, arm), + q(q256, q128), + "IQ4_XS {arm:?}" + ); for ty in [GgmlType::Q4_K, GgmlType::Q5_K, GgmlType::Q6_K] { assert_eq!(site_rel(ty, arm), q(q32, q128), "{ty} {arm:?}"); } @@ -1026,7 +1034,6 @@ mod site_tests { for ty in [ GgmlType::IQ3_S, GgmlType::IQ3_XXS, - GgmlType::IQ4_XS, GgmlType::Q5_1, GgmlType::IQ4_NL, ] { @@ -1091,8 +1098,11 @@ mod site_tests { read("gemm/mod.rs"), read("gemm/grouped.rs"), ); + // The contract is the arm's text, not rustfmt's wrapping of its or-pattern. + let site = site.split_whitespace().collect::>().join(" "); for line in [ - "SiteEntry::Q3k | SiteEntry::Q4k | SiteEntry::Q5k | SiteEntry::Q6k => SiteAct::Q8x128", + // PIN(2026-10-27): IQ4_XS reads the 128-value q8_1 form with the K-quants (`SiteEntry::Iq4xs`), as `site_rel` says. + "SiteEntry::Q3k | SiteEntry::Q4k | SiteEntry::Q5k | SiteEntry::Q6k | SiteEntry::Iq4xs => SiteAct::Q8x128", "SiteEntry::Q8_0 => SiteAct::Q8x32", "SiteEntry::F32 => SiteAct::F32", "SiteEntry::Q5_0 => SiteAct::Q8Blocks32", diff --git a/crates/gpu/src/arch/qwen3moe/plan38.rs b/crates/gpu/src/arch/qwen3moe/plan38.rs index 3340b2497..46714324b 100644 --- a/crates/gpu/src/arch/qwen3moe/plan38.rs +++ b/crates/gpu/src/arch/qwen3moe/plan38.rs @@ -484,7 +484,7 @@ fn gqa_fits(g: &Gqa) -> bool { /// Every layer's plan from its kind, `ple` the PLE site's layer, each weight /// checked against the geometry: type and shape. A dense projection, the /// indexer's two among them, is of a type the common site projects by name -/// (`site::PROJECTED`: the f32 rows' families and the K-quants), each +/// (`site::PROJECTED`: the f32 rows' families, the K-quants and IQ4_XS), each /// tensor's own: the walks pick its launch by the resident form. The joined /// β·α stack is of an f32 rows' family (`site::BY_NAME`). An HC matrix is one /// `HcMatrix::of` reads. The gains, the conv and the injects are F32 (the @@ -648,7 +648,7 @@ const SUPER_BLOCK: usize = 256; /// The resident weight `name`, `rows` rows of `k` values, of a type a /// by-name projection launches (`site::PROJECTED`: every dense family but -/// Q5_0, the K-quants and NVFP4 over the q8_1 rows of their input); another +/// Q5_0, the K-quants, IQ4_XS and NVFP4 over the q8_1 rows of their input); another /// type or shape is refused by name. fn projected_shape(w: &Weights, name: &str, rows: usize, k: usize) -> Result<(), GpuError> { let ty = dense::resident_site(w, WHAT, name, (rows, k), &dense::PROJECTED)?; diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index 33b7877db..691ee01eb 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -23,7 +23,7 @@ //! - [`kgemv`], a site at most eight columns over the q8_1 rows: Q3_K, Q4_K //! and Q6_K their gemvs, row-major (`m` outputs a row; a Q6_K row of an odd //! super-block count the down `_sel` over one expert, one column); Q5_K -//! the K-quant down `_sel` over a stack of one expert, token-major (a slot +//! and IQ4_XS the down `_sel` over a stack of one expert, token-major (a slot //! a column); NVFP4 `nvfp4_gemv` over its planes times the tensor's scale, //! token-major. [`kgemv_order`] is this family's layout and is //! `None` for Q5_0. Where a body copies a row-major output token-major, @@ -109,14 +109,14 @@ pub enum Order { /// file type its form holds ([`SiteTy::of_ggml`]; an F32 plane is the f32 /// site whatever type it was widened from). `None` for a form no dense entry /// reads: a stack in the file's blocks of a type whose entry reads another -/// layout (Q8_0's planes, an i-quant), a derived weight and an NVFP4 stack. +/// layout (Q8_0's planes), a derived weight and an NVFP4 stack. /// The resident form is this crate's, so the map is here, beside the /// launches that read it, and not on the chain's [`SiteTy`]. #[must_use] pub fn of_resident(w: &DevWeight) -> Option { let ty = match w { DevWeight::KQuant { ty, .. } => { - return SiteTy::of_ggml(*ty).filter(|s| s.kquant()); + return SiteTy::of_ggml(*ty).filter(|s| s.kquant() || *s == SiteTy::Iq4Xs); } DevWeight::Q5_0 { .. } => GgmlType::Q5_0, DevWeight::Q5_1 { .. } => GgmlType::Q5_1, @@ -131,7 +131,7 @@ pub fn of_resident(w: &DevWeight) -> Option { SiteTy::of_ggml(ty) } -/// How [`kgemv`] lays a site of type `ty` out: Q3_K, Q4_K and Q6_K row-major, Q5_K and NVFP4 token-major. +/// How [`kgemv`] lays a site out: Q3_K, Q4_K and Q6_K row-major, Q5_K, IQ4_XS and NVFP4 token-major. /// `None` for a type [`kgemv`] does not run: Q8_0, F32, MXFP8, FP8_B128 and BF16 ([`gemv`]), Q5_0 /// ([`blocks32_gemv`], whose row-major layout `y[r · m + c]` is that function's own), and MLX_A4, /// a backend's family no card launch reads. @@ -139,7 +139,7 @@ pub fn of_resident(w: &DevWeight) -> Option { pub fn kgemv_order(ty: SiteTy) -> Option { match ty { SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q6K => Some(Order::RowMajor), - SiteTy::Q5K | SiteTy::Nvfp4 => Some(Order::TokenMajor), + SiteTy::Q5K | SiteTy::Iq4Xs | SiteTy::Nvfp4 => Some(Order::TokenMajor), SiteTy::Q5_0 | SiteTy::Q8_0 | SiteTy::F32 @@ -235,13 +235,14 @@ pub(crate) const BY_NAME: [SiteTy; 5] = [ /// The types a dense site launches by name through [`ByName`] over up to /// [`BY_NAME_COLS`] columns: [`BY_NAME`] and the families that read the q8_1 -/// blocks of 128 (the K-quants, NVFP4), every dense family but Q5_0. A unit +/// blocks of 128 (the K-quants, IQ4_XS, NVFP4), every dense family but Q5_0. A unit /// test holds the list to `placement::accepts::dense_projected`. -pub(crate) const PROJECTED: [SiteTy; 10] = [ +pub(crate) const PROJECTED: [SiteTy; 11] = [ SiteTy::Q3K, SiteTy::Q4K, SiteTy::Q5K, SiteTy::Q6K, + SiteTy::Iq4Xs, SiteTy::Q8_0, SiteTy::F32, SiteTy::Nvfp4, @@ -637,7 +638,7 @@ impl KGemvKernels { /// `y = W · act` for site `name` of type `ty` over the `act.m()` columns of /// the q8_1 rows `act`, laid out as [`kgemv_order`] says: Q4_K, Q6_K -/// and Q3_K their gemvs, row-major; Q5_K the down `_sel` over the site as +/// and Q3_K their gemvs, row-major; Q5_K and IQ4_XS the down `_sel` over the site as /// one expert, token-major, an id past it raising on `fault`; NVFP4 its gemv /// times the tensor's scale, token-major. Q8_0, F32, MXFP8, FP8_B128 and BF16 are [`gemv`]; /// Q5_0 is [`blocks32_gemv`]. Each is refused here by name. @@ -705,6 +706,22 @@ pub fn kgemv_on( y, ) } + SiteTy::Iq4Xs => { + let wt = kq(w, WHAT, name)?; + kk.kq.enqueue_gemv_sel( + stream, + GgmlType::IQ4_XS, + &SelDown { + w: wt, + act, + sel: &kk.one, + n_slots: act.m(), + rows_per_expert: wt.rows(), + }, + fault, + y, + ) + } SiteTy::Nvfp4 => kk.nv.enqueue_gemv(stream, w.nvfp4(WHAT, name)?, act, y), SiteTy::Q8_0 | SiteTy::F32 | SiteTy::Mxfp8 | SiteTy::Fp8B128 | SiteTy::Bf16 => { Err(GpuError::shape( @@ -756,8 +773,8 @@ pub fn blocks32_gemv( pub const BY_NAME_COLS: usize = KGEMV_COLS; /// The launches a [`ByName::gemv`] of a whole weight of type `ty` over `m` -/// columns makes beyond the one an f32-row site makes: a K-quant or NVFP4 -/// site quantizes its columns into q8_1 rows first, and a type whose gemv is +/// columns makes beyond the one an f32-row site makes: a K-quant, IQ4_XS or +/// NVFP4 site quantizes its columns into q8_1 rows first, and a type whose gemv is /// row-major ([`kgemv_order`]) copies its result token-major past one column. /// A body counts its captured walk's launches by this and the weights' /// resident types. @@ -890,8 +907,8 @@ impl ByName { /// `out = W · x` for the resident weight `name`, by the site type of its /// resident form, over the first `m` (1..=[`BY_NAME_COLS`]) columns of /// the f32 rows `x`; a fault of the quantization raises into layer - /// `layer`'s sink. `k` is read only by a K-quant or NVFP4 site, `g32` by - /// an F32 site past one column ([`resident_gemv_cols`]). A Q5_0 site, a + /// `layer`'s sink. `k` is read only by a K-quant, IQ4_XS or NVFP4 site, `g32` + /// by an F32 site past one column ([`resident_gemv_cols`]). A Q5_0 site, a /// form the load made no q8_1 rows for, or a part past the output's /// stride is refused by name. #[allow( @@ -1054,10 +1071,10 @@ pub fn scatter( } /// Where a site reads its weight: the resident tensor of a name -/// (`Weights`), rows of it, or the K-quant words a caller holds — a window -/// of experts laid out as the resident stack of the site's type, kept +/// (`Weights`), rows of it, or the K-quant or IQ4_XS words a caller holds — a +/// window of experts laid out as the resident stack of the site's type, kept /// outside `Weights`. Each carries the site's name, which a refusal names. -/// Words under any other type than a K-quant's, and rows of a site whose +/// Words under a type with no grouped GEMM, and rows of a site whose /// launch reads neither the f32 rows nor q8 blocks of 32, are refused by /// name ([`gemm`], [`gated`], [`resident_gemv_rows`]). #[derive(Clone, Copy)] @@ -1085,14 +1102,16 @@ impl<'a> SiteWeight<'a> { } } - /// Words under a type that is not a K-quant's, and rows of a type whose + /// Words under a type with no grouped GEMM, and rows of a type whose /// launch reads neither the f32 rows nor q8 blocks of 32, refused by /// name as `what`. fn check(self, what: &'static str, ty: SiteTy) -> Result<(), GpuError> { match self { - SiteWeight::Words { name, .. } if !ty.kquant() => Err(GpuError::shape( + SiteWeight::Words { name, .. } if gemm_weight(ty).is_none() => Err(GpuError::shape( what, - format!("{name} is {ty} on words a caller holds; only a K-quant's words launch"), + format!( + "{name} is {ty} on words a caller holds; its grouped GEMM reads no such words" + ), )), SiteWeight::Part { name, .. } if !BY_NAME.contains(&ty) => Err(GpuError::shape( what, @@ -1408,21 +1427,24 @@ pub fn gemm( src.check(WHAT, ty)?; match ty { SiteTy::Nvfp4 => Err(no_wide_gemm(WHAT, name, ty)), - SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q5K | SiteTy::Q6K => k.gemm.enqueue_gemm( - stream, - GemmArgs { - ty: gemm_weight(ty).ok_or(GpuError::state(WHAT, "a K-quant's GEMM type"))?, - w: match src { - SiteWeight::Words { w, .. } => w, - _ => kq(w, WHAT, name)?, + SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q5K | SiteTy::Q6K | SiteTy::Iq4Xs => { + k.gemm.enqueue_gemm( + stream, + GemmArgs { + ty: gemm_weight(ty) + .ok_or(GpuError::state(WHAT, "the selected site's GEMM type"))?, + w: match src { + SiteWeight::Words { w, .. } => w, + _ => kq(w, WHAT, name)?, + }, + rows_per_expert: rows, + act: x.q128.ok_or_else(|| missing("q8_1 blocks of 128"))?, + route, + input, + y, }, - rows_per_expert: rows, - act: x.q128.ok_or_else(|| missing("q8_1 blocks of 128"))?, - route, - input, - y, - }, - ), + ) + } SiteTy::Q8_0 | SiteTy::F32 | SiteTy::Mxfp8 | SiteTy::Fp8B128 | SiteTy::Bf16 => { dense()?; let (site, part) = src.rows(WHAT)?; @@ -1781,8 +1803,8 @@ impl<'a> Gated<'a> { (ty, self.w, src) } - /// What the block refuses whatever the rows: words under a type that is - /// not a K-quant's, and a rule the down's SwiGLU launch lacks + /// What the block refuses whatever the rows: words under a type with no + /// grouped GEMM, and a rule the down's SwiGLU launch lacks /// ([`swiglu_launch`]) — checked before the first launch, so a block /// refused here has launched nothing. fn check(&self) -> Result<(), GpuError> { @@ -2009,15 +2031,13 @@ mod tests { use crate::kquant::Act; use gguf::Split; use gguf::quant::GgmlType; - use gguf::write::{Layout, TensorDecl, Writer}; + use gguf::write::{Layout, TensorDecl}; use model::placement::kernels; - use std::fs::File; - use std::io::BufWriter; /// A by-name projection's extra launches follow the type: none for a site /// that reads the f32 rows or the q8 blocks of 32, the quantization for a - /// K-quant or NVFP4 site, and the token-major copy past one column for - /// the row-major types alone (Q3_K, Q4_K, Q6_K). + /// K-quant, IQ4_XS or NVFP4 site, and the token-major copy past one + /// column for the row-major types alone (Q3_K, Q4_K, Q6_K). #[test] fn a_by_name_projection_counts_its_quantization_and_its_row_major_copy() { for (ty, one, rows) in [ @@ -2030,6 +2050,7 @@ mod tests { (SiteTy::Q4K, 1, 2), (SiteTy::Q6K, 1, 2), (SiteTy::Q5K, 1, 1), + (SiteTy::Iq4Xs, 1, 1), (SiteTy::Nvfp4, 1, 1), ] { assert_eq!(by_name_extra(ty, 1), one, "{ty} at one column"); @@ -2037,83 +2058,170 @@ mod tests { } } - /// A site's type is read from the file's header alone: a Q8_0 matrix of - /// two rows of 64 is read as Q8_0 where Q8_0 is launched, and refused by - /// name where only K-quants are; a Q3_K, a Q5_K, a Q5_0 and an MXFP8 one - /// (bloomery's id, sized by the split) are read as theirs; an IQ4_XS one (a type no site launch reads) and a matrix of - /// another shape are refused by name. + /// Every family is the kernel table's dense row of one file type: a type + /// has a family exactly where the table names a dense entry for it (F16, + /// which loads widened to f32, is the f32 site of its widened rows), the + /// family maps back to that type and is named by it in upper case, its + /// grouped GEMM is the one the table names for the type, it is a + /// `kgemv` exactly where it reads the q8_1 rows of 128 values, it + /// launches by name (`BY_NAME`) exactly where it reads the f32 rows or + /// the q8 blocks of 32, and by projection (`PROJECTED`) exactly where the + /// table names a dense entry but Q5_0's. #[test] - fn a_site_type_comes_from_the_header_and_a_type_with_no_launch_is_refused() { - let dir = std::env::temp_dir().join(format!("bloomery-site-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap_or_else(|e| panic!("{e}")); - let path = dir.join("m.gguf"); - let decls = [ - ("q8", vec![64, 2], 8u32, 2 * 2 * 34), - ("iq", vec![256, 1], 23, 136), - ("q3", vec![256, 2], 11, 2 * 110), - ("q5", vec![256, 2], 13, 2 * 176), - ("q50", vec![64, 2], 6, 2 * 2 * 22), - ("mx", vec![128, 2], GgmlType::MXFP8.as_u32(), 2 * 4 * 33), - ("bf", vec![64, 2], GgmlType::BF16.as_u32(), 2 * 64 * 2), + fn every_family_is_the_tables_dense_row_of_one_type() { + // PIN(2026-10-21): consolidate the three site pins, preserving their assertions. + let own = [ + GgmlType::F8E4M3_ROWS, + GgmlType::MXFP8, + GgmlType::NVFP4_ROWS, + GgmlType::FP8_B128, ]; - let tensors = decls - .iter() - .map(|(name, dims, type_id, nbytes)| TensorDecl { - name: (*name).to_string(), - dims: dims.clone(), - type_id: *type_id, - nbytes: *nbytes, - }) - .collect(); - let layout = Layout::new(&[], tensors).unwrap_or_else(|e| panic!("{e}")); - let file = BufWriter::new(File::create(&path).unwrap_or_else(|e| panic!("{e}"))); - let mut w = Writer::new(file, layout).unwrap_or_else(|e| panic!("{e}")); - for (name, _, _, nbytes) in &decls { - w.tensor(name, &vec![0u8; *nbytes as usize]) - .unwrap_or_else(|e| panic!("{e}")); + for ty in gguf::QROWS.iter().map(|r| r.ty).chain(own) { + let family = SiteTy::of_ggml(ty); + if ty == GgmlType::F16 { + assert_eq!( + family, + Some(SiteTy::F32), + "{ty}: the f32 site of its widened rows" + ); + assert_eq!(kernels::dense(ty), None, "{ty}"); + continue; + } + assert_eq!(family.is_some(), kernels::dense(ty).is_some(), "{ty}"); + if let Some(s) = family { + assert_eq!(s.ggml(), ty, "{s}"); + } } - w.finish().unwrap_or_else(|e| panic!("{e}")); - let split = Split::open(&path).unwrap_or_else(|e| panic!("{e}")); - let all = [ - SiteTy::Q3K, - SiteTy::Q4K, - SiteTy::Q5K, - SiteTy::Q6K, - SiteTy::Q5_0, - SiteTy::Q8_0, - SiteTy::F32, - SiteTy::Nvfp4, - SiteTy::Mxfp8, - SiteTy::Bf16, + let expected = [ + (SiteTy::Q3K, GgmlType::Q3_K, "Q3_K", Some(GemmWeight::Q3K)), + (SiteTy::Q4K, GgmlType::Q4_K, "Q4_K", Some(GemmWeight::Q4K)), + (SiteTy::Q5K, GgmlType::Q5_K, "Q5_K", Some(GemmWeight::Q5K)), + (SiteTy::Q6K, GgmlType::Q6_K, "Q6_K", Some(GemmWeight::Q6K)), + (SiteTy::Q5_0, GgmlType::Q5_0, "Q5_0", None), + (SiteTy::Q8_0, GgmlType::Q8_0, "Q8_0", None), + (SiteTy::F32, GgmlType::F32, "F32", None), + (SiteTy::Nvfp4, GgmlType::NVFP4, "NVFP4", None), + (SiteTy::Mxfp8, GgmlType::MXFP8, "MXFP8", None), + (SiteTy::Fp8B128, GgmlType::FP8_B128, "FP8_B128", None), + (SiteTy::Bf16, GgmlType::BF16, "BF16", None), + ( + SiteTy::Iq4Xs, + GgmlType::IQ4_XS, + "IQ4_XS", + Some(GemmWeight::Iq4Xs), + ), ]; + assert_eq!(SiteTy::ALL.len(), expected.len()); + for (&s, (want, ty, label, grouped)) in SiteTy::ALL.iter().zip(expected) { + assert_eq!(s, want); + assert_eq!(s.ggml(), ty); + assert_eq!(s.to_string(), label); + assert_eq!(gemm_weight(s), grouped); + assert_eq!(kgemv_order(s).is_some(), s.reads() == Form::Q8x128); + let by_name = model::placement::accepts::dense_by_name(ty); + assert_eq!(super::BY_NAME.contains(&s), by_name.is_some(), "{s}"); + if let Some(entry) = by_name { + assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); + } + let projected = model::placement::accepts::dense_projected(ty); + assert_eq!(super::PROJECTED.contains(&s), projected.is_some(), "{s}"); + if let Some(entry) = projected { + assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); + } + } + assert_eq!(SiteTy::Q5_0.reads(), Form::Q8Blocks32); + assert_eq!(kgemv_order(SiteTy::Q5_0), None); + assert_eq!(SiteTy::Nvfp4.reads(), Form::Q8x128); + assert_eq!(kgemv_order(SiteTy::Nvfp4), Some(Order::TokenMajor)); + assert_eq!(SiteTy::Mxfp8.reads(), Form::F32); + assert_eq!(kgemv_order(SiteTy::Iq4Xs), Some(Order::TokenMajor)); + } + + /// A site's type is read from the file's header alone: a Q8_0 matrix of + /// two rows of 64 is read as Q8_0 where Q8_0 is launched, and refused by + /// name where only K-quants are; a Q3_K, a Q5_K, a Q5_0, an IQ4_XS, an MXFP8 + /// (bloomery's id, sized by the split) and a BF16 one are read as theirs; an + /// IQ3_S one (a type no site launch reads), a matrix of another shape and a + /// type outside the allowed list are refused by name. + /// + /// PIN(2026-10-27): the refused type with no launch was IQ4_XS; it is a site family now (`SiteTy::Iq4Xs`), so IQ3_S stands in and IQ4_XS joins the read types. + #[test] + fn a_site_type_comes_from_the_header_and_a_type_with_no_launch_is_refused() { + let decl = |name: &str, ty: GgmlType, bytes| TensorDecl { + name: name.to_owned(), + dims: vec![256, 2], + type_id: ty.as_u32(), + nbytes: bytes, + }; + let layout = Layout::new( + &[], + vec![ + decl("shared", GgmlType::IQ4_XS, 2 * 136), + TensorDecl { + name: "q8".to_owned(), + dims: vec![64, 2], + type_id: GgmlType::Q8_0.as_u32(), + nbytes: 2 * 2 * 34, + }, + decl("q3", GgmlType::Q3_K, 2 * 110), + decl("q5", GgmlType::Q5_K, 2 * 176), + decl("q50", GgmlType::Q5_0, 2 * 8 * 22), + decl("mx", GgmlType::MXFP8, 2 * 8 * 33), + decl("bf", GgmlType::BF16, 2 * 256 * 2), + decl("unsupported", GgmlType::IQ3_S, 2 * 110), + ], + ) + .unwrap(); + let file = Split::header_only("sites.gguf".into(), &layout).unwrap(); for (name, k, want) in [ ("q8", 64, SiteTy::Q8_0), ("q3", 256, SiteTy::Q3K), ("q5", 256, SiteTy::Q5K), - ("q50", 64, SiteTy::Q5_0), - ("mx", 128, SiteTy::Mxfp8), - ("bf", 64, SiteTy::Bf16), + ("q50", 256, SiteTy::Q5_0), + ("mx", 256, SiteTy::Mxfp8), + ("bf", 256, SiteTy::Bf16), ] { - let got = file_site(&split, "test", name, (2, k), &all); - assert_eq!(got.ok(), Some(want), "{name}"); + assert_eq!( + file_site(&file, "test", name, (2, k), SiteTy::ALL).ok(), + Some(want), + "{name}" + ); } - let refused = [ + assert_eq!( + file_site(&file, "test", "shared", (2, 256), &super::PROJECTED).ok(), + Some(SiteTy::Iq4Xs) + ); + for (name, shape, allowed, want) in [ ( "q8", (2, 64), &[SiteTy::Q4K, SiteTy::Q6K][..], "q8 is q8_0 in the file", ), - ("iq", (1, 256), &all[..], "iq is iq4_xs in the file"), - ("q8", (4, 64), &all[..], "it takes 4 rows of 64"), - ]; - for (name, shape, allowed, want) in refused { - match file_site(&split, "test", name, shape, allowed) { - Err(e) => assert!(e.to_string().contains(want), "{name}: {e}"), - Ok(t) => panic!("{name} {shape:?} read as {t}"), - } + ( + "shared", + (2, 256), + &[SiteTy::Q4K][..], + "shared is iq4_xs in the file", + ), + ( + "unsupported", + (2, 256), + SiteTy::ALL, + "unsupported is iq3_s in the file", + ), + ("q8", (4, 64), SiteTy::ALL, "it takes 4 rows of 64"), + ("shared", (1, 256), SiteTy::ALL, "it takes 1 rows of 256"), + ("shared", (2, 512), SiteTy::ALL, "it takes 2 rows of 512"), + ] { + let Err(chain::ChainError::Shape { what, detail }) = + file_site(&file, "test", name, shape, allowed) + else { + panic!("{name} {shape:?}: not refused as a shape error"); + }; + assert_eq!(what, "test"); + assert!(detail.contains(want), "{name}: {detail}"); } - let _ = std::fs::remove_dir_all(&dir); } /// The SwiGLU's launch is the down's form under the rule: every K-quant @@ -2130,7 +2238,13 @@ mod tests { limit: 7.0, }; let launch = |ty, rule| swiglu_launch((ty, "down"), rule); - for ty in [SiteTy::Q3K, SiteTy::Q4K, SiteTy::Q5K, SiteTy::Q6K] { + for ty in [ + SiteTy::Q3K, + SiteTy::Q4K, + SiteTy::Q5K, + SiteTy::Q6K, + SiteTy::Iq4Xs, + ] { for rule in [Act::SiluMul, clamp, oai] { assert_eq!(launch(ty, rule).ok(), Some(SwigluLaunch::Q128), "{ty}"); } @@ -2174,77 +2288,6 @@ mod tests { } } - /// Every family is the kernel table's dense row of one file type: a type - /// has a family exactly where the table names a dense entry for it (F16, - /// which loads widened to f32, is the f32 site of its widened rows), the - /// family maps back to that type and is named by it in upper case, its - /// K-quant GEMM is the one the table names for the type, it is a - /// `kgemv` exactly where it reads the q8_1 rows of 128 values, and it - /// launches by name (`BY_NAME`) exactly where it reads the f32 rows or - /// the q8 blocks of 32. - #[test] - fn every_family_is_the_tables_dense_row_of_one_type() { - // PIN(2026-10-21): consolidate the three site pins, preserving their assertions. - let own = [ - GgmlType::F8E4M3_ROWS, - GgmlType::MXFP8, - GgmlType::NVFP4_ROWS, - GgmlType::FP8_B128, - ]; - for ty in gguf::QROWS.iter().map(|r| r.ty).chain(own) { - let family = SiteTy::of_ggml(ty); - if ty == GgmlType::F16 { - assert_eq!( - family, - Some(SiteTy::F32), - "{ty}: the f32 site of its widened rows" - ); - assert_eq!(kernels::dense(ty), None, "{ty}"); - continue; - } - assert_eq!(family.is_some(), kernels::dense(ty).is_some(), "{ty}"); - if let Some(s) = family { - assert_eq!(s.ggml(), ty, "{s}"); - } - } - let expected = [ - (SiteTy::Q3K, GgmlType::Q3_K, "Q3_K", Some(GemmWeight::Q3K)), - (SiteTy::Q4K, GgmlType::Q4_K, "Q4_K", Some(GemmWeight::Q4K)), - (SiteTy::Q5K, GgmlType::Q5_K, "Q5_K", Some(GemmWeight::Q5K)), - (SiteTy::Q6K, GgmlType::Q6_K, "Q6_K", Some(GemmWeight::Q6K)), - (SiteTy::Q5_0, GgmlType::Q5_0, "Q5_0", None), - (SiteTy::Q8_0, GgmlType::Q8_0, "Q8_0", None), - (SiteTy::F32, GgmlType::F32, "F32", None), - (SiteTy::Nvfp4, GgmlType::NVFP4, "NVFP4", None), - (SiteTy::Mxfp8, GgmlType::MXFP8, "MXFP8", None), - (SiteTy::Fp8B128, GgmlType::FP8_B128, "FP8_B128", None), - (SiteTy::Bf16, GgmlType::BF16, "BF16", None), - ]; - assert_eq!(SiteTy::ALL.len(), expected.len()); - for (&s, (want, ty, label, grouped)) in SiteTy::ALL.iter().zip(expected) { - assert_eq!(s, want); - assert_eq!(s.ggml(), ty); - assert_eq!(s.to_string(), label); - assert_eq!(gemm_weight(s), grouped); - assert_eq!(kgemv_order(s).is_some(), s.reads() == Form::Q8x128); - let by_name = model::placement::accepts::dense_by_name(ty); - assert_eq!(super::BY_NAME.contains(&s), by_name.is_some(), "{s}"); - if let Some(entry) = by_name { - assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); - } - let projected = model::placement::accepts::dense_projected(ty); - assert_eq!(super::PROJECTED.contains(&s), projected.is_some(), "{s}"); - if let Some(entry) = projected { - assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); - } - } - assert_eq!(SiteTy::Q5_0.reads(), Form::Q8Blocks32); - assert_eq!(kgemv_order(SiteTy::Q5_0), None); - assert_eq!(SiteTy::Nvfp4.reads(), Form::Q8x128); - assert_eq!(kgemv_order(SiteTy::Nvfp4), Some(Order::TokenMajor)); - assert_eq!(SiteTy::Mxfp8.reads(), Form::F32); - } - #[test] fn a_bf16_site_reads_the_f32_rows_and_is_a_gemv() { assert_eq!(SiteTy::of_ggml(GgmlType::BF16), Some(SiteTy::Bf16)); diff --git a/crates/model/src/arch/coverage.rs b/crates/model/src/arch/coverage.rs index ba3e2b110..169f31a22 100644 --- a/crates/model/src/arch/coverage.rs +++ b/crates/model/src/arch/coverage.rs @@ -3602,6 +3602,8 @@ mod tests { // PIN(2026-10-26): the K-quants (Q3_K to Q6_K) joined the dense sites' set: both walks project a site by name // (`site::ByName`), the q8_1 quantization and the K-quant gemv by chunks on the ubatch walk. The shared expert's // down keeps the f32-row types: K = 640 is not whole super-blocks. + // PIN(2026-10-27): IQ4_XS moved from refused to q8_1; it reads q8_1 through the selected one-expert dispatch, + // and the shared down still refuses it because its K = 640 is not whole super-blocks. #[test] fn the_qwen38_dense_pins_read_the_projected_sites() { let q8_1 = [ @@ -3609,6 +3611,7 @@ mod tests { GgmlType::Q4_K, GgmlType::Q5_K, GgmlType::Q6_K, + GgmlType::IQ4_XS, ]; let f32_rows = [ GgmlType::Q8_0, @@ -3622,7 +3625,6 @@ mod tests { GgmlType::Q2_K, GgmlType::IQ3_S, GgmlType::IQ4_NL, - GgmlType::IQ4_XS, GgmlType::Q4_0, ]; for (name, role, what) in [ @@ -3651,7 +3653,7 @@ mod tests { assert!(got[0].starts_with(&format!("{ty} {what}")), "{got:?}"); } } - // The shared expert's down is K = 640: a K-quant's rows are whole + // The shared expert's down is K = 640: the q8_1 types' rows are whole // super-blocks of 256, so only the f32-row types read it, and a type // no dense entry reads is refused by both pins. let down = "ffn_down_shexp.weight"; diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index 4282e4c43..5b4273f67 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -248,7 +248,8 @@ pub fn qwen35_leg([gate, up, down]: [GgmlType; 3]) -> Option { /// is `None`. An NVFP4 site has a gemv and no wide GEMM, which the body's /// prompt units run every dense site through, and is `None`. A BF16 site /// reads the words only a load under `ModelTensors::bf16_words` keeps, which -/// the qwen3moe files do not set, and is `None`. The body's load and the +/// the qwen3moe files do not set, and is `None`. Its legacy shared-expert +/// dispatch does not feed IQ4_XS sites either. The body's load and the /// coverage census both read this, so a site the arena cannot feed is /// refused at load and listed together. #[must_use] @@ -256,7 +257,7 @@ pub const fn qwen3moe_dense(ty: GgmlType) -> Option { let Some(entry) = kernels::dense(ty) else { return None; }; - if let SiteEntry::Nvfp4 | SiteEntry::Bf16 = entry { + if let SiteEntry::Nvfp4 | SiteEntry::Bf16 | SiteEntry::Iq4xs = entry { return None; } match entry.reads() { @@ -277,7 +278,7 @@ pub const fn dense_by_name(ty: GgmlType) -> Option { /// The dense sites launched by name over up to eight columns on every family /// the table's dense row names but Q5_0 (`site::ByName`): the f32-row entries -/// beside those that read q8_1 blocks of 128 (the K-quants, NVFP4), whose +/// beside those that read q8_1 blocks of 128 (the K-quants, IQ4_XS, NVFP4), whose /// activation the unit quantizes itself. Q5_0 reads 32-value blocks no /// by-name unit holds. #[must_use] @@ -1128,6 +1129,7 @@ mod tests { assert_eq!(qwen3moe_dense(GgmlType::Q5_0), None); assert_eq!(qwen3moe_dense(GgmlType::NVFP4), None); assert_eq!(qwen3moe_dense(GgmlType::BF16), None); + assert_eq!(qwen3moe_dense(GgmlType::IQ4_XS), None); assert_eq!(head_of_widened(GgmlType::BF16), None); assert_eq!(embed_of_widened(GgmlType::BF16), None); assert!(head_of_widened(GgmlType::Q8_0).is_some()); diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index a9a29eb17..6dfe81f83 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -644,6 +644,34 @@ mod tests { ); } + // PIN(2026-10-16): the dense pairs gain NVFP4 (`nvfp4_gemv` through + // `site::kgemv`); every other pair is unchanged. + // PIN(2026-10-19): and bloomery's own MXFP8 (`mxfp8_gemv` through + // `site::gemv`); every other pair is unchanged. + // PIN(2026-10-20): and bloomery's own FP8_B128 (`fp8_b128_gemv` through + // `site::gemv`); BF16 also uses `bf16_gemv` and `bf16_tile_gemm` over its words. + // PIN(2026-10-26): IQ4_XS adds the existing selected GEMV as an ordinary site (UTC). + #[test] + fn dense_admits_exactly_the_pinned_pairs() { + assert_eq!( + rows(dense), + [ + (GgmlType::F32, SiteEntry::F32), + (GgmlType::Q5_0, SiteEntry::Q5_0), + (GgmlType::Q8_0, SiteEntry::Q8_0), + (GgmlType::Q3_K, SiteEntry::Q3k), + (GgmlType::Q4_K, SiteEntry::Q4k), + (GgmlType::Q5_K, SiteEntry::Q5k), + (GgmlType::Q6_K, SiteEntry::Q6k), + (GgmlType::BF16, SiteEntry::Bf16), + (GgmlType::NVFP4, SiteEntry::Nvfp4), + (GgmlType::IQ4_XS, SiteEntry::Iq4xs), + (GgmlType::MXFP8, SiteEntry::Mxfp8), + (GgmlType::FP8_B128, SiteEntry::Fp8B128), + ] + ); + } + // PIN(2026-10-20): the GEMM pairs gain NVFP4 (`gemm_nvfp4p` over its // planes); every other pair is unchanged. // PIN(2026-10-21): and IQ2_XXS (`gemm_iq2xxs`, which `GemmWeight` already @@ -683,32 +711,6 @@ mod tests { ); } - // PIN(2026-10-16): the dense pairs gain NVFP4 (`nvfp4_gemv` through - // `site::kgemv`); every other pair is unchanged. - // PIN(2026-10-19): and bloomery's own MXFP8 (`mxfp8_gemv` through - // `site::gemv`); every other pair is unchanged. - // PIN(2026-10-20): and bloomery's own FP8_B128 (`fp8_b128_gemv` through - // `site::gemv`); BF16 also uses `bf16_gemv` and `bf16_tile_gemm` over its words. - #[test] - fn dense_admits_exactly_the_pinned_pairs() { - assert_eq!( - rows(dense), - [ - (GgmlType::F32, SiteEntry::F32), - (GgmlType::Q5_0, SiteEntry::Q5_0), - (GgmlType::Q8_0, SiteEntry::Q8_0), - (GgmlType::Q3_K, SiteEntry::Q3k), - (GgmlType::Q4_K, SiteEntry::Q4k), - (GgmlType::Q5_K, SiteEntry::Q5k), - (GgmlType::Q6_K, SiteEntry::Q6k), - (GgmlType::BF16, SiteEntry::Bf16), - (GgmlType::NVFP4, SiteEntry::Nvfp4), - (GgmlType::MXFP8, SiteEntry::Mxfp8), - (GgmlType::FP8_B128, SiteEntry::Fp8B128), - ] - ); - } - /// A dense entry is named by one type's row, so its file type is that /// row's type. #[test] @@ -751,14 +753,13 @@ mod tests { ); } - /// The layout a file tensor of each type loads in, and the one a routed - /// stack loads in under the default expert rule: every type's but q5_K's, - /// which that rule does not admit, and mxfp8's, which has no card expert - /// entry. + /// The layout of each file type and its default routed layout: Q5_K, + /// MXFP8 and IQ4_XS require a program-specific routed rule. // PIN(2026-10-23): NVFP4_ROWS loads as NVFP4's planes (its routed card // columns); every other pair is unchanged. // PIN(2026-10-25): IQ2_XS, IQ1_S, IQ2_S and IQ1_M load as the file's // blocks, as IQ2_XXS does; every other pair is unchanged. + // PIN(2026-10-26): ordinary IQ4_XS matrices retain their file blocks (UTC). #[test] fn the_card_layouts_are_the_pinned_pairs() { assert_eq!( @@ -779,6 +780,7 @@ mod tests { (GgmlType::IQ2_XS, CardFormat::KQuant), (GgmlType::IQ1_S, CardFormat::KQuant), (GgmlType::IQ2_S, CardFormat::KQuant), + (GgmlType::IQ4_XS, CardFormat::KQuant), (GgmlType::IQ1_M, CardFormat::KQuant), (GgmlType::MXFP8, CardFormat::Mxfp8Planes), (GgmlType::NVFP4_ROWS, CardFormat::Nvfp4Planes), @@ -786,8 +788,8 @@ mod tests { ] ); for ty in ALL { - let routed = - CardFormat::of(ty).filter(|_| !matches!(ty, GgmlType::Q5_K | GgmlType::MXFP8)); + let routed = CardFormat::of(ty) + .filter(|_| !matches!(ty, GgmlType::Q5_K | GgmlType::MXFP8 | GgmlType::IQ4_XS)); assert_eq!(CardFormat::of_routed(ty), routed, "{ty:?}"); } } @@ -836,6 +838,7 @@ mod tests { GgmlType::IQ2_XS, GgmlType::IQ1_S, GgmlType::IQ2_S, + GgmlType::IQ4_XS, GgmlType::IQ1_M, ] ); diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 72eb230ce..5a4efc3c7 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,8 +2,8 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. This component tree additionally has a header-only reader and shared tier-handoff -work, but no Flint body, app session or serve seat. Primitive GPU evidence is not Flint model admission. +that snapshot. This component tree has a header-only reader, shared tier-handoff work and the ordinary +IQ4_XS shared site, but no Flint body, app session or serve seat. Primitive GPU evidence is not Flint model admission. ## Six-slot handoff closure checks — 2026-10-26 @@ -114,6 +114,55 @@ capabilities inferred from this primitive. `check-ptx-names` is green with those pinned rows; a GB10 scan replaces them with the observed spill/JIT-local measurements. Offline no-JIT scans cannot supply that column. +### Ordinary IQ4_XS shared-site implementation + +IQ4_XS now has an ordinary file-layout/dense entry in QROWS and a common site family. +The loader keeps its original block words; `site::kgemv` uses the existing IQ4_XS +entry over one matrix and `site::gemm` uses the existing grouped GEMM. The common +gated block selects a split gate/up and sizes the down's q8_1 scratch from the +actual activation form. Its column walk and the shared wide-prompt owner use +those same sites. IQ4_XS is not relabeled as a routed expert or requantized. + +`hw_iq4_shared_loads_and_runs_narrow_columns_and_wide_replay`, owned by +`gate-gpu-ds41-lib`, covers real packed-weight loading, unchanged stored words, +the ordinary `SharedExpert` layout, unequal matrix widths, one/eight-column block +execution and the wide-prompt transition/captured replay. Wide scratch holds only +the q8_1 input and down forms these sites read. Its nonnegative constant-weight +case derives a scalar floating-operation bound; uniform codes do not detect nibble +decode ordering. That remains a separate IQ4 kernel-gate obligation, not a +whole-model accuracy pin or a claim made by this wrapper fixture. + +The pinned current-main merge (`cb8d92ebddd1ce1a325a15450c06a01f57ecf88c`) also +reconciles its existing Qwen3.8 projected-site consumer: `site::PROJECTED` includes +IQ4_XS, and selected dispatch uses the supplied stream. Projected attention/shared +sites are admitted; the K=640 shared down and unsupported types/shapes remain refused. +This reuses the existing program and kernel, not a new model implementation. + +The typed GGUF test `a_site_type_comes_from_the_header_and_a_type_with_no_launch_is_refused` +reads an IQ4_XS shared site through `PROJECTED`, not merely `ALL`. It refuses IQ3_S, another +shape and a type outside the allowed list by name. It is metadata-only and creates no CUDA context. + +Pinned upstream `95a483fc242d00dec7987efe4a05e24b5f651c1f` subsequently adds the +split gate/up path that materializes f32 SwiGLU for a down that reads f32. +The obsolete split-to-f32 refusal is removed, not re-pinned; mismatched gate/up +forms still refuse by semantic Shape/caller/type data. IQ4_XS follows the shared +q8x128 predicate in both row dispatch and `writes_h`, so the GLM prompt consumer +chooses the buffer that is actually written instead of allocating/reading a fake f32 row. +At source `5abe1ae66b651446938e334331c06683cef793a4`, the actual block metadata +consumers passed5 cases,0 failed/ignored,34 filtered, including both IQ4 down forms. +These host cases are not execution evidence for the remaining GPU cases. + +On 2026-10-26, source `203f5125a3c48a5b700c6f43318cfcb9c77ff169` executed the owning +DS41 library on Hikari GPU0 (RTX3090 cc8.6, driver595.91.07):38 passed,0 failed,0 ignored, +28.61s, including the named IQ4 consumer above. Rust1.99.1 / pinned cuda-oxide9df9e7ce +with repository patch31ab6a1d3f58 and LLVM21.1.8 were used. +Its canonical comparison to main15778e2c covered4bins/8carriers/1657entries: all identical, +non-JIT resource rows equal; no JIT-resource comparison is claimed. +All-target native Clippy/formatting and source203f5125 GitHub T0/lint passed. +These are source-specific historical results, not another hardware execution of +later diagnostic, test or documentation changes. Wider affected real-model, +placement and residency coverage remains explicit; this does not establish a +Flint oracle, physical cc12.1, ROCm or performance.