From 55df14add265f2c49f7504e8dfb540c4facddc51 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:32:42 +0900 Subject: [PATCH 01/14] gpu: support ordinary IQ4_XS shared sites --- crates/chain/src/site.rs | 4 + crates/gguf/src/card.rs | 11 +- crates/gguf/src/qtype.rs | 2 +- crates/gpu-deepseek41/src/block.rs | 65 +++--- crates/gpu-deepseek41/src/shared_routed.rs | 185 ++++++++++++++++ crates/gpu-gates/src/act_rule.rs | 45 +--- crates/gpu/src/site.rs | 237 ++++++--------------- crates/placement/src/accepts.rs | 5 +- crates/placement/src/kernels.rs | 114 ---------- docs/research/flint-attach.md | 25 +++ 10 files changed, 334 insertions(+), 359 deletions(-) diff --git a/crates/chain/src/site.rs b/crates/chain/src/site.rs index da5ccefcf..d44ac713e 100644 --- a/crates/chain/src/site.rs +++ b/crates/chain/src/site.rs @@ -38,6 +38,8 @@ pub enum SiteTy { /// entry is none; a backend whose launch reads it names it in the list /// [`file_site`] takes. MlxA4, + /// IQ4_XS in its original blocks, over q8_1 columns, narrow and wide. + Iq4Xs, } impl SiteTy { @@ -56,6 +58,7 @@ impl SiteTy { SiteTy::Mxfp8, SiteTy::Fp8B128, SiteTy::Bf16, + SiteTy::Iq4Xs, ]; /// The family of a file tensor of type `ty`, the one that launches the @@ -99,6 +102,7 @@ impl SiteTy { SiteTy::Mxfp8 => Some(SiteEntry::Mxfp8), SiteTy::Fp8B128 => Some(SiteEntry::Fp8B128), SiteTy::Bf16 => Some(SiteEntry::Bf16), + SiteTy::Iq4Xs => Some(SiteEntry::Iq4xs), SiteTy::MlxA4 => None, } } diff --git a/crates/gguf/src/card.rs b/crates/gguf/src/card.rs index 711ab5dea..48a51c8fe 100644 --- a/crates/gguf/src/card.rs +++ b/crates/gguf/src/card.rs @@ -255,6 +255,9 @@ pub enum SiteEntry { /// `Q8F32Kernels::enqueue_bf16_gemv` over BF16 words and f32 rows; /// wide, `Q8F32Kernels::enqueue_bf16_tile` over the same words and rows. Bf16, + /// `QuantSelKernels::enqueue_gemv_sel` over one matrix in the file's + /// IQ4_XS blocks and `Q8Act` columns; wide, the grouped IQ4_XS GEMM. + Iq4xs, } /// The activation a dense site's launch reads; `crates/gpu/src/site.rs` @@ -281,7 +284,11 @@ impl SiteEntry { #[must_use] pub const fn reads(self) -> SiteAct { match self { - SiteEntry::Q3k | SiteEntry::Q4k | SiteEntry::Q5k | SiteEntry::Q6k => SiteAct::Q8x128, + SiteEntry::Q3k + | SiteEntry::Q4k + | SiteEntry::Q5k + | SiteEntry::Q6k + | SiteEntry::Iq4xs => SiteAct::Q8x128, SiteEntry::Q8_0 => SiteAct::Q8x32, SiteEntry::F32 => SiteAct::F32, SiteEntry::Q5_0 => SiteAct::Q8Blocks32, @@ -300,7 +307,7 @@ impl SiteEntry { } /// One past the last [`SiteEntry`]: the span [`DENSE_TYPE`] covers. -const DENSE_SPAN: usize = SiteEntry::Bf16 as usize + 1; +const DENSE_SPAN: usize = SiteEntry::Iq4xs as usize + 1; /// Dense entry to the file type of the row that names it. The build refuses /// an entry two rows name, an entry no row names, and an entry past diff --git a/crates/gguf/src/qtype.rs b/crates/gguf/src/qtype.rs index a161da0c4..4fdb641ff 100644 --- a/crates/gguf/src/qtype.rs +++ b/crates/gguf/src/qtype.rs @@ -115,7 +115,7 @@ pub const QROWS: &[QRow] = &[ card: card!(- - Iq3s (Iq3s, Q8Act) Iq3s - - -) }, QRow { ty: T::IQ2_S, name: "iq2_s", blck: 256, bytes: 82, dequant: false, host_act: H::None, card: Card::NONE }, QRow { ty: T::IQ4_XS, name: "iq4_xs", blck: 256, bytes: 136, dequant: true, host_act: H::Q8K, - card: card!(- - Iq4xs (Iq4xs, Q8Act) Iq4xs - - -) }, + card: card!(KQuant - Iq4xs (Iq4xs, Q8Act) Iq4xs Iq4xs - -) }, // The C scalars: ggml.c:621-649. QRow { ty: T::I8, name: "i8", blck: 1, bytes: 1, dequant: false, host_act: H::None, card: Card::NONE }, QRow { ty: T::I16, name: "i16", blck: 1, bytes: 2, dequant: false, host_act: H::None, card: Card::NONE }, diff --git a/crates/gpu-deepseek41/src/block.rs b/crates/gpu-deepseek41/src/block.rs index e2f819456..e884d498a 100644 --- a/crates/gpu-deepseek41/src/block.rs +++ b/crates/gpu-deepseek41/src/block.rs @@ -38,7 +38,13 @@ const BLOCK: &str = "gated block"; /// The types a block's matrices read, one launch family each ([`site::file_site`]'s `allowed`); a model's /// coverage pin for the block lists the same set. -pub const BLOCK_TYPES: [SiteTy; 4] = [SiteTy::Q8_0, SiteTy::Q4K, SiteTy::Q5K, SiteTy::Q6K]; +pub const BLOCK_TYPES: [SiteTy; 5] = [ + SiteTy::Q8_0, + SiteTy::Q4K, + SiteTy::Q5K, + SiteTy::Q6K, + SiteTy::Iq4Xs, +]; /// The most columns [`enqueue_block_cols`] takes: the q8_0 block's multi-column launches' cap, kept for every plan. const COLS_MAX: usize = 8; @@ -48,6 +54,11 @@ fn refuse(what: &'static str, detail: String) -> GpuError { GpuError::Shape { what, detail } } +/// A site whose narrow and wide launches both read the 128-value q8_1 form. +fn q128(ty: SiteTy) -> bool { + ty.reads() == site::Form::Q8x128 && site::gemm_weight(ty).is_some() +} + /// A block's gate·up half, picked once at load from the file's types. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum GateUp { @@ -105,19 +116,19 @@ impl BlockPlan { !matches!(self.gate_up, GateUp::Split { .. }) } - /// The plan of a block whose gate, up and down are `tys`: the q8_0 pair or two K-quants for gate·up (the - /// same Q4_K or Q5_K fused, else split), any of [`BLOCK_TYPES`] for the down, which a split gate·up needs a - /// K-quant one of. Another mix is refused by name as `what`'s. + /// The plan of a block whose gate, up and down are `tys`: the q8_0 pair or two sites over + /// 128-value q8_1 blocks (the same Q4_K or Q5_K fused, else split). A split pair needs a + /// down over that same activation form. Another mix is refused by name as `what`'s. /// /// # Errors - /// A gate and up that are not two q8_0 or two K-quants, or a split pair over a down that is not a K-quant. + /// A gate and up whose activation forms disagree, or a split pair over a down of another form. pub fn of_types( what: &'static str, [gate, up, down]: [SiteTy; 3], ) -> Result { let gate_up = match (gate, up) { (SiteTy::Q8_0, SiteTy::Q8_0) => GateUp::Ds41, - (g, u) if g.kquant() && u.kquant() => match (g == u, g) { + (g, u) if q128(g) && q128(u) => match (g == u, g) { (true, SiteTy::Q4K | SiteTy::Q5K) => GateUp::Fused(g), _ => GateUp::Split { gate: g, up: u }, }, @@ -125,12 +136,12 @@ impl BlockPlan { return Err(refuse( what, format!( - "gate is {gate} and up is {up}: a block takes two q8_0 or two K-quants" + "gate is {gate} and up is {up}: a block takes two q8_0 or two sites over 128-value q8_1" ), )); } }; - if matches!(gate_up, GateUp::Split { .. }) && !down.kquant() { + if matches!(gate_up, GateUp::Split { .. }) && !q128(down) { return Err(refuse( what, format!( @@ -167,7 +178,7 @@ impl BlockPlan { /// SwiGLU already wrote it). #[must_use] pub fn extra_launches(self) -> usize { - let down_quantize = usize::from(self.down.kquant()); + let down_quantize = usize::from(q128(self.down)); match self.gate_up { GateUp::Ds41 => down_quantize, GateUp::Fused(_) => 1 + down_quantize, @@ -280,7 +291,7 @@ impl BlockScratch { let (mut any, mut split_ff) = (false, 0); for (plan, ff) in blocks.into_iter().filter(|(p, _)| !p.is_q8()) { any = true; - if plan.down.kquant() && !ha.iter().any(|(f, _)| *f == ff) { + if q128(plan.down) && !ha.iter().any(|(f, _)| *f == ff) { ha.push((ff, Q8Act::with_k(stream, 1, ff)?)); } if matches!(plan.gate_up, GateUp::Split { .. }) { @@ -426,7 +437,7 @@ pub fn enqueue_block_act( gpu.layer_sink(blk.layer)?, h, )?; - if !blk.plan.down.kquant() { + if !q128(blk.plan.down) { return site::resident_gemv(gpu, w, down, h, y); } } @@ -470,7 +481,7 @@ pub fn enqueue_block_act( return site::kgemv(gpu, &kk.gemv, (blk.plan.down, w, down), act, fault, y); } } - if !blk.plan.down.kquant() { + if !q128(blk.plan.down) { return site::resident_gemv(gpu, w, down, h, y); } let (kk, sc) = kit(k, s)?; @@ -620,16 +631,6 @@ mod tests { BlockPlan::of_types(WHAT, tys) } - fn refused(r: Result, words: &[&str]) { - let msg = match r { - Err(GpuError::Shape { what, detail }) => format!("{what}: {detail}"), - other => panic!("expected a named shape refusal, got {other:?}"), - }; - for w in words { - assert!(msg.contains(w), "{msg:?} lacks {w:?}"); - } - } - /// The plan is the types' dispatch: the q8_0 triple is `BlockPlan::Q8`, one Q4_K or Q5_K for gate and up /// is fused, any other K-quant pair splits, and every plan reads back the types it came from. #[test] @@ -654,17 +655,17 @@ mod tests { } } - /// A mix no arm covers is refused by name, with the types: gate and up of two kinds, and a split pair over - /// a q8_0 down. #[test] - fn a_mix_no_arm_covers_is_refused() { - use SiteTy::{Q4K, Q6K, Q8_0}; - refused( - plan([Q8_0, Q4K, Q8_0]), - &["test", "gate is Q8_0", "up is Q4_K"], - ); - refused(plan([Q6K, Q8_0, Q6K]), &["gate is Q6_K", "up is Q8_0"]); - refused(plan([Q6K, Q6K, Q8_0]), &["split", "Q8_0 down"]); + fn incompatible_activation_forms_refuse_before_launch() { + use SiteTy::{Iq4Xs, Q4K, Q6K, Q8_0}; + for types in [ + [Q8_0, Q4K, Q8_0], + [Q6K, Q8_0, Q6K], + [Q6K, Q6K, Q8_0], + [Iq4Xs, Iq4Xs, Q8_0], + ] { + assert!(matches!(plan(types), Err(GpuError::Shape { .. }))); + } } /// The launches beyond the q8_0 block's two, by plan: 0 for the q8_0 block, and the K-quant arms' derived diff --git a/crates/gpu-deepseek41/src/shared_routed.rs b/crates/gpu-deepseek41/src/shared_routed.rs index 17d147ec9..9df96d40b 100644 --- a/crates/gpu-deepseek41/src/shared_routed.rs +++ b/crates/gpu-deepseek41/src/shared_routed.rs @@ -1157,3 +1157,188 @@ fn no_batch() -> GpuError { missing: "the batch's buffers", } } + +#[cfg(test)] +mod tests { + use super::SharedWide; + use crate::block::{ + Block, BlockIo, BlockKernels, BlockPlan, BlockScratch, KBlockKernels, + enqueue_block_cols_act, + }; + use crate::experts::ExpertKernels; + use bloomery_gpu::Gpu; + use bloomery_gpu::gemm::Gemm32Kernels; + use bloomery_gpu::kquant::Act; + use bloomery_gpu::weights::Weights; + use cuda_core::DeviceBuffer; + use gguf::Split; + use gguf::quant::GgmlType; + use gguf::write::{Layout, TensorDecl, Writer}; + + const N: usize = 256; + const FF: usize = 512; + const NAMES: [&str; 3] = [ + "blk.42.ffn_gate_shexp.weight", + "blk.42.ffn_up_shexp.weight", + "blk.42.ffn_down_shexp.weight", + ]; + + fn iq4_shared_file() -> (std::path::PathBuf, Split) { + let path = + std::env::temp_dir().join(format!("bloomery-iq4-shared-{}.gguf", std::process::id())); + let specs = [(N, FF, 0x1c00u16), (N, FF, 0x1800), (FF, N, 0x1800)]; + let payloads: Vec> = specs + .iter() + .enumerate() + .map(|(i, &(k, rows, scale))| { + let mut bytes = Vec::with_capacity(rows * (k / 256) * 136); + for row in 0..rows { + let d = if i == 2 && row % 2 == 1 { + scale - 0x400 + } else { + scale + }; + let mut block = [0x88u8; 136]; + block[..2].copy_from_slice(&d.to_le_bytes()); + block[2..4].copy_from_slice(&0xaaaau16.to_le_bytes()); + block[4..8].fill(0x11); + for _ in 0..k / 256 { + bytes.extend_from_slice(&block); + } + } + bytes + }) + .collect(); + let decls = NAMES + .iter() + .zip(specs) + .zip(&payloads) + .map(|((&name, (k, rows, _)), bytes)| TensorDecl { + name: name.to_owned(), + dims: vec![u64::try_from(k).unwrap(), u64::try_from(rows).unwrap()], + type_id: GgmlType::IQ4_XS.as_u32(), + nbytes: u64::try_from(bytes.len()).unwrap(), + }) + .collect(); + let layout = Layout::new(&[], decls).unwrap(); + let file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .unwrap(); + let mut writer = Writer::new(file, layout).unwrap(); + for (name, bytes) in NAMES.iter().zip(payloads) { + writer.tensor(name, &bytes).unwrap(); + } + writer.finish().unwrap(); + let split = Split::open(&path).unwrap(); + (path, split) + } + + fn check_shared_rows(input: &[f32], rows: &[f32]) { + // Uniform positive rows quantize to code127 without integer rounding. + // The gate is at least128, so SiLU rounds to the gate itself. The + // scalar positive-dot upper bound has four input-width operations, + // two output-width operations and the scale/product roundings. + let operations = 4.0 * 256.0 + 2.0 * 512.0 + 16.0; + let eps = f64::from(f32::EPSILON); + let gamma = operations * eps / (1.0 - operations * eps); + for (c, row) in rows.chunks_exact(N).enumerate() { + let x = f64::from(input[c * N]); + for (r, &got) in row.iter().enumerate() { + let expected = x * x * if r % 2 == 0 { 0.5 } else { 0.25 }; + assert!(got.is_finite()); + assert!( + (f64::from(got) - expected).abs() <= gamma * expected, + "column {c} row {r}: {got} vs {expected}, relative bound {gamma}" + ); + } + } + } + + #[test] + #[ignore = "needs a CUDA card"] + fn hw_iq4_shared_loads_and_runs_narrow_columns_and_wide_replay() { + let (path, split) = iq4_shared_file(); + let gpu = Gpu::new().expect("card"); + let stream = gpu.stream(); + let weights = Weights::load_where(stream, &split, |name| NAMES.contains(&name)).unwrap(); + let plan = BlockPlan::of_file(&split, "IQ4_XS shared", NAMES, (FF, N)).unwrap(); + plan.check_resident(&weights, "IQ4_XS shared", NAMES, (FF, N)) + .unwrap(); + let block = Block { + plan, + names: NAMES, + ff: FF, + limit: 0.0, + layer: 42, + }; + let experts = ExpertKernels::load(gpu.context()).unwrap(); + let quant = KBlockKernels::load(&gpu).unwrap(); + let kernels = BlockKernels { + experts: &experts, + kquant: Some(&quant), + }; + let mut scratch = BlockScratch::new(stream, N, [(plan, FF)]).unwrap(); + let mut h = DeviceBuffer::zeroed(stream, FF).unwrap(); + let g32 = Gemm32Kernels::load(gpu.context()).unwrap(); + for cols in [1, 8, super::GATED_FROM] { + let input: Vec = (0..cols) + .flat_map(|c| std::iter::repeat_n([128.0, 256.0, 384.0][c % 3], N)) + .collect(); + let x = DeviceBuffer::from_host(stream, &input).unwrap(); + let mut y = DeviceBuffer::zeroed(stream, cols * N).unwrap(); + if cols < super::GATED_FROM { + enqueue_block_cols_act( + &gpu, + &weights, + &kernels, + &mut scratch, + (&block, Act::SiluMul), + cols, + BlockIo { + x: &x, + h: &mut h, + y: &mut y, + }, + ) + .unwrap(); + } else { + let types = plan.types(); + let mut wide = + SharedWide::new(&gpu, (&types[..2], &types[2..]), (cols, N, FF)).unwrap(); + wide.run( + (&gpu, &weights, &g32), + (&block, Act::SiluMul), + (cols, N), + (&x, &mut y), + ) + .unwrap(); + let eager = y.to_host_vec(stream).unwrap(); + check_shared_rows(&input, &eager); + let graph = gpu + .capture(|_| { + wide.run( + (&gpu, &weights, &g32), + (&block, Act::SiluMul), + (cols, N), + (&x, &mut y), + ) + }) + .unwrap(); + graph.launch(stream).unwrap(); + let replay = y.to_host_vec(stream).unwrap(); + assert!( + eager + .iter() + .zip(&replay) + .all(|(a, b)| a.to_bits() == b.to_bits()) + ); + } + check_shared_rows(&input, &y.to_host_vec(stream).unwrap()); + assert_eq!(gpu.take_fault().unwrap(), None); + } + stream.synchronize().unwrap(); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/crates/gpu-gates/src/act_rule.rs b/crates/gpu-gates/src/act_rule.rs index a959cb768..33596c379 100644 --- a/crates/gpu-gates/src/act_rule.rs +++ b/crates/gpu-gates/src/act_rule.rs @@ -237,8 +237,11 @@ impl SiteRel { #[must_use] pub fn site_rel(ty: GgmlType, arm: Arm) -> Option { let ours = match (ty, arm) { - (GgmlType::Q3_K | GgmlType::Q4_K | GgmlType::Q5_K | GgmlType::Q6_K, _) - | (GgmlType::IQ3_XXS | GgmlType::IQ4_XS, Arm::Expert) => Some(Q8_1_BLOCK), + ( + GgmlType::Q3_K | GgmlType::Q4_K | GgmlType::Q5_K | GgmlType::Q6_K | GgmlType::IQ4_XS, + _, + ) + | (GgmlType::IQ3_XXS, Arm::Expert) => Some(Q8_1_BLOCK), // Bolt's IQ2_XXS: the routed `iq2_xxs_gemv_sel` and the dense `iq2_xxs_rows` read q8_1. (GgmlType::IQ2_XXS, Arm::Expert | Arm::Gemv) => Some(Q8_1_BLOCK), // Bolt's dense NVFP4 (`nvfp4_gemv`, the ssm_in/ssm_out) reads the engine's q8_1. @@ -1012,7 +1015,6 @@ mod site_tests { for ty in [ GgmlType::IQ3_S, GgmlType::IQ3_XXS, - GgmlType::IQ4_XS, GgmlType::Q5_1, GgmlType::IQ4_NL, ] { @@ -1040,43 +1042,6 @@ mod site_tests { } } - /// Ours is the form the launches read, as their sources say it: the text of the kernel - /// table's `SiteEntry::reads` (which `site.rs`'s `SiteTy::reads` returns) and of the 32-value - /// GEMM family's module doc, so a launch that moves to another form is red here until the - /// table follows it. - #[test] - fn site_rel_reads_the_forms_the_launches_name() { - let gpu = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../gpu/src"); - let read = |file: &str| { - std::fs::read_to_string(gpu.join(file)) - .unwrap_or_else(|e| panic!("crates/gpu/src/{file}: {e}")) - }; - let (site, gemm, grouped) = ( - read("../../gguf/src/card.rs"), - read("gemm/mod.rs"), - read("gemm/grouped.rs"), - ); - for line in [ - "SiteEntry::Q3k | SiteEntry::Q4k | SiteEntry::Q5k | SiteEntry::Q6k => SiteAct::Q8x128", - "SiteEntry::Q8_0 => SiteAct::Q8x32", - "SiteEntry::F32 => SiteAct::F32", - "SiteEntry::Q5_0 => SiteAct::Q8Blocks32", - "q8_1 blocks of 128 values", - ] { - assert!(site.contains(line), "kernels.rs no longer says {line:?}"); - } - assert!( - gemm.contains("(Q8_0, IQ4_NL, Q5_1), over [`GemmAct32`]"), - "gemm/mod.rs no longer names the 32-value family" - ); - for variant in ["Iq3Xxs", "Iq4Xs"] { - assert!( - grouped.contains(variant), - "the grouped GEMM no longer decodes {variant}" - ); - } - } - /// ik's side of the table is the transcription this module already holds for the K-quants /// ([`Rule::of`]), and so is ours for Q3_K and Q4_K; Q5_K's ours differs from it by design (the /// V4.1 gemv reads f32, the bodies' site reads q8_1). diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index cc455f72f..d5d581e54 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -104,14 +104,14 @@ pub enum Order { /// file type its form holds ([`SiteTy::of_ggml`]; an F32 plane is the f32 /// site whatever type it was widened from). `None` for a form no dense entry /// reads: a stack in the file's blocks of a type whose entry reads another -/// layout (Q8_0's planes, an i-quant), a derived weight and an NVFP4 stack. +/// layout (Q8_0's planes), a derived weight and an NVFP4 stack. /// The resident form is this crate's, so the map is here, beside the /// launches that read it, and not on the chain's [`SiteTy`]. #[must_use] pub fn of_resident(w: &DevWeight) -> Option { let ty = match w { DevWeight::KQuant { ty, .. } => { - return SiteTy::of_ggml(*ty).filter(|s| s.kquant()); + return SiteTy::of_ggml(*ty).filter(|s| s.kquant() || *s == SiteTy::Iq4Xs); } DevWeight::Q5_0 { .. } => GgmlType::Q5_0, DevWeight::Q5_1 { .. } => GgmlType::Q5_1, @@ -126,7 +126,7 @@ pub fn of_resident(w: &DevWeight) -> Option { SiteTy::of_ggml(ty) } -/// How [`kgemv`] lays a site of type `ty` out: Q3_K, Q4_K and Q6_K row-major, Q5_K and NVFP4 token-major. +/// How [`kgemv`] lays a site out: Q3_K, Q4_K and Q6_K row-major, Q5_K, IQ4_XS and NVFP4 token-major. /// `None` for a type [`kgemv`] does not run: Q8_0, F32, MXFP8, FP8_B128 and BF16 ([`gemv`]), Q5_0 /// ([`blocks32_gemv`], whose row-major layout `y[r · m + c]` is that function's own), and MLX_A4, /// a backend's family no card launch reads. @@ -134,7 +134,7 @@ pub fn of_resident(w: &DevWeight) -> Option { pub fn kgemv_order(ty: SiteTy) -> Option { match ty { SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q6K => Some(Order::RowMajor), - SiteTy::Q5K | SiteTy::Nvfp4 => Some(Order::TokenMajor), + SiteTy::Q5K | SiteTy::Iq4Xs | SiteTy::Nvfp4 => Some(Order::TokenMajor), SiteTy::Q5_0 | SiteTy::Q8_0 | SiteTy::F32 @@ -666,6 +666,22 @@ pub fn kgemv( y, ) } + SiteTy::Iq4Xs => { + let wt = kq(w, WHAT, name)?; + kk.kq.enqueue_gemv_sel( + gpu.stream(), + GgmlType::IQ4_XS, + &SelDown { + w: wt, + act, + sel: &kk.one, + n_slots: act.m(), + rows_per_expert: wt.rows(), + }, + fault, + y, + ) + } SiteTy::Nvfp4 => kk .nv .enqueue_gemv(gpu.stream(), w.nvfp4(WHAT, name)?, act, y), @@ -752,9 +768,11 @@ impl<'a> SiteWeight<'a> { /// name as `what`. fn check(self, what: &'static str, ty: SiteTy) -> Result<(), GpuError> { match self { - SiteWeight::Words { name, .. } if !ty.kquant() => Err(GpuError::shape( + SiteWeight::Words { name, .. } if gemm_weight(ty).is_none() => Err(GpuError::shape( what, - format!("{name} is {ty} on words a caller holds; only a K-quant's words launch"), + format!( + "{name} is {ty} on words a caller holds; its grouped GEMM reads no such words" + ), )), SiteWeight::Part { name, .. } if !BY_NAME.contains(&ty) => Err(GpuError::shape( what, @@ -1006,21 +1024,23 @@ pub fn gemm( src.check(WHAT, ty)?; match ty { SiteTy::Nvfp4 => Err(no_wide_gemm(WHAT, name, ty)), - SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q5K | SiteTy::Q6K => k.gemm.enqueue_gemm( - stream, - GemmArgs { - ty: gemm_weight(ty).ok_or(GpuError::state(WHAT, "a K-quant's GEMM type"))?, - w: match src { - SiteWeight::Words { w, .. } => w, - _ => kq(w, WHAT, name)?, + SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q5K | SiteTy::Q6K | SiteTy::Iq4Xs => { + k.gemm.enqueue_gemm( + stream, + GemmArgs { + ty: gemm_weight(ty).ok_or(GpuError::state(WHAT, "a K-quant's GEMM type"))?, + w: match src { + SiteWeight::Words { w, .. } => w, + _ => kq(w, WHAT, name)?, + }, + rows_per_expert: rows, + act: x.q128.ok_or_else(|| missing("q8_1 blocks of 128"))?, + route, + input, + y, }, - rows_per_expert: rows, - act: x.q128.ok_or_else(|| missing("q8_1 blocks of 128"))?, - route, - input, - y, - }, - ), + ) + } SiteTy::Q8_0 | SiteTy::F32 | SiteTy::Mxfp8 | SiteTy::Fp8B128 | SiteTy::Bf16 => { dense()?; let (site, part) = src.rows(WHAT)?; @@ -1470,95 +1490,42 @@ fn swiglu( #[cfg(test)] mod tests { use super::{ - Form, Order, SiteTy, SwigluLaunch, WideActs, file_site, gemm_weight, kgemv_order, - swiglu_launch, + Form, SiteTy, SwigluLaunch, WideActs, file_site, gemm_weight, kgemv_order, swiglu_launch, }; - use crate::gemm::GemmWeight; use crate::kquant::Act; use gguf::Split; use gguf::quant::GgmlType; - use gguf::write::{Layout, TensorDecl, Writer}; - use model::placement::kernels; - use std::fs::File; - use std::io::BufWriter; - - /// A site's type is read from the file's header alone: a Q8_0 matrix of - /// two rows of 64 is read as Q8_0 where Q8_0 is launched, and refused by - /// name where only K-quants are; a Q3_K, a Q5_K, a Q5_0 and an MXFP8 one - /// (bloomery's id, sized by the split) are read as theirs; an IQ4_XS one (a type no site launch reads) and a matrix of - /// another shape are refused by name. + use gguf::write::{Layout, TensorDecl}; + #[test] - fn a_site_type_comes_from_the_header_and_a_type_with_no_launch_is_refused() { - let dir = std::env::temp_dir().join(format!("bloomery-site-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap_or_else(|e| panic!("{e}")); - let path = dir.join("m.gguf"); - let decls = [ - ("q8", vec![64, 2], 8u32, 2 * 2 * 34), - ("iq", vec![256, 1], 23, 136), - ("q3", vec![256, 2], 11, 2 * 110), - ("q5", vec![256, 2], 13, 2 * 176), - ("q50", vec![64, 2], 6, 2 * 2 * 22), - ("mx", vec![128, 2], GgmlType::MXFP8.as_u32(), 2 * 4 * 33), - ("bf", vec![64, 2], GgmlType::BF16.as_u32(), 2 * 64 * 2), - ]; - let tensors = decls - .iter() - .map(|(name, dims, type_id, nbytes)| TensorDecl { - name: (*name).to_string(), - dims: dims.clone(), - type_id: *type_id, - nbytes: *nbytes, - }) - .collect(); - let layout = Layout::new(&[], tensors).unwrap_or_else(|e| panic!("{e}")); - let file = BufWriter::new(File::create(&path).unwrap_or_else(|e| panic!("{e}"))); - let mut w = Writer::new(file, layout).unwrap_or_else(|e| panic!("{e}")); - for (name, _, _, nbytes) in &decls { - w.tensor(name, &vec![0u8; *nbytes as usize]) - .unwrap_or_else(|e| panic!("{e}")); - } - w.finish().unwrap_or_else(|e| panic!("{e}")); - let split = Split::open(&path).unwrap_or_else(|e| panic!("{e}")); - let all = [ - SiteTy::Q3K, - SiteTy::Q4K, - SiteTy::Q5K, - SiteTy::Q6K, - SiteTy::Q5_0, - SiteTy::Q8_0, - SiteTy::F32, - SiteTy::Nvfp4, - SiteTy::Mxfp8, - SiteTy::Bf16, - ]; - for (name, k, want) in [ - ("q8", 64, SiteTy::Q8_0), - ("q3", 256, SiteTy::Q3K), - ("q5", 256, SiteTy::Q5K), - ("q50", 64, SiteTy::Q5_0), - ("mx", 128, SiteTy::Mxfp8), - ("bf", 64, SiteTy::Bf16), + fn iq4_sites_keep_shape_and_launch_admission_separate() { + let decl = |name: &str, ty: GgmlType, bytes| TensorDecl { + name: name.to_owned(), + dims: vec![256, 2], + type_id: ty.as_u32(), + nbytes: bytes, + }; + let layout = Layout::new( + &[], + vec![ + decl("shared", GgmlType::IQ4_XS, 2 * 136), + decl("unsupported", GgmlType::IQ3_S, 2 * 110), + ], + ) + .unwrap(); + let file = Split::header_only("sites.gguf".into(), &layout).unwrap(); + assert_eq!( + file_site(&file, "test", "shared", (2, 256), SiteTy::ALL).ok(), + Some(SiteTy::Iq4Xs) + ); + for (name, shape, allowed) in [ + ("shared", (1, 256), SiteTy::ALL), + ("shared", (2, 512), SiteTy::ALL), + ("shared", (2, 256), &[SiteTy::Q4K][..]), + ("unsupported", (2, 256), SiteTy::ALL), ] { - let got = file_site(&split, "test", name, (2, k), &all); - assert_eq!(got.ok(), Some(want), "{name}"); + assert!(file_site(&file, "test", name, shape, allowed).is_err()); } - let refused = [ - ( - "q8", - (2, 64), - &[SiteTy::Q4K, SiteTy::Q6K][..], - "q8 is q8_0 in the file", - ), - ("iq", (1, 256), &all[..], "iq is iq4_xs in the file"), - ("q8", (4, 64), &all[..], "it takes 4 rows of 64"), - ]; - for (name, shape, allowed, want) in refused { - match file_site(&split, "test", name, shape, allowed) { - Err(e) => assert!(e.to_string().contains(want), "{name}: {e}"), - Ok(t) => panic!("{name} {shape:?} read as {t}"), - } - } - let _ = std::fs::remove_dir_all(&dir); } /// The SwiGLU's launch is the down's form under the rule: every K-quant @@ -1616,72 +1583,6 @@ mod tests { } } - /// Every family is the kernel table's dense row of one file type: a type - /// has a family exactly where the table names a dense entry for it (F16, - /// which loads widened to f32, is the f32 site of its widened rows), the - /// family maps back to that type and is named by it in upper case, its - /// K-quant GEMM is the one the table names for the type, it is a - /// `kgemv` exactly where it reads the q8_1 rows of 128 values, and it - /// launches by name (`BY_NAME`) exactly where it reads the f32 rows or - /// the q8 blocks of 32. - #[test] - fn every_family_is_the_tables_dense_row_of_one_type() { - // PIN(2026-10-21): consolidate the three site pins, preserving their assertions. - let own = [ - GgmlType::F8E4M3_ROWS, - GgmlType::MXFP8, - GgmlType::NVFP4_ROWS, - GgmlType::FP8_B128, - ]; - for ty in gguf::QROWS.iter().map(|r| r.ty).chain(own) { - let family = SiteTy::of_ggml(ty); - if ty == GgmlType::F16 { - assert_eq!( - family, - Some(SiteTy::F32), - "{ty}: the f32 site of its widened rows" - ); - assert_eq!(kernels::dense(ty), None, "{ty}"); - continue; - } - assert_eq!(family.is_some(), kernels::dense(ty).is_some(), "{ty}"); - if let Some(s) = family { - assert_eq!(s.ggml(), ty, "{s}"); - } - } - let expected = [ - (SiteTy::Q3K, GgmlType::Q3_K, "Q3_K", Some(GemmWeight::Q3K)), - (SiteTy::Q4K, GgmlType::Q4_K, "Q4_K", Some(GemmWeight::Q4K)), - (SiteTy::Q5K, GgmlType::Q5_K, "Q5_K", Some(GemmWeight::Q5K)), - (SiteTy::Q6K, GgmlType::Q6_K, "Q6_K", Some(GemmWeight::Q6K)), - (SiteTy::Q5_0, GgmlType::Q5_0, "Q5_0", None), - (SiteTy::Q8_0, GgmlType::Q8_0, "Q8_0", None), - (SiteTy::F32, GgmlType::F32, "F32", None), - (SiteTy::Nvfp4, GgmlType::NVFP4, "NVFP4", None), - (SiteTy::Mxfp8, GgmlType::MXFP8, "MXFP8", None), - (SiteTy::Fp8B128, GgmlType::FP8_B128, "FP8_B128", None), - (SiteTy::Bf16, GgmlType::BF16, "BF16", None), - ]; - assert_eq!(SiteTy::ALL.len(), expected.len()); - for (&s, (want, ty, label, grouped)) in SiteTy::ALL.iter().zip(expected) { - assert_eq!(s, want); - assert_eq!(s.ggml(), ty); - assert_eq!(s.to_string(), label); - assert_eq!(gemm_weight(s), grouped); - assert_eq!(kgemv_order(s).is_some(), s.reads() == Form::Q8x128); - let by_name = model::placement::accepts::dense_by_name(ty); - assert_eq!(super::BY_NAME.contains(&s), by_name.is_some(), "{s}"); - if let Some(entry) = by_name { - assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); - } - } - assert_eq!(SiteTy::Q5_0.reads(), Form::Q8Blocks32); - assert_eq!(kgemv_order(SiteTy::Q5_0), None); - assert_eq!(SiteTy::Nvfp4.reads(), Form::Q8x128); - assert_eq!(kgemv_order(SiteTy::Nvfp4), Some(Order::TokenMajor)); - assert_eq!(SiteTy::Mxfp8.reads(), Form::F32); - } - #[test] fn a_bf16_site_reads_the_f32_rows_and_is_a_gemv() { assert_eq!(SiteTy::of_ggml(GgmlType::BF16), Some(SiteTy::Bf16)); diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index 1afdc9890..245bae230 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -231,7 +231,8 @@ pub fn qwen35_leg([gate, up, down]: [GgmlType; 3]) -> Option { /// is `None`. An NVFP4 site has a gemv and no wide GEMM, which the body's /// prompt units run every dense site through, and is `None`. A BF16 site /// reads the words only a load under `ModelTensors::bf16_words` keeps, which -/// the qwen3moe files do not set, and is `None`. The body's load and the +/// the qwen3moe files do not set, and is `None`. Its legacy shared-expert +/// dispatch does not feed IQ4_XS sites either. The body's load and the /// coverage census both read this, so a site the arena cannot feed is /// refused at load and listed together. #[must_use] @@ -239,7 +240,7 @@ pub const fn qwen3moe_dense(ty: GgmlType) -> Option { let Some(entry) = kernels::dense(ty) else { return None; }; - if let SiteEntry::Nvfp4 | SiteEntry::Bf16 = entry { + if let SiteEntry::Nvfp4 | SiteEntry::Bf16 | SiteEntry::Iq4xs = entry { return None; } match entry.reads() { diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 8b9d69940..5d7b4425b 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -621,32 +621,6 @@ mod tests { ); } - // PIN(2026-10-16): the dense pairs gain NVFP4 (`nvfp4_gemv` through - // `site::kgemv`); every other pair is unchanged. - // PIN(2026-10-19): and bloomery's own MXFP8 (`mxfp8_gemv` through - // `site::gemv`); every other pair is unchanged. - // PIN(2026-10-20): and bloomery's own FP8_B128 (`fp8_b128_gemv` through - // `site::gemv`); BF16 also uses `bf16_gemv` and `bf16_tile_gemm` over its words. - #[test] - fn dense_admits_exactly_the_pinned_pairs() { - assert_eq!( - rows(dense), - [ - (GgmlType::F32, SiteEntry::F32), - (GgmlType::Q5_0, SiteEntry::Q5_0), - (GgmlType::Q8_0, SiteEntry::Q8_0), - (GgmlType::Q3_K, SiteEntry::Q3k), - (GgmlType::Q4_K, SiteEntry::Q4k), - (GgmlType::Q5_K, SiteEntry::Q5k), - (GgmlType::Q6_K, SiteEntry::Q6k), - (GgmlType::BF16, SiteEntry::Bf16), - (GgmlType::NVFP4, SiteEntry::Nvfp4), - (GgmlType::MXFP8, SiteEntry::Mxfp8), - (GgmlType::FP8_B128, SiteEntry::Fp8B128), - ] - ); - } - /// A dense entry is named by one type's row, so its file type is that /// row's type. #[test] @@ -686,94 +660,6 @@ mod tests { ); } - /// The layout a file tensor of each type loads in, and the one a routed - /// stack loads in under the default expert rule: every type's but q5_K's, - /// which that rule does not admit, and mxfp8's, which has no card expert - /// entry. - #[test] - fn the_card_layouts_are_the_pinned_pairs() { - assert_eq!( - rows(CardFormat::of), - [ - (GgmlType::F32, CardFormat::F32), - (GgmlType::F16, CardFormat::Bf16AsF32), - (GgmlType::Q5_0, CardFormat::Q5_0), - (GgmlType::Q5_1, CardFormat::Q5_1), - (GgmlType::Q8_0, CardFormat::Q8_0Planes), - (GgmlType::Q3_K, CardFormat::KQuant), - (GgmlType::Q4_K, CardFormat::KQuant), - (GgmlType::Q5_K, CardFormat::KQuant), - (GgmlType::Q6_K, CardFormat::KQuant), - (GgmlType::BF16, CardFormat::Bf16AsF32), - (GgmlType::NVFP4, CardFormat::Nvfp4Planes), - (GgmlType::IQ2_XXS, CardFormat::KQuant), - (GgmlType::MXFP8, CardFormat::Mxfp8Planes), - (GgmlType::FP8_B128, CardFormat::Fp8B128Planes), - ] - ); - for ty in ALL { - let routed = - CardFormat::of(ty).filter(|_| !matches!(ty, GgmlType::Q5_K | GgmlType::MXFP8)); - assert_eq!(CardFormat::of_routed(ty), routed, "{ty:?}"); - } - } - - /// The types each layout holds: the types it is the load layout of, the - /// K-quants as decoded f32, bf16 as raw words, and in the file's blocks - /// every type a routed entry reads in them. - #[test] - fn each_layout_holds_exactly_the_pinned_types() { - let held = |f: CardFormat| -> Vec { - ALL.into_iter().filter(|&ty| f.holds(ty)).collect() - }; - assert_eq!( - held(CardFormat::KQuant), - [ - GgmlType::Q5_1, - GgmlType::Q8_0, - GgmlType::Q3_K, - GgmlType::Q4_K, - GgmlType::Q5_K, - GgmlType::Q6_K, - GgmlType::MXFP4, - GgmlType::IQ2_XXS, - GgmlType::IQ2_XS, - GgmlType::IQ3_XXS, - GgmlType::IQ4_NL, - GgmlType::IQ3_S, - GgmlType::IQ4_XS, - ] - ); - assert_eq!( - held(CardFormat::Bf16AsF32), - [ - GgmlType::F16, - GgmlType::Q3_K, - GgmlType::Q4_K, - GgmlType::Q5_K, - GgmlType::Q6_K, - GgmlType::BF16, - GgmlType::IQ2_XXS, - ] - ); - // PIN(2026-10-22): independent BF16 heads also occupy MXFP8 planes. - assert_eq!( - held(CardFormat::Mxfp8Planes), - [GgmlType::BF16, GgmlType::MXFP8] - ); - for (f, ty) in [ - (CardFormat::Q5_0, GgmlType::Q5_0), - (CardFormat::Q5_1, GgmlType::Q5_1), - (CardFormat::Q8_0Planes, GgmlType::Q8_0), - (CardFormat::Nvfp4Planes, GgmlType::NVFP4), - (CardFormat::Fp8B128Planes, GgmlType::FP8_B128), - (CardFormat::F32, GgmlType::F32), - (CardFormat::Bf16Raw, GgmlType::BF16), - ] { - assert_eq!(held(f), [ty], "{f:?}"); - } - } - /// A card entry reads a layout the card loader can build: every type a /// row names has a [`CardFormat`] of its own, or is one of the file-block /// types a program's routed stack keeps as `KQuant`. diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 0d60c5feb..012ef946d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -66,6 +66,31 @@ Each device slice needs its numerical/fault gates and the maintainer's offered h Legacy metadata booleans share `RouterNorm::from_legacy_bool`: false keeps scores and true uses the existing epsilon-added selected sum. Flint's distinct floor mapping stays in its reader. +### Ordinary IQ4_XS shared-site implementation + +IQ4_XS now has an ordinary file-layout/dense entry in QROWS and a common site family. +The loader keeps its original block words; `site::kgemv` uses the existing IQ4_XS +entry over one matrix and `site::gemm` uses the existing grouped GEMM. The common +gated block selects a split gate/up and sizes the down's q8_1 scratch from the +actual activation form. Its column walk and the shared wide-prompt owner use +those same sites. IQ4_XS is not relabeled as a routed expert or requantized. + +A compiled public `file_site` consumer on the pinned header refused all three +layer42 shared matrices before this change and accepts their IQ4_XS/Q8x128 +bindings after it. This reads no payload and proves neither loading nor arithmetic. +`hw_iq4_shared_loads_and_runs_narrow_columns_and_wide_replay`, owned by +`gate-gpu-ds41-lib`, covers real packed-weight loading, unequal matrix widths, +one/eight-column block execution and the wide-prompt transition/captured replay. +Its nonnegative constant-weight case derives a scalar floating-operation bound; +it is not a whole-model accuracy pin. The hardware case has not been run here. + +Existing program-specific admission remains unchanged: in particular the legacy +Qwen projection/shared dispatch does not gain this site automatically. Flint +still has no Program, attention/router binding, body or serve seat. Run the new +consumer gate and the affected regressions on NVIDIA before claiming device +support; reconcile full-model coverage only with the actual body integration. + + ## 1. Sources and observed artifact From 2f2a8875161ccfed3f33680f794d736445099fc1 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:37:10 +0900 Subject: [PATCH 02/14] gpu: use fixed-size rows in the IQ4 consumer gate --- crates/gpu-deepseek41/src/shared_routed.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/gpu-deepseek41/src/shared_routed.rs b/crates/gpu-deepseek41/src/shared_routed.rs index 9df96d40b..0c2b6063f 100644 --- a/crates/gpu-deepseek41/src/shared_routed.rs +++ b/crates/gpu-deepseek41/src/shared_routed.rs @@ -1243,7 +1243,7 @@ mod tests { let operations = 4.0 * 256.0 + 2.0 * 512.0 + 16.0; let eps = f64::from(f32::EPSILON); let gamma = operations * eps / (1.0 - operations * eps); - for (c, row) in rows.chunks_exact(N).enumerate() { + for (c, row) in rows.as_chunks::().0.iter().enumerate() { let x = f64::from(input[c * N]); for (r, &got) in row.iter().enumerate() { let expected = x * x * if r % 2 == 0 { 0.5 } else { 0.25 }; From 203f5125a3c48a5b700c6f43318cfcb9c77ff169 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:19:09 +0900 Subject: [PATCH 03/14] Keep IQ4 regression checks behavioral and format the merged sites --- crates/gpu-deepseek41/src/block.rs | 5 +++- crates/gpu-deepseek41/src/shared_routed.rs | 3 +- crates/gpu-gates/src/act_rule.rs | 35 ++++------------------ crates/gpu/src/site.rs | 15 ++++++++-- 4 files changed, 24 insertions(+), 34 deletions(-) diff --git a/crates/gpu-deepseek41/src/block.rs b/crates/gpu-deepseek41/src/block.rs index f5c634118..d35f5b674 100644 --- a/crates/gpu-deepseek41/src/block.rs +++ b/crates/gpu-deepseek41/src/block.rs @@ -676,7 +676,10 @@ mod tests { assert_eq!(plan([Q6K, Q6K, Q5K]).unwrap().gate_up, split(Q6K, Q6K)); assert_eq!(plan([Q4K, Q5K, Q4K]).unwrap().gate_up, split(Q4K, Q5K)); assert_eq!(plan([Q3K, Q3K, Q6K]).unwrap().gate_up, split(Q3K, Q3K)); - assert_eq!(plan([Iq4Xs, Iq4Xs, Iq4Xs]).unwrap().gate_up, split(Iq4Xs, Iq4Xs)); + assert_eq!( + plan([Iq4Xs, Iq4Xs, Iq4Xs]).unwrap().gate_up, + split(Iq4Xs, Iq4Xs) + ); assert_eq!( plan([Mxfp8, Mxfp8, Mxfp8]).unwrap().gate_up, GateUp::Site { diff --git a/crates/gpu-deepseek41/src/shared_routed.rs b/crates/gpu-deepseek41/src/shared_routed.rs index d3eb066d7..29bfb4742 100644 --- a/crates/gpu-deepseek41/src/shared_routed.rs +++ b/crates/gpu-deepseek41/src/shared_routed.rs @@ -1189,7 +1189,8 @@ fn no_batch() -> GpuError { mod tests { use super::SharedWide; use crate::block::{ - Block, BlockIo, BlockKernels, BlockPlan, BlockScratch, KBlockKernels, enqueue_block_cols_act, + Block, BlockIo, BlockKernels, BlockPlan, BlockScratch, KBlockKernels, + enqueue_block_cols_act, }; use crate::experts::ExpertKernels; use bloomery_gpu::Gpu; diff --git a/crates/gpu-gates/src/act_rule.rs b/crates/gpu-gates/src/act_rule.rs index b819959ae..589e235fc 100644 --- a/crates/gpu-gates/src/act_rule.rs +++ b/crates/gpu-gates/src/act_rule.rs @@ -982,7 +982,11 @@ mod site_tests { let (q32, q128, q256) = (Some(32), Some(Q8_1_BLOCK), Some(QK_K)); for arm in [Arm::Gemv, Arm::Gemm, Arm::Expert] { assert_eq!(site_rel(GgmlType::Q3_K, arm), q(q256, q128), "Q3_K {arm:?}"); - assert_eq!(site_rel(GgmlType::IQ4_XS, arm), q(q256, q128), "IQ4_XS {arm:?}"); + assert_eq!( + site_rel(GgmlType::IQ4_XS, arm), + q(q256, q128), + "IQ4_XS {arm:?}" + ); for ty in [GgmlType::Q4_K, GgmlType::Q5_K, GgmlType::Q6_K] { assert_eq!(site_rel(ty, arm), q(q32, q128), "{ty} {arm:?}"); } @@ -1043,35 +1047,6 @@ mod site_tests { } } - /// The rounding relation follows the common site's activation forms and - /// the grouped GEMM's existing decoders. - #[test] - fn site_rel_reads_the_forms_the_launches_name() { - use gguf::card::{SiteAct, SiteEntry}; - for entry in [ - SiteEntry::Q3k, - SiteEntry::Q4k, - SiteEntry::Q5k, - SiteEntry::Q6k, - SiteEntry::Iq4xs, - ] { - assert_eq!(entry.reads(), SiteAct::Q8x128); - } - assert_eq!(SiteEntry::Q8_0.reads(), SiteAct::Q8x32); - assert_eq!(SiteEntry::F32.reads(), SiteAct::F32); - assert_eq!(SiteEntry::Q5_0.reads(), SiteAct::Q8Blocks32); - let gpu = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../gpu/src"); - let read = |file: &str| { - std::fs::read_to_string(gpu.join(file)) - .unwrap_or_else(|e| panic!("crates/gpu/src/{file}: {e}")) - }; - assert!(read("gemm/mod.rs").contains("(Q8_0, IQ4_NL, Q5_1), over [`GemmAct32`]")); - let grouped = read("gemm/grouped.rs"); - for variant in ["Iq3Xxs", "Iq4Xs"] { - assert!(grouped.contains(variant), "the grouped GEMM no longer decodes {variant}"); - } - } - /// ik's side of the table is the transcription this module already holds for the K-quants /// ([`Rule::of`]), and so is ours for Q3_K and Q4_K; Q5_K's ours differs from it by design (the /// V4.1 gemv reads f32, the bodies' site reads q8_1). diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index 0e5e7696f..3fe55f491 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -1765,7 +1765,13 @@ mod tests { limit: 7.0, }; let launch = |ty, rule| swiglu_launch((ty, "down"), rule); - for ty in [SiteTy::Q3K, SiteTy::Q4K, SiteTy::Q5K, SiteTy::Q6K, SiteTy::Iq4Xs] { + for ty in [ + SiteTy::Q3K, + SiteTy::Q4K, + SiteTy::Q5K, + SiteTy::Q6K, + SiteTy::Iq4Xs, + ] { for rule in [Act::SiluMul, clamp, oai] { assert_eq!(launch(ty, rule).ok(), Some(SwigluLaunch::Q128), "{ty}"); } @@ -1842,7 +1848,12 @@ mod tests { (SiteTy::Mxfp8, GgmlType::MXFP8, "MXFP8", None), (SiteTy::Fp8B128, GgmlType::FP8_B128, "FP8_B128", None), (SiteTy::Bf16, GgmlType::BF16, "BF16", None), - (SiteTy::Iq4Xs, GgmlType::IQ4_XS, "IQ4_XS", Some(GemmWeight::Iq4Xs)), + ( + SiteTy::Iq4Xs, + GgmlType::IQ4_XS, + "IQ4_XS", + Some(GemmWeight::Iq4Xs), + ), ]; assert_eq!(SiteTy::ALL.len(), expected.len()); for (&s, (want, ty, label, grouped)) in SiteTy::ALL.iter().zip(expected) { From 0b055bb378126f7f97de1ab636c00dcc06f9e82b Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:03:30 +0900 Subject: [PATCH 04/14] Document source-specific ordinary IQ4 hardware proof --- docs/research/flint-attach.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index f58921e84..c73e7787d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,8 +2,8 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. The current tree additionally has a **header-only reader**; it still has no Flint -body, app session or serve seat. No GPU, inference, performance or weight-content validation is claimed. +that snapshot. This component tree has a header-only reader and the ordinary IQ4_XS shared site, +not a Flint body, app session or serve seat. Primitive GPU evidence below is not model admission. ## Host reader status @@ -80,14 +80,27 @@ those same sites. IQ4_XS is not relabeled as a routed expert or requantized. the ordinary `SharedExpert` layout, unequal matrix widths, one/eight-column block execution and the wide-prompt transition/captured replay. Wide scratch holds only the q8_1 input and down forms these sites read. Its nonnegative constant-weight -case derives a scalar floating-operation bound; it is not a whole-model accuracy -pin. Exact-head hardware execution is required before claiming arithmetic or -capture/replay support. +case derives a scalar floating-operation bound; it is not a whole-model accuracy pin. Program-specific admission remains unchanged: in particular the legacy Qwen projection/shared dispatch does not gain this site automatically. This common site contract adds no model admission or kernel. +On 2026-10-26, source `203f5125a3c48a5b700c6f43318cfcb9c77ff169` executed the owning +DS41 library on Hikari GPU0 (RTX3090 cc8.6, driver595.91.07):38 passed,0 failed,0 ignored, +28.61s, including the named IQ4 consumer above. Rust1.99.1 / pinned cuda-oxide9df9e7ce +with repository patch31ab6a1d3f58 and LLVM21.1.8 were used. +Its canonical comparison to main15778e2c covered4bins/8carriers/1657entries: all identical, +non-JIT resource rows equal; no JIT-resource comparison is claimed. +All-target native Clippy/formatting and current-head GitHub T0/lint passed. +The original execution request5a907e86-7ac6-4219-a3b6-e3cf1de84e03 exited0 but credential +cleanup remains unconfirmed; test success does not certify credential revocation. +Raw logs and canonical manifests remain in the contribution workspace's +`flint-closure-20261026` and its matching remote evidence directory. GPU1 was not stopped. +Later documentation-only commits are not a new GPU execution. Wider affected real-model, +placement and residency coverage remains explicit for maintainer review; #782 is review-ready, +not merged, and this does not establish a Flint oracle, physical cc12.1, ROCm or performance. + ## 1. Sources and observed artifact From 5c1c67a7d990c404d7dc700b0c9ee580f5da653a Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:52:05 +0900 Subject: [PATCH 05/14] Preserve IQ4 pin provenance and meaningful refusal diagnostics --- crates/gpu-deepseek41/src/block.rs | 10 +++++++++- crates/gpu/src/site.rs | 13 +++++++++++-- crates/placement/src/kernels.rs | 10 ++++++++-- docs/research/flint-attach.md | 17 ++++++++--------- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/crates/gpu-deepseek41/src/block.rs b/crates/gpu-deepseek41/src/block.rs index d35f5b674..750092b49 100644 --- a/crates/gpu-deepseek41/src/block.rs +++ b/crates/gpu-deepseek41/src/block.rs @@ -711,7 +711,15 @@ mod tests { [Mxfp8, Q8_0, Mxfp8], [Q4K, Mxfp8, Q4K], ] { - assert!(matches!(plan(types), Err(GpuError::Shape { .. }))); + let Err(GpuError::Shape { what, detail }) = plan(types) else { + panic!( + "{types:?}: incompatible activation forms were not refused as a shape error" + ); + }; + assert_eq!(what, WHAT); + for ty in types { + assert!(detail.contains(&ty.to_string()), "{types:?}: {detail}"); + } } } diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index 3fe55f491..b8612ccaf 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -1093,7 +1093,8 @@ pub fn gemm( k.gemm.enqueue_gemm( stream, GemmArgs { - ty: gemm_weight(ty).ok_or(GpuError::state(WHAT, "a K-quant's GEMM type"))?, + ty: gemm_weight(ty) + .ok_or(GpuError::state(WHAT, "the selected site's GEMM type"))?, w: match src { SiteWeight::Words { w, .. } => w, _ => kq(w, WHAT, name)?, @@ -1747,7 +1748,15 @@ mod tests { ("shared", (2, 256), &[SiteTy::Q4K][..]), ("unsupported", (2, 256), SiteTy::ALL), ] { - assert!(file_site(&file, "test", name, shape, allowed).is_err()); + let Err(chain::ChainError::Shape { what, detail }) = + file_site(&file, "test", name, shape, allowed) + else { + panic!( + "{name}: invalid shape or launch admission was not refused as a shape error" + ); + }; + assert_eq!(what, "test"); + assert!(detail.contains(name), "{name}: {detail}"); } } diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index a54404d49..28be171a7 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -634,7 +634,13 @@ mod tests { ); } - // PIN(2026-10-27): IQ4_XS adds the existing selected GEMV as an ordinary site. + // PIN(2026-10-16): the dense pairs gain NVFP4 (`nvfp4_gemv` through + // `site::kgemv`); every other pair is unchanged. + // PIN(2026-10-19): and bloomery's own MXFP8 (`mxfp8_gemv` through + // `site::gemv`); every other pair is unchanged. + // PIN(2026-10-20): and bloomery's own FP8_B128 (`fp8_b128_gemv` through + // `site::gemv`); BF16 also uses `bf16_gemv` and `bf16_tile_gemm` over its words. + // PIN(2026-10-26): IQ4_XS adds the existing selected GEMV as an ordinary site (UTC). #[test] fn dense_admits_exactly_the_pinned_pairs() { assert_eq!( @@ -743,7 +749,7 @@ mod tests { // columns); every other pair is unchanged. // PIN(2026-10-25): IQ2_XS, IQ1_S, IQ2_S and IQ1_M load as the file's // blocks, as IQ2_XXS does; every other pair is unchanged. - // PIN(2026-10-27): ordinary IQ4_XS matrices retain their file blocks. + // PIN(2026-10-26): ordinary IQ4_XS matrices retain their file blocks (UTC). #[test] fn the_card_layouts_are_the_pinned_pairs() { assert_eq!( diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index c73e7787d..23c2e88c4 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -80,7 +80,9 @@ those same sites. IQ4_XS is not relabeled as a routed expert or requantized. the ordinary `SharedExpert` layout, unequal matrix widths, one/eight-column block execution and the wide-prompt transition/captured replay. Wide scratch holds only the q8_1 input and down forms these sites read. Its nonnegative constant-weight -case derives a scalar floating-operation bound; it is not a whole-model accuracy pin. +case derives a scalar floating-operation bound; uniform codes do not detect nibble +decode ordering. That remains a separate IQ4 kernel-gate obligation, not a +whole-model accuracy pin or a claim made by this wrapper fixture. Program-specific admission remains unchanged: in particular the legacy Qwen projection/shared dispatch does not gain this site automatically. This common @@ -92,14 +94,11 @@ DS41 library on Hikari GPU0 (RTX3090 cc8.6, driver595.91.07):38 passed,0 failed, with repository patch31ab6a1d3f58 and LLVM21.1.8 were used. Its canonical comparison to main15778e2c covered4bins/8carriers/1657entries: all identical, non-JIT resource rows equal; no JIT-resource comparison is claimed. -All-target native Clippy/formatting and current-head GitHub T0/lint passed. -The original execution request5a907e86-7ac6-4219-a3b6-e3cf1de84e03 exited0 but credential -cleanup remains unconfirmed; test success does not certify credential revocation. -Raw logs and canonical manifests remain in the contribution workspace's -`flint-closure-20261026` and its matching remote evidence directory. GPU1 was not stopped. -Later documentation-only commits are not a new GPU execution. Wider affected real-model, -placement and residency coverage remains explicit for maintainer review; #782 is review-ready, -not merged, and this does not establish a Flint oracle, physical cc12.1, ROCm or performance. +All-target native Clippy/formatting and source203f5125 GitHub T0/lint passed. +These are source-specific historical results, not another hardware execution of +later diagnostic, test or documentation changes. Wider affected real-model, +placement and residency coverage remains explicit; this does not establish a +Flint oracle, physical cc12.1, ROCm or performance. From 7a7cfdc783cc37281c49f7e01126b9125cca7b5b Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:37:33 +0900 Subject: [PATCH 06/14] Exercise IQ4 projected admission through typed site fixture --- crates/gpu/src/arch/qwen3moe/plan38.rs | 4 ++-- crates/gpu/src/site.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/gpu/src/arch/qwen3moe/plan38.rs b/crates/gpu/src/arch/qwen3moe/plan38.rs index 3340b2497..46714324b 100644 --- a/crates/gpu/src/arch/qwen3moe/plan38.rs +++ b/crates/gpu/src/arch/qwen3moe/plan38.rs @@ -484,7 +484,7 @@ fn gqa_fits(g: &Gqa) -> bool { /// Every layer's plan from its kind, `ple` the PLE site's layer, each weight /// checked against the geometry: type and shape. A dense projection, the /// indexer's two among them, is of a type the common site projects by name -/// (`site::PROJECTED`: the f32 rows' families and the K-quants), each +/// (`site::PROJECTED`: the f32 rows' families, the K-quants and IQ4_XS), each /// tensor's own: the walks pick its launch by the resident form. The joined /// β·α stack is of an f32 rows' family (`site::BY_NAME`). An HC matrix is one /// `HcMatrix::of` reads. The gains, the conv and the injects are F32 (the @@ -648,7 +648,7 @@ const SUPER_BLOCK: usize = 256; /// The resident weight `name`, `rows` rows of `k` values, of a type a /// by-name projection launches (`site::PROJECTED`: every dense family but -/// Q5_0, the K-quants and NVFP4 over the q8_1 rows of their input); another +/// Q5_0, the K-quants, IQ4_XS and NVFP4 over the q8_1 rows of their input); another /// type or shape is refused by name. fn projected_shape(w: &Weights, name: &str, rows: usize, k: usize) -> Result<(), GpuError> { let ty = dense::resident_site(w, WHAT, name, (rows, k), &dense::PROJECTED)?; diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index a1beebdef..b2a2923b4 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -2101,7 +2101,7 @@ mod tests { ); } assert_eq!( - file_site(&file, "test", "shared", (2, 256), SiteTy::ALL).ok(), + file_site(&file, "test", "shared", (2, 256), &super::PROJECTED).ok(), Some(SiteTy::Iq4Xs) ); for (name, shape, allowed) in [ From b0fc05e5349330927ceb33ce6f30e584f25ecf4d Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:45:53 +0900 Subject: [PATCH 07/14] Keep projected-file admission coverage without metadata-copy guard --- crates/gpu/src/site.rs | 68 ------------------------------------------ 1 file changed, 68 deletions(-) diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index b2a2923b4..f2449c320 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -2027,12 +2027,10 @@ mod tests { Form, Order, SiteTy, SwigluLaunch, WideActs, WideForms, by_name_extra, file_site, gemm_weight, kgemv_order, swiglu_launch, }; - use crate::gemm::GemmWeight; use crate::kquant::Act; use gguf::Split; use gguf::quant::GgmlType; use gguf::write::{Layout, TensorDecl}; - use model::placement::kernels; /// A by-name projection's extra launches follow the type: none for a site /// that reads the f32 rows or the q8 blocks of 32, the quantization for a @@ -2186,72 +2184,6 @@ mod tests { } } - /// Every site family is the common table's dense row, with its form and wide launch. - #[test] - fn every_family_is_the_tables_dense_row_of_one_type() { - let own = [ - GgmlType::F8E4M3_ROWS, - GgmlType::MXFP8, - GgmlType::NVFP4_ROWS, - GgmlType::FP8_B128, - ]; - for ty in gguf::QROWS.iter().map(|r| r.ty).chain(own) { - let family = SiteTy::of_ggml(ty); - if ty == GgmlType::F16 { - assert_eq!(family, Some(SiteTy::F32)); - assert_eq!(kernels::dense(ty), None); - continue; - } - assert_eq!(family.is_some(), kernels::dense(ty).is_some(), "{ty}"); - if let Some(s) = family { - assert_eq!(s.ggml(), ty, "{s}"); - } - } - let expected = [ - (SiteTy::Q3K, GgmlType::Q3_K, "Q3_K", Some(GemmWeight::Q3K)), - (SiteTy::Q4K, GgmlType::Q4_K, "Q4_K", Some(GemmWeight::Q4K)), - (SiteTy::Q5K, GgmlType::Q5_K, "Q5_K", Some(GemmWeight::Q5K)), - (SiteTy::Q6K, GgmlType::Q6_K, "Q6_K", Some(GemmWeight::Q6K)), - (SiteTy::Q5_0, GgmlType::Q5_0, "Q5_0", None), - (SiteTy::Q8_0, GgmlType::Q8_0, "Q8_0", None), - (SiteTy::F32, GgmlType::F32, "F32", None), - (SiteTy::Nvfp4, GgmlType::NVFP4, "NVFP4", None), - (SiteTy::Mxfp8, GgmlType::MXFP8, "MXFP8", None), - (SiteTy::Fp8B128, GgmlType::FP8_B128, "FP8_B128", None), - (SiteTy::Bf16, GgmlType::BF16, "BF16", None), - ( - SiteTy::Iq4Xs, - GgmlType::IQ4_XS, - "IQ4_XS", - Some(GemmWeight::Iq4Xs), - ), - ]; - assert_eq!(SiteTy::ALL.len(), expected.len()); - for (&s, (want, ty, label, grouped)) in SiteTy::ALL.iter().zip(expected) { - assert_eq!(s, want); - assert_eq!(s.ggml(), ty); - assert_eq!(s.to_string(), label); - assert_eq!(gemm_weight(s), grouped); - assert_eq!(kgemv_order(s).is_some(), s.reads() == Form::Q8x128); - let by_name = model::placement::accepts::dense_by_name(ty); - assert_eq!(super::BY_NAME.contains(&s), by_name.is_some(), "{s}"); - if let Some(entry) = by_name { - assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); - } - let projected = model::placement::accepts::dense_projected(ty); - assert_eq!(super::PROJECTED.contains(&s), projected.is_some(), "{s}"); - if let Some(entry) = projected { - assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); - } - } - assert_eq!(SiteTy::Q5_0.reads(), Form::Q8Blocks32); - assert_eq!(kgemv_order(SiteTy::Q5_0), None); - assert_eq!(SiteTy::Nvfp4.reads(), Form::Q8x128); - assert_eq!(kgemv_order(SiteTy::Nvfp4), Some(Order::TokenMajor)); - assert_eq!(SiteTy::Mxfp8.reads(), Form::F32); - assert_eq!(kgemv_order(SiteTy::Iq4Xs), Some(Order::TokenMajor)); - } - #[test] fn a_bf16_site_reads_the_f32_rows_and_is_a_gemv() { assert_eq!(SiteTy::of_ggml(GgmlType::BF16), Some(SiteTy::Bf16)); From 6a46aa1ad6282e818f0f1c1d488bc5b6c68d854d Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:05:49 +0900 Subject: [PATCH 08/14] Document projected-file execution and remove obsolete test import --- crates/gpu/src/site.rs | 2 +- docs/research/flint-attach.md | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index f2449c320..29c2287ae 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -2024,7 +2024,7 @@ fn swiglu( #[cfg(test)] mod tests { use super::{ - Form, Order, SiteTy, SwigluLaunch, WideActs, WideForms, by_name_extra, file_site, + Form, SiteTy, SwigluLaunch, WideActs, WideForms, by_name_extra, file_site, gemm_weight, kgemv_order, swiglu_launch, }; use crate::kquant::Act; diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 23c2e88c4..9eb58065d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -84,9 +84,18 @@ case derives a scalar floating-operation bound; uniform codes do not detect nibb decode ordering. That remains a separate IQ4 kernel-gate obligation, not a whole-model accuracy pin or a claim made by this wrapper fixture. -Program-specific admission remains unchanged: in particular the legacy Qwen -projection/shared dispatch does not gain this site automatically. This common -site contract adds no model admission or kernel. +The pinned current-main merge (`cb8d92ebddd1ce1a325a15450c06a01f57ecf88c`) also +reconciles its existing Qwen3.8 projected-site consumer: `site::PROJECTED` includes +IQ4_XS, and selected dispatch uses the supplied stream. Projected attention/shared +sites are admitted; the K=640 shared down and unsupported types/shapes remain refused. +This reuses the existing program and kernel, not a new model implementation. + +At source `b0fc05e5349330927ceb33ce6f30e584f25ecf4d`, the typed GGUF +`iq4_sites_keep_shape_and_launch_admission_separate` consumer passed1 case, +0 failed/ignored,430 filtered. Its successful arm uses `PROJECTED`, not merely +`ALL`; this checks actual file admission rather than copying a metadata registry. +The case is metadata-only and creates no CUDA context; oxide owns its linking. +It is not fresh NVIDIA execution of the current-main stream/launch paths. On 2026-10-26, source `203f5125a3c48a5b700c6f43318cfcb9c77ff169` executed the owning DS41 library on Hikari GPU0 (RTX3090 cc8.6, driver595.91.07):38 passed,0 failed,0 ignored, From 3bd0c0411a475aaa6e7e3340bbe592a5a80f1d0a Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:14:52 +0900 Subject: [PATCH 09/14] Format the cleaned site test imports --- crates/gpu/src/site.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index 29c2287ae..3f6a673ae 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -2024,8 +2024,8 @@ fn swiglu( #[cfg(test)] mod tests { use super::{ - Form, SiteTy, SwigluLaunch, WideActs, WideForms, by_name_extra, file_site, - gemm_weight, kgemv_order, swiglu_launch, + Form, SiteTy, SwigluLaunch, WideActs, WideForms, by_name_extra, file_site, gemm_weight, + kgemv_order, swiglu_launch, }; use crate::kquant::Act; use gguf::Split; From cd034285a0354eba35d82c82b50905a365b8f113 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:56:08 +0900 Subject: [PATCH 10/14] Document upstream split scratch compatibility and host proof --- docs/research/flint-attach.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 9eb58065d..df975eb14 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -97,6 +97,16 @@ At source `b0fc05e5349330927ceb33ce6f30e584f25ecf4d`, the typed GGUF The case is metadata-only and creates no CUDA context; oxide owns its linking. It is not fresh NVIDIA execution of the current-main stream/launch paths. +Pinned upstream `95a483fc242d00dec7987efe4a05e24b5f651c1f` subsequently adds the +split gate/up path that materializes f32 SwiGLU for a down that reads f32. +The obsolete split-to-f32 refusal is removed, not re-pinned; mismatched gate/up +forms still refuse by semantic Shape/caller/type data. IQ4_XS follows the shared +q8x128 predicate in both row dispatch and `writes_h`, so the GLM prompt consumer +chooses the buffer that is actually written instead of allocating/reading a fake f32 row. +At source `5abe1ae66b651446938e334331c06683cef793a4`, the actual block metadata +consumers passed5 cases,0 failed/ignored,34 filtered, including both IQ4 down forms. +These host cases are not execution evidence for the remaining GPU cases. + On 2026-10-26, source `203f5125a3c48a5b700c6f43318cfcb9c77ff169` executed the owning DS41 library on Hikari GPU0 (RTX3090 cc8.6, driver595.91.07):38 passed,0 failed,0 ignored, 28.61s, including the named IQ4 consumer above. Rust1.99.1 / pinned cuda-oxide9df9e7ce From 5dc4f41a33699652e29d142dfd2ae2dcd23c8d92 Mon Sep 17 00:00:00 2001 From: midagedev Date: Tue, 27 Oct 2026 15:22:58 +0900 Subject: [PATCH 11/14] site, act_rule, coverage: restore the site tests and their PIN with IQ4_XS as a site family; the Qwen3.8 dense pins read IQ4_XS through q8_1 Co-Authored-By: Claude Opus 5.5 --- crates/gpu-gates/src/act_rule.rs | 40 ++++++++++ crates/gpu/src/site.rs | 125 +++++++++++++++++++++++++++--- crates/model/src/arch/coverage.rs | 7 ++ 3 files changed, 162 insertions(+), 10 deletions(-) diff --git a/crates/gpu-gates/src/act_rule.rs b/crates/gpu-gates/src/act_rule.rs index e3f42e70a..969255e06 100644 --- a/crates/gpu-gates/src/act_rule.rs +++ b/crates/gpu-gates/src/act_rule.rs @@ -1082,6 +1082,46 @@ mod site_tests { assert_eq!(site_rel(GgmlType::BF16, Arm::Expert), None); } + /// Ours is the form the launches read, as their sources say it: the text of the kernel + /// table's `SiteEntry::reads` (which `site.rs`'s `SiteTy::reads` returns) and of the 32-value + /// GEMM family's module doc, so a launch that moves to another form is red here until the + /// table follows it. + #[test] + fn site_rel_reads_the_forms_the_launches_name() { + let gpu = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../gpu/src"); + let read = |file: &str| { + std::fs::read_to_string(gpu.join(file)) + .unwrap_or_else(|e| panic!("crates/gpu/src/{file}: {e}")) + }; + let (site, gemm, grouped) = ( + read("../../gguf/src/card.rs"), + read("gemm/mod.rs"), + read("gemm/grouped.rs"), + ); + // The contract is the arm's text, not rustfmt's wrapping of its or-pattern. + let site = site.split_whitespace().collect::>().join(" "); + for line in [ + // PIN(2026-10-27): IQ4_XS reads the 128-value q8_1 form with the K-quants (`SiteEntry::Iq4xs`), as `site_rel` says. + "SiteEntry::Q3k | SiteEntry::Q4k | SiteEntry::Q5k | SiteEntry::Q6k | SiteEntry::Iq4xs => SiteAct::Q8x128", + "SiteEntry::Q8_0 => SiteAct::Q8x32", + "SiteEntry::F32 => SiteAct::F32", + "SiteEntry::Q5_0 => SiteAct::Q8Blocks32", + "q8_1 blocks of 128 values", + ] { + assert!(site.contains(line), "kernels.rs no longer says {line:?}"); + } + assert!( + gemm.contains("(Q8_0, IQ4_NL, Q5_1), over [`GemmAct32`]"), + "gemm/mod.rs no longer names the 32-value family" + ); + for variant in ["Iq3Xxs", "Iq4Xs"] { + assert!( + grouped.contains(variant), + "the grouped GEMM no longer decodes {variant}" + ); + } + } + /// ik's side of the table is the transcription this module already holds for the K-quants /// ([`Rule::of`]), and so is ours for Q3_K and Q4_K; Q5_K's ours differs from it by design (the /// V4.1 gemv reads f32, the bodies' site reads q8_1). diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index 3f6a673ae..d2c69b89c 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -2056,8 +2056,16 @@ mod tests { } } + /// A site's type is read from the file's header alone: a Q8_0 matrix of + /// two rows of 64 is read as Q8_0 where Q8_0 is launched, and refused by + /// name where only K-quants are; a Q3_K, a Q5_K, a Q5_0, an IQ4_XS, an MXFP8 + /// (bloomery's id, sized by the split) and a BF16 one are read as theirs; an + /// IQ3_S one (a type no site launch reads), a matrix of another shape and a + /// type outside the allowed list are refused by name. + /// + /// PIN(2026-10-27): the refused type with no launch was IQ4_XS; it is a site family now (`SiteTy::Iq4Xs`), so IQ3_S stands in and IQ4_XS joins the read types. #[test] - fn iq4_sites_keep_shape_and_launch_admission_separate() { + fn a_site_type_comes_from_the_header_and_a_type_with_no_launch_is_refused() { let decl = |name: &str, ty: GgmlType, bytes| TensorDecl { name: name.to_owned(), dims: vec![256, 2], @@ -2102,21 +2110,36 @@ mod tests { file_site(&file, "test", "shared", (2, 256), &super::PROJECTED).ok(), Some(SiteTy::Iq4Xs) ); - for (name, shape, allowed) in [ - ("shared", (1, 256), SiteTy::ALL), - ("shared", (2, 512), SiteTy::ALL), - ("shared", (2, 256), &[SiteTy::Q4K][..]), - ("unsupported", (2, 256), SiteTy::ALL), + for (name, shape, allowed, want) in [ + ( + "q8", + (2, 64), + &[SiteTy::Q4K, SiteTy::Q6K][..], + "q8 is q8_0 in the file", + ), + ( + "shared", + (2, 256), + &[SiteTy::Q4K][..], + "shared is iq4_xs in the file", + ), + ( + "unsupported", + (2, 256), + SiteTy::ALL, + "unsupported is iq3_s in the file", + ), + ("q8", (4, 64), SiteTy::ALL, "it takes 4 rows of 64"), + ("shared", (1, 256), SiteTy::ALL, "it takes 1 rows of 256"), + ("shared", (2, 512), SiteTy::ALL, "it takes 2 rows of 512"), ] { let Err(chain::ChainError::Shape { what, detail }) = file_site(&file, "test", name, shape, allowed) else { - panic!( - "{name}: invalid shape or launch admission was not refused as a shape error" - ); + panic!("{name} {shape:?}: not refused as a shape error"); }; assert_eq!(what, "test"); - assert!(detail.contains(name), "{name}: {detail}"); + assert!(detail.contains(want), "{name}: {detail}"); } } @@ -2184,6 +2207,88 @@ mod tests { } } +<<<<<<< ours +======= + /// Every family is the kernel table's dense row of one file type: a type + /// has a family exactly where the table names a dense entry for it (F16, + /// which loads widened to f32, is the f32 site of its widened rows), the + /// family maps back to that type and is named by it in upper case, its + /// grouped GEMM is the one the table names for the type, it is a + /// `kgemv` exactly where it reads the q8_1 rows of 128 values, it + /// launches by name (`BY_NAME`) exactly where it reads the f32 rows or + /// the q8 blocks of 32, and by projection (`PROJECTED`) exactly where the + /// table names a dense entry but Q5_0's. + #[test] + fn every_family_is_the_tables_dense_row_of_one_type() { + // PIN(2026-10-21): consolidate the three site pins, preserving their assertions. + let own = [ + GgmlType::F8E4M3_ROWS, + GgmlType::MXFP8, + GgmlType::NVFP4_ROWS, + GgmlType::FP8_B128, + ]; + for ty in gguf::QROWS.iter().map(|r| r.ty).chain(own) { + let family = SiteTy::of_ggml(ty); + if ty == GgmlType::F16 { + assert_eq!( + family, + Some(SiteTy::F32), + "{ty}: the f32 site of its widened rows" + ); + assert_eq!(kernels::dense(ty), None, "{ty}"); + continue; + } + assert_eq!(family.is_some(), kernels::dense(ty).is_some(), "{ty}"); + if let Some(s) = family { + assert_eq!(s.ggml(), ty, "{s}"); + } + } + let expected = [ + (SiteTy::Q3K, GgmlType::Q3_K, "Q3_K", Some(GemmWeight::Q3K)), + (SiteTy::Q4K, GgmlType::Q4_K, "Q4_K", Some(GemmWeight::Q4K)), + (SiteTy::Q5K, GgmlType::Q5_K, "Q5_K", Some(GemmWeight::Q5K)), + (SiteTy::Q6K, GgmlType::Q6_K, "Q6_K", Some(GemmWeight::Q6K)), + (SiteTy::Q5_0, GgmlType::Q5_0, "Q5_0", None), + (SiteTy::Q8_0, GgmlType::Q8_0, "Q8_0", None), + (SiteTy::F32, GgmlType::F32, "F32", None), + (SiteTy::Nvfp4, GgmlType::NVFP4, "NVFP4", None), + (SiteTy::Mxfp8, GgmlType::MXFP8, "MXFP8", None), + (SiteTy::Fp8B128, GgmlType::FP8_B128, "FP8_B128", None), + (SiteTy::Bf16, GgmlType::BF16, "BF16", None), + ( + SiteTy::Iq4Xs, + GgmlType::IQ4_XS, + "IQ4_XS", + Some(GemmWeight::Iq4Xs), + ), + ]; + assert_eq!(SiteTy::ALL.len(), expected.len()); + for (&s, (want, ty, label, grouped)) in SiteTy::ALL.iter().zip(expected) { + assert_eq!(s, want); + assert_eq!(s.ggml(), ty); + assert_eq!(s.to_string(), label); + assert_eq!(gemm_weight(s), grouped); + assert_eq!(kgemv_order(s).is_some(), s.reads() == Form::Q8x128); + let by_name = model::placement::accepts::dense_by_name(ty); + assert_eq!(super::BY_NAME.contains(&s), by_name.is_some(), "{s}"); + if let Some(entry) = by_name { + assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); + } + let projected = model::placement::accepts::dense_projected(ty); + assert_eq!(super::PROJECTED.contains(&s), projected.is_some(), "{s}"); + if let Some(entry) = projected { + assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); + } + } + assert_eq!(SiteTy::Q5_0.reads(), Form::Q8Blocks32); + assert_eq!(kgemv_order(SiteTy::Q5_0), None); + assert_eq!(SiteTy::Nvfp4.reads(), Form::Q8x128); + assert_eq!(kgemv_order(SiteTy::Nvfp4), Some(Order::TokenMajor)); + assert_eq!(SiteTy::Mxfp8.reads(), Form::F32); + assert_eq!(kgemv_order(SiteTy::Iq4Xs), Some(Order::TokenMajor)); + } + +>>>>>>> theirs #[test] fn a_bf16_site_reads_the_f32_rows_and_is_a_gemv() { assert_eq!(SiteTy::of_ggml(GgmlType::BF16), Some(SiteTy::Bf16)); diff --git a/crates/model/src/arch/coverage.rs b/crates/model/src/arch/coverage.rs index 9f69c7fbf..d2f6c6240 100644 --- a/crates/model/src/arch/coverage.rs +++ b/crates/model/src/arch/coverage.rs @@ -3602,8 +3602,15 @@ mod tests { // PIN(2026-10-26): the K-quants (Q3_K to Q6_K) joined the dense sites' set: both walks project a site by name // (`site::ByName`), the q8_1 quantization and the K-quant gemv by chunks on the ubatch walk. The shared expert's // down keeps the f32-row types: K = 640 is not whole super-blocks. +<<<<<<< ours // IQ4_XS sites likewise read q8_1 through the selected one-expert dispatch; the shared down // still refuses them because its K = 640 is not whole super-blocks. +======= + // PIN(2026-10-27): IQ4_XS joined the dense sites' set: the kernel table names it a dense entry (`SiteEntry::Iq4xs`) + // that reads the q8_1 blocks of 128 like the K-quants, so `dense_projected` keeps it and `site::PROJECTED` holds it. + // Derivation: the projections by name run it, and the shared expert's down (K = 640, `dense_by_name` omits it) is + // refused as a K-quant's is. +>>>>>>> theirs #[test] fn the_qwen38_dense_pins_read_the_projected_sites() { let q8_1 = [ From 1c8dbbefe40987e52137e95eeddb067856ef3fc7 Mon Sep 17 00:00:00 2001 From: midagedev Date: Tue, 27 Oct 2026 15:34:30 +0900 Subject: [PATCH 12/14] site, coverage: drop the conflict markers the fix commit carried; #782's own coverage pin and site tests stand Co-Authored-By: Claude Opus 5.5 --- crates/gpu/src/site.rs | 82 ------------------------------- crates/model/src/arch/coverage.rs | 7 --- 2 files changed, 89 deletions(-) diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index d2c69b89c..e90824d63 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -2207,88 +2207,6 @@ mod tests { } } -<<<<<<< ours -======= - /// Every family is the kernel table's dense row of one file type: a type - /// has a family exactly where the table names a dense entry for it (F16, - /// which loads widened to f32, is the f32 site of its widened rows), the - /// family maps back to that type and is named by it in upper case, its - /// grouped GEMM is the one the table names for the type, it is a - /// `kgemv` exactly where it reads the q8_1 rows of 128 values, it - /// launches by name (`BY_NAME`) exactly where it reads the f32 rows or - /// the q8 blocks of 32, and by projection (`PROJECTED`) exactly where the - /// table names a dense entry but Q5_0's. - #[test] - fn every_family_is_the_tables_dense_row_of_one_type() { - // PIN(2026-10-21): consolidate the three site pins, preserving their assertions. - let own = [ - GgmlType::F8E4M3_ROWS, - GgmlType::MXFP8, - GgmlType::NVFP4_ROWS, - GgmlType::FP8_B128, - ]; - for ty in gguf::QROWS.iter().map(|r| r.ty).chain(own) { - let family = SiteTy::of_ggml(ty); - if ty == GgmlType::F16 { - assert_eq!( - family, - Some(SiteTy::F32), - "{ty}: the f32 site of its widened rows" - ); - assert_eq!(kernels::dense(ty), None, "{ty}"); - continue; - } - assert_eq!(family.is_some(), kernels::dense(ty).is_some(), "{ty}"); - if let Some(s) = family { - assert_eq!(s.ggml(), ty, "{s}"); - } - } - let expected = [ - (SiteTy::Q3K, GgmlType::Q3_K, "Q3_K", Some(GemmWeight::Q3K)), - (SiteTy::Q4K, GgmlType::Q4_K, "Q4_K", Some(GemmWeight::Q4K)), - (SiteTy::Q5K, GgmlType::Q5_K, "Q5_K", Some(GemmWeight::Q5K)), - (SiteTy::Q6K, GgmlType::Q6_K, "Q6_K", Some(GemmWeight::Q6K)), - (SiteTy::Q5_0, GgmlType::Q5_0, "Q5_0", None), - (SiteTy::Q8_0, GgmlType::Q8_0, "Q8_0", None), - (SiteTy::F32, GgmlType::F32, "F32", None), - (SiteTy::Nvfp4, GgmlType::NVFP4, "NVFP4", None), - (SiteTy::Mxfp8, GgmlType::MXFP8, "MXFP8", None), - (SiteTy::Fp8B128, GgmlType::FP8_B128, "FP8_B128", None), - (SiteTy::Bf16, GgmlType::BF16, "BF16", None), - ( - SiteTy::Iq4Xs, - GgmlType::IQ4_XS, - "IQ4_XS", - Some(GemmWeight::Iq4Xs), - ), - ]; - assert_eq!(SiteTy::ALL.len(), expected.len()); - for (&s, (want, ty, label, grouped)) in SiteTy::ALL.iter().zip(expected) { - assert_eq!(s, want); - assert_eq!(s.ggml(), ty); - assert_eq!(s.to_string(), label); - assert_eq!(gemm_weight(s), grouped); - assert_eq!(kgemv_order(s).is_some(), s.reads() == Form::Q8x128); - let by_name = model::placement::accepts::dense_by_name(ty); - assert_eq!(super::BY_NAME.contains(&s), by_name.is_some(), "{s}"); - if let Some(entry) = by_name { - assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); - } - let projected = model::placement::accepts::dense_projected(ty); - assert_eq!(super::PROJECTED.contains(&s), projected.is_some(), "{s}"); - if let Some(entry) = projected { - assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); - } - } - assert_eq!(SiteTy::Q5_0.reads(), Form::Q8Blocks32); - assert_eq!(kgemv_order(SiteTy::Q5_0), None); - assert_eq!(SiteTy::Nvfp4.reads(), Form::Q8x128); - assert_eq!(kgemv_order(SiteTy::Nvfp4), Some(Order::TokenMajor)); - assert_eq!(SiteTy::Mxfp8.reads(), Form::F32); - assert_eq!(kgemv_order(SiteTy::Iq4Xs), Some(Order::TokenMajor)); - } - ->>>>>>> theirs #[test] fn a_bf16_site_reads_the_f32_rows_and_is_a_gemv() { assert_eq!(SiteTy::of_ggml(GgmlType::BF16), Some(SiteTy::Bf16)); diff --git a/crates/model/src/arch/coverage.rs b/crates/model/src/arch/coverage.rs index d2f6c6240..9f69c7fbf 100644 --- a/crates/model/src/arch/coverage.rs +++ b/crates/model/src/arch/coverage.rs @@ -3602,15 +3602,8 @@ mod tests { // PIN(2026-10-26): the K-quants (Q3_K to Q6_K) joined the dense sites' set: both walks project a site by name // (`site::ByName`), the q8_1 quantization and the K-quant gemv by chunks on the ubatch walk. The shared expert's // down keeps the f32-row types: K = 640 is not whole super-blocks. -<<<<<<< ours // IQ4_XS sites likewise read q8_1 through the selected one-expert dispatch; the shared down // still refuses them because its K = 640 is not whole super-blocks. -======= - // PIN(2026-10-27): IQ4_XS joined the dense sites' set: the kernel table names it a dense entry (`SiteEntry::Iq4xs`) - // that reads the q8_1 blocks of 128 like the K-quants, so `dense_projected` keeps it and `site::PROJECTED` holds it. - // Derivation: the projections by name run it, and the shared expert's down (K = 640, `dense_by_name` omits it) is - // refused as a K-quant's is. ->>>>>>> theirs #[test] fn the_qwen38_dense_pins_read_the_projected_sites() { let q8_1 = [ From 1a76bcd4aa4b92eef75ac29c849ec4805bb51eef Mon Sep 17 00:00:00 2001 From: midagedev Date: Tue, 27 Oct 2026 16:07:44 +0900 Subject: [PATCH 13/14] flint-attach: the site test's contract under its current name, no run history Co-Authored-By: Claude Opus 5.5 --- docs/research/flint-attach.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index e603f58c6..5a4efc3c7 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -138,12 +138,9 @@ IQ4_XS, and selected dispatch uses the supplied stream. Projected attention/shar sites are admitted; the K=640 shared down and unsupported types/shapes remain refused. This reuses the existing program and kernel, not a new model implementation. -At source `b0fc05e5349330927ceb33ce6f30e584f25ecf4d`, the typed GGUF -`iq4_sites_keep_shape_and_launch_admission_separate` consumer passed1 case, -0 failed/ignored,430 filtered. Its successful arm uses `PROJECTED`, not merely -`ALL`; this checks actual file admission rather than copying a metadata registry. -The case is metadata-only and creates no CUDA context; oxide owns its linking. -It is not fresh NVIDIA execution of the current-main stream/launch paths. +The typed GGUF test `a_site_type_comes_from_the_header_and_a_type_with_no_launch_is_refused` +reads an IQ4_XS shared site through `PROJECTED`, not merely `ALL`. It refuses IQ3_S, another +shape and a type outside the allowed list by name. It is metadata-only and creates no CUDA context. Pinned upstream `95a483fc242d00dec7987efe4a05e24b5f651c1f` subsequently adds the split gate/up path that materializes f32 SwiGLU for a down that reads f32. From 56b15d0d2a2da2dff6979bccc482f629d5cceee4 Mon Sep 17 00:00:00 2001 From: midagedev Date: Tue, 27 Oct 2026 16:38:34 +0900 Subject: [PATCH 14/14] site, coverage: restore every_family_is_the_tables_dense_row_of_one_type with IQ4_XS's row and the refusal reasons, IQ4_XS's move dated Co-Authored-By: Claude Opus 5.5 --- crates/gpu/src/site.rs | 85 ++++++++++++++++++++++++++++++- crates/model/src/arch/coverage.rs | 31 +++++++---- crates/placement/src/accepts.rs | 2 +- 3 files changed, 106 insertions(+), 12 deletions(-) diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index e90824d63..691ee01eb 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -2024,13 +2024,15 @@ fn swiglu( #[cfg(test)] mod tests { use super::{ - Form, SiteTy, SwigluLaunch, WideActs, WideForms, by_name_extra, file_site, gemm_weight, - kgemv_order, swiglu_launch, + Form, Order, SiteTy, SwigluLaunch, WideActs, WideForms, by_name_extra, file_site, + gemm_weight, kgemv_order, swiglu_launch, }; + use crate::gemm::GemmWeight; use crate::kquant::Act; use gguf::Split; use gguf::quant::GgmlType; use gguf::write::{Layout, TensorDecl}; + use model::placement::kernels; /// A by-name projection's extra launches follow the type: none for a site /// that reads the f32 rows or the q8 blocks of 32, the quantization for a @@ -2056,6 +2058,85 @@ mod tests { } } + /// Every family is the kernel table's dense row of one file type: a type + /// has a family exactly where the table names a dense entry for it (F16, + /// which loads widened to f32, is the f32 site of its widened rows), the + /// family maps back to that type and is named by it in upper case, its + /// grouped GEMM is the one the table names for the type, it is a + /// `kgemv` exactly where it reads the q8_1 rows of 128 values, it + /// launches by name (`BY_NAME`) exactly where it reads the f32 rows or + /// the q8 blocks of 32, and by projection (`PROJECTED`) exactly where the + /// table names a dense entry but Q5_0's. + #[test] + fn every_family_is_the_tables_dense_row_of_one_type() { + // PIN(2026-10-21): consolidate the three site pins, preserving their assertions. + let own = [ + GgmlType::F8E4M3_ROWS, + GgmlType::MXFP8, + GgmlType::NVFP4_ROWS, + GgmlType::FP8_B128, + ]; + for ty in gguf::QROWS.iter().map(|r| r.ty).chain(own) { + let family = SiteTy::of_ggml(ty); + if ty == GgmlType::F16 { + assert_eq!( + family, + Some(SiteTy::F32), + "{ty}: the f32 site of its widened rows" + ); + assert_eq!(kernels::dense(ty), None, "{ty}"); + continue; + } + assert_eq!(family.is_some(), kernels::dense(ty).is_some(), "{ty}"); + if let Some(s) = family { + assert_eq!(s.ggml(), ty, "{s}"); + } + } + let expected = [ + (SiteTy::Q3K, GgmlType::Q3_K, "Q3_K", Some(GemmWeight::Q3K)), + (SiteTy::Q4K, GgmlType::Q4_K, "Q4_K", Some(GemmWeight::Q4K)), + (SiteTy::Q5K, GgmlType::Q5_K, "Q5_K", Some(GemmWeight::Q5K)), + (SiteTy::Q6K, GgmlType::Q6_K, "Q6_K", Some(GemmWeight::Q6K)), + (SiteTy::Q5_0, GgmlType::Q5_0, "Q5_0", None), + (SiteTy::Q8_0, GgmlType::Q8_0, "Q8_0", None), + (SiteTy::F32, GgmlType::F32, "F32", None), + (SiteTy::Nvfp4, GgmlType::NVFP4, "NVFP4", None), + (SiteTy::Mxfp8, GgmlType::MXFP8, "MXFP8", None), + (SiteTy::Fp8B128, GgmlType::FP8_B128, "FP8_B128", None), + (SiteTy::Bf16, GgmlType::BF16, "BF16", None), + ( + SiteTy::Iq4Xs, + GgmlType::IQ4_XS, + "IQ4_XS", + Some(GemmWeight::Iq4Xs), + ), + ]; + assert_eq!(SiteTy::ALL.len(), expected.len()); + for (&s, (want, ty, label, grouped)) in SiteTy::ALL.iter().zip(expected) { + assert_eq!(s, want); + assert_eq!(s.ggml(), ty); + assert_eq!(s.to_string(), label); + assert_eq!(gemm_weight(s), grouped); + assert_eq!(kgemv_order(s).is_some(), s.reads() == Form::Q8x128); + let by_name = model::placement::accepts::dense_by_name(ty); + assert_eq!(super::BY_NAME.contains(&s), by_name.is_some(), "{s}"); + if let Some(entry) = by_name { + assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); + } + let projected = model::placement::accepts::dense_projected(ty); + assert_eq!(super::PROJECTED.contains(&s), projected.is_some(), "{s}"); + if let Some(entry) = projected { + assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); + } + } + assert_eq!(SiteTy::Q5_0.reads(), Form::Q8Blocks32); + assert_eq!(kgemv_order(SiteTy::Q5_0), None); + assert_eq!(SiteTy::Nvfp4.reads(), Form::Q8x128); + assert_eq!(kgemv_order(SiteTy::Nvfp4), Some(Order::TokenMajor)); + assert_eq!(SiteTy::Mxfp8.reads(), Form::F32); + assert_eq!(kgemv_order(SiteTy::Iq4Xs), Some(Order::TokenMajor)); + } + /// A site's type is read from the file's header alone: a Q8_0 matrix of /// two rows of 64 is read as Q8_0 where Q8_0 is launched, and refused by /// name where only K-quants are; a Q3_K, a Q5_K, a Q5_0, an IQ4_XS, an MXFP8 diff --git a/crates/model/src/arch/coverage.rs b/crates/model/src/arch/coverage.rs index 9f69c7fbf..169f31a22 100644 --- a/crates/model/src/arch/coverage.rs +++ b/crates/model/src/arch/coverage.rs @@ -3602,8 +3602,8 @@ mod tests { // PIN(2026-10-26): the K-quants (Q3_K to Q6_K) joined the dense sites' set: both walks project a site by name // (`site::ByName`), the q8_1 quantization and the K-quant gemv by chunks on the ubatch walk. The shared expert's // down keeps the f32-row types: K = 640 is not whole super-blocks. - // IQ4_XS sites likewise read q8_1 through the selected one-expert dispatch; the shared down - // still refuses them because its K = 640 is not whole super-blocks. + // PIN(2026-10-27): IQ4_XS moved from refused to q8_1; it reads q8_1 through the selected one-expert dispatch, + // and the shared down still refuses it because its K = 640 is not whole super-blocks. #[test] fn the_qwen38_dense_pins_read_the_projected_sites() { let q8_1 = [ @@ -3627,13 +3627,21 @@ mod tests { GgmlType::IQ4_NL, GgmlType::Q4_0, ]; - for (name, role) in [ - ("attn_q.weight", Role::Attention), - ("ssm_out.weight", Role::Attention), - ("indexer.q_proj.weight", Role::Attention), - ("indexer.k_proj.weight", Role::Attention), - ("ffn_gate_shexp.weight", Role::SharedExpert), - ("ffn_up_shexp.weight", Role::SharedExpert), + for (name, role, what) in [ + ("attn_q.weight", Role::Attention, "attention matrices"), + ("ssm_out.weight", Role::Attention, "attention matrices"), + ( + "indexer.q_proj.weight", + Role::Attention, + "attention matrices", + ), + ( + "indexer.k_proj.weight", + Role::Attention, + "attention matrices", + ), + ("ffn_gate_shexp.weight", Role::SharedExpert, "shared expert"), + ("ffn_up_shexp.weight", Role::SharedExpert, "shared expert"), ] { for ty in q8_1.into_iter().chain(f32_rows) { let got = items(Program::Qwen38Body, &matrix(name, role, ty)); @@ -3642,6 +3650,7 @@ mod tests { for ty in refused { let got = items(Program::Qwen38Body, &matrix(name, role, ty)); assert_eq!(got.len(), 1, "{name} {ty}: {got:?}"); + assert!(got[0].starts_with(&format!("{ty} {what}")), "{got:?}"); } } // The shared expert's down is K = 640: the q8_1 types' rows are whole @@ -3655,6 +3664,10 @@ mod tests { for ty in q8_1 { let got = items(Program::Qwen38Body, &matrix(down, Role::SharedExpert, ty)); assert_eq!(got.len(), 1, "{down} {ty}: {got:?}"); + assert!( + got[0].starts_with(&format!("{ty} shared expert down (K = 640")), + "{got:?}" + ); } for ty in refused { let got = items(Program::Qwen38Body, &matrix(down, Role::SharedExpert, ty)); diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index 0bf14a104..5b4273f67 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -278,7 +278,7 @@ pub const fn dense_by_name(ty: GgmlType) -> Option { /// The dense sites launched by name over up to eight columns on every family /// the table's dense row names but Q5_0 (`site::ByName`): the f32-row entries -/// beside those that read q8_1 blocks of 128 (the K-quants, NVFP4), whose +/// beside those that read q8_1 blocks of 128 (the K-quants, IQ4_XS, NVFP4), whose /// activation the unit quantizes itself. Q5_0 reads 32-value blocks no /// by-name unit holds. #[must_use]