From 655b014aa7c0bfb6a66344bc756653bd69dbd9c8 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:24:29 +0900 Subject: [PATCH 1/7] gpu: support six-slot F32 tier handoff --- crates/gpu-deepseek41/src/chain/ffn/batch.rs | 215 +++++++++++++++ crates/gpu/src/host/handoff.rs | 272 +++++++++++++++---- docs/research/flint-attach.md | 24 ++ tools/unsafe-ratchet.txt | 5 +- 4 files changed, 466 insertions(+), 50 deletions(-) diff --git a/crates/gpu-deepseek41/src/chain/ffn/batch.rs b/crates/gpu-deepseek41/src/chain/ffn/batch.rs index fd4f1f741..fe5f6ed57 100644 --- a/crates/gpu-deepseek41/src/chain/ffn/batch.rs +++ b/crates/gpu-deepseek41/src/chain/ffn/batch.rs @@ -2604,3 +2604,218 @@ fn count_of(by_count: &mut [T], m: usize) -> Result<&mut T, GpuError> { detail: format!("a chunk of {m} tokens; the batch holds scratch for 1..={n}"), }) } + +#[cfg(test)] +mod tests { + use super::{CardAccTier, FfnBatchKernels}; + use bloomery_gpu::host::handoff::{Handoff, HandoffKernels}; + use bloomery_gpu::host::page::HandoffLayout; + use bloomery_gpu::host::step::HandoffTarget; + use bloomery_gpu::host::tier::{TierImageLayout, TierTarget}; + use bloomery_gpu::hybrid::HOST; + use bloomery_gpu::{FaultSite, Gpu, GpuError, Window}; + use cuda_core::DeviceBuffer; + + /// The handoff's places feed the production tier sum, not a readback + /// alone: every routed row belongs to exactly one of host, card and + /// tier, and all other copies are poisoned. Both slot entries must + /// conserve that union, including the final routed lane and partial + /// blocks, eagerly and on replay. Invalid ids must still raise. + #[test] + #[ignore = "needs a CUDA device; `just gate-gpu-ds41-lib` runs it on the box"] + fn hw_f32_tier_handoffs_conserve_six_and_eight_consumed_routes() { + const EXPERTS: usize = 16; + const ROW_OFF: usize = 3; + const LAYER: usize = 7; + let gpu = Gpu::new().expect("a GPU"); + let stream = gpu.stream(); + let handoff = HandoffKernels::load(gpu.context()).expect("handoff kernels"); + let sums = FfnBatchKernels::load(gpu.context()).expect("tier sum kernels"); + let mut map = vec![HOST - 1; ROW_OFF + EXPERTS]; + let mut tmap = map.clone(); + for id in 0..EXPERTS { + map[ROW_OFF + id] = if id % 3 == 0 { (id / 3) as u32 } else { HOST }; + tmap[ROW_OFF + id] = if id % 3 == 1 { (id / 3) as u32 } else { HOST }; + } + let map = DeviceBuffer::from_host(stream, &map).expect("card map"); + let tmap = DeviceBuffer::from_host(stream, &tmap).expect("tier map"); + for n_used in [6, 8] { + for hidden in [n_used, 256, 257] { + for bad in [None, Some(EXPERTS as u32), Some(u32::MAX)] { + let mut ids = vec![15, 0, 10, 5, 8, 1, 2, 7]; + ids.truncate(n_used); + if let Some(id) = bad { + ids[n_used - 1] = id; + } + // Neither handoff may read the router's trailing lane. + ids.push(u32::MAX); + let weights: Vec = (0..n_used) + .map(|s| (s + 1) as f32 / 16.0) + .chain([f32::NAN]) + .collect(); + let x: Vec = (0..hidden).map(|d| (d % 7 + 1) as f32 / 8.0).collect(); + let row = |s: usize, d: usize| (ids[s] + 1) as f32 * x[d]; + let mut down = vec![f32::NAN; n_used * hidden]; + let mut trows = down.clone(); + let mut host = vec![0.0f32; hidden]; + let mut want = host.clone(); + for d in 0..hidden { + let mut card_tier = 0.0f32; + for s in 0..n_used { + if ids[s] >= EXPERTS as u32 { + continue; + } + let value = row(s, d); + match ids[s] % 3 { + 0 => down[s * hidden + d] = value, + 1 => trows[s * hidden + d] = value, + _ => host[d] = value.mul_add(weights[s], host[d]), + } + if ids[s] % 3 != 2 { + card_tier = value.mul_add(weights[s], card_tier); + } + } + want[d] = card_tier + host[d]; + } + let ids_d = DeviceBuffer::from_host(stream, &ids).expect("ids"); + let mut weights_d = DeviceBuffer::from_host(stream, &weights).expect("weights"); + let x_d = DeviceBuffer::from_host(stream, &x).expect("activation"); + let seq = DeviceBuffer::from_host(stream, &[19u32]).expect("sequence"); + let down = DeviceBuffer::from_host(stream, &down).expect("card rows"); + let trows = DeviceBuffer::from_host(stream, &trows).expect("tier rows"); + let doubled_host: Vec = host.iter().map(|h| 2.0 * h).collect(); + let mut host = DeviceBuffer::from_host(stream, &host).expect("host sum"); + let lay = HandoffLayout { + seq: 0, + ids: 1, + weights: 1 + n_used, + x: 1 + 2 * n_used, + n_used, + hidden, + }; + let tl = TierImageLayout { + sel: 0, + q3: 0, + d8: 0, + x: n_used, + n_used, + q3_words: 0, + d8_len: 0, + x_len: hidden, + hidden, + cols: 1, + }; + for captured in [false, true] { + let mut image = + DeviceBuffer::zeroed(stream, lay.x + hidden).expect("image"); + let mut timage = + DeviceBuffer::zeroed(stream, tl.x + hidden).expect("tier image"); + let poison = vec![HOST - 1; n_used + 1]; + let mut sel = DeviceBuffer::from_host(stream, &poison).expect("places"); + let mut tsel = + DeviceBuffer::from_host(stream, &poison).expect("tier places"); + let mut acc = DeviceBuffer::zeroed(stream, hidden).expect("sum"); + let mut result = DeviceBuffer::zeroed(stream, hidden).expect("result"); + let mut enqueue = || -> Result<(), GpuError> { + handoff.enqueue_handoff_tier( + stream, + &Handoff { + ids: &ids_d, + weights: &weights_d, + map: &map, + row_off: ROW_OFF, + n_expert: EXPERTS, + }, + &tmap, + ( + HandoffTarget { + x: &x_d, + seq: &seq, + image: &mut image, + layout: lay, + }, + TierTarget { + image: &mut timage, + rows: &trows, + layout: tl, + }, + ), + gpu.layer_sink(LAYER)?, + (&mut sel, &mut tsel), + )?; + let routed_weights = + Window::::of(&image, 4 * lay.weights, n_used)?; + sums.enqueue_card_acc_tier( + stream, + &CardAccTier { + down: &down, + trows: &trows, + w: &routed_weights, + sel: &sel, + tsel: &tsel, + n: hidden, + m: 1, + n_card: EXPERTS.div_ceil(3), + n_tier: EXPERTS.div_ceil(3), + n_used, + }, + &mut acc, + )?; + gpu.elem() + .enqueue_add(stream, &acc, &host, hidden, &mut result) + }; + if captured { + let graph = gpu.capture(|_| enqueue()).expect("capture"); + graph.launch(stream).expect("replay"); + let doubled_weights: Vec = + weights.iter().map(|w| 2.0 * w).collect(); + weights_d + .copy_from_host(stream, &doubled_weights) + .expect("new route weights"); + host.copy_from_host(stream, &doubled_host) + .expect("new host sum"); + graph.launch(stream).expect("second replay"); + stream.synchronize().expect("replays settled"); + } else { + enqueue().expect("eager handoff and consumption"); + } + let scale = if captured { 2.0 } else { 1.0 }; + let got = result.to_host_vec(stream).expect("result"); + let places = sel.to_host_vec(stream).expect("places"); + let tier_places = tsel.to_host_vec(stream).expect("tier places"); + let image = image.to_host_vec(stream).expect("host image"); + let timage = timage.to_host_vec(stream).expect("tier image"); + stream.synchronize().expect("settled"); + let fault = gpu.take_fault().expect("fault word"); + if bad.is_some() { + let fault = fault.expect("an invalid id must raise"); + assert_eq!(fault.layer, LAYER as u32); + assert_eq!(fault.code, FaultSite::ExpertId as u32); + assert_eq!(fault.sites, 1 << FaultSite::ExpertId as u32); + } else { + assert!(fault.is_none(), "{fault:?}"); + } + assert_eq!(places[n_used], HOST - 1, "card places tail"); + assert_eq!(tier_places[n_used], HOST - 1, "tier places tail"); + assert_eq!(image[lay.seq], 19, "sequence"); + assert_eq!(&image[lay.ids..lay.ids + n_used], &ids[..n_used]); + for s in 0..n_used { + assert_eq!(image[lay.weights + s], (weights[s] * scale).to_bits()); + assert_eq!(timage[tl.sel + s], tier_places[s]); + } + for d in 0..hidden { + assert_eq!(image[lay.x + d], x[d].to_bits()); + assert_eq!(timage[tl.x + d], x[d].to_bits()); + } + assert!( + got.iter() + .zip(&want) + .all(|(g, w)| g.to_bits() == (w * scale).to_bits()), + "{n_used} slots, width {hidden}, bad {bad:?}, capture {captured}: result conservation" + ); + } + } + } + } + } +} diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index a653be6db..99f6b4efe 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -18,9 +18,9 @@ //! one launch ([`HandoffKernels::enqueue_handoff_cols`]); //! [`HandoffKernels::enqueue_places_cols`] writes the places alone of such a //! row, for a walk whose routing reaches the host by a download, through -//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_8_tier` is -//! the eight-slot handoff of a layer an expert tier holds experts of: the -//! same writes, and the slots' tier places and the activation in f32 into +//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_6_tier` and +//! `ds41_ffn_handoff_8_tier` hand over six or eight slots of a layer an +//! expert tier holds experts of: the same writes, the slots' tier places and the activation in f32 into //! the row's tier image ([`HandoffKernels::enqueue_handoff_tier`]); //! `ds41_ffn_handoff_10_cols_tier` is `_10_cols`'s of such a layer, every //! column's tier places and activation into the row's tier image @@ -281,6 +281,88 @@ mod handoff_kernels { unsafe { handoff_at::(&a, d, &mut image, &mut sel) }; } + /// [`ds41_ffn_handoff`] and the row's f32 tier image, with the same + /// per-thread writes and following go as [`ds41_ffn_handoff_8_tier`]. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + ids_in.len() >= 6, + w_in.len() >= 6, + map.len() >= row_off + n_expert, + tmap.len() >= row_off + n_expert, + x.len() >= n, + seq.len() >= 1, + n >= 6, + ids_at >= seq_at + 1, + wts_at >= ids_at + 6, + x_at >= wts_at + 6, + image.len() >= x_at + n, + tx_at >= tsel_at + 6, + timage.len() >= tx_at + n, + sel.len() >= 6, + tsel.len() >= 6 + ) + )] + pub fn ds41_ffn_handoff_6_tier( + ids_in: &[u32], + w_in: &[f32], + map: &[u32], + tmap: &[u32], + row_off: u32, + n_expert: u32, + x: &[f32], + seq: &[u32], + n: u32, + seq_at: u32, + ids_at: u32, + wts_at: u32, + x_at: u32, + tsel_at: u32, + tx_at: u32, + fault: FaultSink, + mut image: DisjointSlice, + mut timage: DisjointSlice, + mut sel: DisjointSlice, + mut tsel: DisjointSlice, + ) { + let d = thread::index_1d().get(); + if d >= n as usize { + return; + } + let a = HandoffIn { + ids_in, + w_in, + map, + row_off, + n_expert, + x, + seq, + seq_at, + ids_at, + wts_at, + x_at, + fault, + }; + // SAFETY: d < n, and the launch contract is `handoff_at`'s at + // N_USED slots; thread d is the launch's only thread at d. + unsafe { handoff_at::(&a, d, &mut image, &mut sel) }; + let t = TierIn { + tmap, + tsel_at, + tx_at, + }; + // SAFETY: d < n, and the launch contract is `handoff_tier_at`'s at + // N_USED slots; thread d is the launch's only thread at d. + unsafe { handoff_tier_at::(&a, &t, d, &mut timage, &mut tsel) }; + } + /// [`ds41_ffn_handoff_8`] and the row's f32 tier image /// ([`handoff_tier_at`]): beside the handoff and the stage card's places, /// threads `s < 8` write `tsel[s] = tmap[row_off + id]` — the id's slot @@ -1181,8 +1263,9 @@ impl HandoffKernels { } impl HandoffKernels { - /// Enqueue the handoff of a tier layer of eight slots a token - /// (`ds41_ffn_handoff_8_tier`): [`HandoffKernels::enqueue_handoff`]'s + /// Enqueue the handoff of a tier layer of six or eight slots a token + /// (`ds41_ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): + /// [`HandoffKernels::enqueue_handoff`]'s /// writes into `target`, and each slot's tier place (from `tmap`, the /// slot map's tier copy at the card copy's row offsets) into `tsel` and /// with the activation into the f32 tier image `ttarget`. A layout of @@ -1202,52 +1285,82 @@ impl HandoffKernels { fault: FaultSink, (sel, tsel): (&mut DeviceBuffer, &mut DeviceBuffer), ) -> Result<(), GpuError> { - const WHAT: &str = "ds41_ffn_handoff_8_tier"; let (lay, tl) = (target.layout, ttarget.layout); - if lay.n_used != SLOTS_8 - || tl.n_used != SLOTS_8 - || tl.x_len != lay.hidden - || tl.q3_words != 0 - || tl.d8_len != 0 - { - return Err(GpuError::Shape { - what: WHAT, - detail: format!( - "the boundary carries {} slots of {} values and the tier image {} slots, {} \ - f32, {} code words, {} scales; the entry hands over {SLOTS_8} slots and the \ - activation in f32", - lay.n_used, lay.hidden, tl.n_used, tl.x_len, tl.q3_words, tl.d8_len - ), - }); - } + let slots = check_tier(&lay, &tl)?; + let what = match slots { + HandoffSlots::Six => "ds41_ffn_handoff_6_tier", + _ => "ds41_ffn_handoff_8_tier", + }; let n = lay.hidden; - let grid = launch_u32(WHAT, "grid", n.div_ceil(HANDOFF_THREADS as usize))?; + let grid = launch_u32(what, "grid", n.div_ceil(HANDOFF_THREADS as usize))?; let cfg = LaunchConfig1D::new(grid, HANDOFF_THREADS, 0); - let prep = self.module.prepare_ds41_ffn_handoff_8_tier(cfg)?; - self.module.ds41_ffn_handoff_8_tier( - stream, - &prep, - h.ids, - h.weights, - h.map, - tmap, - launch_u32(WHAT, "row_off", h.row_off)?, - launch_u32(WHAT, "n_expert", h.n_expert)?, - target.x, - target.seq, - launch_u32(WHAT, "n", n)?, - launch_u32(WHAT, "seq_at", lay.seq)?, - launch_u32(WHAT, "ids_at", lay.ids)?, - launch_u32(WHAT, "wts_at", lay.weights)?, - launch_u32(WHAT, "x_at", lay.x)?, - launch_u32(WHAT, "tsel_at", tl.sel)?, - launch_u32(WHAT, "tx_at", tl.x)?, - fault, - target.image, - ttarget.image, - sel, - tsel, - )?; + let row_off = launch_u32(what, "row_off", h.row_off)?; + let n_expert = launch_u32(what, "n_expert", h.n_expert)?; + let at = [ + launch_u32(what, "n", n)?, + launch_u32(what, "seq_at", lay.seq)?, + launch_u32(what, "ids_at", lay.ids)?, + launch_u32(what, "wts_at", lay.weights)?, + launch_u32(what, "x_at", lay.x)?, + launch_u32(what, "tsel_at", tl.sel)?, + launch_u32(what, "tx_at", tl.x)?, + ]; + match slots { + HandoffSlots::Six => { + let prep = self.module.prepare_ds41_ffn_handoff_6_tier(cfg)?; + self.module.ds41_ffn_handoff_6_tier( + stream, + &prep, + h.ids, + h.weights, + h.map, + tmap, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + at[5], + at[6], + fault, + target.image, + ttarget.image, + sel, + tsel, + )?; + } + _ => { + let prep = self.module.prepare_ds41_ffn_handoff_8_tier(cfg)?; + self.module.ds41_ffn_handoff_8_tier( + stream, + &prep, + h.ids, + h.weights, + h.map, + tmap, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + at[5], + at[6], + fault, + target.image, + ttarget.image, + sel, + tsel, + )?; + } + } Ok(()) } @@ -1402,6 +1515,29 @@ impl HandoffKernels { } } +/// The single-token F32 tier handoff's layout contract, shared by both +/// slot entries. The generated launch checks the input and output spans. +fn check_tier(lay: &HandoffLayout, tl: &TierImageLayout) -> Result { + let slots = HandoffSlots::of(lay.n_used); + if !matches!(slots, Ok(HandoffSlots::Six | HandoffSlots::Eight)) + || tl.n_used != lay.n_used + || tl.x_len != lay.hidden + || tl.q3_words != 0 + || tl.d8_len != 0 + { + return Err(GpuError::Shape { + what: "HandoffKernels::enqueue_handoff_tier", + detail: format!( + "the boundary carries {} slots of {} values and the tier image {} slots, {} \ + f32, {} code words, {} scales; the entries hand over matching 6 or 8 slots \ + and the activation in f32", + lay.n_used, lay.hidden, tl.n_used, tl.x_len, tl.q3_words, tl.d8_len + ), + }); + } + slots.map_err(|e| GpuError::shape("HandoffKernels::enqueue_handoff_tier", e.to_string())) +} + /// [`HandoffKernels::enqueue_handoff_cols`]'s refusals, by name before the /// launch: a layout of another slot count, `m` of 0, a pitch under ten, and a /// `sel` of `sel_len` places shorter than `m` columns of ten. @@ -1488,7 +1624,7 @@ pub struct Places<'a> { #[cfg(test)] mod tests { - use super::{HandoffRefused, HandoffSlots, check_cols, check_cols_tier}; + use super::{HandoffRefused, HandoffSlots, check_cols, check_cols_tier, check_tier}; use crate::host::page::HandoffLayout; use crate::host::tier::TierImageLayout; @@ -1539,6 +1675,44 @@ mod tests { } } + #[test] + fn a_token_tier_handoff_requires_matching_f32_six_or_eight_slot_images() { + for n_used in 0..=10 { + let lay = HandoffLayout { + n_used, + ..boundary() + }; + for tier_slots in 0..=10 { + let tl = TierImageLayout { + n_used: tier_slots, + x: tier_slots, + ..image(1) + }; + assert_eq!( + check_tier(&lay, &tl).is_ok(), + matches!(n_used, 6 | 8) && tier_slots == n_used, + "host {n_used}, tier {tier_slots}" + ); + } + } + for n_used in [6, 8] { + let lay = HandoffLayout { + n_used, + ..boundary() + }; + let tl = TierImageLayout { + n_used, + x: n_used, + ..image(1) + }; + for x_len in [0, HIDDEN - 1, HIDDEN + 1, 2 * HIDDEN] { + assert!(check_tier(&lay, &TierImageLayout { x_len, ..tl }).is_err()); + } + assert!(check_tier(&lay, &TierImageLayout { q3_words: 1, ..tl }).is_err()); + assert!(check_tier(&lay, &TierImageLayout { d8_len: 1, ..tl }).is_err()); + } + } + /// A tier handoff of `m` columns over a page of 8 refuses, by name, a /// `tsel` or a `sel` shorter than `10·m`, and takes ones of exactly that: /// the launch writes every column's places. diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 0d60c5feb..e3bfeb200 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -66,6 +66,30 @@ Each device slice needs its numerical/fault gates and the maintainer's offered h Legacy metadata booleans share `RouterNorm::from_legacy_bool`: false keeps scores and true uses the existing epsilon-added selected sum. Flint's distinct floor mapping stays in its reader. +### Six-slot F32 tier handoff implementation + +`HandoffKernels::enqueue_handoff_tier` now selects a genuine six-slot F32 entry +beside the existing eight-slot entry. Both host and tier layouts must have the +same supported slot count and activation width; compressed tier images remain +refused. The entry uses the existing copy/place/fault helpers and the existing +subsequent go/publication protocol; no Flint router or Program is registered. + +The pure layout contract passed locally after an oxide build. +`hw_f32_tier_handoffs_conserve_six_and_eight_consumed_routes`, owned by +`gate-gpu-ds41-lib`, consumes the handoff's weights and emitted places through +the production tier accumulator and adds the host sum. It covers partial widths, +invalid ids, poisoned non-owning rows, trailing sentinels and changed replay +weights. Its host sum is precomputed and activation images are checked by +readback: it is not expert execution or two-card completion/publication proof. +The hardware case has not been run here. + +The new `ds41_ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and +owning-gate evidence; the eight-slot and other existing entries must remain +unchanged. No PTX resource pin is added without an observed NVIDIA JIT run. +Actual six-slot body/tier and two-card completion remain integration work, not +capabilities inferred from this primitive. + + ## 1. Sources and observed artifact diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index 94a2bb78a..902fcf81c 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -528,7 +528,10 @@ gguf 7 2 0 # copy, scale read and MMA instruction), `Mxfp8Cols` (refused, fetch, land, scale_word), the activations' MXFP8 form # body (`mxfp8_cols_body`: its load and its two plane stores) and the two kernel entries' body calls and shared-memory # view. Each block has its SAFETY line. -gpu 2331 306 10 +# PIN(2026-10-24): gpu 2331 -> 2333 blocks, a new six-slot F32 tier entry, not old code relaxed: +# ds41_ffn_handoff_6_tier calls handoff_at and handoff_tier_at under its checked launch contract; +# both blocks state their per-thread bounds and disjoint-write SAFETY invariant. +gpu 2333 306 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line. From 79a2959c302b95e2eb5b67076178e8e31b3c5fad Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:59:23 +0900 Subject: [PATCH 2/7] gpu: name the new six-slot tier primitive generically --- crates/gpu/src/host/handoff.rs | 12 ++++++------ docs/research/flint-attach.md | 6 +++++- tools/unsafe-ratchet.txt | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index 99f6b4efe..f2a362b91 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -18,7 +18,7 @@ //! one launch ([`HandoffKernels::enqueue_handoff_cols`]); //! [`HandoffKernels::enqueue_places_cols`] writes the places alone of such a //! row, for a walk whose routing reaches the host by a download, through -//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_6_tier` and +//! `crate::card_slots`' `ds41_ffn_places`. `ffn_handoff_6_tier` and //! `ds41_ffn_handoff_8_tier` hand over six or eight slots of a layer an //! expert tier holds experts of: the same writes, the slots' tier places and the activation in f32 into //! the row's tier image ([`HandoffKernels::enqueue_handoff_tier`]); @@ -310,7 +310,7 @@ mod handoff_kernels { tsel.len() >= 6 ) )] - pub fn ds41_ffn_handoff_6_tier( + pub fn ffn_handoff_6_tier( ids_in: &[u32], w_in: &[f32], map: &[u32], @@ -1264,7 +1264,7 @@ impl HandoffKernels { impl HandoffKernels { /// Enqueue the handoff of a tier layer of six or eight slots a token - /// (`ds41_ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): + /// (`ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): /// [`HandoffKernels::enqueue_handoff`]'s /// writes into `target`, and each slot's tier place (from `tmap`, the /// slot map's tier copy at the card copy's row offsets) into `tsel` and @@ -1288,7 +1288,7 @@ impl HandoffKernels { let (lay, tl) = (target.layout, ttarget.layout); let slots = check_tier(&lay, &tl)?; let what = match slots { - HandoffSlots::Six => "ds41_ffn_handoff_6_tier", + HandoffSlots::Six => "ffn_handoff_6_tier", _ => "ds41_ffn_handoff_8_tier", }; let n = lay.hidden; @@ -1307,8 +1307,8 @@ impl HandoffKernels { ]; match slots { HandoffSlots::Six => { - let prep = self.module.prepare_ds41_ffn_handoff_6_tier(cfg)?; - self.module.ds41_ffn_handoff_6_tier( + let prep = self.module.prepare_ffn_handoff_6_tier(cfg)?; + self.module.ffn_handoff_6_tier( stream, &prep, h.ids, diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index e3bfeb200..a42a0fc4f 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -83,11 +83,15 @@ weights. Its host sum is precomputed and activation images are checked by readback: it is not expert execution or two-card completion/publication proof. The hardware case has not been run here. -The new `ds41_ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and +The new `ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and owning-gate evidence; the eight-slot and other existing entries must remain unchanged. No PTX resource pin is added without an observed NVIDIA JIT run. Actual six-slot body/tier and two-card completion remain integration work, not capabilities inferred from this primitive. +`check-ptx-names` remains red until observed spill/JIT-local measurements can pin +the new entry in the four existing architecture/capability tables. Offline +no-JIT scans cannot supply that missing column, so no placeholder row is added. + diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index 902fcf81c..cb8465ee7 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -529,7 +529,7 @@ gguf 7 2 0 # body (`mxfp8_cols_body`: its load and its two plane stores) and the two kernel entries' body calls and shared-memory # view. Each block has its SAFETY line. # PIN(2026-10-24): gpu 2331 -> 2333 blocks, a new six-slot F32 tier entry, not old code relaxed: -# ds41_ffn_handoff_6_tier calls handoff_at and handoff_tier_at under its checked launch contract; +# ffn_handoff_6_tier calls handoff_at and handoff_tier_at under its checked launch contract; # both blocks state their per-thread bounds and disjoint-write SAFETY invariant. gpu 2333 306 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: From 29a29b15451641c33399ac419e289ed1ff7796e4 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:19:09 +0900 Subject: [PATCH 3/7] Format the combined tier handoff guard --- crates/gpu/src/host/handoff.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index 474158494..84f35cdc5 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -1634,8 +1634,7 @@ fn check_tier(lay: &HandoffLayout, tl: &TierImageLayout) -> Result Date: Tue, 27 Oct 2026 06:04:10 +0900 Subject: [PATCH 4/7] Use common F32 tier handoff error context --- crates/gpu/src/host/handoff.rs | 6 +----- docs/research/flint-attach.md | 26 ++++++++++++++++++++++++-- tools/model-tax.tsv | 2 +- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index 84f35cdc5..f7217b35d 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -1371,11 +1371,7 @@ impl HandoffKernels { ) -> Result<(), GpuError> { let (lay, tl) = (target.layout, ttarget.layout); let slots = check_tier(&lay, &tl)?; - let what = match slots { - HandoffSlots::Four => "ds41_ffn_handoff_4_tier", - HandoffSlots::Six => "ffn_handoff_6_tier", - _ => "ds41_ffn_handoff_8_tier", - }; + let what = "F32 tier handoff"; let n = lay.hidden; let grid = launch_u32(what, "grid", n.div_ceil(HANDOFF_THREADS as usize))?; let cfg = LaunchConfig1D::new(grid, HANDOFF_THREADS, 0); diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 7cf7dd5aa..8cf68f0c2 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,8 +2,30 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. The current tree additionally has a **header-only reader**; it still has no Flint -body, app session or serve seat. No GPU, inference, performance or weight-content validation is claimed. +that snapshot. This component tree additionally has a header-only reader and shared tier-handoff +work, but no Flint body, app session or serve seat. Primitive GPU evidence is not Flint model admission. + +## Six-slot handoff closure checks — 2026-10-26 + +The existing #781 branch was ordinarily merged with main +`15778e2cb2dd7af428d2dd14259e6b0ea8ca6b89`; its published history was not rewritten. +On source `9bfa55adc257f7e6aae4952aca06246eddaeef70`, the owning DS41 library ran on +Hikari's RTX 3090 (GPU0): **38 passed, 0 failed, 0 ignored**, in 28.75 seconds. +`hw_f32_tier_handoffs_conserve_consumed_routes` exercised four, six and eight routes, +including partial widths, invalid IDs, poisoned unconsumed rows and changed-input graph replay. +The following host-only changes format the guard and replace a redundant slot-dependent +error label with the common `F32 tier handoff` context. The measured common-name census +falls from 39 to 37; the ABI-stable generated kernel names and dispatch stay unchanged. +The earlier GPU run is not relabeled as a run of these later commits. + +The original remote request timed out while its child continued; a separate read-only +collector recovered the original successful log rather than replaying the run. +Original-request credential cleanup remains unconfirmed and is not implied by test success. +Raw evidence is retained outside Git under +`/home/jmtai04/flint-completion-4dadee0-20261025T205648Z/evidence/closure-20261026/`; +local receipts are under `flint-closure-20261026` in the contribution workspace. +Resource-table admission still requires actual measurements for the new entry. +Accessible cards establish only cc8.6, not physical cc12.1; do not synthesize the latter pins. ## Host reader status diff --git a/tools/model-tax.tsv b/tools/model-tax.tsv index 96a9d889c..cb3b46303 100644 --- a/tools/model-tax.tsv +++ b/tools/model-tax.tsv @@ -131,7 +131,7 @@ crates/gpu-vision/src/encoder.rs deepseek 2 baseline D 2: media design: VideoRul crates/gpu/src/arch/hybrid.rs nemotron 1 baseline O 1: own path: an alias row in tools/model-tokens.tsv or a move into the family directory crates/gpu/src/card_slots.rs deepseek 24 PIN(2026-10-21) deepseek: baseline K 20 + 4, the four-slot entry `ds41_ffn_card_acc_4` (MiniMax-M3 routes 4 a token): one more copy of the per-count entry, since a `#[kernel]` entry cannot be generic over the slot count; the deferred rename of the entries by what they compute absorbs it crates/gpu/src/dspark/head.rs deepseek 3 baseline K 3: rename the kernel entries by what they compute (deferred) -crates/gpu/src/host/handoff.rs deepseek 38 PIN(2026-10-24) deepseek: baseline K 31 + 4 + 3, the four-slot tier entry `ds41_ffn_handoff_4_tier` (MiniMax-M3.1's bp) beside the four-slot entry `ds41_ffn_handoff_4` (MiniMax-M3 routes 4 a token): one more copy of the per-count entry, since a `#[kernel]` entry cannot be generic over the slot count; the deferred rename of the entries by what they compute absorbs it +crates/gpu/src/host/handoff.rs deepseek 37 PIN(2026-10-26) deepseek: the tier launcher uses a common F32 error context instead of selecting a family-named label per slot count; existing kernel entry symbols remain ABI-stable crates/hf/src/vocab.rs qwen 4 PIN(2026-10-20) qwen: the qwen35 tokenizer pipeline name and regex; tokenizer programs are not yet described by ModelSpec crates/metal/src/bin/bloomery_serve_metal.rs qwen 3 PIN(2026-10-23) qwen: Metal arch dispatch by program: the Metal bins open the one Metal architecture's session by name until crates/metal/src/arch picks a session by the file's architecture string crates/metal/src/bin/generate_metal.rs qwen 2 PIN(2026-10-23) qwen: Metal arch dispatch by program: the Metal bins open the one Metal architecture's session by name until crates/metal/src/arch picks a session by the file's architecture string From f11d5b15017f3fe8d6a5f4bb9cb30a93b212da4a Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:29:37 +0900 Subject: [PATCH 5/7] Pin measured six-slot handoff resources on cc8.6 --- docs/research/flint-attach.md | 15 +++++++++++++-- tools/ref/ptx-shapes.tsv | 4 ++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 8cf68f0c2..f69a95283 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -24,8 +24,19 @@ Original-request credential cleanup remains unconfirmed and is not implied by te Raw evidence is retained outside Git under `/home/jmtai04/flint-completion-4dadee0-20261025T205648Z/evidence/closure-20261026/`; local receipts are under `flint-closure-20261026` in the contribution workspace. -Resource-table admission still requires actual measurements for the new entry. -Accessible cards establish only cc8.6, not physical cc12.1; do not synthesize the latter pins. +The actual `generate_ds41` carrier at `b9b6e013ebfab5748688be52e9a5d398ac2fc9a3` +subsequently measured `ffn_handoff_6_tier` on sm_86 / physical cc8.6: +ptxas13.3.33 spill0, driver-JIT local0, both register counts26 (driver595.91.07, +JIT CUDA13.2, Rust1.99.1 / cuda-oxide9df9e7ce / repository patch31ab6a1d3f58). +Only that measured row is added. Its normalized entry digest is +`a176a9a5846778e40ba99fc2be637d90` (decl1, 163 lines). +This resource request also exceeded the gateway window; a read-only collector recovered +the original scan, with no remaining owned process and GPU1 active/healthy. +The original script recorded exit23 after producing the complete successful scan; +a post-timeout stdout write failure is the explanation inferred from its final curl step, +not an invented exit0 for the request. The original credential cleanup remains unconfirmed. +Accessible cards establish only cc8.6. The other three sm_86/sm_121/sm_121a + physical +cc12.1 rows remain unmeasured and absent; #781 stays Draft until that gate is dispositioned. ## Host reader status diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index 07613d074..b038d2937 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -74,6 +74,10 @@ generate_ds41 ds41_ffn_handoff_10_cols_tier 0 0 generate_ds41 ds41_ffn_handoff_4 0 0 generate_ds41 ds41_ffn_handoff_8 0 0 generate_ds41 ds41_ffn_handoff_4_tier 0 0 +# PIN(2026-10-26): ffn_handoff_6_tier measured from generate_ds41 at b9b6e013 with +# Rust1.99.1 / cuda-oxide9df9e7ce + repository patch31ab6a1d3f58; sm_86 ptxas13.3.33, +# RTX3090 cc8.6 driver595.91.07 JIT CUDA13.2: spill0 / jit_local0 (both register counts26). +generate_ds41 ffn_handoff_6_tier 0 0 generate_ds41 ds41_ffn_handoff_8_tier 0 0 generate_ds41 ds41_ffn_handoff_9 0 0 generate_ds41 ds41_ffn_handoff_tier 0 0 From 12391442fd2004c9140bc777a58c5d6c44c8e4e1 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:12:14 +0900 Subject: [PATCH 6/7] Keep handoff measurements durable and move operational receipts out of contract docs --- docs/research/flint-attach.md | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index f69a95283..841f93e6d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -18,25 +18,15 @@ error label with the common `F32 tier handoff` context. The measured common-name falls from 39 to 37; the ABI-stable generated kernel names and dispatch stay unchanged. The earlier GPU run is not relabeled as a run of these later commits. -The original remote request timed out while its child continued; a separate read-only -collector recovered the original successful log rather than replaying the run. -Original-request credential cleanup remains unconfirmed and is not implied by test success. -Raw evidence is retained outside Git under -`/home/jmtai04/flint-completion-4dadee0-20261025T205648Z/evidence/closure-20261026/`; -local receipts are under `flint-closure-20261026` in the contribution workspace. The actual `generate_ds41` carrier at `b9b6e013ebfab5748688be52e9a5d398ac2fc9a3` subsequently measured `ffn_handoff_6_tier` on sm_86 / physical cc8.6: ptxas13.3.33 spill0, driver-JIT local0, both register counts26 (driver595.91.07, JIT CUDA13.2, Rust1.99.1 / cuda-oxide9df9e7ce / repository patch31ab6a1d3f58). Only that measured row is added. Its normalized entry digest is `a176a9a5846778e40ba99fc2be637d90` (decl1, 163 lines). -This resource request also exceeded the gateway window; a read-only collector recovered -the original scan, with no remaining owned process and GPU1 active/healthy. -The original script recorded exit23 after producing the complete successful scan; -a post-timeout stdout write failure is the explanation inferred from its final curl step, -not an invented exit0 for the request. The original credential cleanup remains unconfirmed. -Accessible cards establish only cc8.6. The other three sm_86/sm_121/sm_121a + physical -cc12.1 rows remain unmeasured and absent; #781 stays Draft until that gate is dispositioned. +Measurements establish only physical cc8.6. The other three +sm_86/sm_121/sm_121a + physical cc12.1 rows remain unmeasured and absent; +their genuine driver-JIT resource evidence is still required. ## Host reader status From 44d200c0fca555c90c190ddc3a0641485896a442 Mon Sep 17 00:00:00 2001 From: midagedev Date: Tue, 27 Oct 2026 14:55:13 +0900 Subject: [PATCH 7/7] handoff, ptx-shapes: name the eight-slot arm and refuse the rest by name; pin ffn_handoff_6_tier's cc12.1 rows at the cc8.6 values Co-Authored-By: Claude Opus 5.5 --- crates/gpu/src/host/handoff.rs | 8 +++++++- docs/research/flint-attach.md | 11 +++++------ tools/ref/ptx-shapes/sm_121-cc12.1.tsv | 2 ++ tools/ref/ptx-shapes/sm_121a-cc12.1.tsv | 2 ++ tools/ref/ptx-shapes/sm_86-cc12.1.tsv | 2 ++ 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index f7217b35d..780914af6 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -1441,7 +1441,7 @@ impl HandoffKernels { tsel, )?; } - _ => { + HandoffSlots::Eight => { let prep = self.module.prepare_ds41_ffn_handoff_8_tier(cfg)?; self.module.ds41_ffn_handoff_8_tier( stream, @@ -1468,6 +1468,12 @@ impl HandoffKernels { tsel, )?; } + HandoffSlots::Nine | HandoffSlots::Ten => { + return Err(GpuError::shape( + "HandoffKernels::enqueue_handoff_tier", + format!("no F32 tier handoff entry for {slots:?}"), + )); + } } Ok(()) } diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 841f93e6d..72eb230ce 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -25,8 +25,8 @@ JIT CUDA13.2, Rust1.99.1 / cuda-oxide9df9e7ce / repository patch31ab6a1d3f58). Only that measured row is added. Its normalized entry digest is `a176a9a5846778e40ba99fc2be637d90` (decl1, 163 lines). Measurements establish only physical cc8.6. The other three -sm_86/sm_121/sm_121a + physical cc12.1 rows remain unmeasured and absent; -their genuine driver-JIT resource evidence is still required. +sm_86/sm_121/sm_121a + physical cc12.1 rows remain unmeasured: they carry the entry at the cc8.6 +values under a `PIN(2026-10-27)` until a GB10 scan runs. ## Host reader status @@ -108,12 +108,11 @@ main; the final-source hardware result must be recorded separately. The new `ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and owning-gate evidence; the four-slot, eight-slot and other existing entries must remain -unchanged. No PTX resource pin is added without an observed NVIDIA JIT run. +unchanged. The cc12.1 tables pin it at the cc8.6 scan's values until an observed NVIDIA JIT run. Actual six-slot body/tier and two-card completion remain integration work, not capabilities inferred from this primitive. -`check-ptx-names` remains red until observed spill/JIT-local measurements can pin -the new entry in the four existing architecture/capability tables. Offline -no-JIT scans cannot supply that missing column, so no placeholder row is added. +`check-ptx-names` is green with those pinned rows; a GB10 scan replaces them with the +observed spill/JIT-local measurements. Offline no-JIT scans cannot supply that column. diff --git a/tools/ref/ptx-shapes/sm_121-cc12.1.tsv b/tools/ref/ptx-shapes/sm_121-cc12.1.tsv index b31394183..850f9ecdf 100644 --- a/tools/ref/ptx-shapes/sm_121-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_121-cc12.1.tsv @@ -70,6 +70,8 @@ generate_ds41 ds41_ffn_handoff_10_cols_tier 0 0 generate_ds41 ds41_ffn_handoff_4 0 0 generate_ds41 ds41_ffn_handoff_8 0 0 generate_ds41 ds41_ffn_handoff_4_tier 0 0 +# PIN(2026-10-27): ffn_handoff_6_tier at the cc 8.6 scan's value (tools/ref/ptx-shapes.tsv), not measured on cc 12.1, until a GB10 scan runs. +generate_ds41 ffn_handoff_6_tier 0 0 generate_ds41 ds41_ffn_handoff_8_tier 0 0 generate_ds41 ds41_ffn_handoff_tier 0 0 generate_ds41 ds41_ffn_places 0 0 diff --git a/tools/ref/ptx-shapes/sm_121a-cc12.1.tsv b/tools/ref/ptx-shapes/sm_121a-cc12.1.tsv index 500bf10e5..c951ae267 100644 --- a/tools/ref/ptx-shapes/sm_121a-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_121a-cc12.1.tsv @@ -85,6 +85,8 @@ generate_ds41 ds41_ffn_handoff_10_cols_tier 0 0 generate_ds41 ds41_ffn_handoff_4 0 0 generate_ds41 ds41_ffn_handoff_8 0 0 generate_ds41 ds41_ffn_handoff_4_tier 0 0 +# PIN(2026-10-27): ffn_handoff_6_tier at the cc 8.6 scan's value (tools/ref/ptx-shapes.tsv), not measured on cc 12.1, until a GB10 scan runs. +generate_ds41 ffn_handoff_6_tier 0 0 generate_ds41 ds41_ffn_handoff_8_tier 0 0 generate_ds41 ds41_ffn_handoff_9 0 0 generate_ds41 ds41_ffn_handoff_tier 0 0 diff --git a/tools/ref/ptx-shapes/sm_86-cc12.1.tsv b/tools/ref/ptx-shapes/sm_86-cc12.1.tsv index bf7802d0f..de7cb39c0 100644 --- a/tools/ref/ptx-shapes/sm_86-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_86-cc12.1.tsv @@ -69,6 +69,8 @@ generate_ds41 ds41_ffn_handoff_10_cols_tier 0 0 generate_ds41 ds41_ffn_handoff_4 0 0 generate_ds41 ds41_ffn_handoff_8 0 0 generate_ds41 ds41_ffn_handoff_4_tier 0 0 +# PIN(2026-10-27): ffn_handoff_6_tier at the cc 8.6 scan's value (tools/ref/ptx-shapes.tsv), not measured on cc 12.1, until a GB10 scan runs. +generate_ds41 ffn_handoff_6_tier 0 0 generate_ds41 ds41_ffn_handoff_8_tier 0 0 generate_ds41 ds41_ffn_handoff_tier 0 0 generate_ds41 ds41_ffn_places 0 0